155 files changed, 11923 insertions, 8826 deletions

diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 7af443672626..a6feb78c404c 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -23,7 +23,7 @@ config DRM_KMS_HELPER
         tristate
         depends on DRM
         select FB
-        select FRAMEBUFFER_CONSOLE if !EMBEDDED
+        select FRAMEBUFFER_CONSOLE if !EXPERT
         help
           FB and CRTC helpers for KMS drivers.
 
@@ -73,52 +73,47 @@ source "drivers/gpu/drm/radeon/Kconfig"
 
 config DRM_I810
         tristate "Intel I810"
-        # BKL usage in order to avoid AB-BA deadlocks, may become BROKEN_ON_SMP
-        depends on DRM && AGP && AGP_INTEL && BKL
+        # !PREEMPT because of missing ioctl locking
+        depends on DRM && AGP && AGP_INTEL && (!PREEMPT || BROKEN)
         help
           Choose this option if you have an Intel I810 graphics card.  If M is
           selected, the module will be called i810.  AGP support is required
           for this driver to work.
 
-choice
-        prompt "Intel 830M, 845G, 852GM, 855GM, 865G"
-        depends on DRM && AGP && AGP_INTEL
-        optional
-
-config DRM_I830
-        tristate "i830 driver"
-        # BKL usage in order to avoid AB-BA deadlocks, i830 may get removed
-        depends on BKL
-        help
-          Choose this option if you have a system that has Intel 830M, 845G,
-          852GM, 855GM or 865G integrated graphics.  If M is selected, the
-          module will be called i830.  AGP support is required for this driver
-          to work. This driver is used by the older X releases X.org 6.7 and
-          XFree86 4.3. If unsure, build this and i915 as modules and the X server
-          will load the correct one.
-
 config DRM_I915
-        tristate "i915 driver"
+        tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics"
+        depends on DRM
+        depends on AGP
         depends on AGP_INTEL
+        # we need shmfs for the swappable backing store, and in particular
+        # the shmem_readpage() which depends upon tmpfs
         select SHMEM
+        select TMPFS
         select DRM_KMS_HELPER
         select FB_CFB_FILLRECT
         select FB_CFB_COPYAREA
         select FB_CFB_IMAGEBLIT
         # i915 depends on ACPI_VIDEO when ACPI is enabled
         # but for select to work, need to select ACPI_VIDEO's dependencies, ick
-        select VIDEO_OUTPUT_CONTROL if ACPI
         select BACKLIGHT_CLASS_DEVICE if ACPI
         select INPUT if ACPI
         select ACPI_VIDEO if ACPI
         select ACPI_BUTTON if ACPI
         help
-          Choose this option if you have a system that has Intel 830M, 845G,
-          852GM, 855GM 865G or 915G integrated graphics.  If M is selected, the
-          module will be called i915.  AGP support is required for this driver
-          to work. This driver is used by the Intel driver in X.org 6.8 and
-          XFree86 4.4 and above. If unsure, build this and i830 as modules and
-          the X server will load the correct one.
+          Choose this option if you have a system that has "Intel Graphics
+          Media Accelerator" or "HD Graphics" integrated graphics,
+          including 830M, 845G, 852GM, 855GM, 865G, 915G, 945G, 965G,
+          G35, G41, G43, G45 chipsets and Celeron, Pentium, Core i3,
+          Core i5, Core i7 as well as Atom CPUs with integrated graphics.
+          If M is selected, the module will be called i915.  AGP support
+          is required for this driver to work. This driver is used by
+          the Intel driver in X.org 6.8 and XFree86 4.4 and above. It
+          replaces the older i830 module that supported a subset of the
+          hardware in older X.org releases.
+
+          Note that the older i810/i815 chipsets require the use of the
+          i810 driver instead, and the Atom z5xx series has an entirely
+          different implementation.
 
 config DRM_I915_KMS
         bool "Enable modesetting on intel by default"
@@ -130,8 +125,6 @@ config DRM_I915_KMS
           the driver to bind to PCI devices, which precludes loading things
           like intelfb.
 
-endchoice
-
 config DRM_MGA
         tristate "Matrox g200/g400"
         depends on DRM && PCI
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 997c43d04909..89cf05a72d1c 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -12,7 +12,7 @@ drm-y       :=	drm_auth.o drm_buffer.o drm_bufs.o drm_cache.o \
                 drm_platform.o drm_sysfs.o drm_hashtab.o drm_sman.o drm_mm.o \
                 drm_crtc.o drm_modes.o drm_edid.o \
                 drm_info.o drm_debugfs.o drm_encoder_slave.o \
-                drm_trace_points.o drm_global.o
+                drm_trace_points.o drm_global.o drm_usb.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 
@@ -29,7 +29,6 @@ obj-$(CONFIG_DRM_R128)	+= r128/
 obj-$(CONFIG_DRM_RADEON)+= radeon/
 obj-$(CONFIG_DRM_MGA)   += mga/
 obj-$(CONFIG_DRM_I810)  += i810/
-obj-$(CONFIG_DRM_I830)  += i830/
 obj-$(CONFIG_DRM_I915)  += i915/
 obj-$(CONFIG_DRM_SIS)   += sis/
 obj-$(CONFIG_DRM_SAVAGE)+= savage/
diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c
index 252fdb98b73a..0cb2ba50af53 100644
--- a/drivers/gpu/drm/drm_agpsupport.c
+++ b/drivers/gpu/drm/drm_agpsupport.c
@@ -466,10 +466,4 @@ drm_agp_bind_pages(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_agp_bind_pages);
 
-void drm_agp_chipset_flush(struct drm_device *dev)
-{
-        agp_flush_chipset(dev->agp->bridge);
-}
-EXPORT_SYMBOL(drm_agp_chipset_flush);
-
 #endif /* __OS_HAS_AGP */
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 6985cb1da72c..4c95b5fd9df3 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -156,12 +156,12 @@ static struct drm_conn_prop_enum_list drm_connector_enum_list[] =
         { DRM_MODE_CONNECTOR_SVIDEO, "SVIDEO", 0 },
         { DRM_MODE_CONNECTOR_LVDS, "LVDS", 0 },
         { DRM_MODE_CONNECTOR_Component, "Component", 0 },
-        { DRM_MODE_CONNECTOR_9PinDIN, "9-pin DIN", 0 },
-        { DRM_MODE_CONNECTOR_DisplayPort, "DisplayPort", 0 },
-        { DRM_MODE_CONNECTOR_HDMIA, "HDMI Type A", 0 },
-        { DRM_MODE_CONNECTOR_HDMIB, "HDMI Type B", 0 },
+        { DRM_MODE_CONNECTOR_9PinDIN, "DIN", 0 },
+        { DRM_MODE_CONNECTOR_DisplayPort, "DP", 0 },
+        { DRM_MODE_CONNECTOR_HDMIA, "HDMI-A", 0 },
+        { DRM_MODE_CONNECTOR_HDMIB, "HDMI-B", 0 },
         { DRM_MODE_CONNECTOR_TV, "TV", 0 },
-        { DRM_MODE_CONNECTOR_eDP, "Embedded DisplayPort", 0 },
+        { DRM_MODE_CONNECTOR_eDP, "eDP", 0 },
 };
 
 static struct drm_prop_enum_list drm_encoder_enum_list[] =
@@ -2674,3 +2674,56 @@ out:
         mutex_unlock(&dev->mode_config.mutex);
         return ret;
 }
+
+void drm_mode_config_reset(struct drm_device *dev)
+{
+        struct drm_crtc *crtc;
+        struct drm_encoder *encoder;
+        struct drm_connector *connector;
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+                if (crtc->funcs->reset)
+                        crtc->funcs->reset(crtc);
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
+                if (encoder->funcs->reset)
+                        encoder->funcs->reset(encoder);
+
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
+                if (connector->funcs->reset)
+                        connector->funcs->reset(connector);
+}
+EXPORT_SYMBOL(drm_mode_config_reset);
+
+int drm_mode_create_dumb_ioctl(struct drm_device *dev,
+                               void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_create_dumb *args = data;
+
+        if (!dev->driver->dumb_create)
+                return -ENOSYS;
+        return dev->driver->dumb_create(file_priv, dev, args);
+}
+
+int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
+                             void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_map_dumb *args = data;
+
+        /* call driver ioctl to get mmap offset */
+        if (!dev->driver->dumb_map_offset)
+                return -ENOSYS;
+
+        return dev->driver->dumb_map_offset(file_priv, dev, args->handle, &args->offset);
+}
+
+int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
+                                void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_destroy_dumb *args = data;
+
+        if (!dev->driver->dumb_destroy)
+                return -ENOSYS;
+
+        return dev->driver->dumb_destroy(file_priv, dev, args->handle);
+}
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 4c200931a6bc..92369655dca3 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -343,13 +343,12 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
         struct drm_encoder *encoder;
         bool ret = true;
 
-        adjusted_mode = drm_mode_duplicate(dev, mode);
-
         crtc->enabled = drm_helper_crtc_in_use(crtc);
-
         if (!crtc->enabled)
                 return true;
 
+        adjusted_mode = drm_mode_duplicate(dev, mode);
+
         saved_hwmode = crtc->hwmode;
         saved_mode = crtc->mode;
         saved_x = crtc->x;
@@ -437,10 +436,9 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
          */
         drm_calc_timestamping_constants(crtc);
 
-        /* XXX free adjustedmode */
-        drm_mode_destroy(dev, adjusted_mode);
         /* FIXME: add subpixel order */
 done:
+        drm_mode_destroy(dev, adjusted_mode);
         if (!ret) {
                 crtc->hwmode = saved_hwmode;
                 crtc->mode = saved_mode;
@@ -482,6 +480,7 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
         int count = 0, ro, fail = 0;
         struct drm_crtc_helper_funcs *crtc_funcs;
         int ret = 0;
+        int i;
 
         DRM_DEBUG_KMS("\n");
 
@@ -496,14 +495,17 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 
         crtc_funcs = set->crtc->helper_private;
 
+        if (!set->mode)
+                set->fb = NULL;
+
         if (set->fb) {
                 DRM_DEBUG_KMS("[CRTC:%d] [FB:%d] #connectors=%d (x y) (%i %i)\n",
                                 set->crtc->base.id, set->fb->base.id,
                                 (int)set->num_connectors, set->x, set->y);
         } else {
-                DRM_DEBUG_KMS("[CRTC:%d] [NOFB] #connectors=%d (x y) (%i %i)\n",
-                                set->crtc->base.id, (int)set->num_connectors,
-                                set->x, set->y);
+                DRM_DEBUG_KMS("[CRTC:%d] [NOFB]\n", set->crtc->base.id);
+                set->mode = NULL;
+                set->num_connectors = 0;
         }
 
         dev = set->crtc->dev;
@@ -648,8 +650,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                 mode_changed = true;
 
         if (mode_changed) {
-                set->crtc->enabled = (set->mode != NULL);
-                if (set->mode != NULL) {
+                set->crtc->enabled = drm_helper_crtc_in_use(set->crtc);
+                if (set->crtc->enabled) {
                         DRM_DEBUG_KMS("attempting to set mode from"
                                         " userspace\n");
                         drm_mode_debug_printmodeline(set->mode);
@@ -660,9 +662,16 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                                                       old_fb)) {
                                 DRM_ERROR("failed to set mode on [CRTC:%d]\n",
                                           set->crtc->base.id);
+                                set->crtc->fb = old_fb;
                                 ret = -EINVAL;
                                 goto fail;
                         }
+                        DRM_DEBUG_KMS("Setting connector DPMS state to on\n");
+                        for (i = 0; i < set->num_connectors; i++) {
+                                DRM_DEBUG_KMS("\t[CONNECTOR:%d:%s] set DPMS on\n", set->connectors[i]->base.id,
+                                              drm_get_connector_name(set->connectors[i]));
+                                set->connectors[i]->dpms = DRM_MODE_DPMS_ON;
+                        }
                 }
                 drm_helper_disable_unused_functions(dev);
         } else if (fb_changed) {
@@ -674,8 +683,10 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                         set->crtc->fb = set->fb;
                 ret = crtc_funcs->mode_set_base(set->crtc,
                                                 set->x, set->y, old_fb);
-                if (ret != 0)
+                if (ret != 0) {
+                        set->crtc->fb = old_fb;
                         goto fail;
+                }
         }
 
         kfree(save_connectors);
@@ -852,7 +863,7 @@ static void output_poll_execute(struct work_struct *work)
         struct delayed_work *delayed_work = to_delayed_work(work);
         struct drm_device *dev = container_of(delayed_work, struct drm_device, mode_config.output_poll_work);
         struct drm_connector *connector;
-        enum drm_connector_status old_status, status;
+        enum drm_connector_status old_status;
         bool repoll = false, changed = false;
 
         if (!drm_kms_helper_poll)
@@ -877,8 +888,12 @@ static void output_poll_execute(struct work_struct *work)
                     !(connector->polled & DRM_CONNECTOR_POLL_HPD))
                         continue;
 
-                status = connector->funcs->detect(connector, false);
-                if (old_status != status)
+                connector->status = connector->funcs->detect(connector, false);
+                DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
+                              connector->base.id,
+                              drm_get_connector_name(connector),
+                              old_status, connector->status);
+                if (old_status != connector->status)
                         changed = true;
         }
 
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 271835a71570..0d04914eb058 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -150,7 +150,10 @@ static struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-        DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED)
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED)
 };
 
 #define DRM_CORE_IOCTL_COUNT    ARRAY_SIZE( drm_ioctls )
@@ -234,49 +237,6 @@ int drm_lastclose(struct drm_device * dev)
         return 0;
 }
 
-/**
- * Module initialization. Called via init_module at module load time, or via
- * linux/init/main.c (this is not currently supported).
- *
- * \return zero on success or a negative number on failure.
- *
- * Initializes an array of drm_device structures, and attempts to
- * initialize all available devices, using consecutive minors, registering the
- * stubs and initializing the device.
- *
- * Expands the \c DRIVER_PREINIT and \c DRIVER_POST_INIT macros before and
- * after the initialization for driver customization.
- */
-int drm_init(struct drm_driver *driver)
-{
-        DRM_DEBUG("\n");
-        INIT_LIST_HEAD(&driver->device_list);
-
-        if (driver->driver_features & DRIVER_USE_PLATFORM_DEVICE)
-                return drm_platform_init(driver);
-        else
-                return drm_pci_init(driver);
-}
-
-EXPORT_SYMBOL(drm_init);
-
-void drm_exit(struct drm_driver *driver)
-{
-        struct drm_device *dev, *tmp;
-        DRM_DEBUG("\n");
-
-        if (driver->driver_features & DRIVER_MODESET) {
-                pci_unregister_driver(&driver->pci_driver);
-        } else {
-                list_for_each_entry_safe(dev, tmp, &driver->device_list, driver_item)
-                        drm_put_dev(dev);
-        }
-
-        DRM_INFO("Module unloaded\n");
-}
-
-EXPORT_SYMBOL(drm_exit);
-
 /** File operations structure */
 static const struct file_operations drm_stub_fops = {
         .owner = THIS_MODULE,
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index a245d17165ae..af60d9be9632 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -449,12 +449,11 @@ static void edid_fixup_preferred(struct drm_connector *connector,
 struct drm_display_mode *drm_mode_find_dmt(struct drm_device *dev,
                                            int hsize, int vsize, int fresh)
 {
+        struct drm_display_mode *mode = NULL;
         int i;
-        struct drm_display_mode *ptr, *mode;
 
-        mode = NULL;
         for (i = 0; i < drm_num_dmt_modes; i++) {
-                ptr = &drm_dmt_modes[i];
+                const struct drm_display_mode *ptr = &drm_dmt_modes[i];
                 if (hsize == ptr->hdisplay &&
                         vsize == ptr->vdisplay &&
                         fresh == drm_mode_vrefresh(ptr)) {
@@ -885,7 +884,7 @@ static struct drm_display_mode *drm_mode_detailed(struct drm_device *dev,
 }
 
 static bool
-mode_is_rb(struct drm_display_mode *mode)
+mode_is_rb(const struct drm_display_mode *mode)
 {
         return (mode->htotal - mode->hdisplay == 160) &&
                (mode->hsync_end - mode->hdisplay == 80) &&
@@ -894,7 +893,8 @@ mode_is_rb(struct drm_display_mode *mode)
 }
 
 static bool
-mode_in_hsync_range(struct drm_display_mode *mode, struct edid *edid, u8 *t)
+mode_in_hsync_range(const struct drm_display_mode *mode,
+                    struct edid *edid, u8 *t)
 {
         int hsync, hmin, hmax;
 
@@ -910,7 +910,8 @@ mode_in_hsync_range(struct drm_display_mode *mode, struct edid *edid, u8 *t)
 }
 
 static bool
-mode_in_vsync_range(struct drm_display_mode *mode, struct edid *edid, u8 *t)
+mode_in_vsync_range(const struct drm_display_mode *mode,
+                    struct edid *edid, u8 *t)
 {
         int vsync, vmin, vmax;
 
@@ -941,7 +942,7 @@ range_pixel_clock(struct edid *edid, u8 *t)
 }
 
 static bool
-mode_in_range(struct drm_display_mode *mode, struct edid *edid,
+mode_in_range(const struct drm_display_mode *mode, struct edid *edid,
               struct detailed_timing *timing)
 {
         u32 max_clock;
@@ -1472,7 +1473,7 @@ int drm_add_modes_noedid(struct drm_connector *connector,
                         int hdisplay, int vdisplay)
 {
         int i, count, num_modes = 0;
-        struct drm_display_mode *mode, *ptr;
+        struct drm_display_mode *mode;
         struct drm_device *dev = connector->dev;
 
         count = sizeof(drm_dmt_modes) / sizeof(struct drm_display_mode);
@@ -1482,7 +1483,7 @@ int drm_add_modes_noedid(struct drm_connector *connector,
                 vdisplay = 0;
 
         for (i = 0; i < count; i++) {
-                ptr = &drm_dmt_modes[i];
+                const struct drm_display_mode *ptr = &drm_dmt_modes[i];
                 if (hdisplay && vdisplay) {
                         /*
                          * Only when two are valid, they will be used to check
diff --git a/drivers/gpu/drm/drm_edid_modes.h b/drivers/gpu/drm/drm_edid_modes.h
index 6eb7592e152f..5f2064489fd5 100644
--- a/drivers/gpu/drm/drm_edid_modes.h
+++ b/drivers/gpu/drm/drm_edid_modes.h
@@ -32,7 +32,7 @@
  * This table is copied from xfree86/modes/xf86EdidModes.c.
  * But the mode with Reduced blank feature is deleted.
  */
-static struct drm_display_mode drm_dmt_modes[] = {
+static const struct drm_display_mode drm_dmt_modes[] = {
         /* 640x350@85Hz */
         { DRM_MODE("640x350", DRM_MODE_TYPE_DRIVER, 31500, 640, 672,
                    736, 832, 0, 350, 382, 385, 445, 0,
@@ -266,7 +266,7 @@ static struct drm_display_mode drm_dmt_modes[] = {
 static const int drm_num_dmt_modes =
         sizeof(drm_dmt_modes) / sizeof(struct drm_display_mode);
 
-static struct drm_display_mode edid_est_modes[] = {
+static const struct drm_display_mode edid_est_modes[] = {
         { DRM_MODE("800x600", DRM_MODE_TYPE_DRIVER, 40000, 800, 840,
                    968, 1056, 0, 600, 601, 605, 628, 0,
                    DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) }, /* 800x600@60Hz */
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index d2849e4ea4d0..d421f9d58d46 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -627,6 +627,11 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
                 value = (red << info->var.red.offset) |
                         (green << info->var.green.offset) |
                         (blue << info->var.blue.offset);
+                if (info->var.transp.length > 0) {
+                        u32 mask = (1 << info->var.transp.length) - 1;
+                        mask <<= info->var.transp.offset;
+                        value |= mask;
+                }
                 palette[regno] = value;
                 return 0;
         }
@@ -985,6 +990,8 @@ void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
         info->fix.type = FB_TYPE_PACKED_PIXELS;
         info->fix.visual = depth == 8 ? FB_VISUAL_PSEUDOCOLOR :
                 FB_VISUAL_TRUECOLOR;
+        info->fix.mmio_start = 0;
+        info->fix.mmio_len = 0;
         info->fix.type_aux = 0;
         info->fix.xpanstep = 1; /* doing it in hw */
         info->fix.ypanstep = 1; /* doing it in hw */
@@ -1005,6 +1012,7 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe
         info->var.xres_virtual = fb->width;
         info->var.yres_virtual = fb->height;
         info->var.bits_per_pixel = fb->bits_per_pixel;
+        info->var.accel_flags = FB_ACCELF_TEXT;
         info->var.xoffset = 0;
         info->var.yoffset = 0;
         info->var.activate = FB_ACTIVATE_NOW;
@@ -1530,3 +1538,24 @@ bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 }
 EXPORT_SYMBOL(drm_fb_helper_hotplug_event);
 
+/* The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT)
+ * but the module doesn't depend on any fb console symbols.  At least
+ * attempt to load fbcon to avoid leaving the system without a usable console.
+ */
+#if defined(CONFIG_FRAMEBUFFER_CONSOLE_MODULE) && !defined(CONFIG_EXPERT)
+static int __init drm_fb_helper_modinit(void)
+{
+        const char *name = "fbcon";
+        struct module *fbcon;
+
+        mutex_lock(&module_mutex);
+        fbcon = find_module(name);
+        mutex_unlock(&module_mutex);
+
+        if (!fbcon)
+                request_module_nowait(name);
+        return 0;
+}
+
+module_init(drm_fb_helper_modinit);
+#endif
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index a39794bac04b..2ec7d48fc4a8 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -236,6 +236,8 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
                 return -EBUSY;  /* No exclusive opens */
         if (!drm_cpu_valid())
                 return -EINVAL;
+        if (dev->switch_power_state != DRM_SWITCH_POWER_ON)
+                return -EINVAL;
 
         DRM_DEBUG("pid = %d, minor = %d\n", task_pid_nr(current), minor_id);
 
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index ea1c4b019ebf..57ce27c9a747 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -101,7 +101,7 @@ drm_gem_init(struct drm_device *dev)
 
         dev->mm_private = mm;
 
-        if (drm_ht_create(&mm->offset_hash, 19)) {
+        if (drm_ht_create(&mm->offset_hash, 12)) {
                 kfree(mm);
                 return -ENOMEM;
         }
@@ -181,7 +181,7 @@ EXPORT_SYMBOL(drm_gem_object_alloc);
 /**
  * Removes the mapping from handle to filp for this object.
  */
-static int
+int
 drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 {
         struct drm_device *dev;
@@ -214,6 +214,7 @@ drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 
         return 0;
 }
+EXPORT_SYMBOL(drm_gem_handle_delete);
 
 /**
  * Create a handle for this object. This adds a handle reference
diff --git a/drivers/gpu/drm/drm_hashtab.c b/drivers/gpu/drm/drm_hashtab.c
index a93d7b4ddaa6..e3a75688f3cd 100644
--- a/drivers/gpu/drm/drm_hashtab.c
+++ b/drivers/gpu/drm/drm_hashtab.c
@@ -39,27 +39,18 @@
 
 int drm_ht_create(struct drm_open_hash *ht, unsigned int order)
 {
-        unsigned int i;
+        unsigned int size = 1 << order;
 
-        ht->size = 1 << order;
         ht->order = order;
-        ht->fill = 0;
         ht->table = NULL;
-        ht->use_vmalloc = ((ht->size * sizeof(*ht->table)) > PAGE_SIZE);
-        if (!ht->use_vmalloc) {
-                ht->table = kcalloc(ht->size, sizeof(*ht->table), GFP_KERNEL);
-        }
-        if (!ht->table) {
-                ht->use_vmalloc = 1;
-                ht->table = vmalloc(ht->size*sizeof(*ht->table));
-        }
+        if (size <= PAGE_SIZE / sizeof(*ht->table))
+                ht->table = kcalloc(size, sizeof(*ht->table), GFP_KERNEL);
+        else
+                ht->table = vzalloc(size*sizeof(*ht->table));
         if (!ht->table) {
                 DRM_ERROR("Out of memory for hash table\n");
                 return -ENOMEM;
         }
-        for (i=0; i< ht->size; ++i) {
-                INIT_HLIST_HEAD(&ht->table[i]);
-        }
         return 0;
 }
 EXPORT_SYMBOL(drm_ht_create);
@@ -180,7 +171,6 @@ int drm_ht_remove_key(struct drm_open_hash *ht, unsigned long key)
         list = drm_ht_find_key(ht, key);
         if (list) {
                 hlist_del_init(list);
-                ht->fill--;
                 return 0;
         }
         return -EINVAL;
@@ -189,7 +179,6 @@ int drm_ht_remove_key(struct drm_open_hash *ht, unsigned long key)
 int drm_ht_remove_item(struct drm_open_hash *ht, struct drm_hash_item *item)
 {
         hlist_del_init(&item->head);
-        ht->fill--;
         return 0;
 }
 EXPORT_SYMBOL(drm_ht_remove_item);
@@ -197,10 +186,10 @@ EXPORT_SYMBOL(drm_ht_remove_item);
 void drm_ht_remove(struct drm_open_hash *ht)
 {
         if (ht->table) {
-                if (ht->use_vmalloc)
-                        vfree(ht->table);
-                else
+                if ((PAGE_SIZE / sizeof(*ht->table)) >> ht->order)
                         kfree(ht->table);
+                else
+                        vfree(ht->table);
                 ht->table = NULL;
         }
 }
diff --git a/drivers/gpu/drm/drm_info.c b/drivers/gpu/drm/drm_info.c
index 3cdbaf379bb5..812aaac4438a 100644
--- a/drivers/gpu/drm/drm_info.c
+++ b/drivers/gpu/drm/drm_info.c
@@ -47,30 +47,19 @@ int drm_name_info(struct seq_file *m, void *data)
         struct drm_minor *minor = node->minor;
         struct drm_device *dev = minor->dev;
         struct drm_master *master = minor->master;
-
+        const char *bus_name;
         if (!master)
                 return 0;
 
-        if (drm_core_check_feature(dev, DRIVER_USE_PLATFORM_DEVICE)) {
-                if (master->unique) {
-                        seq_printf(m, "%s %s %s\n",
-                                        dev->driver->platform_device->name,
-                                        dev_name(dev->dev), master->unique);
-                } else {
-                        seq_printf(m, "%s\n",
-                                dev->driver->platform_device->name);
-                }
+        bus_name = dev->driver->bus->get_name(dev);
+        if (master->unique) {
+                seq_printf(m, "%s %s %s\n",
+                           bus_name,
+                           dev_name(dev->dev), master->unique);
         } else {
-                if (master->unique) {
-                        seq_printf(m, "%s %s %s\n",
-                                dev->driver->pci_driver.name,
-                                dev_name(dev->dev), master->unique);
-                } else {
-                        seq_printf(m, "%s %s\n", dev->driver->pci_driver.name,
-                                dev_name(dev->dev));
-                }
+                seq_printf(m, "%s %s\n",
+                           bus_name, dev_name(dev->dev));
         }
-
         return 0;
 }
 
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 47db4df37a69..117490590f56 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -96,7 +96,7 @@ int drm_setunique(struct drm_device *dev, void *data,
 {
         struct drm_unique *u = data;
         struct drm_master *master = file_priv->master;
-        int domain, bus, slot, func, ret;
+        int ret;
 
         if (master->unique_len || master->unique)
                 return -EBUSY;
@@ -104,50 +104,12 @@ int drm_setunique(struct drm_device *dev, void *data,
         if (!u->unique_len || u->unique_len > 1024)
                 return -EINVAL;
 
-        master->unique_len = u->unique_len;
-        master->unique_size = u->unique_len + 1;
-        master->unique = kmalloc(master->unique_size, GFP_KERNEL);
-        if (!master->unique) {
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        if (copy_from_user(master->unique, u->unique, master->unique_len)) {
-                ret = -EFAULT;
-                goto err;
-        }
-
-        master->unique[master->unique_len] = '\0';
-
-        dev->devname = kmalloc(strlen(dev->driver->pci_driver.name) +
-                               strlen(master->unique) + 2, GFP_KERNEL);
-        if (!dev->devname) {
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        sprintf(dev->devname, "%s@%s", dev->driver->pci_driver.name,
-                master->unique);
-
-        /* Return error if the busid submitted doesn't match the device's actual
-         * busid.
-         */
-        ret = sscanf(master->unique, "PCI:%d:%d:%d", &bus, &slot, &func);
-        if (ret != 3) {
-                ret = -EINVAL;
-                goto err;
-        }
-
-        domain = bus >> 8;
-        bus &= 0xff;
+        if (!dev->driver->bus->set_unique)
+                return -EINVAL;
 
-        if ((domain != drm_get_pci_domain(dev)) ||
-            (bus != dev->pdev->bus->number) ||
-            (slot != PCI_SLOT(dev->pdev->devfn)) ||
-            (func != PCI_FUNC(dev->pdev->devfn))) {
-                ret = -EINVAL;
+        ret = dev->driver->bus->set_unique(dev, master, u);
+        if (ret)
                 goto err;
-        }
 
         return 0;
 
@@ -159,74 +121,15 @@ err:
 static int drm_set_busid(struct drm_device *dev, struct drm_file *file_priv)
 {
         struct drm_master *master = file_priv->master;
-        int len, ret;
+        int ret;
 
         if (master->unique != NULL)
                 drm_unset_busid(dev, master);
 
-        if (drm_core_check_feature(dev, DRIVER_USE_PLATFORM_DEVICE)) {
-                master->unique_len = 10 + strlen(dev->platformdev->name);
-                master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
-
-                if (master->unique == NULL)
-                        return -ENOMEM;
-
-                len = snprintf(master->unique, master->unique_len,
-                        "platform:%s", dev->platformdev->name);
-
-                if (len > master->unique_len) {
-                        DRM_ERROR("Unique buffer overflowed\n");
-                        ret = -EINVAL;
-                        goto err;
-                }
-
-                dev->devname =
-                        kmalloc(strlen(dev->platformdev->name) +
-                                master->unique_len + 2, GFP_KERNEL);
-
-                if (dev->devname == NULL) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-
-                sprintf(dev->devname, "%s@%s", dev->platformdev->name,
-                        master->unique);
-
-        } else {
-                master->unique_len = 40;
-                master->unique_size = master->unique_len;
-                master->unique = kmalloc(master->unique_size, GFP_KERNEL);
-                if (master->unique == NULL)
-                        return -ENOMEM;
-
-                len = snprintf(master->unique, master->unique_len,
-                        "pci:%04x:%02x:%02x.%d",
-                        drm_get_pci_domain(dev),
-                        dev->pdev->bus->number,
-                        PCI_SLOT(dev->pdev->devfn),
-                        PCI_FUNC(dev->pdev->devfn));
-                if (len >= master->unique_len) {
-                        DRM_ERROR("buffer overflow");
-                        ret = -EINVAL;
-                        goto err;
-                } else
-                        master->unique_len = len;
-
-                dev->devname =
-                        kmalloc(strlen(dev->driver->pci_driver.name) +
-                                master->unique_len + 2, GFP_KERNEL);
-
-                if (dev->devname == NULL) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-
-                sprintf(dev->devname, "%s@%s", dev->driver->pci_driver.name,
-                        master->unique);
-        }
-
+        ret = dev->driver->bus->set_busid(dev, master);
+        if (ret)
+                goto err;
         return 0;
-
 err:
         drm_unset_busid(dev, master);
         return ret;
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 4e82d0d3c378..cb49685bde01 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -74,23 +74,13 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
 {
         struct drm_irq_busid *p = data;
 
-        if (drm_core_check_feature(dev, DRIVER_USE_PLATFORM_DEVICE))
+        if (!dev->driver->bus->irq_by_busid)
                 return -EINVAL;
 
         if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
                 return -EINVAL;
 
-        if ((p->busnum >> 8) != drm_get_pci_domain(dev) ||
-            (p->busnum & 0xff) != dev->pdev->bus->number ||
-            p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn))
-                return -EINVAL;
-
-        p->irq = dev->pdev->irq;
-
-        DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum,
-                  p->irq);
-
-        return 0;
+        return dev->driver->bus->irq_by_busid(dev, p);
 }
 
 /*
@@ -1047,10 +1037,13 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
         struct timeval now;
         unsigned long flags;
         unsigned int seq;
+        int ret;
 
         e = kzalloc(sizeof *e, GFP_KERNEL);
-        if (e == NULL)
-                return -ENOMEM;
+        if (e == NULL) {
+                ret = -ENOMEM;
+                goto err_put;
+        }
 
         e->pipe = pipe;
         e->base.pid = current->pid;
@@ -1064,9 +1057,8 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
         spin_lock_irqsave(&dev->event_lock, flags);
 
         if (file_priv->event_space < sizeof e->event) {
-                spin_unlock_irqrestore(&dev->event_lock, flags);
-                kfree(e);
-                return -ENOMEM;
+                ret = -EBUSY;
+                goto err_unlock;
         }
 
         file_priv->event_space -= sizeof e->event;
@@ -1086,20 +1078,30 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
 
         e->event.sequence = vblwait->request.sequence;
         if ((seq - vblwait->request.sequence) <= (1 << 23)) {
+                e->event.sequence = seq;
                 e->event.tv_sec = now.tv_sec;
                 e->event.tv_usec = now.tv_usec;
-                drm_vblank_put(dev, e->pipe);
+                drm_vblank_put(dev, pipe);
                 list_add_tail(&e->base.link, &e->base.file_priv->event_list);
                 wake_up_interruptible(&e->base.file_priv->event_wait);
+                vblwait->reply.sequence = seq;
                 trace_drm_vblank_event_delivered(current->pid, pipe,
                                                  vblwait->request.sequence);
         } else {
                 list_add_tail(&e->base.link, &dev->vblank_event_list);
+                vblwait->reply.sequence = vblwait->request.sequence;
         }
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
         return 0;
+
+err_unlock:
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+        kfree(e);
+err_put:
+        drm_vblank_put(dev, pipe);
+        return ret;
 }
 
 /**
@@ -1238,7 +1240,7 @@ void drm_handle_vblank_events(struct drm_device *dev, int crtc)
  * Drivers should call this routine in their vblank interrupt handlers to
  * update the vblank counter and send any signals that may be pending.
  */
-void drm_handle_vblank(struct drm_device *dev, int crtc)
+bool drm_handle_vblank(struct drm_device *dev, int crtc)
 {
         u32 vblcount;
         s64 diff_ns;
@@ -1246,7 +1248,7 @@ void drm_handle_vblank(struct drm_device *dev, int crtc)
         unsigned long irqflags;
 
         if (!dev->num_crtcs)
-                return;
+                return false;
 
         /* Need timestamp lock to prevent concurrent execution with
          * vblank enable/disable, as this would cause inconsistent
@@ -1257,7 +1259,7 @@ void drm_handle_vblank(struct drm_device *dev, int crtc)
         /* Vblank irq handling disabled. Nothing to do. */
         if (!dev->vblank_enabled[crtc]) {
                 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
-                return;
+                return false;
         }
 
         /* Fetch corresponding timestamp for this vblank interval from
@@ -1299,5 +1301,6 @@ void drm_handle_vblank(struct drm_device *dev, int crtc)
         drm_handle_vblank_events(dev, crtc);
 
         spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        return true;
 }
 EXPORT_SYMBOL(drm_handle_vblank);
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index a6bfc302ed90..add1737dae0d 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -64,8 +64,8 @@ static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic)
                 else {
                         child =
                             list_entry(mm->unused_nodes.next,
-                                       struct drm_mm_node, free_stack);
-                        list_del(&child->free_stack);
+                                       struct drm_mm_node, node_list);
+                        list_del(&child->node_list);
                         --mm->num_unused;
                 }
                 spin_unlock(&mm->unused_lock);
@@ -94,195 +94,242 @@ int drm_mm_pre_get(struct drm_mm *mm)
                         return ret;
                 }
                 ++mm->num_unused;
-                list_add_tail(&node->free_stack, &mm->unused_nodes);
+                list_add_tail(&node->node_list, &mm->unused_nodes);
         }
         spin_unlock(&mm->unused_lock);
         return 0;
 }
 EXPORT_SYMBOL(drm_mm_pre_get);
 
-static int drm_mm_create_tail_node(struct drm_mm *mm,
-                                   unsigned long start,
-                                   unsigned long size, int atomic)
+static inline unsigned long drm_mm_hole_node_start(struct drm_mm_node *hole_node)
 {
-        struct drm_mm_node *child;
-
-        child = drm_mm_kmalloc(mm, atomic);
-        if (unlikely(child == NULL))
-                return -ENOMEM;
-
-        child->free = 1;
-        child->size = size;
-        child->start = start;
-        child->mm = mm;
+        return hole_node->start + hole_node->size;
+}
 
-        list_add_tail(&child->node_list, &mm->node_list);
-        list_add_tail(&child->free_stack, &mm->free_stack);
+static inline unsigned long drm_mm_hole_node_end(struct drm_mm_node *hole_node)
+{
+        struct drm_mm_node *next_node =
+                list_entry(hole_node->node_list.next, struct drm_mm_node,
+                           node_list);
 
-        return 0;
+        return next_node->start;
 }
 
-static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent,
-                                                 unsigned long size,
-                                                 int atomic)
+static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
+                                 struct drm_mm_node *node,
+                                 unsigned long size, unsigned alignment)
 {
-        struct drm_mm_node *child;
+        struct drm_mm *mm = hole_node->mm;
+        unsigned long tmp = 0, wasted = 0;
+        unsigned long hole_start = drm_mm_hole_node_start(hole_node);
+        unsigned long hole_end = drm_mm_hole_node_end(hole_node);
 
-        child = drm_mm_kmalloc(parent->mm, atomic);
-        if (unlikely(child == NULL))
-                return NULL;
+        BUG_ON(!hole_node->hole_follows || node->allocated);
 
-        INIT_LIST_HEAD(&child->free_stack);
+        if (alignment)
+                tmp = hole_start % alignment;
 
-        child->size = size;
-        child->start = parent->start;
-        child->mm = parent->mm;
+        if (!tmp) {
+                hole_node->hole_follows = 0;
+                list_del_init(&hole_node->hole_stack);
+        } else
+                wasted = alignment - tmp;
 
-        list_add_tail(&child->node_list, &parent->node_list);
-        INIT_LIST_HEAD(&child->free_stack);
+        node->start = hole_start + wasted;
+        node->size = size;
+        node->mm = mm;
+        node->allocated = 1;
 
-        parent->size -= size;
-        parent->start += size;
-        return child;
-}
+        INIT_LIST_HEAD(&node->hole_stack);
+        list_add(&node->node_list, &hole_node->node_list);
 
+        BUG_ON(node->start + node->size > hole_end);
+
+        if (node->start + node->size < hole_end) {
+                list_add(&node->hole_stack, &mm->hole_stack);
+                node->hole_follows = 1;
+        } else {
+                node->hole_follows = 0;
+        }
+}
 
-struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *node,
+struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *hole_node,
                                              unsigned long size,
                                              unsigned alignment,
                                              int atomic)
 {
+        struct drm_mm_node *node;
+
+        node = drm_mm_kmalloc(hole_node->mm, atomic);
+        if (unlikely(node == NULL))
+                return NULL;
 
-        struct drm_mm_node *align_splitoff = NULL;
-        unsigned tmp = 0;
+        drm_mm_insert_helper(hole_node, node, size, alignment);
 
-        if (alignment)
-                tmp = node->start % alignment;
+        return node;
+}
+EXPORT_SYMBOL(drm_mm_get_block_generic);
 
-        if (tmp) {
-                align_splitoff =
-                    drm_mm_split_at_start(node, alignment - tmp, atomic);
-                if (unlikely(align_splitoff == NULL))
-                        return NULL;
-        }
+/**
+ * Search for free space and insert a preallocated memory node. Returns
+ * -ENOSPC if no suitable free area is available. The preallocated memory node
+ * must be cleared.
+ */
+int drm_mm_insert_node(struct drm_mm *mm, struct drm_mm_node *node,
+                       unsigned long size, unsigned alignment)
+{
+        struct drm_mm_node *hole_node;
 
-        if (node->size == size) {
-                list_del_init(&node->free_stack);
-                node->free = 0;
-        } else {
-                node = drm_mm_split_at_start(node, size, atomic);
-        }
+        hole_node = drm_mm_search_free(mm, size, alignment, 0);
+        if (!hole_node)
+                return -ENOSPC;
 
-        if (align_splitoff)
-                drm_mm_put_block(align_splitoff);
+        drm_mm_insert_helper(hole_node, node, size, alignment);
 
-        return node;
+        return 0;
 }
-EXPORT_SYMBOL(drm_mm_get_block_generic);
+EXPORT_SYMBOL(drm_mm_insert_node);
 
-struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *node,
-                                                unsigned long size,
-                                                unsigned alignment,
-                                                unsigned long start,
-                                                unsigned long end,
-                                                int atomic)
+static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
+                                       struct drm_mm_node *node,
+                                       unsigned long size, unsigned alignment,
+                                       unsigned long start, unsigned long end)
 {
-        struct drm_mm_node *align_splitoff = NULL;
-        unsigned tmp = 0;
-        unsigned wasted = 0;
+        struct drm_mm *mm = hole_node->mm;
+        unsigned long tmp = 0, wasted = 0;
+        unsigned long hole_start = drm_mm_hole_node_start(hole_node);
+        unsigned long hole_end = drm_mm_hole_node_end(hole_node);
 
-        if (node->start < start)
-                wasted += start - node->start;
+        BUG_ON(!hole_node->hole_follows || node->allocated);
+
+        if (hole_start < start)
+                wasted += start - hole_start;
         if (alignment)
-                tmp = ((node->start + wasted) % alignment);
+                tmp = (hole_start + wasted) % alignment;
 
         if (tmp)
                 wasted += alignment - tmp;
-        if (wasted) {
-                align_splitoff = drm_mm_split_at_start(node, wasted, atomic);
-                if (unlikely(align_splitoff == NULL))
-                        return NULL;
+
+        if (!wasted) {
+                hole_node->hole_follows = 0;
+                list_del_init(&hole_node->hole_stack);
         }
 
-        if (node->size == size) {
-                list_del_init(&node->free_stack);
-                node->free = 0;
+        node->start = hole_start + wasted;
+        node->size = size;
+        node->mm = mm;
+        node->allocated = 1;
+
+        INIT_LIST_HEAD(&node->hole_stack);
+        list_add(&node->node_list, &hole_node->node_list);
+
+        BUG_ON(node->start + node->size > hole_end);
+        BUG_ON(node->start + node->size > end);
+
+        if (node->start + node->size < hole_end) {
+                list_add(&node->hole_stack, &mm->hole_stack);
+                node->hole_follows = 1;
         } else {
-                node = drm_mm_split_at_start(node, size, atomic);
+                node->hole_follows = 0;
         }
+}
+
+struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *hole_node,
+                                                unsigned long size,
+                                                unsigned alignment,
+                                                unsigned long start,
+                                                unsigned long end,
+                                                int atomic)
+{
+        struct drm_mm_node *node;
 
-        if (align_splitoff)
-                drm_mm_put_block(align_splitoff);
+        node = drm_mm_kmalloc(hole_node->mm, atomic);
+        if (unlikely(node == NULL))
+                return NULL;
+
+        drm_mm_insert_helper_range(hole_node, node, size, alignment,
+                                   start, end);
 
         return node;
 }
 EXPORT_SYMBOL(drm_mm_get_block_range_generic);
 
-/*
- * Put a block. Merge with the previous and / or next block if they are free.
- * Otherwise add to the free stack.
+/**
+ * Search for free space and insert a preallocated memory node. Returns
+ * -ENOSPC if no suitable free area is available. This is for range
+ * restricted allocations. The preallocated memory node must be cleared.
  */
-
-void drm_mm_put_block(struct drm_mm_node *cur)
+int drm_mm_insert_node_in_range(struct drm_mm *mm, struct drm_mm_node *node,
+                                unsigned long size, unsigned alignment,
+                                unsigned long start, unsigned long end)
 {
+        struct drm_mm_node *hole_node;
 
-        struct drm_mm *mm = cur->mm;
-        struct list_head *cur_head = &cur->node_list;
-        struct list_head *root_head = &mm->node_list;
-        struct drm_mm_node *prev_node = NULL;
-        struct drm_mm_node *next_node;
+        hole_node = drm_mm_search_free_in_range(mm, size, alignment,
+                                                start, end, 0);
+        if (!hole_node)
+                return -ENOSPC;
 
-        int merged = 0;
+        drm_mm_insert_helper_range(hole_node, node, size, alignment,
+                                   start, end);
 
-        BUG_ON(cur->scanned_block || cur->scanned_prev_free
-                                  || cur->scanned_next_free);
+        return 0;
+}
+EXPORT_SYMBOL(drm_mm_insert_node_in_range);
 
-        if (cur_head->prev != root_head) {
-                prev_node =
-                    list_entry(cur_head->prev, struct drm_mm_node, node_list);
-                if (prev_node->free) {
-                        prev_node->size += cur->size;
-                        merged = 1;
-                }
-        }
-        if (cur_head->next != root_head) {
-                next_node =
-                    list_entry(cur_head->next, struct drm_mm_node, node_list);
-                if (next_node->free) {
-                        if (merged) {
-                                prev_node->size += next_node->size;
-                                list_del(&next_node->node_list);
-                                list_del(&next_node->free_stack);
-                                spin_lock(&mm->unused_lock);
-                                if (mm->num_unused < MM_UNUSED_TARGET) {
-                                        list_add(&next_node->free_stack,
-                                                 &mm->unused_nodes);
-                                        ++mm->num_unused;
-                                } else
-                                        kfree(next_node);
-                                spin_unlock(&mm->unused_lock);
-                        } else {
-                                next_node->size += cur->size;
-                                next_node->start = cur->start;
-                                merged = 1;
-                        }
-                }
-        }
-        if (!merged) {
-                cur->free = 1;
-                list_add(&cur->free_stack, &mm->free_stack);
-        } else {
-                list_del(&cur->node_list);
-                spin_lock(&mm->unused_lock);
-                if (mm->num_unused < MM_UNUSED_TARGET) {
-                        list_add(&cur->free_stack, &mm->unused_nodes);
-                        ++mm->num_unused;
-                } else
-                        kfree(cur);
-                spin_unlock(&mm->unused_lock);
-        }
+/**
+ * Remove a memory node from the allocator.
+ */
+void drm_mm_remove_node(struct drm_mm_node *node)
+{
+        struct drm_mm *mm = node->mm;
+        struct drm_mm_node *prev_node;
+
+        BUG_ON(node->scanned_block || node->scanned_prev_free
+                                   || node->scanned_next_free);
+
+        prev_node =
+            list_entry(node->node_list.prev, struct drm_mm_node, node_list);
+
+        if (node->hole_follows) {
+                BUG_ON(drm_mm_hole_node_start(node)
+                                == drm_mm_hole_node_end(node));
+                list_del(&node->hole_stack);
+        } else
+                BUG_ON(drm_mm_hole_node_start(node)
+                                != drm_mm_hole_node_end(node));
+
+        if (!prev_node->hole_follows) {
+                prev_node->hole_follows = 1;
+                list_add(&prev_node->hole_stack, &mm->hole_stack);
+        } else
+                list_move(&prev_node->hole_stack, &mm->hole_stack);
+
+        list_del(&node->node_list);
+        node->allocated = 0;
 }
+EXPORT_SYMBOL(drm_mm_remove_node);
+
+/*
+ * Remove a memory node from the allocator and free the allocated struct
+ * drm_mm_node. Only to be used on a struct drm_mm_node obtained by one of the
+ * drm_mm_get_block functions.
+ */
+void drm_mm_put_block(struct drm_mm_node *node)
+{
+
+        struct drm_mm *mm = node->mm;
+
+        drm_mm_remove_node(node);
 
+        spin_lock(&mm->unused_lock);
+        if (mm->num_unused < MM_UNUSED_TARGET) {
+                list_add(&node->node_list, &mm->unused_nodes);
+                ++mm->num_unused;
+        } else
+                kfree(node);
+        spin_unlock(&mm->unused_lock);
+}
 EXPORT_SYMBOL(drm_mm_put_block);
 
 static int check_free_hole(unsigned long start, unsigned long end,
@@ -319,8 +366,10 @@ struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm,
         best = NULL;
         best_size = ~0UL;
 
-        list_for_each_entry(entry, &mm->free_stack, free_stack) {
-                if (!check_free_hole(entry->start, entry->start + entry->size,
+        list_for_each_entry(entry, &mm->hole_stack, hole_stack) {
+                BUG_ON(!entry->hole_follows);
+                if (!check_free_hole(drm_mm_hole_node_start(entry),
+                                     drm_mm_hole_node_end(entry),
                                      size, alignment))
                         continue;
 
@@ -353,12 +402,13 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm,
         best = NULL;
         best_size = ~0UL;
 
-        list_for_each_entry(entry, &mm->free_stack, free_stack) {
-                unsigned long adj_start = entry->start < start ?
-                        start : entry->start;
-                unsigned long adj_end = entry->start + entry->size > end ?
-                        end : entry->start + entry->size;
+        list_for_each_entry(entry, &mm->hole_stack, hole_stack) {
+                unsigned long adj_start = drm_mm_hole_node_start(entry) < start ?
+                        start : drm_mm_hole_node_start(entry);
+                unsigned long adj_end = drm_mm_hole_node_end(entry) > end ?
+                        end : drm_mm_hole_node_end(entry);
 
+                BUG_ON(!entry->hole_follows);
                 if (!check_free_hole(adj_start, adj_end, size, alignment))
                         continue;
 
@@ -376,6 +426,23 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm,
 EXPORT_SYMBOL(drm_mm_search_free_in_range);
 
 /**
+ * Moves an allocation. To be used with embedded struct drm_mm_node.
+ */
+void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
+{
+        list_replace(&old->node_list, &new->node_list);
+        list_replace(&old->node_list, &new->hole_stack);
+        new->hole_follows = old->hole_follows;
+        new->mm = old->mm;
+        new->start = old->start;
+        new->size = old->size;
+
+        old->allocated = 0;
+        new->allocated = 1;
+}
+EXPORT_SYMBOL(drm_mm_replace_node);
+
+/**
  * Initializa lru scanning.
  *
  * This simply sets up the scanning routines with the parameters for the desired
@@ -392,10 +459,38 @@ void drm_mm_init_scan(struct drm_mm *mm, unsigned long size,
         mm->scanned_blocks = 0;
         mm->scan_hit_start = 0;
         mm->scan_hit_size = 0;
+        mm->scan_check_range = 0;
+        mm->prev_scanned_node = NULL;
 }
 EXPORT_SYMBOL(drm_mm_init_scan);
 
 /**
+ * Initializa lru scanning.
+ *
+ * This simply sets up the scanning routines with the parameters for the desired
+ * hole. This version is for range-restricted scans.
+ *
+ * Warning: As long as the scan list is non-empty, no other operations than
+ * adding/removing nodes to/from the scan list are allowed.
+ */
+void drm_mm_init_scan_with_range(struct drm_mm *mm, unsigned long size,
+                                 unsigned alignment,
+                                 unsigned long start,
+                                 unsigned long end)
+{
+        mm->scan_alignment = alignment;
+        mm->scan_size = size;
+        mm->scanned_blocks = 0;
+        mm->scan_hit_start = 0;
+        mm->scan_hit_size = 0;
+        mm->scan_start = start;
+        mm->scan_end = end;
+        mm->scan_check_range = 1;
+        mm->prev_scanned_node = NULL;
+}
+EXPORT_SYMBOL(drm_mm_init_scan_with_range);
+
+/**
  * Add a node to the scan list that might be freed to make space for the desired
  * hole.
  *
@@ -404,58 +499,42 @@ EXPORT_SYMBOL(drm_mm_init_scan);
 int drm_mm_scan_add_block(struct drm_mm_node *node)
 {
         struct drm_mm *mm = node->mm;
-        struct list_head *prev_free, *next_free;
-        struct drm_mm_node *prev_node, *next_node;
+        struct drm_mm_node *prev_node;
+        unsigned long hole_start, hole_end;
+        unsigned long adj_start;
+        unsigned long adj_end;
 
         mm->scanned_blocks++;
 
-        prev_free = next_free = NULL;
-
-        BUG_ON(node->free);
+        BUG_ON(node->scanned_block);
         node->scanned_block = 1;
-        node->free = 1;
-
-        if (node->node_list.prev != &mm->node_list) {
-                prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
-                                       node_list);
-
-                if (prev_node->free) {
-                        list_del(&prev_node->node_list);
-
-                        node->start = prev_node->start;
-                        node->size += prev_node->size;
-
-                        prev_node->scanned_prev_free = 1;
-
-                        prev_free = &prev_node->free_stack;
-                }
-        }
-
-        if (node->node_list.next != &mm->node_list) {
-                next_node = list_entry(node->node_list.next, struct drm_mm_node,
-                                       node_list);
 
-                if (next_node->free) {
-                        list_del(&next_node->node_list);
-
-                        node->size += next_node->size;
-
-                        next_node->scanned_next_free = 1;
-
-                        next_free = &next_node->free_stack;
-                }
+        prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
+                               node_list);
+
+        node->scanned_preceeds_hole = prev_node->hole_follows;
+        prev_node->hole_follows = 1;
+        list_del(&node->node_list);
+        node->node_list.prev = &prev_node->node_list;
+        node->node_list.next = &mm->prev_scanned_node->node_list;
+        mm->prev_scanned_node = node;
+
+        hole_start = drm_mm_hole_node_start(prev_node);
+        hole_end = drm_mm_hole_node_end(prev_node);
+        if (mm->scan_check_range) {
+                adj_start = hole_start < mm->scan_start ?
+                        mm->scan_start : hole_start;
+                adj_end = hole_end > mm->scan_end ?
+                        mm->scan_end : hole_end;
+        } else {
+                adj_start = hole_start;
+                adj_end = hole_end;
         }
 
-        /* The free_stack list is not used for allocated objects, so these two
-         * pointers can be abused (as long as no allocations in this memory
-         * manager happens). */
-        node->free_stack.prev = prev_free;
-        node->free_stack.next = next_free;
-
-        if (check_free_hole(node->start, node->start + node->size,
+        if (check_free_hole(adj_start , adj_end,
                             mm->scan_size, mm->scan_alignment)) {
-                mm->scan_hit_start = node->start;
-                mm->scan_hit_size = node->size;
+                mm->scan_hit_start = hole_start;
+                mm->scan_hit_size = hole_end;
 
                 return 1;
         }
@@ -481,39 +560,19 @@ EXPORT_SYMBOL(drm_mm_scan_add_block);
 int drm_mm_scan_remove_block(struct drm_mm_node *node)
 {
         struct drm_mm *mm = node->mm;
-        struct drm_mm_node *prev_node, *next_node;
+        struct drm_mm_node *prev_node;
 
         mm->scanned_blocks--;
 
         BUG_ON(!node->scanned_block);
         node->scanned_block = 0;
-        node->free = 0;
 
-        prev_node = list_entry(node->free_stack.prev, struct drm_mm_node,
-                               free_stack);
-        next_node = list_entry(node->free_stack.next, struct drm_mm_node,
-                               free_stack);
+        prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
+                               node_list);
 
-        if (prev_node) {
-                BUG_ON(!prev_node->scanned_prev_free);
-                prev_node->scanned_prev_free = 0;
-
-                list_add_tail(&prev_node->node_list, &node->node_list);
-
-                node->start = prev_node->start + prev_node->size;
-                node->size -= prev_node->size;
-        }
-
-        if (next_node) {
-                BUG_ON(!next_node->scanned_next_free);
-                next_node->scanned_next_free = 0;
-
-                list_add(&next_node->node_list, &node->node_list);
-
-                node->size -= next_node->size;
-        }
-
-        INIT_LIST_HEAD(&node->free_stack);
+        prev_node->hole_follows = node->scanned_preceeds_hole;
+        INIT_LIST_HEAD(&node->node_list);
+        list_add(&node->node_list, &prev_node->node_list);
 
         /* Only need to check for containement because start&size for the
          * complete resulting free block (not just the desired part) is
@@ -530,7 +589,7 @@ EXPORT_SYMBOL(drm_mm_scan_remove_block);
 
 int drm_mm_clean(struct drm_mm * mm)
 {
-        struct list_head *head = &mm->node_list;
+        struct list_head *head = &mm->head_node.node_list;
 
         return (head->next->next == head);
 }
@@ -538,38 +597,40 @@ EXPORT_SYMBOL(drm_mm_clean);
 
 int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
 {
-        INIT_LIST_HEAD(&mm->node_list);
-        INIT_LIST_HEAD(&mm->free_stack);
+        INIT_LIST_HEAD(&mm->hole_stack);
         INIT_LIST_HEAD(&mm->unused_nodes);
         mm->num_unused = 0;
         mm->scanned_blocks = 0;
         spin_lock_init(&mm->unused_lock);
 
-        return drm_mm_create_tail_node(mm, start, size, 0);
+        /* Clever trick to avoid a special case in the free hole tracking. */
+        INIT_LIST_HEAD(&mm->head_node.node_list);
+        INIT_LIST_HEAD(&mm->head_node.hole_stack);
+        mm->head_node.hole_follows = 1;
+        mm->head_node.scanned_block = 0;
+        mm->head_node.scanned_prev_free = 0;
+        mm->head_node.scanned_next_free = 0;
+        mm->head_node.mm = mm;
+        mm->head_node.start = start + size;
+        mm->head_node.size = start - mm->head_node.start;
+        list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
+
+        return 0;
 }
 EXPORT_SYMBOL(drm_mm_init);
 
 void drm_mm_takedown(struct drm_mm * mm)
 {
-        struct list_head *bnode = mm->free_stack.next;
-        struct drm_mm_node *entry;
-        struct drm_mm_node *next;
-
-        entry = list_entry(bnode, struct drm_mm_node, free_stack);
+        struct drm_mm_node *entry, *next;
 
-        if (entry->node_list.next != &mm->node_list ||
-            entry->free_stack.next != &mm->free_stack) {
+        if (!list_empty(&mm->head_node.node_list)) {
                 DRM_ERROR("Memory manager not clean. Delaying takedown\n");
                 return;
         }
 
-        list_del(&entry->free_stack);
-        list_del(&entry->node_list);
-        kfree(entry);
-
         spin_lock(&mm->unused_lock);
-        list_for_each_entry_safe(entry, next, &mm->unused_nodes, free_stack) {
-                list_del(&entry->free_stack);
+        list_for_each_entry_safe(entry, next, &mm->unused_nodes, node_list) {
+                list_del(&entry->node_list);
                 kfree(entry);
                 --mm->num_unused;
         }
@@ -582,19 +643,37 @@ EXPORT_SYMBOL(drm_mm_takedown);
 void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
 {
         struct drm_mm_node *entry;
-        int total_used = 0, total_free = 0, total = 0;
-
-        list_for_each_entry(entry, &mm->node_list, node_list) {
-                printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8ld: %s\n",
+        unsigned long total_used = 0, total_free = 0, total = 0;
+        unsigned long hole_start, hole_end, hole_size;
+
+        hole_start = drm_mm_hole_node_start(&mm->head_node);
+        hole_end = drm_mm_hole_node_end(&mm->head_node);
+        hole_size = hole_end - hole_start;
+        if (hole_size)
+                printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
+                        prefix, hole_start, hole_end,
+                        hole_size);
+        total_free += hole_size;
+
+        drm_mm_for_each_node(entry, mm) {
+                printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: used\n",
                         prefix, entry->start, entry->start + entry->size,
-                        entry->size, entry->free ? "free" : "used");
-                total += entry->size;
-                if (entry->free)
-                        total_free += entry->size;
-                else
-                        total_used += entry->size;
+                        entry->size);
+                total_used += entry->size;
+
+                if (entry->hole_follows) {
+                        hole_start = drm_mm_hole_node_start(entry);
+                        hole_end = drm_mm_hole_node_end(entry);
+                        hole_size = hole_end - hole_start;
+                        printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
+                                prefix, hole_start, hole_end,
+                                hole_size);
+                        total_free += hole_size;
+                }
         }
-        printk(KERN_DEBUG "%s total: %d, used %d free %d\n", prefix, total,
+        total = total_free + total_used;
+
+        printk(KERN_DEBUG "%s total: %lu, used %lu free %lu\n", prefix, total,
                 total_used, total_free);
 }
 EXPORT_SYMBOL(drm_mm_debug_table);
@@ -603,17 +682,34 @@ EXPORT_SYMBOL(drm_mm_debug_table);
 int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm)
 {
         struct drm_mm_node *entry;
-        int total_used = 0, total_free = 0, total = 0;
-
-        list_for_each_entry(entry, &mm->node_list, node_list) {
-                seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: %s\n", entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used");
-                total += entry->size;
-                if (entry->free)
-                        total_free += entry->size;
-                else
-                        total_used += entry->size;
+        unsigned long total_used = 0, total_free = 0, total = 0;
+        unsigned long hole_start, hole_end, hole_size;
+
+        hole_start = drm_mm_hole_node_start(&mm->head_node);
+        hole_end = drm_mm_hole_node_end(&mm->head_node);
+        hole_size = hole_end - hole_start;
+        if (hole_size)
+                seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
+                                hole_start, hole_end, hole_size);
+        total_free += hole_size;
+
+        drm_mm_for_each_node(entry, mm) {
+                seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: used\n",
+                                entry->start, entry->start + entry->size,
+                                entry->size);
+                total_used += entry->size;
+                if (entry->hole_follows) {
+                        hole_start = drm_mm_hole_node_start(&mm->head_node);
+                        hole_end = drm_mm_hole_node_end(&mm->head_node);
+                        hole_size = hole_end - hole_start;
+                        seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
+                                        hole_start, hole_end, hole_size);
+                        total_free += hole_size;
+                }
         }
-        seq_printf(m, "total: %d, used %d free %d\n", total, total_used, total_free);
+        total = total_free + total_used;
+
+        seq_printf(m, "total: %lu, used %lu free %lu\n", total, total_used, total_free);
         return 0;
 }
 EXPORT_SYMBOL(drm_mm_dump_table);
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 58e65f92c232..25bf87390f53 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -593,7 +593,7 @@ EXPORT_SYMBOL(drm_mode_height);
  *
  * Return @modes's hsync rate in kHz, rounded to the nearest int.
  */
-int drm_mode_hsync(struct drm_display_mode *mode)
+int drm_mode_hsync(const struct drm_display_mode *mode)
 {
         unsigned int calc_val;
 
@@ -627,7 +627,7 @@ EXPORT_SYMBOL(drm_mode_hsync);
  * If it is 70.288, it will return 70Hz.
  * If it is 59.6, it will return 60Hz.
  */
-int drm_mode_vrefresh(struct drm_display_mode *mode)
+int drm_mode_vrefresh(const struct drm_display_mode *mode)
 {
         int refresh = 0;
         unsigned int calc_val;
@@ -725,7 +725,7 @@ EXPORT_SYMBOL(drm_mode_set_crtcinfo);
  * a pointer to it.  Used to create new instances of established modes.
  */
 struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,
-                                            struct drm_display_mode *mode)
+                                            const struct drm_display_mode *mode)
 {
         struct drm_display_mode *nmode;
         int new_id;
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index f5bd9e590c80..e1aee4f6a7c6 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -125,6 +125,176 @@ void drm_pci_free(struct drm_device * dev, drm_dma_handle_t * dmah)
 EXPORT_SYMBOL(drm_pci_free);
 
 #ifdef CONFIG_PCI
+
+static int drm_get_pci_domain(struct drm_device *dev)
+{
+#ifndef __alpha__
+        /* For historical reasons, drm_get_pci_domain() is busticated
+         * on most archs and has to remain so for userspace interface
+         * < 1.4, except on alpha which was right from the beginning
+         */
+        if (dev->if_version < 0x10004)
+                return 0;
+#endif /* __alpha__ */
+
+        return pci_domain_nr(dev->pdev->bus);
+}
+
+static int drm_pci_get_irq(struct drm_device *dev)
+{
+        return dev->pdev->irq;
+}
+
+static const char *drm_pci_get_name(struct drm_device *dev)
+{
+        struct pci_driver *pdriver = dev->driver->kdriver.pci;
+        return pdriver->name;
+}
+
+int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master)
+{
+        int len, ret;
+        struct pci_driver *pdriver = dev->driver->kdriver.pci;
+        master->unique_len = 40;
+        master->unique_size = master->unique_len;
+        master->unique = kmalloc(master->unique_size, GFP_KERNEL);
+        if (master->unique == NULL)
+                return -ENOMEM;
+
+
+        len = snprintf(master->unique, master->unique_len,
+                       "pci:%04x:%02x:%02x.%d",
+                       drm_get_pci_domain(dev),
+                       dev->pdev->bus->number,
+                       PCI_SLOT(dev->pdev->devfn),
+                       PCI_FUNC(dev->pdev->devfn));
+
+        if (len >= master->unique_len) {
+                DRM_ERROR("buffer overflow");
+                ret = -EINVAL;
+                goto err;
+        } else
+                master->unique_len = len;
+
+        dev->devname =
+                kmalloc(strlen(pdriver->name) +
+                        master->unique_len + 2, GFP_KERNEL);
+
+        if (dev->devname == NULL) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        sprintf(dev->devname, "%s@%s", pdriver->name,
+                master->unique);
+
+        return 0;
+err:
+        return ret;
+}
+
+int drm_pci_set_unique(struct drm_device *dev,
+                       struct drm_master *master,
+                       struct drm_unique *u)
+{
+        int domain, bus, slot, func, ret;
+        const char *bus_name;
+
+        master->unique_len = u->unique_len;
+        master->unique_size = u->unique_len + 1;
+        master->unique = kmalloc(master->unique_size, GFP_KERNEL);
+        if (!master->unique) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        if (copy_from_user(master->unique, u->unique, master->unique_len)) {
+                ret = -EFAULT;
+                goto err;
+        }
+
+        master->unique[master->unique_len] = '\0';
+
+        bus_name = dev->driver->bus->get_name(dev);
+        dev->devname = kmalloc(strlen(bus_name) +
+                               strlen(master->unique) + 2, GFP_KERNEL);
+        if (!dev->devname) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        sprintf(dev->devname, "%s@%s", bus_name,
+                master->unique);
+
+        /* Return error if the busid submitted doesn't match the device's actual
+         * busid.
+         */
+        ret = sscanf(master->unique, "PCI:%d:%d:%d", &bus, &slot, &func);
+        if (ret != 3) {
+                ret = -EINVAL;
+                goto err;
+        }
+
+        domain = bus >> 8;
+        bus &= 0xff;
+
+        if ((domain != drm_get_pci_domain(dev)) ||
+            (bus != dev->pdev->bus->number) ||
+            (slot != PCI_SLOT(dev->pdev->devfn)) ||
+            (func != PCI_FUNC(dev->pdev->devfn))) {
+                ret = -EINVAL;
+                goto err;
+        }
+        return 0;
+err:
+        return ret;
+}
+
+
+int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p)
+{
+        if ((p->busnum >> 8) != drm_get_pci_domain(dev) ||
+            (p->busnum & 0xff) != dev->pdev->bus->number ||
+            p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn))
+                return -EINVAL;
+
+        p->irq = dev->pdev->irq;
+
+        DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum,
+                  p->irq);
+        return 0;
+}
+
+int drm_pci_agp_init(struct drm_device *dev)
+{
+        if (drm_core_has_AGP(dev)) {
+                if (drm_pci_device_is_agp(dev))
+                        dev->agp = drm_agp_init(dev);
+                if (drm_core_check_feature(dev, DRIVER_REQUIRE_AGP)
+                    && (dev->agp == NULL)) {
+                        DRM_ERROR("Cannot initialize the agpgart module.\n");
+                        return -EINVAL;
+                }
+                if (drm_core_has_MTRR(dev)) {
+                        if (dev->agp)
+                                dev->agp->agp_mtrr =
+                                        mtrr_add(dev->agp->agp_info.aper_base,
+                                                 dev->agp->agp_info.aper_size *
+                                                 1024 * 1024, MTRR_TYPE_WRCOMB, 1);
+                }
+        }
+        return 0;
+}
+
+static struct drm_bus drm_pci_bus = {
+        .bus_type = DRIVER_BUS_PCI,
+        .get_irq = drm_pci_get_irq,
+        .get_name = drm_pci_get_name,
+        .set_busid = drm_pci_set_busid,
+        .set_unique = drm_pci_set_unique,
+        .agp_init = drm_pci_agp_init,
+};
+
 /**
  * Register.
  *
@@ -219,7 +389,7 @@ err_g1:
 EXPORT_SYMBOL(drm_get_pci_dev);
 
 /**
- * PCI device initialization. Called via drm_init at module load time,
+ * PCI device initialization. Called direct from modules at load time.
  *
  * \return zero on success or a negative number on failure.
  *
@@ -229,18 +399,24 @@ EXPORT_SYMBOL(drm_get_pci_dev);
  * Expands the \c DRIVER_PREINIT and \c DRIVER_POST_INIT macros before and
  * after the initialization for driver customization.
  */
-int drm_pci_init(struct drm_driver *driver)
+int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 {
         struct pci_dev *pdev = NULL;
         const struct pci_device_id *pid;
         int i;
 
+        DRM_DEBUG("\n");
+
+        INIT_LIST_HEAD(&driver->device_list);
+        driver->kdriver.pci = pdriver;
+        driver->bus = &drm_pci_bus;
+
         if (driver->driver_features & DRIVER_MODESET)
-                return pci_register_driver(&driver->pci_driver);
+                return pci_register_driver(pdriver);
 
         /* If not using KMS, fall back to stealth mode manual scanning. */
-        for (i = 0; driver->pci_driver.id_table[i].vendor != 0; i++) {
-                pid = &driver->pci_driver.id_table[i];
+        for (i = 0; pdriver->id_table[i].vendor != 0; i++) {
+                pid = &pdriver->id_table[i];
 
                 /* Loop around setting up a DRM device for each PCI device
                  * matching our ID and device class.  If we had the internal
@@ -265,10 +441,27 @@ int drm_pci_init(struct drm_driver *driver)
 
 #else
 
-int drm_pci_init(struct drm_driver *driver)
+int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 {
         return -1;
 }
 
 #endif
+
+EXPORT_SYMBOL(drm_pci_init);
+
 /*@}*/
+void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
+{
+        struct drm_device *dev, *tmp;
+        DRM_DEBUG("\n");
+
+        if (driver->driver_features & DRIVER_MODESET) {
+                pci_unregister_driver(pdriver);
+        } else {
+                list_for_each_entry_safe(dev, tmp, &driver->device_list, driver_item)
+                        drm_put_dev(dev);
+        }
+        DRM_INFO("Module unloaded\n");
+}
+EXPORT_SYMBOL(drm_pci_exit);
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 92d1d0fb7b75..7223f06d8e58 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -109,8 +109,60 @@ err_g1:
 }
 EXPORT_SYMBOL(drm_get_platform_dev);
 
+static int drm_platform_get_irq(struct drm_device *dev)
+{
+        return platform_get_irq(dev->platformdev, 0);
+}
+
+static const char *drm_platform_get_name(struct drm_device *dev)
+{
+        return dev->platformdev->name;
+}
+
+static int drm_platform_set_busid(struct drm_device *dev, struct drm_master *master)
+{
+        int len, ret;
+
+        master->unique_len = 10 + strlen(dev->platformdev->name);
+        master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
+
+        if (master->unique == NULL)
+                return -ENOMEM;
+
+        len = snprintf(master->unique, master->unique_len,
+                       "platform:%s", dev->platformdev->name);
+
+        if (len > master->unique_len) {
+                DRM_ERROR("Unique buffer overflowed\n");
+                ret = -EINVAL;
+                goto err;
+        }
+
+        dev->devname =
+                kmalloc(strlen(dev->platformdev->name) +
+                        master->unique_len + 2, GFP_KERNEL);
+
+        if (dev->devname == NULL) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        sprintf(dev->devname, "%s@%s", dev->platformdev->name,
+                master->unique);
+        return 0;
+err:
+        return ret;
+}
+
+static struct drm_bus drm_platform_bus = {
+        .bus_type = DRIVER_BUS_PLATFORM,
+        .get_irq = drm_platform_get_irq,
+        .get_name = drm_platform_get_name,
+        .set_busid = drm_platform_set_busid,
+};
+
 /**
- * Platform device initialization. Called via drm_init at module load time,
+ * Platform device initialization. Called direct from modules.
  *
  * \return zero on success or a negative number on failure.
  *
@@ -121,7 +173,24 @@ EXPORT_SYMBOL(drm_get_platform_dev);
  * after the initialization for driver customization.
  */
 
-int drm_platform_init(struct drm_driver *driver)
+int drm_platform_init(struct drm_driver *driver, struct platform_device *platform_device)
 {
-        return drm_get_platform_dev(driver->platform_device, driver);
+        DRM_DEBUG("\n");
+
+        driver->kdriver.platform_device = platform_device;
+        driver->bus = &drm_platform_bus;
+        INIT_LIST_HEAD(&driver->device_list);
+        return drm_get_platform_dev(platform_device, driver);
+}
+EXPORT_SYMBOL(drm_platform_init);
+
+void drm_platform_exit(struct drm_driver *driver, struct platform_device *platform_device)
+{
+        struct drm_device *dev, *tmp;
+        DRM_DEBUG("\n");
+
+        list_for_each_entry_safe(dev, tmp, &driver->device_list, driver_item)
+                drm_put_dev(dev);
+        DRM_INFO("Module unloaded\n");
 }
+EXPORT_SYMBOL(drm_platform_exit);
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index d59edc18301f..001273d57f2d 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -269,25 +269,14 @@ int drm_fill_in_dev(struct drm_device *dev,
 
         dev->driver = driver;
 
-        if (drm_core_has_AGP(dev)) {
-                if (drm_device_is_agp(dev))
-                        dev->agp = drm_agp_init(dev);
-                if (drm_core_check_feature(dev, DRIVER_REQUIRE_AGP)
-                    && (dev->agp == NULL)) {
-                        DRM_ERROR("Cannot initialize the agpgart module.\n");
-                        retcode = -EINVAL;
+        if (dev->driver->bus->agp_init) {
+                retcode = dev->driver->bus->agp_init(dev);
+                if (retcode)
                         goto error_out_unreg;
-                }
-                if (drm_core_has_MTRR(dev)) {
-                        if (dev->agp)
-                                dev->agp->agp_mtrr =
-                                    mtrr_add(dev->agp->agp_info.aper_base,
-                                             dev->agp->agp_info.aper_size *
-                                             1024 * 1024, MTRR_TYPE_WRCOMB, 1);
-                }
         }
 
 
+
         retcode = drm_ctxbitmap_init(dev);
         if (retcode) {
                 DRM_ERROR("Cannot allocate memory for context bitmap.\n");
@@ -425,7 +414,6 @@ int drm_put_minor(struct drm_minor **minor_p)
  *
  * Cleans up all DRM device, calling drm_lastclose().
  *
- * \sa drm_init
  */
 void drm_put_dev(struct drm_device *dev)
 {
@@ -475,6 +463,7 @@ void drm_put_dev(struct drm_device *dev)
 
         drm_put_minor(&dev->primary);
 
+        list_del(&dev->driver_item);
         if (dev->devname) {
                 kfree(dev->devname);
                 dev->devname = NULL;
diff --git a/drivers/gpu/drm/drm_usb.c b/drivers/gpu/drm/drm_usb.c
new file mode 100644
index 000000000000..206d2300d873
--- /dev/null
+++ b/drivers/gpu/drm/drm_usb.c
@@ -0,0 +1,117 @@
+#include "drmP.h"
+#include <linux/usb.h>
+
+#ifdef CONFIG_USB
+int drm_get_usb_dev(struct usb_interface *interface,
+                    const struct usb_device_id *id,
+                    struct drm_driver *driver)
+{
+        struct drm_device *dev;
+        struct usb_device *usbdev;
+        int ret;
+
+        DRM_DEBUG("\n");
+
+        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+        if (!dev)
+                return -ENOMEM;
+
+        usbdev = interface_to_usbdev(interface);
+        dev->usbdev = usbdev;
+        dev->dev = &usbdev->dev;
+
+        mutex_lock(&drm_global_mutex);
+
+        ret = drm_fill_in_dev(dev, NULL, driver);
+        if (ret) {
+                printk(KERN_ERR "DRM: Fill_in_dev failed.\n");
+                goto err_g1;
+        }
+
+        usb_set_intfdata(interface, dev);
+        ret = drm_get_minor(dev, &dev->control, DRM_MINOR_CONTROL);
+        if (ret)
+                goto err_g1;
+
+        ret = drm_get_minor(dev, &dev->primary, DRM_MINOR_LEGACY);
+        if (ret)
+                goto err_g2;
+
+        if (dev->driver->load) {
+                ret = dev->driver->load(dev, 0);
+                if (ret)
+                        goto err_g3;
+        }
+
+        /* setup the grouping for the legacy output */
+        ret = drm_mode_group_init_legacy_group(dev,
+                                               &dev->primary->mode_group);
+        if (ret)
+                goto err_g3;
+
+        list_add_tail(&dev->driver_item, &driver->device_list);
+
+        mutex_unlock(&drm_global_mutex);
+
+        DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+                 driver->name, driver->major, driver->minor, driver->patchlevel,
+                 driver->date, dev->primary->index);
+
+        return 0;
+
+err_g3:
+        drm_put_minor(&dev->primary);
+err_g2:
+        drm_put_minor(&dev->control);
+err_g1:
+        kfree(dev);
+        mutex_unlock(&drm_global_mutex);
+        return ret;
+
+}
+EXPORT_SYMBOL(drm_get_usb_dev);
+
+static int drm_usb_get_irq(struct drm_device *dev)
+{
+        return 0;
+}
+
+static const char *drm_usb_get_name(struct drm_device *dev)
+{
+        return "USB";
+}
+
+static int drm_usb_set_busid(struct drm_device *dev,
+                               struct drm_master *master)
+{
+        return 0;
+}
+
+static struct drm_bus drm_usb_bus = {
+        .bus_type = DRIVER_BUS_USB,
+        .get_irq = drm_usb_get_irq,
+        .get_name = drm_usb_get_name,
+        .set_busid = drm_usb_set_busid,
+};
+    
+int drm_usb_init(struct drm_driver *driver, struct usb_driver *udriver)
+{
+        int res;
+        DRM_DEBUG("\n");
+
+        INIT_LIST_HEAD(&driver->device_list);
+        driver->kdriver.usb = udriver;
+        driver->bus = &drm_usb_bus;
+
+        res = usb_register(udriver);
+        return res;
+}
+EXPORT_SYMBOL(drm_usb_init);
+
+void drm_usb_exit(struct drm_driver *driver,
+                  struct usb_driver *udriver)
+{
+        usb_deregister(udriver);
+}
+EXPORT_SYMBOL(drm_usb_exit);
+#endif
diff --git a/drivers/gpu/drm/i810/i810_dma.c b/drivers/gpu/drm/i810/i810_dma.c
index ff33e53bbbf8..8f371e8d630f 100644
--- a/drivers/gpu/drm/i810/i810_dma.c
+++ b/drivers/gpu/drm/i810/i810_dma.c
@@ -37,7 +37,6 @@
 #include <linux/interrupt.h>    /* For task queue support */
 #include <linux/delay.h>
 #include <linux/slab.h>
-#include <linux/smp_lock.h>
 #include <linux/pagemap.h>
 
 #define I810_BUF_FREE           2
@@ -94,7 +93,6 @@ static int i810_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
         struct drm_buf *buf;
         drm_i810_buf_priv_t *buf_priv;
 
-        lock_kernel();
         dev = priv->minor->dev;
         dev_priv = dev->dev_private;
         buf = dev_priv->mmap_buffer;
@@ -104,7 +102,6 @@ static int i810_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
         vma->vm_file = filp;
 
         buf_priv->currently_mapped = I810_BUF_MAPPED;
-        unlock_kernel();
 
         if (io_remap_pfn_range(vma, vma->vm_start,
                                vma->vm_pgoff,
@@ -116,7 +113,7 @@ static int i810_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
 static const struct file_operations i810_buffer_fops = {
         .open = drm_open,
         .release = drm_release,
-        .unlocked_ioctl = i810_ioctl,
+        .unlocked_ioctl = drm_ioctl,
         .mmap = i810_mmap_buffers,
         .fasync = drm_fasync,
         .llseek = noop_llseek,
@@ -1242,19 +1239,6 @@ int i810_driver_dma_quiescent(struct drm_device *dev)
         return 0;
 }
 
-/*
- * call the drm_ioctl under the big kernel lock because
- * to lock against the i810_mmap_buffers function.
- */
-long i810_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-        int ret;
-        lock_kernel();
-        ret = drm_ioctl(file, cmd, arg);
-        unlock_kernel();
-        return ret;
-}
-
 struct drm_ioctl_desc i810_ioctls[] = {
         DRM_IOCTL_DEF_DRV(I810_INIT, i810_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
         DRM_IOCTL_DEF_DRV(I810_VERTEX, i810_dma_vertex, DRM_AUTH|DRM_UNLOCKED),
diff --git a/drivers/gpu/drm/i810/i810_drv.c b/drivers/gpu/drm/i810/i810_drv.c
index 88bcd331e7c5..6f98d059f68a 100644
--- a/drivers/gpu/drm/i810/i810_drv.c
+++ b/drivers/gpu/drm/i810/i810_drv.c
@@ -57,18 +57,13 @@ static struct drm_driver driver = {
                  .owner = THIS_MODULE,
                  .open = drm_open,
                  .release = drm_release,
-                 .unlocked_ioctl = i810_ioctl,
+                 .unlocked_ioctl = drm_ioctl,
                  .mmap = drm_mmap,
                  .poll = drm_poll,
                  .fasync = drm_fasync,
                  .llseek = noop_llseek,
         },
 
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
-
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
         .date = DRIVER_DATE,
@@ -77,15 +72,24 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver i810_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init i810_init(void)
 {
+        if (num_possible_cpus() > 1) {
+                pr_err("drm/i810 does not support SMP\n");
+                return -EINVAL;
+        }
         driver.num_ioctls = i810_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &i810_pci_driver);
 }
 
 static void __exit i810_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &i810_pci_driver);
 }
 
 module_init(i810_init);
diff --git a/drivers/gpu/drm/i830/Makefile b/drivers/gpu/drm/i830/Makefile
deleted file mode 100644
index c642ee0b238c..000000000000
--- a/drivers/gpu/drm/i830/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-#
-# Makefile for the drm device driver.  This driver provides support for the
-# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
-
-ccflags-y := -Iinclude/drm
-i830-y := i830_drv.o i830_dma.o i830_irq.o
-
-obj-$(CONFIG_DRM_I830) += i830.o
diff --git a/drivers/gpu/drm/i830/i830_dma.c b/drivers/gpu/drm/i830/i830_dma.c
deleted file mode 100644
index ca6f31ff0eec..000000000000
--- a/drivers/gpu/drm/i830/i830_dma.c
+++ /dev/null
@@ -1,1560 +0,0 @@
-/* i830_dma.c -- DMA support for the I830 -*- linux-c -*-
- * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors: Rickard E. (Rik) Faith <faith@valinux.com>
- *          Jeff Hartmann <jhartmann@valinux.com>
- *          Keith Whitwell <keith@tungstengraphics.com>
- *          Abraham vd Merwe <abraham@2d3d.co.za>
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "i830_drm.h"
-#include "i830_drv.h"
-#include <linux/interrupt.h>    /* For task queue support */
-#include <linux/smp_lock.h>
-#include <linux/pagemap.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <asm/uaccess.h>
-
-#define I830_BUF_FREE           2
-#define I830_BUF_CLIENT         1
-#define I830_BUF_HARDWARE       0
-
-#define I830_BUF_UNMAPPED 0
-#define I830_BUF_MAPPED   1
-
-static struct drm_buf *i830_freelist_get(struct drm_device * dev)
-{
-        struct drm_device_dma *dma = dev->dma;
-        int i;
-        int used;
-
-        /* Linear search might not be the best solution */
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-                /* In use is already a pointer */
-                used = cmpxchg(buf_priv->in_use, I830_BUF_FREE,
-                               I830_BUF_CLIENT);
-                if (used == I830_BUF_FREE)
-                        return buf;
-        }
-        return NULL;
-}
-
-/* This should only be called if the buffer is not sent to the hardware
- * yet, the hardware updates in use for us once its on the ring buffer.
- */
-
-static int i830_freelist_put(struct drm_device *dev, struct drm_buf *buf)
-{
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        int used;
-
-        /* In use is already a pointer */
-        used = cmpxchg(buf_priv->in_use, I830_BUF_CLIENT, I830_BUF_FREE);
-        if (used != I830_BUF_CLIENT) {
-                DRM_ERROR("Freeing buffer thats not in use : %d\n", buf->idx);
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-static int i830_mmap_buffers(struct file *filp, struct vm_area_struct *vma)
-{
-        struct drm_file *priv = filp->private_data;
-        struct drm_device *dev;
-        drm_i830_private_t *dev_priv;
-        struct drm_buf *buf;
-        drm_i830_buf_priv_t *buf_priv;
-
-        lock_kernel();
-        dev = priv->minor->dev;
-        dev_priv = dev->dev_private;
-        buf = dev_priv->mmap_buffer;
-        buf_priv = buf->dev_private;
-
-        vma->vm_flags |= (VM_IO | VM_DONTCOPY);
-        vma->vm_file = filp;
-
-        buf_priv->currently_mapped = I830_BUF_MAPPED;
-        unlock_kernel();
-
-        if (io_remap_pfn_range(vma, vma->vm_start,
-                               vma->vm_pgoff,
-                               vma->vm_end - vma->vm_start, vma->vm_page_prot))
-                return -EAGAIN;
-        return 0;
-}
-
-static const struct file_operations i830_buffer_fops = {
-        .open = drm_open,
-        .release = drm_release,
-        .unlocked_ioctl = i830_ioctl,
-        .mmap = i830_mmap_buffers,
-        .fasync = drm_fasync,
-        .llseek = noop_llseek,
-};
-
-static int i830_map_buffer(struct drm_buf *buf, struct drm_file *file_priv)
-{
-        struct drm_device *dev = file_priv->minor->dev;
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        const struct file_operations *old_fops;
-        unsigned long virtual;
-        int retcode = 0;
-
-        if (buf_priv->currently_mapped == I830_BUF_MAPPED)
-                return -EINVAL;
-
-        down_write(&current->mm->mmap_sem);
-        old_fops = file_priv->filp->f_op;
-        file_priv->filp->f_op = &i830_buffer_fops;
-        dev_priv->mmap_buffer = buf;
-        virtual = do_mmap(file_priv->filp, 0, buf->total, PROT_READ | PROT_WRITE,
-                          MAP_SHARED, buf->bus_address);
-        dev_priv->mmap_buffer = NULL;
-        file_priv->filp->f_op = old_fops;
-        if (IS_ERR((void *)virtual)) {  /* ugh */
-                /* Real error */
-                DRM_ERROR("mmap error\n");
-                retcode = PTR_ERR((void *)virtual);
-                buf_priv->virtual = NULL;
-        } else {
-                buf_priv->virtual = (void __user *)virtual;
-        }
-        up_write(&current->mm->mmap_sem);
-
-        return retcode;
-}
-
-static int i830_unmap_buffer(struct drm_buf *buf)
-{
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        int retcode = 0;
-
-        if (buf_priv->currently_mapped != I830_BUF_MAPPED)
-                return -EINVAL;
-
-        down_write(&current->mm->mmap_sem);
-        retcode = do_munmap(current->mm,
-                            (unsigned long)buf_priv->virtual,
-                            (size_t) buf->total);
-        up_write(&current->mm->mmap_sem);
-
-        buf_priv->currently_mapped = I830_BUF_UNMAPPED;
-        buf_priv->virtual = NULL;
-
-        return retcode;
-}
-
-static int i830_dma_get_buffer(struct drm_device *dev, drm_i830_dma_t *d,
-                               struct drm_file *file_priv)
-{
-        struct drm_buf *buf;
-        drm_i830_buf_priv_t *buf_priv;
-        int retcode = 0;
-
-        buf = i830_freelist_get(dev);
-        if (!buf) {
-                retcode = -ENOMEM;
-                DRM_DEBUG("retcode=%d\n", retcode);
-                return retcode;
-        }
-
-        retcode = i830_map_buffer(buf, file_priv);
-        if (retcode) {
-                i830_freelist_put(dev, buf);
-                DRM_ERROR("mapbuf failed, retcode %d\n", retcode);
-                return retcode;
-        }
-        buf->file_priv = file_priv;
-        buf_priv = buf->dev_private;
-        d->granted = 1;
-        d->request_idx = buf->idx;
-        d->request_size = buf->total;
-        d->virtual = buf_priv->virtual;
-
-        return retcode;
-}
-
-static int i830_dma_cleanup(struct drm_device *dev)
-{
-        struct drm_device_dma *dma = dev->dma;
-
-        /* Make sure interrupts are disabled here because the uninstall ioctl
-         * may not have been called from userspace and after dev_private
-         * is freed, it's too late.
-         */
-        if (dev->irq_enabled)
-                drm_irq_uninstall(dev);
-
-        if (dev->dev_private) {
-                int i;
-                drm_i830_private_t *dev_priv =
-                    (drm_i830_private_t *) dev->dev_private;
-
-                if (dev_priv->ring.virtual_start)
-                        drm_core_ioremapfree(&dev_priv->ring.map, dev);
-                if (dev_priv->hw_status_page) {
-                        pci_free_consistent(dev->pdev, PAGE_SIZE,
-                                            dev_priv->hw_status_page,
-                                            dev_priv->dma_status_page);
-                        /* Need to rewrite hardware status page */
-                        I830_WRITE(0x02080, 0x1ffff000);
-                }
-
-                kfree(dev->dev_private);
-                dev->dev_private = NULL;
-
-                for (i = 0; i < dma->buf_count; i++) {
-                        struct drm_buf *buf = dma->buflist[i];
-                        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-                        if (buf_priv->kernel_virtual && buf->total)
-                                drm_core_ioremapfree(&buf_priv->map, dev);
-                }
-        }
-        return 0;
-}
-
-int i830_wait_ring(struct drm_device *dev, int n, const char *caller)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_ring_buffer_t *ring = &(dev_priv->ring);
-        int iters = 0;
-        unsigned long end;
-        unsigned int last_head = I830_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-
-        end = jiffies + (HZ * 3);
-        while (ring->space < n) {
-                ring->head = I830_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->Size;
-
-                if (ring->head != last_head) {
-                        end = jiffies + (HZ * 3);
-                        last_head = ring->head;
-                }
-
-                iters++;
-                if (time_before(end, jiffies)) {
-                        DRM_ERROR("space: %d wanted %d\n", ring->space, n);
-                        DRM_ERROR("lockup\n");
-                        goto out_wait_ring;
-                }
-                udelay(1);
-                dev_priv->sarea_priv->perf_boxes |= I830_BOX_WAIT;
-        }
-
-out_wait_ring:
-        return iters;
-}
-
-static void i830_kernel_lost_context(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_ring_buffer_t *ring = &(dev_priv->ring);
-
-        ring->head = I830_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-        ring->tail = I830_READ(LP_RING + RING_TAIL) & TAIL_ADDR;
-        ring->space = ring->head - (ring->tail + 8);
-        if (ring->space < 0)
-                ring->space += ring->Size;
-
-        if (ring->head == ring->tail)
-                dev_priv->sarea_priv->perf_boxes |= I830_BOX_RING_EMPTY;
-}
-
-static int i830_freelist_init(struct drm_device *dev, drm_i830_private_t *dev_priv)
-{
-        struct drm_device_dma *dma = dev->dma;
-        int my_idx = 36;
-        u32 *hw_status = (u32 *) (dev_priv->hw_status_page + my_idx);
-        int i;
-
-        if (dma->buf_count > 1019) {
-                /* Not enough space in the status page for the freelist */
-                return -EINVAL;
-        }
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-
-                buf_priv->in_use = hw_status++;
-                buf_priv->my_use_idx = my_idx;
-                my_idx += 4;
-
-                *buf_priv->in_use = I830_BUF_FREE;
-
-                buf_priv->map.offset = buf->bus_address;
-                buf_priv->map.size = buf->total;
-                buf_priv->map.type = _DRM_AGP;
-                buf_priv->map.flags = 0;
-                buf_priv->map.mtrr = 0;
-
-                drm_core_ioremap(&buf_priv->map, dev);
-                buf_priv->kernel_virtual = buf_priv->map.handle;
-        }
-        return 0;
-}
-
-static int i830_dma_initialize(struct drm_device *dev,
-                               drm_i830_private_t *dev_priv,
-                               drm_i830_init_t *init)
-{
-        struct drm_map_list *r_list;
-
-        memset(dev_priv, 0, sizeof(drm_i830_private_t));
-
-        list_for_each_entry(r_list, &dev->maplist, head) {
-                if (r_list->map &&
-                    r_list->map->type == _DRM_SHM &&
-                    r_list->map->flags & _DRM_CONTAINS_LOCK) {
-                        dev_priv->sarea_map = r_list->map;
-                        break;
-                }
-        }
-
-        if (!dev_priv->sarea_map) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not find sarea!\n");
-                return -EINVAL;
-        }
-        dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
-        if (!dev_priv->mmio_map) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not find mmio map!\n");
-                return -EINVAL;
-        }
-        dev->agp_buffer_token = init->buffers_offset;
-        dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
-        if (!dev->agp_buffer_map) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not find dma buffer map!\n");
-                return -EINVAL;
-        }
-
-        dev_priv->sarea_priv = (drm_i830_sarea_t *)
-            ((u8 *) dev_priv->sarea_map->handle + init->sarea_priv_offset);
-
-        dev_priv->ring.Start = init->ring_start;
-        dev_priv->ring.End = init->ring_end;
-        dev_priv->ring.Size = init->ring_size;
-
-        dev_priv->ring.map.offset = dev->agp->base + init->ring_start;
-        dev_priv->ring.map.size = init->ring_size;
-        dev_priv->ring.map.type = _DRM_AGP;
-        dev_priv->ring.map.flags = 0;
-        dev_priv->ring.map.mtrr = 0;
-
-        drm_core_ioremap(&dev_priv->ring.map, dev);
-
-        if (dev_priv->ring.map.handle == NULL) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("can not ioremap virtual address for"
-                          " ring buffer\n");
-                return -ENOMEM;
-        }
-
-        dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
-
-        dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
-
-        dev_priv->w = init->w;
-        dev_priv->h = init->h;
-        dev_priv->pitch = init->pitch;
-        dev_priv->back_offset = init->back_offset;
-        dev_priv->depth_offset = init->depth_offset;
-        dev_priv->front_offset = init->front_offset;
-
-        dev_priv->front_di1 = init->front_offset | init->pitch_bits;
-        dev_priv->back_di1 = init->back_offset | init->pitch_bits;
-        dev_priv->zi1 = init->depth_offset | init->pitch_bits;
-
-        DRM_DEBUG("front_di1 %x\n", dev_priv->front_di1);
-        DRM_DEBUG("back_offset %x\n", dev_priv->back_offset);
-        DRM_DEBUG("back_di1 %x\n", dev_priv->back_di1);
-        DRM_DEBUG("pitch_bits %x\n", init->pitch_bits);
-
-        dev_priv->cpp = init->cpp;
-        /* We are using separate values as placeholders for mechanisms for
-         * private backbuffer/depthbuffer usage.
-         */
-
-        dev_priv->back_pitch = init->back_pitch;
-        dev_priv->depth_pitch = init->depth_pitch;
-        dev_priv->do_boxes = 0;
-        dev_priv->use_mi_batchbuffer_start = 0;
-
-        /* Program Hardware Status Page */
-        dev_priv->hw_status_page =
-            pci_alloc_consistent(dev->pdev, PAGE_SIZE,
-                                 &dev_priv->dma_status_page);
-        if (!dev_priv->hw_status_page) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("Can not allocate hardware status page\n");
-                return -ENOMEM;
-        }
-        memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
-        DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
-
-        I830_WRITE(0x02080, dev_priv->dma_status_page);
-        DRM_DEBUG("Enabled hardware status page\n");
-
-        /* Now we need to init our freelist */
-        if (i830_freelist_init(dev, dev_priv) != 0) {
-                dev->dev_private = (void *)dev_priv;
-                i830_dma_cleanup(dev);
-                DRM_ERROR("Not enough space in the status page for"
-                          " the freelist\n");
-                return -ENOMEM;
-        }
-        dev->dev_private = (void *)dev_priv;
-
-        return 0;
-}
-
-static int i830_dma_init(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv;
-        drm_i830_init_t *init = data;
-        int retcode = 0;
-
-        switch (init->func) {
-        case I830_INIT_DMA:
-                dev_priv = kmalloc(sizeof(drm_i830_private_t), GFP_KERNEL);
-                if (dev_priv == NULL)
-                        return -ENOMEM;
-                retcode = i830_dma_initialize(dev, dev_priv, init);
-                break;
-        case I830_CLEANUP_DMA:
-                retcode = i830_dma_cleanup(dev);
-                break;
-        default:
-                retcode = -EINVAL;
-                break;
-        }
-
-        return retcode;
-}
-
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define ST1_ENABLE               (1<<16)
-#define ST1_MASK                 (0xffff)
-
-/* Most efficient way to verify state for the i830 is as it is
- * emitted.  Non-conformant state is silently dropped.
- */
-static void i830EmitContextVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i, j = 0;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        BEGIN_LP_RING(I830_CTX_SETUP_SIZE + 4);
-
-        for (i = 0; i < I830_CTXREG_BLENDCOLR0; i++) {
-                tmp = code[i];
-                if ((tmp & (7 << 29)) == CMD_3D &&
-                    (tmp & (0x1f << 24)) < (0x1d << 24)) {
-                        OUT_RING(tmp);
-                        j++;
-                } else {
-                        DRM_ERROR("Skipping %d\n", i);
-                }
-        }
-
-        OUT_RING(STATE3D_CONST_BLEND_COLOR_CMD);
-        OUT_RING(code[I830_CTXREG_BLENDCOLR]);
-        j += 2;
-
-        for (i = I830_CTXREG_VF; i < I830_CTXREG_MCSB0; i++) {
-                tmp = code[i];
-                if ((tmp & (7 << 29)) == CMD_3D &&
-                    (tmp & (0x1f << 24)) < (0x1d << 24)) {
-                        OUT_RING(tmp);
-                        j++;
-                } else {
-                        DRM_ERROR("Skipping %d\n", i);
-                }
-        }
-
-        OUT_RING(STATE3D_MAP_COORD_SETBIND_CMD);
-        OUT_RING(code[I830_CTXREG_MCSB1]);
-        j += 2;
-
-        if (j & 1)
-                OUT_RING(0);
-
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitTexVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i, j = 0;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        if (code[I830_TEXREG_MI0] == GFX_OP_MAP_INFO ||
-            (code[I830_TEXREG_MI0] & ~(0xf * LOAD_TEXTURE_MAP0)) ==
-            (STATE3D_LOAD_STATE_IMMEDIATE_2 | 4)) {
-
-                BEGIN_LP_RING(I830_TEX_SETUP_SIZE);
-
-                OUT_RING(code[I830_TEXREG_MI0]);        /* TM0LI */
-                OUT_RING(code[I830_TEXREG_MI1]);        /* TM0S0 */
-                OUT_RING(code[I830_TEXREG_MI2]);        /* TM0S1 */
-                OUT_RING(code[I830_TEXREG_MI3]);        /* TM0S2 */
-                OUT_RING(code[I830_TEXREG_MI4]);        /* TM0S3 */
-                OUT_RING(code[I830_TEXREG_MI5]);        /* TM0S4 */
-
-                for (i = 6; i < I830_TEX_SETUP_SIZE; i++) {
-                        tmp = code[i];
-                        OUT_RING(tmp);
-                        j++;
-                }
-
-                if (j & 1)
-                        OUT_RING(0);
-
-                ADVANCE_LP_RING();
-        } else
-                printk("rejected packet %x\n", code[0]);
-}
-
-static void i830EmitTexBlendVerified(struct drm_device *dev,
-                                     unsigned int *code, unsigned int num)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i, j = 0;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        if (!num)
-                return;
-
-        BEGIN_LP_RING(num + 1);
-
-        for (i = 0; i < num; i++) {
-                tmp = code[i];
-                OUT_RING(tmp);
-                j++;
-        }
-
-        if (j & 1)
-                OUT_RING(0);
-
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitTexPalette(struct drm_device *dev,
-                               unsigned int *palette, int number, int is_shared)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        int i;
-        RING_LOCALS;
-
-        return;
-
-        BEGIN_LP_RING(258);
-
-        if (is_shared == 1) {
-                OUT_RING(CMD_OP_MAP_PALETTE_LOAD |
-                         MAP_PALETTE_NUM(0) | MAP_PALETTE_BOTH);
-        } else {
-                OUT_RING(CMD_OP_MAP_PALETTE_LOAD | MAP_PALETTE_NUM(number));
-        }
-        for (i = 0; i < 256; i++)
-                OUT_RING(palette[i]);
-        OUT_RING(0);
-        /* KW:  WHERE IS THE ADVANCE_LP_RING?  This is effectively a noop!
-         */
-}
-
-/* Need to do some additional checking when setting the dest buffer.
- */
-static void i830EmitDestVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        unsigned int tmp;
-        RING_LOCALS;
-
-        BEGIN_LP_RING(I830_DEST_SETUP_SIZE + 10);
-
-        tmp = code[I830_DESTREG_CBUFADDR];
-        if (tmp == dev_priv->front_di1 || tmp == dev_priv->back_di1) {
-                if (((int)outring) & 8) {
-                        OUT_RING(0);
-                        OUT_RING(0);
-                }
-
-                OUT_RING(CMD_OP_DESTBUFFER_INFO);
-                OUT_RING(BUF_3D_ID_COLOR_BACK |
-                         BUF_3D_PITCH(dev_priv->back_pitch * dev_priv->cpp) |
-                         BUF_3D_USE_FENCE);
-                OUT_RING(tmp);
-                OUT_RING(0);
-
-                OUT_RING(CMD_OP_DESTBUFFER_INFO);
-                OUT_RING(BUF_3D_ID_DEPTH | BUF_3D_USE_FENCE |
-                         BUF_3D_PITCH(dev_priv->depth_pitch * dev_priv->cpp));
-                OUT_RING(dev_priv->zi1);
-                OUT_RING(0);
-        } else {
-                DRM_ERROR("bad di1 %x (allow %x or %x)\n",
-                          tmp, dev_priv->front_di1, dev_priv->back_di1);
-        }
-
-        /* invarient:
-         */
-
-        OUT_RING(GFX_OP_DESTBUFFER_VARS);
-        OUT_RING(code[I830_DESTREG_DV1]);
-
-        OUT_RING(GFX_OP_DRAWRECT_INFO);
-        OUT_RING(code[I830_DESTREG_DR1]);
-        OUT_RING(code[I830_DESTREG_DR2]);
-        OUT_RING(code[I830_DESTREG_DR3]);
-        OUT_RING(code[I830_DESTREG_DR4]);
-
-        /* Need to verify this */
-        tmp = code[I830_DESTREG_SENABLE];
-        if ((tmp & ~0x3) == GFX_OP_SCISSOR_ENABLE) {
-                OUT_RING(tmp);
-        } else {
-                DRM_ERROR("bad scissor enable\n");
-                OUT_RING(0);
-        }
-
-        OUT_RING(GFX_OP_SCISSOR_RECT);
-        OUT_RING(code[I830_DESTREG_SR1]);
-        OUT_RING(code[I830_DESTREG_SR2]);
-        OUT_RING(0);
-
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitStippleVerified(struct drm_device *dev, unsigned int *code)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        BEGIN_LP_RING(2);
-        OUT_RING(GFX_OP_STIPPLE);
-        OUT_RING(code[1]);
-        ADVANCE_LP_RING();
-}
-
-static void i830EmitState(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        unsigned int dirty = sarea_priv->dirty;
-
-        DRM_DEBUG("%s %x\n", __func__, dirty);
-
-        if (dirty & I830_UPLOAD_BUFFERS) {
-                i830EmitDestVerified(dev, sarea_priv->BufferState);
-                sarea_priv->dirty &= ~I830_UPLOAD_BUFFERS;
-        }
-
-        if (dirty & I830_UPLOAD_CTX) {
-                i830EmitContextVerified(dev, sarea_priv->ContextState);
-                sarea_priv->dirty &= ~I830_UPLOAD_CTX;
-        }
-
-        if (dirty & I830_UPLOAD_TEX0) {
-                i830EmitTexVerified(dev, sarea_priv->TexState[0]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX0;
-        }
-
-        if (dirty & I830_UPLOAD_TEX1) {
-                i830EmitTexVerified(dev, sarea_priv->TexState[1]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX1;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND0) {
-                i830EmitTexBlendVerified(dev, sarea_priv->TexBlendState[0],
-                                         sarea_priv->TexBlendStateWordsUsed[0]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND0;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND1) {
-                i830EmitTexBlendVerified(dev, sarea_priv->TexBlendState[1],
-                                         sarea_priv->TexBlendStateWordsUsed[1]);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND1;
-        }
-
-        if (dirty & I830_UPLOAD_TEX_PALETTE_SHARED) {
-                i830EmitTexPalette(dev, sarea_priv->Palette[0], 0, 1);
-        } else {
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(0)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette[0], 0, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(0);
-                }
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(1)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette[1], 1, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(1);
-                }
-
-                /* 1.3:
-                 */
-#if 0
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(2)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette2[0], 0, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(2);
-                }
-                if (dirty & I830_UPLOAD_TEX_PALETTE_N(3)) {
-                        i830EmitTexPalette(dev, sarea_priv->Palette2[1], 1, 0);
-                        sarea_priv->dirty &= ~I830_UPLOAD_TEX_PALETTE_N(2);
-                }
-#endif
-        }
-
-        /* 1.3:
-         */
-        if (dirty & I830_UPLOAD_STIPPLE) {
-                i830EmitStippleVerified(dev, sarea_priv->StippleState);
-                sarea_priv->dirty &= ~I830_UPLOAD_STIPPLE;
-        }
-
-        if (dirty & I830_UPLOAD_TEX2) {
-                i830EmitTexVerified(dev, sarea_priv->TexState2);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX2;
-        }
-
-        if (dirty & I830_UPLOAD_TEX3) {
-                i830EmitTexVerified(dev, sarea_priv->TexState3);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEX3;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND2) {
-                i830EmitTexBlendVerified(dev,
-                                         sarea_priv->TexBlendState2,
-                                         sarea_priv->TexBlendStateWordsUsed2);
-
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND2;
-        }
-
-        if (dirty & I830_UPLOAD_TEXBLEND3) {
-                i830EmitTexBlendVerified(dev,
-                                         sarea_priv->TexBlendState3,
-                                         sarea_priv->TexBlendStateWordsUsed3);
-                sarea_priv->dirty &= ~I830_UPLOAD_TEXBLEND3;
-        }
-}
-
-/* ================================================================
- * Performance monitoring functions
- */
-
-static void i830_fill_box(struct drm_device *dev,
-                          int x, int y, int w, int h, int r, int g, int b)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        u32 color;
-        unsigned int BR13, CMD;
-        RING_LOCALS;
-
-        BR13 = (0xF0 << 16) | (dev_priv->pitch * dev_priv->cpp) | (1 << 24);
-        CMD = XY_COLOR_BLT_CMD;
-        x += dev_priv->sarea_priv->boxes[0].x1;
-        y += dev_priv->sarea_priv->boxes[0].y1;
-
-        if (dev_priv->cpp == 4) {
-                BR13 |= (1 << 25);
-                CMD |= (XY_COLOR_BLT_WRITE_ALPHA | XY_COLOR_BLT_WRITE_RGB);
-                color = (((0xff) << 24) | (r << 16) | (g << 8) | b);
-        } else {
-                color = (((r & 0xf8) << 8) |
-                         ((g & 0xfc) << 3) | ((b & 0xf8) >> 3));
-        }
-
-        BEGIN_LP_RING(6);
-        OUT_RING(CMD);
-        OUT_RING(BR13);
-        OUT_RING((y << 16) | x);
-        OUT_RING(((y + h) << 16) | (x + w));
-
-        if (dev_priv->current_page == 1)
-                OUT_RING(dev_priv->front_offset);
-        else
-                OUT_RING(dev_priv->back_offset);
-
-        OUT_RING(color);
-        ADVANCE_LP_RING();
-}
-
-static void i830_cp_performance_boxes(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        /* Purple box for page flipping
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_FLIP)
-                i830_fill_box(dev, 4, 4, 8, 8, 255, 0, 255);
-
-        /* Red box if we have to wait for idle at any point
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_WAIT)
-                i830_fill_box(dev, 16, 4, 8, 8, 255, 0, 0);
-
-        /* Blue box: lost context?
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_LOST_CONTEXT)
-                i830_fill_box(dev, 28, 4, 8, 8, 0, 0, 255);
-
-        /* Yellow box for texture swaps
-         */
-        if (dev_priv->sarea_priv->perf_boxes & I830_BOX_TEXTURE_LOAD)
-                i830_fill_box(dev, 40, 4, 8, 8, 255, 255, 0);
-
-        /* Green box if hardware never idles (as far as we can tell)
-         */
-        if (!(dev_priv->sarea_priv->perf_boxes & I830_BOX_RING_EMPTY))
-                i830_fill_box(dev, 64, 4, 8, 8, 0, 255, 0);
-
-        /* Draw bars indicating number of buffers allocated
-         * (not a great measure, easily confused)
-         */
-        if (dev_priv->dma_used) {
-                int bar = dev_priv->dma_used / 10240;
-                if (bar > 100)
-                        bar = 100;
-                if (bar < 1)
-                        bar = 1;
-                i830_fill_box(dev, 4, 16, bar, 4, 196, 128, 128);
-                dev_priv->dma_used = 0;
-        }
-
-        dev_priv->sarea_priv->perf_boxes = 0;
-}
-
-static void i830_dma_dispatch_clear(struct drm_device *dev, int flags,
-                                    unsigned int clear_color,
-                                    unsigned int clear_zval,
-                                    unsigned int clear_depthmask)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        int nbox = sarea_priv->nbox;
-        struct drm_clip_rect *pbox = sarea_priv->boxes;
-        int pitch = dev_priv->pitch;
-        int cpp = dev_priv->cpp;
-        int i;
-        unsigned int BR13, CMD, D_CMD;
-        RING_LOCALS;
-
-        if (dev_priv->current_page == 1) {
-                unsigned int tmp = flags;
-
-                flags &= ~(I830_FRONT | I830_BACK);
-                if (tmp & I830_FRONT)
-                        flags |= I830_BACK;
-                if (tmp & I830_BACK)
-                        flags |= I830_FRONT;
-        }
-
-        i830_kernel_lost_context(dev);
-
-        switch (cpp) {
-        case 2:
-                BR13 = (0xF0 << 16) | (pitch * cpp) | (1 << 24);
-                D_CMD = CMD = XY_COLOR_BLT_CMD;
-                break;
-        case 4:
-                BR13 = (0xF0 << 16) | (pitch * cpp) | (1 << 24) | (1 << 25);
-                CMD = (XY_COLOR_BLT_CMD | XY_COLOR_BLT_WRITE_ALPHA |
-                       XY_COLOR_BLT_WRITE_RGB);
-                D_CMD = XY_COLOR_BLT_CMD;
-                if (clear_depthmask & 0x00ffffff)
-                        D_CMD |= XY_COLOR_BLT_WRITE_RGB;
-                if (clear_depthmask & 0xff000000)
-                        D_CMD |= XY_COLOR_BLT_WRITE_ALPHA;
-                break;
-        default:
-                BR13 = (0xF0 << 16) | (pitch * cpp) | (1 << 24);
-                D_CMD = CMD = XY_COLOR_BLT_CMD;
-                break;
-        }
-
-        if (nbox > I830_NR_SAREA_CLIPRECTS)
-                nbox = I830_NR_SAREA_CLIPRECTS;
-
-        for (i = 0; i < nbox; i++, pbox++) {
-                if (pbox->x1 > pbox->x2 ||
-                    pbox->y1 > pbox->y2 ||
-                    pbox->x2 > dev_priv->w || pbox->y2 > dev_priv->h)
-                        continue;
-
-                if (flags & I830_FRONT) {
-                        DRM_DEBUG("clear front\n");
-                        BEGIN_LP_RING(6);
-                        OUT_RING(CMD);
-                        OUT_RING(BR13);
-                        OUT_RING((pbox->y1 << 16) | pbox->x1);
-                        OUT_RING((pbox->y2 << 16) | pbox->x2);
-                        OUT_RING(dev_priv->front_offset);
-                        OUT_RING(clear_color);
-                        ADVANCE_LP_RING();
-                }
-
-                if (flags & I830_BACK) {
-                        DRM_DEBUG("clear back\n");
-                        BEGIN_LP_RING(6);
-                        OUT_RING(CMD);
-                        OUT_RING(BR13);
-                        OUT_RING((pbox->y1 << 16) | pbox->x1);
-                        OUT_RING((pbox->y2 << 16) | pbox->x2);
-                        OUT_RING(dev_priv->back_offset);
-                        OUT_RING(clear_color);
-                        ADVANCE_LP_RING();
-                }
-
-                if (flags & I830_DEPTH) {
-                        DRM_DEBUG("clear depth\n");
-                        BEGIN_LP_RING(6);
-                        OUT_RING(D_CMD);
-                        OUT_RING(BR13);
-                        OUT_RING((pbox->y1 << 16) | pbox->x1);
-                        OUT_RING((pbox->y2 << 16) | pbox->x2);
-                        OUT_RING(dev_priv->depth_offset);
-                        OUT_RING(clear_zval);
-                        ADVANCE_LP_RING();
-                }
-        }
-}
-
-static void i830_dma_dispatch_swap(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        int nbox = sarea_priv->nbox;
-        struct drm_clip_rect *pbox = sarea_priv->boxes;
-        int pitch = dev_priv->pitch;
-        int cpp = dev_priv->cpp;
-        int i;
-        unsigned int CMD, BR13;
-        RING_LOCALS;
-
-        DRM_DEBUG("swapbuffers\n");
-
-        i830_kernel_lost_context(dev);
-
-        if (dev_priv->do_boxes)
-                i830_cp_performance_boxes(dev);
-
-        switch (cpp) {
-        case 2:
-                BR13 = (pitch * cpp) | (0xCC << 16) | (1 << 24);
-                CMD = XY_SRC_COPY_BLT_CMD;
-                break;
-        case 4:
-                BR13 = (pitch * cpp) | (0xCC << 16) | (1 << 24) | (1 << 25);
-                CMD = (XY_SRC_COPY_BLT_CMD | XY_SRC_COPY_BLT_WRITE_ALPHA |
-                       XY_SRC_COPY_BLT_WRITE_RGB);
-                break;
-        default:
-                BR13 = (pitch * cpp) | (0xCC << 16) | (1 << 24);
-                CMD = XY_SRC_COPY_BLT_CMD;
-                break;
-        }
-
-        if (nbox > I830_NR_SAREA_CLIPRECTS)
-                nbox = I830_NR_SAREA_CLIPRECTS;
-
-        for (i = 0; i < nbox; i++, pbox++) {
-                if (pbox->x1 > pbox->x2 ||
-                    pbox->y1 > pbox->y2 ||
-                    pbox->x2 > dev_priv->w || pbox->y2 > dev_priv->h)
-                        continue;
-
-                DRM_DEBUG("dispatch swap %d,%d-%d,%d!\n",
-                          pbox->x1, pbox->y1, pbox->x2, pbox->y2);
-
-                BEGIN_LP_RING(8);
-                OUT_RING(CMD);
-                OUT_RING(BR13);
-                OUT_RING((pbox->y1 << 16) | pbox->x1);
-                OUT_RING((pbox->y2 << 16) | pbox->x2);
-
-                if (dev_priv->current_page == 0)
-                        OUT_RING(dev_priv->front_offset);
-                else
-                        OUT_RING(dev_priv->back_offset);
-
-                OUT_RING((pbox->y1 << 16) | pbox->x1);
-                OUT_RING(BR13 & 0xffff);
-
-                if (dev_priv->current_page == 0)
-                        OUT_RING(dev_priv->back_offset);
-                else
-                        OUT_RING(dev_priv->front_offset);
-
-                ADVANCE_LP_RING();
-        }
-}
-
-static void i830_dma_dispatch_flip(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
-                  __func__,
-                  dev_priv->current_page,
-                  dev_priv->sarea_priv->pf_current_page);
-
-        i830_kernel_lost_context(dev);
-
-        if (dev_priv->do_boxes) {
-                dev_priv->sarea_priv->perf_boxes |= I830_BOX_FLIP;
-                i830_cp_performance_boxes(dev);
-        }
-
-        BEGIN_LP_RING(2);
-        OUT_RING(INST_PARSER_CLIENT | INST_OP_FLUSH | INST_FLUSH_MAP_CACHE);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        BEGIN_LP_RING(6);
-        OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
-        OUT_RING(0);
-        if (dev_priv->current_page == 0) {
-                OUT_RING(dev_priv->back_offset);
-                dev_priv->current_page = 1;
-        } else {
-                OUT_RING(dev_priv->front_offset);
-                dev_priv->current_page = 0;
-        }
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        BEGIN_LP_RING(2);
-        OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
-}
-
-static void i830_dma_dispatch_vertex(struct drm_device *dev,
-                                     struct drm_buf *buf, int discard, int used)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-        drm_i830_sarea_t *sarea_priv = dev_priv->sarea_priv;
-        struct drm_clip_rect *box = sarea_priv->boxes;
-        int nbox = sarea_priv->nbox;
-        unsigned long address = (unsigned long)buf->bus_address;
-        unsigned long start = address - dev->agp->base;
-        int i = 0, u;
-        RING_LOCALS;
-
-        i830_kernel_lost_context(dev);
-
-        if (nbox > I830_NR_SAREA_CLIPRECTS)
-                nbox = I830_NR_SAREA_CLIPRECTS;
-
-        if (discard) {
-                u = cmpxchg(buf_priv->in_use, I830_BUF_CLIENT,
-                            I830_BUF_HARDWARE);
-                if (u != I830_BUF_CLIENT)
-                        DRM_DEBUG("xxxx 2\n");
-        }
-
-        if (used > 4 * 1023)
-                used = 0;
-
-        if (sarea_priv->dirty)
-                i830EmitState(dev);
-
-        DRM_DEBUG("dispatch vertex addr 0x%lx, used 0x%x nbox %d\n",
-                  address, used, nbox);
-
-        dev_priv->counter++;
-        DRM_DEBUG("dispatch counter : %ld\n", dev_priv->counter);
-        DRM_DEBUG("i830_dma_dispatch\n");
-        DRM_DEBUG("start : %lx\n", start);
-        DRM_DEBUG("used : %d\n", used);
-        DRM_DEBUG("start + used - 4 : %ld\n", start + used - 4);
-
-        if (buf_priv->currently_mapped == I830_BUF_MAPPED) {
-                u32 *vp = buf_priv->kernel_virtual;
-
-                vp[0] = (GFX_OP_PRIMITIVE |
-                         sarea_priv->vertex_prim | ((used / 4) - 2));
-
-                if (dev_priv->use_mi_batchbuffer_start) {
-                        vp[used / 4] = MI_BATCH_BUFFER_END;
-                        used += 4;
-                }
-
-                if (used & 4) {
-                        vp[used / 4] = 0;
-                        used += 4;
-                }
-
-                i830_unmap_buffer(buf);
-        }
-
-        if (used) {
-                do {
-                        if (i < nbox) {
-                                BEGIN_LP_RING(6);
-                                OUT_RING(GFX_OP_DRAWRECT_INFO);
-                                OUT_RING(sarea_priv->
-                                         BufferState[I830_DESTREG_DR1]);
-                                OUT_RING(box[i].x1 | (box[i].y1 << 16));
-                                OUT_RING(box[i].x2 | (box[i].y2 << 16));
-                                OUT_RING(sarea_priv->
-                                         BufferState[I830_DESTREG_DR4]);
-                                OUT_RING(0);
-                                ADVANCE_LP_RING();
-                        }
-
-                        if (dev_priv->use_mi_batchbuffer_start) {
-                                BEGIN_LP_RING(2);
-                                OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
-                                OUT_RING(start | MI_BATCH_NON_SECURE);
-                                ADVANCE_LP_RING();
-                        } else {
-                                BEGIN_LP_RING(4);
-                                OUT_RING(MI_BATCH_BUFFER);
-                                OUT_RING(start | MI_BATCH_NON_SECURE);
-                                OUT_RING(start + used - 4);
-                                OUT_RING(0);
-                                ADVANCE_LP_RING();
-                        }
-
-                } while (++i < nbox);
-        }
-
-        if (discard) {
-                dev_priv->counter++;
-
-                (void)cmpxchg(buf_priv->in_use, I830_BUF_CLIENT,
-                              I830_BUF_HARDWARE);
-
-                BEGIN_LP_RING(8);
-                OUT_RING(CMD_STORE_DWORD_IDX);
-                OUT_RING(20);
-                OUT_RING(dev_priv->counter);
-                OUT_RING(CMD_STORE_DWORD_IDX);
-                OUT_RING(buf_priv->my_use_idx);
-                OUT_RING(I830_BUF_FREE);
-                OUT_RING(CMD_REPORT_HEAD);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        }
-}
-
-static void i830_dma_quiescent(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        i830_kernel_lost_context(dev);
-
-        BEGIN_LP_RING(4);
-        OUT_RING(INST_PARSER_CLIENT | INST_OP_FLUSH | INST_FLUSH_MAP_CACHE);
-        OUT_RING(CMD_REPORT_HEAD);
-        OUT_RING(0);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        i830_wait_ring(dev, dev_priv->ring.Size - 8, __func__);
-}
-
-static int i830_flush_queue(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        struct drm_device_dma *dma = dev->dma;
-        int i, ret = 0;
-        RING_LOCALS;
-
-        i830_kernel_lost_context(dev);
-
-        BEGIN_LP_RING(2);
-        OUT_RING(CMD_REPORT_HEAD);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
-
-        i830_wait_ring(dev, dev_priv->ring.Size - 8, __func__);
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-
-                int used = cmpxchg(buf_priv->in_use, I830_BUF_HARDWARE,
-                                   I830_BUF_FREE);
-
-                if (used == I830_BUF_HARDWARE)
-                        DRM_DEBUG("reclaimed from HARDWARE\n");
-                if (used == I830_BUF_CLIENT)
-                        DRM_DEBUG("still on client\n");
-        }
-
-        return ret;
-}
-
-/* Must be called with the lock held */
-static void i830_reclaim_buffers(struct drm_device *dev, struct drm_file *file_priv)
-{
-        struct drm_device_dma *dma = dev->dma;
-        int i;
-
-        if (!dma)
-                return;
-        if (!dev->dev_private)
-                return;
-        if (!dma->buflist)
-                return;
-
-        i830_flush_queue(dev);
-
-        for (i = 0; i < dma->buf_count; i++) {
-                struct drm_buf *buf = dma->buflist[i];
-                drm_i830_buf_priv_t *buf_priv = buf->dev_private;
-
-                if (buf->file_priv == file_priv && buf_priv) {
-                        int used = cmpxchg(buf_priv->in_use, I830_BUF_CLIENT,
-                                           I830_BUF_FREE);
-
-                        if (used == I830_BUF_CLIENT)
-                                DRM_DEBUG("reclaimed from client\n");
-                        if (buf_priv->currently_mapped == I830_BUF_MAPPED)
-                                buf_priv->currently_mapped = I830_BUF_UNMAPPED;
-                }
-        }
-}
-
-static int i830_flush_ioctl(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv)
-{
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        i830_flush_queue(dev);
-        return 0;
-}
-
-static int i830_dma_vertex(struct drm_device *dev, void *data,
-                           struct drm_file *file_priv)
-{
-        struct drm_device_dma *dma = dev->dma;
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u32 *hw_status = dev_priv->hw_status_page;
-        drm_i830_sarea_t *sarea_priv = (drm_i830_sarea_t *)
-            dev_priv->sarea_priv;
-        drm_i830_vertex_t *vertex = data;
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        DRM_DEBUG("i830 dma vertex, idx %d used %d discard %d\n",
-                  vertex->idx, vertex->used, vertex->discard);
-
-        if (vertex->idx < 0 || vertex->idx > dma->buf_count)
-                return -EINVAL;
-
-        i830_dma_dispatch_vertex(dev,
-                                 dma->buflist[vertex->idx],
-                                 vertex->discard, vertex->used);
-
-        sarea_priv->last_enqueue = dev_priv->counter - 1;
-        sarea_priv->last_dispatch = (int)hw_status[5];
-
-        return 0;
-}
-
-static int i830_clear_bufs(struct drm_device *dev, void *data,
-                           struct drm_file *file_priv)
-{
-        drm_i830_clear_t *clear = data;
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        /* GH: Someone's doing nasty things... */
-        if (!dev->dev_private)
-                return -EINVAL;
-
-        i830_dma_dispatch_clear(dev, clear->flags,
-                                clear->clear_color,
-                                clear->clear_depth, clear->clear_depthmask);
-        return 0;
-}
-
-static int i830_swap_bufs(struct drm_device *dev, void *data,
-                          struct drm_file *file_priv)
-{
-        DRM_DEBUG("i830_swap_bufs\n");
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        i830_dma_dispatch_swap(dev);
-        return 0;
-}
-
-/* Not sure why this isn't set all the time:
- */
-static void i830_do_init_pageflip(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        DRM_DEBUG("%s\n", __func__);
-        dev_priv->page_flipping = 1;
-        dev_priv->current_page = 0;
-        dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
-}
-
-static int i830_do_cleanup_pageflip(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        DRM_DEBUG("%s\n", __func__);
-        if (dev_priv->current_page != 0)
-                i830_dma_dispatch_flip(dev);
-
-        dev_priv->page_flipping = 0;
-        return 0;
-}
-
-static int i830_flip_bufs(struct drm_device *dev, void *data,
-                          struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-
-        DRM_DEBUG("%s\n", __func__);
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        if (!dev_priv->page_flipping)
-                i830_do_init_pageflip(dev);
-
-        i830_dma_dispatch_flip(dev);
-        return 0;
-}
-
-static int i830_getage(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u32 *hw_status = dev_priv->hw_status_page;
-        drm_i830_sarea_t *sarea_priv = (drm_i830_sarea_t *)
-            dev_priv->sarea_priv;
-
-        sarea_priv->last_dispatch = (int)hw_status[5];
-        return 0;
-}
-
-static int i830_getbuf(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        int retcode = 0;
-        drm_i830_dma_t *d = data;
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u32 *hw_status = dev_priv->hw_status_page;
-        drm_i830_sarea_t *sarea_priv = (drm_i830_sarea_t *)
-            dev_priv->sarea_priv;
-
-        DRM_DEBUG("getbuf\n");
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        d->granted = 0;
-
-        retcode = i830_dma_get_buffer(dev, d, file_priv);
-
-        DRM_DEBUG("i830_dma: %d returning %d, granted = %d\n",
-                  task_pid_nr(current), retcode, d->granted);
-
-        sarea_priv->last_dispatch = (int)hw_status[5];
-
-        return retcode;
-}
-
-static int i830_copybuf(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
-{
-        /* Never copy - 2.4.x doesn't need it */
-        return 0;
-}
-
-static int i830_docopy(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        return 0;
-}
-
-static int i830_getparam(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_getparam_t *param = data;
-        int value;
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        switch (param->param) {
-        case I830_PARAM_IRQ_ACTIVE:
-                value = dev->irq_enabled;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        if (copy_to_user(param->value, &value, sizeof(int))) {
-                DRM_ERROR("copy_to_user\n");
-                return -EFAULT;
-        }
-
-        return 0;
-}
-
-static int i830_setparam(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_setparam_t *param = data;
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        switch (param->param) {
-        case I830_SETPARAM_USE_MI_BATCHBUFFER_START:
-                dev_priv->use_mi_batchbuffer_start = param->value;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        return 0;
-}
-
-int i830_driver_load(struct drm_device *dev, unsigned long flags)
-{
-        /* i830 has 4 more counters */
-        dev->counters += 4;
-        dev->types[6] = _DRM_STAT_IRQ;
-        dev->types[7] = _DRM_STAT_PRIMARY;
-        dev->types[8] = _DRM_STAT_SECONDARY;
-        dev->types[9] = _DRM_STAT_DMA;
-
-        return 0;
-}
-
-void i830_driver_lastclose(struct drm_device *dev)
-{
-        i830_dma_cleanup(dev);
-}
-
-void i830_driver_preclose(struct drm_device *dev, struct drm_file *file_priv)
-{
-        if (dev->dev_private) {
-                drm_i830_private_t *dev_priv = dev->dev_private;
-                if (dev_priv->page_flipping)
-                        i830_do_cleanup_pageflip(dev);
-        }
-}
-
-void i830_driver_reclaim_buffers_locked(struct drm_device *dev, struct drm_file *file_priv)
-{
-        i830_reclaim_buffers(dev, file_priv);
-}
-
-int i830_driver_dma_quiescent(struct drm_device *dev)
-{
-        i830_dma_quiescent(dev);
-        return 0;
-}
-
-/*
- * call the drm_ioctl under the big kernel lock because
- * to lock against the i830_mmap_buffers function.
- */
-long i830_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-        int ret;
-        lock_kernel();
-        ret = drm_ioctl(file, cmd, arg);
-        unlock_kernel();
-        return ret;
-}
-
-struct drm_ioctl_desc i830_ioctls[] = {
-        DRM_IOCTL_DEF_DRV(I830_INIT, i830_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_VERTEX, i830_dma_vertex, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_CLEAR, i830_clear_bufs, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_FLUSH, i830_flush_ioctl, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_GETAGE, i830_getage, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_GETBUF, i830_getbuf, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_SWAP, i830_swap_bufs, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_COPY, i830_copybuf, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_DOCOPY, i830_docopy, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_FLIP, i830_flip_bufs, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_IRQ_EMIT, i830_irq_emit, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_IRQ_WAIT, i830_irq_wait, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_GETPARAM, i830_getparam, DRM_AUTH|DRM_UNLOCKED),
-        DRM_IOCTL_DEF_DRV(I830_SETPARAM, i830_setparam, DRM_AUTH|DRM_UNLOCKED),
-};
-
-int i830_max_ioctl = DRM_ARRAY_SIZE(i830_ioctls);
-
-/**
- * Determine if the device really is AGP or not.
- *
- * All Intel graphics chipsets are treated as AGP, even if they are really
- * PCI-e.
- *
- * \param dev   The device to be tested.
- *
- * \returns
- * A value of 1 is always retured to indictate every i8xx is AGP.
- */
-int i830_driver_device_is_agp(struct drm_device *dev)
-{
-        return 1;
-}
diff --git a/drivers/gpu/drm/i830/i830_drv.c b/drivers/gpu/drm/i830/i830_drv.c
deleted file mode 100644
index f655ab7977da..000000000000
--- a/drivers/gpu/drm/i830/i830_drv.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/* i830_drv.c -- I810 driver -*- linux-c -*-
- * Created: Mon Dec 13 01:56:22 1999 by jhartmann@precisioninsight.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Rickard E. (Rik) Faith <faith@valinux.com>
- *    Jeff Hartmann <jhartmann@valinux.com>
- *    Gareth Hughes <gareth@valinux.com>
- *    Abraham vd Merwe <abraham@2d3d.co.za>
- *    Keith Whitwell <keith@tungstengraphics.com>
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "i830_drm.h"
-#include "i830_drv.h"
-
-#include "drm_pciids.h"
-
-static struct pci_device_id pciidlist[] = {
-        i830_PCI_IDS
-};
-
-static struct drm_driver driver = {
-        .driver_features =
-            DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | DRIVER_USE_MTRR |
-            DRIVER_HAVE_DMA | DRIVER_DMA_QUEUE,
-#if USE_IRQS
-        .driver_features |= DRIVER_HAVE_IRQ | DRIVER_SHARED_IRQ,
-#endif
-        .dev_priv_size = sizeof(drm_i830_buf_priv_t),
-        .load = i830_driver_load,
-        .lastclose = i830_driver_lastclose,
-        .preclose = i830_driver_preclose,
-        .device_is_agp = i830_driver_device_is_agp,
-        .reclaim_buffers_locked = i830_driver_reclaim_buffers_locked,
-        .dma_quiescent = i830_driver_dma_quiescent,
-#if USE_IRQS
-        .irq_preinstall = i830_driver_irq_preinstall,
-        .irq_postinstall = i830_driver_irq_postinstall,
-        .irq_uninstall = i830_driver_irq_uninstall,
-        .irq_handler = i830_driver_irq_handler,
-#endif
-        .ioctls = i830_ioctls,
-        .fops = {
-                 .owner = THIS_MODULE,
-                 .open = drm_open,
-                 .release = drm_release,
-                 .unlocked_ioctl = i830_ioctl,
-                 .mmap = drm_mmap,
-                 .poll = drm_poll,
-                 .fasync = drm_fasync,
-                 .llseek = noop_llseek,
-        },
-
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
-
-        .name = DRIVER_NAME,
-        .desc = DRIVER_DESC,
-        .date = DRIVER_DATE,
-        .major = DRIVER_MAJOR,
-        .minor = DRIVER_MINOR,
-        .patchlevel = DRIVER_PATCHLEVEL,
-};
-
-static int __init i830_init(void)
-{
-        driver.num_ioctls = i830_max_ioctl;
-        return drm_init(&driver);
-}
-
-static void __exit i830_exit(void)
-{
-        drm_exit(&driver);
-}
-
-module_init(i830_init);
-module_exit(i830_exit);
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/i830/i830_drv.h b/drivers/gpu/drm/i830/i830_drv.h
deleted file mode 100644
index 0df1c720560b..000000000000
--- a/drivers/gpu/drm/i830/i830_drv.h
+++ /dev/null
@@ -1,295 +0,0 @@
-/* i830_drv.h -- Private header for the I830 driver -*- linux-c -*-
- * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com
- *
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
- * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
- * All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors: Rickard E. (Rik) Faith <faith@valinux.com>
- *          Jeff Hartmann <jhartmann@valinux.com>
- *
- */
-
-#ifndef _I830_DRV_H_
-#define _I830_DRV_H_
-
-/* General customization:
- */
-
-#define DRIVER_AUTHOR           "VA Linux Systems Inc."
-
-#define DRIVER_NAME             "i830"
-#define DRIVER_DESC             "Intel 830M"
-#define DRIVER_DATE             "20021108"
-
-/* Interface history:
- *
- * 1.1: Original.
- * 1.2: ?
- * 1.3: New irq emit/wait ioctls.
- *      New pageflip ioctl.
- *      New getparam ioctl.
- *      State for texunits 3&4 in sarea.
- *      New (alternative) layout for texture state.
- */
-#define DRIVER_MAJOR            1
-#define DRIVER_MINOR            3
-#define DRIVER_PATCHLEVEL       2
-
-/* Driver will work either way: IRQ's save cpu time when waiting for
- * the card, but are subject to subtle interactions between bios,
- * hardware and the driver.
- */
-/* XXX: Add vblank support? */
-#define USE_IRQS 0
-
-typedef struct drm_i830_buf_priv {
-        u32 *in_use;
-        int my_use_idx;
-        int currently_mapped;
-        void __user *virtual;
-        void *kernel_virtual;
-        drm_local_map_t map;
-} drm_i830_buf_priv_t;
-
-typedef struct _drm_i830_ring_buffer {
-        int tail_mask;
-        unsigned long Start;
-        unsigned long End;
-        unsigned long Size;
-        u8 *virtual_start;
-        int head;
-        int tail;
-        int space;
-        drm_local_map_t map;
-} drm_i830_ring_buffer_t;
-
-typedef struct drm_i830_private {
-        struct drm_local_map *sarea_map;
-        struct drm_local_map *mmio_map;
-
-        drm_i830_sarea_t *sarea_priv;
-        drm_i830_ring_buffer_t ring;
-
-        void *hw_status_page;
-        unsigned long counter;
-
-        dma_addr_t dma_status_page;
-
-        struct drm_buf *mmap_buffer;
-
-        u32 front_di1, back_di1, zi1;
-
-        int back_offset;
-        int depth_offset;
-        int front_offset;
-        int w, h;
-        int pitch;
-        int back_pitch;
-        int depth_pitch;
-        unsigned int cpp;
-
-        int do_boxes;
-        int dma_used;
-
-        int current_page;
-        int page_flipping;
-
-        wait_queue_head_t irq_queue;
-        atomic_t irq_received;
-        atomic_t irq_emitted;
-
-        int use_mi_batchbuffer_start;
-
-} drm_i830_private_t;
-
-long i830_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
-extern struct drm_ioctl_desc i830_ioctls[];
-extern int i830_max_ioctl;
-
-/* i830_irq.c */
-extern int i830_irq_emit(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv);
-extern int i830_irq_wait(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv);
-
-extern irqreturn_t i830_driver_irq_handler(DRM_IRQ_ARGS);
-extern void i830_driver_irq_preinstall(struct drm_device *dev);
-extern void i830_driver_irq_postinstall(struct drm_device *dev);
-extern void i830_driver_irq_uninstall(struct drm_device *dev);
-extern int i830_driver_load(struct drm_device *, unsigned long flags);
-extern void i830_driver_preclose(struct drm_device *dev,
-                                 struct drm_file *file_priv);
-extern void i830_driver_lastclose(struct drm_device *dev);
-extern void i830_driver_reclaim_buffers_locked(struct drm_device *dev,
-                                               struct drm_file *file_priv);
-extern int i830_driver_dma_quiescent(struct drm_device *dev);
-extern int i830_driver_device_is_agp(struct drm_device *dev);
-
-#define I830_READ(reg)          DRM_READ32(dev_priv->mmio_map, reg)
-#define I830_WRITE(reg, val)    DRM_WRITE32(dev_priv->mmio_map, reg, val)
-#define I830_READ16(reg)        DRM_READ16(dev_priv->mmio_map, reg)
-#define I830_WRITE16(reg, val)  DRM_WRITE16(dev_priv->mmio_map, reg, val)
-
-#define I830_VERBOSE 0
-
-#define RING_LOCALS     unsigned int outring, ringmask, outcount; \
-                        volatile char *virt;
-
-#define BEGIN_LP_RING(n) do {                           \
-        if (I830_VERBOSE)                               \
-                printk("BEGIN_LP_RING(%d)\n", (n));     \
-        if (dev_priv->ring.space < n*4)                 \
-                i830_wait_ring(dev, n*4, __func__);     \
-        outcount = 0;                                   \
-        outring = dev_priv->ring.tail;                  \
-        ringmask = dev_priv->ring.tail_mask;            \
-        virt = dev_priv->ring.virtual_start;            \
-} while (0)
-
-#define OUT_RING(n) do {                                        \
-        if (I830_VERBOSE)                                       \
-                printk("   OUT_RING %x\n", (int)(n));           \
-        *(volatile unsigned int *)(virt + outring) = n;         \
-        outcount++;                                             \
-        outring += 4;                                           \
-        outring &= ringmask;                                    \
-} while (0)
-
-#define ADVANCE_LP_RING() do {                                  \
-        if (I830_VERBOSE)                                       \
-                printk("ADVANCE_LP_RING %x\n", outring);        \
-        dev_priv->ring.tail = outring;                          \
-        dev_priv->ring.space -= outcount * 4;                   \
-        I830_WRITE(LP_RING + RING_TAIL, outring);               \
-} while (0)
-
-extern int i830_wait_ring(struct drm_device *dev, int n, const char *caller);
-
-#define GFX_OP_USER_INTERRUPT           ((0<<29)|(2<<23))
-#define GFX_OP_BREAKPOINT_INTERRUPT     ((0<<29)|(1<<23))
-#define CMD_REPORT_HEAD                 (7<<23)
-#define CMD_STORE_DWORD_IDX             ((0x21<<23) | 0x1)
-#define CMD_OP_BATCH_BUFFER  ((0x0<<29)|(0x30<<23)|0x1)
-
-#define STATE3D_LOAD_STATE_IMMEDIATE_2      ((0x3<<29)|(0x1d<<24)|(0x03<<16))
-#define LOAD_TEXTURE_MAP0                   (1<<11)
-
-#define INST_PARSER_CLIENT   0x00000000
-#define INST_OP_FLUSH        0x02000000
-#define INST_FLUSH_MAP_CACHE 0x00000001
-
-#define BB1_START_ADDR_MASK   (~0x7)
-#define BB1_PROTECTED         (1<<0)
-#define BB1_UNPROTECTED       (0<<0)
-#define BB2_END_ADDR_MASK     (~0x7)
-
-#define I830REG_HWSTAM          0x02098
-#define I830REG_INT_IDENTITY_R  0x020a4
-#define I830REG_INT_MASK_R      0x020a8
-#define I830REG_INT_ENABLE_R    0x020a0
-
-#define I830_IRQ_RESERVED ((1<<13)|(3<<2))
-
-#define LP_RING                 0x2030
-#define HP_RING                 0x2040
-#define RING_TAIL               0x00
-#define TAIL_ADDR               0x001FFFF8
-#define RING_HEAD               0x04
-#define HEAD_WRAP_COUNT         0xFFE00000
-#define HEAD_WRAP_ONE           0x00200000
-#define HEAD_ADDR               0x001FFFFC
-#define RING_START              0x08
-#define START_ADDR              0x0xFFFFF000
-#define RING_LEN                0x0C
-#define RING_NR_PAGES           0x001FF000
-#define RING_REPORT_MASK        0x00000006
-#define RING_REPORT_64K         0x00000002
-#define RING_REPORT_128K        0x00000004
-#define RING_NO_REPORT          0x00000000
-#define RING_VALID_MASK         0x00000001
-#define RING_VALID              0x00000001
-#define RING_INVALID            0x00000000
-
-#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define SC_UPDATE_SCISSOR       (0x1<<1)
-#define SC_ENABLE_MASK          (0x1<<0)
-#define SC_ENABLE               (0x1<<0)
-
-#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
-#define SCI_YMIN_MASK      (0xffff<<16)
-#define SCI_XMIN_MASK      (0xffff<<0)
-#define SCI_YMAX_MASK      (0xffff<<16)
-#define SCI_XMAX_MASK      (0xffff<<0)
-
-#define GFX_OP_SCISSOR_ENABLE    ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define GFX_OP_SCISSOR_RECT      ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
-#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
-#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
-#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
-#define GFX_OP_PRIMITIVE         ((0x3<<29)|(0x1f<<24))
-
-#define CMD_OP_DESTBUFFER_INFO   ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
-
-#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
-#define ASYNC_FLIP                (1<<22)
-
-#define CMD_3D                          (0x3<<29)
-#define STATE3D_CONST_BLEND_COLOR_CMD   (CMD_3D|(0x1d<<24)|(0x88<<16))
-#define STATE3D_MAP_COORD_SETBIND_CMD   (CMD_3D|(0x1d<<24)|(0x02<<16))
-
-#define BR00_BITBLT_CLIENT   0x40000000
-#define BR00_OP_COLOR_BLT    0x10000000
-#define BR00_OP_SRC_COPY_BLT 0x10C00000
-#define BR13_SOLID_PATTERN   0x80000000
-
-#define BUF_3D_ID_COLOR_BACK    (0x3<<24)
-#define BUF_3D_ID_DEPTH         (0x7<<24)
-#define BUF_3D_USE_FENCE        (1<<23)
-#define BUF_3D_PITCH(x)         (((x)/4)<<2)
-
-#define CMD_OP_MAP_PALETTE_LOAD ((3<<29)|(0x1d<<24)|(0x82<<16)|255)
-#define MAP_PALETTE_NUM(x)      ((x<<8) & (1<<8))
-#define MAP_PALETTE_BOTH        (1<<11)
-
-#define XY_COLOR_BLT_CMD                ((2<<29)|(0x50<<22)|0x4)
-#define XY_COLOR_BLT_WRITE_ALPHA        (1<<21)
-#define XY_COLOR_BLT_WRITE_RGB          (1<<20)
-
-#define XY_SRC_COPY_BLT_CMD             ((2<<29)|(0x53<<22)|6)
-#define XY_SRC_COPY_BLT_WRITE_ALPHA     (1<<21)
-#define XY_SRC_COPY_BLT_WRITE_RGB       (1<<20)
-
-#define MI_BATCH_BUFFER         ((0x30<<23)|1)
-#define MI_BATCH_BUFFER_START   (0x31<<23)
-#define MI_BATCH_BUFFER_END     (0xA<<23)
-#define MI_BATCH_NON_SECURE     (1)
-
-#define MI_WAIT_FOR_EVENT       ((0x3<<23))
-#define MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
-#define MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
-
-#define MI_LOAD_SCAN_LINES_INCL  ((0x12<<23))
-
-#endif
diff --git a/drivers/gpu/drm/i830/i830_irq.c b/drivers/gpu/drm/i830/i830_irq.c
deleted file mode 100644
index d1a6b95d631d..000000000000
--- a/drivers/gpu/drm/i830/i830_irq.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/* i830_dma.c -- DMA support for the I830 -*- linux-c -*-
- *
- * Copyright 2002 Tungsten Graphics, Inc.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors: Keith Whitwell <keith@tungstengraphics.com>
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "i830_drm.h"
-#include "i830_drv.h"
-#include <linux/interrupt.h>    /* For task queue support */
-#include <linux/delay.h>
-
-irqreturn_t i830_driver_irq_handler(DRM_IRQ_ARGS)
-{
-        struct drm_device *dev = (struct drm_device *) arg;
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        u16 temp;
-
-        temp = I830_READ16(I830REG_INT_IDENTITY_R);
-        DRM_DEBUG("%x\n", temp);
-
-        if (!(temp & 2))
-                return IRQ_NONE;
-
-        I830_WRITE16(I830REG_INT_IDENTITY_R, temp);
-
-        atomic_inc(&dev_priv->irq_received);
-        wake_up_interruptible(&dev_priv->irq_queue);
-
-        return IRQ_HANDLED;
-}
-
-static int i830_emit_irq(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        RING_LOCALS;
-
-        DRM_DEBUG("%s\n", __func__);
-
-        atomic_inc(&dev_priv->irq_emitted);
-
-        BEGIN_LP_RING(2);
-        OUT_RING(0);
-        OUT_RING(GFX_OP_USER_INTERRUPT);
-        ADVANCE_LP_RING();
-
-        return atomic_read(&dev_priv->irq_emitted);
-}
-
-static int i830_wait_irq(struct drm_device *dev, int irq_nr)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        DECLARE_WAITQUEUE(entry, current);
-        unsigned long end = jiffies + HZ * 3;
-        int ret = 0;
-
-        DRM_DEBUG("%s\n", __func__);
-
-        if (atomic_read(&dev_priv->irq_received) >= irq_nr)
-                return 0;
-
-        dev_priv->sarea_priv->perf_boxes |= I830_BOX_WAIT;
-
-        add_wait_queue(&dev_priv->irq_queue, &entry);
-
-        for (;;) {
-                __set_current_state(TASK_INTERRUPTIBLE);
-                if (atomic_read(&dev_priv->irq_received) >= irq_nr)
-                        break;
-                if ((signed)(end - jiffies) <= 0) {
-                        DRM_ERROR("timeout iir %x imr %x ier %x hwstam %x\n",
-                                  I830_READ16(I830REG_INT_IDENTITY_R),
-                                  I830_READ16(I830REG_INT_MASK_R),
-                                  I830_READ16(I830REG_INT_ENABLE_R),
-                                  I830_READ16(I830REG_HWSTAM));
-
-                        ret = -EBUSY;   /* Lockup?  Missed irq? */
-                        break;
-                }
-                schedule_timeout(HZ * 3);
-                if (signal_pending(current)) {
-                        ret = -EINTR;
-                        break;
-                }
-        }
-
-        __set_current_state(TASK_RUNNING);
-        remove_wait_queue(&dev_priv->irq_queue, &entry);
-        return ret;
-}
-
-/* Needs the lock as it touches the ring.
- */
-int i830_irq_emit(struct drm_device *dev, void *data,
-                  struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_irq_emit_t *emit = data;
-        int result;
-
-        LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        result = i830_emit_irq(dev);
-
-        if (copy_to_user(emit->irq_seq, &result, sizeof(int))) {
-                DRM_ERROR("copy_to_user\n");
-                return -EFAULT;
-        }
-
-        return 0;
-}
-
-/* Doesn't need the hardware lock.
- */
-int i830_irq_wait(struct drm_device *dev, void *data,
-                  struct drm_file *file_priv)
-{
-        drm_i830_private_t *dev_priv = dev->dev_private;
-        drm_i830_irq_wait_t *irqwait = data;
-
-        if (!dev_priv) {
-                DRM_ERROR("%s called with no initialization\n", __func__);
-                return -EINVAL;
-        }
-
-        return i830_wait_irq(dev, irqwait->irq_seq);
-}
-
-/* drm_dma.h hooks
-*/
-void i830_driver_irq_preinstall(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-
-        I830_WRITE16(I830REG_HWSTAM, 0xffff);
-        I830_WRITE16(I830REG_INT_MASK_R, 0x0);
-        I830_WRITE16(I830REG_INT_ENABLE_R, 0x0);
-        atomic_set(&dev_priv->irq_received, 0);
-        atomic_set(&dev_priv->irq_emitted, 0);
-        init_waitqueue_head(&dev_priv->irq_queue);
-}
-
-void i830_driver_irq_postinstall(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-
-        I830_WRITE16(I830REG_INT_ENABLE_R, 0x2);
-}
-
-void i830_driver_irq_uninstall(struct drm_device *dev)
-{
-        drm_i830_private_t *dev_priv = (drm_i830_private_t *) dev->dev_private;
-        if (!dev_priv)
-                return;
-
-        I830_WRITE16(I830REG_INT_MASK_R, 0xffff);
-        I830_WRITE16(I830REG_INT_ENABLE_R, 0x0);
-}
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index fdc833d5cc7b..0ae6a7c5020f 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -9,6 +9,8 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
           i915_gem.o \
           i915_gem_debug.o \
           i915_gem_evict.o \
+          i915_gem_execbuffer.o \
+          i915_gem_gtt.o \
           i915_gem_tiling.o \
           i915_trace_points.o \
           intel_display.o \
diff --git a/drivers/gpu/drm/i915/dvo_ch7017.c b/drivers/gpu/drm/i915/dvo_ch7017.c
index af70337567ce..d3e8c540f778 100644
--- a/drivers/gpu/drm/i915/dvo_ch7017.c
+++ b/drivers/gpu/drm/i915/dvo_ch7017.c
@@ -242,7 +242,7 @@ fail:
 
 static enum drm_connector_status ch7017_detect(struct intel_dvo_device *dvo)
 {
-        return connector_status_unknown;
+        return connector_status_connected;
 }
 
 static enum drm_mode_status ch7017_mode_valid(struct intel_dvo_device *dvo,
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 1f4f3ceb63c7..3601466c5502 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -32,6 +32,7 @@
 #include "drmP.h"
 #include "drm.h"
 #include "intel_drv.h"
+#include "intel_ringbuffer.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
 
@@ -72,7 +73,6 @@ static int i915_capabilities(struct seq_file *m, void *data)
         B(is_broadwater);
         B(is_crestline);
         B(has_fbc);
-        B(has_rc6);
         B(has_pipe_cxsr);
         B(has_hotplug);
         B(cursor_needs_physical);
@@ -86,19 +86,19 @@ static int i915_capabilities(struct seq_file *m, void *data)
         return 0;
 }
 
-static const char *get_pin_flag(struct drm_i915_gem_object *obj_priv)
+static const char *get_pin_flag(struct drm_i915_gem_object *obj)
 {
-        if (obj_priv->user_pin_count > 0)
+        if (obj->user_pin_count > 0)
                 return "P";
-        else if (obj_priv->pin_count > 0)
+        else if (obj->pin_count > 0)
                 return "p";
         else
                 return " ";
 }
 
-static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv)
+static const char *get_tiling_flag(struct drm_i915_gem_object *obj)
 {
-    switch (obj_priv->tiling_mode) {
+    switch (obj->tiling_mode) {
     default:
     case I915_TILING_NONE: return " ";
     case I915_TILING_X: return "X";
@@ -106,10 +106,19 @@ static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv)
     }
 }
 
+static const char *agp_type_str(int type)
+{
+        switch (type) {
+        case 0: return " uncached";
+        case 1: return " snooped";
+        default: return "";
+        }
+}
+
 static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
-        seq_printf(m, "%p: %s%s %8zd %08x %08x %d%s%s",
+        seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s%s",
                    &obj->base,
                    get_pin_flag(obj),
                    get_tiling_flag(obj),
@@ -117,6 +126,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
                    obj->base.read_domains,
                    obj->base.write_domain,
                    obj->last_rendering_seqno,
+                   obj->last_fenced_seqno,
+                   agp_type_str(obj->agp_type == AGP_USER_CACHED_MEMORY),
                    obj->dirty ? " dirty" : "",
                    obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
         if (obj->base.name)
@@ -124,7 +135,17 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
         if (obj->fence_reg != I915_FENCE_REG_NONE)
                 seq_printf(m, " (fence: %d)", obj->fence_reg);
         if (obj->gtt_space != NULL)
-                seq_printf(m, " (gtt_offset: %08x)", obj->gtt_offset);
+                seq_printf(m, " (gtt offset: %08x, size: %08x)",
+                           obj->gtt_offset, (unsigned int)obj->gtt_space->size);
+        if (obj->pin_mappable || obj->fault_mappable) {
+                char s[3], *t = s;
+                if (obj->pin_mappable)
+                        *t++ = 'p';
+                if (obj->fault_mappable)
+                        *t++ = 'f';
+                *t = '\0';
+                seq_printf(m, " (%s mappable)", s);
+        }
         if (obj->ring != NULL)
                 seq_printf(m, " (%s)", obj->ring->name);
 }
@@ -136,7 +157,7 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
         struct list_head *head;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         size_t total_obj_size, total_gtt_size;
         int count, ret;
 
@@ -171,12 +192,12 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
         }
 
         total_obj_size = total_gtt_size = count = 0;
-        list_for_each_entry(obj_priv, head, mm_list) {
+        list_for_each_entry(obj, head, mm_list) {
                 seq_printf(m, "   ");
-                describe_obj(m, obj_priv);
+                describe_obj(m, obj);
                 seq_printf(m, "\n");
-                total_obj_size += obj_priv->base.size;
-                total_gtt_size += obj_priv->gtt_space->size;
+                total_obj_size += obj->base.size;
+                total_gtt_size += obj->gtt_space->size;
                 count++;
         }
         mutex_unlock(&dev->struct_mutex);
@@ -186,30 +207,116 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
         return 0;
 }
 
+#define count_objects(list, member) do { \
+        list_for_each_entry(obj, list, member) { \
+                size += obj->gtt_space->size; \
+                ++count; \
+                if (obj->map_and_fenceable) { \
+                        mappable_size += obj->gtt_space->size; \
+                        ++mappable_count; \
+                } \
+        } \
+} while(0)
+
 static int i915_gem_object_info(struct seq_file *m, void* data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 count, mappable_count;
+        size_t size, mappable_size;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
         if (ret)
                 return ret;
 
-        seq_printf(m, "%u objects\n", dev_priv->mm.object_count);
-        seq_printf(m, "%zu object bytes\n", dev_priv->mm.object_memory);
-        seq_printf(m, "%u pinned\n", dev_priv->mm.pin_count);
-        seq_printf(m, "%zu pin bytes\n", dev_priv->mm.pin_memory);
-        seq_printf(m, "%u objects in gtt\n", dev_priv->mm.gtt_count);
-        seq_printf(m, "%zu gtt bytes\n", dev_priv->mm.gtt_memory);
-        seq_printf(m, "%zu gtt total\n", dev_priv->mm.gtt_total);
+        seq_printf(m, "%u objects, %zu bytes\n",
+                   dev_priv->mm.object_count,
+                   dev_priv->mm.object_memory);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.gtt_list, gtt_list);
+        seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.active_list, mm_list);
+        count_objects(&dev_priv->mm.flushing_list, mm_list);
+        seq_printf(m, "  %u [%u] active objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.pinned_list, mm_list);
+        seq_printf(m, "  %u [%u] pinned objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.inactive_list, mm_list);
+        seq_printf(m, "  %u [%u] inactive objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        count_objects(&dev_priv->mm.deferred_free_list, mm_list);
+        seq_printf(m, "  %u [%u] freed objects, %zu [%zu] bytes\n",
+                   count, mappable_count, size, mappable_size);
+
+        size = count = mappable_size = mappable_count = 0;
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                if (obj->fault_mappable) {
+                        size += obj->gtt_space->size;
+                        ++count;
+                }
+                if (obj->pin_mappable) {
+                        mappable_size += obj->gtt_space->size;
+                        ++mappable_count;
+                }
+        }
+        seq_printf(m, "%u pinned mappable objects, %zu bytes\n",
+                   mappable_count, mappable_size);
+        seq_printf(m, "%u fault mappable objects, %zu bytes\n",
+                   count, size);
+
+        seq_printf(m, "%zu [%zu] gtt total\n",
+                   dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total);
 
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
 }
 
+static int i915_gem_gtt_info(struct seq_file *m, void* data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+        size_t total_obj_size, total_gtt_size;
+        int count, ret;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        total_obj_size = total_gtt_size = count = 0;
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                seq_printf(m, "   ");
+                describe_obj(m, obj);
+                seq_printf(m, "\n");
+                total_obj_size += obj->base.size;
+                total_gtt_size += obj->gtt_space->size;
+                count++;
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n",
+                   count, total_obj_size, total_gtt_size);
+
+        return 0;
+}
+
 
 static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 {
@@ -243,14 +350,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
                         seq_printf(m, "%d prepares\n", work->pending);
 
                         if (work->old_fb_obj) {
-                                struct drm_i915_gem_object *obj_priv = to_intel_bo(work->old_fb_obj);
-                                if(obj_priv)
-                                        seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset );
+                                struct drm_i915_gem_object *obj = work->old_fb_obj;
+                                if (obj)
+                                        seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
                         }
                         if (work->pending_flip_obj) {
-                                struct drm_i915_gem_object *obj_priv = to_intel_bo(work->pending_flip_obj);
-                                if(obj_priv)
-                                        seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset );
+                                struct drm_i915_gem_object *obj = work->pending_flip_obj;
+                                if (obj)
+                                        seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
                         }
                 }
                 spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -265,44 +372,80 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_request *gem_request;
-        int ret;
+        int ret, count;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
         if (ret)
                 return ret;
 
-        seq_printf(m, "Request:\n");
-        list_for_each_entry(gem_request, &dev_priv->render_ring.request_list,
-                        list) {
-                seq_printf(m, "    %d @ %d\n",
-                           gem_request->seqno,
-                           (int) (jiffies - gem_request->emitted_jiffies));
+        count = 0;
+        if (!list_empty(&dev_priv->ring[RCS].request_list)) {
+                seq_printf(m, "Render requests:\n");
+                list_for_each_entry(gem_request,
+                                    &dev_priv->ring[RCS].request_list,
+                                    list) {
+                        seq_printf(m, "    %d @ %d\n",
+                                   gem_request->seqno,
+                                   (int) (jiffies - gem_request->emitted_jiffies));
+                }
+                count++;
+        }
+        if (!list_empty(&dev_priv->ring[VCS].request_list)) {
+                seq_printf(m, "BSD requests:\n");
+                list_for_each_entry(gem_request,
+                                    &dev_priv->ring[VCS].request_list,
+                                    list) {
+                        seq_printf(m, "    %d @ %d\n",
+                                   gem_request->seqno,
+                                   (int) (jiffies - gem_request->emitted_jiffies));
+                }
+                count++;
+        }
+        if (!list_empty(&dev_priv->ring[BCS].request_list)) {
+                seq_printf(m, "BLT requests:\n");
+                list_for_each_entry(gem_request,
+                                    &dev_priv->ring[BCS].request_list,
+                                    list) {
+                        seq_printf(m, "    %d @ %d\n",
+                                   gem_request->seqno,
+                                   (int) (jiffies - gem_request->emitted_jiffies));
+                }
+                count++;
         }
         mutex_unlock(&dev->struct_mutex);
 
+        if (count == 0)
+                seq_printf(m, "No requests\n");
+
         return 0;
 }
 
+static void i915_ring_seqno_info(struct seq_file *m,
+                                 struct intel_ring_buffer *ring)
+{
+        if (ring->get_seqno) {
+                seq_printf(m, "Current sequence (%s): %d\n",
+                           ring->name, ring->get_seqno(ring));
+                seq_printf(m, "Waiter sequence (%s):  %d\n",
+                           ring->name, ring->waiting_seqno);
+                seq_printf(m, "IRQ sequence (%s):     %d\n",
+                           ring->name, ring->irq_seqno);
+        }
+}
+
 static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
+        int ret, i;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
         if (ret)
                 return ret;
 
-        if (dev_priv->render_ring.status_page.page_addr != NULL) {
-                seq_printf(m, "Current sequence: %d\n",
-                           dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring));
-        } else {
-                seq_printf(m, "Current sequence: hws uninitialized\n");
-        }
-        seq_printf(m, "Waiter sequence:  %d\n",
-                        dev_priv->mm.waiting_gem_seqno);
-        seq_printf(m, "IRQ sequence:     %d\n", dev_priv->mm.irq_gem_seqno);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                i915_ring_seqno_info(m, &dev_priv->ring[i]);
 
         mutex_unlock(&dev->struct_mutex);
 
@@ -315,7 +458,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
+        int ret, i;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
         if (ret)
@@ -354,16 +497,14 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         }
         seq_printf(m, "Interrupts received: %d\n",
                    atomic_read(&dev_priv->irq_received));
-        if (dev_priv->render_ring.status_page.page_addr != NULL) {
-                seq_printf(m, "Current sequence:    %d\n",
-                           dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring));
-        } else {
-                seq_printf(m, "Current sequence:    hws uninitialized\n");
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                if (IS_GEN6(dev)) {
+                        seq_printf(m, "Graphics Interrupt mask (%s):    %08x\n",
+                                   dev_priv->ring[i].name,
+                                   I915_READ_IMR(&dev_priv->ring[i]));
+                }
+                i915_ring_seqno_info(m, &dev_priv->ring[i]);
         }
-        seq_printf(m, "Waiter sequence:     %d\n",
-                   dev_priv->mm.waiting_gem_seqno);
-        seq_printf(m, "IRQ sequence:        %d\n",
-                   dev_priv->mm.irq_gem_seqno);
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
@@ -383,29 +524,17 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
         seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start);
         seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
         for (i = 0; i < dev_priv->num_fence_regs; i++) {
-                struct drm_gem_object *obj = dev_priv->fence_regs[i].obj;
+                struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
 
-                if (obj == NULL) {
-                        seq_printf(m, "Fenced object[%2d] = unused\n", i);
-                } else {
-                        struct drm_i915_gem_object *obj_priv;
-
-                        obj_priv = to_intel_bo(obj);
-                        seq_printf(m, "Fenced object[%2d] = %p: %s "
-                                   "%08x %08zx %08x %s %08x %08x %d",
-                                   i, obj, get_pin_flag(obj_priv),
-                                   obj_priv->gtt_offset,
-                                   obj->size, obj_priv->stride,
-                                   get_tiling_flag(obj_priv),
-                                   obj->read_domains, obj->write_domain,
-                                   obj_priv->last_rendering_seqno);
-                        if (obj->name)
-                                seq_printf(m, " (name: %d)", obj->name);
-                        seq_printf(m, "\n");
-                }
+                seq_printf(m, "Fenced object[%2d] = ", i);
+                if (obj == NULL)
+                        seq_printf(m, "unused");
+                else
+                        describe_obj(m, obj);
+                seq_printf(m, "\n");
         }
-        mutex_unlock(&dev->struct_mutex);
 
+        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -414,10 +543,12 @@ static int i915_hws_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int i;
+        struct intel_ring_buffer *ring;
         volatile u32 *hws;
+        int i;
 
-        hws = (volatile u32 *)dev_priv->render_ring.status_page.page_addr;
+        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
+        hws = (volatile u32 *)ring->status_page.page_addr;
         if (hws == NULL)
                 return 0;
 
@@ -431,14 +562,14 @@ static int i915_hws_info(struct seq_file *m, void *data)
 
 static void i915_dump_object(struct seq_file *m,
                              struct io_mapping *mapping,
-                             struct drm_i915_gem_object *obj_priv)
+                             struct drm_i915_gem_object *obj)
 {
         int page, page_count, i;
 
-        page_count = obj_priv->base.size / PAGE_SIZE;
+        page_count = obj->base.size / PAGE_SIZE;
         for (page = 0; page < page_count; page++) {
                 u32 *mem = io_mapping_map_wc(mapping,
-                                             obj_priv->gtt_offset + page * PAGE_SIZE);
+                                             obj->gtt_offset + page * PAGE_SIZE);
                 for (i = 0; i < PAGE_SIZE; i += 4)
                         seq_printf(m, "%08x :  %08x\n", i, mem[i / 4]);
                 io_mapping_unmap(mem);
@@ -450,25 +581,21 @@ static int i915_batchbuffer_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
         if (ret)
                 return ret;
 
-        list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
-                obj = &obj_priv->base;
-                if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) {
-                    seq_printf(m, "--- gtt_offset = 0x%08x\n",
-                               obj_priv->gtt_offset);
-                    i915_dump_object(m, dev_priv->mm.gtt_mapping, obj_priv);
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
+                if (obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) {
+                    seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
+                    i915_dump_object(m, dev_priv->mm.gtt_mapping, obj);
                 }
         }
 
         mutex_unlock(&dev->struct_mutex);
-
         return 0;
 }
 
@@ -477,19 +604,21 @@ static int i915_ringbuffer_data(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
         if (ret)
                 return ret;
 
-        if (!dev_priv->render_ring.gem_object) {
+        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
+        if (!ring->obj) {
                 seq_printf(m, "No ringbuffer setup\n");
         } else {
-                u8 *virt = dev_priv->render_ring.virtual_start;
+                u8 *virt = ring->virtual_start;
                 uint32_t off;
 
-                for (off = 0; off < dev_priv->render_ring.size; off += 4) {
+                for (off = 0; off < ring->size; off += 4) {
                         uint32_t *ptr = (uint32_t *)(virt + off);
                         seq_printf(m, "%08x :  %08x\n", off, *ptr);
                 }
@@ -504,19 +633,38 @@ static int i915_ringbuffer_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        unsigned int head, tail;
+        struct intel_ring_buffer *ring;
 
-        head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-        tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
+        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
+        if (ring->size == 0)
+                return 0;
 
-        seq_printf(m, "RingHead :  %08x\n", head);
-        seq_printf(m, "RingTail :  %08x\n", tail);
-        seq_printf(m, "RingSize :  %08lx\n", dev_priv->render_ring.size);
-        seq_printf(m, "Acthd :     %08x\n", I915_READ(INTEL_INFO(dev)->gen >= 4 ? ACTHD_I965 : ACTHD));
+        seq_printf(m, "Ring %s:\n", ring->name);
+        seq_printf(m, "  Head :    %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR);
+        seq_printf(m, "  Tail :    %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR);
+        seq_printf(m, "  Size :    %08x\n", ring->size);
+        seq_printf(m, "  Active :  %08x\n", intel_ring_get_active_head(ring));
+        seq_printf(m, "  NOPID :   %08x\n", I915_READ_NOPID(ring));
+        if (IS_GEN6(dev)) {
+                seq_printf(m, "  Sync 0 :   %08x\n", I915_READ_SYNC_0(ring));
+                seq_printf(m, "  Sync 1 :   %08x\n", I915_READ_SYNC_1(ring));
+        }
+        seq_printf(m, "  Control : %08x\n", I915_READ_CTL(ring));
+        seq_printf(m, "  Start :   %08x\n", I915_READ_START(ring));
 
         return 0;
 }
 
+static const char *ring_str(int ring)
+{
+        switch (ring) {
+        case RING_RENDER: return " render";
+        case RING_BSD: return " bsd";
+        case RING_BLT: return " blt";
+        default: return "";
+        }
+}
+
 static const char *pin_flag(int pinned)
 {
         if (pinned > 0)
@@ -547,6 +695,37 @@ static const char *purgeable_flag(int purgeable)
         return purgeable ? " purgeable" : "";
 }
 
+static void print_error_buffers(struct seq_file *m,
+                                const char *name,
+                                struct drm_i915_error_buffer *err,
+                                int count)
+{
+        seq_printf(m, "%s [%d]:\n", name, count);
+
+        while (count--) {
+                seq_printf(m, "  %08x %8u %04x %04x %08x%s%s%s%s%s%s",
+                           err->gtt_offset,
+                           err->size,
+                           err->read_domains,
+                           err->write_domain,
+                           err->seqno,
+                           pin_flag(err->pinned),
+                           tiling_flag(err->tiling),
+                           dirty_flag(err->dirty),
+                           purgeable_flag(err->purgeable),
+                           ring_str(err->ring),
+                           agp_type_str(err->agp_type));
+
+                if (err->name)
+                        seq_printf(m, " (name: %d)", err->name);
+                if (err->fence_reg != I915_FENCE_REG_NONE)
+                        seq_printf(m, " (fence: %d)", err->fence_reg);
+
+                seq_printf(m, "\n");
+                err++;
+        }
+}
+
 static int i915_error_state(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -568,47 +747,54 @@ static int i915_error_state(struct seq_file *m, void *unused)
                    error->time.tv_usec);
         seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
         seq_printf(m, "EIR: 0x%08x\n", error->eir);
-        seq_printf(m, "  PGTBL_ER: 0x%08x\n", error->pgtbl_er);
-        seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm);
+        seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
+        if (INTEL_INFO(dev)->gen >= 6) {
+                seq_printf(m, "ERROR: 0x%08x\n", error->error);
+                seq_printf(m, "Blitter command stream:\n");
+                seq_printf(m, "  ACTHD:    0x%08x\n", error->bcs_acthd);
+                seq_printf(m, "  IPEIR:    0x%08x\n", error->bcs_ipeir);
+                seq_printf(m, "  IPEHR:    0x%08x\n", error->bcs_ipehr);
+                seq_printf(m, "  INSTDONE: 0x%08x\n", error->bcs_instdone);
+                seq_printf(m, "  seqno:    0x%08x\n", error->bcs_seqno);
+                seq_printf(m, "Video (BSD) command stream:\n");
+                seq_printf(m, "  ACTHD:    0x%08x\n", error->vcs_acthd);
+                seq_printf(m, "  IPEIR:    0x%08x\n", error->vcs_ipeir);
+                seq_printf(m, "  IPEHR:    0x%08x\n", error->vcs_ipehr);
+                seq_printf(m, "  INSTDONE: 0x%08x\n", error->vcs_instdone);
+                seq_printf(m, "  seqno:    0x%08x\n", error->vcs_seqno);
+        }
+        seq_printf(m, "Render command stream:\n");
+        seq_printf(m, "  ACTHD: 0x%08x\n", error->acthd);
         seq_printf(m, "  IPEIR: 0x%08x\n", error->ipeir);
         seq_printf(m, "  IPEHR: 0x%08x\n", error->ipehr);
         seq_printf(m, "  INSTDONE: 0x%08x\n", error->instdone);
-        seq_printf(m, "  ACTHD: 0x%08x\n", error->acthd);
         if (INTEL_INFO(dev)->gen >= 4) {
-                seq_printf(m, "  INSTPS: 0x%08x\n", error->instps);
                 seq_printf(m, "  INSTDONE1: 0x%08x\n", error->instdone1);
+                seq_printf(m, "  INSTPS: 0x%08x\n", error->instps);
         }
-        seq_printf(m, "seqno: 0x%08x\n", error->seqno);
-
-        if (error->active_bo_count) {
-                seq_printf(m, "Buffers [%d]:\n", error->active_bo_count);
-
-                for (i = 0; i < error->active_bo_count; i++) {
-                        seq_printf(m, "  %08x %8zd %08x %08x %08x%s%s%s%s",
-                                   error->active_bo[i].gtt_offset,
-                                   error->active_bo[i].size,
-                                   error->active_bo[i].read_domains,
-                                   error->active_bo[i].write_domain,
-                                   error->active_bo[i].seqno,
-                                   pin_flag(error->active_bo[i].pinned),
-                                   tiling_flag(error->active_bo[i].tiling),
-                                   dirty_flag(error->active_bo[i].dirty),
-                                   purgeable_flag(error->active_bo[i].purgeable));
-
-                        if (error->active_bo[i].name)
-                                seq_printf(m, " (name: %d)", error->active_bo[i].name);
-                        if (error->active_bo[i].fence_reg != I915_FENCE_REG_NONE)
-                                seq_printf(m, " (fence: %d)", error->active_bo[i].fence_reg);
-
-                        seq_printf(m, "\n");
-                }
-        }
+        seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm);
+        seq_printf(m, "  seqno: 0x%08x\n", error->seqno);
+
+        for (i = 0; i < 16; i++)
+                seq_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
+
+        if (error->active_bo)
+                print_error_buffers(m, "Active",
+                                    error->active_bo,
+                                    error->active_bo_count);
+
+        if (error->pinned_bo)
+                print_error_buffers(m, "Pinned",
+                                    error->pinned_bo,
+                                    error->pinned_bo_count);
 
         for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) {
                 if (error->batchbuffer[i]) {
                         struct drm_i915_error_object *obj = error->batchbuffer[i];
 
-                        seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
+                        seq_printf(m, "%s --- gtt_offset = 0x%08x\n",
+                                   dev_priv->ring[i].name,
+                                   obj->gtt_offset);
                         offset = 0;
                         for (page = 0; page < obj->page_count; page++) {
                                 for (elt = 0; elt < PAGE_SIZE/4; elt++) {
@@ -635,6 +821,9 @@ static int i915_error_state(struct seq_file *m, void *unused)
         if (error->overlay)
                 intel_overlay_print_error_state(m, error->overlay);
 
+        if (error->display)
+                intel_display_print_error_state(m, dev, error->display);
+
 out:
         spin_unlock_irqrestore(&dev_priv->error_lock, flags);
 
@@ -658,15 +847,51 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        u16 rgvswctl = I915_READ16(MEMSWCTL);
-        u16 rgvstat = I915_READ16(MEMSTAT_ILK);
 
-        seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf);
-        seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f);
-        seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >>
-                   MEMSTAT_VID_SHIFT);
-        seq_printf(m, "Current P-state: %d\n",
-                   (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
+        if (IS_GEN5(dev)) {
+                u16 rgvswctl = I915_READ16(MEMSWCTL);
+                u16 rgvstat = I915_READ16(MEMSTAT_ILK);
+
+                seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf);
+                seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f);
+                seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >>
+                           MEMSTAT_VID_SHIFT);
+                seq_printf(m, "Current P-state: %d\n",
+                           (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
+        } else if (IS_GEN6(dev)) {
+                u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
+                u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
+                u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+                int max_freq;
+
+                /* RPSTAT1 is in the GT power well */
+                __gen6_force_wake_get(dev_priv);
+
+                seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
+                seq_printf(m, "RPSTAT1: 0x%08x\n", I915_READ(GEN6_RPSTAT1));
+                seq_printf(m, "Render p-state ratio: %d\n",
+                           (gt_perf_status & 0xff00) >> 8);
+                seq_printf(m, "Render p-state VID: %d\n",
+                           gt_perf_status & 0xff);
+                seq_printf(m, "Render p-state limit: %d\n",
+                           rp_state_limits & 0xff);
+
+                max_freq = (rp_state_cap & 0xff0000) >> 16;
+                seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
+                           max_freq * 100);
+
+                max_freq = (rp_state_cap & 0xff00) >> 8;
+                seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
+                           max_freq * 100);
+
+                max_freq = rp_state_cap & 0xff;
+                seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
+                           max_freq * 100);
+
+                __gen6_force_wake_put(dev_priv);
+        } else {
+                seq_printf(m, "no P-state info available\n");
+        }
 
         return 0;
 }
@@ -715,7 +940,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
         struct drm_device *dev = node->minor->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         u32 rgvmodectl = I915_READ(MEMMODECTL);
-        u32 rstdbyctl = I915_READ(MCHBAR_RENDER_STANDBY);
+        u32 rstdbyctl = I915_READ(RSTDBYCTL);
         u16 crstandvid = I915_READ16(CRSTANDVID);
 
         seq_printf(m, "HD boost: %s\n", (rgvmodectl & MEMMODE_BOOST_EN) ?
@@ -738,6 +963,30 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
         seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f));
         seq_printf(m, "Render standby enabled: %s\n",
                    (rstdbyctl & RCX_SW_EXIT) ? "no" : "yes");
+        seq_printf(m, "Current RS state: ");
+        switch (rstdbyctl & RSX_STATUS_MASK) {
+        case RSX_STATUS_ON:
+                seq_printf(m, "on\n");
+                break;
+        case RSX_STATUS_RC1:
+                seq_printf(m, "RC1\n");
+                break;
+        case RSX_STATUS_RC1E:
+                seq_printf(m, "RC1E\n");
+                break;
+        case RSX_STATUS_RS1:
+                seq_printf(m, "RS1\n");
+                break;
+        case RSX_STATUS_RS2:
+                seq_printf(m, "RS2 (RC6)\n");
+                break;
+        case RSX_STATUS_RS3:
+                seq_printf(m, "RC3 (RC6+)\n");
+                break;
+        default:
+                seq_printf(m, "unknown\n");
+                break;
+        }
 
         return 0;
 }
@@ -794,7 +1043,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
         drm_i915_private_t *dev_priv = dev->dev_private;
         bool sr_enabled = false;
 
-        if (IS_GEN5(dev))
+        if (HAS_PCH_SPLIT(dev))
                 sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
         else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev))
                 sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
@@ -886,7 +1135,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
                    fb->base.height,
                    fb->base.depth,
                    fb->base.bits_per_pixel);
-        describe_obj(m, to_intel_bo(fb->obj));
+        describe_obj(m, fb->obj);
         seq_printf(m, "\n");
 
         list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) {
@@ -898,7 +1147,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
                            fb->base.height,
                            fb->base.depth,
                            fb->base.bits_per_pixel);
-                describe_obj(m, to_intel_bo(fb->obj));
+                describe_obj(m, fb->obj);
                 seq_printf(m, "\n");
         }
 
@@ -943,7 +1192,6 @@ i915_wedged_write(struct file *filp,
                   loff_t *ppos)
 {
         struct drm_device *dev = filp->private_data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
         char buf[20];
         int val = 1;
 
@@ -959,12 +1207,7 @@ i915_wedged_write(struct file *filp,
         }
 
         DRM_INFO("Manually setting wedged to %d\n", val);
-
-        atomic_set(&dev_priv->mm.wedged, val);
-        if (val) {
-                wake_up_all(&dev_priv->irq_queue);
-                queue_work(dev_priv->wq, &dev_priv->error_work);
-        }
+        i915_handle_error(dev, val);
 
         return cnt;
 }
@@ -1018,6 +1261,7 @@ static int i915_wedged_create(struct dentry *root, struct drm_minor *minor)
 static struct drm_info_list i915_debugfs_list[] = {
         {"i915_capabilities", i915_capabilities, 0, 0},
         {"i915_gem_objects", i915_gem_object_info, 0},
+        {"i915_gem_gtt", i915_gem_gtt_info, 0},
         {"i915_gem_active", i915_gem_object_list_info, 0, (void *) ACTIVE_LIST},
         {"i915_gem_flushing", i915_gem_object_list_info, 0, (void *) FLUSHING_LIST},
         {"i915_gem_inactive", i915_gem_object_list_info, 0, (void *) INACTIVE_LIST},
@@ -1028,9 +1272,15 @@ static struct drm_info_list i915_debugfs_list[] = {
         {"i915_gem_seqno", i915_gem_seqno_info, 0},
         {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
         {"i915_gem_interrupt", i915_interrupt_info, 0},
-        {"i915_gem_hws", i915_hws_info, 0},
-        {"i915_ringbuffer_data", i915_ringbuffer_data, 0},
-        {"i915_ringbuffer_info", i915_ringbuffer_info, 0},
+        {"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
+        {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
+        {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
+        {"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS},
+        {"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS},
+        {"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS},
+        {"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS},
+        {"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS},
+        {"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS},
         {"i915_batchbuffers", i915_batchbuffer_info, 0},
         {"i915_error_state", i915_error_state, 0},
         {"i915_rstdby_delays", i915_rstdby_delays, 0},
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 7a26f4dd21ae..17bd766f2081 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -34,6 +34,7 @@
 #include "i915_drm.h"
 #include "i915_drv.h"
 #include "i915_trace.h"
+#include "../../../platform/x86/intel_ips.h"
 #include <linux/pci.h>
 #include <linux/vgaarb.h>
 #include <linux/acpi.h>
@@ -49,6 +50,8 @@
 static int i915_init_phys_hws(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
+
         /* Program Hardware Status Page */
         dev_priv->status_page_dmah =
                 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE);
@@ -57,11 +60,10 @@ static int i915_init_phys_hws(struct drm_device *dev)
                 DRM_ERROR("Can not allocate hardware status page\n");
                 return -ENOMEM;
         }
-        dev_priv->render_ring.status_page.page_addr
-                = dev_priv->status_page_dmah->vaddr;
+        ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
         dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
 
-        memset(dev_priv->render_ring.status_page.page_addr, 0, PAGE_SIZE);
+        memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 
         if (INTEL_INFO(dev)->gen >= 4)
                 dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) &
@@ -79,13 +81,15 @@ static int i915_init_phys_hws(struct drm_device *dev)
 static void i915_free_hws(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
+
         if (dev_priv->status_page_dmah) {
                 drm_pci_free(dev, dev_priv->status_page_dmah);
                 dev_priv->status_page_dmah = NULL;
         }
 
-        if (dev_priv->render_ring.status_page.gfx_addr) {
-                dev_priv->render_ring.status_page.gfx_addr = 0;
+        if (ring->status_page.gfx_addr) {
+                ring->status_page.gfx_addr = 0;
                 drm_core_ioremapfree(&dev_priv->hws_map, dev);
         }
 
@@ -97,7 +101,7 @@ void i915_kernel_lost_context(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv;
-        struct intel_ring_buffer *ring = &dev_priv->render_ring;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         /*
          * We should never lose context on the ring with modesetting
@@ -106,8 +110,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
-        ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-        ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
+        ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
+        ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
         ring->space = ring->head - (ring->tail + 8);
         if (ring->space < 0)
                 ring->space += ring->size;
@@ -123,6 +127,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
 static int i915_dma_cleanup(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int i;
+
         /* Make sure interrupts are disabled here because the uninstall ioctl
          * may not have been called from userspace and after dev_private
          * is freed, it's too late.
@@ -131,9 +137,8 @@ static int i915_dma_cleanup(struct drm_device * dev)
                 drm_irq_uninstall(dev);
 
         mutex_lock(&dev->struct_mutex);
-        intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
-        intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
-        intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                intel_cleanup_ring_buffer(&dev_priv->ring[i]);
         mutex_unlock(&dev->struct_mutex);
 
         /* Clear the HWS virtual address at teardown */
@@ -147,6 +152,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
+        int ret;
 
         master_priv->sarea = drm_getsarea(dev);
         if (master_priv->sarea) {
@@ -157,33 +163,22 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
         }
 
         if (init->ring_size != 0) {
-                if (dev_priv->render_ring.gem_object != NULL) {
+                if (LP_RING(dev_priv)->obj != NULL) {
                         i915_dma_cleanup(dev);
                         DRM_ERROR("Client tried to initialize ringbuffer in "
                                   "GEM mode\n");
                         return -EINVAL;
                 }
 
-                dev_priv->render_ring.size = init->ring_size;
-
-                dev_priv->render_ring.map.offset = init->ring_start;
-                dev_priv->render_ring.map.size = init->ring_size;
-                dev_priv->render_ring.map.type = 0;
-                dev_priv->render_ring.map.flags = 0;
-                dev_priv->render_ring.map.mtrr = 0;
-
-                drm_core_ioremap_wc(&dev_priv->render_ring.map, dev);
-
-                if (dev_priv->render_ring.map.handle == NULL) {
+                ret = intel_render_ring_init_dri(dev,
+                                                 init->ring_start,
+                                                 init->ring_size);
+                if (ret) {
                         i915_dma_cleanup(dev);
-                        DRM_ERROR("can not ioremap virtual address for"
-                                  " ring buffer\n");
-                        return -ENOMEM;
+                        return ret;
                 }
         }
 
-        dev_priv->render_ring.virtual_start = dev_priv->render_ring.map.handle;
-
         dev_priv->cpp = init->cpp;
         dev_priv->back_offset = init->back_offset;
         dev_priv->front_offset = init->front_offset;
@@ -201,12 +196,10 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 static int i915_dma_resume(struct drm_device * dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
-        struct intel_ring_buffer *ring;
         DRM_DEBUG_DRIVER("%s\n", __func__);
 
-        ring = &dev_priv->render_ring;
-
         if (ring->map.handle == NULL) {
                 DRM_ERROR("can not ioremap virtual address for"
                           " ring buffer\n");
@@ -221,7 +214,7 @@ static int i915_dma_resume(struct drm_device * dev)
         DRM_DEBUG_DRIVER("hw status page @ %p\n",
                                 ring->status_page.page_addr);
         if (ring->status_page.gfx_addr != 0)
-                intel_ring_setup_status_page(dev, ring);
+                intel_ring_setup_status_page(ring);
         else
                 I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
 
@@ -263,7 +256,7 @@ static int i915_dma_init(struct drm_device *dev, void *data,
  * instruction detected will be given a size of zero, which is a
  * signal to abort the rest of the buffer.
  */
-static int do_validate_cmd(int cmd)
+static int validate_cmd(int cmd)
 {
         switch (((cmd >> 29) & 0x7)) {
         case 0x0:
@@ -321,40 +314,27 @@ static int do_validate_cmd(int cmd)
         return 0;
 }
 
-static int validate_cmd(int cmd)
-{
-        int ret = do_validate_cmd(cmd);
-
-/*      printk("validate_cmd( %x ): %d\n", cmd, ret); */
-
-        return ret;
-}
-
 static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int i;
+        int i, ret;
 
-        if ((dwords+1) * sizeof(int) >= dev_priv->render_ring.size - 8)
+        if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8)
                 return -EINVAL;
 
-        BEGIN_LP_RING((dwords+1)&~1);
-
         for (i = 0; i < dwords;) {
-                int cmd, sz;
-
-                cmd = buffer[i];
-
-                if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
+                int sz = validate_cmd(buffer[i]);
+                if (sz == 0 || i + sz > dwords)
                         return -EINVAL;
-
-                OUT_RING(cmd);
-
-                while (++i, --sz) {
-                        OUT_RING(buffer[i]);
-                }
+                i += sz;
         }
 
+        ret = BEGIN_LP_RING((dwords+1)&~1);
+        if (ret)
+                return ret;
+
+        for (i = 0; i < dwords; i++)
+                OUT_RING(buffer[i]);
         if (dwords & 1)
                 OUT_RING(0);
 
@@ -365,34 +345,41 @@ static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
 
 int
 i915_emit_box(struct drm_device *dev,
-              struct drm_clip_rect *boxes,
-              int i, int DR1, int DR4)
+              struct drm_clip_rect *box,
+              int DR1, int DR4)
 {
-        struct drm_clip_rect box = boxes[i];
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
 
-        if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) {
+        if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
+            box->y2 <= 0 || box->x2 <= 0) {
                 DRM_ERROR("Bad box %d,%d..%d,%d\n",
-                          box.x1, box.y1, box.x2, box.y2);
+                          box->x1, box->y1, box->x2, box->y2);
                 return -EINVAL;
         }
 
         if (INTEL_INFO(dev)->gen >= 4) {
-                BEGIN_LP_RING(4);
+                ret = BEGIN_LP_RING(4);
+                if (ret)
+                        return ret;
+
                 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
-                OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
-                OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
+                OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
+                OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
                 OUT_RING(DR4);
-                ADVANCE_LP_RING();
         } else {
-                BEGIN_LP_RING(6);
+                ret = BEGIN_LP_RING(6);
+                if (ret)
+                        return ret;
+
                 OUT_RING(GFX_OP_DRAWRECT_INFO);
                 OUT_RING(DR1);
-                OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
-                OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
+                OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
+                OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
                 OUT_RING(DR4);
                 OUT_RING(0);
-                ADVANCE_LP_RING();
         }
+        ADVANCE_LP_RING();
 
         return 0;
 }
@@ -412,12 +399,13 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
         if (master_priv->sarea_priv)
                 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_STORE_DWORD_INDEX);
-        OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        OUT_RING(dev_priv->counter);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
+        if (BEGIN_LP_RING(4) == 0) {
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(dev_priv->counter);
+                OUT_RING(0);
+                ADVANCE_LP_RING();
+        }
 }
 
 static int i915_dispatch_cmdbuffer(struct drm_device * dev,
@@ -439,7 +427,7 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev,
 
         for (i = 0; i < count; i++) {
                 if (i < nbox) {
-                        ret = i915_emit_box(dev, cliprects, i,
+                        ret = i915_emit_box(dev, &cliprects[i],
                                             cmd->DR1, cmd->DR4);
                         if (ret)
                                 return ret;
@@ -458,8 +446,9 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
                                      drm_i915_batchbuffer_t * batch,
                                      struct drm_clip_rect *cliprects)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int nbox = batch->num_cliprects;
-        int i = 0, count;
+        int i, count, ret;
 
         if ((batch->start | batch->used) & 0x7) {
                 DRM_ERROR("alignment");
@@ -469,17 +458,19 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
         i915_kernel_lost_context(dev);
 
         count = nbox ? nbox : 1;
-
         for (i = 0; i < count; i++) {
                 if (i < nbox) {
-                        int ret = i915_emit_box(dev, cliprects, i,
-                                                batch->DR1, batch->DR4);
+                        ret = i915_emit_box(dev, &cliprects[i],
+                                            batch->DR1, batch->DR4);
                         if (ret)
                                 return ret;
                 }
 
                 if (!IS_I830(dev) && !IS_845G(dev)) {
-                        BEGIN_LP_RING(2);
+                        ret = BEGIN_LP_RING(2);
+                        if (ret)
+                                return ret;
+
                         if (INTEL_INFO(dev)->gen >= 4) {
                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
                                 OUT_RING(batch->start);
@@ -487,26 +478,29 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
                                 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
                         }
-                        ADVANCE_LP_RING();
                 } else {
-                        BEGIN_LP_RING(4);
+                        ret = BEGIN_LP_RING(4);
+                        if (ret)
+                                return ret;
+
                         OUT_RING(MI_BATCH_BUFFER);
                         OUT_RING(batch->start | MI_BATCH_NON_SECURE);
                         OUT_RING(batch->start + batch->used - 4);
                         OUT_RING(0);
-                        ADVANCE_LP_RING();
                 }
+                ADVANCE_LP_RING();
         }
 
 
         if (IS_G4X(dev) || IS_GEN5(dev)) {
-                BEGIN_LP_RING(2);
-                OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
-                OUT_RING(MI_NOOP);
-                ADVANCE_LP_RING();
+                if (BEGIN_LP_RING(2) == 0) {
+                        OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
+                        OUT_RING(MI_NOOP);
+                        ADVANCE_LP_RING();
+                }
         }
-        i915_emit_breadcrumb(dev);
 
+        i915_emit_breadcrumb(dev);
         return 0;
 }
 
@@ -515,6 +509,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv =
                 dev->primary->master->driver_priv;
+        int ret;
 
         if (!master_priv->sarea_priv)
                 return -EINVAL;
@@ -526,12 +521,13 @@ static int i915_dispatch_flip(struct drm_device * dev)
 
         i915_kernel_lost_context(dev);
 
-        BEGIN_LP_RING(2);
+        ret = BEGIN_LP_RING(10);
+        if (ret)
+                return ret;
+
         OUT_RING(MI_FLUSH | MI_READ_FLUSH);
         OUT_RING(0);
-        ADVANCE_LP_RING();
 
-        BEGIN_LP_RING(6);
         OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
         OUT_RING(0);
         if (dev_priv->current_page == 0) {
@@ -542,33 +538,32 @@ static int i915_dispatch_flip(struct drm_device * dev)
                 dev_priv->current_page = 0;
         }
         OUT_RING(0);
-        ADVANCE_LP_RING();
 
-        BEGIN_LP_RING(2);
         OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
         OUT_RING(0);
+
         ADVANCE_LP_RING();
 
         master_priv->sarea_priv->last_enqueue = dev_priv->counter++;
 
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_STORE_DWORD_INDEX);
-        OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        OUT_RING(dev_priv->counter);
-        OUT_RING(0);
-        ADVANCE_LP_RING();
+        if (BEGIN_LP_RING(4) == 0) {
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(dev_priv->counter);
+                OUT_RING(0);
+                ADVANCE_LP_RING();
+        }
 
         master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
         return 0;
 }
 
-static int i915_quiescent(struct drm_device * dev)
+static int i915_quiescent(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = LP_RING(dev->dev_private);
 
         i915_kernel_lost_context(dev);
-        return intel_wait_ring_buffer(dev, &dev_priv->render_ring,
-                                      dev_priv->render_ring.size - 8);
+        return intel_wait_ring_buffer(ring, ring->size - 8);
 }
 
 static int i915_flush_ioctl(struct drm_device *dev, void *data,
@@ -767,6 +762,15 @@ static int i915_getparam(struct drm_device *dev, void *data,
         case I915_PARAM_HAS_BLT:
                 value = HAS_BLT(dev);
                 break;
+        case I915_PARAM_HAS_RELAXED_FENCING:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_COHERENT_RINGS:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_EXEC_CONSTANTS:
+                value = INTEL_INFO(dev)->gen >= 4;
+                break;
         default:
                 DRM_DEBUG_DRIVER("Unknown parameter %d\n",
                                  param->param);
@@ -822,7 +826,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         drm_i915_hws_addr_t *hws = data;
-        struct intel_ring_buffer *ring = &dev_priv->render_ring;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         if (!I915_NEED_GFX_HWS(dev))
                 return -EINVAL;
@@ -1001,73 +1005,47 @@ intel_teardown_mchbar(struct drm_device *dev)
 #define PTE_VALID                       (1 << 0)
 
 /**
- * i915_gtt_to_phys - take a GTT address and turn it into a physical one
+ * i915_stolen_to_phys - take an offset into stolen memory and turn it into
+ *                       a physical one
  * @dev: drm device
- * @gtt_addr: address to translate
+ * @offset: address to translate
  *
- * Some chip functions require allocations from stolen space but need the
- * physical address of the memory in question.  We use this routine
- * to get a physical address suitable for register programming from a given
- * GTT address.
+ * Some chip functions require allocations from stolen space and need the
+ * physical address of the memory in question.
  */
-static unsigned long i915_gtt_to_phys(struct drm_device *dev,
-                                      unsigned long gtt_addr)
+static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset)
 {
-        unsigned long *gtt;
-        unsigned long entry, phys;
-        int gtt_bar = IS_GEN2(dev) ? 1 : 0;
-        int gtt_offset, gtt_size;
-
-        if (INTEL_INFO(dev)->gen >= 4) {
-                if (IS_G4X(dev) || INTEL_INFO(dev)->gen > 4) {
-                        gtt_offset = 2*1024*1024;
-                        gtt_size = 2*1024*1024;
-                } else {
-                        gtt_offset = 512*1024;
-                        gtt_size = 512*1024;
-                }
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct pci_dev *pdev = dev_priv->bridge_dev;
+        u32 base;
+
+#if 0
+        /* On the machines I have tested the Graphics Base of Stolen Memory
+         * is unreliable, so compute the base by subtracting the stolen memory
+         * from the Top of Low Usable DRAM which is where the BIOS places
+         * the graphics stolen memory.
+         */
+        if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+                /* top 32bits are reserved = 0 */
+                pci_read_config_dword(pdev, 0xA4, &base);
         } else {
-                gtt_bar = 3;
-                gtt_offset = 0;
-                gtt_size = pci_resource_len(dev->pdev, gtt_bar);
-        }
-
-        gtt = ioremap_wc(pci_resource_start(dev->pdev, gtt_bar) + gtt_offset,
-                         gtt_size);
-        if (!gtt) {
-                DRM_ERROR("ioremap of GTT failed\n");
-                return 0;
-        }
-
-        entry = *(volatile u32 *)(gtt + (gtt_addr / 1024));
-
-        DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, PTE: 0x%08lx\n", gtt_addr, entry);
-
-        /* Mask out these reserved bits on this hardware. */
-        if (INTEL_INFO(dev)->gen < 4 && !IS_G33(dev))
-                entry &= ~PTE_ADDRESS_MASK_HIGH;
-
-        /* If it's not a mapping type we know, then bail. */
-        if ((entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_UNCACHED &&
-            (entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_CACHED) {
-                iounmap(gtt);
-                return 0;
-        }
-
-        if (!(entry & PTE_VALID)) {
-                DRM_ERROR("bad GTT entry in stolen space\n");
-                iounmap(gtt);
-                return 0;
+                /* XXX presume 8xx is the same as i915 */
+                pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base);
+        }
+#else
+        if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+                u16 val;
+                pci_read_config_word(pdev, 0xb0, &val);
+                base = val >> 4 << 20;
+        } else {
+                u8 val;
+                pci_read_config_byte(pdev, 0x9c, &val);
+                base = val >> 3 << 27;
         }
+        base -= dev_priv->mm.gtt->stolen_size;
+#endif
 
-        iounmap(gtt);
-
-        phys =(entry & PTE_ADDRESS_MASK) |
-                ((uint64_t)(entry & PTE_ADDRESS_MASK_HIGH) << (32 - 4));
-
-        DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, phys addr: 0x%08lx\n", gtt_addr, phys);
-
-        return phys;
+        return base + offset;
 }
 
 static void i915_warn_stolen(struct drm_device *dev)
@@ -1083,54 +1061,35 @@ static void i915_setup_compression(struct drm_device *dev, int size)
         unsigned long cfb_base;
         unsigned long ll_base = 0;
 
-        /* Leave 1M for line length buffer & misc. */
-        compressed_fb = drm_mm_search_free(&dev_priv->mm.vram, size, 4096, 0);
-        if (!compressed_fb) {
-                dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
-                i915_warn_stolen(dev);
-                return;
-        }
-
-        compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
-        if (!compressed_fb) {
-                i915_warn_stolen(dev);
-                dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
-                return;
-        }
-
-        cfb_base = i915_gtt_to_phys(dev, compressed_fb->start);
-        if (!cfb_base) {
-                DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
-                drm_mm_put_block(compressed_fb);
-        }
+        compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
+        if (compressed_fb)
+                compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
+        if (!compressed_fb)
+                goto err;
 
-        if (!(IS_GM45(dev) || IS_IRONLAKE_M(dev))) {
-                compressed_llb = drm_mm_search_free(&dev_priv->mm.vram, 4096,
-                                                    4096, 0);
-                if (!compressed_llb) {
-                        i915_warn_stolen(dev);
-                        return;
-                }
+        cfb_base = i915_stolen_to_phys(dev, compressed_fb->start);
+        if (!cfb_base)
+                goto err_fb;
 
-                compressed_llb = drm_mm_get_block(compressed_llb, 4096, 4096);
-                if (!compressed_llb) {
-                        i915_warn_stolen(dev);
-                        return;
-                }
+        if (!(IS_GM45(dev) || HAS_PCH_SPLIT(dev))) {
+                compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
+                                                    4096, 4096, 0);
+                if (compressed_llb)
+                        compressed_llb = drm_mm_get_block(compressed_llb,
+                                                          4096, 4096);
+                if (!compressed_llb)
+                        goto err_fb;
 
-                ll_base = i915_gtt_to_phys(dev, compressed_llb->start);
-                if (!ll_base) {
-                        DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
-                        drm_mm_put_block(compressed_fb);
-                        drm_mm_put_block(compressed_llb);
-                }
+                ll_base = i915_stolen_to_phys(dev, compressed_llb->start);
+                if (!ll_base)
+                        goto err_llb;
         }
 
         dev_priv->cfb_size = size;
 
         intel_disable_fbc(dev);
         dev_priv->compressed_fb = compressed_fb;
-        if (IS_IRONLAKE_M(dev))
+        if (HAS_PCH_SPLIT(dev))
                 I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
         else if (IS_GM45(dev)) {
                 I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
@@ -1140,8 +1099,17 @@ static void i915_setup_compression(struct drm_device *dev, int size)
                 dev_priv->compressed_llb = compressed_llb;
         }
 
-        DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", cfb_base,
-                  ll_base, size >> 20);
+        DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n",
+                      cfb_base, ll_base, size >> 20);
+        return;
+
+err_llb:
+        drm_mm_put_block(compressed_llb);
+err_fb:
+        drm_mm_put_block(compressed_fb);
+err:
+        dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
+        i915_warn_stolen(dev);
 }
 
 static void i915_cleanup_compression(struct drm_device *dev)
@@ -1172,12 +1140,16 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_
         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
         if (state == VGA_SWITCHEROO_ON) {
                 printk(KERN_INFO "i915: switched on\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 /* i915 resume handler doesn't set to D0 */
                 pci_set_power_state(dev->pdev, PCI_D0);
                 i915_resume(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
         } else {
                 printk(KERN_ERR "i915: switched off\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 i915_suspend(dev, pmm);
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
 
@@ -1192,17 +1164,20 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
         return can_switch;
 }
 
-static int i915_load_modeset_init(struct drm_device *dev,
-                                  unsigned long prealloc_size,
-                                  unsigned long agp_size)
+static int i915_load_modeset_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long prealloc_size, gtt_size, mappable_size;
         int ret = 0;
 
-        /* Basic memrange allocator for stolen space (aka mm.vram) */
-        drm_mm_init(&dev_priv->mm.vram, 0, prealloc_size);
+        prealloc_size = dev_priv->mm.gtt->stolen_size;
+        gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
+        mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
+
+        /* Basic memrange allocator for stolen space */
+        drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
 
-        /* Let GEM Manage from end of prealloc space to end of aperture.
+        /* Let GEM Manage all of the aperture.
          *
          * However, leave one page at the end still bound to the scratch page.
          * There are a number of places where the hardware apparently
@@ -1211,7 +1186,7 @@ static int i915_load_modeset_init(struct drm_device *dev,
          * at the last page of the aperture.  One page should be enough to
          * keep any prefetching inside of the aperture.
          */
-        i915_gem_do_init(dev, prealloc_size, agp_size - 4096);
+        i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE);
 
         mutex_lock(&dev->struct_mutex);
         ret = i915_gem_init_ringbuffer(dev);
@@ -1223,31 +1198,39 @@ static int i915_load_modeset_init(struct drm_device *dev,
         if (I915_HAS_FBC(dev) && i915_powersave) {
                 int cfb_size;
 
-                /* Try to get an 8M buffer... */
-                if (prealloc_size > (9*1024*1024))
-                        cfb_size = 8*1024*1024;
+                /* Leave 1M for line length buffer & misc. */
+
+                /* Try to get a 32M buffer... */
+                if (prealloc_size > (36*1024*1024))
+                        cfb_size = 32*1024*1024;
                 else /* fall back to 7/8 of the stolen space */
                         cfb_size = prealloc_size * 7 / 8;
                 i915_setup_compression(dev, cfb_size);
         }
 
-        /* Allow hardware batchbuffers unless told otherwise.
-         */
+        /* Allow hardware batchbuffers unless told otherwise. */
         dev_priv->allow_batchbuffer = 1;
 
         ret = intel_parse_bios(dev);
         if (ret)
                 DRM_INFO("failed to find VBIOS tables\n");
 
-        /* if we have > 1 VGA cards, then disable the radeon VGA resources */
+        /* If we have > 1 VGA cards, then we need to arbitrate access
+         * to the common VGA resources.
+         *
+         * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
+         * then we do not take part in VGA arbitration and the
+         * vga_client_register() fails with -ENODEV.
+         */
         ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
-        if (ret)
+        if (ret && ret != -ENODEV)
                 goto cleanup_ringbuffer;
 
         intel_register_dsm_handler();
 
         ret = vga_switcheroo_register_client(dev->pdev,
                                              i915_switcheroo_set_state,
+                                             NULL,
                                              i915_switcheroo_can_switch);
         if (ret)
                 goto cleanup_vga_client;
@@ -1422,152 +1405,12 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev)
         }
 }
 
-struct v_table {
-        u8 vid;
-        unsigned long vd; /* in .1 mil */
-        unsigned long vm; /* in .1 mil */
-        u8 pvid;
-};
-
-static struct v_table v_table[] = {
-        { 0, 16125, 15000, 0x7f, },
-        { 1, 16000, 14875, 0x7e, },
-        { 2, 15875, 14750, 0x7d, },
-        { 3, 15750, 14625, 0x7c, },
-        { 4, 15625, 14500, 0x7b, },
-        { 5, 15500, 14375, 0x7a, },
-        { 6, 15375, 14250, 0x79, },
-        { 7, 15250, 14125, 0x78, },
-        { 8, 15125, 14000, 0x77, },
-        { 9, 15000, 13875, 0x76, },
-        { 10, 14875, 13750, 0x75, },
-        { 11, 14750, 13625, 0x74, },
-        { 12, 14625, 13500, 0x73, },
-        { 13, 14500, 13375, 0x72, },
-        { 14, 14375, 13250, 0x71, },
-        { 15, 14250, 13125, 0x70, },
-        { 16, 14125, 13000, 0x6f, },
-        { 17, 14000, 12875, 0x6e, },
-        { 18, 13875, 12750, 0x6d, },
-        { 19, 13750, 12625, 0x6c, },
-        { 20, 13625, 12500, 0x6b, },
-        { 21, 13500, 12375, 0x6a, },
-        { 22, 13375, 12250, 0x69, },
-        { 23, 13250, 12125, 0x68, },
-        { 24, 13125, 12000, 0x67, },
-        { 25, 13000, 11875, 0x66, },
-        { 26, 12875, 11750, 0x65, },
-        { 27, 12750, 11625, 0x64, },
-        { 28, 12625, 11500, 0x63, },
-        { 29, 12500, 11375, 0x62, },
-        { 30, 12375, 11250, 0x61, },
-        { 31, 12250, 11125, 0x60, },
-        { 32, 12125, 11000, 0x5f, },
-        { 33, 12000, 10875, 0x5e, },
-        { 34, 11875, 10750, 0x5d, },
-        { 35, 11750, 10625, 0x5c, },
-        { 36, 11625, 10500, 0x5b, },
-        { 37, 11500, 10375, 0x5a, },
-        { 38, 11375, 10250, 0x59, },
-        { 39, 11250, 10125, 0x58, },
-        { 40, 11125, 10000, 0x57, },
-        { 41, 11000, 9875, 0x56, },
-        { 42, 10875, 9750, 0x55, },
-        { 43, 10750, 9625, 0x54, },
-        { 44, 10625, 9500, 0x53, },
-        { 45, 10500, 9375, 0x52, },
-        { 46, 10375, 9250, 0x51, },
-        { 47, 10250, 9125, 0x50, },
-        { 48, 10125, 9000, 0x4f, },
-        { 49, 10000, 8875, 0x4e, },
-        { 50, 9875, 8750, 0x4d, },
-        { 51, 9750, 8625, 0x4c, },
-        { 52, 9625, 8500, 0x4b, },
-        { 53, 9500, 8375, 0x4a, },
-        { 54, 9375, 8250, 0x49, },
-        { 55, 9250, 8125, 0x48, },
-        { 56, 9125, 8000, 0x47, },
-        { 57, 9000, 7875, 0x46, },
-        { 58, 8875, 7750, 0x45, },
-        { 59, 8750, 7625, 0x44, },
-        { 60, 8625, 7500, 0x43, },
-        { 61, 8500, 7375, 0x42, },
-        { 62, 8375, 7250, 0x41, },
-        { 63, 8250, 7125, 0x40, },
-        { 64, 8125, 7000, 0x3f, },
-        { 65, 8000, 6875, 0x3e, },
-        { 66, 7875, 6750, 0x3d, },
-        { 67, 7750, 6625, 0x3c, },
-        { 68, 7625, 6500, 0x3b, },
-        { 69, 7500, 6375, 0x3a, },
-        { 70, 7375, 6250, 0x39, },
-        { 71, 7250, 6125, 0x38, },
-        { 72, 7125, 6000, 0x37, },
-        { 73, 7000, 5875, 0x36, },
-        { 74, 6875, 5750, 0x35, },
-        { 75, 6750, 5625, 0x34, },
-        { 76, 6625, 5500, 0x33, },
-        { 77, 6500, 5375, 0x32, },
-        { 78, 6375, 5250, 0x31, },
-        { 79, 6250, 5125, 0x30, },
-        { 80, 6125, 5000, 0x2f, },
-        { 81, 6000, 4875, 0x2e, },
-        { 82, 5875, 4750, 0x2d, },
-        { 83, 5750, 4625, 0x2c, },
-        { 84, 5625, 4500, 0x2b, },
-        { 85, 5500, 4375, 0x2a, },
-        { 86, 5375, 4250, 0x29, },
-        { 87, 5250, 4125, 0x28, },
-        { 88, 5125, 4000, 0x27, },
-        { 89, 5000, 3875, 0x26, },
-        { 90, 4875, 3750, 0x25, },
-        { 91, 4750, 3625, 0x24, },
-        { 92, 4625, 3500, 0x23, },
-        { 93, 4500, 3375, 0x22, },
-        { 94, 4375, 3250, 0x21, },
-        { 95, 4250, 3125, 0x20, },
-        { 96, 4125, 3000, 0x1f, },
-        { 97, 4125, 3000, 0x1e, },
-        { 98, 4125, 3000, 0x1d, },
-        { 99, 4125, 3000, 0x1c, },
-        { 100, 4125, 3000, 0x1b, },
-        { 101, 4125, 3000, 0x1a, },
-        { 102, 4125, 3000, 0x19, },
-        { 103, 4125, 3000, 0x18, },
-        { 104, 4125, 3000, 0x17, },
-        { 105, 4125, 3000, 0x16, },
-        { 106, 4125, 3000, 0x15, },
-        { 107, 4125, 3000, 0x14, },
-        { 108, 4125, 3000, 0x13, },
-        { 109, 4125, 3000, 0x12, },
-        { 110, 4125, 3000, 0x11, },
-        { 111, 4125, 3000, 0x10, },
-        { 112, 4125, 3000, 0x0f, },
-        { 113, 4125, 3000, 0x0e, },
-        { 114, 4125, 3000, 0x0d, },
-        { 115, 4125, 3000, 0x0c, },
-        { 116, 4125, 3000, 0x0b, },
-        { 117, 4125, 3000, 0x0a, },
-        { 118, 4125, 3000, 0x09, },
-        { 119, 4125, 3000, 0x08, },
-        { 120, 1125, 0, 0x07, },
-        { 121, 1000, 0, 0x06, },
-        { 122, 875, 0, 0x05, },
-        { 123, 750, 0, 0x04, },
-        { 124, 625, 0, 0x03, },
-        { 125, 500, 0, 0x02, },
-        { 126, 375, 0, 0x01, },
-        { 127, 0, 0, 0x00, },
-};
-
-struct cparams {
-        int i;
-        int t;
-        int m;
-        int c;
-};
-
-static struct cparams cparams[] = {
+static const struct cparams {
+        u16 i;
+        u16 t;
+        u16 m;
+        u16 c;
+} cparams[] = {
         { 1, 1333, 301, 28664 },
         { 1, 1066, 294, 24460 },
         { 1, 800, 294, 25192 },
@@ -1633,21 +1476,145 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
         return ((m * x) / 127) - b;
 }
 
-static unsigned long pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
+static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
 {
-        unsigned long val = 0;
-        int i;
-
-        for (i = 0; i < ARRAY_SIZE(v_table); i++) {
-                if (v_table[i].pvid == pxvid) {
-                        if (IS_MOBILE(dev_priv->dev))
-                                val = v_table[i].vm;
-                        else
-                                val = v_table[i].vd;
-                }
-        }
-
-        return val;
+        static const struct v_table {
+                u16 vd; /* in .1 mil */
+                u16 vm; /* in .1 mil */
+        } v_table[] = {
+                { 0, 0, },
+                { 375, 0, },
+                { 500, 0, },
+                { 625, 0, },
+                { 750, 0, },
+                { 875, 0, },
+                { 1000, 0, },
+                { 1125, 0, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4125, 3000, },
+                { 4250, 3125, },
+                { 4375, 3250, },
+                { 4500, 3375, },
+                { 4625, 3500, },
+                { 4750, 3625, },
+                { 4875, 3750, },
+                { 5000, 3875, },
+                { 5125, 4000, },
+                { 5250, 4125, },
+                { 5375, 4250, },
+                { 5500, 4375, },
+                { 5625, 4500, },
+                { 5750, 4625, },
+                { 5875, 4750, },
+                { 6000, 4875, },
+                { 6125, 5000, },
+                { 6250, 5125, },
+                { 6375, 5250, },
+                { 6500, 5375, },
+                { 6625, 5500, },
+                { 6750, 5625, },
+                { 6875, 5750, },
+                { 7000, 5875, },
+                { 7125, 6000, },
+                { 7250, 6125, },
+                { 7375, 6250, },
+                { 7500, 6375, },
+                { 7625, 6500, },
+                { 7750, 6625, },
+                { 7875, 6750, },
+                { 8000, 6875, },
+                { 8125, 7000, },
+                { 8250, 7125, },
+                { 8375, 7250, },
+                { 8500, 7375, },
+                { 8625, 7500, },
+                { 8750, 7625, },
+                { 8875, 7750, },
+                { 9000, 7875, },
+                { 9125, 8000, },
+                { 9250, 8125, },
+                { 9375, 8250, },
+                { 9500, 8375, },
+                { 9625, 8500, },
+                { 9750, 8625, },
+                { 9875, 8750, },
+                { 10000, 8875, },
+                { 10125, 9000, },
+                { 10250, 9125, },
+                { 10375, 9250, },
+                { 10500, 9375, },
+                { 10625, 9500, },
+                { 10750, 9625, },
+                { 10875, 9750, },
+                { 11000, 9875, },
+                { 11125, 10000, },
+                { 11250, 10125, },
+                { 11375, 10250, },
+                { 11500, 10375, },
+                { 11625, 10500, },
+                { 11750, 10625, },
+                { 11875, 10750, },
+                { 12000, 10875, },
+                { 12125, 11000, },
+                { 12250, 11125, },
+                { 12375, 11250, },
+                { 12500, 11375, },
+                { 12625, 11500, },
+                { 12750, 11625, },
+                { 12875, 11750, },
+                { 13000, 11875, },
+                { 13125, 12000, },
+                { 13250, 12125, },
+                { 13375, 12250, },
+                { 13500, 12375, },
+                { 13625, 12500, },
+                { 13750, 12625, },
+                { 13875, 12750, },
+                { 14000, 12875, },
+                { 14125, 13000, },
+                { 14250, 13125, },
+                { 14375, 13250, },
+                { 14500, 13375, },
+                { 14625, 13500, },
+                { 14750, 13625, },
+                { 14875, 13750, },
+                { 15000, 13875, },
+                { 15125, 14000, },
+                { 15250, 14125, },
+                { 15375, 14250, },
+                { 15500, 14375, },
+                { 15625, 14500, },
+                { 15750, 14625, },
+                { 15875, 14750, },
+                { 16000, 14875, },
+                { 16125, 15000, },
+        };
+        if (dev_priv->info->is_mobile)
+                return v_table[pxvid].vm;
+        else
+                return v_table[pxvid].vd;
 }
 
 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
@@ -1868,6 +1835,26 @@ out_unlock:
 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
 
 /**
+ * Tells the intel_ips driver that the i915 driver is now loaded, if
+ * IPS got loaded first.
+ *
+ * This awkward dance is so that neither module has to depend on the
+ * other in order for IPS to do the appropriate communication of
+ * GPU turbo limits to i915.
+ */
+static void
+ips_ping_for_i915_load(void)
+{
+        void (*link)(void);
+
+        link = symbol_get(ips_link_to_i915_driver);
+        if (link) {
+                link();
+                symbol_put(ips_link_to_i915_driver);
+        }
+}
+
+/**
  * i915_driver_load - setup chip and create an initial config
  * @dev: DRM device
  * @flags: startup flags
@@ -1881,9 +1868,9 @@ EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
 int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
         struct drm_i915_private *dev_priv;
-        resource_size_t base, size;
         int ret = 0, mmio_bar;
-        uint32_t agp_size, prealloc_size;
+        uint32_t agp_size;
+
         /* i915 has 4 more counters */
         dev->counters += 4;
         dev->types[6] = _DRM_STAT_IRQ;
@@ -1899,11 +1886,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         dev_priv->dev = dev;
         dev_priv->info = (struct intel_device_info *) flags;
 
-        /* Add register map (needed for suspend/resume) */
-        mmio_bar = IS_GEN2(dev) ? 1 : 0;
-        base = pci_resource_start(dev->pdev, mmio_bar);
-        size = pci_resource_len(dev->pdev, mmio_bar);
-
         if (i915_get_bridge_dev(dev)) {
                 ret = -EIO;
                 goto free_priv;
@@ -1913,16 +1895,25 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         if (IS_GEN2(dev))
                 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
 
-        dev_priv->regs = ioremap(base, size);
+        mmio_bar = IS_GEN2(dev) ? 1 : 0;
+        dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0);
         if (!dev_priv->regs) {
                 DRM_ERROR("failed to map registers\n");
                 ret = -EIO;
                 goto put_bridge;
         }
 
+        dev_priv->mm.gtt = intel_gtt_get();
+        if (!dev_priv->mm.gtt) {
+                DRM_ERROR("Failed to initialize GTT\n");
+                ret = -ENODEV;
+                goto out_iomapfree;
+        }
+
+        agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
+
         dev_priv->mm.gtt_mapping =
-                io_mapping_create_wc(dev->agp->base,
-                                     dev->agp->agp_info.aper_size * 1024*1024);
+                io_mapping_create_wc(dev->agp->base, agp_size);
         if (dev_priv->mm.gtt_mapping == NULL) {
                 ret = -EIO;
                 goto out_rmmap;
@@ -1934,24 +1925,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
          * MTRR if present.  Even if a UC MTRR isn't present.
          */
         dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base,
-                                         dev->agp->agp_info.aper_size *
-                                         1024 * 1024,
+                                         agp_size,
                                          MTRR_TYPE_WRCOMB, 1);
         if (dev_priv->mm.gtt_mtrr < 0) {
                 DRM_INFO("MTRR allocation failed.  Graphics "
                          "performance may suffer.\n");
         }
 
-        dev_priv->mm.gtt = intel_gtt_get();
-        if (!dev_priv->mm.gtt) {
-                DRM_ERROR("Failed to initialize GTT\n");
-                ret = -ENODEV;
-                goto out_iomapfree;
-        }
-
-        prealloc_size = dev_priv->mm.gtt->gtt_stolen_entries << PAGE_SHIFT;
-        agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
-
         /* The i915 workqueue is primarily used for batched retirement of
          * requests (and thus managing bo) once the task has been completed
          * by the GPU. i915_gem_retire_requests() is called directly when we
@@ -1959,7 +1939,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
          * bo.
          *
          * It is also used for periodic low-priority events, such as
-         * idle-timers and hangcheck.
+         * idle-timers and recording error state.
          *
          * All tasks on the workqueue are expected to acquire the dev mutex
          * so there is no point in running more than one instance of the
@@ -1977,22 +1957,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         /* enable GEM by default */
         dev_priv->has_gem = 1;
 
-        if (prealloc_size > agp_size * 3 / 4) {
-                DRM_ERROR("Detected broken video BIOS with %d/%dkB of video "
-                          "memory stolen.\n",
-                          prealloc_size / 1024, agp_size / 1024);
-                DRM_ERROR("Disabling GEM. (try reducing stolen memory or "
-                          "updating the BIOS to fix).\n");
-                dev_priv->has_gem = 0;
-        }
-
-        if (dev_priv->has_gem == 0 &&
-            drm_core_check_feature(dev, DRIVER_MODESET)) {
-                DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n");
-                ret = -ENODEV;
-                goto out_iomapfree;
-        }
-
         dev->driver->get_vblank_counter = i915_get_vblank_counter;
         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
         if (IS_G4X(dev) || IS_GEN5(dev) || IS_GEN6(dev)) {
@@ -2013,8 +1977,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         /* Init HWS */
         if (!I915_NEED_GFX_HWS(dev)) {
                 ret = i915_init_phys_hws(dev);
-                if (ret != 0)
-                        goto out_workqueue_free;
+                if (ret)
+                        goto out_gem_unload;
         }
 
         if (IS_PINEVIEW(dev))
@@ -2036,16 +2000,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         if (!IS_I945G(dev) && !IS_I945GM(dev))
                 pci_enable_msi(dev->pdev);
 
-        spin_lock_init(&dev_priv->user_irq_lock);
+        spin_lock_init(&dev_priv->irq_lock);
         spin_lock_init(&dev_priv->error_lock);
         dev_priv->trace_irq_seqno = 0;
 
         ret = drm_vblank_init(dev, I915_NUM_PIPE);
-
-        if (ret) {
-                (void) i915_driver_unload(dev);
-                return ret;
-        }
+        if (ret)
+                goto out_gem_unload;
 
         /* Start out suspended */
         dev_priv->mm.suspended = 1;
@@ -2053,10 +2014,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         intel_detect_pch(dev);
 
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = i915_load_modeset_init(dev, prealloc_size, agp_size);
+                ret = i915_load_modeset_init(dev);
                 if (ret < 0) {
                         DRM_ERROR("failed to init modeset\n");
-                        goto out_workqueue_free;
+                        goto out_gem_unload;
                 }
         }
 
@@ -2072,14 +2033,21 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         dev_priv->mchdev_lock = &mchdev_lock;
         spin_unlock(&mchdev_lock);
 
+        ips_ping_for_i915_load();
+
         return 0;
 
-out_workqueue_free:
+out_gem_unload:
+        if (dev->pdev->msi_enabled)
+                pci_disable_msi(dev->pdev);
+
+        intel_teardown_gmbus(dev);
+        intel_teardown_mchbar(dev);
         destroy_workqueue(dev_priv->wq);
 out_iomapfree:
         io_mapping_free(dev_priv->mm.gtt_mapping);
 out_rmmap:
-        iounmap(dev_priv->regs);
+        pci_iounmap(dev->pdev, dev_priv->regs);
 put_bridge:
         pci_dev_put(dev_priv->bridge_dev);
 free_priv:
@@ -2096,6 +2064,9 @@ int i915_driver_unload(struct drm_device *dev)
         i915_mch_dev = NULL;
         spin_unlock(&mchdev_lock);
 
+        if (dev_priv->mm.inactive_shrinker.shrink)
+                unregister_shrinker(&dev_priv->mm.inactive_shrinker);
+
         mutex_lock(&dev->struct_mutex);
         ret = i915_gpu_idle(dev);
         if (ret)
@@ -2153,7 +2124,7 @@ int i915_driver_unload(struct drm_device *dev)
                 mutex_unlock(&dev->struct_mutex);
                 if (I915_HAS_FBC(dev) && i915_powersave)
                         i915_cleanup_compression(dev);
-                drm_mm_takedown(&dev_priv->mm.vram);
+                drm_mm_takedown(&dev_priv->mm.stolen);
 
                 intel_cleanup_overlay(dev);
 
@@ -2162,7 +2133,7 @@ int i915_driver_unload(struct drm_device *dev)
         }
 
         if (dev_priv->regs != NULL)
-                iounmap(dev_priv->regs);
+                pci_iounmap(dev->pdev, dev_priv->regs);
 
         intel_teardown_gmbus(dev);
         intel_teardown_mchbar(dev);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index f737960712e6..9ad42d583493 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -49,12 +49,18 @@ module_param_named(powersave, i915_powersave, int, 0600);
 unsigned int i915_lvds_downclock = 0;
 module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400);
 
+unsigned int i915_panel_use_ssc = 1;
+module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600);
+
+bool i915_try_reset = true;
+module_param_named(reset, i915_try_reset, bool, 0600);
+
 static struct drm_driver driver;
 extern int intel_agp_enabled;
 
 #define INTEL_VGA_DEVICE(id, info) {            \
         .class = PCI_CLASS_DISPLAY_VGA << 8,    \
-        .class_mask = 0xffff00,                 \
+        .class_mask = 0xff0000,                 \
         .vendor = 0x8086,                       \
         .device = id,                           \
         .subvendor = PCI_ANY_ID,                \
@@ -111,7 +117,7 @@ static const struct intel_device_info intel_i965g_info = {
 
 static const struct intel_device_info intel_i965gm_info = {
         .gen = 4, .is_crestline = 1,
-        .is_mobile = 1, .has_fbc = 1, .has_rc6 = 1, .has_hotplug = 1,
+        .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
         .has_overlay = 1,
         .supports_tv = 1,
 };
@@ -130,7 +136,7 @@ static const struct intel_device_info intel_g45_info = {
 
 static const struct intel_device_info intel_gm45_info = {
         .gen = 4, .is_g4x = 1,
-        .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_rc6 = 1,
+        .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
         .has_pipe_cxsr = 1, .has_hotplug = 1,
         .supports_tv = 1,
         .has_bsd_ring = 1,
@@ -150,7 +156,7 @@ static const struct intel_device_info intel_ironlake_d_info = {
 
 static const struct intel_device_info intel_ironlake_m_info = {
         .gen = 5, .is_mobile = 1,
-        .need_gfx_hws = 1, .has_rc6 = 1, .has_hotplug = 1,
+        .need_gfx_hws = 1, .has_hotplug = 1,
         .has_fbc = 0, /* disabled due to buggy hardware */
         .has_bsd_ring = 1,
 };
@@ -165,6 +171,7 @@ static const struct intel_device_info intel_sandybridge_d_info = {
 static const struct intel_device_info intel_sandybridge_m_info = {
         .gen = 6, .is_mobile = 1,
         .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 1,
         .has_bsd_ring = 1,
         .has_blt_ring = 1,
 };
@@ -244,10 +251,34 @@ void intel_detect_pch (struct drm_device *dev)
         }
 }
 
+void __gen6_force_wake_get(struct drm_i915_private *dev_priv)
+{
+        int count;
+
+        count = 0;
+        while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1))
+                udelay(10);
+
+        I915_WRITE_NOTRACE(FORCEWAKE, 1);
+        POSTING_READ(FORCEWAKE);
+
+        count = 0;
+        while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0)
+                udelay(10);
+}
+
+void __gen6_force_wake_put(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE_NOTRACE(FORCEWAKE, 0);
+        POSTING_READ(FORCEWAKE);
+}
+
 static int i915_drm_freeze(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        drm_kms_helper_poll_disable(dev);
+
         pci_save_state(dev->pdev);
 
         /* If KMS is active, we do the leavevt stuff here */
@@ -284,7 +315,9 @@ int i915_suspend(struct drm_device *dev, pm_message_t state)
         if (state.event == PM_EVENT_PRETHAW)
                 return 0;
 
-        drm_kms_helper_poll_disable(dev);
+
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
 
         error = i915_drm_freeze(dev);
         if (error)
@@ -304,6 +337,12 @@ static int i915_drm_thaw(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int error = 0;
 
+        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+                mutex_lock(&dev->struct_mutex);
+                i915_gem_restore_gtt_mappings(dev);
+                mutex_unlock(&dev->struct_mutex);
+        }
+
         i915_restore_state(dev);
         intel_opregion_setup(dev);
 
@@ -315,10 +354,14 @@ static int i915_drm_thaw(struct drm_device *dev)
                 error = i915_gem_init_ringbuffer(dev);
                 mutex_unlock(&dev->struct_mutex);
 
+                drm_mode_config_reset(dev);
                 drm_irq_install(dev);
 
                 /* Resume the modeset for every activated CRTC */
                 drm_helper_resume_force_mode(dev);
+
+                if (dev_priv->renderctx && dev_priv->pwrctx)
+                        ironlake_enable_rc6(dev);
         }
 
         intel_opregion_init(dev);
@@ -332,6 +375,9 @@ int i915_resume(struct drm_device *dev)
 {
         int ret;
 
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         if (pci_enable_device(dev->pdev))
                 return -EIO;
 
@@ -405,6 +451,14 @@ static int ironlake_do_reset(struct drm_device *dev, u8 flags)
         return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 }
 
+static int gen6_do_reset(struct drm_device *dev, u8 flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(GEN6_GDRST, GEN6_GRDOM_FULL);
+        return wait_for((I915_READ(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
+}
+
 /**
  * i965_reset - reset chip after a hang
  * @dev: drm device to reset
@@ -431,7 +485,11 @@ int i915_reset(struct drm_device *dev, u8 flags)
         bool need_display = true;
         int ret;
 
-        mutex_lock(&dev->struct_mutex);
+        if (!i915_try_reset)
+                return 0;
+
+        if (!mutex_trylock(&dev->struct_mutex))
+                return -EBUSY;
 
         i915_gem_reset(dev);
 
@@ -439,6 +497,9 @@ int i915_reset(struct drm_device *dev, u8 flags)
         if (get_seconds() - dev_priv->last_gpu_reset < 5) {
                 DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
         } else switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                ret = gen6_do_reset(dev, flags);
+                break;
         case 5:
                 ret = ironlake_do_reset(dev, flags);
                 break;
@@ -472,11 +533,17 @@ int i915_reset(struct drm_device *dev, u8 flags)
          */
         if (drm_core_check_feature(dev, DRIVER_MODESET) ||
                         !dev_priv->mm.suspended) {
-                struct intel_ring_buffer *ring = &dev_priv->render_ring;
                 dev_priv->mm.suspended = 0;
-                ring->init(dev, ring);
+
+                dev_priv->ring[RCS].init(&dev_priv->ring[RCS]);
+                if (HAS_BSD(dev))
+                    dev_priv->ring[VCS].init(&dev_priv->ring[VCS]);
+                if (HAS_BLT(dev))
+                    dev_priv->ring[BCS].init(&dev_priv->ring[BCS]);
+
                 mutex_unlock(&dev->struct_mutex);
                 drm_irq_uninstall(dev);
+                drm_mode_config_reset(dev);
                 drm_irq_install(dev);
                 mutex_lock(&dev->struct_mutex);
         }
@@ -501,6 +568,14 @@ int i915_reset(struct drm_device *dev, u8 flags)
 static int __devinit
 i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
+        /* Only bind to function 0 of the device. Early generations
+         * used function 1 as a placeholder for multi-head. This causes
+         * us confusion instead, especially on the systems where both
+         * functions have the same PCI-ID!
+         */
+        if (PCI_FUNC(pdev->devfn))
+                return -ENODEV;
+
         return drm_get_pci_dev(pdev, ent, &driver);
 }
 
@@ -523,6 +598,9 @@ static int i915_pm_suspend(struct device *dev)
                 return -ENODEV;
         }
 
+        if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         error = i915_drm_freeze(drm_dev);
         if (error)
                 return error;
@@ -606,6 +684,8 @@ static struct drm_driver driver = {
         .device_is_agp = i915_driver_device_is_agp,
         .enable_vblank = i915_enable_vblank,
         .disable_vblank = i915_disable_vblank,
+        .get_vblank_timestamp = i915_get_vblank_timestamp,
+        .get_scanout_position = i915_get_crtc_scanoutpos,
         .irq_preinstall = i915_driver_irq_preinstall,
         .irq_postinstall = i915_driver_irq_postinstall,
         .irq_uninstall = i915_driver_irq_uninstall,
@@ -620,6 +700,9 @@ static struct drm_driver driver = {
         .gem_init_object = i915_gem_init_object,
         .gem_free_object = i915_gem_free_object,
         .gem_vm_ops = &i915_gem_vm_ops,
+        .dumb_create = i915_gem_dumb_create,
+        .dumb_map_offset = i915_gem_mmap_gtt,
+        .dumb_destroy = i915_gem_dumb_destroy,
         .ioctls = i915_ioctls,
         .fops = {
                  .owner = THIS_MODULE,
@@ -636,14 +719,6 @@ static struct drm_driver driver = {
                  .llseek = noop_llseek,
         },
 
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-                 .probe = i915_pci_probe,
-                 .remove = i915_pci_remove,
-                 .driver.pm = &i915_pm_ops,
-        },
-
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
         .date = DRIVER_DATE,
@@ -652,6 +727,14 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver i915_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+        .probe = i915_pci_probe,
+        .remove = i915_pci_remove,
+        .driver.pm = &i915_pm_ops,
+};
+
 static int __init i915_init(void)
 {
         if (!intel_agp_enabled) {
@@ -661,8 +744,6 @@ static int __init i915_init(void)
 
         driver.num_ioctls = i915_max_ioctl;
 
-        i915_gem_shrinker_init();
-
         /*
          * If CONFIG_DRM_I915_KMS is set, default to KMS unless
          * explicitly disabled with the module pararmeter.
@@ -684,18 +765,15 @@ static int __init i915_init(void)
                 driver.driver_features &= ~DRIVER_MODESET;
 #endif
 
-        if (!(driver.driver_features & DRIVER_MODESET)) {
-                driver.suspend = i915_suspend;
-                driver.resume = i915_resume;
-        }
+        if (!(driver.driver_features & DRIVER_MODESET))
+                driver.get_vblank_timestamp = NULL;
 
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &i915_pci_driver);
 }
 
 static void __exit i915_exit(void)
 {
-        i915_gem_shrinker_exit();
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &i915_pci_driver);
 }
 
 module_init(i915_init);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 409826da3099..a78197d43ce6 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -89,7 +89,7 @@ struct drm_i915_gem_phys_object {
         int id;
         struct page **page_list;
         drm_dma_handle_t *handle;
-        struct drm_gem_object *cur_obj;
+        struct drm_i915_gem_object *cur_obj;
 };
 
 struct mem_block {
@@ -124,9 +124,9 @@ struct drm_i915_master_private {
 #define I915_FENCE_REG_NONE -1
 
 struct drm_i915_fence_reg {
-        struct drm_gem_object *obj;
         struct list_head lru_list;
-        bool gpu;
+        struct drm_i915_gem_object *obj;
+        uint32_t setup_seqno;
 };
 
 struct sdvo_device_mapping {
@@ -139,6 +139,8 @@ struct sdvo_device_mapping {
         u8 ddc_pin;
 };
 
+struct intel_display_error_state;
+
 struct drm_i915_error_state {
         u32 eir;
         u32 pgtbl_er;
@@ -148,32 +150,47 @@ struct drm_i915_error_state {
         u32 ipehr;
         u32 instdone;
         u32 acthd;
+        u32 error; /* gen6+ */
+        u32 bcs_acthd; /* gen6+ blt engine */
+        u32 bcs_ipehr;
+        u32 bcs_ipeir;
+        u32 bcs_instdone;
+        u32 bcs_seqno;
+        u32 vcs_acthd; /* gen6+ bsd engine */
+        u32 vcs_ipehr;
+        u32 vcs_ipeir;
+        u32 vcs_instdone;
+        u32 vcs_seqno;
         u32 instpm;
         u32 instps;
         u32 instdone1;
         u32 seqno;
         u64 bbaddr;
+        u64 fence[16];
         struct timeval time;
         struct drm_i915_error_object {
                 int page_count;
                 u32 gtt_offset;
                 u32 *pages[0];
-        } *ringbuffer, *batchbuffer[2];
+        } *ringbuffer, *batchbuffer[I915_NUM_RINGS];
         struct drm_i915_error_buffer {
-                size_t size;
+                u32 size;
                 u32 name;
                 u32 seqno;
                 u32 gtt_offset;
                 u32 read_domains;
                 u32 write_domain;
-                u32 fence_reg;
+                s32 fence_reg:5;
                 s32 pinned:2;
                 u32 tiling:2;
                 u32 dirty:1;
                 u32 purgeable:1;
-        } *active_bo;
-        u32 active_bo_count;
+                u32 ring:4;
+                u32 agp_type:1;
+        } *active_bo, *pinned_bo;
+        u32 active_bo_count, pinned_bo_count;
         struct intel_overlay_error_state *overlay;
+        struct intel_display_error_state *display;
 };
 
 struct drm_i915_display_funcs {
@@ -207,7 +224,6 @@ struct intel_device_info {
         u8 is_broadwater : 1;
         u8 is_crestline : 1;
         u8 has_fbc : 1;
-        u8 has_rc6 : 1;
         u8 has_pipe_cxsr : 1;
         u8 has_hotplug : 1;
         u8 cursor_needs_physical : 1;
@@ -243,6 +259,7 @@ typedef struct drm_i915_private {
         const struct intel_device_info *info;
 
         int has_gem;
+        int relative_constants_mode;
 
         void __iomem *regs;
 
@@ -253,20 +270,15 @@ typedef struct drm_i915_private {
         } *gmbus;
 
         struct pci_dev *bridge_dev;
-        struct intel_ring_buffer render_ring;
-        struct intel_ring_buffer bsd_ring;
-        struct intel_ring_buffer blt_ring;
+        struct intel_ring_buffer ring[I915_NUM_RINGS];
         uint32_t next_seqno;
 
         drm_dma_handle_t *status_page_dmah;
-        void *seqno_page;
         dma_addr_t dma_status_page;
         uint32_t counter;
-        unsigned int seqno_gfx_addr;
         drm_local_map_t hws_map;
-        struct drm_gem_object *seqno_obj;
-        struct drm_gem_object *pwrctx;
-        struct drm_gem_object *renderctx;
+        struct drm_i915_gem_object *pwrctx;
+        struct drm_i915_gem_object *renderctx;
 
         struct resource mch_res;
 
@@ -275,25 +287,17 @@ typedef struct drm_i915_private {
         int front_offset;
         int current_page;
         int page_flipping;
-#define I915_DEBUG_READ (1<<0)
-#define I915_DEBUG_WRITE (1<<1)
-        unsigned long debug_flags;
 
-        wait_queue_head_t irq_queue;
         atomic_t irq_received;
-        /** Protects user_irq_refcount and irq_mask_reg */
-        spinlock_t user_irq_lock;
         u32 trace_irq_seqno;
+
+        /* protects the irq masks */
+        spinlock_t irq_lock;
         /** Cached value of IMR to avoid reads in updating the bitfield */
-        u32 irq_mask_reg;
         u32 pipestat[2];
-        /** splitted irq regs for graphics and display engine on Ironlake,
-            irq_mask_reg is still used for display irq. */
-        u32 gt_irq_mask_reg;
-        u32 gt_irq_enable_reg;
-        u32 de_irq_enable_reg;
-        u32 pch_irq_mask_reg;
-        u32 pch_irq_enable_reg;
+        u32 irq_mask;
+        u32 gt_irq_mask;
+        u32 pch_irq_mask;
 
         u32 hotplug_supported_mask;
         struct work_struct hotplug_work;
@@ -306,7 +310,7 @@ typedef struct drm_i915_private {
         int num_pipe;
 
         /* For hangcheck timer */
-#define DRM_I915_HANGCHECK_PERIOD 250 /* in ms */
+#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
         struct timer_list hangcheck_timer;
         int hangcheck_count;
         uint32_t last_acthd;
@@ -329,6 +333,7 @@ typedef struct drm_i915_private {
 
         /* LVDS info */
         int backlight_level;  /* restore backlight to this value */
+        bool backlight_enabled;
         struct drm_display_mode *panel_fixed_mode;
         struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */
         struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */
@@ -530,23 +535,24 @@ typedef struct drm_i915_private {
 
         struct {
                 /** Bridge to intel-gtt-ko */
-                struct intel_gtt *gtt;
+                const struct intel_gtt *gtt;
                 /** Memory allocator for GTT stolen memory */
-                struct drm_mm vram;
+                struct drm_mm stolen;
                 /** Memory allocator for GTT */
                 struct drm_mm gtt_space;
+                /** List of all objects in gtt_space. Used to restore gtt
+                 * mappings on resume */
+                struct list_head gtt_list;
+
+                /** Usable portion of the GTT for GEM */
+                unsigned long gtt_start;
+                unsigned long gtt_mappable_end;
+                unsigned long gtt_end;
 
                 struct io_mapping *gtt_mapping;
                 int gtt_mtrr;
 
-                /**
-                 * Membership on list of all loaded devices, used to evict
-                 * inactive buffers under memory pressure.
-                 *
-                 * Modifications should only be done whilst holding the
-                 * shrink_list_lock spinlock.
-                 */
-                struct list_head shrink_list;
+                struct shrinker inactive_shrinker;
 
                 /**
                  * List of objects currently involved in rendering.
@@ -609,16 +615,6 @@ typedef struct drm_i915_private {
                 struct delayed_work retire_work;
 
                 /**
-                 * Waiting sequence number, if any
-                 */
-                uint32_t waiting_gem_seqno;
-
-                /**
-                 * Last seq seen at irq time
-                 */
-                uint32_t irq_gem_seqno;
-
-                /**
                  * Flag if the X Server, and thus DRM, is not currently in
                  * control of the device.
                  *
@@ -645,16 +641,11 @@ typedef struct drm_i915_private {
                 /* storage for physical objects */
                 struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
 
-                uint32_t flush_rings;
-
                 /* accounting, useful for userland debugging */
-                size_t object_memory;
-                size_t pin_memory;
-                size_t gtt_memory;
                 size_t gtt_total;
+                size_t mappable_gtt_total;
+                size_t object_memory;
                 u32 object_count;
-                u32 pin_count;
-                u32 gtt_count;
         } mm;
         struct sdvo_device_mapping sdvo_mappings[2];
         /* indicate whether the LVDS_BORDER should be enabled or not */
@@ -688,14 +679,14 @@ typedef struct drm_i915_private {
         u8 fmax;
         u8 fstart;
 
-        u64 last_count1;
-        unsigned long last_time1;
-        u64 last_count2;
-        struct timespec last_time2;
-        unsigned long gfx_power;
-        int c_m;
-        int r_t;
-        u8 corr;
+        u64 last_count1;
+        unsigned long last_time1;
+        u64 last_count2;
+        struct timespec last_time2;
+        unsigned long gfx_power;
+        int c_m;
+        int r_t;
+        u8 corr;
         spinlock_t *mchdev_lock;
 
         enum no_fbc_reason no_fbc_reason;
@@ -709,20 +700,20 @@ typedef struct drm_i915_private {
         struct intel_fbdev *fbdev;
 } drm_i915_private_t;
 
-/** driver private structure attached to each drm_gem_object */
 struct drm_i915_gem_object {
         struct drm_gem_object base;
 
         /** Current space allocated to this object in the GTT, if any. */
         struct drm_mm_node *gtt_space;
+        struct list_head gtt_list;
 
         /** This object's place on the active/flushing/inactive lists */
         struct list_head ring_list;
         struct list_head mm_list;
         /** This object's place on GPU write list */
         struct list_head gpu_write_list;
-        /** This object's place on eviction list */
-        struct list_head evict_list;
+        /** This object's place in the batchbuffer or on the eviction list */
+        struct list_head exec_list;
 
         /**
          * This is set if the object is on the active or flushing lists
@@ -738,6 +729,12 @@ struct drm_i915_gem_object {
         unsigned int dirty : 1;
 
         /**
+         * This is set if the object has been written to since the last
+         * GPU flush.
+         */
+        unsigned int pending_gpu_write : 1;
+
+        /**
          * Fence register bits (if any) for this object.  Will be set
          * as needed when mapped into the GTT.
          * Protected by dev->struct_mutex.
@@ -747,29 +744,15 @@ struct drm_i915_gem_object {
         signed int fence_reg : 5;
 
         /**
-         * Used for checking the object doesn't appear more than once
-         * in an execbuffer object list.
-         */
-        unsigned int in_execbuffer : 1;
-
-        /**
          * Advice: are the backing pages purgeable?
          */
         unsigned int madv : 2;
 
         /**
-         * Refcount for the pages array. With the current locking scheme, there
-         * are at most two concurrent users: Binding a bo to the gtt and
-         * pwrite/pread using physical addresses. So two bits for a maximum
-         * of two users are enough.
-         */
-        unsigned int pages_refcount : 2;
-#define DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT 0x3
-
-        /**
          * Current tiling mode for the object.
          */
         unsigned int tiling_mode : 2;
+        unsigned int tiling_changed : 1;
 
         /** How many users have pinned this object in GTT space. The following
          * users can each hold at most one reference: pwrite/pread, pin_ioctl
@@ -783,28 +766,55 @@ struct drm_i915_gem_object {
         unsigned int pin_count : 4;
 #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
 
-        /** AGP memory structure for our GTT binding. */
-        DRM_AGP_MEM *agp_mem;
+        /**
+         * Is the object at the current location in the gtt mappable and
+         * fenceable? Used to avoid costly recalculations.
+         */
+        unsigned int map_and_fenceable : 1;
+
+        /**
+         * Whether the current gtt mapping needs to be mappable (and isn't just
+         * mappable by accident). Track pin and fault separate for a more
+         * accurate mappable working set.
+         */
+        unsigned int fault_mappable : 1;
+        unsigned int pin_mappable : 1;
+
+        /*
+         * Is the GPU currently using a fence to access this buffer,
+         */
+        unsigned int pending_fenced_gpu_access:1;
+        unsigned int fenced_gpu_access:1;
 
         struct page **pages;
 
         /**
-         * Current offset of the object in GTT space.
-         *
-         * This is the same as gtt_space->start
+         * DMAR support
          */
-        uint32_t gtt_offset;
+        struct scatterlist *sg_list;
+        int num_sg;
 
-        /* Which ring is refering to is this object */
-        struct intel_ring_buffer *ring;
+        /**
+         * Used for performing relocations during execbuffer insertion.
+         */
+        struct hlist_node exec_node;
+        unsigned long exec_handle;
+        struct drm_i915_gem_exec_object2 *exec_entry;
 
         /**
-         * Fake offset for use by mmap(2)
+         * Current offset of the object in GTT space.
+         *
+         * This is the same as gtt_space->start
          */
-        uint64_t mmap_offset;
+        uint32_t gtt_offset;
 
         /** Breadcrumb of last rendering to the buffer. */
         uint32_t last_rendering_seqno;
+        struct intel_ring_buffer *ring;
+
+        /** Breadcrumb of last fenced GPU access to the buffer. */
+        uint32_t last_fenced_seqno;
+        struct intel_ring_buffer *last_fenced_ring;
 
         /** Current tiling stride for the object, if it's tiled. */
         uint32_t stride;
@@ -880,11 +890,74 @@ enum intel_chip_family {
         CHIP_I965 = 0x08,
 };
 
+#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
+
+#define IS_I830(dev)            ((dev)->pci_device == 0x3577)
+#define IS_845G(dev)            ((dev)->pci_device == 0x2562)
+#define IS_I85X(dev)            (INTEL_INFO(dev)->is_i85x)
+#define IS_I865G(dev)           ((dev)->pci_device == 0x2572)
+#define IS_I915G(dev)           (INTEL_INFO(dev)->is_i915g)
+#define IS_I915GM(dev)          ((dev)->pci_device == 0x2592)
+#define IS_I945G(dev)           ((dev)->pci_device == 0x2772)
+#define IS_I945GM(dev)          (INTEL_INFO(dev)->is_i945gm)
+#define IS_BROADWATER(dev)      (INTEL_INFO(dev)->is_broadwater)
+#define IS_CRESTLINE(dev)       (INTEL_INFO(dev)->is_crestline)
+#define IS_GM45(dev)            ((dev)->pci_device == 0x2A42)
+#define IS_G4X(dev)             (INTEL_INFO(dev)->is_g4x)
+#define IS_PINEVIEW_G(dev)      ((dev)->pci_device == 0xa001)
+#define IS_PINEVIEW_M(dev)      ((dev)->pci_device == 0xa011)
+#define IS_PINEVIEW(dev)        (INTEL_INFO(dev)->is_pineview)
+#define IS_G33(dev)             (INTEL_INFO(dev)->is_g33)
+#define IS_IRONLAKE_D(dev)      ((dev)->pci_device == 0x0042)
+#define IS_IRONLAKE_M(dev)      ((dev)->pci_device == 0x0046)
+#define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
+
+#define IS_GEN2(dev)    (INTEL_INFO(dev)->gen == 2)
+#define IS_GEN3(dev)    (INTEL_INFO(dev)->gen == 3)
+#define IS_GEN4(dev)    (INTEL_INFO(dev)->gen == 4)
+#define IS_GEN5(dev)    (INTEL_INFO(dev)->gen == 5)
+#define IS_GEN6(dev)    (INTEL_INFO(dev)->gen == 6)
+
+#define HAS_BSD(dev)            (INTEL_INFO(dev)->has_bsd_ring)
+#define HAS_BLT(dev)            (INTEL_INFO(dev)->has_blt_ring)
+#define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
+
+#define HAS_OVERLAY(dev)                (INTEL_INFO(dev)->has_overlay)
+#define OVERLAY_NEEDS_PHYSICAL(dev)     (INTEL_INFO(dev)->overlay_needs_physical)
+
+/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
+ * rows, which changed the alignment requirements and fence programming.
+ */
+#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
+                                                      IS_I915GM(dev)))
+#define SUPPORTS_DIGITAL_OUTPUTS(dev)   (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
+#define SUPPORTS_INTEGRATED_HDMI(dev)   (IS_G4X(dev) || IS_GEN5(dev))
+#define SUPPORTS_INTEGRATED_DP(dev)     (IS_G4X(dev) || IS_GEN5(dev))
+#define SUPPORTS_EDP(dev)               (IS_IRONLAKE_M(dev))
+#define SUPPORTS_TV(dev)                (INTEL_INFO(dev)->supports_tv)
+#define I915_HAS_HOTPLUG(dev)            (INTEL_INFO(dev)->has_hotplug)
+/* dsparb controlled by hw only */
+#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
+
+#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
+#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
+#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
+
+#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev))
+#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev))
+
+#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
+#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
+#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
+
+#include "i915_trace.h"
+
 extern struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
 extern unsigned int i915_fbpercrtc;
 extern unsigned int i915_powersave;
 extern unsigned int i915_lvds_downclock;
+extern unsigned int i915_panel_use_ssc;
 
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
@@ -907,8 +980,8 @@ extern int i915_driver_device_is_agp(struct drm_device * dev);
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
                               unsigned long arg);
 extern int i915_emit_box(struct drm_device *dev,
-                         struct drm_clip_rect *boxes,
-                         int i, int DR1, int DR4);
+                         struct drm_clip_rect *box,
+                         int DR1, int DR4);
 extern int i915_reset(struct drm_device *dev, u8 flags);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
@@ -918,6 +991,7 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 
 /* i915_irq.c */
 void i915_hangcheck_elapsed(unsigned long data);
+void i915_handle_error(struct drm_device *dev, bool wedged);
 extern int i915_irq_emit(struct drm_device *dev, void *data,
                          struct drm_file *file_priv);
 extern int i915_irq_wait(struct drm_device *dev, void *data,
@@ -939,12 +1013,6 @@ extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc);
 extern u32 gm45_get_vblank_counter(struct drm_device *dev, int crtc);
 extern int i915_vblank_swap(struct drm_device *dev, void *data,
                             struct drm_file *file_priv);
-extern void i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask);
-extern void i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask);
-extern void ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv,
-                u32 mask);
-extern void ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv,
-                u32 mask);
 
 void
 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
@@ -953,6 +1021,13 @@ void
 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 
 void intel_enable_asle (struct drm_device *dev);
+int i915_get_vblank_timestamp(struct drm_device *dev, int crtc,
+                              int *max_error,
+                              struct timeval *vblank_time,
+                              unsigned flags);
+
+int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
+                             int *vpos, int *hpos);
 
 #ifdef CONFIG_DEBUG_FS
 extern void i915_destroy_error_state(struct drm_device *dev);
@@ -1017,15 +1092,35 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
 void i915_gem_load(struct drm_device *dev);
 int i915_gem_init_object(struct drm_gem_object *obj);
-struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
-                                              size_t size);
+int __must_check i915_gem_flush_ring(struct drm_device *dev,
+                                     struct intel_ring_buffer *ring,
+                                     uint32_t invalidate_domains,
+                                     uint32_t flush_domains);
+struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
+                                                  size_t size);
 void i915_gem_free_object(struct drm_gem_object *obj);
-int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment);
-void i915_gem_object_unpin(struct drm_gem_object *obj);
-int i915_gem_object_unbind(struct drm_gem_object *obj);
-void i915_gem_release_mmap(struct drm_gem_object *obj);
+int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
+                                     uint32_t alignment,
+                                     bool map_and_fenceable);
+void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
+void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 
+int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
+int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
+                                                bool interruptible);
+void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
+                                    struct intel_ring_buffer *ring,
+                                    u32 seqno);
+
+int i915_gem_dumb_create(struct drm_file *file_priv,
+                         struct drm_device *dev,
+                         struct drm_mode_create_dumb *args);
+int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
+                      uint32_t handle, uint64_t *offset);
+int i915_gem_dumb_destroy(struct drm_file *file_priv, struct drm_device *dev,
+                          uint32_t handle);                       
 /**
  * Returns true if seq1 is later than seq2.
  */
@@ -1035,73 +1130,88 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
         return (int32_t)(seq1 - seq2) >= 0;
 }
 
-int i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
-                                  bool interruptible);
-int i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
-                                  bool interruptible);
+static inline u32
+i915_gem_next_request_seqno(struct drm_device *dev,
+                            struct intel_ring_buffer *ring)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        return ring->outstanding_lazy_request = dev_priv->next_seqno;
+}
+
+int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
+                                           struct intel_ring_buffer *pipelined,
+                                           bool interruptible);
+int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
+
 void i915_gem_retire_requests(struct drm_device *dev);
 void i915_gem_reset(struct drm_device *dev);
-void i915_gem_clflush_object(struct drm_gem_object *obj);
-int i915_gem_object_set_domain(struct drm_gem_object *obj,
-                               uint32_t read_domains,
-                               uint32_t write_domain);
-int i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
-                              bool interruptible);
-int i915_gem_init_ringbuffer(struct drm_device *dev);
+void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
+                                            uint32_t read_domains,
+                                            uint32_t write_domain);
+int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
+                                           bool interruptible);
+int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
-int i915_gem_do_init(struct drm_device *dev, unsigned long start,
-                     unsigned long end);
-int i915_gpu_idle(struct drm_device *dev);
-int i915_gem_idle(struct drm_device *dev);
-uint32_t i915_add_request(struct drm_device *dev,
-                          struct drm_file *file_priv,
-                          struct drm_i915_gem_request *request,
-                          struct intel_ring_buffer *ring);
-int i915_do_wait_request(struct drm_device *dev,
-                         uint32_t seqno,
-                         bool interruptible,
-                         struct intel_ring_buffer *ring);
+void i915_gem_do_init(struct drm_device *dev,
+                      unsigned long start,
+                      unsigned long mappable_end,
+                      unsigned long end);
+int __must_check i915_gpu_idle(struct drm_device *dev);
+int __must_check i915_gem_idle(struct drm_device *dev);
+int __must_check i915_add_request(struct drm_device *dev,
+                                  struct drm_file *file_priv,
+                                  struct drm_i915_gem_request *request,
+                                  struct intel_ring_buffer *ring);
+int __must_check i915_do_wait_request(struct drm_device *dev,
+                                      uint32_t seqno,
+                                      bool interruptible,
+                                      struct intel_ring_buffer *ring);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
-int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
-                                      int write);
-int i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
-                                         bool pipelined);
+int __must_check
+i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
+                                  bool write);
+int __must_check
+i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+                                     struct intel_ring_buffer *pipelined);
 int i915_gem_attach_phys_object(struct drm_device *dev,
-                                struct drm_gem_object *obj,
+                                struct drm_i915_gem_object *obj,
                                 int id,
                                 int align);
 void i915_gem_detach_phys_object(struct drm_device *dev,
-                                 struct drm_gem_object *obj);
+                                 struct drm_i915_gem_object *obj);
 void i915_gem_free_all_phys_object(struct drm_device *dev);
-void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv);
+void i915_gem_release(struct drm_device *dev, struct drm_file *file);
 
-void i915_gem_shrinker_init(void);
-void i915_gem_shrinker_exit(void);
+/* i915_gem_gtt.c */
+void i915_gem_restore_gtt_mappings(struct drm_device *dev);
+int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
+void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 
 /* i915_gem_evict.c */
-int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment);
-int i915_gem_evict_everything(struct drm_device *dev);
-int i915_gem_evict_inactive(struct drm_device *dev);
+int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size,
+                                          unsigned alignment, bool mappable);
+int __must_check i915_gem_evict_everything(struct drm_device *dev,
+                                           bool purgeable_only);
+int __must_check i915_gem_evict_inactive(struct drm_device *dev,
+                                         bool purgeable_only);
 
 /* i915_gem_tiling.c */
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
-void i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj);
-void i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj);
-bool i915_tiling_ok(struct drm_device *dev, int stride, int size,
-                    int tiling_mode);
-bool i915_gem_object_fence_offset_ok(struct drm_gem_object *obj,
-                                     int tiling_mode);
+void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
+void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
 
 /* i915_gem_debug.c */
-void i915_gem_dump_object(struct drm_gem_object *obj, int len,
+void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
                           const char *where, uint32_t mark);
 #if WATCH_LISTS
 int i915_verify_lists(struct drm_device *dev);
 #else
 #define i915_verify_lists(dev) 0
 #endif
-void i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle);
-void i915_gem_dump_object(struct drm_gem_object *obj, int len,
+void i915_gem_object_check_coherency(struct drm_i915_gem_object *obj,
+                                     int handle);
+void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
                           const char *where, uint32_t mark);
 
 /* i915_debugfs.c */
@@ -1163,6 +1273,8 @@ extern void intel_disable_fbc(struct drm_device *dev);
 extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval);
 extern bool intel_fbc_enabled(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
+extern void ironlake_enable_rc6(struct drm_device *dev);
+extern void gen6_set_rps(struct drm_device *dev, u8 val);
 extern void intel_detect_pch (struct drm_device *dev);
 extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
 
@@ -1170,79 +1282,120 @@ extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
 #ifdef CONFIG_DEBUG_FS
 extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
 extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error);
+
+extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
+extern void intel_display_print_error_state(struct seq_file *m,
+                                            struct drm_device *dev,
+                                            struct intel_display_error_state *error);
 #endif
 
+#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
+
+#define BEGIN_LP_RING(n) \
+        intel_ring_begin(LP_RING(dev_priv), (n))
+
+#define OUT_RING(x) \
+        intel_ring_emit(LP_RING(dev_priv), x)
+
+#define ADVANCE_LP_RING() \
+        intel_ring_advance(LP_RING(dev_priv))
+
 /**
  * Lock test for when it's just for synchronization of ring access.
  *
  * In that case, we don't need to do it when GEM is initialized as nobody else
  * has access to the ring.
  */
-#define RING_LOCK_TEST_WITH_RETURN(dev, file_priv) do {                 \
-        if (((drm_i915_private_t *)dev->dev_private)->render_ring.gem_object \
-                        == NULL)                                        \
-                LOCK_TEST_WITH_RETURN(dev, file_priv);                  \
+#define RING_LOCK_TEST_WITH_RETURN(dev, file) do {                      \
+        if (LP_RING(dev->dev_private)->obj == NULL)                     \
+                LOCK_TEST_WITH_RETURN(dev, file);                       \
 } while (0)
 
-static inline u32 i915_read(struct drm_i915_private *dev_priv, u32 reg)
+
+#define __i915_read(x, y) \
+static inline u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
+        u##x val = read##y(dev_priv->regs + reg); \
+        trace_i915_reg_rw('R', reg, val, sizeof(val)); \
+        return val; \
+}
+__i915_read(8, b)
+__i915_read(16, w)
+__i915_read(32, l)
+__i915_read(64, q)
+#undef __i915_read
+
+#define __i915_write(x, y) \
+static inline void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
+        trace_i915_reg_rw('W', reg, val, sizeof(val)); \
+        write##y(val, dev_priv->regs + reg); \
+}
+__i915_write(8, b)
+__i915_write(16, w)
+__i915_write(32, l)
+__i915_write(64, q)
+#undef __i915_write
+
+#define I915_READ8(reg)         i915_read8(dev_priv, (reg))
+#define I915_WRITE8(reg, val)   i915_write8(dev_priv, (reg), (val))
+
+#define I915_READ16(reg)        i915_read16(dev_priv, (reg))
+#define I915_WRITE16(reg, val)  i915_write16(dev_priv, (reg), (val))
+#define I915_READ16_NOTRACE(reg)        readw(dev_priv->regs + (reg))
+#define I915_WRITE16_NOTRACE(reg, val)  writew(val, dev_priv->regs + (reg))
+
+#define I915_READ(reg)          i915_read32(dev_priv, (reg))
+#define I915_WRITE(reg, val)    i915_write32(dev_priv, (reg), (val))
+#define I915_READ_NOTRACE(reg)          readl(dev_priv->regs + (reg))
+#define I915_WRITE_NOTRACE(reg, val)    writel(val, dev_priv->regs + (reg))
+
+#define I915_WRITE64(reg, val)  i915_write64(dev_priv, (reg), (val))
+#define I915_READ64(reg)        i915_read64(dev_priv, (reg))
+
+#define POSTING_READ(reg)       (void)I915_READ_NOTRACE(reg)
+#define POSTING_READ16(reg)     (void)I915_READ16_NOTRACE(reg)
+
+
+/* On SNB platform, before reading ring registers forcewake bit
+ * must be set to prevent GT core from power down and stale values being
+ * returned.
+ */
+void __gen6_force_wake_get(struct drm_i915_private *dev_priv);
+void __gen6_force_wake_put (struct drm_i915_private *dev_priv);
+static inline u32 i915_safe_read(struct drm_i915_private *dev_priv, u32 reg)
 {
         u32 val;
 
-        val = readl(dev_priv->regs + reg);
-        if (dev_priv->debug_flags & I915_DEBUG_READ)
-                printk(KERN_ERR "read 0x%08x from 0x%08x\n", val, reg);
+        if (dev_priv->info->gen >= 6) {
+                __gen6_force_wake_get(dev_priv);
+                val = I915_READ(reg);
+                __gen6_force_wake_put(dev_priv);
+        } else
+                val = I915_READ(reg);
+
         return val;
 }
 
-static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg,
-                              u32 val)
+static inline void
+i915_write(struct drm_i915_private *dev_priv, u32 reg, u64 val, int len)
 {
-        writel(val, dev_priv->regs + reg);
-        if (dev_priv->debug_flags & I915_DEBUG_WRITE)
-                printk(KERN_ERR "wrote 0x%08x to 0x%08x\n", val, reg);
+       /* Trace down the write operation before the real write */
+       trace_i915_reg_rw('W', reg, val, len);
+       switch (len) {
+       case 8:
+               writeq(val, dev_priv->regs + reg);
+               break;
+       case 4:
+               writel(val, dev_priv->regs + reg);
+               break;
+       case 2:
+               writew(val, dev_priv->regs + reg);
+               break;
+       case 1:
+               writeb(val, dev_priv->regs + reg);
+               break;
+       }
 }
 
-#define I915_READ(reg)          i915_read(dev_priv, (reg))
-#define I915_WRITE(reg, val)    i915_write(dev_priv, (reg), (val))
-#define I915_READ16(reg)        readw(dev_priv->regs + (reg))
-#define I915_WRITE16(reg, val)  writel(val, dev_priv->regs + (reg))
-#define I915_READ8(reg)         readb(dev_priv->regs + (reg))
-#define I915_WRITE8(reg, val)   writeb(val, dev_priv->regs + (reg))
-#define I915_WRITE64(reg, val)  writeq(val, dev_priv->regs + (reg))
-#define I915_READ64(reg)        readq(dev_priv->regs + (reg))
-#define POSTING_READ(reg)       (void)I915_READ(reg)
-#define POSTING_READ16(reg)     (void)I915_READ16(reg)
-
-#define I915_DEBUG_ENABLE_IO() (dev_priv->debug_flags |= I915_DEBUG_READ | \
-                                I915_DEBUG_WRITE)
-#define I915_DEBUG_DISABLE_IO() (dev_priv->debug_flags &= ~(I915_DEBUG_READ | \
-                                                            I915_DEBUG_WRITE))
-
-#define I915_VERBOSE 0
-
-#define BEGIN_LP_RING(n)  do { \
-        drm_i915_private_t *dev_priv__ = dev->dev_private;                \
-        if (I915_VERBOSE)                                               \
-                DRM_DEBUG("   BEGIN_LP_RING %x\n", (int)(n));           \
-        intel_ring_begin(dev, &dev_priv__->render_ring, (n));           \
-} while (0)
-
-
-#define OUT_RING(x) do {                                                \
-        drm_i915_private_t *dev_priv__ = dev->dev_private;              \
-        if (I915_VERBOSE)                                               \
-                DRM_DEBUG("   OUT_RING %x\n", (int)(x));                \
-        intel_ring_emit(dev, &dev_priv__->render_ring, x);              \
-} while (0)
-
-#define ADVANCE_LP_RING() do {                                          \
-        drm_i915_private_t *dev_priv__ = dev->dev_private;                \
-        if (I915_VERBOSE)                                               \
-                DRM_DEBUG("ADVANCE_LP_RING %x\n",                       \
-                                dev_priv__->render_ring.tail);          \
-        intel_ring_advance(dev, &dev_priv__->render_ring);              \
-} while(0)
-
 /**
  * Reads a dword out of the status page, which is written to from the command
  * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
@@ -1259,72 +1412,9 @@ static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg,
  * The area from dword 0x20 to 0x3ff is available for driver usage.
  */
 #define READ_HWSP(dev_priv, reg)  (((volatile u32 *)\
-                        (dev_priv->render_ring.status_page.page_addr))[reg])
+                        (LP_RING(dev_priv)->status_page.page_addr))[reg])
 #define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
 #define I915_GEM_HWS_INDEX              0x20
 #define I915_BREADCRUMB_INDEX           0x21
 
-#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
-
-#define IS_I830(dev)            ((dev)->pci_device == 0x3577)
-#define IS_845G(dev)            ((dev)->pci_device == 0x2562)
-#define IS_I85X(dev)            (INTEL_INFO(dev)->is_i85x)
-#define IS_I865G(dev)           ((dev)->pci_device == 0x2572)
-#define IS_I915G(dev)           (INTEL_INFO(dev)->is_i915g)
-#define IS_I915GM(dev)          ((dev)->pci_device == 0x2592)
-#define IS_I945G(dev)           ((dev)->pci_device == 0x2772)
-#define IS_I945GM(dev)          (INTEL_INFO(dev)->is_i945gm)
-#define IS_BROADWATER(dev)      (INTEL_INFO(dev)->is_broadwater)
-#define IS_CRESTLINE(dev)       (INTEL_INFO(dev)->is_crestline)
-#define IS_GM45(dev)            ((dev)->pci_device == 0x2A42)
-#define IS_G4X(dev)             (INTEL_INFO(dev)->is_g4x)
-#define IS_PINEVIEW_G(dev)      ((dev)->pci_device == 0xa001)
-#define IS_PINEVIEW_M(dev)      ((dev)->pci_device == 0xa011)
-#define IS_PINEVIEW(dev)        (INTEL_INFO(dev)->is_pineview)
-#define IS_G33(dev)             (INTEL_INFO(dev)->is_g33)
-#define IS_IRONLAKE_D(dev)      ((dev)->pci_device == 0x0042)
-#define IS_IRONLAKE_M(dev)      ((dev)->pci_device == 0x0046)
-#define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
-
-#define IS_GEN2(dev)    (INTEL_INFO(dev)->gen == 2)
-#define IS_GEN3(dev)    (INTEL_INFO(dev)->gen == 3)
-#define IS_GEN4(dev)    (INTEL_INFO(dev)->gen == 4)
-#define IS_GEN5(dev)    (INTEL_INFO(dev)->gen == 5)
-#define IS_GEN6(dev)    (INTEL_INFO(dev)->gen == 6)
-
-#define HAS_BSD(dev)            (INTEL_INFO(dev)->has_bsd_ring)
-#define HAS_BLT(dev)            (INTEL_INFO(dev)->has_blt_ring)
-#define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
-
-#define HAS_OVERLAY(dev)                (INTEL_INFO(dev)->has_overlay)
-#define OVERLAY_NEEDS_PHYSICAL(dev)     (INTEL_INFO(dev)->overlay_needs_physical)
-
-/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
- * rows, which changed the alignment requirements and fence programming.
- */
-#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
-                                                      IS_I915GM(dev)))
-#define SUPPORTS_DIGITAL_OUTPUTS(dev)   (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
-#define SUPPORTS_INTEGRATED_HDMI(dev)   (IS_G4X(dev) || IS_GEN5(dev))
-#define SUPPORTS_INTEGRATED_DP(dev)     (IS_G4X(dev) || IS_GEN5(dev))
-#define SUPPORTS_EDP(dev)               (IS_IRONLAKE_M(dev))
-#define SUPPORTS_TV(dev)                (INTEL_INFO(dev)->supports_tv)
-#define I915_HAS_HOTPLUG(dev)            (INTEL_INFO(dev)->has_hotplug)
-/* dsparb controlled by hw only */
-#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
-
-#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
-#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
-#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
-#define I915_HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6)
-
-#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev))
-#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev))
-
-#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
-#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
-#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
-
-#define PRIMARY_RINGBUFFER_SIZE         (128*1024)
-
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 17b1cba3b5f1..bc7f06b8fbca 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -34,39 +34,31 @@
 #include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/pci.h>
-#include <linux/intel-gtt.h>
-
-static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
-
-static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
-                                                  bool pipelined);
-static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
-static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
-static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
-                                             int write);
-static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
-                                                     uint64_t offset,
-                                                     uint64_t size);
-static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
-static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
-                                          bool interruptible);
-static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
-                                           unsigned alignment);
-static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
-static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
+
+static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
+static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
+static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
+static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
+                                                          bool write);
+static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
+                                                                  uint64_t offset,
+                                                                  uint64_t size);
+static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
+static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
+                                                    unsigned alignment,
+                                                    bool map_and_fenceable);
+static void i915_gem_clear_fence_reg(struct drm_device *dev,
+                                     struct drm_i915_fence_reg *reg);
+static int i915_gem_phys_pwrite(struct drm_device *dev,
+                                struct drm_i915_gem_object *obj,
                                 struct drm_i915_gem_pwrite *args,
-                                struct drm_file *file_priv);
-static void i915_gem_free_object_tail(struct drm_gem_object *obj);
+                                struct drm_file *file);
+static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
 
-static int
-i915_gem_object_get_pages(struct drm_gem_object *obj,
-                          gfp_t gfpmask);
+static int i915_gem_inactive_shrink(struct shrinker *shrinker,
+                                    int nr_to_scan,
+                                    gfp_t gfp_mask);
 
-static void
-i915_gem_object_put_pages(struct drm_gem_object *obj);
-
-static LIST_HEAD(shrink_list);
-static DEFINE_SPINLOCK(shrink_list_lock);
 
 /* some bookkeeping */
 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
@@ -83,34 +75,6 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
         dev_priv->mm.object_memory -= size;
 }
 
-static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
-                                  size_t size)
-{
-        dev_priv->mm.gtt_count++;
-        dev_priv->mm.gtt_memory += size;
-}
-
-static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
-                                     size_t size)
-{
-        dev_priv->mm.gtt_count--;
-        dev_priv->mm.gtt_memory -= size;
-}
-
-static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
-                                  size_t size)
-{
-        dev_priv->mm.pin_count++;
-        dev_priv->mm.pin_memory += size;
-}
-
-static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
-                                     size_t size)
-{
-        dev_priv->mm.pin_count--;
-        dev_priv->mm.pin_memory -= size;
-}
-
 int
 i915_gem_check_is_wedged(struct drm_device *dev)
 {
@@ -141,7 +105,7 @@ i915_gem_check_is_wedged(struct drm_device *dev)
         return -EIO;
 }
 
-static int i915_mutex_lock_interruptible(struct drm_device *dev)
+int i915_mutex_lock_interruptible(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
@@ -164,124 +128,141 @@ static int i915_mutex_lock_interruptible(struct drm_device *dev)
 }
 
 static inline bool
-i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
+i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 {
-        return obj_priv->gtt_space &&
-                !obj_priv->active &&
-                obj_priv->pin_count == 0;
+        return obj->gtt_space && !obj->active && obj->pin_count == 0;
 }
 
-int i915_gem_do_init(struct drm_device *dev,
-                     unsigned long start,
-                     unsigned long end)
+void i915_gem_do_init(struct drm_device *dev,
+                      unsigned long start,
+                      unsigned long mappable_end,
+                      unsigned long end)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
 
-        if (start >= end ||
-            (start & (PAGE_SIZE - 1)) != 0 ||
-            (end & (PAGE_SIZE - 1)) != 0) {
-                return -EINVAL;
-        }
-
-        drm_mm_init(&dev_priv->mm.gtt_space, start,
-                    end - start);
+        drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
 
+        dev_priv->mm.gtt_start = start;
+        dev_priv->mm.gtt_mappable_end = mappable_end;
+        dev_priv->mm.gtt_end = end;
         dev_priv->mm.gtt_total = end - start;
+        dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 
-        return 0;
+        /* Take over this portion of the GTT */
+        intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 }
 
 int
 i915_gem_init_ioctl(struct drm_device *dev, void *data,
-                    struct drm_file *file_priv)
+                    struct drm_file *file)
 {
         struct drm_i915_gem_init *args = data;
-        int ret;
+
+        if (args->gtt_start >= args->gtt_end ||
+            (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
+                return -EINVAL;
 
         mutex_lock(&dev->struct_mutex);
-        ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
+        i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
         mutex_unlock(&dev->struct_mutex);
 
-        return ret;
+        return 0;
 }
 
 int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv)
+                            struct drm_file *file)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_get_aperture *args = data;
+        struct drm_i915_gem_object *obj;
+        size_t pinned;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
                 return -ENODEV;
 
+        pinned = 0;
         mutex_lock(&dev->struct_mutex);
-        args->aper_size = dev_priv->mm.gtt_total;
-        args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
+        list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
+                pinned += obj->gtt_space->size;
         mutex_unlock(&dev->struct_mutex);
 
+        args->aper_size = dev_priv->mm.gtt_total;
+        args->aper_available_size = args->aper_size -pinned;
+
         return 0;
 }
 
-
-/**
- * Creates a new mm object and returns a handle to it.
- */
-int
-i915_gem_create_ioctl(struct drm_device *dev, void *data,
-                      struct drm_file *file_priv)
+static int
+i915_gem_create(struct drm_file *file,
+                struct drm_device *dev,
+                uint64_t size,
+                uint32_t *handle_p)
 {
-        struct drm_i915_gem_create *args = data;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int ret;
         u32 handle;
 
-        args->size = roundup(args->size, PAGE_SIZE);
+        size = roundup(size, PAGE_SIZE);
 
         /* Allocate the new object */
-        obj = i915_gem_alloc_object(dev, args->size);
+        obj = i915_gem_alloc_object(dev, size);
         if (obj == NULL)
                 return -ENOMEM;
 
-        ret = drm_gem_handle_create(file_priv, obj, &handle);
+        ret = drm_gem_handle_create(file, &obj->base, &handle);
         if (ret) {
-                drm_gem_object_release(obj);
-                i915_gem_info_remove_obj(dev->dev_private, obj->size);
+                drm_gem_object_release(&obj->base);
+                i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
                 kfree(obj);
                 return ret;
         }
 
         /* drop reference from allocate - handle holds it now */
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         trace_i915_gem_object_create(obj);
 
-        args->handle = handle;
+        *handle_p = handle;
         return 0;
 }
 
-static inline int
-fast_shmem_read(struct page **pages,
-                loff_t page_base, int page_offset,
-                char __user *data,
-                int length)
+int
+i915_gem_dumb_create(struct drm_file *file,
+                     struct drm_device *dev,
+                     struct drm_mode_create_dumb *args)
 {
-        char *vaddr;
-        int ret;
+        /* have to work out size/pitch and return them */
+        args->pitch = ALIGN(args->width & ((args->bpp + 1) / 8), 64);
+        args->size = args->pitch * args->height;
+        return i915_gem_create(file, dev,
+                               args->size, &args->handle);
+}
 
-        vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
-        ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
-        kunmap_atomic(vaddr);
+int i915_gem_dumb_destroy(struct drm_file *file,
+                          struct drm_device *dev,
+                          uint32_t handle)
+{
+        return drm_gem_handle_delete(file, handle);
+}
 
-        return ret;
+/**
+ * Creates a new mm object and returns a handle to it.
+ */
+int
+i915_gem_create_ioctl(struct drm_device *dev, void *data,
+                      struct drm_file *file)
+{
+        struct drm_i915_gem_create *args = data;
+        return i915_gem_create(file, dev,
+                               args->size, &args->handle);
 }
 
-static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
+static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = obj->dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 
         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
-                obj_priv->tiling_mode != I915_TILING_NONE;
+                obj->tiling_mode != I915_TILING_NONE;
 }
 
 static inline void
@@ -357,38 +338,51 @@ slow_shmem_bit17_copy(struct page *gpu_page,
  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
  */
 static int
-i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pread_fast(struct drm_device *dev,
+                          struct drm_i915_gem_object *obj,
                           struct drm_i915_gem_pread *args,
-                          struct drm_file *file_priv)
+                          struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         ssize_t remain;
-        loff_t offset, page_base;
+        loff_t offset;
         char __user *user_data;
         int page_offset, page_length;
 
         user_data = (char __user *) (uintptr_t) args->data_ptr;
         remain = args->size;
 
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
 
         while (remain > 0) {
+                struct page *page;
+                char *vaddr;
+                int ret;
+
                 /* Operation in this page
                  *
-                 * page_base = page offset within aperture
                  * page_offset = offset within page
                  * page_length = bytes to copy for this page
                  */
-                page_base = (offset & ~(PAGE_SIZE-1));
                 page_offset = offset & (PAGE_SIZE-1);
                 page_length = remain;
                 if ((page_offset + remain) > PAGE_SIZE)
                         page_length = PAGE_SIZE - page_offset;
 
-                if (fast_shmem_read(obj_priv->pages,
-                                    page_base, page_offset,
-                                    user_data, page_length))
+                page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
+                                           GFP_HIGHUSER | __GFP_RECLAIMABLE);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
+                vaddr = kmap_atomic(page);
+                ret = __copy_to_user_inatomic(user_data,
+                                              vaddr + page_offset,
+                                              page_length);
+                kunmap_atomic(vaddr);
+
+                mark_page_accessed(page);
+                page_cache_release(page);
+                if (ret)
                         return -EFAULT;
 
                 remain -= page_length;
@@ -399,30 +393,6 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
         return 0;
 }
 
-static int
-i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
-{
-        int ret;
-
-        ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
-
-        /* If we've insufficient memory to map in the pages, attempt
-         * to make some space by throwing out some old buffers.
-         */
-        if (ret == -ENOMEM) {
-                struct drm_device *dev = obj->dev;
-
-                ret = i915_gem_evict_something(dev, obj->size,
-                                               i915_gem_get_gtt_alignment(obj));
-                if (ret)
-                        return ret;
-
-                ret = i915_gem_object_get_pages(obj, 0);
-        }
-
-        return ret;
-}
-
 /**
  * This is the fallback shmem pread path, which allocates temporary storage
  * in kernel space to copy_to_user into outside of the struct_mutex, so we
@@ -430,18 +400,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
  * and not take page faults.
  */
 static int
-i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pread_slow(struct drm_device *dev,
+                          struct drm_i915_gem_object *obj,
                           struct drm_i915_gem_pread *args,
-                          struct drm_file *file_priv)
+                          struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         struct mm_struct *mm = current->mm;
         struct page **user_pages;
         ssize_t remain;
         loff_t offset, pinned_pages, i;
         loff_t first_data_page, last_data_page, num_pages;
-        int shmem_page_index, shmem_page_offset;
-        int data_page_index,  data_page_offset;
+        int shmem_page_offset;
+        int data_page_index, data_page_offset;
         int page_length;
         int ret;
         uint64_t data_ptr = args->data_ptr;
@@ -480,19 +451,18 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 
         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
 
         while (remain > 0) {
+                struct page *page;
+
                 /* Operation in this page
                  *
-                 * shmem_page_index = page number within shmem file
                  * shmem_page_offset = offset within page in shmem file
                  * data_page_index = page number in get_user_pages return
                  * data_page_offset = offset with data_page_index page.
                  * page_length = bytes to copy for this page
                  */
-                shmem_page_index = offset / PAGE_SIZE;
                 shmem_page_offset = offset & ~PAGE_MASK;
                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
                 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -503,8 +473,13 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 if ((data_page_offset + page_length) > PAGE_SIZE)
                         page_length = PAGE_SIZE - data_page_offset;
 
+                page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
+                                           GFP_HIGHUSER | __GFP_RECLAIMABLE);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
                 if (do_bit17_swizzling) {
-                        slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
+                        slow_shmem_bit17_copy(page,
                                               shmem_page_offset,
                                               user_pages[data_page_index],
                                               data_page_offset,
@@ -513,11 +488,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 } else {
                         slow_shmem_copy(user_pages[data_page_index],
                                         data_page_offset,
-                                        obj_priv->pages[shmem_page_index],
+                                        page,
                                         shmem_page_offset,
                                         page_length);
                 }
 
+                mark_page_accessed(page);
+                page_cache_release(page);
+
                 remain -= page_length;
                 data_ptr += page_length;
                 offset += page_length;
@@ -526,6 +504,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 out:
         for (i = 0; i < pinned_pages; i++) {
                 SetPageDirty(user_pages[i]);
+                mark_page_accessed(user_pages[i]);
                 page_cache_release(user_pages[i]);
         }
         drm_free_large(user_pages);
@@ -540,11 +519,10 @@ out:
  */
 int
 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
-                     struct drm_file *file_priv)
+                     struct drm_file *file)
 {
         struct drm_i915_gem_pread *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret = 0;
 
         if (args->size == 0)
@@ -564,39 +542,33 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
         /* Bounds check source.  */
-        if (args->offset > obj->size || args->size > obj->size - args->offset) {
+        if (args->offset > obj->base.size ||
+            args->size > obj->base.size - args->offset) {
                 ret = -EINVAL;
                 goto out;
         }
 
-        ret = i915_gem_object_get_pages_or_evict(obj);
-        if (ret)
-                goto out;
-
         ret = i915_gem_object_set_cpu_read_domain_range(obj,
                                                         args->offset,
                                                         args->size);
         if (ret)
-                goto out_put;
+                goto out;
 
         ret = -EFAULT;
         if (!i915_gem_object_needs_bit17_swizzle(obj))
-                ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
+                ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
         if (ret == -EFAULT)
-                ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
+                ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
 
-out_put:
-        i915_gem_object_put_pages(obj);
 out:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -646,32 +618,16 @@ slow_kernel_write(struct io_mapping *mapping,
         io_mapping_unmap(dst_vaddr);
 }
 
-static inline int
-fast_shmem_write(struct page **pages,
-                 loff_t page_base, int page_offset,
-                 char __user *data,
-                 int length)
-{
-        char *vaddr;
-        int ret;
-
-        vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
-        ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
-        kunmap_atomic(vaddr);
-
-        return ret;
-}
-
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
  */
 static int
-i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_gtt_pwrite_fast(struct drm_device *dev,
+                         struct drm_i915_gem_object *obj,
                          struct drm_i915_gem_pwrite *args,
-                         struct drm_file *file_priv)
+                         struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         drm_i915_private_t *dev_priv = dev->dev_private;
         ssize_t remain;
         loff_t offset, page_base;
@@ -681,8 +637,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
         user_data = (char __user *) (uintptr_t) args->data_ptr;
         remain = args->size;
 
-        obj_priv = to_intel_bo(obj);
-        offset = obj_priv->gtt_offset + args->offset;
+        offset = obj->gtt_offset + args->offset;
 
         while (remain > 0) {
                 /* Operation in this page
@@ -722,11 +677,11 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
  */
 static int
-i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_gtt_pwrite_slow(struct drm_device *dev,
+                         struct drm_i915_gem_object *obj,
                          struct drm_i915_gem_pwrite *args,
-                         struct drm_file *file_priv)
+                         struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         drm_i915_private_t *dev_priv = dev->dev_private;
         ssize_t remain;
         loff_t gtt_page_base, offset;
@@ -763,12 +718,15 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 goto out_unpin_pages;
         }
 
-        ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+        ret = i915_gem_object_set_to_gtt_domain(obj, true);
+        if (ret)
+                goto out_unpin_pages;
+
+        ret = i915_gem_object_put_fence(obj);
         if (ret)
                 goto out_unpin_pages;
 
-        obj_priv = to_intel_bo(obj);
-        offset = obj_priv->gtt_offset + args->offset;
+        offset = obj->gtt_offset + args->offset;
 
         while (remain > 0) {
                 /* Operation in this page
@@ -814,39 +772,58 @@ out_unpin_pages:
  * copy_from_user into the kmapped pages backing the object.
  */
 static int
-i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pwrite_fast(struct drm_device *dev,
+                           struct drm_i915_gem_object *obj,
                            struct drm_i915_gem_pwrite *args,
-                           struct drm_file *file_priv)
+                           struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         ssize_t remain;
-        loff_t offset, page_base;
+        loff_t offset;
         char __user *user_data;
         int page_offset, page_length;
 
         user_data = (char __user *) (uintptr_t) args->data_ptr;
         remain = args->size;
 
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
-        obj_priv->dirty = 1;
+        obj->dirty = 1;
 
         while (remain > 0) {
+                struct page *page;
+                char *vaddr;
+                int ret;
+
                 /* Operation in this page
                  *
-                 * page_base = page offset within aperture
                  * page_offset = offset within page
                  * page_length = bytes to copy for this page
                  */
-                page_base = (offset & ~(PAGE_SIZE-1));
                 page_offset = offset & (PAGE_SIZE-1);
                 page_length = remain;
                 if ((page_offset + remain) > PAGE_SIZE)
                         page_length = PAGE_SIZE - page_offset;
 
-                if (fast_shmem_write(obj_priv->pages,
-                                       page_base, page_offset,
-                                       user_data, page_length))
+                page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
+                                           GFP_HIGHUSER | __GFP_RECLAIMABLE);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
+                vaddr = kmap_atomic(page, KM_USER0);
+                ret = __copy_from_user_inatomic(vaddr + page_offset,
+                                                user_data,
+                                                page_length);
+                kunmap_atomic(vaddr, KM_USER0);
+
+                set_page_dirty(page);
+                mark_page_accessed(page);
+                page_cache_release(page);
+
+                /* If we get a fault while copying data, then (presumably) our
+                 * source page isn't available.  Return the error and we'll
+                 * retry in the slow path.
+                 */
+                if (ret)
                         return -EFAULT;
 
                 remain -= page_length;
@@ -865,17 +842,18 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
  * struct_mutex is held.
  */
 static int
-i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_shmem_pwrite_slow(struct drm_device *dev,
+                           struct drm_i915_gem_object *obj,
                            struct drm_i915_gem_pwrite *args,
-                           struct drm_file *file_priv)
+                           struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         struct mm_struct *mm = current->mm;
         struct page **user_pages;
         ssize_t remain;
         loff_t offset, pinned_pages, i;
         loff_t first_data_page, last_data_page, num_pages;
-        int shmem_page_index, shmem_page_offset;
+        int shmem_page_offset;
         int data_page_index,  data_page_offset;
         int page_length;
         int ret;
@@ -913,20 +891,19 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 
         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 
-        obj_priv = to_intel_bo(obj);
         offset = args->offset;
-        obj_priv->dirty = 1;
+        obj->dirty = 1;
 
         while (remain > 0) {
+                struct page *page;
+
                 /* Operation in this page
                  *
-                 * shmem_page_index = page number within shmem file
                  * shmem_page_offset = offset within page in shmem file
                  * data_page_index = page number in get_user_pages return
                  * data_page_offset = offset with data_page_index page.
                  * page_length = bytes to copy for this page
                  */
-                shmem_page_index = offset / PAGE_SIZE;
                 shmem_page_offset = offset & ~PAGE_MASK;
                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
                 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -937,21 +914,32 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 if ((data_page_offset + page_length) > PAGE_SIZE)
                         page_length = PAGE_SIZE - data_page_offset;
 
+                page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
+                                           GFP_HIGHUSER | __GFP_RECLAIMABLE);
+                if (IS_ERR(page)) {
+                        ret = PTR_ERR(page);
+                        goto out;
+                }
+
                 if (do_bit17_swizzling) {
-                        slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
+                        slow_shmem_bit17_copy(page,
                                               shmem_page_offset,
                                               user_pages[data_page_index],
                                               data_page_offset,
                                               page_length,
                                               0);
                 } else {
-                        slow_shmem_copy(obj_priv->pages[shmem_page_index],
+                        slow_shmem_copy(page,
                                         shmem_page_offset,
                                         user_pages[data_page_index],
                                         data_page_offset,
                                         page_length);
                 }
 
+                set_page_dirty(page);
+                mark_page_accessed(page);
+                page_cache_release(page);
+
                 remain -= page_length;
                 data_ptr += page_length;
                 offset += page_length;
@@ -975,8 +963,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                       struct drm_file *file)
 {
         struct drm_i915_gem_pwrite *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         if (args->size == 0)
@@ -996,15 +983,15 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
         /* Bounds check destination. */
-        if (args->offset > obj->size || args->size > obj->size - args->offset) {
+        if (args->offset > obj->base.size ||
+            args->size > obj->base.size - args->offset) {
                 ret = -EINVAL;
                 goto out;
         }
@@ -1015,16 +1002,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
          * pread/pwrite currently are reading and writing from the CPU
          * perspective, requiring manual detiling by the client.
          */
-        if (obj_priv->phys_obj)
+        if (obj->phys_obj)
                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
-        else if (obj_priv->tiling_mode == I915_TILING_NONE &&
-                 obj_priv->gtt_space &&
-                 obj->write_domain != I915_GEM_DOMAIN_CPU) {
-                ret = i915_gem_object_pin(obj, 0);
+        else if (obj->gtt_space &&
+                 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
+                ret = i915_gem_object_pin(obj, 0, true);
                 if (ret)
                         goto out;
 
-                ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+                ret = i915_gem_object_set_to_gtt_domain(obj, true);
+                if (ret)
+                        goto out_unpin;
+
+                ret = i915_gem_object_put_fence(obj);
                 if (ret)
                         goto out_unpin;
 
@@ -1035,26 +1025,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 out_unpin:
                 i915_gem_object_unpin(obj);
         } else {
-                ret = i915_gem_object_get_pages_or_evict(obj);
-                if (ret)
-                        goto out;
-
                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
                 if (ret)
-                        goto out_put;
+                        goto out;
 
                 ret = -EFAULT;
                 if (!i915_gem_object_needs_bit17_swizzle(obj))
                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
                 if (ret == -EFAULT)
                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
-
-out_put:
-                i915_gem_object_put_pages(obj);
         }
 
 out:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -1066,12 +1049,10 @@ unlock:
  */
 int
 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
-                          struct drm_file *file_priv)
+                          struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_set_domain *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         uint32_t read_domains = args->read_domains;
         uint32_t write_domain = args->write_domain;
         int ret;
@@ -1096,28 +1077,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
-
-        intel_mark_busy(dev, obj);
 
         if (read_domains & I915_GEM_DOMAIN_GTT) {
                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 
-                /* Update the LRU on the fence for the CPU access that's
-                 * about to occur.
-                 */
-                if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
-                        struct drm_i915_fence_reg *reg =
-                                &dev_priv->fence_regs[obj_priv->fence_reg];
-                        list_move_tail(&reg->lru_list,
-                                       &dev_priv->mm.fence_list);
-                }
-
                 /* Silently promote "you're not bound, there was nothing to do"
                  * to success, since the client was just asking us to
                  * make sure everything was done.
@@ -1128,11 +1096,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
         }
 
-        /* Maintain LRU order of "inactive" objects */
-        if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
-                list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
-
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -1143,10 +1107,10 @@ unlock:
  */
 int
 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
-                      struct drm_file *file_priv)
+                         struct drm_file *file)
 {
         struct drm_i915_gem_sw_finish *args = data;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int ret = 0;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1156,17 +1120,17 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
 
         /* Pinned buffers may be scanout, so flush the cache */
-        if (to_intel_bo(obj)->pin_count)
+        if (obj->pin_count)
                 i915_gem_object_flush_cpu_write_domain(obj);
 
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -1181,8 +1145,9 @@ unlock:
  */
 int
 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                    struct drm_file *file)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_mmap *args = data;
         struct drm_gem_object *obj;
         loff_t offset;
@@ -1191,10 +1156,15 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
         if (!(dev->driver->driver_features & DRIVER_GEM))
                 return -ENODEV;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = drm_gem_object_lookup(dev, file, args->handle);
         if (obj == NULL)
                 return -ENOENT;
 
+        if (obj->size > dev_priv->mm.gtt_mappable_end) {
+                drm_gem_object_unreference_unlocked(obj);
+                return -E2BIG;
+        }
+
         offset = args->offset;
 
         down_write(&current->mm->mmap_sem);
@@ -1229,10 +1199,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  */
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
-        struct drm_gem_object *obj = vma->vm_private_data;
-        struct drm_device *dev = obj->dev;
+        struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         pgoff_t page_offset;
         unsigned long pfn;
         int ret = 0;
@@ -1244,27 +1213,35 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
         /* Now bind it into the GTT if needed */
         mutex_lock(&dev->struct_mutex);
-        if (!obj_priv->gtt_space) {
-                ret = i915_gem_object_bind_to_gtt(obj, 0);
-                if (ret)
-                        goto unlock;
 
-                ret = i915_gem_object_set_to_gtt_domain(obj, write);
+        if (!obj->map_and_fenceable) {
+                ret = i915_gem_object_unbind(obj);
                 if (ret)
                         goto unlock;
         }
-
-        /* Need a new fence register? */
-        if (obj_priv->tiling_mode != I915_TILING_NONE) {
-                ret = i915_gem_object_get_fence_reg(obj, true);
+        if (!obj->gtt_space) {
+                ret = i915_gem_object_bind_to_gtt(obj, 0, true);
                 if (ret)
                         goto unlock;
         }
 
-        if (i915_gem_object_is_inactive(obj_priv))
-                list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
+        ret = i915_gem_object_set_to_gtt_domain(obj, write);
+        if (ret)
+                goto unlock;
+
+        if (obj->tiling_mode == I915_TILING_NONE)
+                ret = i915_gem_object_put_fence(obj);
+        else
+                ret = i915_gem_object_get_fence(obj, NULL, true);
+        if (ret)
+                goto unlock;
+
+        if (i915_gem_object_is_inactive(obj))
+                list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
-        pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
+        obj->fault_mappable = true;
+
+        pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
                 page_offset;
 
         /* Finally, remap it using the new GTT offset */
@@ -1273,11 +1250,12 @@ unlock:
         mutex_unlock(&dev->struct_mutex);
 
         switch (ret) {
+        case -EAGAIN:
+                set_need_resched();
         case 0:
         case -ERESTARTSYS:
                 return VM_FAULT_NOPAGE;
         case -ENOMEM:
-        case -EAGAIN:
                 return VM_FAULT_OOM;
         default:
                 return VM_FAULT_SIGBUS;
@@ -1296,37 +1274,39 @@ unlock:
  * This routine allocates and attaches a fake offset for @obj.
  */
 static int
-i915_gem_create_mmap_offset(struct drm_gem_object *obj)
+i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         struct drm_gem_mm *mm = dev->mm_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct drm_map_list *list;
         struct drm_local_map *map;
         int ret = 0;
 
         /* Set the object up for mmap'ing */
-        list = &obj->map_list;
+        list = &obj->base.map_list;
         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
         if (!list->map)
                 return -ENOMEM;
 
         map = list->map;
         map->type = _DRM_GEM;
-        map->size = obj->size;
+        map->size = obj->base.size;
         map->handle = obj;
 
         /* Get a DRM GEM mmap offset allocated... */
         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
-                                                    obj->size / PAGE_SIZE, 0, 0);
+                                                    obj->base.size / PAGE_SIZE,
+                                                    0, 0);
         if (!list->file_offset_node) {
-                DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
+                DRM_ERROR("failed to allocate offset for bo %d\n",
+                          obj->base.name);
                 ret = -ENOSPC;
                 goto out_free_list;
         }
 
         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
-                                                  obj->size / PAGE_SIZE, 0);
+                                                  obj->base.size / PAGE_SIZE,
+                                                  0);
         if (!list->file_offset_node) {
                 ret = -ENOMEM;
                 goto out_free_list;
@@ -1339,16 +1319,13 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj)
                 goto out_free_mm;
         }
 
-        /* By now we should be all set, any drm_mmap request on the offset
-         * below will get to our mmap & fault handler */
-        obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
-
         return 0;
 
 out_free_mm:
         drm_mm_put_block(list->file_offset_node);
 out_free_list:
         kfree(list->map);
+        list->map = NULL;
 
         return ret;
 }
@@ -1368,38 +1345,51 @@ out_free_list:
  * fixup by i915_gem_fault().
  */
 void
-i915_gem_release_mmap(struct drm_gem_object *obj)
+i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        if (!obj->fault_mappable)
+                return;
+
+        unmap_mapping_range(obj->base.dev->dev_mapping,
+                            (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
+                            obj->base.size, 1);
 
-        if (dev->dev_mapping)
-                unmap_mapping_range(dev->dev_mapping,
-                                    obj_priv->mmap_offset, obj->size, 1);
+        obj->fault_mappable = false;
 }
 
 static void
-i915_gem_free_mmap_offset(struct drm_gem_object *obj)
+i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
         struct drm_gem_mm *mm = dev->mm_private;
-        struct drm_map_list *list;
+        struct drm_map_list *list = &obj->base.map_list;
 
-        list = &obj->map_list;
         drm_ht_remove_item(&mm->offset_hash, &list->hash);
+        drm_mm_put_block(list->file_offset_node);
+        kfree(list->map);
+        list->map = NULL;
+}
 
-        if (list->file_offset_node) {
-                drm_mm_put_block(list->file_offset_node);
-                list->file_offset_node = NULL;
-        }
+static uint32_t
+i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
+        uint32_t size;
 
-        if (list->map) {
-                kfree(list->map);
-                list->map = NULL;
-        }
+        if (INTEL_INFO(dev)->gen >= 4 ||
+            obj->tiling_mode == I915_TILING_NONE)
+                return obj->base.size;
 
-        obj_priv->mmap_offset = 0;
+        /* Previous chips need a power-of-two fence region when tiling */
+        if (INTEL_INFO(dev)->gen == 3)
+                size = 1024*1024;
+        else
+                size = 512*1024;
+
+        while (size < obj->base.size)
+                size <<= 1;
+
+        return size;
 }
 
 /**
@@ -1407,59 +1397,71 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj)
  * @obj: object to check
  *
  * Return the required GTT alignment for an object, taking into account
- * potential fence register mapping if needed.
+ * potential fence register mapping.
  */
 static uint32_t
-i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
+i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int start, i;
+        struct drm_device *dev = obj->base.dev;
 
         /*
          * Minimum alignment is 4k (GTT page size), but might be greater
          * if a fence register is needed for the object.
          */
-        if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
+        if (INTEL_INFO(dev)->gen >= 4 ||
+            obj->tiling_mode == I915_TILING_NONE)
                 return 4096;
 
         /*
          * Previous chips need to be aligned to the size of the smallest
          * fence register that can contain the object.
          */
-        if (INTEL_INFO(dev)->gen == 3)
-                start = 1024*1024;
-        else
-                start = 512*1024;
-
-        for (i = start; i < obj->size; i <<= 1)
-                ;
-
-        return i;
+        return i915_gem_get_gtt_size(obj);
 }
 
 /**
- * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
- * @dev: DRM device
- * @data: GTT mapping ioctl data
- * @file_priv: GEM object info
- *
- * Simply returns the fake offset to userspace so it can mmap it.
- * The mmap call will end up in drm_gem_mmap(), which will set things
- * up so we can get faults in the handler above.
+ * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
+ *                                       unfenced object
+ * @obj: object to check
  *
- * The fault handler will take care of binding the object into the GTT
- * (since it may have been evicted to make room for something), allocating
- * a fence register, and mapping the appropriate aperture address into
- * userspace.
+ * Return the required GTT alignment for an object, only taking into account
+ * unfenced tiled surface requirements.
  */
+static uint32_t
+i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
+        int tile_height;
+
+        /*
+         * Minimum alignment is 4k (GTT page size) for sane hw.
+         */
+        if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
+            obj->tiling_mode == I915_TILING_NONE)
+                return 4096;
+
+        /*
+         * Older chips need unfenced tiled buffers to be aligned to the left
+         * edge of an even tile row (where tile rows are counted as if the bo is
+         * placed in a fenced gtt region).
+         */
+        if (IS_GEN2(dev) ||
+            (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
+                tile_height = 32;
+        else
+                tile_height = 8;
+
+        return tile_height * obj->stride * 2;
+}
+
 int
-i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
+i915_gem_mmap_gtt(struct drm_file *file,
+                  struct drm_device *dev,
+                  uint32_t handle,
+                  uint64_t *offset)
 {
-        struct drm_i915_gem_mmap_gtt *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1469,130 +1471,224 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
-        if (obj_priv->madv != I915_MADV_WILLNEED) {
+        if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
+                ret = -E2BIG;
+                goto unlock;
+        }
+
+        if (obj->madv != I915_MADV_WILLNEED) {
                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
                 ret = -EINVAL;
                 goto out;
         }
 
-        if (!obj_priv->mmap_offset) {
+        if (!obj->base.map_list.map) {
                 ret = i915_gem_create_mmap_offset(obj);
                 if (ret)
                         goto out;
         }
 
-        args->offset = obj_priv->mmap_offset;
-
-        /*
-         * Pull it into the GTT so that we have a page list (makes the
-         * initial fault faster and any subsequent flushing possible).
-         */
-        if (!obj_priv->agp_mem) {
-                ret = i915_gem_object_bind_to_gtt(obj, 0);
-                if (ret)
-                        goto out;
-        }
+        *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
 
 out:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
 }
 
-static void
-i915_gem_object_put_pages(struct drm_gem_object *obj)
+/**
+ * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
+ * @dev: DRM device
+ * @data: GTT mapping ioctl data
+ * @file: GEM object info
+ *
+ * Simply returns the fake offset to userspace so it can mmap it.
+ * The mmap call will end up in drm_gem_mmap(), which will set things
+ * up so we can get faults in the handler above.
+ *
+ * The fault handler will take care of binding the object into the GTT
+ * (since it may have been evicted to make room for something), allocating
+ * a fence register, and mapping the appropriate aperture address into
+ * userspace.
+ */
+int
+i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
+                        struct drm_file *file)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count = obj->size / PAGE_SIZE;
-        int i;
+        struct drm_i915_gem_mmap_gtt *args = data;
 
-        BUG_ON(obj_priv->pages_refcount == 0);
-        BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
+        if (!(dev->driver->driver_features & DRIVER_GEM))
+                return -ENODEV;
 
-        if (--obj_priv->pages_refcount != 0)
-                return;
+        return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
+}
 
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
-                i915_gem_object_save_bit_17_swizzle(obj);
 
-        if (obj_priv->madv == I915_MADV_DONTNEED)
-                obj_priv->dirty = 0;
+static int
+i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
+                              gfp_t gfpmask)
+{
+        int page_count, i;
+        struct address_space *mapping;
+        struct inode *inode;
+        struct page *page;
 
-        for (i = 0; i < page_count; i++) {
-                if (obj_priv->dirty)
-                        set_page_dirty(obj_priv->pages[i]);
+        /* Get the list of pages out of our struct file.  They'll be pinned
+         * at this point until we release them.
+         */
+        page_count = obj->base.size / PAGE_SIZE;
+        BUG_ON(obj->pages != NULL);
+        obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
+        if (obj->pages == NULL)
+                return -ENOMEM;
 
-                if (obj_priv->madv == I915_MADV_WILLNEED)
-                        mark_page_accessed(obj_priv->pages[i]);
+        inode = obj->base.filp->f_path.dentry->d_inode;
+        mapping = inode->i_mapping;
+        for (i = 0; i < page_count; i++) {
+                page = read_cache_page_gfp(mapping, i,
+                                           GFP_HIGHUSER |
+                                           __GFP_COLD |
+                                           __GFP_RECLAIMABLE |
+                                           gfpmask);
+                if (IS_ERR(page))
+                        goto err_pages;
 
-                page_cache_release(obj_priv->pages[i]);
+                obj->pages[i] = page;
         }
-        obj_priv->dirty = 0;
 
-        drm_free_large(obj_priv->pages);
-        obj_priv->pages = NULL;
+        if (obj->tiling_mode != I915_TILING_NONE)
+                i915_gem_object_do_bit_17_swizzle(obj);
+
+        return 0;
+
+err_pages:
+        while (i--)
+                page_cache_release(obj->pages[i]);
+
+        drm_free_large(obj->pages);
+        obj->pages = NULL;
+        return PTR_ERR(page);
 }
 
-static uint32_t
-i915_gem_next_request_seqno(struct drm_device *dev,
-                            struct intel_ring_buffer *ring)
+static void
+i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        int page_count = obj->base.size / PAGE_SIZE;
+        int i;
+
+        BUG_ON(obj->madv == __I915_MADV_PURGED);
+
+        if (obj->tiling_mode != I915_TILING_NONE)
+                i915_gem_object_save_bit_17_swizzle(obj);
+
+        if (obj->madv == I915_MADV_DONTNEED)
+                obj->dirty = 0;
+
+        for (i = 0; i < page_count; i++) {
+                if (obj->dirty)
+                        set_page_dirty(obj->pages[i]);
+
+                if (obj->madv == I915_MADV_WILLNEED)
+                        mark_page_accessed(obj->pages[i]);
 
-        ring->outstanding_lazy_request = true;
-        return dev_priv->next_seqno;
+                page_cache_release(obj->pages[i]);
+        }
+        obj->dirty = 0;
+
+        drm_free_large(obj->pages);
+        obj->pages = NULL;
 }
 
-static void
-i915_gem_object_move_to_active(struct drm_gem_object *obj,
-                               struct intel_ring_buffer *ring)
+void
+i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
+                               struct intel_ring_buffer *ring,
+                               u32 seqno)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
 
         BUG_ON(ring == NULL);
-        obj_priv->ring = ring;
+        obj->ring = ring;
 
         /* Add a reference if we're newly entering the active list. */
-        if (!obj_priv->active) {
-                drm_gem_object_reference(obj);
-                obj_priv->active = 1;
+        if (!obj->active) {
+                drm_gem_object_reference(&obj->base);
+                obj->active = 1;
         }
 
         /* Move from whatever list we were on to the tail of execution. */
-        list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
-        list_move_tail(&obj_priv->ring_list, &ring->active_list);
-        obj_priv->last_rendering_seqno = seqno;
+        list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
+        list_move_tail(&obj->ring_list, &ring->active_list);
+
+        obj->last_rendering_seqno = seqno;
+        if (obj->fenced_gpu_access) {
+                struct drm_i915_fence_reg *reg;
+
+                BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
+
+                obj->last_fenced_seqno = seqno;
+                obj->last_fenced_ring = ring;
+
+                reg = &dev_priv->fence_regs[obj->fence_reg];
+                list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+        }
 }
 
 static void
-i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
+i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        list_del_init(&obj->ring_list);
+        obj->last_rendering_seqno = 0;
+}
+
+static void
+i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 
-        BUG_ON(!obj_priv->active);
-        list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
-        list_del_init(&obj_priv->ring_list);
-        obj_priv->last_rendering_seqno = 0;
+        BUG_ON(!obj->active);
+        list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
+
+        i915_gem_object_move_off_active(obj);
+}
+
+static void
+i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (obj->pin_count != 0)
+                list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
+        else
+                list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
+
+        BUG_ON(!list_empty(&obj->gpu_write_list));
+        BUG_ON(!obj->active);
+        obj->ring = NULL;
+
+        i915_gem_object_move_off_active(obj);
+        obj->fenced_gpu_access = false;
+
+        obj->active = 0;
+        obj->pending_gpu_write = false;
+        drm_gem_object_unreference(&obj->base);
+
+        WARN_ON(i915_verify_lists(dev));
 }
 
 /* Immediately discard the backing storage */
 static void
-i915_gem_object_truncate(struct drm_gem_object *obj)
+i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct inode *inode;
 
         /* Our goal here is to return as much of the memory as
@@ -1601,42 +1697,18 @@ i915_gem_object_truncate(struct drm_gem_object *obj)
          * backing pages, *now*. Here we mirror the actions taken
          * when by shmem_delete_inode() to release the backing store.
          */
-        inode = obj->filp->f_path.dentry->d_inode;
+        inode = obj->base.filp->f_path.dentry->d_inode;
         truncate_inode_pages(inode->i_mapping, 0);
         if (inode->i_op->truncate_range)
                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
 
-        obj_priv->madv = __I915_MADV_PURGED;
+        obj->madv = __I915_MADV_PURGED;
 }
 
 static inline int
-i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
-{
-        return obj_priv->madv == I915_MADV_DONTNEED;
-}
-
-static void
-i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
+i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        if (obj_priv->pin_count != 0)
-                list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
-        else
-                list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
-        list_del_init(&obj_priv->ring_list);
-
-        BUG_ON(!list_empty(&obj_priv->gpu_write_list));
-
-        obj_priv->last_rendering_seqno = 0;
-        obj_priv->ring = NULL;
-        if (obj_priv->active) {
-                obj_priv->active = 0;
-                drm_gem_object_unreference(obj);
-        }
-        WARN_ON(i915_verify_lists(dev));
+        return obj->madv == I915_MADV_DONTNEED;
 }
 
 static void
@@ -1644,37 +1716,27 @@ i915_gem_process_flushing_list(struct drm_device *dev,
                                uint32_t flush_domains,
                                struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv, *next;
+        struct drm_i915_gem_object *obj, *next;
 
-        list_for_each_entry_safe(obj_priv, next,
+        list_for_each_entry_safe(obj, next,
                                  &ring->gpu_write_list,
                                  gpu_write_list) {
-                struct drm_gem_object *obj = &obj_priv->base;
-
-                if (obj->write_domain & flush_domains) {
-                        uint32_t old_write_domain = obj->write_domain;
+                if (obj->base.write_domain & flush_domains) {
+                        uint32_t old_write_domain = obj->base.write_domain;
 
-                        obj->write_domain = 0;
-                        list_del_init(&obj_priv->gpu_write_list);
-                        i915_gem_object_move_to_active(obj, ring);
-
-                        /* update the fence lru list */
-                        if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
-                                struct drm_i915_fence_reg *reg =
-                                        &dev_priv->fence_regs[obj_priv->fence_reg];
-                                list_move_tail(&reg->lru_list,
-                                                &dev_priv->mm.fence_list);
-                        }
+                        obj->base.write_domain = 0;
+                        list_del_init(&obj->gpu_write_list);
+                        i915_gem_object_move_to_active(obj, ring,
+                                                       i915_gem_next_request_seqno(dev, ring));
 
                         trace_i915_gem_object_change_domain(obj,
-                                                            obj->read_domains,
+                                                            obj->base.read_domains,
                                                             old_write_domain);
                 }
         }
 }
 
-uint32_t
+int
 i915_add_request(struct drm_device *dev,
                  struct drm_file *file,
                  struct drm_i915_gem_request *request,
@@ -1684,17 +1746,17 @@ i915_add_request(struct drm_device *dev,
         struct drm_i915_file_private *file_priv = NULL;
         uint32_t seqno;
         int was_empty;
+        int ret;
+
+        BUG_ON(request == NULL);
 
         if (file != NULL)
                 file_priv = file->driver_priv;
 
-        if (request == NULL) {
-                request = kzalloc(sizeof(*request), GFP_KERNEL);
-                if (request == NULL)
-                        return 0;
-        }
+        ret = ring->add_request(ring, &seqno);
+        if (ret)
+            return ret;
 
-        seqno = ring->add_request(dev, ring, 0);
         ring->outstanding_lazy_request = false;
 
         request->seqno = seqno;
@@ -1718,26 +1780,7 @@ i915_add_request(struct drm_device *dev,
                         queue_delayed_work(dev_priv->wq,
                                            &dev_priv->mm.retire_work, HZ);
         }
-        return seqno;
-}
-
-/**
- * Command execution barrier
- *
- * Ensures that all commands in the ring are finished
- * before signalling the CPU
- */
-static void
-i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
-{
-        uint32_t flush_domains = 0;
-
-        /* The sampler always gets flushed on i965 (sigh) */
-        if (INTEL_INFO(dev)->gen >= 4)
-                flush_domains |= I915_GEM_DOMAIN_SAMPLER;
-
-        ring->flush(dev, ring,
-                        I915_GEM_DOMAIN_COMMAND, flush_domains);
+        return 0;
 }
 
 static inline void
@@ -1770,62 +1813,76 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
         }
 
         while (!list_empty(&ring->active_list)) {
-                struct drm_i915_gem_object *obj_priv;
+                struct drm_i915_gem_object *obj;
+
+                obj = list_first_entry(&ring->active_list,
+                                       struct drm_i915_gem_object,
+                                       ring_list);
+
+                obj->base.write_domain = 0;
+                list_del_init(&obj->gpu_write_list);
+                i915_gem_object_move_to_inactive(obj);
+        }
+}
+
+static void i915_gem_reset_fences(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
+
+        for (i = 0; i < 16; i++) {
+                struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
+                struct drm_i915_gem_object *obj = reg->obj;
+
+                if (!obj)
+                        continue;
 
-                obj_priv = list_first_entry(&ring->active_list,
-                                            struct drm_i915_gem_object,
-                                            ring_list);
+                if (obj->tiling_mode)
+                        i915_gem_release_mmap(obj);
 
-                obj_priv->base.write_domain = 0;
-                list_del_init(&obj_priv->gpu_write_list);
-                i915_gem_object_move_to_inactive(&obj_priv->base);
+                reg->obj->fence_reg = I915_FENCE_REG_NONE;
+                reg->obj->fenced_gpu_access = false;
+                reg->obj->last_fenced_seqno = 0;
+                reg->obj->last_fenced_ring = NULL;
+                i915_gem_clear_fence_reg(dev, reg);
         }
 }
 
 void i915_gem_reset(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int i;
 
-        i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
-        i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
-        i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
 
         /* Remove anything from the flushing lists. The GPU cache is likely
          * to be lost on reset along with the data, so simply move the
          * lost bo to the inactive list.
          */
         while (!list_empty(&dev_priv->mm.flushing_list)) {
-                obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
-                                            struct drm_i915_gem_object,
-                                            mm_list);
+                obj= list_first_entry(&dev_priv->mm.flushing_list,
+                                      struct drm_i915_gem_object,
+                                      mm_list);
 
-                obj_priv->base.write_domain = 0;
-                list_del_init(&obj_priv->gpu_write_list);
-                i915_gem_object_move_to_inactive(&obj_priv->base);
+                obj->base.write_domain = 0;
+                list_del_init(&obj->gpu_write_list);
+                i915_gem_object_move_to_inactive(obj);
         }
 
         /* Move everything out of the GPU domains to ensure we do any
          * necessary invalidation upon reuse.
          */
-        list_for_each_entry(obj_priv,
+        list_for_each_entry(obj,
                             &dev_priv->mm.inactive_list,
                             mm_list)
         {
-                obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
+                obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
         }
 
         /* The fence registers are invalidated so clear them out */
-        for (i = 0; i < 16; i++) {
-                struct drm_i915_fence_reg *reg;
-
-                reg = &dev_priv->fence_regs[i];
-                if (!reg->obj)
-                        continue;
-
-                i915_gem_clear_fence_reg(reg->obj);
-        }
+        i915_gem_reset_fences(dev);
 }
 
 /**
@@ -1837,6 +1894,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         uint32_t seqno;
+        int i;
 
         if (!ring->status_page.page_addr ||
             list_empty(&ring->request_list))
@@ -1844,7 +1902,12 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
 
         WARN_ON(i915_verify_lists(dev));
 
-        seqno = ring->get_seqno(dev, ring);
+        seqno = ring->get_seqno(ring);
+
+        for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
+                if (seqno >= ring->sync_seqno[i])
+                        ring->sync_seqno[i] = 0;
+
         while (!list_empty(&ring->request_list)) {
                 struct drm_i915_gem_request *request;
 
@@ -1866,18 +1929,16 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
          * by the ringbuffer to the flushing/inactive lists as appropriate.
          */
         while (!list_empty(&ring->active_list)) {
-                struct drm_gem_object *obj;
-                struct drm_i915_gem_object *obj_priv;
+                struct drm_i915_gem_object *obj;
 
-                obj_priv = list_first_entry(&ring->active_list,
-                                            struct drm_i915_gem_object,
-                                            ring_list);
+                obj= list_first_entry(&ring->active_list,
+                                      struct drm_i915_gem_object,
+                                      ring_list);
 
-                if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
+                if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
                         break;
 
-                obj = &obj_priv->base;
-                if (obj->write_domain != 0)
+                if (obj->base.write_domain != 0)
                         i915_gem_object_move_to_flushing(obj);
                 else
                         i915_gem_object_move_to_inactive(obj);
@@ -1885,7 +1946,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
 
         if (unlikely (dev_priv->trace_irq_seqno &&
                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
-                ring->user_irq_put(dev, ring);
+                ring->irq_put(ring);
                 dev_priv->trace_irq_seqno = 0;
         }
 
@@ -1896,24 +1957,24 @@ void
 i915_gem_retire_requests(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int i;
 
         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
-            struct drm_i915_gem_object *obj_priv, *tmp;
+            struct drm_i915_gem_object *obj, *next;
 
             /* We must be careful that during unbind() we do not
              * accidentally infinitely recurse into retire requests.
              * Currently:
              *   retire -> free -> unbind -> wait -> retire_ring
              */
-            list_for_each_entry_safe(obj_priv, tmp,
+            list_for_each_entry_safe(obj, next,
                                      &dev_priv->mm.deferred_free_list,
                                      mm_list)
-                    i915_gem_free_object_tail(&obj_priv->base);
+                    i915_gem_free_object_tail(obj);
         }
 
-        i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
-        i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
-        i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]);
 }
 
 static void
@@ -1921,6 +1982,8 @@ i915_gem_retire_work_handler(struct work_struct *work)
 {
         drm_i915_private_t *dev_priv;
         struct drm_device *dev;
+        bool idle;
+        int i;
 
         dev_priv = container_of(work, drm_i915_private_t,
                                 mm.retire_work.work);
@@ -1934,11 +1997,31 @@ i915_gem_retire_work_handler(struct work_struct *work)
 
         i915_gem_retire_requests(dev);
 
-        if (!dev_priv->mm.suspended &&
-                (!list_empty(&dev_priv->render_ring.request_list) ||
-                 !list_empty(&dev_priv->bsd_ring.request_list) ||
-                 !list_empty(&dev_priv->blt_ring.request_list)))
+        /* Send a periodic flush down the ring so we don't hold onto GEM
+         * objects indefinitely.
+         */
+        idle = true;
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct intel_ring_buffer *ring = &dev_priv->ring[i];
+
+                if (!list_empty(&ring->gpu_write_list)) {
+                        struct drm_i915_gem_request *request;
+                        int ret;
+
+                        ret = i915_gem_flush_ring(dev, ring, 0,
+                                                  I915_GEM_GPU_DOMAINS);
+                        request = kzalloc(sizeof(*request), GFP_KERNEL);
+                        if (ret || request == NULL ||
+                            i915_add_request(dev, NULL, request, ring))
+                            kfree(request);
+                }
+
+                idle &= list_empty(&ring->request_list);
+        }
+
+        if (!dev_priv->mm.suspended && !idle)
                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+
         mutex_unlock(&dev->struct_mutex);
 }
 
@@ -1955,14 +2038,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
         if (atomic_read(&dev_priv->mm.wedged))
                 return -EAGAIN;
 
-        if (ring->outstanding_lazy_request) {
-                seqno = i915_add_request(dev, NULL, NULL, ring);
-                if (seqno == 0)
+        if (seqno == ring->outstanding_lazy_request) {
+                struct drm_i915_gem_request *request;
+
+                request = kzalloc(sizeof(*request), GFP_KERNEL);
+                if (request == NULL)
                         return -ENOMEM;
+
+                ret = i915_add_request(dev, NULL, request, ring);
+                if (ret) {
+                        kfree(request);
+                        return ret;
+                }
+
+                seqno = request->seqno;
         }
-        BUG_ON(seqno == dev_priv->next_seqno);
 
-        if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
+        if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
                 if (HAS_PCH_SPLIT(dev))
                         ier = I915_READ(DEIER) | I915_READ(GTIER);
                 else
@@ -1976,21 +2068,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
 
                 trace_i915_gem_request_wait_begin(dev, seqno);
 
-                ring->waiting_gem_seqno = seqno;
-                ring->user_irq_get(dev, ring);
-                if (interruptible)
-                        ret = wait_event_interruptible(ring->irq_queue,
-                                i915_seqno_passed(
-                                        ring->get_seqno(dev, ring), seqno)
-                                || atomic_read(&dev_priv->mm.wedged));
-                else
-                        wait_event(ring->irq_queue,
-                                i915_seqno_passed(
-                                        ring->get_seqno(dev, ring), seqno)
-                                || atomic_read(&dev_priv->mm.wedged));
+                ring->waiting_seqno = seqno;
+                if (ring->irq_get(ring)) {
+                        if (interruptible)
+                                ret = wait_event_interruptible(ring->irq_queue,
+                                                               i915_seqno_passed(ring->get_seqno(ring), seqno)
+                                                               || atomic_read(&dev_priv->mm.wedged));
+                        else
+                                wait_event(ring->irq_queue,
+                                           i915_seqno_passed(ring->get_seqno(ring), seqno)
+                                           || atomic_read(&dev_priv->mm.wedged));
 
-                ring->user_irq_put(dev, ring);
-                ring->waiting_gem_seqno = 0;
+                        ring->irq_put(ring);
+                } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
+                                                      seqno) ||
+                                    atomic_read(&dev_priv->mm.wedged), 3000))
+                        ret = -EBUSY;
+                ring->waiting_seqno = 0;
 
                 trace_i915_gem_request_wait_end(dev, seqno);
         }
@@ -1999,7 +2093,7 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
 
         if (ret && ret != -ERESTARTSYS)
                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
-                          __func__, ret, seqno, ring->get_seqno(dev, ring),
+                          __func__, ret, seqno, ring->get_seqno(ring),
                           dev_priv->next_seqno);
 
         /* Directly dispatch request retiring.  While we have the work queue
@@ -2024,70 +2118,30 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno,
         return i915_do_wait_request(dev, seqno, 1, ring);
 }
 
-static void
-i915_gem_flush_ring(struct drm_device *dev,
-                    struct drm_file *file_priv,
-                    struct intel_ring_buffer *ring,
-                    uint32_t invalidate_domains,
-                    uint32_t flush_domains)
-{
-        ring->flush(dev, ring, invalidate_domains, flush_domains);
-        i915_gem_process_flushing_list(dev, flush_domains, ring);
-}
-
-static void
-i915_gem_flush(struct drm_device *dev,
-               struct drm_file *file_priv,
-               uint32_t invalidate_domains,
-               uint32_t flush_domains,
-               uint32_t flush_rings)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-
-        if (flush_domains & I915_GEM_DOMAIN_CPU)
-                drm_agp_chipset_flush(dev);
-
-        if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
-                if (flush_rings & RING_RENDER)
-                        i915_gem_flush_ring(dev, file_priv,
-                                            &dev_priv->render_ring,
-                                            invalidate_domains, flush_domains);
-                if (flush_rings & RING_BSD)
-                        i915_gem_flush_ring(dev, file_priv,
-                                            &dev_priv->bsd_ring,
-                                            invalidate_domains, flush_domains);
-                if (flush_rings & RING_BLT)
-                        i915_gem_flush_ring(dev, file_priv,
-                                            &dev_priv->blt_ring,
-                                            invalidate_domains, flush_domains);
-        }
-}
-
 /**
  * Ensures that all rendering to the object has completed and the object is
  * safe to unbind from the GTT or access from the CPU.
  */
-static int
-i915_gem_object_wait_rendering(struct drm_gem_object *obj,
+int
+i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
                                bool interruptible)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
         int ret;
 
         /* This function only exists to support waiting for existing rendering,
          * not for emitting required flushes.
          */
-        BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
+        BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
 
         /* If there is rendering queued on the buffer being evicted, wait for
          * it.
          */
-        if (obj_priv->active) {
+        if (obj->active) {
                 ret = i915_do_wait_request(dev,
-                                           obj_priv->last_rendering_seqno,
+                                           obj->last_rendering_seqno,
                                            interruptible,
-                                           obj_priv->ring);
+                                           obj->ring);
                 if (ret)
                         return ret;
         }
@@ -2099,17 +2153,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  * Unbinds an object from the GTT aperture.
  */
 int
-i915_gem_object_unbind(struct drm_gem_object *obj)
+i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret = 0;
 
-        if (obj_priv->gtt_space == NULL)
+        if (obj->gtt_space == NULL)
                 return 0;
 
-        if (obj_priv->pin_count != 0) {
+        if (obj->pin_count != 0) {
                 DRM_ERROR("Attempting to unbind pinned buffer\n");
                 return -EINVAL;
         }
@@ -2132,27 +2183,27 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
          */
         if (ret) {
                 i915_gem_clflush_object(obj);
-                obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
+                obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
         }
 
         /* release the fence reg _after_ flushing */
-        if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-                i915_gem_clear_fence_reg(obj);
-
-        drm_unbind_agp(obj_priv->agp_mem);
-        drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
+        ret = i915_gem_object_put_fence(obj);
+        if (ret == -ERESTARTSYS)
+                return ret;
 
-        i915_gem_object_put_pages(obj);
-        BUG_ON(obj_priv->pages_refcount);
+        i915_gem_gtt_unbind_object(obj);
+        i915_gem_object_put_pages_gtt(obj);
 
-        i915_gem_info_remove_gtt(dev_priv, obj->size);
-        list_del_init(&obj_priv->mm_list);
+        list_del_init(&obj->gtt_list);
+        list_del_init(&obj->mm_list);
+        /* Avoid an unnecessary call to unbind on rebind. */
+        obj->map_and_fenceable = true;
 
-        drm_mm_put_block(obj_priv->gtt_space);
-        obj_priv->gtt_space = NULL;
-        obj_priv->gtt_offset = 0;
+        drm_mm_put_block(obj->gtt_space);
+        obj->gtt_space = NULL;
+        obj->gtt_offset = 0;
 
-        if (i915_gem_object_is_purgeable(obj_priv))
+        if (i915_gem_object_is_purgeable(obj))
                 i915_gem_object_truncate(obj);
 
         trace_i915_gem_object_unbind(obj);
@@ -2160,14 +2211,37 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
         return ret;
 }
 
+int
+i915_gem_flush_ring(struct drm_device *dev,
+                    struct intel_ring_buffer *ring,
+                    uint32_t invalidate_domains,
+                    uint32_t flush_domains)
+{
+        int ret;
+
+        ret = ring->flush(ring, invalidate_domains, flush_domains);
+        if (ret)
+                return ret;
+
+        i915_gem_process_flushing_list(dev, flush_domains, ring);
+        return 0;
+}
+
 static int i915_ring_idle(struct drm_device *dev,
                           struct intel_ring_buffer *ring)
 {
+        int ret;
+
         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
                 return 0;
 
-        i915_gem_flush_ring(dev, NULL, ring,
-                            I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+        if (!list_empty(&ring->gpu_write_list)) {
+                ret = i915_gem_flush_ring(dev, ring,
+                                    I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+                if (ret)
+                        return ret;
+        }
+
         return i915_wait_request(dev,
                                  i915_gem_next_request_seqno(dev, ring),
                                  ring);
@@ -2178,7 +2252,7 @@ i915_gpu_idle(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         bool lists_empty;
-        int ret;
+        int ret, i;
 
         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
                        list_empty(&dev_priv->mm.active_list));
@@ -2186,258 +2260,305 @@ i915_gpu_idle(struct drm_device *dev)
                 return 0;
 
         /* Flush everything onto the inactive list. */
-        ret = i915_ring_idle(dev, &dev_priv->render_ring);
-        if (ret)
-                return ret;
-
-        ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
-        if (ret)
-                return ret;
-
-        ret = i915_ring_idle(dev, &dev_priv->blt_ring);
-        if (ret)
-                return ret;
-
-        return 0;
-}
-
-static int
-i915_gem_object_get_pages(struct drm_gem_object *obj,
-                          gfp_t gfpmask)
-{
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count, i;
-        struct address_space *mapping;
-        struct inode *inode;
-        struct page *page;
-
-        BUG_ON(obj_priv->pages_refcount
-                        == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
-
-        if (obj_priv->pages_refcount++ != 0)
-                return 0;
-
-        /* Get the list of pages out of our struct file.  They'll be pinned
-         * at this point until we release them.
-         */
-        page_count = obj->size / PAGE_SIZE;
-        BUG_ON(obj_priv->pages != NULL);
-        obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
-        if (obj_priv->pages == NULL) {
-                obj_priv->pages_refcount--;
-                return -ENOMEM;
-        }
-
-        inode = obj->filp->f_path.dentry->d_inode;
-        mapping = inode->i_mapping;
-        for (i = 0; i < page_count; i++) {
-                page = read_cache_page_gfp(mapping, i,
-                                           GFP_HIGHUSER |
-                                           __GFP_COLD |
-                                           __GFP_RECLAIMABLE |
-                                           gfpmask);
-                if (IS_ERR(page))
-                        goto err_pages;
-
-                obj_priv->pages[i] = page;
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                ret = i915_ring_idle(dev, &dev_priv->ring[i]);
+                if (ret)
+                        return ret;
         }
 
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
-                i915_gem_object_do_bit_17_swizzle(obj);
-
         return 0;
-
-err_pages:
-        while (i--)
-                page_cache_release(obj_priv->pages[i]);
-
-        drm_free_large(obj_priv->pages);
-        obj_priv->pages = NULL;
-        obj_priv->pages_refcount--;
-        return PTR_ERR(page);
 }
 
-static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
+                                       struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        int regnum = obj->fence_reg;
         uint64_t val;
 
-        val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
-                    0xfffff000) << 32;
-        val |= obj_priv->gtt_offset & 0xfffff000;
-        val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
+        val = (uint64_t)((obj->gtt_offset + size - 4096) &
+                         0xfffff000) << 32;
+        val |= obj->gtt_offset & 0xfffff000;
+        val |= (uint64_t)((obj->stride / 128) - 1) <<
                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
         val |= I965_FENCE_REG_VALID;
 
-        I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 6);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
+                intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
+                intel_ring_emit(pipelined, (u32)val);
+                intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
+                intel_ring_emit(pipelined, (u32)(val >> 32));
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
+
+        return 0;
 }
 
-static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
+                                struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        int regnum = obj->fence_reg;
         uint64_t val;
 
-        val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
+        val = (uint64_t)((obj->gtt_offset + size - 4096) &
                     0xfffff000) << 32;
-        val |= obj_priv->gtt_offset & 0xfffff000;
-        val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        val |= obj->gtt_offset & 0xfffff000;
+        val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
         val |= I965_FENCE_REG_VALID;
 
-        I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 6);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
+                intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
+                intel_ring_emit(pipelined, (u32)val);
+                intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
+                intel_ring_emit(pipelined, (u32)(val >> 32));
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
+
+        return 0;
 }
 
-static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
+                                struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        u32 fence_reg, val, pitch_val;
         int tile_width;
-        uint32_t fence_reg, val;
-        uint32_t pitch_val;
 
-        if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
-            (obj_priv->gtt_offset & (obj->size - 1))) {
-                WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
-                     __func__, obj_priv->gtt_offset, obj->size);
-                return;
-        }
+        if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
+                 (size & -size) != size ||
+                 (obj->gtt_offset & (size - 1)),
+                 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
+                 obj->gtt_offset, obj->map_and_fenceable, size))
+                return -EINVAL;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y &&
-            HAS_128_BYTE_Y_TILING(dev))
+        if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
                 tile_width = 128;
         else
                 tile_width = 512;
 
         /* Note: pitch better be a power of two tile widths */
-        pitch_val = obj_priv->stride / tile_width;
+        pitch_val = obj->stride / tile_width;
         pitch_val = ffs(pitch_val) - 1;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y &&
-            HAS_128_BYTE_Y_TILING(dev))
-                WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
-        else
-                WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
-
-        val = obj_priv->gtt_offset;
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        val = obj->gtt_offset;
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-        val |= I915_FENCE_SIZE_BITS(obj->size);
+        val |= I915_FENCE_SIZE_BITS(size);
         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
         val |= I830_FENCE_REG_VALID;
 
-        if (regnum < 8)
-                fence_reg = FENCE_REG_830_0 + (regnum * 4);
+        fence_reg = obj->fence_reg;
+        if (fence_reg < 8)
+                fence_reg = FENCE_REG_830_0 + fence_reg * 4;
         else
-                fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
-        I915_WRITE(fence_reg, val);
+                fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
+
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 4);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
+                intel_ring_emit(pipelined, fence_reg);
+                intel_ring_emit(pipelined, val);
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE(fence_reg, val);
+
+        return 0;
 }
 
-static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
+static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
+                                struct intel_ring_buffer *pipelined)
 {
-        struct drm_gem_object *obj = reg->obj;
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int regnum = obj_priv->fence_reg;
+        u32 size = obj->gtt_space->size;
+        int regnum = obj->fence_reg;
         uint32_t val;
         uint32_t pitch_val;
-        uint32_t fence_size_bits;
 
-        if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
-            (obj_priv->gtt_offset & (obj->size - 1))) {
-                WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
-                     __func__, obj_priv->gtt_offset);
-                return;
-        }
+        if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
+                 (size & -size) != size ||
+                 (obj->gtt_offset & (size - 1)),
+                 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
+                 obj->gtt_offset, size))
+                return -EINVAL;
 
-        pitch_val = obj_priv->stride / 128;
+        pitch_val = obj->stride / 128;
         pitch_val = ffs(pitch_val) - 1;
-        WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
 
-        val = obj_priv->gtt_offset;
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        val = obj->gtt_offset;
+        if (obj->tiling_mode == I915_TILING_Y)
                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-        fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
-        WARN_ON(fence_size_bits & ~0x00000f00);
-        val |= fence_size_bits;
+        val |= I830_FENCE_SIZE_BITS(size);
         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
         val |= I830_FENCE_REG_VALID;
 
-        I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
+        if (pipelined) {
+                int ret = intel_ring_begin(pipelined, 4);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(pipelined, MI_NOOP);
+                intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
+                intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
+                intel_ring_emit(pipelined, val);
+                intel_ring_advance(pipelined);
+        } else
+                I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
+
+        return 0;
 }
 
-static int i915_find_fence_reg(struct drm_device *dev,
-                               bool interruptible)
+static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
+{
+        return i915_seqno_passed(ring->get_seqno(ring), seqno);
+}
+
+static int
+i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
+                            struct intel_ring_buffer *pipelined,
+                            bool interruptible)
+{
+        int ret;
+
+        if (obj->fenced_gpu_access) {
+                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                        ret = i915_gem_flush_ring(obj->base.dev,
+                                                  obj->last_fenced_ring,
+                                                  0, obj->base.write_domain);
+                        if (ret)
+                                return ret;
+                }
+
+                obj->fenced_gpu_access = false;
+        }
+
+        if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
+                if (!ring_passed_seqno(obj->last_fenced_ring,
+                                       obj->last_fenced_seqno)) {
+                        ret = i915_do_wait_request(obj->base.dev,
+                                                   obj->last_fenced_seqno,
+                                                   interruptible,
+                                                   obj->last_fenced_ring);
+                        if (ret)
+                                return ret;
+                }
+
+                obj->last_fenced_seqno = 0;
+                obj->last_fenced_ring = NULL;
+        }
+
+        /* Ensure that all CPU reads are completed before installing a fence
+         * and all writes before removing the fence.
+         */
+        if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
+                mb();
+
+        return 0;
+}
+
+int
+i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
+{
+        int ret;
+
+        if (obj->tiling_mode)
+                i915_gem_release_mmap(obj);
+
+        ret = i915_gem_object_flush_fence(obj, NULL, true);
+        if (ret)
+                return ret;
+
+        if (obj->fence_reg != I915_FENCE_REG_NONE) {
+                struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+                i915_gem_clear_fence_reg(obj->base.dev,
+                                         &dev_priv->fence_regs[obj->fence_reg]);
+
+                obj->fence_reg = I915_FENCE_REG_NONE;
+        }
+
+        return 0;
+}
+
+static struct drm_i915_fence_reg *
+i915_find_fence_reg(struct drm_device *dev,
+                    struct intel_ring_buffer *pipelined)
 {
-        struct drm_i915_fence_reg *reg = NULL;
-        struct drm_i915_gem_object *obj_priv = NULL;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj = NULL;
-        int i, avail, ret;
+        struct drm_i915_fence_reg *reg, *first, *avail;
+        int i;
 
         /* First try to find a free reg */
-        avail = 0;
+        avail = NULL;
         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
                 reg = &dev_priv->fence_regs[i];
                 if (!reg->obj)
-                        return i;
+                        return reg;
 
-                obj_priv = to_intel_bo(reg->obj);
-                if (!obj_priv->pin_count)
-                    avail++;
+                if (!reg->obj->pin_count)
+                        avail = reg;
         }
 
-        if (avail == 0)
-                return -ENOSPC;
+        if (avail == NULL)
+                return NULL;
 
         /* None available, try to steal one or wait for a user to finish */
-        i = I915_FENCE_REG_NONE;
-        list_for_each_entry(reg, &dev_priv->mm.fence_list,
-                            lru_list) {
-                obj = reg->obj;
-                obj_priv = to_intel_bo(obj);
-
-                if (obj_priv->pin_count)
+        avail = first = NULL;
+        list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
+                if (reg->obj->pin_count)
                         continue;
 
-                /* found one! */
-                i = obj_priv->fence_reg;
-                break;
-        }
+                if (first == NULL)
+                        first = reg;
 
-        BUG_ON(i == I915_FENCE_REG_NONE);
+                if (!pipelined ||
+                    !reg->obj->last_fenced_ring ||
+                    reg->obj->last_fenced_ring == pipelined) {
+                        avail = reg;
+                        break;
+                }
+        }
 
-        /* We only have a reference on obj from the active list. put_fence_reg
-         * might drop that one, causing a use-after-free in it. So hold a
-         * private reference to obj like the other callers of put_fence_reg
-         * (set_tiling ioctl) do. */
-        drm_gem_object_reference(obj);
-        ret = i915_gem_object_put_fence_reg(obj, interruptible);
-        drm_gem_object_unreference(obj);
-        if (ret != 0)
-                return ret;
+        if (avail == NULL)
+                avail = first;
 
-        return i;
+        return avail;
 }
 
 /**
- * i915_gem_object_get_fence_reg - set up a fence reg for an object
+ * i915_gem_object_get_fence - set up a fence reg for an object
  * @obj: object to map through a fence reg
+ * @pipelined: ring on which to queue the change, or NULL for CPU access
+ * @interruptible: must we wait uninterruptibly for the register to retire?
  *
  * When mapping objects through the GTT, userspace wants to be able to write
  * to them without having to worry about swizzling if the object is tiled.
@@ -2449,72 +2570,141 @@ static int i915_find_fence_reg(struct drm_device *dev,
  * and tiling format.
  */
 int
-i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
-                              bool interruptible)
+i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
+                          struct intel_ring_buffer *pipelined,
+                          bool interruptible)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        struct drm_i915_fence_reg *reg = NULL;
+        struct drm_i915_fence_reg *reg;
         int ret;
 
-        /* Just update our place in the LRU if our fence is getting used. */
-        if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
-                reg = &dev_priv->fence_regs[obj_priv->fence_reg];
+        /* XXX disable pipelining. There are bugs. Shocking. */
+        pipelined = NULL;
+
+        /* Just update our place in the LRU if our fence is getting reused. */
+        if (obj->fence_reg != I915_FENCE_REG_NONE) {
+                reg = &dev_priv->fence_regs[obj->fence_reg];
                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+
+                if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
+                        pipelined = NULL;
+
+                if (!pipelined) {
+                        if (reg->setup_seqno) {
+                                if (!ring_passed_seqno(obj->last_fenced_ring,
+                                                       reg->setup_seqno)) {
+                                        ret = i915_do_wait_request(obj->base.dev,
+                                                                   reg->setup_seqno,
+                                                                   interruptible,
+                                                                   obj->last_fenced_ring);
+                                        if (ret)
+                                                return ret;
+                                }
+
+                                reg->setup_seqno = 0;
+                        }
+                } else if (obj->last_fenced_ring &&
+                           obj->last_fenced_ring != pipelined) {
+                        ret = i915_gem_object_flush_fence(obj,
+                                                          pipelined,
+                                                          interruptible);
+                        if (ret)
+                                return ret;
+                } else if (obj->tiling_changed) {
+                        if (obj->fenced_gpu_access) {
+                                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                                        ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
+                                                                  0, obj->base.write_domain);
+                                        if (ret)
+                                                return ret;
+                                }
+
+                                obj->fenced_gpu_access = false;
+                        }
+                }
+
+                if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
+                        pipelined = NULL;
+                BUG_ON(!pipelined && reg->setup_seqno);
+
+                if (obj->tiling_changed) {
+                        if (pipelined) {
+                                reg->setup_seqno =
+                                        i915_gem_next_request_seqno(dev, pipelined);
+                                obj->last_fenced_seqno = reg->setup_seqno;
+                                obj->last_fenced_ring = pipelined;
+                        }
+                        goto update;
+                }
+
                 return 0;
         }
 
-        switch (obj_priv->tiling_mode) {
-        case I915_TILING_NONE:
-                WARN(1, "allocating a fence for non-tiled object?\n");
-                break;
-        case I915_TILING_X:
-                if (!obj_priv->stride)
-                        return -EINVAL;
-                WARN((obj_priv->stride & (512 - 1)),
-                     "object 0x%08x is X tiled but has non-512B pitch\n",
-                     obj_priv->gtt_offset);
-                break;
-        case I915_TILING_Y:
-                if (!obj_priv->stride)
-                        return -EINVAL;
-                WARN((obj_priv->stride & (128 - 1)),
-                     "object 0x%08x is Y tiled but has non-128B pitch\n",
-                     obj_priv->gtt_offset);
-                break;
-        }
+        reg = i915_find_fence_reg(dev, pipelined);
+        if (reg == NULL)
+                return -ENOSPC;
 
-        ret = i915_find_fence_reg(dev, interruptible);
-        if (ret < 0)
+        ret = i915_gem_object_flush_fence(obj, pipelined, interruptible);
+        if (ret)
                 return ret;
 
-        obj_priv->fence_reg = ret;
-        reg = &dev_priv->fence_regs[obj_priv->fence_reg];
-        list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+        if (reg->obj) {
+                struct drm_i915_gem_object *old = reg->obj;
+
+                drm_gem_object_reference(&old->base);
+
+                if (old->tiling_mode)
+                        i915_gem_release_mmap(old);
+
+                ret = i915_gem_object_flush_fence(old,
+                                                  pipelined,
+                                                  interruptible);
+                if (ret) {
+                        drm_gem_object_unreference(&old->base);
+                        return ret;
+                }
+
+                if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
+                        pipelined = NULL;
+
+                old->fence_reg = I915_FENCE_REG_NONE;
+                old->last_fenced_ring = pipelined;
+                old->last_fenced_seqno =
+                        pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
+
+                drm_gem_object_unreference(&old->base);
+        } else if (obj->last_fenced_seqno == 0)
+                pipelined = NULL;
 
         reg->obj = obj;
+        list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+        obj->fence_reg = reg - dev_priv->fence_regs;
+        obj->last_fenced_ring = pipelined;
 
+        reg->setup_seqno =
+                pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
+        obj->last_fenced_seqno = reg->setup_seqno;
+
+update:
+        obj->tiling_changed = false;
         switch (INTEL_INFO(dev)->gen) {
         case 6:
-                sandybridge_write_fence_reg(reg);
+                ret = sandybridge_write_fence_reg(obj, pipelined);
                 break;
         case 5:
         case 4:
-                i965_write_fence_reg(reg);
+                ret = i965_write_fence_reg(obj, pipelined);
                 break;
         case 3:
-                i915_write_fence_reg(reg);
+                ret = i915_write_fence_reg(obj, pipelined);
                 break;
         case 2:
-                i830_write_fence_reg(reg);
+                ret = i830_write_fence_reg(obj, pipelined);
                 break;
         }
 
-        trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
-                        obj_priv->tiling_mode);
-
-        return 0;
+        return ret;
 }
 
 /**
@@ -2522,154 +2712,125 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
  * @obj: object to clear
  *
  * Zeroes out the fence register itself and clears out the associated
- * data structures in dev_priv and obj_priv.
+ * data structures in dev_priv and obj.
  */
 static void
-i915_gem_clear_fence_reg(struct drm_gem_object *obj)
+i915_gem_clear_fence_reg(struct drm_device *dev,
+                         struct drm_i915_fence_reg *reg)
 {
-        struct drm_device *dev = obj->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        struct drm_i915_fence_reg *reg =
-                &dev_priv->fence_regs[obj_priv->fence_reg];
-        uint32_t fence_reg;
+        uint32_t fence_reg = reg - dev_priv->fence_regs;
 
         switch (INTEL_INFO(dev)->gen) {
         case 6:
-                I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
-                             (obj_priv->fence_reg * 8), 0);
+                I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
                 break;
         case 5:
         case 4:
-                I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
+                I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
                 break;
         case 3:
-                if (obj_priv->fence_reg >= 8)
-                        fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
+                if (fence_reg >= 8)
+                        fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
                 else
         case 2:
-                        fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
+                        fence_reg = FENCE_REG_830_0 + fence_reg * 4;
 
                 I915_WRITE(fence_reg, 0);
                 break;
         }
 
-        reg->obj = NULL;
-        obj_priv->fence_reg = I915_FENCE_REG_NONE;
         list_del_init(&reg->lru_list);
-}
-
-/**
- * i915_gem_object_put_fence_reg - waits on outstanding fenced access
- * to the buffer to finish, and then resets the fence register.
- * @obj: tiled object holding a fence register.
- * @bool: whether the wait upon the fence is interruptible
- *
- * Zeroes out the fence register itself and clears out the associated
- * data structures in dev_priv and obj_priv.
- */
-int
-i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
-                              bool interruptible)
-{
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        struct drm_i915_fence_reg *reg;
-
-        if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
-                return 0;
-
-        /* If we've changed tiling, GTT-mappings of the object
-         * need to re-fault to ensure that the correct fence register
-         * setup is in place.
-         */
-        i915_gem_release_mmap(obj);
-
-        /* On the i915, GPU access to tiled buffers is via a fence,
-         * therefore we must wait for any outstanding access to complete
-         * before clearing the fence.
-         */
-        reg = &dev_priv->fence_regs[obj_priv->fence_reg];
-        if (reg->gpu) {
-                int ret;
-
-                ret = i915_gem_object_flush_gpu_write_domain(obj, true);
-                if (ret)
-                        return ret;
-
-                ret = i915_gem_object_wait_rendering(obj, interruptible);
-                if (ret)
-                        return ret;
-
-                reg->gpu = false;
-        }
-
-        i915_gem_object_flush_gtt_write_domain(obj);
-        i915_gem_clear_fence_reg(obj);
-
-        return 0;
+        reg->obj = NULL;
+        reg->setup_seqno = 0;
 }
 
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
 static int
-i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
+i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
+                            unsigned alignment,
+                            bool map_and_fenceable)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct drm_mm_node *free_space;
-        gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
+        gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
+        u32 size, fence_size, fence_alignment, unfenced_alignment;
+        bool mappable, fenceable;
         int ret;
 
-        if (obj_priv->madv != I915_MADV_WILLNEED) {
+        if (obj->madv != I915_MADV_WILLNEED) {
                 DRM_ERROR("Attempting to bind a purgeable object\n");
                 return -EINVAL;
         }
 
+        fence_size = i915_gem_get_gtt_size(obj);
+        fence_alignment = i915_gem_get_gtt_alignment(obj);
+        unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
+
         if (alignment == 0)
-                alignment = i915_gem_get_gtt_alignment(obj);
-        if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
+                alignment = map_and_fenceable ? fence_alignment :
+                                                unfenced_alignment;
+        if (map_and_fenceable && alignment & (fence_alignment - 1)) {
                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
                 return -EINVAL;
         }
 
+        size = map_and_fenceable ? fence_size : obj->base.size;
+
         /* If the object is bigger than the entire aperture, reject it early
          * before evicting everything in a vain attempt to find space.
          */
-        if (obj->size > dev_priv->mm.gtt_total) {
+        if (obj->base.size >
+            (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
                 return -E2BIG;
         }
 
  search_free:
-        free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
-                                        obj->size, alignment, 0);
-        if (free_space != NULL)
-                obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
-                                                       alignment);
-        if (obj_priv->gtt_space == NULL) {
+        if (map_and_fenceable)
+                free_space =
+                        drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
+                                                    size, alignment, 0,
+                                                    dev_priv->mm.gtt_mappable_end,
+                                                    0);
+        else
+                free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
+                                                size, alignment, 0);
+
+        if (free_space != NULL) {
+                if (map_and_fenceable)
+                        obj->gtt_space =
+                                drm_mm_get_block_range_generic(free_space,
+                                                               size, alignment, 0,
+                                                               dev_priv->mm.gtt_mappable_end,
+                                                               0);
+                else
+                        obj->gtt_space =
+                                drm_mm_get_block(free_space, size, alignment);
+        }
+        if (obj->gtt_space == NULL) {
                 /* If the gtt is empty and we're still having trouble
                  * fitting our object in, we're out of memory.
                  */
-                ret = i915_gem_evict_something(dev, obj->size, alignment);
+                ret = i915_gem_evict_something(dev, size, alignment,
+                                               map_and_fenceable);
                 if (ret)
                         return ret;
 
                 goto search_free;
         }
 
-        ret = i915_gem_object_get_pages(obj, gfpmask);
+        ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
         if (ret) {
-                drm_mm_put_block(obj_priv->gtt_space);
-                obj_priv->gtt_space = NULL;
+                drm_mm_put_block(obj->gtt_space);
+                obj->gtt_space = NULL;
 
                 if (ret == -ENOMEM) {
-                        /* first try to clear up some space from the GTT */
-                        ret = i915_gem_evict_something(dev, obj->size,
-                                                       alignment);
+                        /* first try to reclaim some memory by clearing the GTT */
+                        ret = i915_gem_evict_everything(dev, false);
                         if (ret) {
                                 /* now try to shrink everyone else */
                                 if (gfpmask) {
@@ -2677,7 +2838,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
                                         goto search_free;
                                 }
 
-                                return ret;
+                                return -ENOMEM;
                         }
 
                         goto search_free;
@@ -2686,126 +2847,116 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
                 return ret;
         }
 
-        /* Create an AGP memory structure pointing at our pages, and bind it
-         * into the GTT.
-         */
-        obj_priv->agp_mem = drm_agp_bind_pages(dev,
-                                               obj_priv->pages,
-                                               obj->size >> PAGE_SHIFT,
-                                               obj_priv->gtt_space->start,
-                                               obj_priv->agp_type);
-        if (obj_priv->agp_mem == NULL) {
-                i915_gem_object_put_pages(obj);
-                drm_mm_put_block(obj_priv->gtt_space);
-                obj_priv->gtt_space = NULL;
-
-                ret = i915_gem_evict_something(dev, obj->size, alignment);
-                if (ret)
+        ret = i915_gem_gtt_bind_object(obj);
+        if (ret) {
+                i915_gem_object_put_pages_gtt(obj);
+                drm_mm_put_block(obj->gtt_space);
+                obj->gtt_space = NULL;
+
+                if (i915_gem_evict_everything(dev, false))
                         return ret;
 
                 goto search_free;
         }
 
-        /* keep track of bounds object by adding it to the inactive list */
-        list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
-        i915_gem_info_add_gtt(dev_priv, obj->size);
+        list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
+        list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
         /* Assert that the object is not currently in any GPU domain. As it
          * wasn't in the GTT, there shouldn't be any way it could have been in
          * a GPU cache
          */
-        BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
-        BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
+        BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
+        BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
 
-        obj_priv->gtt_offset = obj_priv->gtt_space->start;
-        trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
+        obj->gtt_offset = obj->gtt_space->start;
 
+        fenceable =
+                obj->gtt_space->size == fence_size &&
+                (obj->gtt_space->start & (fence_alignment -1)) == 0;
+
+        mappable =
+                obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
+
+        obj->map_and_fenceable = mappable && fenceable;
+
+        trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
         return 0;
 }
 
 void
-i915_gem_clflush_object(struct drm_gem_object *obj)
+i915_gem_clflush_object(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
-
         /* If we don't have a page list set up, then we're not pinned
          * to GPU, and we can ignore the cache flush because it'll happen
          * again at bind time.
          */
-        if (obj_priv->pages == NULL)
+        if (obj->pages == NULL)
                 return;
 
         trace_i915_gem_object_clflush(obj);
 
-        drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
+        drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
 }
 
 /** Flushes any GPU write domain for the object if it's dirty. */
 static int
-i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
-                                       bool pipelined)
+i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
-        uint32_t old_write_domain;
+        struct drm_device *dev = obj->base.dev;
 
-        if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
+        if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
                 return 0;
 
         /* Queue the GPU write cache flushing we need. */
-        old_write_domain = obj->write_domain;
-        i915_gem_flush_ring(dev, NULL,
-                            to_intel_bo(obj)->ring,
-                            0, obj->write_domain);
-        BUG_ON(obj->write_domain);
-
-        trace_i915_gem_object_change_domain(obj,
-                                            obj->read_domains,
-                                            old_write_domain);
-
-        if (pipelined)
-                return 0;
-
-        return i915_gem_object_wait_rendering(obj, true);
+        return i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
 }
 
 /** Flushes the GTT write domain for the object if it's dirty. */
 static void
-i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
+i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
 {
         uint32_t old_write_domain;
 
-        if (obj->write_domain != I915_GEM_DOMAIN_GTT)
+        if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
                 return;
 
-        /* No actual flushing is required for the GTT write domain.   Writes
+        /* No actual flushing is required for the GTT write domain.  Writes
          * to it immediately go to main memory as far as we know, so there's
          * no chipset flush.  It also doesn't land in render cache.
+         *
+         * However, we do have to enforce the order so that all writes through
+         * the GTT land before any writes to the device, such as updates to
+         * the GATT itself.
          */
-        old_write_domain = obj->write_domain;
-        obj->write_domain = 0;
+        wmb();
+
+        i915_gem_release_mmap(obj);
+
+        old_write_domain = obj->base.write_domain;
+        obj->base.write_domain = 0;
 
         trace_i915_gem_object_change_domain(obj,
-                                            obj->read_domains,
+                                            obj->base.read_domains,
                                             old_write_domain);
 }
 
 /** Flushes the CPU write domain for the object if it's dirty. */
 static void
-i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
+i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
         uint32_t old_write_domain;
 
-        if (obj->write_domain != I915_GEM_DOMAIN_CPU)
+        if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
                 return;
 
         i915_gem_clflush_object(obj);
-        drm_agp_chipset_flush(dev);
-        old_write_domain = obj->write_domain;
-        obj->write_domain = 0;
+        intel_gtt_chipset_flush();
+        old_write_domain = obj->base.write_domain;
+        obj->base.write_domain = 0;
 
         trace_i915_gem_object_change_domain(obj,
-                                            obj->read_domains,
+                                            obj->base.read_domains,
                                             old_write_domain);
 }
 
@@ -2816,40 +2967,39 @@ i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
  * flushes to occur.
  */
 int
-i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
+i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
         /* Not valid to be called on unbound objects. */
-        if (obj_priv->gtt_space == NULL)
+        if (obj->gtt_space == NULL)
                 return -EINVAL;
 
-        ret = i915_gem_object_flush_gpu_write_domain(obj, false);
-        if (ret != 0)
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
                 return ret;
 
-        i915_gem_object_flush_cpu_write_domain(obj);
-
-        if (write) {
+        if (obj->pending_gpu_write || write) {
                 ret = i915_gem_object_wait_rendering(obj, true);
                 if (ret)
                         return ret;
         }
 
-        old_write_domain = obj->write_domain;
-        old_read_domains = obj->read_domains;
+        i915_gem_object_flush_cpu_write_domain(obj);
+
+        old_write_domain = obj->base.write_domain;
+        old_read_domains = obj->base.read_domains;
 
         /* It should now be out of any other write domains, and we can update
          * the domain values for our changes.
          */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
-        obj->read_domains |= I915_GEM_DOMAIN_GTT;
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
         if (write) {
-                obj->read_domains = I915_GEM_DOMAIN_GTT;
-                obj->write_domain = I915_GEM_DOMAIN_GTT;
-                obj_priv->dirty = 1;
+                obj->base.read_domains = I915_GEM_DOMAIN_GTT;
+                obj->base.write_domain = I915_GEM_DOMAIN_GTT;
+                obj->dirty = 1;
         }
 
         trace_i915_gem_object_change_domain(obj,
@@ -2864,23 +3014,23 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
  * wait, as in modesetting process we're not supposed to be interrupted.
  */
 int
-i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
-                                     bool pipelined)
+i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+                                     struct intel_ring_buffer *pipelined)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         uint32_t old_read_domains;
         int ret;
 
         /* Not valid to be called on unbound objects. */
-        if (obj_priv->gtt_space == NULL)
+        if (obj->gtt_space == NULL)
                 return -EINVAL;
 
-        ret = i915_gem_object_flush_gpu_write_domain(obj, true);
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
         if (ret)
                 return ret;
 
+
         /* Currently, we are always called from an non-interruptible context. */
-        if (!pipelined) {
+        if (pipelined != obj->ring) {
                 ret = i915_gem_object_wait_rendering(obj, false);
                 if (ret)
                         return ret;
@@ -2888,12 +3038,12 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
 
         i915_gem_object_flush_cpu_write_domain(obj);
 
-        old_read_domains = obj->read_domains;
-        obj->read_domains |= I915_GEM_DOMAIN_GTT;
+        old_read_domains = obj->base.read_domains;
+        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
 
         trace_i915_gem_object_change_domain(obj,
                                             old_read_domains,
-                                            obj->write_domain);
+                                            obj->base.write_domain);
 
         return 0;
 }
@@ -2902,14 +3052,19 @@ int
 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
                           bool interruptible)
 {
+        int ret;
+
         if (!obj->active)
                 return 0;
 
-        if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
-                i915_gem_flush_ring(obj->base.dev, NULL, obj->ring,
-                                    0, obj->base.write_domain);
+        if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
+                                          0, obj->base.write_domain);
+                if (ret)
+                        return ret;
+        }
 
-        return i915_gem_object_wait_rendering(&obj->base, interruptible);
+        return i915_gem_object_wait_rendering(obj, interruptible);
 }
 
 /**
@@ -2919,13 +3074,17 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
  * flushes to occur.
  */
 static int
-i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
+i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 {
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
-        ret = i915_gem_object_flush_gpu_write_domain(obj, false);
-        if (ret != 0)
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
+                return ret;
+
+        ret = i915_gem_object_wait_rendering(obj, true);
+        if (ret)
                 return ret;
 
         i915_gem_object_flush_gtt_write_domain(obj);
@@ -2935,33 +3094,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
          */
         i915_gem_object_set_to_full_cpu_read_domain(obj);
 
-        if (write) {
-                ret = i915_gem_object_wait_rendering(obj, true);
-                if (ret)
-                        return ret;
-        }
-
-        old_write_domain = obj->write_domain;
-        old_read_domains = obj->read_domains;
+        old_write_domain = obj->base.write_domain;
+        old_read_domains = obj->base.read_domains;
 
         /* Flush the CPU cache if it's still invalid. */
-        if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
+        if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
                 i915_gem_clflush_object(obj);
 
-                obj->read_domains |= I915_GEM_DOMAIN_CPU;
+                obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
         }
 
         /* It should now be out of any other write domains, and we can update
          * the domain values for our changes.
          */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
         /* If we're writing through the CPU, then the GPU read domains will
          * need to be invalidated at next use.
          */
         if (write) {
-                obj->read_domains = I915_GEM_DOMAIN_CPU;
-                obj->write_domain = I915_GEM_DOMAIN_CPU;
+                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
         }
 
         trace_i915_gem_object_change_domain(obj,
@@ -2971,184 +3124,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
         return 0;
 }
 
-/*
- * Set the next domain for the specified object. This
- * may not actually perform the necessary flushing/invaliding though,
- * as that may want to be batched with other set_domain operations
- *
- * This is (we hope) the only really tricky part of gem. The goal
- * is fairly simple -- track which caches hold bits of the object
- * and make sure they remain coherent. A few concrete examples may
- * help to explain how it works. For shorthand, we use the notation
- * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
- * a pair of read and write domain masks.
- *
- * Case 1: the batch buffer
- *
- *      1. Allocated
- *      2. Written by CPU
- *      3. Mapped to GTT
- *      4. Read by GPU
- *      5. Unmapped from GTT
- *      6. Freed
- *
- *      Let's take these a step at a time
- *
- *      1. Allocated
- *              Pages allocated from the kernel may still have
- *              cache contents, so we set them to (CPU, CPU) always.
- *      2. Written by CPU (using pwrite)
- *              The pwrite function calls set_domain (CPU, CPU) and
- *              this function does nothing (as nothing changes)
- *      3. Mapped by GTT
- *              This function asserts that the object is not
- *              currently in any GPU-based read or write domains
- *      4. Read by GPU
- *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
- *              As write_domain is zero, this function adds in the
- *              current read domains (CPU+COMMAND, 0).
- *              flush_domains is set to CPU.
- *              invalidate_domains is set to COMMAND
- *              clflush is run to get data out of the CPU caches
- *              then i915_dev_set_domain calls i915_gem_flush to
- *              emit an MI_FLUSH and drm_agp_chipset_flush
- *      5. Unmapped from GTT
- *              i915_gem_object_unbind calls set_domain (CPU, CPU)
- *              flush_domains and invalidate_domains end up both zero
- *              so no flushing/invalidating happens
- *      6. Freed
- *              yay, done
- *
- * Case 2: The shared render buffer
- *
- *      1. Allocated
- *      2. Mapped to GTT
- *      3. Read/written by GPU
- *      4. set_domain to (CPU,CPU)
- *      5. Read/written by CPU
- *      6. Read/written by GPU
- *
- *      1. Allocated
- *              Same as last example, (CPU, CPU)
- *      2. Mapped to GTT
- *              Nothing changes (assertions find that it is not in the GPU)
- *      3. Read/written by GPU
- *              execbuffer calls set_domain (RENDER, RENDER)
- *              flush_domains gets CPU
- *              invalidate_domains gets GPU
- *              clflush (obj)
- *              MI_FLUSH and drm_agp_chipset_flush
- *      4. set_domain (CPU, CPU)
- *              flush_domains gets GPU
- *              invalidate_domains gets CPU
- *              wait_rendering (obj) to make sure all drawing is complete.
- *              This will include an MI_FLUSH to get the data from GPU
- *              to memory
- *              clflush (obj) to invalidate the CPU cache
- *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
- *      5. Read/written by CPU
- *              cache lines are loaded and dirtied
- *      6. Read written by GPU
- *              Same as last GPU access
- *
- * Case 3: The constant buffer
- *
- *      1. Allocated
- *      2. Written by CPU
- *      3. Read by GPU
- *      4. Updated (written) by CPU again
- *      5. Read by GPU
- *
- *      1. Allocated
- *              (CPU, CPU)
- *      2. Written by CPU
- *              (CPU, CPU)
- *      3. Read by GPU
- *              (CPU+RENDER, 0)
- *              flush_domains = CPU
- *              invalidate_domains = RENDER
- *              clflush (obj)
- *              MI_FLUSH
- *              drm_agp_chipset_flush
- *      4. Updated (written) by CPU again
- *              (CPU, CPU)
- *              flush_domains = 0 (no previous write domain)
- *              invalidate_domains = 0 (no new read domains)
- *      5. Read by GPU
- *              (CPU+RENDER, 0)
- *              flush_domains = CPU
- *              invalidate_domains = RENDER
- *              clflush (obj)
- *              MI_FLUSH
- *              drm_agp_chipset_flush
- */
-static void
-i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
-                                  struct intel_ring_buffer *ring)
-{
-        struct drm_device               *dev = obj->dev;
-        struct drm_i915_private         *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
-        uint32_t                        invalidate_domains = 0;
-        uint32_t                        flush_domains = 0;
-        uint32_t                        old_read_domains;
-
-        intel_mark_busy(dev, obj);
-
-        /*
-         * If the object isn't moving to a new write domain,
-         * let the object stay in multiple read domains
-         */
-        if (obj->pending_write_domain == 0)
-                obj->pending_read_domains |= obj->read_domains;
-        else
-                obj_priv->dirty = 1;
-
-        /*
-         * Flush the current write domain if
-         * the new read domains don't match. Invalidate
-         * any read domains which differ from the old
-         * write domain
-         */
-        if (obj->write_domain &&
-            (obj->write_domain != obj->pending_read_domains ||
-             obj_priv->ring != ring)) {
-                flush_domains |= obj->write_domain;
-                invalidate_domains |=
-                        obj->pending_read_domains & ~obj->write_domain;
-        }
-        /*
-         * Invalidate any read caches which may have
-         * stale data. That is, any new read domains.
-         */
-        invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
-        if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
-                i915_gem_clflush_object(obj);
-
-        old_read_domains = obj->read_domains;
-
-        /* The actual obj->write_domain will be updated with
-         * pending_write_domain after we emit the accumulated flush for all
-         * of our domain changes in execbuffers (which clears objects'
-         * write_domains).  So if we have a current write domain that we
-         * aren't changing, set pending_write_domain to that.
-         */
-        if (flush_domains == 0 && obj->pending_write_domain == 0)
-                obj->pending_write_domain = obj->write_domain;
-        obj->read_domains = obj->pending_read_domains;
-
-        dev->invalidate_domains |= invalidate_domains;
-        dev->flush_domains |= flush_domains;
-        if (flush_domains & I915_GEM_GPU_DOMAINS)
-                dev_priv->mm.flush_rings |= obj_priv->ring->id;
-        if (invalidate_domains & I915_GEM_GPU_DOMAINS)
-                dev_priv->mm.flush_rings |= ring->id;
-
-        trace_i915_gem_object_change_domain(obj,
-                                            old_read_domains,
-                                            obj->write_domain);
-}
-
 /**
  * Moves the object from a partially CPU read to a full one.
  *
@@ -3156,30 +3131,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
  */
 static void
-i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
+i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        if (!obj_priv->page_cpu_valid)
+        if (!obj->page_cpu_valid)
                 return;
 
         /* If we're partially in the CPU read domain, finish moving it in.
          */
-        if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
+        if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
                 int i;
 
-                for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
-                        if (obj_priv->page_cpu_valid[i])
+                for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
+                        if (obj->page_cpu_valid[i])
                                 continue;
-                        drm_clflush_pages(obj_priv->pages + i, 1);
+                        drm_clflush_pages(obj->pages + i, 1);
                 }
         }
 
         /* Free the page_cpu_valid mappings which are now stale, whether
          * or not we've got I915_GEM_DOMAIN_CPU.
          */
-        kfree(obj_priv->page_cpu_valid);
-        obj_priv->page_cpu_valid = NULL;
+        kfree(obj->page_cpu_valid);
+        obj->page_cpu_valid = NULL;
 }
 
 /**
@@ -3195,354 +3168,65 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
  * flushes to occur.
  */
 static int
-i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
+i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
                                           uint64_t offset, uint64_t size)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         uint32_t old_read_domains;
         int i, ret;
 
-        if (offset == 0 && size == obj->size)
+        if (offset == 0 && size == obj->base.size)
                 return i915_gem_object_set_to_cpu_domain(obj, 0);
 
-        ret = i915_gem_object_flush_gpu_write_domain(obj, false);
-        if (ret != 0)
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
                 return ret;
+
+        ret = i915_gem_object_wait_rendering(obj, true);
+        if (ret)
+                return ret;
+
         i915_gem_object_flush_gtt_write_domain(obj);
 
         /* If we're already fully in the CPU read domain, we're done. */
-        if (obj_priv->page_cpu_valid == NULL &&
-            (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
+        if (obj->page_cpu_valid == NULL &&
+            (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
                 return 0;
 
         /* Otherwise, create/clear the per-page CPU read domain flag if we're
          * newly adding I915_GEM_DOMAIN_CPU
          */
-        if (obj_priv->page_cpu_valid == NULL) {
-                obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
-                                                   GFP_KERNEL);
-                if (obj_priv->page_cpu_valid == NULL)
+        if (obj->page_cpu_valid == NULL) {
+                obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
+                                              GFP_KERNEL);
+                if (obj->page_cpu_valid == NULL)
                         return -ENOMEM;
-        } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
-                memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
+        } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
+                memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
 
         /* Flush the cache on any pages that are still invalid from the CPU's
          * perspective.
          */
         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
              i++) {
-                if (obj_priv->page_cpu_valid[i])
+                if (obj->page_cpu_valid[i])
                         continue;
 
-                drm_clflush_pages(obj_priv->pages + i, 1);
+                drm_clflush_pages(obj->pages + i, 1);
 
-                obj_priv->page_cpu_valid[i] = 1;
+                obj->page_cpu_valid[i] = 1;
         }
 
         /* It should now be out of any other write domains, and we can update
          * the domain values for our changes.
          */
-        BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
-        old_read_domains = obj->read_domains;
-        obj->read_domains |= I915_GEM_DOMAIN_CPU;
+        old_read_domains = obj->base.read_domains;
+        obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
 
         trace_i915_gem_object_change_domain(obj,
                                             old_read_domains,
-                                            obj->write_domain);
-
-        return 0;
-}
-
-/**
- * Pin an object to the GTT and evaluate the relocations landing in it.
- */
-static int
-i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
-                             struct drm_file *file_priv,
-                             struct drm_i915_gem_exec_object2 *entry)
-{
-        struct drm_device *dev = obj->base.dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_relocation_entry __user *user_relocs;
-        struct drm_gem_object *target_obj = NULL;
-        uint32_t target_handle = 0;
-        int i, ret = 0;
-
-        user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
-        for (i = 0; i < entry->relocation_count; i++) {
-                struct drm_i915_gem_relocation_entry reloc;
-                uint32_t target_offset;
-
-                if (__copy_from_user_inatomic(&reloc,
-                                              user_relocs+i,
-                                              sizeof(reloc))) {
-                        ret = -EFAULT;
-                        break;
-                }
-
-                if (reloc.target_handle != target_handle) {
-                        drm_gem_object_unreference(target_obj);
-
-                        target_obj = drm_gem_object_lookup(dev, file_priv,
-                                                           reloc.target_handle);
-                        if (target_obj == NULL) {
-                                ret = -ENOENT;
-                                break;
-                        }
-
-                        target_handle = reloc.target_handle;
-                }
-                target_offset = to_intel_bo(target_obj)->gtt_offset;
-
-#if WATCH_RELOC
-                DRM_INFO("%s: obj %p offset %08x target %d "
-                         "read %08x write %08x gtt %08x "
-                         "presumed %08x delta %08x\n",
-                         __func__,
-                         obj,
-                         (int) reloc.offset,
-                         (int) reloc.target_handle,
-                         (int) reloc.read_domains,
-                         (int) reloc.write_domain,
-                         (int) target_offset,
-                         (int) reloc.presumed_offset,
-                         reloc.delta);
-#endif
-
-                /* The target buffer should have appeared before us in the
-                 * exec_object list, so it should have a GTT space bound by now.
-                 */
-                if (target_offset == 0) {
-                        DRM_ERROR("No GTT space found for object %d\n",
-                                  reloc.target_handle);
-                        ret = -EINVAL;
-                        break;
-                }
-
-                /* Validate that the target is in a valid r/w GPU domain */
-                if (reloc.write_domain & (reloc.write_domain - 1)) {
-                        DRM_ERROR("reloc with multiple write domains: "
-                                  "obj %p target %d offset %d "
-                                  "read %08x write %08x",
-                                  obj, reloc.target_handle,
-                                  (int) reloc.offset,
-                                  reloc.read_domains,
-                                  reloc.write_domain);
-                        ret = -EINVAL;
-                        break;
-                }
-                if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
-                    reloc.read_domains & I915_GEM_DOMAIN_CPU) {
-                        DRM_ERROR("reloc with read/write CPU domains: "
-                                  "obj %p target %d offset %d "
-                                  "read %08x write %08x",
-                                  obj, reloc.target_handle,
-                                  (int) reloc.offset,
-                                  reloc.read_domains,
-                                  reloc.write_domain);
-                        ret = -EINVAL;
-                        break;
-                }
-                if (reloc.write_domain && target_obj->pending_write_domain &&
-                    reloc.write_domain != target_obj->pending_write_domain) {
-                        DRM_ERROR("Write domain conflict: "
-                                  "obj %p target %d offset %d "
-                                  "new %08x old %08x\n",
-                                  obj, reloc.target_handle,
-                                  (int) reloc.offset,
-                                  reloc.write_domain,
-                                  target_obj->pending_write_domain);
-                        ret = -EINVAL;
-                        break;
-                }
-
-                target_obj->pending_read_domains |= reloc.read_domains;
-                target_obj->pending_write_domain |= reloc.write_domain;
-
-                /* If the relocation already has the right value in it, no
-                 * more work needs to be done.
-                 */
-                if (target_offset == reloc.presumed_offset)
-                        continue;
-
-                /* Check that the relocation address is valid... */
-                if (reloc.offset > obj->base.size - 4) {
-                        DRM_ERROR("Relocation beyond object bounds: "
-                                  "obj %p target %d offset %d size %d.\n",
-                                  obj, reloc.target_handle,
-                                  (int) reloc.offset, (int) obj->base.size);
-                        ret = -EINVAL;
-                        break;
-                }
-                if (reloc.offset & 3) {
-                        DRM_ERROR("Relocation not 4-byte aligned: "
-                                  "obj %p target %d offset %d.\n",
-                                  obj, reloc.target_handle,
-                                  (int) reloc.offset);
-                        ret = -EINVAL;
-                        break;
-                }
-
-                /* and points to somewhere within the target object. */
-                if (reloc.delta >= target_obj->size) {
-                        DRM_ERROR("Relocation beyond target object bounds: "
-                                  "obj %p target %d delta %d size %d.\n",
-                                  obj, reloc.target_handle,
-                                  (int) reloc.delta, (int) target_obj->size);
-                        ret = -EINVAL;
-                        break;
-                }
-
-                reloc.delta += target_offset;
-                if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
-                        uint32_t page_offset = reloc.offset & ~PAGE_MASK;
-                        char *vaddr;
-
-                        vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
-                        *(uint32_t *)(vaddr + page_offset) = reloc.delta;
-                        kunmap_atomic(vaddr);
-                } else {
-                        uint32_t __iomem *reloc_entry;
-                        void __iomem *reloc_page;
-
-                        ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
-                        if (ret)
-                                break;
-
-                        /* Map the page containing the relocation we're going to perform.  */
-                        reloc.offset += obj->gtt_offset;
-                        reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
-                                                              reloc.offset & PAGE_MASK);
-                        reloc_entry = (uint32_t __iomem *)
-                                (reloc_page + (reloc.offset & ~PAGE_MASK));
-                        iowrite32(reloc.delta, reloc_entry);
-                        io_mapping_unmap_atomic(reloc_page);
-                }
-
-                /* and update the user's relocation entry */
-                reloc.presumed_offset = target_offset;
-                if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
-                                              &reloc.presumed_offset,
-                                              sizeof(reloc.presumed_offset))) {
-                    ret = -EFAULT;
-                    break;
-                }
-        }
-
-        drm_gem_object_unreference(target_obj);
-        return ret;
-}
-
-static int
-i915_gem_execbuffer_pin(struct drm_device *dev,
-                        struct drm_file *file,
-                        struct drm_gem_object **object_list,
-                        struct drm_i915_gem_exec_object2 *exec_list,
-                        int count)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret, i, retry;
-
-        /* attempt to pin all of the buffers into the GTT */
-        for (retry = 0; retry < 2; retry++) {
-                ret = 0;
-                for (i = 0; i < count; i++) {
-                        struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
-                        struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
-                        bool need_fence =
-                                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
-                                obj->tiling_mode != I915_TILING_NONE;
-
-                        /* Check fence reg constraints and rebind if necessary */
-                        if (need_fence &&
-                            !i915_gem_object_fence_offset_ok(&obj->base,
-                                                             obj->tiling_mode)) {
-                                ret = i915_gem_object_unbind(&obj->base);
-                                if (ret)
-                                        break;
-                        }
-
-                        ret = i915_gem_object_pin(&obj->base, entry->alignment);
-                        if (ret)
-                                break;
-
-                        /*
-                         * Pre-965 chips need a fence register set up in order
-                         * to properly handle blits to/from tiled surfaces.
-                         */
-                        if (need_fence) {
-                                ret = i915_gem_object_get_fence_reg(&obj->base, true);
-                                if (ret) {
-                                        i915_gem_object_unpin(&obj->base);
-                                        break;
-                                }
-
-                                dev_priv->fence_regs[obj->fence_reg].gpu = true;
-                        }
-
-                        entry->offset = obj->gtt_offset;
-                }
-
-                while (i--)
-                        i915_gem_object_unpin(object_list[i]);
-
-                if (ret == 0)
-                        break;
-
-                if (ret != -ENOSPC || retry)
-                        return ret;
-
-                ret = i915_gem_evict_everything(dev);
-                if (ret)
-                        return ret;
-        }
-
-        return 0;
-}
-
-static int
-i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
-                                struct drm_file *file,
-                                struct intel_ring_buffer *ring,
-                                struct drm_gem_object **objects,
-                                int count)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret, i;
-
-        /* Zero the global flush/invalidate flags. These
-         * will be modified as new domains are computed
-         * for each object
-         */
-        dev->invalidate_domains = 0;
-        dev->flush_domains = 0;
-        dev_priv->mm.flush_rings = 0;
-        for (i = 0; i < count; i++)
-                i915_gem_object_set_to_gpu_domain(objects[i], ring);
-
-        if (dev->invalidate_domains | dev->flush_domains) {
-#if WATCH_EXEC
-                DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
-                          __func__,
-                         dev->invalidate_domains,
-                         dev->flush_domains);
-#endif
-                i915_gem_flush(dev, file,
-                               dev->invalidate_domains,
-                               dev->flush_domains,
-                               dev_priv->mm.flush_rings);
-        }
-
-        for (i = 0; i < count; i++) {
-                struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
-                /* XXX replace with semaphores */
-                if (obj->ring && ring != obj->ring) {
-                        ret = i915_gem_object_wait_rendering(&obj->base, true);
-                        if (ret)
-                                return ret;
-                }
-        }
+                                            obj->base.write_domain);
 
         return 0;
 }
@@ -3582,586 +3266,129 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
                 return 0;
 
         ret = 0;
-        if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
+        if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
                 /* And wait for the seqno passing without holding any locks and
                  * causing extra latency for others. This is safe as the irq
                  * generation is designed to be run atomically and so is
                  * lockless.
                  */
-                ring->user_irq_get(dev, ring);
-                ret = wait_event_interruptible(ring->irq_queue,
-                                               i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
-                                               || atomic_read(&dev_priv->mm.wedged));
-                ring->user_irq_put(dev, ring);
-
-                if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
-                        ret = -EIO;
-        }
-
-        if (ret == 0)
-                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
-
-        return ret;
-}
-
-static int
-i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
-                          uint64_t exec_offset)
-{
-        uint32_t exec_start, exec_len;
-
-        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
-        exec_len = (uint32_t) exec->batch_len;
-
-        if ((exec_start | exec_len) & 0x7)
-                return -EINVAL;
-
-        if (!exec_start)
-                return -EINVAL;
-
-        return 0;
-}
-
-static int
-validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
-                   int count)
-{
-        int i;
-
-        for (i = 0; i < count; i++) {
-                char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
-                size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
-
-                if (!access_ok(VERIFY_READ, ptr, length))
-                        return -EFAULT;
-
-                /* we may also need to update the presumed offsets */
-                if (!access_ok(VERIFY_WRITE, ptr, length))
-                        return -EFAULT;
-
-                if (fault_in_pages_readable(ptr, length))
-                        return -EFAULT;
-        }
-
-        return 0;
-}
-
-static int
-i915_gem_do_execbuffer(struct drm_device *dev, void *data,
-                       struct drm_file *file,
-                       struct drm_i915_gem_execbuffer2 *args,
-                       struct drm_i915_gem_exec_object2 *exec_list)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object **object_list = NULL;
-        struct drm_gem_object *batch_obj;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_clip_rect *cliprects = NULL;
-        struct drm_i915_gem_request *request = NULL;
-        int ret, i, flips;
-        uint64_t exec_offset;
-
-        struct intel_ring_buffer *ring = NULL;
-
-        ret = i915_gem_check_is_wedged(dev);
-        if (ret)
-                return ret;
-
-        ret = validate_exec_list(exec_list, args->buffer_count);
-        if (ret)
-                return ret;
-
-#if WATCH_EXEC
-        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
-                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
-#endif
-        switch (args->flags & I915_EXEC_RING_MASK) {
-        case I915_EXEC_DEFAULT:
-        case I915_EXEC_RENDER:
-                ring = &dev_priv->render_ring;
-                break;
-        case I915_EXEC_BSD:
-                if (!HAS_BSD(dev)) {
-                        DRM_ERROR("execbuf with invalid ring (BSD)\n");
-                        return -EINVAL;
-                }
-                ring = &dev_priv->bsd_ring;
-                break;
-        case I915_EXEC_BLT:
-                if (!HAS_BLT(dev)) {
-                        DRM_ERROR("execbuf with invalid ring (BLT)\n");
-                        return -EINVAL;
-                }
-                ring = &dev_priv->blt_ring;
-                break;
-        default:
-                DRM_ERROR("execbuf with unknown ring: %d\n",
-                          (int)(args->flags & I915_EXEC_RING_MASK));
-                return -EINVAL;
-        }
-
-        if (args->buffer_count < 1) {
-                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
-                return -EINVAL;
-        }
-        object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
-        if (object_list == NULL) {
-                DRM_ERROR("Failed to allocate object list for %d buffers\n",
-                          args->buffer_count);
-                ret = -ENOMEM;
-                goto pre_mutex_err;
-        }
-
-        if (args->num_cliprects != 0) {
-                cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
-                                    GFP_KERNEL);
-                if (cliprects == NULL) {
-                        ret = -ENOMEM;
-                        goto pre_mutex_err;
-                }
-
-                ret = copy_from_user(cliprects,
-                                     (struct drm_clip_rect __user *)
-                                     (uintptr_t) args->cliprects_ptr,
-                                     sizeof(*cliprects) * args->num_cliprects);
-                if (ret != 0) {
-                        DRM_ERROR("copy %d cliprects failed: %d\n",
-                                  args->num_cliprects, ret);
-                        ret = -EFAULT;
-                        goto pre_mutex_err;
-                }
-        }
-
-        request = kzalloc(sizeof(*request), GFP_KERNEL);
-        if (request == NULL) {
-                ret = -ENOMEM;
-                goto pre_mutex_err;
-        }
-
-        ret = i915_mutex_lock_interruptible(dev);
-        if (ret)
-                goto pre_mutex_err;
-
-        if (dev_priv->mm.suspended) {
-                mutex_unlock(&dev->struct_mutex);
-                ret = -EBUSY;
-                goto pre_mutex_err;
-        }
-
-        /* Look up object handles */
-        for (i = 0; i < args->buffer_count; i++) {
-                object_list[i] = drm_gem_object_lookup(dev, file,
-                                                       exec_list[i].handle);
-                if (object_list[i] == NULL) {
-                        DRM_ERROR("Invalid object handle %d at index %d\n",
-                                   exec_list[i].handle, i);
-                        /* prevent error path from reading uninitialized data */
-                        args->buffer_count = i + 1;
-                        ret = -ENOENT;
-                        goto err;
-                }
-
-                obj_priv = to_intel_bo(object_list[i]);
-                if (obj_priv->in_execbuffer) {
-                        DRM_ERROR("Object %p appears more than once in object list\n",
-                                   object_list[i]);
-                        /* prevent error path from reading uninitialized data */
-                        args->buffer_count = i + 1;
-                        ret = -EINVAL;
-                        goto err;
-                }
-                obj_priv->in_execbuffer = true;
-        }
-
-        /* Move the objects en-masse into the GTT, evicting if necessary. */
-        ret = i915_gem_execbuffer_pin(dev, file,
-                                      object_list, exec_list,
-                                      args->buffer_count);
-        if (ret)
-                goto err;
-
-        /* The objects are in their final locations, apply the relocations. */
-        for (i = 0; i < args->buffer_count; i++) {
-                struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
-                obj->base.pending_read_domains = 0;
-                obj->base.pending_write_domain = 0;
-                ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
-                if (ret)
-                        goto err;
-        }
-
-        /* Set the pending read domains for the batch buffer to COMMAND */
-        batch_obj = object_list[args->buffer_count-1];
-        if (batch_obj->pending_write_domain) {
-                DRM_ERROR("Attempting to use self-modifying batch buffer\n");
-                ret = -EINVAL;
-                goto err;
-        }
-        batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
-
-        /* Sanity check the batch buffer */
-        exec_offset = to_intel_bo(batch_obj)->gtt_offset;
-        ret = i915_gem_check_execbuffer(args, exec_offset);
-        if (ret != 0) {
-                DRM_ERROR("execbuf with invalid offset/length\n");
-                goto err;
-        }
-
-        ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
-                                              object_list, args->buffer_count);
-        if (ret)
-                goto err;
-
-        for (i = 0; i < args->buffer_count; i++) {
-                struct drm_gem_object *obj = object_list[i];
-                uint32_t old_write_domain = obj->write_domain;
-                obj->write_domain = obj->pending_write_domain;
-                trace_i915_gem_object_change_domain(obj,
-                                                    obj->read_domains,
-                                                    old_write_domain);
-        }
-
-#if WATCH_COHERENCY
-        for (i = 0; i < args->buffer_count; i++) {
-                i915_gem_object_check_coherency(object_list[i],
-                                                exec_list[i].handle);
-        }
-#endif
-
-#if WATCH_EXEC
-        i915_gem_dump_object(batch_obj,
-                              args->batch_len,
-                              __func__,
-                              ~0);
-#endif
-
-        /* Check for any pending flips. As we only maintain a flip queue depth
-         * of 1, we can simply insert a WAIT for the next display flip prior
-         * to executing the batch and avoid stalling the CPU.
-         */
-        flips = 0;
-        for (i = 0; i < args->buffer_count; i++) {
-                if (object_list[i]->write_domain)
-                        flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
-        }
-        if (flips) {
-                int plane, flip_mask;
-
-                for (plane = 0; flips >> plane; plane++) {
-                        if (((flips >> plane) & 1) == 0)
-                                continue;
-
-                        if (plane)
-                                flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
-                        else
-                                flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-
-                        intel_ring_begin(dev, ring, 2);
-                        intel_ring_emit(dev, ring,
-                                        MI_WAIT_FOR_EVENT | flip_mask);
-                        intel_ring_emit(dev, ring, MI_NOOP);
-                        intel_ring_advance(dev, ring);
-                }
-        }
-
-        /* Exec the batchbuffer */
-        ret = ring->dispatch_gem_execbuffer(dev, ring, args,
-                                            cliprects, exec_offset);
-        if (ret) {
-                DRM_ERROR("dispatch failed %d\n", ret);
-                goto err;
-        }
-
-        /*
-         * Ensure that the commands in the batch buffer are
-         * finished before the interrupt fires
-         */
-        i915_retire_commands(dev, ring);
-
-        for (i = 0; i < args->buffer_count; i++) {
-                struct drm_gem_object *obj = object_list[i];
-
-                i915_gem_object_move_to_active(obj, ring);
-                if (obj->write_domain)
-                        list_move_tail(&to_intel_bo(obj)->gpu_write_list,
-                                       &ring->gpu_write_list);
-        }
-
-        i915_add_request(dev, file, request, ring);
-        request = NULL;
-
-err:
-        for (i = 0; i < args->buffer_count; i++) {
-                if (object_list[i]) {
-                        obj_priv = to_intel_bo(object_list[i]);
-                        obj_priv->in_execbuffer = false;
-                }
-                drm_gem_object_unreference(object_list[i]);
-        }
-
-        mutex_unlock(&dev->struct_mutex);
-
-pre_mutex_err:
-        drm_free_large(object_list);
-        kfree(cliprects);
-        kfree(request);
-
-        return ret;
-}
-
-/*
- * Legacy execbuffer just creates an exec2 list from the original exec object
- * list array and passes it to the real function.
- */
-int
-i915_gem_execbuffer(struct drm_device *dev, void *data,
-                    struct drm_file *file_priv)
-{
-        struct drm_i915_gem_execbuffer *args = data;
-        struct drm_i915_gem_execbuffer2 exec2;
-        struct drm_i915_gem_exec_object *exec_list = NULL;
-        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-        int ret, i;
-
-#if WATCH_EXEC
-        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
-                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
-#endif
-
-        if (args->buffer_count < 1) {
-                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
-                return -EINVAL;
-        }
-
-        /* Copy in the exec list from userland */
-        exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
-        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
-        if (exec_list == NULL || exec2_list == NULL) {
-                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
-                          args->buffer_count);
-                drm_free_large(exec_list);
-                drm_free_large(exec2_list);
-                return -ENOMEM;
-        }
-        ret = copy_from_user(exec_list,
-                             (struct drm_i915_relocation_entry __user *)
-                             (uintptr_t) args->buffers_ptr,
-                             sizeof(*exec_list) * args->buffer_count);
-        if (ret != 0) {
-                DRM_ERROR("copy %d exec entries failed %d\n",
-                          args->buffer_count, ret);
-                drm_free_large(exec_list);
-                drm_free_large(exec2_list);
-                return -EFAULT;
-        }
-
-        for (i = 0; i < args->buffer_count; i++) {
-                exec2_list[i].handle = exec_list[i].handle;
-                exec2_list[i].relocation_count = exec_list[i].relocation_count;
-                exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
-                exec2_list[i].alignment = exec_list[i].alignment;
-                exec2_list[i].offset = exec_list[i].offset;
-                if (INTEL_INFO(dev)->gen < 4)
-                        exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
-                else
-                        exec2_list[i].flags = 0;
-        }
+                if (ring->irq_get(ring)) {
+                        ret = wait_event_interruptible(ring->irq_queue,
+                                                       i915_seqno_passed(ring->get_seqno(ring), seqno)
+                                                       || atomic_read(&dev_priv->mm.wedged));
+                        ring->irq_put(ring);
 
-        exec2.buffers_ptr = args->buffers_ptr;
-        exec2.buffer_count = args->buffer_count;
-        exec2.batch_start_offset = args->batch_start_offset;
-        exec2.batch_len = args->batch_len;
-        exec2.DR1 = args->DR1;
-        exec2.DR4 = args->DR4;
-        exec2.num_cliprects = args->num_cliprects;
-        exec2.cliprects_ptr = args->cliprects_ptr;
-        exec2.flags = I915_EXEC_RENDER;
-
-        ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
-        if (!ret) {
-                /* Copy the new buffer offsets back to the user's exec list. */
-                for (i = 0; i < args->buffer_count; i++)
-                        exec_list[i].offset = exec2_list[i].offset;
-                /* ... and back out to userspace */
-                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
-                                   (uintptr_t) args->buffers_ptr,
-                                   exec_list,
-                                   sizeof(*exec_list) * args->buffer_count);
-                if (ret) {
-                        ret = -EFAULT;
-                        DRM_ERROR("failed to copy %d exec entries "
-                                  "back to user (%d)\n",
-                                  args->buffer_count, ret);
+                        if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
+                                ret = -EIO;
                 }
         }
 
-        drm_free_large(exec_list);
-        drm_free_large(exec2_list);
-        return ret;
-}
-
-int
-i915_gem_execbuffer2(struct drm_device *dev, void *data,
-                     struct drm_file *file_priv)
-{
-        struct drm_i915_gem_execbuffer2 *args = data;
-        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-        int ret;
-
-#if WATCH_EXEC
-        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
-                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
-#endif
-
-        if (args->buffer_count < 1) {
-                DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
-                return -EINVAL;
-        }
-
-        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
-        if (exec2_list == NULL) {
-                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
-                          args->buffer_count);
-                return -ENOMEM;
-        }
-        ret = copy_from_user(exec2_list,
-                             (struct drm_i915_relocation_entry __user *)
-                             (uintptr_t) args->buffers_ptr,
-                             sizeof(*exec2_list) * args->buffer_count);
-        if (ret != 0) {
-                DRM_ERROR("copy %d exec entries failed %d\n",
-                          args->buffer_count, ret);
-                drm_free_large(exec2_list);
-                return -EFAULT;
-        }
-
-        ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
-        if (!ret) {
-                /* Copy the new buffer offsets back to the user's exec list. */
-                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
-                                   (uintptr_t) args->buffers_ptr,
-                                   exec2_list,
-                                   sizeof(*exec2_list) * args->buffer_count);
-                if (ret) {
-                        ret = -EFAULT;
-                        DRM_ERROR("failed to copy %d exec entries "
-                                  "back to user (%d)\n",
-                                  args->buffer_count, ret);
-                }
-        }
+        if (ret == 0)
+                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 
-        drm_free_large(exec2_list);
         return ret;
 }
 
 int
-i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
+i915_gem_object_pin(struct drm_i915_gem_object *obj,
+                    uint32_t alignment,
+                    bool map_and_fenceable)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret;
 
-        BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
+        BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
         WARN_ON(i915_verify_lists(dev));
 
-        if (obj_priv->gtt_space != NULL) {
-                if (alignment == 0)
-                        alignment = i915_gem_get_gtt_alignment(obj);
-                if (obj_priv->gtt_offset & (alignment - 1)) {
-                        WARN(obj_priv->pin_count,
-                             "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n",
-                             obj_priv->gtt_offset, alignment);
+        if (obj->gtt_space != NULL) {
+                if ((alignment && obj->gtt_offset & (alignment - 1)) ||
+                    (map_and_fenceable && !obj->map_and_fenceable)) {
+                        WARN(obj->pin_count,
+                             "bo is already pinned with incorrect alignment:"
+                             " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
+                             " obj->map_and_fenceable=%d\n",
+                             obj->gtt_offset, alignment,
+                             map_and_fenceable,
+                             obj->map_and_fenceable);
                         ret = i915_gem_object_unbind(obj);
                         if (ret)
                                 return ret;
                 }
         }
 
-        if (obj_priv->gtt_space == NULL) {
-                ret = i915_gem_object_bind_to_gtt(obj, alignment);
+        if (obj->gtt_space == NULL) {
+                ret = i915_gem_object_bind_to_gtt(obj, alignment,
+                                                  map_and_fenceable);
                 if (ret)
                         return ret;
         }
 
-        obj_priv->pin_count++;
-
-        /* If the object is not active and not pending a flush,
-         * remove it from the inactive list
-         */
-        if (obj_priv->pin_count == 1) {
-                i915_gem_info_add_pin(dev_priv, obj->size);
-                if (!obj_priv->active)
-                        list_move_tail(&obj_priv->mm_list,
+        if (obj->pin_count++ == 0) {
+                if (!obj->active)
+                        list_move_tail(&obj->mm_list,
                                        &dev_priv->mm.pinned_list);
         }
+        obj->pin_mappable |= map_and_fenceable;
 
         WARN_ON(i915_verify_lists(dev));
         return 0;
 }
 
 void
-i915_gem_object_unpin(struct drm_gem_object *obj)
+i915_gem_object_unpin(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 
         WARN_ON(i915_verify_lists(dev));
-        obj_priv->pin_count--;
-        BUG_ON(obj_priv->pin_count < 0);
-        BUG_ON(obj_priv->gtt_space == NULL);
+        BUG_ON(obj->pin_count == 0);
+        BUG_ON(obj->gtt_space == NULL);
 
-        /* If the object is no longer pinned, and is
-         * neither active nor being flushed, then stick it on
-         * the inactive list
-         */
-        if (obj_priv->pin_count == 0) {
-                if (!obj_priv->active)
-                        list_move_tail(&obj_priv->mm_list,
+        if (--obj->pin_count == 0) {
+                if (!obj->active)
+                        list_move_tail(&obj->mm_list,
                                        &dev_priv->mm.inactive_list);
-                i915_gem_info_remove_pin(dev_priv, obj->size);
+                obj->pin_mappable = false;
         }
         WARN_ON(i915_verify_lists(dev));
 }
 
 int
 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                   struct drm_file *file)
 {
         struct drm_i915_gem_pin *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
-        if (obj_priv->madv != I915_MADV_WILLNEED) {
+        if (obj->madv != I915_MADV_WILLNEED) {
                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
                 ret = -EINVAL;
                 goto out;
         }
 
-        if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
+        if (obj->pin_filp != NULL && obj->pin_filp != file) {
                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
                 ret = -EINVAL;
                 goto out;
         }
 
-        obj_priv->user_pin_count++;
-        obj_priv->pin_filp = file_priv;
-        if (obj_priv->user_pin_count == 1) {
-                ret = i915_gem_object_pin(obj, args->alignment);
+        obj->user_pin_count++;
+        obj->pin_filp = file;
+        if (obj->user_pin_count == 1) {
+                ret = i915_gem_object_pin(obj, args->alignment, true);
                 if (ret)
                         goto out;
         }
@@ -4170,9 +3397,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
          * as the X server doesn't manage domains yet
          */
         i915_gem_object_flush_cpu_write_domain(obj);
-        args->offset = obj_priv->gtt_offset;
+        args->offset = obj->gtt_offset;
 out:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -4180,38 +3407,36 @@ unlock:
 
 int
 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
-                     struct drm_file *file_priv)
+                     struct drm_file *file)
 {
         struct drm_i915_gem_pin *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
-        if (obj_priv->pin_filp != file_priv) {
+        if (obj->pin_filp != file) {
                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
                 ret = -EINVAL;
                 goto out;
         }
-        obj_priv->user_pin_count--;
-        if (obj_priv->user_pin_count == 0) {
-                obj_priv->pin_filp = NULL;
+        obj->user_pin_count--;
+        if (obj->user_pin_count == 0) {
+                obj->pin_filp = NULL;
                 i915_gem_object_unpin(obj);
         }
 
 out:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -4219,52 +3444,64 @@ unlock:
 
 int
 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
-                    struct drm_file *file_priv)
+                    struct drm_file *file)
 {
         struct drm_i915_gem_busy *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
         /* Count all active objects as busy, even if they are currently not used
          * by the gpu. Users of this interface expect objects to eventually
          * become non-busy without any further actions, therefore emit any
          * necessary flushes here.
          */
-        args->busy = obj_priv->active;
+        args->busy = obj->active;
         if (args->busy) {
                 /* Unconditionally flush objects, even when the gpu still uses this
                  * object. Userspace calling this function indicates that it wants to
                  * use this buffer rather sooner than later, so issuing the required
                  * flush earlier is beneficial.
                  */
-                if (obj->write_domain & I915_GEM_GPU_DOMAINS)
-                        i915_gem_flush_ring(dev, file_priv,
-                                            obj_priv->ring,
-                                            0, obj->write_domain);
+                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                        ret = i915_gem_flush_ring(dev, obj->ring,
+                                                  0, obj->base.write_domain);
+                } else if (obj->ring->outstanding_lazy_request ==
+                           obj->last_rendering_seqno) {
+                        struct drm_i915_gem_request *request;
+
+                        /* This ring is not being cleared by active usage,
+                         * so emit a request to do so.
+                         */
+                        request = kzalloc(sizeof(*request), GFP_KERNEL);
+                        if (request)
+                                ret = i915_add_request(dev,
+                                                       NULL, request,
+                                                       obj->ring);
+                        else
+                                ret = -ENOMEM;
+                }
 
                 /* Update the active list for the hardware's current position.
                  * Otherwise this only updates on a delayed timer or when irqs
                  * are actually unmasked, and our working set ends up being
                  * larger than required.
                  */
-                i915_gem_retire_requests_ring(dev, obj_priv->ring);
+                i915_gem_retire_requests_ring(dev, obj->ring);
 
-                args->busy = obj_priv->active;
+                args->busy = obj->active;
         }
 
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -4282,8 +3519,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
 {
         struct drm_i915_gem_madvise *args = data;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         switch (args->madv) {
@@ -4298,37 +3534,36 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
         if (obj == NULL) {
                 ret = -ENOENT;
                 goto unlock;
         }
-        obj_priv = to_intel_bo(obj);
 
-        if (obj_priv->pin_count) {
+        if (obj->pin_count) {
                 ret = -EINVAL;
                 goto out;
         }
 
-        if (obj_priv->madv != __I915_MADV_PURGED)
-                obj_priv->madv = args->madv;
+        if (obj->madv != __I915_MADV_PURGED)
+                obj->madv = args->madv;
 
         /* if the object is no longer bound, discard its backing storage */
-        if (i915_gem_object_is_purgeable(obj_priv) &&
-            obj_priv->gtt_space == NULL)
+        if (i915_gem_object_is_purgeable(obj) &&
+            obj->gtt_space == NULL)
                 i915_gem_object_truncate(obj);
 
-        args->retained = obj_priv->madv != __I915_MADV_PURGED;
+        args->retained = obj->madv != __I915_MADV_PURGED;
 
 out:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
 }
 
-struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
-                                              size_t size)
+struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
+                                                  size_t size)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
@@ -4351,11 +3586,15 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
         obj->base.driver_private = NULL;
         obj->fence_reg = I915_FENCE_REG_NONE;
         INIT_LIST_HEAD(&obj->mm_list);
+        INIT_LIST_HEAD(&obj->gtt_list);
         INIT_LIST_HEAD(&obj->ring_list);
+        INIT_LIST_HEAD(&obj->exec_list);
         INIT_LIST_HEAD(&obj->gpu_write_list);
         obj->madv = I915_MADV_WILLNEED;
+        /* Avoid an unnecessary call to unbind on the first bind. */
+        obj->map_and_fenceable = true;
 
-        return &obj->base;
+        return obj;
 }
 
 int i915_gem_init_object(struct drm_gem_object *obj)
@@ -4365,42 +3604,41 @@ int i915_gem_init_object(struct drm_gem_object *obj)
         return 0;
 }
 
-static void i915_gem_free_object_tail(struct drm_gem_object *obj)
+static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret;
 
         ret = i915_gem_object_unbind(obj);
         if (ret == -ERESTARTSYS) {
-                list_move(&obj_priv->mm_list,
+                list_move(&obj->mm_list,
                           &dev_priv->mm.deferred_free_list);
                 return;
         }
 
-        if (obj_priv->mmap_offset)
+        if (obj->base.map_list.map)
                 i915_gem_free_mmap_offset(obj);
 
-        drm_gem_object_release(obj);
-        i915_gem_info_remove_obj(dev_priv, obj->size);
+        drm_gem_object_release(&obj->base);
+        i915_gem_info_remove_obj(dev_priv, obj->base.size);
 
-        kfree(obj_priv->page_cpu_valid);
-        kfree(obj_priv->bit_17);
-        kfree(obj_priv);
+        kfree(obj->page_cpu_valid);
+        kfree(obj->bit_17);
+        kfree(obj);
 }
 
-void i915_gem_free_object(struct drm_gem_object *obj)
+void i915_gem_free_object(struct drm_gem_object *gem_obj)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
+        struct drm_device *dev = obj->base.dev;
 
         trace_i915_gem_object_destroy(obj);
 
-        while (obj_priv->pin_count > 0)
+        while (obj->pin_count > 0)
                 i915_gem_object_unpin(obj);
 
-        if (obj_priv->phys_obj)
+        if (obj->phys_obj)
                 i915_gem_detach_phys_object(dev, obj);
 
         i915_gem_free_object_tail(obj);
@@ -4427,13 +3665,15 @@ i915_gem_idle(struct drm_device *dev)
 
         /* Under UMS, be paranoid and evict. */
         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = i915_gem_evict_inactive(dev);
+                ret = i915_gem_evict_inactive(dev, false);
                 if (ret) {
                         mutex_unlock(&dev->struct_mutex);
                         return ret;
                 }
         }
 
+        i915_gem_reset_fences(dev);
+
         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
          * We need to replace this with a semaphore, or something.
          * And not confound mm.suspended!
@@ -4452,82 +3692,15 @@ i915_gem_idle(struct drm_device *dev)
         return 0;
 }
 
-/*
- * 965+ support PIPE_CONTROL commands, which provide finer grained control
- * over cache flushing.
- */
-static int
-i915_gem_init_pipe_control(struct drm_device *dev)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret;
-
-        obj = i915_gem_alloc_object(dev, 4096);
-        if (obj == NULL) {
-                DRM_ERROR("Failed to allocate seqno page\n");
-                ret = -ENOMEM;
-                goto err;
-        }
-        obj_priv = to_intel_bo(obj);
-        obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
-
-        ret = i915_gem_object_pin(obj, 4096);
-        if (ret)
-                goto err_unref;
-
-        dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
-        dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
-        if (dev_priv->seqno_page == NULL)
-                goto err_unpin;
-
-        dev_priv->seqno_obj = obj;
-        memset(dev_priv->seqno_page, 0, PAGE_SIZE);
-
-        return 0;
-
-err_unpin:
-        i915_gem_object_unpin(obj);
-err_unref:
-        drm_gem_object_unreference(obj);
-err:
-        return ret;
-}
-
-
-static void
-i915_gem_cleanup_pipe_control(struct drm_device *dev)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-
-        obj = dev_priv->seqno_obj;
-        obj_priv = to_intel_bo(obj);
-        kunmap(obj_priv->pages[0]);
-        i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
-        dev_priv->seqno_obj = NULL;
-
-        dev_priv->seqno_page = NULL;
-}
-
 int
 i915_gem_init_ringbuffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         int ret;
 
-        if (HAS_PIPE_CONTROL(dev)) {
-                ret = i915_gem_init_pipe_control(dev);
-                if (ret)
-                        return ret;
-        }
-
         ret = intel_init_render_ring_buffer(dev);
         if (ret)
-                goto cleanup_pipe_control;
+                return ret;
 
         if (HAS_BSD(dev)) {
                 ret = intel_init_bsd_ring_buffer(dev);
@@ -4546,12 +3719,9 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
         return 0;
 
 cleanup_bsd_ring:
-        intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
+        intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
 cleanup_render_ring:
-        intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
-cleanup_pipe_control:
-        if (HAS_PIPE_CONTROL(dev))
-                i915_gem_cleanup_pipe_control(dev);
+        intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
         return ret;
 }
 
@@ -4559,12 +3729,10 @@ void
 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        int i;
 
-        intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
-        intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
-        intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
-        if (HAS_PIPE_CONTROL(dev))
-                i915_gem_cleanup_pipe_control(dev);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                intel_cleanup_ring_buffer(&dev_priv->ring[i]);
 }
 
 int
@@ -4572,7 +3740,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
+        int ret, i;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return 0;
@@ -4592,14 +3760,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
         }
 
         BUG_ON(!list_empty(&dev_priv->mm.active_list));
-        BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
-        BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
-        BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
-        BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
-        BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
-        BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
+                BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
+        }
         mutex_unlock(&dev->struct_mutex);
 
         ret = drm_irq_install(dev);
@@ -4661,17 +3827,14 @@ i915_gem_load(struct drm_device *dev)
         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
-        init_ring_lists(&dev_priv->render_ring);
-        init_ring_lists(&dev_priv->bsd_ring);
-        init_ring_lists(&dev_priv->blt_ring);
+        INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                init_ring_lists(&dev_priv->ring[i]);
         for (i = 0; i < 16; i++)
                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
                           i915_gem_retire_work_handler);
         init_completion(&dev_priv->error_completion);
-        spin_lock(&shrink_list_lock);
-        list_add(&dev_priv->mm.shrink_list, &shrink_list);
-        spin_unlock(&shrink_list_lock);
 
         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
         if (IS_GEN3(dev)) {
@@ -4683,6 +3846,8 @@ i915_gem_load(struct drm_device *dev)
                 }
         }
 
+        dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
+
         /* Old X drivers will take 0-2 for front, back, depth buffers */
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 dev_priv->fence_reg_start = 3;
@@ -4714,6 +3879,10 @@ i915_gem_load(struct drm_device *dev)
         }
         i915_gem_detect_bit_6_swizzle(dev);
         init_waitqueue_head(&dev_priv->pending_flip_queue);
+
+        dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
+        dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
+        register_shrinker(&dev_priv->mm.inactive_shrinker);
 }
 
 /*
@@ -4783,47 +3952,47 @@ void i915_gem_free_all_phys_object(struct drm_device *dev)
 }
 
 void i915_gem_detach_phys_object(struct drm_device *dev,
-                                 struct drm_gem_object *obj)
+                                 struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv;
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
+        char *vaddr;
         int i;
-        int ret;
         int page_count;
 
-        obj_priv = to_intel_bo(obj);
-        if (!obj_priv->phys_obj)
+        if (!obj->phys_obj)
                 return;
+        vaddr = obj->phys_obj->handle->vaddr;
 
-        ret = i915_gem_object_get_pages(obj, 0);
-        if (ret)
-                goto out;
-
-        page_count = obj->size / PAGE_SIZE;
-
+        page_count = obj->base.size / PAGE_SIZE;
         for (i = 0; i < page_count; i++) {
-                char *dst = kmap_atomic(obj_priv->pages[i]);
-                char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
-
-                memcpy(dst, src, PAGE_SIZE);
-                kunmap_atomic(dst);
+                struct page *page = read_cache_page_gfp(mapping, i,
+                                                        GFP_HIGHUSER | __GFP_RECLAIMABLE);
+                if (!IS_ERR(page)) {
+                        char *dst = kmap_atomic(page);
+                        memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
+                        kunmap_atomic(dst);
+
+                        drm_clflush_pages(&page, 1);
+
+                        set_page_dirty(page);
+                        mark_page_accessed(page);
+                        page_cache_release(page);
+                }
         }
-        drm_clflush_pages(obj_priv->pages, page_count);
-        drm_agp_chipset_flush(dev);
+        intel_gtt_chipset_flush();
 
-        i915_gem_object_put_pages(obj);
-out:
-        obj_priv->phys_obj->cur_obj = NULL;
-        obj_priv->phys_obj = NULL;
+        obj->phys_obj->cur_obj = NULL;
+        obj->phys_obj = NULL;
 }
 
 int
 i915_gem_attach_phys_object(struct drm_device *dev,
-                            struct drm_gem_object *obj,
+                            struct drm_i915_gem_object *obj,
                             int id,
                             int align)
 {
+        struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
         int ret = 0;
         int page_count;
         int i;
@@ -4831,10 +4000,8 @@ i915_gem_attach_phys_object(struct drm_device *dev,
         if (id > I915_MAX_PHYS_OBJECT)
                 return -EINVAL;
 
-        obj_priv = to_intel_bo(obj);
-
-        if (obj_priv->phys_obj) {
-                if (obj_priv->phys_obj->id == id)
+        if (obj->phys_obj) {
+                if (obj->phys_obj->id == id)
                         return 0;
                 i915_gem_detach_phys_object(dev, obj);
         }
@@ -4842,51 +4009,50 @@ i915_gem_attach_phys_object(struct drm_device *dev,
         /* create a new object */
         if (!dev_priv->mm.phys_objs[id - 1]) {
                 ret = i915_gem_init_phys_object(dev, id,
-                                                obj->size, align);
+                                                obj->base.size, align);
                 if (ret) {
-                        DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
-                        goto out;
+                        DRM_ERROR("failed to init phys object %d size: %zu\n",
+                                  id, obj->base.size);
+                        return ret;
                 }
         }
 
         /* bind to the object */
-        obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
-        obj_priv->phys_obj->cur_obj = obj;
+        obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
+        obj->phys_obj->cur_obj = obj;
 
-        ret = i915_gem_object_get_pages(obj, 0);
-        if (ret) {
-                DRM_ERROR("failed to get page list\n");
-                goto out;
-        }
-
-        page_count = obj->size / PAGE_SIZE;
+        page_count = obj->base.size / PAGE_SIZE;
 
         for (i = 0; i < page_count; i++) {
-                char *src = kmap_atomic(obj_priv->pages[i]);
-                char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
+                struct page *page;
+                char *dst, *src;
+
+                page = read_cache_page_gfp(mapping, i,
+                                           GFP_HIGHUSER | __GFP_RECLAIMABLE);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
 
+                src = kmap_atomic(page);
+                dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
                 memcpy(dst, src, PAGE_SIZE);
                 kunmap_atomic(src);
-        }
 
-        i915_gem_object_put_pages(obj);
+                mark_page_accessed(page);
+                page_cache_release(page);
+        }
 
         return 0;
-out:
-        return ret;
 }
 
 static int
-i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
+i915_gem_phys_pwrite(struct drm_device *dev,
+                     struct drm_i915_gem_object *obj,
                      struct drm_i915_gem_pwrite *args,
                      struct drm_file *file_priv)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
+        void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
 
-        DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
-
         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
                 unsigned long unwritten;
 
@@ -4901,7 +4067,7 @@ i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
                         return -EFAULT;
         }
 
-        drm_agp_chipset_flush(dev);
+        intel_gtt_chipset_flush();
         return 0;
 }
 
@@ -4939,144 +4105,68 @@ i915_gpu_is_active(struct drm_device *dev)
 }
 
 static int
-i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
+i915_gem_inactive_shrink(struct shrinker *shrinker,
+                         int nr_to_scan,
+                         gfp_t gfp_mask)
 {
-        drm_i915_private_t *dev_priv, *next_dev;
-        struct drm_i915_gem_object *obj_priv, *next_obj;
-        int cnt = 0;
-        int would_deadlock = 1;
+        struct drm_i915_private *dev_priv =
+                container_of(shrinker,
+                             struct drm_i915_private,
+                             mm.inactive_shrinker);
+        struct drm_device *dev = dev_priv->dev;
+        struct drm_i915_gem_object *obj, *next;
+        int cnt;
+
+        if (!mutex_trylock(&dev->struct_mutex))
+                return 0;
 
         /* "fast-path" to count number of available objects */
         if (nr_to_scan == 0) {
-                spin_lock(&shrink_list_lock);
-                list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
-                        struct drm_device *dev = dev_priv->dev;
-
-                        if (mutex_trylock(&dev->struct_mutex)) {
-                                list_for_each_entry(obj_priv,
-                                                    &dev_priv->mm.inactive_list,
-                                                    mm_list)
-                                        cnt++;
-                                mutex_unlock(&dev->struct_mutex);
-                        }
-                }
-                spin_unlock(&shrink_list_lock);
-
-                return (cnt / 100) * sysctl_vfs_cache_pressure;
+                cnt = 0;
+                list_for_each_entry(obj,
+                                    &dev_priv->mm.inactive_list,
+                                    mm_list)
+                        cnt++;
+                mutex_unlock(&dev->struct_mutex);
+                return cnt / 100 * sysctl_vfs_cache_pressure;
         }
 
-        spin_lock(&shrink_list_lock);
-
 rescan:
         /* first scan for clean buffers */
-        list_for_each_entry_safe(dev_priv, next_dev,
-                                 &shrink_list, mm.shrink_list) {
-                struct drm_device *dev = dev_priv->dev;
-
-                if (! mutex_trylock(&dev->struct_mutex))
-                        continue;
-
-                spin_unlock(&shrink_list_lock);
-                i915_gem_retire_requests(dev);
+        i915_gem_retire_requests(dev);
 
-                list_for_each_entry_safe(obj_priv, next_obj,
-                                         &dev_priv->mm.inactive_list,
-                                         mm_list) {
-                        if (i915_gem_object_is_purgeable(obj_priv)) {
-                                i915_gem_object_unbind(&obj_priv->base);
-                                if (--nr_to_scan <= 0)
-                                        break;
-                        }
+        list_for_each_entry_safe(obj, next,
+                                 &dev_priv->mm.inactive_list,
+                                 mm_list) {
+                if (i915_gem_object_is_purgeable(obj)) {
+                        if (i915_gem_object_unbind(obj) == 0 &&
+                            --nr_to_scan == 0)
+                                break;
                 }
-
-                spin_lock(&shrink_list_lock);
-                mutex_unlock(&dev->struct_mutex);
-
-                would_deadlock = 0;
-
-                if (nr_to_scan <= 0)
-                        break;
         }
 
         /* second pass, evict/count anything still on the inactive list */
-        list_for_each_entry_safe(dev_priv, next_dev,
-                                 &shrink_list, mm.shrink_list) {
-                struct drm_device *dev = dev_priv->dev;
-
-                if (! mutex_trylock(&dev->struct_mutex))
-                        continue;
-
-                spin_unlock(&shrink_list_lock);
-
-                list_for_each_entry_safe(obj_priv, next_obj,
-                                         &dev_priv->mm.inactive_list,
-                                         mm_list) {
-                        if (nr_to_scan > 0) {
-                                i915_gem_object_unbind(&obj_priv->base);
-                                nr_to_scan--;
-                        } else
-                                cnt++;
-                }
-
-                spin_lock(&shrink_list_lock);
-                mutex_unlock(&dev->struct_mutex);
-
-                would_deadlock = 0;
+        cnt = 0;
+        list_for_each_entry_safe(obj, next,
+                                 &dev_priv->mm.inactive_list,
+                                 mm_list) {
+                if (nr_to_scan &&
+                    i915_gem_object_unbind(obj) == 0)
+                        nr_to_scan--;
+                else
+                        cnt++;
         }
 
-        if (nr_to_scan) {
-                int active = 0;
-
+        if (nr_to_scan && i915_gpu_is_active(dev)) {
                 /*
                  * We are desperate for pages, so as a last resort, wait
                  * for the GPU to finish and discard whatever we can.
                  * This has a dramatic impact to reduce the number of
                  * OOM-killer events whilst running the GPU aggressively.
                  */
-                list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
-                        struct drm_device *dev = dev_priv->dev;
-
-                        if (!mutex_trylock(&dev->struct_mutex))
-                                continue;
-
-                        spin_unlock(&shrink_list_lock);
-
-                        if (i915_gpu_is_active(dev)) {
-                                i915_gpu_idle(dev);
-                                active++;
-                        }
-
-                        spin_lock(&shrink_list_lock);
-                        mutex_unlock(&dev->struct_mutex);
-                }
-
-                if (active)
+                if (i915_gpu_idle(dev) == 0)
                         goto rescan;
         }
-
-        spin_unlock(&shrink_list_lock);
-
-        if (would_deadlock)
-                return -1;
-        else if (cnt > 0)
-                return (cnt / 100) * sysctl_vfs_cache_pressure;
-        else
-                return 0;
-}
-
-static struct shrinker shrinker = {
-        .shrink = i915_gem_shrink,
-        .seeks = DEFAULT_SEEKS,
-};
-
-__init void
-i915_gem_shrinker_init(void)
-{
-    register_shrinker(&shrinker);
-}
-
-__exit void
-i915_gem_shrinker_exit(void)
-{
-    unregister_shrinker(&shrinker);
+        mutex_unlock(&dev->struct_mutex);
+        return cnt / 100 * sysctl_vfs_cache_pressure;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
index 48644b840a8d..29d014c48ca2 100644
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -152,13 +152,12 @@ i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end,
 }
 
 void
-i915_gem_dump_object(struct drm_gem_object *obj, int len,
+i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
                      const char *where, uint32_t mark)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int page;
 
-        DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset);
+        DRM_INFO("%s: object at offset %08x\n", where, obj->gtt_offset);
         for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) {
                 int page_len, chunk, chunk_len;
 
@@ -170,9 +169,9 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len,
                         chunk_len = page_len - chunk;
                         if (chunk_len > 128)
                                 chunk_len = 128;
-                        i915_gem_dump_page(obj_priv->pages[page],
+                        i915_gem_dump_page(obj->pages[page],
                                            chunk, chunk + chunk_len,
-                                           obj_priv->gtt_offset +
+                                           obj->gtt_offset +
                                            page * PAGE_SIZE,
                                            mark);
                 }
@@ -182,21 +181,19 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len,
 
 #if WATCH_COHERENCY
 void
-i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
+i915_gem_object_check_coherency(struct drm_i915_gem_object *obj, int handle)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+        struct drm_device *dev = obj->base.dev;
         int page;
         uint32_t *gtt_mapping;
         uint32_t *backing_map = NULL;
         int bad_count = 0;
 
         DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n",
-                 __func__, obj, obj_priv->gtt_offset, handle,
+                 __func__, obj, obj->gtt_offset, handle,
                  obj->size / 1024);
 
-        gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset,
-                              obj->size);
+        gtt_mapping = ioremap(dev->agp->base + obj->gtt_offset, obj->base.size);
         if (gtt_mapping == NULL) {
                 DRM_ERROR("failed to map GTT space\n");
                 return;
@@ -205,7 +202,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
         for (page = 0; page < obj->size / PAGE_SIZE; page++) {
                 int i;
 
-                backing_map = kmap_atomic(obj_priv->pages[page], KM_USER0);
+                backing_map = kmap_atomic(obj->pages[page], KM_USER0);
 
                 if (backing_map == NULL) {
                         DRM_ERROR("failed to map backing page\n");
@@ -220,7 +217,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
                         if (cpuval != gttval) {
                                 DRM_INFO("incoherent CPU vs GPU at 0x%08x: "
                                          "0x%08x vs 0x%08x\n",
-                                         (int)(obj_priv->gtt_offset +
+                                         (int)(obj->gtt_offset +
                                                page * PAGE_SIZE + i * 4),
                                          cpuval, gttval);
                                 if (bad_count++ >= 8) {
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index d8ae7d1d0cc6..3d39005540aa 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -32,28 +32,36 @@
 #include "i915_drm.h"
 
 static bool
-mark_free(struct drm_i915_gem_object *obj_priv,
-           struct list_head *unwind)
+mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
 {
-        list_add(&obj_priv->evict_list, unwind);
-        drm_gem_object_reference(&obj_priv->base);
-        return drm_mm_scan_add_block(obj_priv->gtt_space);
+        list_add(&obj->exec_list, unwind);
+        drm_gem_object_reference(&obj->base);
+        return drm_mm_scan_add_block(obj->gtt_space);
 }
 
 int
-i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment)
+i915_gem_evict_something(struct drm_device *dev, int min_size,
+                         unsigned alignment, bool mappable)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct list_head eviction_list, unwind_list;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret = 0;
 
         i915_gem_retire_requests(dev);
 
         /* Re-check for free space after retiring requests */
-        if (drm_mm_search_free(&dev_priv->mm.gtt_space,
-                               min_size, alignment, 0))
-                return 0;
+        if (mappable) {
+                if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
+                                                min_size, alignment, 0,
+                                                dev_priv->mm.gtt_mappable_end,
+                                                0))
+                        return 0;
+        } else {
+                if (drm_mm_search_free(&dev_priv->mm.gtt_space,
+                                       min_size, alignment, 0))
+                        return 0;
+        }
 
         /*
          * The goal is to evict objects and amalgamate space in LRU order.
@@ -79,45 +87,56 @@ i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignmen
          */
 
         INIT_LIST_HEAD(&unwind_list);
-        drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
+        if (mappable)
+                drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size,
+                                            alignment, 0,
+                                            dev_priv->mm.gtt_mappable_end);
+        else
+                drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
 
         /* First see if there is a large enough contiguous idle region... */
-        list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) {
-                if (mark_free(obj_priv, &unwind_list))
+        list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
 
         /* Now merge in the soon-to-be-expired objects... */
-        list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
                 /* Does the object require an outstanding flush? */
-                if (obj_priv->base.write_domain || obj_priv->pin_count)
+                if (obj->base.write_domain || obj->pin_count)
                         continue;
 
-                if (mark_free(obj_priv, &unwind_list))
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
 
         /* Finally add anything with a pending flush (in order of retirement) */
-        list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) {
-                if (obj_priv->pin_count)
+        list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
+                if (obj->pin_count)
                         continue;
 
-                if (mark_free(obj_priv, &unwind_list))
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
-        list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
-                if (! obj_priv->base.write_domain || obj_priv->pin_count)
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
+                if (! obj->base.write_domain || obj->pin_count)
                         continue;
 
-                if (mark_free(obj_priv, &unwind_list))
+                if (mark_free(obj, &unwind_list))
                         goto found;
         }
 
         /* Nothing found, clean up and bail out! */
-        list_for_each_entry(obj_priv, &unwind_list, evict_list) {
-                ret = drm_mm_scan_remove_block(obj_priv->gtt_space);
+        while (!list_empty(&unwind_list)) {
+                obj = list_first_entry(&unwind_list,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+
+                ret = drm_mm_scan_remove_block(obj->gtt_space);
                 BUG_ON(ret);
-                drm_gem_object_unreference(&obj_priv->base);
+
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
         }
 
         /* We expect the caller to unpin, evict all and try again, or give up.
@@ -131,33 +150,34 @@ found:
          * temporary list. */
         INIT_LIST_HEAD(&eviction_list);
         while (!list_empty(&unwind_list)) {
-                obj_priv = list_first_entry(&unwind_list,
-                                            struct drm_i915_gem_object,
-                                            evict_list);
-                if (drm_mm_scan_remove_block(obj_priv->gtt_space)) {
-                        list_move(&obj_priv->evict_list, &eviction_list);
+                obj = list_first_entry(&unwind_list,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                if (drm_mm_scan_remove_block(obj->gtt_space)) {
+                        list_move(&obj->exec_list, &eviction_list);
                         continue;
                 }
-                list_del(&obj_priv->evict_list);
-                drm_gem_object_unreference(&obj_priv->base);
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
         }
 
         /* Unbinding will emit any required flushes */
         while (!list_empty(&eviction_list)) {
-                obj_priv = list_first_entry(&eviction_list,
-                                            struct drm_i915_gem_object,
-                                            evict_list);
+                obj = list_first_entry(&eviction_list,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
                 if (ret == 0)
-                        ret = i915_gem_object_unbind(&obj_priv->base);
-                list_del(&obj_priv->evict_list);
-                drm_gem_object_unreference(&obj_priv->base);
+                        ret = i915_gem_object_unbind(obj);
+
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
         }
 
         return ret;
 }
 
 int
-i915_gem_evict_everything(struct drm_device *dev)
+i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         int ret;
@@ -176,36 +196,22 @@ i915_gem_evict_everything(struct drm_device *dev)
 
         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
 
-        ret = i915_gem_evict_inactive(dev);
-        if (ret)
-                return ret;
-
-        lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
-                       list_empty(&dev_priv->mm.flushing_list) &&
-                       list_empty(&dev_priv->mm.active_list));
-        BUG_ON(!lists_empty);
-
-        return 0;
+        return i915_gem_evict_inactive(dev, purgeable_only);
 }
 
 /** Unbinds all inactive objects. */
 int
-i915_gem_evict_inactive(struct drm_device *dev)
+i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-
-        while (!list_empty(&dev_priv->mm.inactive_list)) {
-                struct drm_gem_object *obj;
-                int ret;
-
-                obj = &list_first_entry(&dev_priv->mm.inactive_list,
-                                        struct drm_i915_gem_object,
-                                        mm_list)->base;
-
-                ret = i915_gem_object_unbind(obj);
-                if (ret != 0) {
-                        DRM_ERROR("Error unbinding object: %d\n", ret);
-                        return ret;
+        struct drm_i915_gem_object *obj, *next;
+
+        list_for_each_entry_safe(obj, next,
+                                 &dev_priv->mm.inactive_list, mm_list) {
+                if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
+                        int ret = i915_gem_object_unbind(obj);
+                        if (ret)
+                                return ret;
                 }
         }
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
new file mode 100644
index 000000000000..d2f445e825f2
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -0,0 +1,1377 @@
+/*
+ * Copyright © 2008,2010 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *    Chris Wilson <chris@chris-wilson.co.uk>
+ *
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "intel_drv.h"
+
+struct change_domains {
+        uint32_t invalidate_domains;
+        uint32_t flush_domains;
+        uint32_t flush_rings;
+};
+
+/*
+ * Set the next domain for the specified object. This
+ * may not actually perform the necessary flushing/invaliding though,
+ * as that may want to be batched with other set_domain operations
+ *
+ * This is (we hope) the only really tricky part of gem. The goal
+ * is fairly simple -- track which caches hold bits of the object
+ * and make sure they remain coherent. A few concrete examples may
+ * help to explain how it works. For shorthand, we use the notation
+ * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
+ * a pair of read and write domain masks.
+ *
+ * Case 1: the batch buffer
+ *
+ *      1. Allocated
+ *      2. Written by CPU
+ *      3. Mapped to GTT
+ *      4. Read by GPU
+ *      5. Unmapped from GTT
+ *      6. Freed
+ *
+ *      Let's take these a step at a time
+ *
+ *      1. Allocated
+ *              Pages allocated from the kernel may still have
+ *              cache contents, so we set them to (CPU, CPU) always.
+ *      2. Written by CPU (using pwrite)
+ *              The pwrite function calls set_domain (CPU, CPU) and
+ *              this function does nothing (as nothing changes)
+ *      3. Mapped by GTT
+ *              This function asserts that the object is not
+ *              currently in any GPU-based read or write domains
+ *      4. Read by GPU
+ *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
+ *              As write_domain is zero, this function adds in the
+ *              current read domains (CPU+COMMAND, 0).
+ *              flush_domains is set to CPU.
+ *              invalidate_domains is set to COMMAND
+ *              clflush is run to get data out of the CPU caches
+ *              then i915_dev_set_domain calls i915_gem_flush to
+ *              emit an MI_FLUSH and drm_agp_chipset_flush
+ *      5. Unmapped from GTT
+ *              i915_gem_object_unbind calls set_domain (CPU, CPU)
+ *              flush_domains and invalidate_domains end up both zero
+ *              so no flushing/invalidating happens
+ *      6. Freed
+ *              yay, done
+ *
+ * Case 2: The shared render buffer
+ *
+ *      1. Allocated
+ *      2. Mapped to GTT
+ *      3. Read/written by GPU
+ *      4. set_domain to (CPU,CPU)
+ *      5. Read/written by CPU
+ *      6. Read/written by GPU
+ *
+ *      1. Allocated
+ *              Same as last example, (CPU, CPU)
+ *      2. Mapped to GTT
+ *              Nothing changes (assertions find that it is not in the GPU)
+ *      3. Read/written by GPU
+ *              execbuffer calls set_domain (RENDER, RENDER)
+ *              flush_domains gets CPU
+ *              invalidate_domains gets GPU
+ *              clflush (obj)
+ *              MI_FLUSH and drm_agp_chipset_flush
+ *      4. set_domain (CPU, CPU)
+ *              flush_domains gets GPU
+ *              invalidate_domains gets CPU
+ *              wait_rendering (obj) to make sure all drawing is complete.
+ *              This will include an MI_FLUSH to get the data from GPU
+ *              to memory
+ *              clflush (obj) to invalidate the CPU cache
+ *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
+ *      5. Read/written by CPU
+ *              cache lines are loaded and dirtied
+ *      6. Read written by GPU
+ *              Same as last GPU access
+ *
+ * Case 3: The constant buffer
+ *
+ *      1. Allocated
+ *      2. Written by CPU
+ *      3. Read by GPU
+ *      4. Updated (written) by CPU again
+ *      5. Read by GPU
+ *
+ *      1. Allocated
+ *              (CPU, CPU)
+ *      2. Written by CPU
+ *              (CPU, CPU)
+ *      3. Read by GPU
+ *              (CPU+RENDER, 0)
+ *              flush_domains = CPU
+ *              invalidate_domains = RENDER
+ *              clflush (obj)
+ *              MI_FLUSH
+ *              drm_agp_chipset_flush
+ *      4. Updated (written) by CPU again
+ *              (CPU, CPU)
+ *              flush_domains = 0 (no previous write domain)
+ *              invalidate_domains = 0 (no new read domains)
+ *      5. Read by GPU
+ *              (CPU+RENDER, 0)
+ *              flush_domains = CPU
+ *              invalidate_domains = RENDER
+ *              clflush (obj)
+ *              MI_FLUSH
+ *              drm_agp_chipset_flush
+ */
+static void
+i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
+                                  struct intel_ring_buffer *ring,
+                                  struct change_domains *cd)
+{
+        uint32_t invalidate_domains = 0, flush_domains = 0;
+
+        /*
+         * If the object isn't moving to a new write domain,
+         * let the object stay in multiple read domains
+         */
+        if (obj->base.pending_write_domain == 0)
+                obj->base.pending_read_domains |= obj->base.read_domains;
+
+        /*
+         * Flush the current write domain if
+         * the new read domains don't match. Invalidate
+         * any read domains which differ from the old
+         * write domain
+         */
+        if (obj->base.write_domain &&
+            (((obj->base.write_domain != obj->base.pending_read_domains ||
+               obj->ring != ring)) ||
+             (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
+                flush_domains |= obj->base.write_domain;
+                invalidate_domains |=
+                        obj->base.pending_read_domains & ~obj->base.write_domain;
+        }
+        /*
+         * Invalidate any read caches which may have
+         * stale data. That is, any new read domains.
+         */
+        invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
+        if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
+                i915_gem_clflush_object(obj);
+
+        /* blow away mappings if mapped through GTT */
+        if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
+                i915_gem_release_mmap(obj);
+
+        /* The actual obj->write_domain will be updated with
+         * pending_write_domain after we emit the accumulated flush for all
+         * of our domain changes in execbuffers (which clears objects'
+         * write_domains).  So if we have a current write domain that we
+         * aren't changing, set pending_write_domain to that.
+         */
+        if (flush_domains == 0 && obj->base.pending_write_domain == 0)
+                obj->base.pending_write_domain = obj->base.write_domain;
+
+        cd->invalidate_domains |= invalidate_domains;
+        cd->flush_domains |= flush_domains;
+        if (flush_domains & I915_GEM_GPU_DOMAINS)
+                cd->flush_rings |= obj->ring->id;
+        if (invalidate_domains & I915_GEM_GPU_DOMAINS)
+                cd->flush_rings |= ring->id;
+}
+
+struct eb_objects {
+        int and;
+        struct hlist_head buckets[0];
+};
+
+static struct eb_objects *
+eb_create(int size)
+{
+        struct eb_objects *eb;
+        int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
+        while (count > size)
+                count >>= 1;
+        eb = kzalloc(count*sizeof(struct hlist_head) +
+                     sizeof(struct eb_objects),
+                     GFP_KERNEL);
+        if (eb == NULL)
+                return eb;
+
+        eb->and = count - 1;
+        return eb;
+}
+
+static void
+eb_reset(struct eb_objects *eb)
+{
+        memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
+}
+
+static void
+eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
+{
+        hlist_add_head(&obj->exec_node,
+                       &eb->buckets[obj->exec_handle & eb->and]);
+}
+
+static struct drm_i915_gem_object *
+eb_get_object(struct eb_objects *eb, unsigned long handle)
+{
+        struct hlist_head *head;
+        struct hlist_node *node;
+        struct drm_i915_gem_object *obj;
+
+        head = &eb->buckets[handle & eb->and];
+        hlist_for_each(node, head) {
+                obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
+                if (obj->exec_handle == handle)
+                        return obj;
+        }
+
+        return NULL;
+}
+
+static void
+eb_destroy(struct eb_objects *eb)
+{
+        kfree(eb);
+}
+
+static int
+i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
+                                   struct eb_objects *eb,
+                                   struct drm_i915_gem_relocation_entry *reloc)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_gem_object *target_obj;
+        uint32_t target_offset;
+        int ret = -EINVAL;
+
+        /* we've already hold a reference to all valid objects */
+        target_obj = &eb_get_object(eb, reloc->target_handle)->base;
+        if (unlikely(target_obj == NULL))
+                return -ENOENT;
+
+        target_offset = to_intel_bo(target_obj)->gtt_offset;
+
+#if WATCH_RELOC
+        DRM_INFO("%s: obj %p offset %08x target %d "
+                 "read %08x write %08x gtt %08x "
+                 "presumed %08x delta %08x\n",
+                 __func__,
+                 obj,
+                 (int) reloc->offset,
+                 (int) reloc->target_handle,
+                 (int) reloc->read_domains,
+                 (int) reloc->write_domain,
+                 (int) target_offset,
+                 (int) reloc->presumed_offset,
+                 reloc->delta);
+#endif
+
+        /* The target buffer should have appeared before us in the
+         * exec_object list, so it should have a GTT space bound by now.
+         */
+        if (unlikely(target_offset == 0)) {
+                DRM_ERROR("No GTT space found for object %d\n",
+                          reloc->target_handle);
+                return ret;
+        }
+
+        /* Validate that the target is in a valid r/w GPU domain */
+        if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
+                DRM_ERROR("reloc with multiple write domains: "
+                          "obj %p target %d offset %d "
+                          "read %08x write %08x",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          reloc->read_domains,
+                          reloc->write_domain);
+                return ret;
+        }
+        if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
+                DRM_ERROR("reloc with read/write CPU domains: "
+                          "obj %p target %d offset %d "
+                          "read %08x write %08x",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          reloc->read_domains,
+                          reloc->write_domain);
+                return ret;
+        }
+        if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
+                     reloc->write_domain != target_obj->pending_write_domain)) {
+                DRM_ERROR("Write domain conflict: "
+                          "obj %p target %d offset %d "
+                          "new %08x old %08x\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          reloc->write_domain,
+                          target_obj->pending_write_domain);
+                return ret;
+        }
+
+        target_obj->pending_read_domains |= reloc->read_domains;
+        target_obj->pending_write_domain |= reloc->write_domain;
+
+        /* If the relocation already has the right value in it, no
+         * more work needs to be done.
+         */
+        if (target_offset == reloc->presumed_offset)
+                return 0;
+
+        /* Check that the relocation address is valid... */
+        if (unlikely(reloc->offset > obj->base.size - 4)) {
+                DRM_ERROR("Relocation beyond object bounds: "
+                          "obj %p target %d offset %d size %d.\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset,
+                          (int) obj->base.size);
+                return ret;
+        }
+        if (unlikely(reloc->offset & 3)) {
+                DRM_ERROR("Relocation not 4-byte aligned: "
+                          "obj %p target %d offset %d.\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->offset);
+                return ret;
+        }
+
+        /* and points to somewhere within the target object. */
+        if (unlikely(reloc->delta >= target_obj->size)) {
+                DRM_ERROR("Relocation beyond target object bounds: "
+                          "obj %p target %d delta %d size %d.\n",
+                          obj, reloc->target_handle,
+                          (int) reloc->delta,
+                          (int) target_obj->size);
+                return ret;
+        }
+
+        reloc->delta += target_offset;
+        if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
+                uint32_t page_offset = reloc->offset & ~PAGE_MASK;
+                char *vaddr;
+
+                vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
+                *(uint32_t *)(vaddr + page_offset) = reloc->delta;
+                kunmap_atomic(vaddr);
+        } else {
+                struct drm_i915_private *dev_priv = dev->dev_private;
+                uint32_t __iomem *reloc_entry;
+                void __iomem *reloc_page;
+
+                ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+                if (ret)
+                        return ret;
+
+                /* Map the page containing the relocation we're going to perform.  */
+                reloc->offset += obj->gtt_offset;
+                reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+                                                      reloc->offset & PAGE_MASK);
+                reloc_entry = (uint32_t __iomem *)
+                        (reloc_page + (reloc->offset & ~PAGE_MASK));
+                iowrite32(reloc->delta, reloc_entry);
+                io_mapping_unmap_atomic(reloc_page);
+        }
+
+        /* and update the user's relocation entry */
+        reloc->presumed_offset = target_offset;
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
+                                    struct eb_objects *eb)
+{
+        struct drm_i915_gem_relocation_entry __user *user_relocs;
+        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+        int i, ret;
+
+        user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
+        for (i = 0; i < entry->relocation_count; i++) {
+                struct drm_i915_gem_relocation_entry reloc;
+
+                if (__copy_from_user_inatomic(&reloc,
+                                              user_relocs+i,
+                                              sizeof(reloc)))
+                        return -EFAULT;
+
+                ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc);
+                if (ret)
+                        return ret;
+
+                if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
+                                            &reloc.presumed_offset,
+                                            sizeof(reloc.presumed_offset)))
+                        return -EFAULT;
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
+                                         struct eb_objects *eb,
+                                         struct drm_i915_gem_relocation_entry *relocs)
+{
+        const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+        int i, ret;
+
+        for (i = 0; i < entry->relocation_count; i++) {
+                ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_relocate(struct drm_device *dev,
+                             struct eb_objects *eb,
+                             struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        int ret;
+
+        list_for_each_entry(obj, objects, exec_list) {
+                ret = i915_gem_execbuffer_relocate_object(obj, eb);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
+                            struct drm_file *file,
+                            struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        int ret, retry;
+        bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
+        struct list_head ordered_objects;
+
+        INIT_LIST_HEAD(&ordered_objects);
+        while (!list_empty(objects)) {
+                struct drm_i915_gem_exec_object2 *entry;
+                bool need_fence, need_mappable;
+
+                obj = list_first_entry(objects,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                entry = obj->exec_entry;
+
+                need_fence =
+                        has_fenced_gpu_access &&
+                        entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                        obj->tiling_mode != I915_TILING_NONE;
+                need_mappable =
+                        entry->relocation_count ? true : need_fence;
+
+                if (need_mappable)
+                        list_move(&obj->exec_list, &ordered_objects);
+                else
+                        list_move_tail(&obj->exec_list, &ordered_objects);
+
+                obj->base.pending_read_domains = 0;
+                obj->base.pending_write_domain = 0;
+        }
+        list_splice(&ordered_objects, objects);
+
+        /* Attempt to pin all of the buffers into the GTT.
+         * This is done in 3 phases:
+         *
+         * 1a. Unbind all objects that do not match the GTT constraints for
+         *     the execbuffer (fenceable, mappable, alignment etc).
+         * 1b. Increment pin count for already bound objects.
+         * 2.  Bind new objects.
+         * 3.  Decrement pin count.
+         *
+         * This avoid unnecessary unbinding of later objects in order to makr
+         * room for the earlier objects *unless* we need to defragment.
+         */
+        retry = 0;
+        do {
+                ret = 0;
+
+                /* Unbind any ill-fitting objects or pin. */
+                list_for_each_entry(obj, objects, exec_list) {
+                        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+                        bool need_fence, need_mappable;
+                        if (!obj->gtt_space)
+                                continue;
+
+                        need_fence =
+                                has_fenced_gpu_access &&
+                                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                                obj->tiling_mode != I915_TILING_NONE;
+                        need_mappable =
+                                entry->relocation_count ? true : need_fence;
+
+                        if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
+                            (need_mappable && !obj->map_and_fenceable))
+                                ret = i915_gem_object_unbind(obj);
+                        else
+                                ret = i915_gem_object_pin(obj,
+                                                          entry->alignment,
+                                                          need_mappable);
+                        if (ret)
+                                goto err;
+
+                        entry++;
+                }
+
+                /* Bind fresh objects */
+                list_for_each_entry(obj, objects, exec_list) {
+                        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+                        bool need_fence;
+
+                        need_fence =
+                                has_fenced_gpu_access &&
+                                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                                obj->tiling_mode != I915_TILING_NONE;
+
+                        if (!obj->gtt_space) {
+                                bool need_mappable =
+                                        entry->relocation_count ? true : need_fence;
+
+                                ret = i915_gem_object_pin(obj,
+                                                          entry->alignment,
+                                                          need_mappable);
+                                if (ret)
+                                        break;
+                        }
+
+                        if (has_fenced_gpu_access) {
+                                if (need_fence) {
+                                        ret = i915_gem_object_get_fence(obj, ring, 1);
+                                        if (ret)
+                                                break;
+                                } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
+                                           obj->tiling_mode == I915_TILING_NONE) {
+                                        /* XXX pipelined! */
+                                        ret = i915_gem_object_put_fence(obj);
+                                        if (ret)
+                                                break;
+                                }
+                                obj->pending_fenced_gpu_access = need_fence;
+                        }
+
+                        entry->offset = obj->gtt_offset;
+                }
+
+                /* Decrement pin count for bound objects */
+                list_for_each_entry(obj, objects, exec_list) {
+                        if (obj->gtt_space)
+                                i915_gem_object_unpin(obj);
+                }
+
+                if (ret != -ENOSPC || retry > 1)
+                        return ret;
+
+                /* First attempt, just clear anything that is purgeable.
+                 * Second attempt, clear the entire GTT.
+                 */
+                ret = i915_gem_evict_everything(ring->dev, retry == 0);
+                if (ret)
+                        return ret;
+
+                retry++;
+        } while (1);
+
+err:
+        obj = list_entry(obj->exec_list.prev,
+                         struct drm_i915_gem_object,
+                         exec_list);
+        while (objects != &obj->exec_list) {
+                if (obj->gtt_space)
+                        i915_gem_object_unpin(obj);
+
+                obj = list_entry(obj->exec_list.prev,
+                                 struct drm_i915_gem_object,
+                                 exec_list);
+        }
+
+        return ret;
+}
+
+static int
+i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
+                                  struct drm_file *file,
+                                  struct intel_ring_buffer *ring,
+                                  struct list_head *objects,
+                                  struct eb_objects *eb,
+                                  struct drm_i915_gem_exec_object2 *exec,
+                                  int count)
+{
+        struct drm_i915_gem_relocation_entry *reloc;
+        struct drm_i915_gem_object *obj;
+        int *reloc_offset;
+        int i, total, ret;
+
+        /* We may process another execbuffer during the unlock... */
+        while (!list_empty(objects)) {
+                obj = list_first_entry(objects,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        total = 0;
+        for (i = 0; i < count; i++)
+                total += exec[i].relocation_count;
+
+        reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
+        reloc = drm_malloc_ab(total, sizeof(*reloc));
+        if (reloc == NULL || reloc_offset == NULL) {
+                drm_free_large(reloc);
+                drm_free_large(reloc_offset);
+                mutex_lock(&dev->struct_mutex);
+                return -ENOMEM;
+        }
+
+        total = 0;
+        for (i = 0; i < count; i++) {
+                struct drm_i915_gem_relocation_entry __user *user_relocs;
+
+                user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
+
+                if (copy_from_user(reloc+total, user_relocs,
+                                   exec[i].relocation_count * sizeof(*reloc))) {
+                        ret = -EFAULT;
+                        mutex_lock(&dev->struct_mutex);
+                        goto err;
+                }
+
+                reloc_offset[i] = total;
+                total += exec[i].relocation_count;
+        }
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret) {
+                mutex_lock(&dev->struct_mutex);
+                goto err;
+        }
+
+        /* reacquire the objects */
+        eb_reset(eb);
+        for (i = 0; i < count; i++) {
+                struct drm_i915_gem_object *obj;
+
+                obj = to_intel_bo(drm_gem_object_lookup(dev, file,
+                                                        exec[i].handle));
+                if (obj == NULL) {
+                        DRM_ERROR("Invalid object handle %d at index %d\n",
+                                   exec[i].handle, i);
+                        ret = -ENOENT;
+                        goto err;
+                }
+
+                list_add_tail(&obj->exec_list, objects);
+                obj->exec_handle = exec[i].handle;
+                obj->exec_entry = &exec[i];
+                eb_add_object(eb, obj);
+        }
+
+        ret = i915_gem_execbuffer_reserve(ring, file, objects);
+        if (ret)
+                goto err;
+
+        list_for_each_entry(obj, objects, exec_list) {
+                int offset = obj->exec_entry - exec;
+                ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
+                                                               reloc + reloc_offset[offset]);
+                if (ret)
+                        goto err;
+        }
+
+        /* Leave the user relocations as are, this is the painfully slow path,
+         * and we want to avoid the complication of dropping the lock whilst
+         * having buffers reserved in the aperture and so causing spurious
+         * ENOSPC for random operations.
+         */
+
+err:
+        drm_free_large(reloc);
+        drm_free_large(reloc_offset);
+        return ret;
+}
+
+static int
+i915_gem_execbuffer_flush(struct drm_device *dev,
+                          uint32_t invalidate_domains,
+                          uint32_t flush_domains,
+                          uint32_t flush_rings)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int i, ret;
+
+        if (flush_domains & I915_GEM_DOMAIN_CPU)
+                intel_gtt_chipset_flush();
+
+        if (flush_domains & I915_GEM_DOMAIN_GTT)
+                wmb();
+
+        if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
+                for (i = 0; i < I915_NUM_RINGS; i++)
+                        if (flush_rings & (1 << i)) {
+                                ret = i915_gem_flush_ring(dev,
+                                                          &dev_priv->ring[i],
+                                                          invalidate_domains,
+                                                          flush_domains);
+                                if (ret)
+                                        return ret;
+                        }
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
+                               struct intel_ring_buffer *to)
+{
+        struct intel_ring_buffer *from = obj->ring;
+        u32 seqno;
+        int ret, idx;
+
+        if (from == NULL || to == from)
+                return 0;
+
+        /* XXX gpu semaphores are currently causing hard hangs on SNB mobile */
+        if (INTEL_INFO(obj->base.dev)->gen < 6 || IS_MOBILE(obj->base.dev))
+                return i915_gem_object_wait_rendering(obj, true);
+
+        idx = intel_ring_sync_index(from, to);
+
+        seqno = obj->last_rendering_seqno;
+        if (seqno <= from->sync_seqno[idx])
+                return 0;
+
+        if (seqno == from->outstanding_lazy_request) {
+                struct drm_i915_gem_request *request;
+
+                request = kzalloc(sizeof(*request), GFP_KERNEL);
+                if (request == NULL)
+                        return -ENOMEM;
+
+                ret = i915_add_request(obj->base.dev, NULL, request, from);
+                if (ret) {
+                        kfree(request);
+                        return ret;
+                }
+
+                seqno = request->seqno;
+        }
+
+        from->sync_seqno[idx] = seqno;
+        return intel_ring_sync(to, from, seqno - 1);
+}
+
+static int
+i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
+                                struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        struct change_domains cd;
+        int ret;
+
+        cd.invalidate_domains = 0;
+        cd.flush_domains = 0;
+        cd.flush_rings = 0;
+        list_for_each_entry(obj, objects, exec_list)
+                i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
+
+        if (cd.invalidate_domains | cd.flush_domains) {
+#if WATCH_EXEC
+                DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
+                          __func__,
+                         cd.invalidate_domains,
+                         cd.flush_domains);
+#endif
+                ret = i915_gem_execbuffer_flush(ring->dev,
+                                                cd.invalidate_domains,
+                                                cd.flush_domains,
+                                                cd.flush_rings);
+                if (ret)
+                        return ret;
+        }
+
+        list_for_each_entry(obj, objects, exec_list) {
+                ret = i915_gem_execbuffer_sync_rings(obj, ring);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static bool
+i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
+{
+        return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
+}
+
+static int
+validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
+                   int count)
+{
+        int i;
+
+        for (i = 0; i < count; i++) {
+                char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
+                int length; /* limited by fault_in_pages_readable() */
+
+                /* First check for malicious input causing overflow */
+                if (exec[i].relocation_count >
+                    INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
+                        return -EINVAL;
+
+                length = exec[i].relocation_count *
+                        sizeof(struct drm_i915_gem_relocation_entry);
+                if (!access_ok(VERIFY_READ, ptr, length))
+                        return -EFAULT;
+
+                /* we may also need to update the presumed offsets */
+                if (!access_ok(VERIFY_WRITE, ptr, length))
+                        return -EFAULT;
+
+                if (fault_in_pages_readable(ptr, length))
+                        return -EFAULT;
+        }
+
+        return 0;
+}
+
+static int
+i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring,
+                                   struct list_head *objects)
+{
+        struct drm_i915_gem_object *obj;
+        int flips;
+
+        /* Check for any pending flips. As we only maintain a flip queue depth
+         * of 1, we can simply insert a WAIT for the next display flip prior
+         * to executing the batch and avoid stalling the CPU.
+         */
+        flips = 0;
+        list_for_each_entry(obj, objects, exec_list) {
+                if (obj->base.write_domain)
+                        flips |= atomic_read(&obj->pending_flip);
+        }
+        if (flips) {
+                int plane, flip_mask, ret;
+
+                for (plane = 0; flips >> plane; plane++) {
+                        if (((flips >> plane) & 1) == 0)
+                                continue;
+
+                        if (plane)
+                                flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
+                        else
+                                flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
+
+                        ret = intel_ring_begin(ring, 2);
+                        if (ret)
+                                return ret;
+
+                        intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
+                        intel_ring_emit(ring, MI_NOOP);
+                        intel_ring_advance(ring);
+                }
+        }
+
+        return 0;
+}
+
+static void
+i915_gem_execbuffer_move_to_active(struct list_head *objects,
+                                   struct intel_ring_buffer *ring,
+                                   u32 seqno)
+{
+        struct drm_i915_gem_object *obj;
+
+        list_for_each_entry(obj, objects, exec_list) {
+                obj->base.read_domains = obj->base.pending_read_domains;
+                obj->base.write_domain = obj->base.pending_write_domain;
+                obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
+
+                i915_gem_object_move_to_active(obj, ring, seqno);
+                if (obj->base.write_domain) {
+                        obj->dirty = 1;
+                        obj->pending_gpu_write = true;
+                        list_move_tail(&obj->gpu_write_list,
+                                       &ring->gpu_write_list);
+                        intel_mark_busy(ring->dev, obj);
+                }
+
+                trace_i915_gem_object_change_domain(obj,
+                                                    obj->base.read_domains,
+                                                    obj->base.write_domain);
+        }
+}
+
+static void
+i915_gem_execbuffer_retire_commands(struct drm_device *dev,
+                                    struct drm_file *file,
+                                    struct intel_ring_buffer *ring)
+{
+        struct drm_i915_gem_request *request;
+        u32 invalidate;
+
+        /*
+         * Ensure that the commands in the batch buffer are
+         * finished before the interrupt fires.
+         *
+         * The sampler always gets flushed on i965 (sigh).
+         */
+        invalidate = I915_GEM_DOMAIN_COMMAND;
+        if (INTEL_INFO(dev)->gen >= 4)
+                invalidate |= I915_GEM_DOMAIN_SAMPLER;
+        if (ring->flush(ring, invalidate, 0)) {
+                i915_gem_next_request_seqno(dev, ring);
+                return;
+        }
+
+        /* Add a breadcrumb for the completion of the batch buffer */
+        request = kzalloc(sizeof(*request), GFP_KERNEL);
+        if (request == NULL || i915_add_request(dev, file, request, ring)) {
+                i915_gem_next_request_seqno(dev, ring);
+                kfree(request);
+        }
+}
+
+static int
+i915_gem_do_execbuffer(struct drm_device *dev, void *data,
+                       struct drm_file *file,
+                       struct drm_i915_gem_execbuffer2 *args,
+                       struct drm_i915_gem_exec_object2 *exec)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct list_head objects;
+        struct eb_objects *eb;
+        struct drm_i915_gem_object *batch_obj;
+        struct drm_clip_rect *cliprects = NULL;
+        struct intel_ring_buffer *ring;
+        u32 exec_start, exec_len;
+        u32 seqno;
+        int ret, mode, i;
+
+        if (!i915_gem_check_execbuffer(args)) {
+                DRM_ERROR("execbuf with invalid offset/length\n");
+                return -EINVAL;
+        }
+
+        ret = validate_exec_list(exec, args->buffer_count);
+        if (ret)
+                return ret;
+
+#if WATCH_EXEC
+        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
+                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
+#endif
+        switch (args->flags & I915_EXEC_RING_MASK) {
+        case I915_EXEC_DEFAULT:
+        case I915_EXEC_RENDER:
+                ring = &dev_priv->ring[RCS];
+                break;
+        case I915_EXEC_BSD:
+                if (!HAS_BSD(dev)) {
+                        DRM_ERROR("execbuf with invalid ring (BSD)\n");
+                        return -EINVAL;
+                }
+                ring = &dev_priv->ring[VCS];
+                break;
+        case I915_EXEC_BLT:
+                if (!HAS_BLT(dev)) {
+                        DRM_ERROR("execbuf with invalid ring (BLT)\n");
+                        return -EINVAL;
+                }
+                ring = &dev_priv->ring[BCS];
+                break;
+        default:
+                DRM_ERROR("execbuf with unknown ring: %d\n",
+                          (int)(args->flags & I915_EXEC_RING_MASK));
+                return -EINVAL;
+        }
+
+        mode = args->flags & I915_EXEC_CONSTANTS_MASK;
+        switch (mode) {
+        case I915_EXEC_CONSTANTS_REL_GENERAL:
+        case I915_EXEC_CONSTANTS_ABSOLUTE:
+        case I915_EXEC_CONSTANTS_REL_SURFACE:
+                if (ring == &dev_priv->ring[RCS] &&
+                    mode != dev_priv->relative_constants_mode) {
+                        if (INTEL_INFO(dev)->gen < 4)
+                                return -EINVAL;
+
+                        if (INTEL_INFO(dev)->gen > 5 &&
+                            mode == I915_EXEC_CONSTANTS_REL_SURFACE)
+                                return -EINVAL;
+
+                        ret = intel_ring_begin(ring, 4);
+                        if (ret)
+                                return ret;
+
+                        intel_ring_emit(ring, MI_NOOP);
+                        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+                        intel_ring_emit(ring, INSTPM);
+                        intel_ring_emit(ring,
+                                        I915_EXEC_CONSTANTS_MASK << 16 | mode);
+                        intel_ring_advance(ring);
+
+                        dev_priv->relative_constants_mode = mode;
+                }
+                break;
+        default:
+                DRM_ERROR("execbuf with unknown constants: %d\n", mode);
+                return -EINVAL;
+        }
+
+        if (args->buffer_count < 1) {
+                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
+                return -EINVAL;
+        }
+
+        if (args->num_cliprects != 0) {
+                if (ring != &dev_priv->ring[RCS]) {
+                        DRM_ERROR("clip rectangles are only valid with the render ring\n");
+                        return -EINVAL;
+                }
+
+                cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
+                                    GFP_KERNEL);
+                if (cliprects == NULL) {
+                        ret = -ENOMEM;
+                        goto pre_mutex_err;
+                }
+
+                if (copy_from_user(cliprects,
+                                     (struct drm_clip_rect __user *)(uintptr_t)
+                                     args->cliprects_ptr,
+                                     sizeof(*cliprects)*args->num_cliprects)) {
+                        ret = -EFAULT;
+                        goto pre_mutex_err;
+                }
+        }
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                goto pre_mutex_err;
+
+        if (dev_priv->mm.suspended) {
+                mutex_unlock(&dev->struct_mutex);
+                ret = -EBUSY;
+                goto pre_mutex_err;
+        }
+
+        eb = eb_create(args->buffer_count);
+        if (eb == NULL) {
+                mutex_unlock(&dev->struct_mutex);
+                ret = -ENOMEM;
+                goto pre_mutex_err;
+        }
+
+        /* Look up object handles */
+        INIT_LIST_HEAD(&objects);
+        for (i = 0; i < args->buffer_count; i++) {
+                struct drm_i915_gem_object *obj;
+
+                obj = to_intel_bo(drm_gem_object_lookup(dev, file,
+                                                        exec[i].handle));
+                if (obj == NULL) {
+                        DRM_ERROR("Invalid object handle %d at index %d\n",
+                                   exec[i].handle, i);
+                        /* prevent error path from reading uninitialized data */
+                        ret = -ENOENT;
+                        goto err;
+                }
+
+                if (!list_empty(&obj->exec_list)) {
+                        DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
+                                   obj, exec[i].handle, i);
+                        ret = -EINVAL;
+                        goto err;
+                }
+
+                list_add_tail(&obj->exec_list, &objects);
+                obj->exec_handle = exec[i].handle;
+                obj->exec_entry = &exec[i];
+                eb_add_object(eb, obj);
+        }
+
+        /* take note of the batch buffer before we might reorder the lists */
+        batch_obj = list_entry(objects.prev,
+                               struct drm_i915_gem_object,
+                               exec_list);
+
+        /* Move the objects en-masse into the GTT, evicting if necessary. */
+        ret = i915_gem_execbuffer_reserve(ring, file, &objects);
+        if (ret)
+                goto err;
+
+        /* The objects are in their final locations, apply the relocations. */
+        ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
+        if (ret) {
+                if (ret == -EFAULT) {
+                        ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
+                                                                &objects, eb,
+                                                                exec,
+                                                                args->buffer_count);
+                        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+                }
+                if (ret)
+                        goto err;
+        }
+
+        /* Set the pending read domains for the batch buffer to COMMAND */
+        if (batch_obj->base.pending_write_domain) {
+                DRM_ERROR("Attempting to use self-modifying batch buffer\n");
+                ret = -EINVAL;
+                goto err;
+        }
+        batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
+
+        ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
+        if (ret)
+                goto err;
+
+        ret = i915_gem_execbuffer_wait_for_flips(ring, &objects);
+        if (ret)
+                goto err;
+
+        seqno = i915_gem_next_request_seqno(dev, ring);
+        for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) {
+                if (seqno < ring->sync_seqno[i]) {
+                        /* The GPU can not handle its semaphore value wrapping,
+                         * so every billion or so execbuffers, we need to stall
+                         * the GPU in order to reset the counters.
+                         */
+                        ret = i915_gpu_idle(dev);
+                        if (ret)
+                                goto err;
+
+                        BUG_ON(ring->sync_seqno[i]);
+                }
+        }
+
+        exec_start = batch_obj->gtt_offset + args->batch_start_offset;
+        exec_len = args->batch_len;
+        if (cliprects) {
+                for (i = 0; i < args->num_cliprects; i++) {
+                        ret = i915_emit_box(dev, &cliprects[i],
+                                            args->DR1, args->DR4);
+                        if (ret)
+                                goto err;
+
+                        ret = ring->dispatch_execbuffer(ring,
+                                                        exec_start, exec_len);
+                        if (ret)
+                                goto err;
+                }
+        } else {
+                ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
+                if (ret)
+                        goto err;
+        }
+
+        i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
+        i915_gem_execbuffer_retire_commands(dev, file, ring);
+
+err:
+        eb_destroy(eb);
+        while (!list_empty(&objects)) {
+                struct drm_i915_gem_object *obj;
+
+                obj = list_first_entry(&objects,
+                                       struct drm_i915_gem_object,
+                                       exec_list);
+                list_del_init(&obj->exec_list);
+                drm_gem_object_unreference(&obj->base);
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+pre_mutex_err:
+        kfree(cliprects);
+        return ret;
+}
+
+/*
+ * Legacy execbuffer just creates an exec2 list from the original exec object
+ * list array and passes it to the real function.
+ */
+int
+i915_gem_execbuffer(struct drm_device *dev, void *data,
+                    struct drm_file *file)
+{
+        struct drm_i915_gem_execbuffer *args = data;
+        struct drm_i915_gem_execbuffer2 exec2;
+        struct drm_i915_gem_exec_object *exec_list = NULL;
+        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
+        int ret, i;
+
+#if WATCH_EXEC
+        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
+                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
+#endif
+
+        if (args->buffer_count < 1) {
+                DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
+                return -EINVAL;
+        }
+
+        /* Copy in the exec list from userland */
+        exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
+        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
+        if (exec_list == NULL || exec2_list == NULL) {
+                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
+                          args->buffer_count);
+                drm_free_large(exec_list);
+                drm_free_large(exec2_list);
+                return -ENOMEM;
+        }
+        ret = copy_from_user(exec_list,
+                             (struct drm_i915_relocation_entry __user *)
+                             (uintptr_t) args->buffers_ptr,
+                             sizeof(*exec_list) * args->buffer_count);
+        if (ret != 0) {
+                DRM_ERROR("copy %d exec entries failed %d\n",
+                          args->buffer_count, ret);
+                drm_free_large(exec_list);
+                drm_free_large(exec2_list);
+                return -EFAULT;
+        }
+
+        for (i = 0; i < args->buffer_count; i++) {
+                exec2_list[i].handle = exec_list[i].handle;
+                exec2_list[i].relocation_count = exec_list[i].relocation_count;
+                exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
+                exec2_list[i].alignment = exec_list[i].alignment;
+                exec2_list[i].offset = exec_list[i].offset;
+                if (INTEL_INFO(dev)->gen < 4)
+                        exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
+                else
+                        exec2_list[i].flags = 0;
+        }
+
+        exec2.buffers_ptr = args->buffers_ptr;
+        exec2.buffer_count = args->buffer_count;
+        exec2.batch_start_offset = args->batch_start_offset;
+        exec2.batch_len = args->batch_len;
+        exec2.DR1 = args->DR1;
+        exec2.DR4 = args->DR4;
+        exec2.num_cliprects = args->num_cliprects;
+        exec2.cliprects_ptr = args->cliprects_ptr;
+        exec2.flags = I915_EXEC_RENDER;
+
+        ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
+        if (!ret) {
+                /* Copy the new buffer offsets back to the user's exec list. */
+                for (i = 0; i < args->buffer_count; i++)
+                        exec_list[i].offset = exec2_list[i].offset;
+                /* ... and back out to userspace */
+                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
+                                   (uintptr_t) args->buffers_ptr,
+                                   exec_list,
+                                   sizeof(*exec_list) * args->buffer_count);
+                if (ret) {
+                        ret = -EFAULT;
+                        DRM_ERROR("failed to copy %d exec entries "
+                                  "back to user (%d)\n",
+                                  args->buffer_count, ret);
+                }
+        }
+
+        drm_free_large(exec_list);
+        drm_free_large(exec2_list);
+        return ret;
+}
+
+int
+i915_gem_execbuffer2(struct drm_device *dev, void *data,
+                     struct drm_file *file)
+{
+        struct drm_i915_gem_execbuffer2 *args = data;
+        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
+        int ret;
+
+#if WATCH_EXEC
+        DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
+                  (int) args->buffers_ptr, args->buffer_count, args->batch_len);
+#endif
+
+        if (args->buffer_count < 1) {
+                DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
+                return -EINVAL;
+        }
+
+        exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
+        if (exec2_list == NULL) {
+                DRM_ERROR("Failed to allocate exec list for %d buffers\n",
+                          args->buffer_count);
+                return -ENOMEM;
+        }
+        ret = copy_from_user(exec2_list,
+                             (struct drm_i915_relocation_entry __user *)
+                             (uintptr_t) args->buffers_ptr,
+                             sizeof(*exec2_list) * args->buffer_count);
+        if (ret != 0) {
+                DRM_ERROR("copy %d exec entries failed %d\n",
+                          args->buffer_count, ret);
+                drm_free_large(exec2_list);
+                return -EFAULT;
+        }
+
+        ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
+        if (!ret) {
+                /* Copy the new buffer offsets back to the user's exec list. */
+                ret = copy_to_user((struct drm_i915_relocation_entry __user *)
+                                   (uintptr_t) args->buffers_ptr,
+                                   exec2_list,
+                                   sizeof(*exec2_list) * args->buffer_count);
+                if (ret) {
+                        ret = -EFAULT;
+                        DRM_ERROR("failed to copy %d exec entries "
+                                  "back to user (%d)\n",
+                                  args->buffer_count, ret);
+                }
+        }
+
+        drm_free_large(exec2_list);
+        return ret;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
new file mode 100644
index 000000000000..b0abdc64aa9f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright © 2010 Daniel Vetter
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "intel_drv.h"
+
+void i915_gem_restore_gtt_mappings(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+
+        /* First fill our portion of the GTT with scratch pages */
+        intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
+                              (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
+
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                i915_gem_clflush_object(obj);
+
+                if (dev_priv->mm.gtt->needs_dmar) {
+                        BUG_ON(!obj->sg_list);
+
+                        intel_gtt_insert_sg_entries(obj->sg_list,
+                                                    obj->num_sg,
+                                                    obj->gtt_space->start
+                                                        >> PAGE_SHIFT,
+                                                    obj->agp_type);
+                } else
+                        intel_gtt_insert_pages(obj->gtt_space->start
+                                                   >> PAGE_SHIFT,
+                                               obj->base.size >> PAGE_SHIFT,
+                                               obj->pages,
+                                               obj->agp_type);
+        }
+
+        intel_gtt_chipset_flush();
+}
+
+int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        if (dev_priv->mm.gtt->needs_dmar) {
+                ret = intel_gtt_map_memory(obj->pages,
+                                           obj->base.size >> PAGE_SHIFT,
+                                           &obj->sg_list,
+                                           &obj->num_sg);
+                if (ret != 0)
+                        return ret;
+
+                intel_gtt_insert_sg_entries(obj->sg_list,
+                                            obj->num_sg,
+                                            obj->gtt_space->start >> PAGE_SHIFT,
+                                            obj->agp_type);
+        } else
+                intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
+                                       obj->base.size >> PAGE_SHIFT,
+                                       obj->pages,
+                                       obj->agp_type);
+
+        return 0;
+}
+
+void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
+{
+        intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
+                              obj->base.size >> PAGE_SHIFT);
+
+        if (obj->sg_list) {
+                intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
+                obj->sg_list = NULL;
+        }
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index af352de70be1..22a32b9932c5 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -181,7 +181,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 }
 
 /* Check pitch constriants for all chips & tiling formats */
-bool
+static bool
 i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 {
         int tile_width;
@@ -232,32 +232,44 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
         return true;
 }
 
-bool
-i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode)
+/* Is the current GTT allocation valid for the change in tiling? */
+static bool
+i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
 {
-        struct drm_device *dev = obj->dev;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-
-        if (obj_priv->gtt_space == NULL)
-                return true;
+        u32 size;
 
         if (tiling_mode == I915_TILING_NONE)
                 return true;
 
-        if (INTEL_INFO(dev)->gen >= 4)
+        if (INTEL_INFO(obj->base.dev)->gen >= 4)
                 return true;
 
-        if (obj_priv->gtt_offset & (obj->size - 1))
-                return false;
-
-        if (IS_GEN3(dev)) {
-                if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK)
+        if (INTEL_INFO(obj->base.dev)->gen == 3) {
+                if (obj->gtt_offset & ~I915_FENCE_START_MASK)
                         return false;
         } else {
-                if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK)
+                if (obj->gtt_offset & ~I830_FENCE_START_MASK)
                         return false;
         }
 
+        /*
+         * Previous chips need to be aligned to the size of the smallest
+         * fence register that can contain the object.
+         */
+        if (INTEL_INFO(obj->base.dev)->gen == 3)
+                size = 1024*1024;
+        else
+                size = 512*1024;
+
+        while (size < obj->base.size)
+                size <<= 1;
+
+        if (obj->gtt_space->size != size)
+                return false;
+
+        if (obj->gtt_offset & (size - 1))
+                return false;
+
         return true;
 }
 
@@ -267,30 +279,29 @@ i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode)
  */
 int
 i915_gem_set_tiling(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                   struct drm_file *file)
 {
         struct drm_i915_gem_set_tiling *args = data;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ret = i915_gem_check_is_wedged(dev);
         if (ret)
                 return ret;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL)
                 return -ENOENT;
-        obj_priv = to_intel_bo(obj);
 
-        if (!i915_tiling_ok(dev, args->stride, obj->size, args->tiling_mode)) {
-                drm_gem_object_unreference_unlocked(obj);
+        if (!i915_tiling_ok(dev,
+                            args->stride, obj->base.size, args->tiling_mode)) {
+                drm_gem_object_unreference_unlocked(&obj->base);
                 return -EINVAL;
         }
 
-        if (obj_priv->pin_count) {
-                drm_gem_object_unreference_unlocked(obj);
+        if (obj->pin_count) {
+                drm_gem_object_unreference_unlocked(&obj->base);
                 return -EBUSY;
         }
 
@@ -324,34 +335,28 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
         }
 
         mutex_lock(&dev->struct_mutex);
-        if (args->tiling_mode != obj_priv->tiling_mode ||
-            args->stride != obj_priv->stride) {
+        if (args->tiling_mode != obj->tiling_mode ||
+            args->stride != obj->stride) {
                 /* We need to rebind the object if its current allocation
                  * no longer meets the alignment restrictions for its new
                  * tiling mode. Otherwise we can just leave it alone, but
                  * need to ensure that any fence register is cleared.
                  */
-                if (!i915_gem_object_fence_offset_ok(obj, args->tiling_mode))
-                        ret = i915_gem_object_unbind(obj);
-                else if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-                        ret = i915_gem_object_put_fence_reg(obj, true);
-                else
-                        i915_gem_release_mmap(obj);
+                i915_gem_release_mmap(obj);
 
-                if (ret != 0) {
-                        args->tiling_mode = obj_priv->tiling_mode;
-                        args->stride = obj_priv->stride;
-                        goto err;
-                }
+                obj->map_and_fenceable =
+                        obj->gtt_space == NULL ||
+                        (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end &&
+                         i915_gem_object_fence_ok(obj, args->tiling_mode));
 
-                obj_priv->tiling_mode = args->tiling_mode;
-                obj_priv->stride = args->stride;
+                obj->tiling_changed = true;
+                obj->tiling_mode = args->tiling_mode;
+                obj->stride = args->stride;
         }
-err:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 
-        return ret;
+        return 0;
 }
 
 /**
@@ -359,22 +364,20 @@ err:
  */
 int
 i915_gem_get_tiling(struct drm_device *dev, void *data,
-                   struct drm_file *file_priv)
+                   struct drm_file *file)
 {
         struct drm_i915_gem_get_tiling *args = data;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
 
-        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
         if (obj == NULL)
                 return -ENOENT;
-        obj_priv = to_intel_bo(obj);
 
         mutex_lock(&dev->struct_mutex);
 
-        args->tiling_mode = obj_priv->tiling_mode;
-        switch (obj_priv->tiling_mode) {
+        args->tiling_mode = obj->tiling_mode;
+        switch (obj->tiling_mode) {
         case I915_TILING_X:
                 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
                 break;
@@ -394,7 +397,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
         if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
                 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
@@ -424,46 +427,44 @@ i915_gem_swizzle_page(struct page *page)
 }
 
 void
-i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj)
+i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count = obj->size >> PAGE_SHIFT;
+        int page_count = obj->base.size >> PAGE_SHIFT;
         int i;
 
         if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
                 return;
 
-        if (obj_priv->bit_17 == NULL)
+        if (obj->bit_17 == NULL)
                 return;
 
         for (i = 0; i < page_count; i++) {
-                char new_bit_17 = page_to_phys(obj_priv->pages[i]) >> 17;
+                char new_bit_17 = page_to_phys(obj->pages[i]) >> 17;
                 if ((new_bit_17 & 0x1) !=
-                    (test_bit(i, obj_priv->bit_17) != 0)) {
-                        i915_gem_swizzle_page(obj_priv->pages[i]);
-                        set_page_dirty(obj_priv->pages[i]);
+                    (test_bit(i, obj->bit_17) != 0)) {
+                        i915_gem_swizzle_page(obj->pages[i]);
+                        set_page_dirty(obj->pages[i]);
                 }
         }
 }
 
 void
-i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
+i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct drm_device *dev = obj->dev;
+        struct drm_device *dev = obj->base.dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
-        int page_count = obj->size >> PAGE_SHIFT;
+        int page_count = obj->base.size >> PAGE_SHIFT;
         int i;
 
         if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
                 return;
 
-        if (obj_priv->bit_17 == NULL) {
-                obj_priv->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
+        if (obj->bit_17 == NULL) {
+                obj->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
                                            sizeof(long), GFP_KERNEL);
-                if (obj_priv->bit_17 == NULL) {
+                if (obj->bit_17 == NULL) {
                         DRM_ERROR("Failed to allocate memory for bit 17 "
                                   "record\n");
                         return;
@@ -471,9 +472,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
         }
 
         for (i = 0; i < page_count; i++) {
-                if (page_to_phys(obj_priv->pages[i]) & (1 << 17))
-                        __set_bit(i, obj_priv->bit_17);
+                if (page_to_phys(obj->pages[i]) & (1 << 17))
+                        __set_bit(i, obj->bit_17);
                 else
-                        __clear_bit(i, obj_priv->bit_17);
+                        __clear_bit(i, obj->bit_17);
         }
 }
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 729fd0c91d7b..97f946dcc1aa 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -64,64 +64,24 @@
 #define DRM_I915_VBLANK_PIPE_ALL        (DRM_I915_VBLANK_PIPE_A | \
                                          DRM_I915_VBLANK_PIPE_B)
 
-void
-ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->gt_irq_mask_reg & mask) != 0) {
-                dev_priv->gt_irq_mask_reg &= ~mask;
-                I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
-                (void) I915_READ(GTIMR);
-        }
-}
-
-void
-ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->gt_irq_mask_reg & mask) != mask) {
-                dev_priv->gt_irq_mask_reg |= mask;
-                I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
-                (void) I915_READ(GTIMR);
-        }
-}
-
 /* For display hotplug interrupt */
 static void
 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
-        if ((dev_priv->irq_mask_reg & mask) != 0) {
-                dev_priv->irq_mask_reg &= ~mask;
-                I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(DEIMR);
+        if ((dev_priv->irq_mask & mask) != 0) {
+                dev_priv->irq_mask &= ~mask;
+                I915_WRITE(DEIMR, dev_priv->irq_mask);
+                POSTING_READ(DEIMR);
         }
 }
 
 static inline void
 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
-        if ((dev_priv->irq_mask_reg & mask) != mask) {
-                dev_priv->irq_mask_reg |= mask;
-                I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(DEIMR);
-        }
-}
-
-void
-i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->irq_mask_reg & mask) != 0) {
-                dev_priv->irq_mask_reg &= ~mask;
-                I915_WRITE(IMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(IMR);
-        }
-}
-
-void
-i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
-{
-        if ((dev_priv->irq_mask_reg & mask) != mask) {
-                dev_priv->irq_mask_reg |= mask;
-                I915_WRITE(IMR, dev_priv->irq_mask_reg);
-                (void) I915_READ(IMR);
+        if ((dev_priv->irq_mask & mask) != mask) {
+                dev_priv->irq_mask |= mask;
+                I915_WRITE(DEIMR, dev_priv->irq_mask);
+                POSTING_READ(DEIMR);
         }
 }
 
@@ -144,7 +104,7 @@ i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
                 dev_priv->pipestat[pipe] |= mask;
                 /* Enable the interrupt, clear any pending status */
                 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
-                (void) I915_READ(reg);
+                POSTING_READ(reg);
         }
 }
 
@@ -156,16 +116,19 @@ i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 
                 dev_priv->pipestat[pipe] &= ~mask;
                 I915_WRITE(reg, dev_priv->pipestat[pipe]);
-                (void) I915_READ(reg);
+                POSTING_READ(reg);
         }
 }
 
 /**
  * intel_enable_asle - enable ASLE interrupt for OpRegion
  */
-void intel_enable_asle (struct drm_device *dev)
+void intel_enable_asle(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        unsigned long irqflags;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 
         if (HAS_PCH_SPLIT(dev))
                 ironlake_enable_display_irq(dev_priv, DE_GSE);
@@ -176,6 +139,8 @@ void intel_enable_asle (struct drm_device *dev)
                         i915_enable_pipestat(dev_priv, 0,
                                              PIPE_LEGACY_BLC_EVENT_ENABLE);
         }
+
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
 /**
@@ -243,6 +208,103 @@ u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
         return I915_READ(reg);
 }
 
+int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
+                             int *vpos, int *hpos)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        u32 vbl = 0, position = 0;
+        int vbl_start, vbl_end, htotal, vtotal;
+        bool in_vbl = true;
+        int ret = 0;
+
+        if (!i915_pipe_enabled(dev, pipe)) {
+                DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
+                                        "pipe %d\n", pipe);
+                return 0;
+        }
+
+        /* Get vtotal. */
+        vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff);
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                /* No obvious pixelcount register. Only query vertical
+                 * scanout position from Display scan line register.
+                 */
+                position = I915_READ(PIPEDSL(pipe));
+
+                /* Decode into vertical scanout position. Don't have
+                 * horizontal scanout position.
+                 */
+                *vpos = position & 0x1fff;
+                *hpos = 0;
+        } else {
+                /* Have access to pixelcount since start of frame.
+                 * We can split this into vertical and horizontal
+                 * scanout position.
+                 */
+                position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
+
+                htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff);
+                *vpos = position / htotal;
+                *hpos = position - (*vpos * htotal);
+        }
+
+        /* Query vblank area. */
+        vbl = I915_READ(VBLANK(pipe));
+
+        /* Test position against vblank region. */
+        vbl_start = vbl & 0x1fff;
+        vbl_end = (vbl >> 16) & 0x1fff;
+
+        if ((*vpos < vbl_start) || (*vpos > vbl_end))
+                in_vbl = false;
+
+        /* Inside "upper part" of vblank area? Apply corrective offset: */
+        if (in_vbl && (*vpos >= vbl_start))
+                *vpos = *vpos - vtotal;
+
+        /* Readouts valid? */
+        if (vbl > 0)
+                ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
+
+        /* In vblank? */
+        if (in_vbl)
+                ret |= DRM_SCANOUTPOS_INVBL;
+
+        return ret;
+}
+
+int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
+                              int *max_error,
+                              struct timeval *vblank_time,
+                              unsigned flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc;
+
+        if (pipe < 0 || pipe >= dev_priv->num_pipe) {
+                DRM_ERROR("Invalid crtc %d\n", pipe);
+                return -EINVAL;
+        }
+
+        /* Get drm_crtc to timestamp: */
+        crtc = intel_get_crtc_for_pipe(dev, pipe);
+        if (crtc == NULL) {
+                DRM_ERROR("Invalid crtc %d\n", pipe);
+                return -EINVAL;
+        }
+
+        if (!crtc->enabled) {
+                DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
+                return -EBUSY;
+        }
+
+        /* Helper routine in DRM core does all the work: */
+        return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
+                                                     vblank_time, flags,
+                                                     crtc);
+}
+
 /*
  * Handle hotplug events outside the interrupt handler proper.
  */
@@ -297,20 +359,109 @@ static void notify_ring(struct drm_device *dev,
                         struct intel_ring_buffer *ring)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 seqno = ring->get_seqno(dev, ring);
-        ring->irq_gem_seqno = seqno;
+        u32 seqno;
+
+        if (ring->obj == NULL)
+                return;
+
+        seqno = ring->get_seqno(ring);
         trace_i915_gem_request_complete(dev, seqno);
+
+        ring->irq_seqno = seqno;
         wake_up_all(&ring->irq_queue);
+
         dev_priv->hangcheck_count = 0;
         mod_timer(&dev_priv->hangcheck_timer,
                   jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
 }
 
+static void gen6_pm_irq_handler(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        u8 new_delay = dev_priv->cur_delay;
+        u32 pm_iir;
+
+        pm_iir = I915_READ(GEN6_PMIIR);
+        if (!pm_iir)
+                return;
+
+        if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
+                if (dev_priv->cur_delay != dev_priv->max_delay)
+                        new_delay = dev_priv->cur_delay + 1;
+                if (new_delay > dev_priv->max_delay)
+                        new_delay = dev_priv->max_delay;
+        } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) {
+                if (dev_priv->cur_delay != dev_priv->min_delay)
+                        new_delay = dev_priv->cur_delay - 1;
+                if (new_delay < dev_priv->min_delay) {
+                        new_delay = dev_priv->min_delay;
+                        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                                   I915_READ(GEN6_RP_INTERRUPT_LIMITS) |
+                                   ((new_delay << 16) & 0x3f0000));
+                } else {
+                        /* Make sure we continue to get down interrupts
+                         * until we hit the minimum frequency */
+                        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                                   I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000);
+                }
+
+        }
+
+        gen6_set_rps(dev, new_delay);
+        dev_priv->cur_delay = new_delay;
+
+        I915_WRITE(GEN6_PMIIR, pm_iir);
+}
+
+static void pch_irq_handler(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        u32 pch_iir;
+
+        pch_iir = I915_READ(SDEIIR);
+
+        if (pch_iir & SDE_AUDIO_POWER_MASK)
+                DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
+                                 (pch_iir & SDE_AUDIO_POWER_MASK) >>
+                                 SDE_AUDIO_POWER_SHIFT);
+
+        if (pch_iir & SDE_GMBUS)
+                DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
+
+        if (pch_iir & SDE_AUDIO_HDCP_MASK)
+                DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
+
+        if (pch_iir & SDE_AUDIO_TRANS_MASK)
+                DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
+
+        if (pch_iir & SDE_POISON)
+                DRM_ERROR("PCH poison interrupt\n");
+
+        if (pch_iir & SDE_FDI_MASK) {
+                u32 fdia, fdib;
+
+                fdia = I915_READ(FDI_RXA_IIR);
+                fdib = I915_READ(FDI_RXB_IIR);
+                DRM_DEBUG_DRIVER("PCH FDI RX interrupt; FDI RXA IIR: 0x%08x, FDI RXB IIR: 0x%08x\n", fdia, fdib);
+        }
+
+        if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
+                DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
+
+        if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
+                DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
+
+        if (pch_iir & SDE_TRANSB_FIFO_UNDER)
+                DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n");
+        if (pch_iir & SDE_TRANSA_FIFO_UNDER)
+                DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n");
+}
+
 static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         int ret = IRQ_NONE;
-        u32 de_iir, gt_iir, de_ier, pch_iir;
+        u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
         u32 hotplug_mask;
         struct drm_i915_master_private *master_priv;
         u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT;
@@ -321,13 +472,15 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
         /* disable master interrupt before clearing iir  */
         de_ier = I915_READ(DEIER);
         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
-        (void)I915_READ(DEIER);
+        POSTING_READ(DEIER);
 
         de_iir = I915_READ(DEIIR);
         gt_iir = I915_READ(GTIIR);
         pch_iir = I915_READ(SDEIIR);
+        pm_iir = I915_READ(GEN6_PMIIR);
 
-        if (de_iir == 0 && gt_iir == 0 && pch_iir == 0)
+        if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
+            (!IS_GEN6(dev) || pm_iir == 0))
                 goto done;
 
         if (HAS_PCH_CPT(dev))
@@ -344,12 +497,12 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
                                 READ_BREADCRUMB(dev_priv);
         }
 
-        if (gt_iir & GT_PIPE_NOTIFY)
-                notify_ring(dev, &dev_priv->render_ring);
+        if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
+                notify_ring(dev, &dev_priv->ring[RCS]);
         if (gt_iir & bsd_usr_interrupt)
-                notify_ring(dev, &dev_priv->bsd_ring);
-        if (HAS_BLT(dev) && gt_iir & GT_BLT_USER_INTERRUPT)
-                notify_ring(dev, &dev_priv->blt_ring);
+                notify_ring(dev, &dev_priv->ring[VCS]);
+        if (gt_iir & GT_BLT_USER_INTERRUPT)
+                notify_ring(dev, &dev_priv->ring[BCS]);
 
         if (de_iir & DE_GSE)
                 intel_opregion_gse_intr(dev);
@@ -371,14 +524,20 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
                 drm_handle_vblank(dev, 1);
 
         /* check event from PCH */
-        if ((de_iir & DE_PCH_EVENT) && (pch_iir & hotplug_mask))
-                queue_work(dev_priv->wq, &dev_priv->hotplug_work);
+        if (de_iir & DE_PCH_EVENT) {
+                if (pch_iir & hotplug_mask)
+                        queue_work(dev_priv->wq, &dev_priv->hotplug_work);
+                pch_irq_handler(dev);
+        }
 
         if (de_iir & DE_PCU_EVENT) {
                 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
                 i915_handle_rps_change(dev);
         }
 
+        if (IS_GEN6(dev))
+                gen6_pm_irq_handler(dev);
+
         /* should clear PCH hotplug event before clear CPU irq */
         I915_WRITE(SDEIIR, pch_iir);
         I915_WRITE(GTIIR, gt_iir);
@@ -386,7 +545,7 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
 
 done:
         I915_WRITE(DEIER, de_ier);
-        (void)I915_READ(DEIER);
+        POSTING_READ(DEIER);
 
         return ret;
 }
@@ -422,29 +581,23 @@ static void i915_error_work_func(struct work_struct *work)
 
 #ifdef CONFIG_DEBUG_FS
 static struct drm_i915_error_object *
-i915_error_object_create(struct drm_device *dev,
-                         struct drm_gem_object *src)
+i915_error_object_create(struct drm_i915_private *dev_priv,
+                         struct drm_i915_gem_object *src)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_error_object *dst;
-        struct drm_i915_gem_object *src_priv;
         int page, page_count;
         u32 reloc_offset;
 
-        if (src == NULL)
+        if (src == NULL || src->pages == NULL)
                 return NULL;
 
-        src_priv = to_intel_bo(src);
-        if (src_priv->pages == NULL)
-                return NULL;
-
-        page_count = src->size / PAGE_SIZE;
+        page_count = src->base.size / PAGE_SIZE;
 
         dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC);
         if (dst == NULL)
                 return NULL;
 
-        reloc_offset = src_priv->gtt_offset;
+        reloc_offset = src->gtt_offset;
         for (page = 0; page < page_count; page++) {
                 unsigned long flags;
                 void __iomem *s;
@@ -466,7 +619,7 @@ i915_error_object_create(struct drm_device *dev,
                 reloc_offset += PAGE_SIZE;
         }
         dst->page_count = page_count;
-        dst->gtt_offset = src_priv->gtt_offset;
+        dst->gtt_offset = src->gtt_offset;
 
         return dst;
 
@@ -503,53 +656,98 @@ i915_error_state_free(struct drm_device *dev,
         kfree(error);
 }
 
-static u32
-i915_get_bbaddr(struct drm_device *dev, u32 *ring)
+static u32 capture_bo_list(struct drm_i915_error_buffer *err,
+                           int count,
+                           struct list_head *head)
 {
-        u32 cmd;
+        struct drm_i915_gem_object *obj;
+        int i = 0;
+
+        list_for_each_entry(obj, head, mm_list) {
+                err->size = obj->base.size;
+                err->name = obj->base.name;
+                err->seqno = obj->last_rendering_seqno;
+                err->gtt_offset = obj->gtt_offset;
+                err->read_domains = obj->base.read_domains;
+                err->write_domain = obj->base.write_domain;
+                err->fence_reg = obj->fence_reg;
+                err->pinned = 0;
+                if (obj->pin_count > 0)
+                        err->pinned = 1;
+                if (obj->user_pin_count > 0)
+                        err->pinned = -1;
+                err->tiling = obj->tiling_mode;
+                err->dirty = obj->dirty;
+                err->purgeable = obj->madv != I915_MADV_WILLNEED;
+                err->ring = obj->ring ? obj->ring->id : 0;
+                err->agp_type = obj->agp_type == AGP_USER_CACHED_MEMORY;
+
+                if (++i == count)
+                        break;
 
-        if (IS_I830(dev) || IS_845G(dev))
-                cmd = MI_BATCH_BUFFER;
-        else if (INTEL_INFO(dev)->gen >= 4)
-                cmd = (MI_BATCH_BUFFER_START | (2 << 6) |
-                       MI_BATCH_NON_SECURE_I965);
-        else
-                cmd = (MI_BATCH_BUFFER_START | (2 << 6));
+                err++;
+        }
 
-        return ring[0] == cmd ? ring[1] : 0;
+        return i;
 }
 
-static u32
-i915_ringbuffer_last_batch(struct drm_device *dev)
+static void i915_gem_record_fences(struct drm_device *dev,
+                                   struct drm_i915_error_state *error)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 head, bbaddr;
-        u32 *ring;
+        int i;
+
+        /* Fences */
+        switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                for (i = 0; i < 16; i++)
+                        error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
+                break;
+        case 5:
+        case 4:
+                for (i = 0; i < 16; i++)
+                        error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
+                break;
+        case 3:
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
+        case 2:
+                for (i = 0; i < 8; i++)
+                        error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
+                break;
 
-        /* Locate the current position in the ringbuffer and walk back
-         * to find the most recently dispatched batch buffer.
-         */
-        bbaddr = 0;
-        head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-        ring = (u32 *)(dev_priv->render_ring.virtual_start + head);
-
-        while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) {
-                bbaddr = i915_get_bbaddr(dev, ring);
-                if (bbaddr)
-                        break;
         }
+}
 
-        if (bbaddr == 0) {
-                ring = (u32 *)(dev_priv->render_ring.virtual_start
-                                + dev_priv->render_ring.size);
-                while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) {
-                        bbaddr = i915_get_bbaddr(dev, ring);
-                        if (bbaddr)
-                                break;
-                }
+static struct drm_i915_error_object *
+i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
+                             struct intel_ring_buffer *ring)
+{
+        struct drm_i915_gem_object *obj;
+        u32 seqno;
+
+        if (!ring->get_seqno)
+                return NULL;
+
+        seqno = ring->get_seqno(ring);
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
+                if (obj->ring != ring)
+                        continue;
+
+                if (i915_seqno_passed(seqno, obj->last_rendering_seqno))
+                        continue;
+
+                if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
+                        continue;
+
+                /* We need to copy these to an anonymous buffer as the simplest
+                 * method to avoid being overwritten by userspace.
+                 */
+                return i915_error_object_create(dev_priv, obj);
         }
 
-        return bbaddr;
+        return NULL;
 }
 
 /**
@@ -564,12 +762,10 @@ i915_ringbuffer_last_batch(struct drm_device *dev)
 static void i915_capture_error_state(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct drm_i915_error_state *error;
-        struct drm_gem_object *batchbuffer[2];
         unsigned long flags;
-        u32 bbaddr;
-        int count;
+        int i;
 
         spin_lock_irqsave(&dev_priv->error_lock, flags);
         error = dev_priv->first_error;
@@ -585,20 +781,33 @@ static void i915_capture_error_state(struct drm_device *dev)
 
         DRM_DEBUG_DRIVER("generating error event\n");
 
-        error->seqno =
-                dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring);
+        error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]);
         error->eir = I915_READ(EIR);
         error->pgtbl_er = I915_READ(PGTBL_ER);
         error->pipeastat = I915_READ(PIPEASTAT);
         error->pipebstat = I915_READ(PIPEBSTAT);
         error->instpm = I915_READ(INSTPM);
-        if (INTEL_INFO(dev)->gen < 4) {
-                error->ipeir = I915_READ(IPEIR);
-                error->ipehr = I915_READ(IPEHR);
-                error->instdone = I915_READ(INSTDONE);
-                error->acthd = I915_READ(ACTHD);
-                error->bbaddr = 0;
-        } else {
+        error->error = 0;
+        if (INTEL_INFO(dev)->gen >= 6) {
+                error->error = I915_READ(ERROR_GEN6);
+
+                error->bcs_acthd = I915_READ(BCS_ACTHD);
+                error->bcs_ipehr = I915_READ(BCS_IPEHR);
+                error->bcs_ipeir = I915_READ(BCS_IPEIR);
+                error->bcs_instdone = I915_READ(BCS_INSTDONE);
+                error->bcs_seqno = 0;
+                if (dev_priv->ring[BCS].get_seqno)
+                        error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]);
+
+                error->vcs_acthd = I915_READ(VCS_ACTHD);
+                error->vcs_ipehr = I915_READ(VCS_IPEHR);
+                error->vcs_ipeir = I915_READ(VCS_IPEIR);
+                error->vcs_instdone = I915_READ(VCS_INSTDONE);
+                error->vcs_seqno = 0;
+                if (dev_priv->ring[VCS].get_seqno)
+                        error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]);
+        }
+        if (INTEL_INFO(dev)->gen >= 4) {
                 error->ipeir = I915_READ(IPEIR_I965);
                 error->ipehr = I915_READ(IPEHR_I965);
                 error->instdone = I915_READ(INSTDONE_I965);
@@ -606,118 +815,63 @@ static void i915_capture_error_state(struct drm_device *dev)
                 error->instdone1 = I915_READ(INSTDONE1);
                 error->acthd = I915_READ(ACTHD_I965);
                 error->bbaddr = I915_READ64(BB_ADDR);
+        } else {
+                error->ipeir = I915_READ(IPEIR);
+                error->ipehr = I915_READ(IPEHR);
+                error->instdone = I915_READ(INSTDONE);
+                error->acthd = I915_READ(ACTHD);
+                error->bbaddr = 0;
         }
+        i915_gem_record_fences(dev, error);
 
-        bbaddr = i915_ringbuffer_last_batch(dev);
-
-        /* Grab the current batchbuffer, most likely to have crashed. */
-        batchbuffer[0] = NULL;
-        batchbuffer[1] = NULL;
-        count = 0;
-        list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
-                struct drm_gem_object *obj = &obj_priv->base;
-
-                if (batchbuffer[0] == NULL &&
-                    bbaddr >= obj_priv->gtt_offset &&
-                    bbaddr < obj_priv->gtt_offset + obj->size)
-                        batchbuffer[0] = obj;
-
-                if (batchbuffer[1] == NULL &&
-                    error->acthd >= obj_priv->gtt_offset &&
-                    error->acthd < obj_priv->gtt_offset + obj->size)
-                        batchbuffer[1] = obj;
-
-                count++;
-        }
-        /* Scan the other lists for completeness for those bizarre errors. */
-        if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
-                list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) {
-                        struct drm_gem_object *obj = &obj_priv->base;
-
-                        if (batchbuffer[0] == NULL &&
-                            bbaddr >= obj_priv->gtt_offset &&
-                            bbaddr < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[0] = obj;
-
-                        if (batchbuffer[1] == NULL &&
-                            error->acthd >= obj_priv->gtt_offset &&
-                            error->acthd < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[1] = obj;
-
-                        if (batchbuffer[0] && batchbuffer[1])
-                                break;
-                }
-        }
-        if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
-                list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) {
-                        struct drm_gem_object *obj = &obj_priv->base;
-
-                        if (batchbuffer[0] == NULL &&
-                            bbaddr >= obj_priv->gtt_offset &&
-                            bbaddr < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[0] = obj;
-
-                        if (batchbuffer[1] == NULL &&
-                            error->acthd >= obj_priv->gtt_offset &&
-                            error->acthd < obj_priv->gtt_offset + obj->size)
-                                batchbuffer[1] = obj;
-
-                        if (batchbuffer[0] && batchbuffer[1])
-                                break;
-                }
-        }
-
-        /* We need to copy these to an anonymous buffer as the simplest
-         * method to avoid being overwritten by userspace.
-         */
-        error->batchbuffer[0] = i915_error_object_create(dev, batchbuffer[0]);
-        if (batchbuffer[1] != batchbuffer[0])
-                error->batchbuffer[1] = i915_error_object_create(dev, batchbuffer[1]);
-        else
-                error->batchbuffer[1] = NULL;
+        /* Record the active batchbuffers */
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                error->batchbuffer[i] =
+                        i915_error_first_batchbuffer(dev_priv,
+                                                     &dev_priv->ring[i]);
 
         /* Record the ringbuffer */
-        error->ringbuffer = i915_error_object_create(dev,
-                        dev_priv->render_ring.gem_object);
+        error->ringbuffer = i915_error_object_create(dev_priv,
+                                                     dev_priv->ring[RCS].obj);
 
-        /* Record buffers on the active list. */
+        /* Record buffers on the active and pinned lists. */
         error->active_bo = NULL;
-        error->active_bo_count = 0;
+        error->pinned_bo = NULL;
 
-        if (count)
-                error->active_bo = kmalloc(sizeof(*error->active_bo)*count,
-                                           GFP_ATOMIC);
+        i = 0;
+        list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
+                i++;
+        error->active_bo_count = i;
+        list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
+                i++;
+        error->pinned_bo_count = i - error->active_bo_count;
 
-        if (error->active_bo) {
-                int i = 0;
-                list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
-                        struct drm_gem_object *obj = &obj_priv->base;
-
-                        error->active_bo[i].size = obj->size;
-                        error->active_bo[i].name = obj->name;
-                        error->active_bo[i].seqno = obj_priv->last_rendering_seqno;
-                        error->active_bo[i].gtt_offset = obj_priv->gtt_offset;
-                        error->active_bo[i].read_domains = obj->read_domains;
-                        error->active_bo[i].write_domain = obj->write_domain;
-                        error->active_bo[i].fence_reg = obj_priv->fence_reg;
-                        error->active_bo[i].pinned = 0;
-                        if (obj_priv->pin_count > 0)
-                                error->active_bo[i].pinned = 1;
-                        if (obj_priv->user_pin_count > 0)
-                                error->active_bo[i].pinned = -1;
-                        error->active_bo[i].tiling = obj_priv->tiling_mode;
-                        error->active_bo[i].dirty = obj_priv->dirty;
-                        error->active_bo[i].purgeable = obj_priv->madv != I915_MADV_WILLNEED;
-
-                        if (++i == count)
-                                break;
-                }
-                error->active_bo_count = i;
+        error->active_bo = NULL;
+        error->pinned_bo = NULL;
+        if (i) {
+                error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
+                                           GFP_ATOMIC);
+                if (error->active_bo)
+                        error->pinned_bo =
+                                error->active_bo + error->active_bo_count;
         }
 
+        if (error->active_bo)
+                error->active_bo_count =
+                        capture_bo_list(error->active_bo,
+                                        error->active_bo_count,
+                                        &dev_priv->mm.active_list);
+
+        if (error->pinned_bo)
+                error->pinned_bo_count =
+                        capture_bo_list(error->pinned_bo,
+                                        error->pinned_bo_count,
+                                        &dev_priv->mm.pinned_list);
+
         do_gettimeofday(&error->time);
 
         error->overlay = intel_overlay_capture_error_state(dev);
+        error->display = intel_display_capture_error_state(dev);
 
         spin_lock_irqsave(&dev_priv->error_lock, flags);
         if (dev_priv->first_error == NULL) {
@@ -775,7 +929,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
                                I915_READ(ACTHD_I965));
                         I915_WRITE(IPEIR_I965, ipeir);
-                        (void)I915_READ(IPEIR_I965);
+                        POSTING_READ(IPEIR_I965);
                 }
                 if (eir & GM45_ERROR_PAGE_TABLE) {
                         u32 pgtbl_err = I915_READ(PGTBL_ER);
@@ -783,7 +937,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
                                pgtbl_err);
                         I915_WRITE(PGTBL_ER, pgtbl_err);
-                        (void)I915_READ(PGTBL_ER);
+                        POSTING_READ(PGTBL_ER);
                 }
         }
 
@@ -794,7 +948,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
                                pgtbl_err);
                         I915_WRITE(PGTBL_ER, pgtbl_err);
-                        (void)I915_READ(PGTBL_ER);
+                        POSTING_READ(PGTBL_ER);
                 }
         }
 
@@ -825,7 +979,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
                                I915_READ(ACTHD));
                         I915_WRITE(IPEIR, ipeir);
-                        (void)I915_READ(IPEIR);
+                        POSTING_READ(IPEIR);
                 } else {
                         u32 ipeir = I915_READ(IPEIR_I965);
 
@@ -842,12 +996,12 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                         printk(KERN_ERR "  ACTHD: 0x%08x\n",
                                I915_READ(ACTHD_I965));
                         I915_WRITE(IPEIR_I965, ipeir);
-                        (void)I915_READ(IPEIR_I965);
+                        POSTING_READ(IPEIR_I965);
                 }
         }
 
         I915_WRITE(EIR, eir);
-        (void)I915_READ(EIR);
+        POSTING_READ(EIR);
         eir = I915_READ(EIR);
         if (eir) {
                 /*
@@ -870,7 +1024,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
  * so userspace knows something bad happened (should trigger collection
  * of a ring dump etc.).
  */
-static void i915_handle_error(struct drm_device *dev, bool wedged)
+void i915_handle_error(struct drm_device *dev, bool wedged)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -884,11 +1038,11 @@ static void i915_handle_error(struct drm_device *dev, bool wedged)
                 /*
                  * Wakeup waiting processes so they don't hang
                  */
-                wake_up_all(&dev_priv->render_ring.irq_queue);
+                wake_up_all(&dev_priv->ring[RCS].irq_queue);
                 if (HAS_BSD(dev))
-                        wake_up_all(&dev_priv->bsd_ring.irq_queue);
+                        wake_up_all(&dev_priv->ring[VCS].irq_queue);
                 if (HAS_BLT(dev))
-                        wake_up_all(&dev_priv->blt_ring.irq_queue);
+                        wake_up_all(&dev_priv->ring[BCS].irq_queue);
         }
 
         queue_work(dev_priv->wq, &dev_priv->error_work);
@@ -899,7 +1053,7 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct intel_unpin_work *work;
         unsigned long flags;
         bool stall_detected;
@@ -918,13 +1072,13 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
         }
 
         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
-        obj_priv = to_intel_bo(work->pending_flip_obj);
+        obj = work->pending_flip_obj;
         if (INTEL_INFO(dev)->gen >= 4) {
                 int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF;
-                stall_detected = I915_READ(dspsurf) == obj_priv->gtt_offset;
+                stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
         } else {
                 int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR;
-                stall_detected = I915_READ(dspaddr) == (obj_priv->gtt_offset +
+                stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
                                                         crtc->y * crtc->fb->pitch +
                                                         crtc->x * crtc->fb->bits_per_pixel/8);
         }
@@ -970,7 +1124,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                  * It doesn't set the bit in iir again, but it still produces
                  * interrupts (for non-MSI).
                  */
-                spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 pipea_stats = I915_READ(PIPEASTAT);
                 pipeb_stats = I915_READ(PIPEBSTAT);
 
@@ -993,7 +1147,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                         I915_WRITE(PIPEBSTAT, pipeb_stats);
                         irq_received = 1;
                 }
-                spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
                 if (!irq_received)
                         break;
@@ -1026,9 +1180,9 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                 }
 
                 if (iir & I915_USER_INTERRUPT)
-                        notify_ring(dev, &dev_priv->render_ring);
-                if (HAS_BSD(dev) && (iir & I915_BSD_USER_INTERRUPT))
-                        notify_ring(dev, &dev_priv->bsd_ring);
+                        notify_ring(dev, &dev_priv->ring[RCS]);
+                if (iir & I915_BSD_USER_INTERRUPT)
+                        notify_ring(dev, &dev_priv->ring[VCS]);
 
                 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
                         intel_prepare_page_flip(dev, 0);
@@ -1042,18 +1196,18 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                                 intel_finish_page_flip_plane(dev, 1);
                 }
 
-                if (pipea_stats & vblank_status) {
+                if (pipea_stats & vblank_status &&
+                    drm_handle_vblank(dev, 0)) {
                         vblank++;
-                        drm_handle_vblank(dev, 0);
                         if (!dev_priv->flip_pending_is_done) {
                                 i915_pageflip_stall_check(dev, 0);
                                 intel_finish_page_flip(dev, 0);
                         }
                 }
 
-                if (pipeb_stats & vblank_status) {
+                if (pipeb_stats & vblank_status &&
+                    drm_handle_vblank(dev, 1)) {
                         vblank++;
-                        drm_handle_vblank(dev, 1);
                         if (!dev_priv->flip_pending_is_done) {
                                 i915_pageflip_stall_check(dev, 1);
                                 intel_finish_page_flip(dev, 1);
@@ -1101,12 +1255,13 @@ static int i915_emit_irq(struct drm_device * dev)
         if (master_priv->sarea_priv)
                 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
-        BEGIN_LP_RING(4);
-        OUT_RING(MI_STORE_DWORD_INDEX);
-        OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        OUT_RING(dev_priv->counter);
-        OUT_RING(MI_USER_INTERRUPT);
-        ADVANCE_LP_RING();
+        if (BEGIN_LP_RING(4) == 0) {
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(dev_priv->counter);
+                OUT_RING(MI_USER_INTERRUPT);
+                ADVANCE_LP_RING();
+        }
 
         return dev_priv->counter;
 }
@@ -1114,12 +1269,11 @@ static int i915_emit_irq(struct drm_device * dev)
 void i915_trace_irq_get(struct drm_device *dev, u32 seqno)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
-
-        if (dev_priv->trace_irq_seqno == 0)
-                render_ring->user_irq_get(dev, render_ring);
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
-        dev_priv->trace_irq_seqno = seqno;
+        if (dev_priv->trace_irq_seqno == 0 &&
+            ring->irq_get(ring))
+                dev_priv->trace_irq_seqno = seqno;
 }
 
 static int i915_wait_irq(struct drm_device * dev, int irq_nr)
@@ -1127,7 +1281,7 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
         int ret = 0;
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
+        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
                   READ_BREADCRUMB(dev_priv));
@@ -1141,10 +1295,12 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
         if (master_priv->sarea_priv)
                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 
-        render_ring->user_irq_get(dev, render_ring);
-        DRM_WAIT_ON(ret, dev_priv->render_ring.irq_queue, 3 * DRM_HZ,
-                    READ_BREADCRUMB(dev_priv) >= irq_nr);
-        render_ring->user_irq_put(dev, render_ring);
+        if (ring->irq_get(ring)) {
+                DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ,
+                            READ_BREADCRUMB(dev_priv) >= irq_nr);
+                ring->irq_put(ring);
+        } else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000))
+                ret = -EBUSY;
 
         if (ret == -EBUSY) {
                 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
@@ -1163,7 +1319,7 @@ int i915_irq_emit(struct drm_device *dev, void *data,
         drm_i915_irq_emit_t *emit = data;
         int result;
 
-        if (!dev_priv || !dev_priv->render_ring.virtual_start) {
+        if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
                 DRM_ERROR("called with no initialization\n");
                 return -EINVAL;
         }
@@ -1209,9 +1365,9 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
         if (!i915_pipe_enabled(dev, pipe))
                 return -EINVAL;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
         if (HAS_PCH_SPLIT(dev))
-                ironlake_enable_display_irq(dev_priv, (pipe == 0) ? 
+                ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
                                             DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
         else if (INTEL_INFO(dev)->gen >= 4)
                 i915_enable_pipestat(dev_priv, pipe,
@@ -1219,7 +1375,7 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
         else
                 i915_enable_pipestat(dev_priv, pipe,
                                      PIPE_VBLANK_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
         return 0;
 }
 
@@ -1231,15 +1387,15 @@ void i915_disable_vblank(struct drm_device *dev, int pipe)
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         unsigned long irqflags;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
         if (HAS_PCH_SPLIT(dev))
-                ironlake_disable_display_irq(dev_priv, (pipe == 0) ? 
+                ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
                                              DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
         else
                 i915_disable_pipestat(dev_priv, pipe,
                                       PIPE_VBLANK_INTERRUPT_ENABLE |
                                       PIPE_START_VBLANK_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
 void i915_enable_interrupt (struct drm_device *dev)
@@ -1306,12 +1462,50 @@ int i915_vblank_swap(struct drm_device *dev, void *data,
         return -EINVAL;
 }
 
-static struct drm_i915_gem_request *
-i915_get_tail_request(struct drm_device *dev)
+static u32
+ring_last_seqno(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        return list_entry(dev_priv->render_ring.request_list.prev,
-                        struct drm_i915_gem_request, list);
+        return list_entry(ring->request_list.prev,
+                          struct drm_i915_gem_request, list)->seqno;
+}
+
+static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
+{
+        if (list_empty(&ring->request_list) ||
+            i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
+                /* Issue a wake-up to catch stuck h/w. */
+                if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) {
+                        DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
+                                  ring->name,
+                                  ring->waiting_seqno,
+                                  ring->get_seqno(ring));
+                        wake_up_all(&ring->irq_queue);
+                        *err = true;
+                }
+                return true;
+        }
+        return false;
+}
+
+static bool kick_ring(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp = I915_READ_CTL(ring);
+        if (tmp & RING_WAIT) {
+                DRM_ERROR("Kicking stuck wait on %s\n",
+                          ring->name);
+                I915_WRITE_CTL(ring, tmp);
+                return true;
+        }
+        if (IS_GEN6(dev) &&
+            (tmp & RING_WAIT_SEMAPHORE)) {
+                DRM_ERROR("Kicking stuck semaphore on %s\n",
+                          ring->name);
+                I915_WRITE_CTL(ring, tmp);
+                return true;
+        }
+        return false;
 }
 
 /**
@@ -1325,6 +1519,17 @@ void i915_hangcheck_elapsed(unsigned long data)
         struct drm_device *dev = (struct drm_device *)data;
         drm_i915_private_t *dev_priv = dev->dev_private;
         uint32_t acthd, instdone, instdone1;
+        bool err = false;
+
+        /* If all work is done then ACTHD clearly hasn't advanced. */
+        if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
+            i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
+            i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) {
+                dev_priv->hangcheck_count = 0;
+                if (err)
+                        goto repeat;
+                return;
+        }
 
         if (INTEL_INFO(dev)->gen < 4) {
                 acthd = I915_READ(ACTHD);
@@ -1336,38 +1541,6 @@ void i915_hangcheck_elapsed(unsigned long data)
                 instdone1 = I915_READ(INSTDONE1);
         }
 
-        /* If all work is done then ACTHD clearly hasn't advanced. */
-        if (list_empty(&dev_priv->render_ring.request_list) ||
-                i915_seqno_passed(dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring),
-                                  i915_get_tail_request(dev)->seqno)) {
-                bool missed_wakeup = false;
-
-                dev_priv->hangcheck_count = 0;
-
-                /* Issue a wake-up to catch stuck h/w. */
-                if (dev_priv->render_ring.waiting_gem_seqno &&
-                    waitqueue_active(&dev_priv->render_ring.irq_queue)) {
-                        wake_up_all(&dev_priv->render_ring.irq_queue);
-                        missed_wakeup = true;
-                }
-
-                if (dev_priv->bsd_ring.waiting_gem_seqno &&
-                    waitqueue_active(&dev_priv->bsd_ring.irq_queue)) {
-                        wake_up_all(&dev_priv->bsd_ring.irq_queue);
-                        missed_wakeup = true;
-                }
-
-                if (dev_priv->blt_ring.waiting_gem_seqno &&
-                    waitqueue_active(&dev_priv->blt_ring.irq_queue)) {
-                        wake_up_all(&dev_priv->blt_ring.irq_queue);
-                        missed_wakeup = true;
-                }
-
-                if (missed_wakeup)
-                        DRM_ERROR("Hangcheck timer elapsed... GPU idle, missed IRQ.\n");
-                return;
-        }
-
         if (dev_priv->last_acthd == acthd &&
             dev_priv->last_instdone == instdone &&
             dev_priv->last_instdone1 == instdone1) {
@@ -1380,12 +1553,17 @@ void i915_hangcheck_elapsed(unsigned long data)
                                  * and break the hang. This should work on
                                  * all but the second generation chipsets.
                                  */
-                                u32 tmp = I915_READ(PRB0_CTL);
-                                if (tmp & RING_WAIT) {
-                                        I915_WRITE(PRB0_CTL, tmp);
-                                        POSTING_READ(PRB0_CTL);
-                                        goto out;
-                                }
+
+                                if (kick_ring(&dev_priv->ring[RCS]))
+                                        goto repeat;
+
+                                if (HAS_BSD(dev) &&
+                                    kick_ring(&dev_priv->ring[VCS]))
+                                        goto repeat;
+
+                                if (HAS_BLT(dev) &&
+                                    kick_ring(&dev_priv->ring[BCS]))
+                                        goto repeat;
                         }
 
                         i915_handle_error(dev, true);
@@ -1399,7 +1577,7 @@ void i915_hangcheck_elapsed(unsigned long data)
                 dev_priv->last_instdone1 = instdone1;
         }
 
-out:
+repeat:
         /* Reset timer case chip hangs without another request being added */
         mod_timer(&dev_priv->hangcheck_timer,
                   jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
@@ -1417,17 +1595,17 @@ static void ironlake_irq_preinstall(struct drm_device *dev)
 
         I915_WRITE(DEIMR, 0xffffffff);
         I915_WRITE(DEIER, 0x0);
-        (void) I915_READ(DEIER);
+        POSTING_READ(DEIER);
 
         /* and GT */
         I915_WRITE(GTIMR, 0xffffffff);
         I915_WRITE(GTIER, 0x0);
-        (void) I915_READ(GTIER);
+        POSTING_READ(GTIER);
 
         /* south display irq */
         I915_WRITE(SDEIMR, 0xffffffff);
         I915_WRITE(SDEIER, 0x0);
-        (void) I915_READ(SDEIER);
+        POSTING_READ(SDEIER);
 }
 
 static int ironlake_irq_postinstall(struct drm_device *dev)
@@ -1436,38 +1614,34 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
         /* enable kind of interrupts always enabled */
         u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
                            DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
-        u32 render_mask = GT_PIPE_NOTIFY | GT_BSD_USER_INTERRUPT;
+        u32 render_irqs;
         u32 hotplug_mask;
 
-        dev_priv->irq_mask_reg = ~display_mask;
-        dev_priv->de_irq_enable_reg = display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK;
+        dev_priv->irq_mask = ~display_mask;
 
         /* should always can generate irq */
         I915_WRITE(DEIIR, I915_READ(DEIIR));
-        I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
-        I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
-        (void) I915_READ(DEIER);
-
-        if (IS_GEN6(dev)) {
-                render_mask =
-                        GT_PIPE_NOTIFY |
-                        GT_GEN6_BSD_USER_INTERRUPT |
-                        GT_BLT_USER_INTERRUPT;
-        }
+        I915_WRITE(DEIMR, dev_priv->irq_mask);
+        I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
+        POSTING_READ(DEIER);
 
-        dev_priv->gt_irq_mask_reg = ~render_mask;
-        dev_priv->gt_irq_enable_reg = render_mask;
+        dev_priv->gt_irq_mask = ~0;
 
         I915_WRITE(GTIIR, I915_READ(GTIIR));
-        I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
-        if (IS_GEN6(dev)) {
-                I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT);
-                I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_IMR_USER_INTERRUPT);
-                I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT);
-        }
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 
-        I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg);
-        (void) I915_READ(GTIER);
+        if (IS_GEN6(dev))
+                render_irqs =
+                        GT_USER_INTERRUPT |
+                        GT_GEN6_BSD_USER_INTERRUPT |
+                        GT_BLT_USER_INTERRUPT;
+        else
+                render_irqs =
+                        GT_USER_INTERRUPT |
+                        GT_PIPE_NOTIFY |
+                        GT_BSD_USER_INTERRUPT;
+        I915_WRITE(GTIER, render_irqs);
+        POSTING_READ(GTIER);
 
         if (HAS_PCH_CPT(dev)) {
                 hotplug_mask = SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT  |
@@ -1475,15 +1649,17 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
         } else {
                 hotplug_mask = SDE_CRT_HOTPLUG | SDE_PORTB_HOTPLUG |
                                SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG;
+                hotplug_mask |= SDE_AUX_MASK | SDE_FDI_MASK | SDE_TRANS_MASK;
+                I915_WRITE(FDI_RXA_IMR, 0);
+                I915_WRITE(FDI_RXB_IMR, 0);
         }
 
-        dev_priv->pch_irq_mask_reg = ~hotplug_mask;
-        dev_priv->pch_irq_enable_reg = hotplug_mask;
+        dev_priv->pch_irq_mask = ~hotplug_mask;
 
         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
-        I915_WRITE(SDEIMR, dev_priv->pch_irq_mask_reg);
-        I915_WRITE(SDEIER, dev_priv->pch_irq_enable_reg);
-        (void) I915_READ(SDEIER);
+        I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
+        I915_WRITE(SDEIER, hotplug_mask);
+        POSTING_READ(SDEIER);
 
         if (IS_IRONLAKE_M(dev)) {
                 /* Clear & enable PCU event interrupts */
@@ -1519,7 +1695,7 @@ void i915_driver_irq_preinstall(struct drm_device * dev)
         I915_WRITE(PIPEBSTAT, 0);
         I915_WRITE(IMR, 0xffffffff);
         I915_WRITE(IER, 0x0);
-        (void) I915_READ(IER);
+        POSTING_READ(IER);
 }
 
 /*
@@ -1532,11 +1708,11 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
         u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
         u32 error_mask;
 
-        DRM_INIT_WAITQUEUE(&dev_priv->render_ring.irq_queue);
+        DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
         if (HAS_BSD(dev))
-                DRM_INIT_WAITQUEUE(&dev_priv->bsd_ring.irq_queue);
+                DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
         if (HAS_BLT(dev))
-                DRM_INIT_WAITQUEUE(&dev_priv->blt_ring.irq_queue);
+                DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);
 
         dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 
@@ -1544,7 +1720,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
                 return ironlake_irq_postinstall(dev);
 
         /* Unmask the interrupts that we always want on. */
-        dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
+        dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
 
         dev_priv->pipestat[0] = 0;
         dev_priv->pipestat[1] = 0;
@@ -1553,7 +1729,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
                 /* Enable in IER... */
                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
                 /* and unmask in IMR */
-                dev_priv->irq_mask_reg &= ~I915_DISPLAY_PORT_INTERRUPT;
+                dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
         }
 
         /*
@@ -1571,9 +1747,9 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
         }
         I915_WRITE(EMR, error_mask);
 
-        I915_WRITE(IMR, dev_priv->irq_mask_reg);
+        I915_WRITE(IMR, dev_priv->irq_mask);
         I915_WRITE(IER, enable_mask);
-        (void) I915_READ(IER);
+        POSTING_READ(IER);
 
         if (I915_HAS_HOTPLUG(dev)) {
                 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 25ed911a3112..5cfc68940f17 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -78,6 +78,12 @@
 #define  GRDOM_RENDER   (1<<2)
 #define  GRDOM_MEDIA    (3<<2)
 
+#define GEN6_GDRST      0x941c
+#define  GEN6_GRDOM_FULL                (1 << 0)
+#define  GEN6_GRDOM_RENDER              (1 << 1)
+#define  GEN6_GRDOM_MEDIA               (1 << 2)
+#define  GEN6_GRDOM_BLT                 (1 << 3)
+
 /* VGA stuff */
 
 #define VGA_ST01_MDA 0x3ba
@@ -139,6 +145,8 @@
 #define   MI_END_SCENE          (1 << 4) /* flush binner and incr scene count */
 #define   MI_INVALIDATE_ISP     (1 << 5) /* invalidate indirect state pointers */
 #define MI_BATCH_BUFFER_END     MI_INSTR(0x0a, 0)
+#define MI_SUSPEND_FLUSH        MI_INSTR(0x0b, 0)
+#define   MI_SUSPEND_FLUSH_EN   (1<<0)
 #define MI_REPORT_HEAD          MI_INSTR(0x07, 0)
 #define MI_OVERLAY_FLIP         MI_INSTR(0x11,0)
 #define   MI_OVERLAY_CONTINUE   (0x0<<21)
@@ -153,17 +161,29 @@
 #define   MI_MM_SPACE_PHYSICAL          (0<<8)
 #define   MI_SAVE_EXT_STATE_EN          (1<<3)
 #define   MI_RESTORE_EXT_STATE_EN       (1<<2)
+#define   MI_FORCE_RESTORE              (1<<1)
 #define   MI_RESTORE_INHIBIT            (1<<0)
 #define MI_STORE_DWORD_IMM      MI_INSTR(0x20, 1)
 #define   MI_MEM_VIRTUAL        (1 << 22) /* 965+ only */
 #define MI_STORE_DWORD_INDEX    MI_INSTR(0x21, 1)
 #define   MI_STORE_DWORD_INDEX_SHIFT 2
-#define MI_LOAD_REGISTER_IMM    MI_INSTR(0x22, 1)
+/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
+ * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
+ *   simply ignores the register load under certain conditions.
+ * - One can actually load arbitrary many arbitrary registers: Simply issue x
+ *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
+ */
+#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1)
 #define MI_FLUSH_DW             MI_INSTR(0x26, 2) /* for GEN6 */
 #define MI_BATCH_BUFFER         MI_INSTR(0x30, 1)
 #define   MI_BATCH_NON_SECURE   (1)
 #define   MI_BATCH_NON_SECURE_I965 (1<<8)
 #define MI_BATCH_BUFFER_START   MI_INSTR(0x31, 0)
+#define MI_SEMAPHORE_MBOX       MI_INSTR(0x16, 1) /* gen6+ */
+#define  MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
+#define  MI_SEMAPHORE_UPDATE        (1<<21)
+#define  MI_SEMAPHORE_COMPARE       (1<<20)
+#define  MI_SEMAPHORE_REGISTER      (1<<18)
 /*
  * 3D instructions used by the kernel
  */
@@ -256,10 +276,6 @@
  * Instruction and interrupt control regs
  */
 #define PGTBL_ER        0x02024
-#define PRB0_TAIL       0x02030
-#define PRB0_HEAD       0x02034
-#define PRB0_START      0x02038
-#define PRB0_CTL        0x0203c
 #define RENDER_RING_BASE        0x02000
 #define BSD_RING_BASE           0x04000
 #define GEN6_BSD_RING_BASE      0x12000
@@ -268,9 +284,14 @@
 #define RING_HEAD(base)         ((base)+0x34)
 #define RING_START(base)        ((base)+0x38)
 #define RING_CTL(base)          ((base)+0x3c)
+#define RING_SYNC_0(base)       ((base)+0x40)
+#define RING_SYNC_1(base)       ((base)+0x44)
+#define RING_MAX_IDLE(base)     ((base)+0x54)
 #define RING_HWS_PGA(base)      ((base)+0x80)
 #define RING_HWS_PGA_GEN6(base) ((base)+0x2080)
 #define RING_ACTHD(base)        ((base)+0x74)
+#define RING_NOPID(base)        ((base)+0x94)
+#define RING_IMR(base)          ((base)+0xa8)
 #define   TAIL_ADDR             0x001FFFF8
 #define   HEAD_WRAP_COUNT       0xFFE00000
 #define   HEAD_WRAP_ONE         0x00200000
@@ -285,10 +306,17 @@
 #define   RING_INVALID          0x00000000
 #define   RING_WAIT_I8XX        (1<<0) /* gen2, PRBx_HEAD */
 #define   RING_WAIT             (1<<11) /* gen3+, PRBx_CTL */
+#define   RING_WAIT_SEMAPHORE   (1<<10) /* gen6+ */
+#if 0
+#define PRB0_TAIL       0x02030
+#define PRB0_HEAD       0x02034
+#define PRB0_START      0x02038
+#define PRB0_CTL        0x0203c
 #define PRB1_TAIL       0x02040 /* 915+ only */
 #define PRB1_HEAD       0x02044 /* 915+ only */
 #define PRB1_START      0x02048 /* 915+ only */
 #define PRB1_CTL        0x0204c /* 915+ only */
+#endif
 #define IPEIR_I965      0x02064
 #define IPEHR_I965      0x02068
 #define INSTDONE_I965   0x0206c
@@ -305,11 +333,42 @@
 #define INSTDONE        0x02090
 #define NOPID           0x02094
 #define HWSTAM          0x02098
+#define VCS_INSTDONE    0x1206C
+#define VCS_IPEIR       0x12064
+#define VCS_IPEHR       0x12068
+#define VCS_ACTHD       0x12074
+#define BCS_INSTDONE    0x2206C
+#define BCS_IPEIR       0x22064
+#define BCS_IPEHR       0x22068
+#define BCS_ACTHD       0x22074
+
+#define ERROR_GEN6      0x040a0
+
+/* GM45+ chicken bits -- debug workaround bits that may be required
+ * for various sorts of correct behavior.  The top 16 bits of each are
+ * the enables for writing to the corresponding low bit.
+ */
+#define _3D_CHICKEN     0x02084
+#define _3D_CHICKEN2    0x0208c
+/* Disables pipelining of read flushes past the SF-WIZ interface.
+ * Required on all Ironlake steppings according to the B-Spec, but the
+ * particular danger of not doing so is not specified.
+ */
+# define _3D_CHICKEN2_WM_READ_PIPELINED                 (1 << 14)
+#define _3D_CHICKEN3    0x02090
 
 #define MI_MODE         0x0209c
 # define VS_TIMER_DISPATCH                              (1 << 6)
 # define MI_FLUSH_ENABLE                                (1 << 11)
 
+#define GFX_MODE        0x02520
+#define   GFX_RUN_LIST_ENABLE           (1<<15)
+#define   GFX_TLB_INVALIDATE_ALWAYS     (1<<13)
+#define   GFX_SURFACE_FAULT_ENABLE      (1<<12)
+#define   GFX_REPLAY_MODE               (1<<11)
+#define   GFX_PSMI_GRANULARITY          (1<<10)
+#define   GFX_PPGTT_ENABLE              (1<<9)
+
 #define SCPD0           0x0209c /* 915+ only */
 #define IER             0x020a0
 #define IIR             0x020a4
@@ -454,6 +513,10 @@
 #define   GEN6_BLITTER_SYNC_STATUS                      (1 << 24)
 #define   GEN6_BLITTER_USER_INTERRUPT                   (1 << 22)
 
+#define GEN6_BLITTER_ECOSKPD    0x221d0
+#define   GEN6_BLITTER_LOCK_SHIFT                       16
+#define   GEN6_BLITTER_FBC_NOTIFY                       (1<<3)
+
 #define GEN6_BSD_SLEEP_PSMI_CONTROL     0x12050
 #define   GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK       (1 << 16)
 #define   GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE           (1 << 0)
@@ -461,7 +524,7 @@
 #define   GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR                    (1 << 3)
 
 #define GEN6_BSD_IMR                    0x120a8
-#define   GEN6_BSD_IMR_USER_INTERRUPT   (1 << 12)
+#define   GEN6_BSD_USER_INTERRUPT       (1 << 12)
 
 #define GEN6_BSD_RNCID                  0x12198
 
@@ -541,6 +604,18 @@
 
 #define ILK_DISPLAY_CHICKEN1    0x42000
 #define   ILK_FBCQ_DIS          (1<<22)
+#define   ILK_PABSTRETCH_DIS    (1<<21)
+
+
+/*
+ * Framebuffer compression for Sandybridge
+ *
+ * The following two registers are of type GTTMMADR
+ */
+#define SNB_DPFC_CTL_SA         0x100100
+#define   SNB_CPU_FENCE_ENABLE  (1<<29)
+#define DPFC_CPU_FENCE_OFFSET   0x100104
+
 
 /*
  * GPIO regs
@@ -900,6 +975,8 @@
  */
 #define MCHBAR_MIRROR_BASE      0x10000
 
+#define MCHBAR_MIRROR_BASE_SNB  0x140000
+
 /** 915-945 and GM965 MCH register controlling DRAM channel access */
 #define DCC                     0x10200
 #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL              (0 << 0)
@@ -1061,9 +1138,50 @@
 #define RCBMINAVG               0x111a0
 #define RCUPEI                  0x111b0
 #define RCDNEI                  0x111b4
-#define MCHBAR_RENDER_STANDBY           0x111b8
-#define   RCX_SW_EXIT           (1<<23)
-#define   RSX_STATUS_MASK       0x00700000
+#define RSTDBYCTL               0x111b8
+#define   RS1EN                 (1<<31)
+#define   RS2EN                 (1<<30)
+#define   RS3EN                 (1<<29)
+#define   D3RS3EN               (1<<28) /* Display D3 imlies RS3 */
+#define   SWPROMORSX            (1<<27) /* RSx promotion timers ignored */
+#define   RCWAKERW              (1<<26) /* Resetwarn from PCH causes wakeup */
+#define   DPRSLPVREN            (1<<25) /* Fast voltage ramp enable */
+#define   GFXTGHYST             (1<<24) /* Hysteresis to allow trunk gating */
+#define   RCX_SW_EXIT           (1<<23) /* Leave RSx and prevent re-entry */
+#define   RSX_STATUS_MASK       (7<<20)
+#define   RSX_STATUS_ON         (0<<20)
+#define   RSX_STATUS_RC1        (1<<20)
+#define   RSX_STATUS_RC1E       (2<<20)
+#define   RSX_STATUS_RS1        (3<<20)
+#define   RSX_STATUS_RS2        (4<<20) /* aka rc6 */
+#define   RSX_STATUS_RSVD       (5<<20) /* deep rc6 unsupported on ilk */
+#define   RSX_STATUS_RS3        (6<<20) /* rs3 unsupported on ilk */
+#define   RSX_STATUS_RSVD2      (7<<20)
+#define   UWRCRSXE              (1<<19) /* wake counter limit prevents rsx */
+#define   RSCRP                 (1<<18) /* rs requests control on rs1/2 reqs */
+#define   JRSC                  (1<<17) /* rsx coupled to cpu c-state */
+#define   RS2INC0               (1<<16) /* allow rs2 in cpu c0 */
+#define   RS1CONTSAV_MASK       (3<<14)
+#define   RS1CONTSAV_NO_RS1     (0<<14) /* rs1 doesn't save/restore context */
+#define   RS1CONTSAV_RSVD       (1<<14)
+#define   RS1CONTSAV_SAVE_RS1   (2<<14) /* rs1 saves context */
+#define   RS1CONTSAV_FULL_RS1   (3<<14) /* rs1 saves and restores context */
+#define   NORMSLEXLAT_MASK      (3<<12)
+#define   SLOW_RS123            (0<<12)
+#define   SLOW_RS23             (1<<12)
+#define   SLOW_RS3              (2<<12)
+#define   NORMAL_RS123          (3<<12)
+#define   RCMODE_TIMEOUT        (1<<11) /* 0 is eval interval method */
+#define   IMPROMOEN             (1<<10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */
+#define   RCENTSYNC             (1<<9) /* rs coupled to cpu c-state (3/6/7) */
+#define   STATELOCK             (1<<7) /* locked to rs_cstate if 0 */
+#define   RS_CSTATE_MASK        (3<<4)
+#define   RS_CSTATE_C367_RS1    (0<<4)
+#define   RS_CSTATE_C36_RS1_C7_RS2 (1<<4)
+#define   RS_CSTATE_RSVD        (2<<4)
+#define   RS_CSTATE_C367_RS2    (3<<4)
+#define   REDSAVES              (1<<3) /* no context save if was idle during rs0 */
+#define   REDRESTORES           (1<<2) /* no restore if was idle during rs0 */
 #define VIDCTL                  0x111c0
 #define VIDSTS                  0x111c8
 #define VIDSTART                0x111cc /* 8 bits */
@@ -1119,6 +1237,10 @@
 #define DDRMPLL1                0X12c20
 #define PEG_BAND_GAP_DATA       0x14d68
 
+#define GEN6_GT_PERF_STATUS     0x145948
+#define GEN6_RP_STATE_LIMITS    0x145994
+#define GEN6_RP_STATE_CAP       0x145998
+
 /*
  * Logical Context regs
  */
@@ -1168,7 +1290,6 @@
 #define VTOTAL(pipe) _PIPE(pipe, VTOTAL_A, VTOTAL_B)
 #define VBLANK(pipe) _PIPE(pipe, VBLANK_A, VBLANK_B)
 #define VSYNC(pipe) _PIPE(pipe, VSYNC_A, VSYNC_B)
-#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC)
 #define BCLRPAT(pipe) _PIPE(pipe, BCLRPAT_A, BCLRPAT_B)
 
 /* VGA port control */
@@ -2182,8 +2303,10 @@
 #define   PIPE_6BPC                             (2 << 5)
 #define   PIPE_12BPC                            (3 << 5)
 
+#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC)
 #define PIPECONF(pipe) _PIPE(pipe, PIPEACONF, PIPEBCONF)
 #define PIPEDSL(pipe)  _PIPE(pipe, PIPEADSL, PIPEBDSL)
+#define PIPEFRAMEPIXEL(pipe)  _PIPE(pipe, PIPEAFRAMEPIXEL, PIPEBFRAMEPIXEL)
 
 #define DSPARB                  0x70030
 #define   DSPARB_CSTART_MASK    (0x7f << 7)
@@ -2271,8 +2394,13 @@
 
 /* Memory latency timer register */
 #define MLTR_ILK                0x11222
+#define  MLTR_WM1_SHIFT         0
+#define  MLTR_WM2_SHIFT         8
 /* the unit of memory self-refresh latency time is 0.5us */
 #define  ILK_SRLT_MASK          0x3f
+#define ILK_LATENCY(shift)      (I915_READ(MLTR_ILK) >> (shift) & ILK_SRLT_MASK)
+#define ILK_READ_WM1_LATENCY()  ILK_LATENCY(MLTR_WM1_SHIFT)
+#define ILK_READ_WM2_LATENCY()  ILK_LATENCY(MLTR_WM2_SHIFT)
 
 /* define the fifo size on Ironlake */
 #define ILK_DISPLAY_FIFO        128
@@ -2291,6 +2419,40 @@
 
 #define ILK_FIFO_LINE_SIZE      64
 
+/* define the WM info on Sandybridge */
+#define SNB_DISPLAY_FIFO        128
+#define SNB_DISPLAY_MAXWM       0x7f    /* bit 16:22 */
+#define SNB_DISPLAY_DFTWM       8
+#define SNB_CURSOR_FIFO         32
+#define SNB_CURSOR_MAXWM        0x1f    /* bit 4:0 */
+#define SNB_CURSOR_DFTWM        8
+
+#define SNB_DISPLAY_SR_FIFO     512
+#define SNB_DISPLAY_MAX_SRWM    0x1ff   /* bit 16:8 */
+#define SNB_DISPLAY_DFT_SRWM    0x3f
+#define SNB_CURSOR_SR_FIFO      64
+#define SNB_CURSOR_MAX_SRWM     0x3f    /* bit 5:0 */
+#define SNB_CURSOR_DFT_SRWM     8
+
+#define SNB_FBC_MAX_SRWM        0xf     /* bit 23:20 */
+
+#define SNB_FIFO_LINE_SIZE      64
+
+
+/* the address where we get all kinds of latency value */
+#define SSKPD                   0x5d10
+#define SSKPD_WM_MASK           0x3f
+#define SSKPD_WM0_SHIFT         0
+#define SSKPD_WM1_SHIFT         8
+#define SSKPD_WM2_SHIFT         16
+#define SSKPD_WM3_SHIFT         24
+
+#define SNB_LATENCY(shift)      (I915_READ(MCHBAR_MIRROR_BASE_SNB + SSKPD) >> (shift) & SSKPD_WM_MASK)
+#define SNB_READ_WM0_LATENCY()          SNB_LATENCY(SSKPD_WM0_SHIFT)
+#define SNB_READ_WM1_LATENCY()          SNB_LATENCY(SSKPD_WM1_SHIFT)
+#define SNB_READ_WM2_LATENCY()          SNB_LATENCY(SSKPD_WM2_SHIFT)
+#define SNB_READ_WM3_LATENCY()          SNB_LATENCY(SSKPD_WM3_SHIFT)
+
 /*
  * The two pipe frame counter registers are not synchronized, so
  * reading a stable value is somewhat tricky. The following code
@@ -2351,6 +2513,10 @@
 #define CURBBASE                0x700c4
 #define CURBPOS                 0x700c8
 
+#define CURCNTR(pipe) _PIPE(pipe, CURACNTR, CURBCNTR)
+#define CURBASE(pipe) _PIPE(pipe, CURABASE, CURBBASE)
+#define CURPOS(pipe) _PIPE(pipe, CURAPOS, CURBPOS)
+
 /* Display A control */
 #define DSPACNTR                0x70180
 #define   DISPLAY_PLANE_ENABLE                  (1<<31)
@@ -2464,6 +2630,8 @@
 #define DISPLAY_PORT_PLL_BIOS_2         0x46014
 
 #define PCH_DSPCLK_GATE_D       0x42020
+# define DPFCUNIT_CLOCK_GATE_DISABLE            (1 << 9)
+# define DPFCRUNIT_CLOCK_GATE_DISABLE           (1 << 8)
 # define DPFDUNIT_CLOCK_GATE_DISABLE            (1 << 7)
 # define DPARBUNIT_CLOCK_GATE_DISABLE           (1 << 5)
 
@@ -2471,6 +2639,9 @@
 # define MARIUNIT_CLOCK_GATE_DISABLE            (1 << 18)
 # define SVSMUNIT_CLOCK_GATE_DISABLE            (1 << 1)
 
+#define PCH_3DCGDIS1            0x46024
+# define VFMUNIT_CLOCK_GATE_DISABLE             (1 << 11)
+
 #define FDI_PLL_FREQ_CTL        0x46030
 #define  FDI_PLL_FREQ_CHANGE_REQUEST    (1<<24)
 #define  FDI_PLL_FREQ_LOCK_LIMIT_MASK   0xfff00
@@ -2586,10 +2757,21 @@
 #define GTIER   0x4401c
 
 #define ILK_DISPLAY_CHICKEN2    0x42004
+/* Required on all Ironlake and Sandybridge according to the B-Spec. */
+#define  ILK_ELPIN_409_SELECT   (1 << 25)
 #define  ILK_DPARB_GATE (1<<22)
 #define  ILK_VSDPFD_FULL        (1<<21)
+#define ILK_DISPLAY_CHICKEN_FUSES       0x42014
+#define  ILK_INTERNAL_GRAPHICS_DISABLE  (1<<31)
+#define  ILK_INTERNAL_DISPLAY_DISABLE   (1<<30)
+#define  ILK_DISPLAY_DEBUG_DISABLE      (1<<29)
+#define  ILK_HDCP_DISABLE               (1<<25)
+#define  ILK_eDP_A_DISABLE              (1<<24)
+#define  ILK_DESKTOP                    (1<<23)
 #define ILK_DSPCLK_GATE         0x42020
 #define  ILK_DPARB_CLK_GATE     (1<<5)
+#define  ILK_DPFD_CLK_GATE      (1<<7)
+
 /* According to spec this bit 7/8/9 of 0x42020 should be set to enable FBC */
 #define   ILK_CLK_FBC           (1<<7)
 #define   ILK_DPFC_DIS1         (1<<8)
@@ -2602,12 +2784,41 @@
 /* PCH */
 
 /* south display engine interrupt */
+#define SDE_AUDIO_POWER_D       (1 << 27)
+#define SDE_AUDIO_POWER_C       (1 << 26)
+#define SDE_AUDIO_POWER_B       (1 << 25)
+#define SDE_AUDIO_POWER_SHIFT   (25)
+#define SDE_AUDIO_POWER_MASK    (7 << SDE_AUDIO_POWER_SHIFT)
+#define SDE_GMBUS               (1 << 24)
+#define SDE_AUDIO_HDCP_TRANSB   (1 << 23)
+#define SDE_AUDIO_HDCP_TRANSA   (1 << 22)
+#define SDE_AUDIO_HDCP_MASK     (3 << 22)
+#define SDE_AUDIO_TRANSB        (1 << 21)
+#define SDE_AUDIO_TRANSA        (1 << 20)
+#define SDE_AUDIO_TRANS_MASK    (3 << 20)
+#define SDE_POISON              (1 << 19)
+/* 18 reserved */
+#define SDE_FDI_RXB             (1 << 17)
+#define SDE_FDI_RXA             (1 << 16)
+#define SDE_FDI_MASK            (3 << 16)
+#define SDE_AUXD                (1 << 15)
+#define SDE_AUXC                (1 << 14)
+#define SDE_AUXB                (1 << 13)
+#define SDE_AUX_MASK            (7 << 13)
+/* 12 reserved */
 #define SDE_CRT_HOTPLUG         (1 << 11)
 #define SDE_PORTD_HOTPLUG       (1 << 10)
 #define SDE_PORTC_HOTPLUG       (1 << 9)
 #define SDE_PORTB_HOTPLUG       (1 << 8)
 #define SDE_SDVOB_HOTPLUG       (1 << 6)
 #define SDE_HOTPLUG_MASK        (0xf << 8)
+#define SDE_TRANSB_CRC_DONE     (1 << 5)
+#define SDE_TRANSB_CRC_ERR      (1 << 4)
+#define SDE_TRANSB_FIFO_UNDER   (1 << 3)
+#define SDE_TRANSA_CRC_DONE     (1 << 2)
+#define SDE_TRANSA_CRC_ERR      (1 << 1)
+#define SDE_TRANSA_FIFO_UNDER   (1 << 0)
+#define SDE_TRANS_MASK          (0x3f)
 /* CPT */
 #define SDE_CRT_HOTPLUG_CPT     (1 << 19)
 #define SDE_PORTD_HOTPLUG_CPT   (1 << 23)
@@ -2669,6 +2880,7 @@
 #define PCH_DPLL(pipe) _PIPE(pipe, PCH_DPLL_A, PCH_DPLL_B)
 
 #define PCH_FPA0                0xc6040
+#define  FP_CB_TUNE             (0x3<<22)
 #define PCH_FPA1                0xc6044
 #define PCH_FPB0                0xc6048
 #define PCH_FPB1                0xc604c
@@ -3033,6 +3245,7 @@
 #define  TRANS_DP_10BPC         (1<<9)
 #define  TRANS_DP_6BPC          (2<<9)
 #define  TRANS_DP_12BPC         (3<<9)
+#define  TRANS_DP_BPC_MASK      (3<<9)
 #define  TRANS_DP_VSYNC_ACTIVE_HIGH     (1<<4)
 #define  TRANS_DP_VSYNC_ACTIVE_LOW      0
 #define  TRANS_DP_HSYNC_ACTIVE_HIGH     (1<<3)
@@ -3046,10 +3259,74 @@
 #define  EDP_LINK_TRAIN_600MV_3_5DB_SNB_A       (0x01<<22)
 #define  EDP_LINK_TRAIN_800MV_0DB_SNB_A         (0x0<<22)
 /* SNB B-stepping */
-#define  EDP_LINK_TRAIN_400MV_0DB_SNB_B         (0x0<<22)
-#define  EDP_LINK_TRAIN_400MV_6DB_SNB_B         (0x3a<<22)
-#define  EDP_LINK_TRAIN_600MV_3_5DB_SNB_B       (0x39<<22)
-#define  EDP_LINK_TRAIN_800MV_0DB_SNB_B         (0x38<<22)
+#define  EDP_LINK_TRAIN_400_600MV_0DB_SNB_B     (0x0<<22)
+#define  EDP_LINK_TRAIN_400MV_3_5DB_SNB_B       (0x1<<22)
+#define  EDP_LINK_TRAIN_400_600MV_6DB_SNB_B     (0x3a<<22)
+#define  EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B   (0x39<<22)
+#define  EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B    (0x38<<22)
 #define  EDP_LINK_TRAIN_VOL_EMP_MASK_SNB        (0x3f<<22)
 
+#define  FORCEWAKE                              0xA18C
+#define  FORCEWAKE_ACK                          0x130090
+
+#define GEN6_RPNSWREQ                           0xA008
+#define   GEN6_TURBO_DISABLE                    (1<<31)
+#define   GEN6_FREQUENCY(x)                     ((x)<<25)
+#define   GEN6_OFFSET(x)                        ((x)<<19)
+#define   GEN6_AGGRESSIVE_TURBO                 (0<<15)
+#define GEN6_RC_VIDEO_FREQ                      0xA00C
+#define GEN6_RC_CONTROL                         0xA090
+#define   GEN6_RC_CTL_RC6pp_ENABLE              (1<<16)
+#define   GEN6_RC_CTL_RC6p_ENABLE               (1<<17)
+#define   GEN6_RC_CTL_RC6_ENABLE                (1<<18)
+#define   GEN6_RC_CTL_RC1e_ENABLE               (1<<20)
+#define   GEN6_RC_CTL_RC7_ENABLE                (1<<22)
+#define   GEN6_RC_CTL_EI_MODE(x)                ((x)<<27)
+#define   GEN6_RC_CTL_HW_ENABLE                 (1<<31)
+#define GEN6_RP_DOWN_TIMEOUT                    0xA010
+#define GEN6_RP_INTERRUPT_LIMITS                0xA014
+#define GEN6_RPSTAT1                            0xA01C
+#define GEN6_RP_CONTROL                         0xA024
+#define   GEN6_RP_MEDIA_TURBO                   (1<<11)
+#define   GEN6_RP_USE_NORMAL_FREQ               (1<<9)
+#define   GEN6_RP_MEDIA_IS_GFX                  (1<<8)
+#define   GEN6_RP_ENABLE                        (1<<7)
+#define   GEN6_RP_UP_BUSY_MAX                   (0x2<<3)
+#define   GEN6_RP_DOWN_BUSY_MIN                 (0x2<<0)
+#define GEN6_RP_UP_THRESHOLD                    0xA02C
+#define GEN6_RP_DOWN_THRESHOLD                  0xA030
+#define GEN6_RP_UP_EI                           0xA068
+#define GEN6_RP_DOWN_EI                         0xA06C
+#define GEN6_RP_IDLE_HYSTERSIS                  0xA070
+#define GEN6_RC_STATE                           0xA094
+#define GEN6_RC1_WAKE_RATE_LIMIT                0xA098
+#define GEN6_RC6_WAKE_RATE_LIMIT                0xA09C
+#define GEN6_RC6pp_WAKE_RATE_LIMIT              0xA0A0
+#define GEN6_RC_EVALUATION_INTERVAL             0xA0A8
+#define GEN6_RC_IDLE_HYSTERSIS                  0xA0AC
+#define GEN6_RC_SLEEP                           0xA0B0
+#define GEN6_RC1e_THRESHOLD                     0xA0B4
+#define GEN6_RC6_THRESHOLD                      0xA0B8
+#define GEN6_RC6p_THRESHOLD                     0xA0BC
+#define GEN6_RC6pp_THRESHOLD                    0xA0C0
+#define GEN6_PMINTRMSK                          0xA168
+
+#define GEN6_PMISR                              0x44020
+#define GEN6_PMIMR                              0x44024
+#define GEN6_PMIIR                              0x44028
+#define GEN6_PMIER                              0x4402C
+#define  GEN6_PM_MBOX_EVENT                     (1<<25)
+#define  GEN6_PM_THERMAL_EVENT                  (1<<24)
+#define  GEN6_PM_RP_DOWN_TIMEOUT                (1<<6)
+#define  GEN6_PM_RP_UP_THRESHOLD                (1<<5)
+#define  GEN6_PM_RP_DOWN_THRESHOLD              (1<<4)
+#define  GEN6_PM_RP_UP_EI_EXPIRED               (1<<2)
+#define  GEN6_PM_RP_DOWN_EI_EXPIRED             (1<<1)
+
+#define GEN6_PCODE_MAILBOX                      0x138124
+#define   GEN6_PCODE_READY                      (1<<31)
+#define   GEN6_READ_OC_PARAMS                   0xc
+#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE       0x9
+#define GEN6_PCODE_DATA                         0x138128
+
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 454c064f8ef7..0521ecf26017 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -235,10 +235,21 @@ static void i915_restore_vga(struct drm_device *dev)
 static void i915_save_modeset_reg(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
+        /* Cursor state */
+        dev_priv->saveCURACNTR = I915_READ(CURACNTR);
+        dev_priv->saveCURAPOS = I915_READ(CURAPOS);
+        dev_priv->saveCURABASE = I915_READ(CURABASE);
+        dev_priv->saveCURBCNTR = I915_READ(CURBCNTR);
+        dev_priv->saveCURBPOS = I915_READ(CURBPOS);
+        dev_priv->saveCURBBASE = I915_READ(CURBBASE);
+        if (IS_GEN2(dev))
+                dev_priv->saveCURSIZE = I915_READ(CURSIZE);
+
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->savePCH_DREF_CONTROL = I915_READ(PCH_DREF_CONTROL);
                 dev_priv->saveDISP_ARB_CTL = I915_READ(DISP_ARB_CTL);
@@ -357,6 +368,28 @@ static void i915_save_modeset_reg(struct drm_device *dev)
         }
         i915_save_palette(dev, PIPE_B);
         dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
+
+        /* Fences */
+        switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                for (i = 0; i < 16; i++)
+                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
+                break;
+        case 5:
+        case 4:
+                for (i = 0; i < 16; i++)
+                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
+                break;
+        case 3:
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
+        case 2:
+                for (i = 0; i < 8; i++)
+                        dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
+                break;
+        }
+
         return;
 }
 
@@ -365,10 +398,33 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int dpll_a_reg, fpa0_reg, fpa1_reg;
         int dpll_b_reg, fpb0_reg, fpb1_reg;
+        int i;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
 
+        /* Fences */
+        switch (INTEL_INFO(dev)->gen) {
+        case 6:
+                for (i = 0; i < 16; i++)
+                        I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
+                break;
+        case 5:
+        case 4:
+                for (i = 0; i < 16; i++)
+                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
+                break;
+        case 3:
+        case 2:
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
+                for (i = 0; i < 8; i++)
+                        I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
+                break;
+        }
+
+
         if (HAS_PCH_SPLIT(dev)) {
                 dpll_a_reg = PCH_DPLL_A;
                 dpll_b_reg = PCH_DPLL_B;
@@ -529,6 +585,16 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
         I915_WRITE(DSPBCNTR, dev_priv->saveDSPBCNTR);
         I915_WRITE(DSPBADDR, I915_READ(DSPBADDR));
 
+        /* Cursor state */
+        I915_WRITE(CURAPOS, dev_priv->saveCURAPOS);
+        I915_WRITE(CURACNTR, dev_priv->saveCURACNTR);
+        I915_WRITE(CURABASE, dev_priv->saveCURABASE);
+        I915_WRITE(CURBPOS, dev_priv->saveCURBPOS);
+        I915_WRITE(CURBCNTR, dev_priv->saveCURBCNTR);
+        I915_WRITE(CURBBASE, dev_priv->saveCURBBASE);
+        if (IS_GEN2(dev))
+                I915_WRITE(CURSIZE, dev_priv->saveCURSIZE);
+
         return;
 }
 
@@ -543,16 +609,6 @@ void i915_save_display(struct drm_device *dev)
         /* Don't save them in KMS mode */
         i915_save_modeset_reg(dev);
 
-        /* Cursor state */
-        dev_priv->saveCURACNTR = I915_READ(CURACNTR);
-        dev_priv->saveCURAPOS = I915_READ(CURAPOS);
-        dev_priv->saveCURABASE = I915_READ(CURABASE);
-        dev_priv->saveCURBCNTR = I915_READ(CURBCNTR);
-        dev_priv->saveCURBPOS = I915_READ(CURBPOS);
-        dev_priv->saveCURBBASE = I915_READ(CURBBASE);
-        if (IS_GEN2(dev))
-                dev_priv->saveCURSIZE = I915_READ(CURSIZE);
-
         /* CRT state */
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->saveADPA = I915_READ(PCH_ADPA);
@@ -657,16 +713,6 @@ void i915_restore_display(struct drm_device *dev)
         /* Don't restore them in KMS mode */
         i915_restore_modeset_reg(dev);
 
-        /* Cursor state */
-        I915_WRITE(CURAPOS, dev_priv->saveCURAPOS);
-        I915_WRITE(CURACNTR, dev_priv->saveCURACNTR);
-        I915_WRITE(CURABASE, dev_priv->saveCURABASE);
-        I915_WRITE(CURBPOS, dev_priv->saveCURBPOS);
-        I915_WRITE(CURBCNTR, dev_priv->saveCURBCNTR);
-        I915_WRITE(CURBBASE, dev_priv->saveCURBBASE);
-        if (IS_GEN2(dev))
-                I915_WRITE(CURSIZE, dev_priv->saveCURSIZE);
-
         /* CRT state */
         if (HAS_PCH_SPLIT(dev))
                 I915_WRITE(PCH_ADPA, dev_priv->saveADPA);
@@ -694,7 +740,7 @@ void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(PCH_PP_OFF_DELAYS, dev_priv->savePP_OFF_DELAYS);
                 I915_WRITE(PCH_PP_DIVISOR, dev_priv->savePP_DIVISOR);
                 I915_WRITE(PCH_PP_CONTROL, dev_priv->savePP_CONTROL);
-                I915_WRITE(MCHBAR_RENDER_STANDBY,
+                I915_WRITE(RSTDBYCTL,
                            dev_priv->saveMCHBAR_RENDER_STANDBY);
         } else {
                 I915_WRITE(PFIT_PGM_RATIOS, dev_priv->savePFIT_PGM_RATIOS);
@@ -765,14 +811,16 @@ int i915_save_state(struct drm_device *dev)
                 dev_priv->saveFDI_RXA_IMR = I915_READ(FDI_RXA_IMR);
                 dev_priv->saveFDI_RXB_IMR = I915_READ(FDI_RXB_IMR);
                 dev_priv->saveMCHBAR_RENDER_STANDBY =
-                        I915_READ(MCHBAR_RENDER_STANDBY);
+                        I915_READ(RSTDBYCTL);
         } else {
                 dev_priv->saveIER = I915_READ(IER);
                 dev_priv->saveIMR = I915_READ(IMR);
         }
 
-        if (HAS_PCH_SPLIT(dev))
+        if (IS_IRONLAKE_M(dev))
                 ironlake_disable_drps(dev);
+        if (IS_GEN6(dev))
+                gen6_disable_rps(dev);
 
         /* Cache mode state */
         dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
@@ -788,28 +836,6 @@ int i915_save_state(struct drm_device *dev)
         for (i = 0; i < 3; i++)
                 dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
 
-        /* Fences */
-        switch (INTEL_INFO(dev)->gen) {
-        case 6:
-                for (i = 0; i < 16; i++)
-                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
-                break;
-        case 5:
-        case 4:
-                for (i = 0; i < 16; i++)
-                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
-                break;
-        case 3:
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
-        case 2:
-                for (i = 0; i < 8; i++)
-                        dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
-                break;
-
-        }
-
         return 0;
 }
 
@@ -823,27 +849,6 @@ int i915_restore_state(struct drm_device *dev)
         /* Hardware status page */
         I915_WRITE(HWS_PGA, dev_priv->saveHWS);
 
-        /* Fences */
-        switch (INTEL_INFO(dev)->gen) {
-        case 6:
-                for (i = 0; i < 16; i++)
-                        I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
-                break;
-        case 5:
-        case 4:
-                for (i = 0; i < 16; i++)
-                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
-                break;
-        case 3:
-        case 2:
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
-                for (i = 0; i < 8; i++)
-                        I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
-                break;
-        }
-
         i915_restore_display(dev);
 
         /* Interrupt state */
@@ -860,13 +865,16 @@ int i915_restore_state(struct drm_device *dev)
         }
 
         /* Clock gating state */
-        intel_init_clock_gating(dev);
+        intel_enable_clock_gating(dev);
 
-        if (HAS_PCH_SPLIT(dev)) {
+        if (IS_IRONLAKE_M(dev)) {
                 ironlake_enable_drps(dev);
                 intel_init_emon(dev);
         }
 
+        if (IS_GEN6(dev))
+                gen6_enable_rps(dev_priv);
+
         /* Cache mode state */
         I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000);
 
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index fea97a21cc14..7f0fc3ed61aa 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -6,6 +6,7 @@
 #include <linux/tracepoint.h>
 
 #include <drm/drmP.h>
+#include "i915_drv.h"
 
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM i915
@@ -16,18 +17,18 @@
 
 TRACE_EVENT(i915_gem_object_create,
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_PROTO(struct drm_i915_gem_object *obj),
 
             TP_ARGS(obj),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              __field(u32, size)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
-                           __entry->size = obj->size;
+                           __entry->size = obj->base.size;
                            ),
 
             TP_printk("obj=%p, size=%u", __entry->obj, __entry->size)
@@ -35,40 +36,43 @@ TRACE_EVENT(i915_gem_object_create,
 
 TRACE_EVENT(i915_gem_object_bind,
 
-            TP_PROTO(struct drm_gem_object *obj, u32 gtt_offset),
+            TP_PROTO(struct drm_i915_gem_object *obj, u32 gtt_offset, bool mappable),
 
-            TP_ARGS(obj, gtt_offset),
+            TP_ARGS(obj, gtt_offset, mappable),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              __field(u32, gtt_offset)
+                             __field(bool, mappable)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
                            __entry->gtt_offset = gtt_offset;
+                           __entry->mappable = mappable;
                            ),
 
-            TP_printk("obj=%p, gtt_offset=%08x",
-                      __entry->obj, __entry->gtt_offset)
+            TP_printk("obj=%p, gtt_offset=%08x%s",
+                      __entry->obj, __entry->gtt_offset,
+                      __entry->mappable ? ", mappable" : "")
 );
 
 TRACE_EVENT(i915_gem_object_change_domain,
 
-            TP_PROTO(struct drm_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain),
+            TP_PROTO(struct drm_i915_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain),
 
             TP_ARGS(obj, old_read_domains, old_write_domain),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              __field(u32, read_domains)
                              __field(u32, write_domain)
                              ),
 
             TP_fast_assign(
                            __entry->obj = obj;
-                           __entry->read_domains = obj->read_domains | (old_read_domains << 16);
-                           __entry->write_domain = obj->write_domain | (old_write_domain << 16);
+                           __entry->read_domains = obj->base.read_domains | (old_read_domains << 16);
+                           __entry->write_domain = obj->base.write_domain | (old_write_domain << 16);
                            ),
 
             TP_printk("obj=%p, read=%04x, write=%04x",
@@ -76,36 +80,14 @@ TRACE_EVENT(i915_gem_object_change_domain,
                       __entry->read_domains, __entry->write_domain)
 );
 
-TRACE_EVENT(i915_gem_object_get_fence,
-
-            TP_PROTO(struct drm_gem_object *obj, int fence, int tiling_mode),
-
-            TP_ARGS(obj, fence, tiling_mode),
-
-            TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
-                             __field(int, fence)
-                             __field(int, tiling_mode)
-                             ),
-
-            TP_fast_assign(
-                           __entry->obj = obj;
-                           __entry->fence = fence;
-                           __entry->tiling_mode = tiling_mode;
-                           ),
-
-            TP_printk("obj=%p, fence=%d, tiling=%d",
-                      __entry->obj, __entry->fence, __entry->tiling_mode)
-);
-
 DECLARE_EVENT_CLASS(i915_gem_object,
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_PROTO(struct drm_i915_gem_object *obj),
 
             TP_ARGS(obj),
 
             TP_STRUCT__entry(
-                             __field(struct drm_gem_object *, obj)
+                             __field(struct drm_i915_gem_object *, obj)
                              ),
 
             TP_fast_assign(
@@ -117,21 +99,21 @@ DECLARE_EVENT_CLASS(i915_gem_object,
 
 DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush,
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_PROTO(struct drm_i915_gem_object *obj),
 
             TP_ARGS(obj)
 );
 
 DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind,
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_PROTO(struct drm_i915_gem_object *obj),
 
             TP_ARGS(obj)
 );
 
 DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
 
-            TP_PROTO(struct drm_gem_object *obj),
+            TP_PROTO(struct drm_i915_gem_object *obj),
 
             TP_ARGS(obj)
 );
@@ -263,13 +245,13 @@ DEFINE_EVENT(i915_ring, i915_ring_wait_end,
 );
 
 TRACE_EVENT(i915_flip_request,
-            TP_PROTO(int plane, struct drm_gem_object *obj),
+            TP_PROTO(int plane, struct drm_i915_gem_object *obj),
 
             TP_ARGS(plane, obj),
 
             TP_STRUCT__entry(
                     __field(int, plane)
-                    __field(struct drm_gem_object *, obj)
+                    __field(struct drm_i915_gem_object *, obj)
                     ),
 
             TP_fast_assign(
@@ -281,13 +263,13 @@ TRACE_EVENT(i915_flip_request,
 );
 
 TRACE_EVENT(i915_flip_complete,
-            TP_PROTO(int plane, struct drm_gem_object *obj),
+            TP_PROTO(int plane, struct drm_i915_gem_object *obj),
 
             TP_ARGS(plane, obj),
 
             TP_STRUCT__entry(
                     __field(int, plane)
-                    __field(struct drm_gem_object *, obj)
+                    __field(struct drm_i915_gem_object *, obj)
                     ),
 
             TP_fast_assign(
@@ -298,6 +280,29 @@ TRACE_EVENT(i915_flip_complete,
             TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
 );
 
+TRACE_EVENT(i915_reg_rw,
+           TP_PROTO(int cmd, uint32_t reg, uint64_t val, int len),
+
+           TP_ARGS(cmd, reg, val, len),
+
+           TP_STRUCT__entry(
+                   __field(int, cmd)
+                   __field(uint32_t, reg)
+                   __field(uint64_t, val)
+                   __field(int, len)
+                   ),
+
+           TP_fast_assign(
+                   __entry->cmd = cmd;
+                   __entry->reg = reg;
+                   __entry->val = (uint64_t)val;
+                   __entry->len = len;
+                   ),
+
+           TP_printk("cmd=%c, reg=0x%x, val=0x%llx, len=%d",
+                     __entry->cmd, __entry->reg, __entry->val, __entry->len)
+);
+
 #endif /* _I915_TRACE_H_ */
 
 /* This part must be outside protection */
diff --git a/drivers/gpu/drm/i915/intel_acpi.c b/drivers/gpu/drm/i915/intel_acpi.c
index 65c88f9ba12c..2cb8e0b9f1ee 100644
--- a/drivers/gpu/drm/i915/intel_acpi.c
+++ b/drivers/gpu/drm/i915/intel_acpi.c
@@ -190,37 +190,6 @@ out:
         kfree(output.pointer);
 }
 
-static int intel_dsm_switchto(enum vga_switcheroo_client_id id)
-{
-        return 0;
-}
-
-static int intel_dsm_power_state(enum vga_switcheroo_client_id id,
-                                 enum vga_switcheroo_state state)
-{
-        return 0;
-}
-
-static int intel_dsm_init(void)
-{
-        return 0;
-}
-
-static int intel_dsm_get_client_id(struct pci_dev *pdev)
-{
-        if (intel_dsm_priv.dhandle == DEVICE_ACPI_HANDLE(&pdev->dev))
-                return VGA_SWITCHEROO_IGD;
-        else
-                return VGA_SWITCHEROO_DIS;
-}
-
-static struct vga_switcheroo_handler intel_dsm_handler = {
-        .switchto = intel_dsm_switchto,
-        .power_state = intel_dsm_power_state,
-        .init = intel_dsm_init,
-        .get_client_id = intel_dsm_get_client_id,
-};
-
 static bool intel_dsm_pci_probe(struct pci_dev *pdev)
 {
         acpi_handle dhandle, intel_handle;
@@ -276,11 +245,8 @@ void intel_register_dsm_handler(void)
 {
         if (!intel_dsm_detect())
                 return;
-
-        vga_switcheroo_register_handler(&intel_dsm_handler);
 }
 
 void intel_unregister_dsm_handler(void)
 {
-        vga_switcheroo_unregister_handler();
 }
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index b0b1200ed650..0b44956c336b 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -264,17 +264,12 @@ parse_general_features(struct drm_i915_private *dev_priv,
                 dev_priv->int_crt_support = general->int_crt_support;
                 dev_priv->lvds_use_ssc = general->enable_ssc;
 
-                if (dev_priv->lvds_use_ssc) {
-                        if (IS_I85X(dev))
-                                dev_priv->lvds_ssc_freq =
-                                        general->ssc_freq ? 66 : 48;
-                        else if (IS_GEN5(dev) || IS_GEN6(dev))
-                                dev_priv->lvds_ssc_freq =
-                                        general->ssc_freq ? 100 : 120;
-                        else
-                                dev_priv->lvds_ssc_freq =
-                                        general->ssc_freq ? 100 : 96;
-                }
+                if (IS_I85X(dev))
+                        dev_priv->lvds_ssc_freq = general->ssc_freq ? 66 : 48;
+                else if (IS_GEN5(dev) || IS_GEN6(dev))
+                        dev_priv->lvds_ssc_freq = general->ssc_freq ? 100 : 120;
+                else
+                        dev_priv->lvds_ssc_freq = general->ssc_freq ? 100 : 96;
         }
 }
 
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 8df574316063..8a77ff4a7237 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -30,6 +30,7 @@
 #include "drm.h"
 #include "drm_crtc.h"
 #include "drm_crtc_helper.h"
+#include "drm_edid.h"
 #include "intel_drv.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
@@ -287,8 +288,9 @@ static bool intel_crt_ddc_probe(struct drm_i915_private *dev_priv, int ddc_bus)
         return i2c_transfer(&dev_priv->gmbus[ddc_bus].adapter, msgs, 1) == 1;
 }
 
-static bool intel_crt_detect_ddc(struct intel_crt *crt)
+static bool intel_crt_detect_ddc(struct drm_connector *connector)
 {
+        struct intel_crt *crt = intel_attached_crt(connector);
         struct drm_i915_private *dev_priv = crt->base.base.dev->dev_private;
 
         /* CRT should always be at 0, but check anyway */
@@ -301,8 +303,26 @@ static bool intel_crt_detect_ddc(struct intel_crt *crt)
         }
 
         if (intel_ddc_probe(&crt->base, dev_priv->crt_ddc_pin)) {
-                DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n");
-                return true;
+                struct edid *edid;
+                bool is_digital = false;
+
+                edid = drm_get_edid(connector,
+                        &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
+                /*
+                 * This may be a DVI-I connector with a shared DDC
+                 * link between analog and digital outputs, so we
+                 * have to check the EDID input spec of the attached device.
+                 */
+                if (edid != NULL) {
+                        is_digital = edid->input & DRM_EDID_INPUT_DIGITAL;
+                        connector->display_info.raw_edid = NULL;
+                        kfree(edid);
+                }
+
+                if (!is_digital) {
+                        DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n");
+                        return true;
+                }
         }
 
         return false;
@@ -458,7 +478,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
                 }
         }
 
-        if (intel_crt_detect_ddc(crt))
+        if (intel_crt_detect_ddc(connector))
                 return connector_status_connected;
 
         if (!force)
@@ -472,7 +492,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
                 crtc = intel_get_load_detect_pipe(&crt->base, connector,
                                                   NULL, &dpms_mode);
                 if (crtc) {
-                        if (intel_crt_detect_ddc(crt))
+                        if (intel_crt_detect_ddc(connector))
                                 status = connector_status_connected;
                         else
                                 status = intel_crt_load_detect(crtc, crt);
@@ -515,6 +535,15 @@ static int intel_crt_set_property(struct drm_connector *connector,
         return 0;
 }
 
+static void intel_crt_reset(struct drm_connector *connector)
+{
+        struct drm_device *dev = connector->dev;
+        struct intel_crt *crt = intel_attached_crt(connector);
+
+        if (HAS_PCH_SPLIT(dev))
+                crt->force_hotplug_required = 1;
+}
+
 /*
  * Routines for controlling stuff on the analog port
  */
@@ -528,6 +557,7 @@ static const struct drm_encoder_helper_funcs intel_crt_helper_funcs = {
 };
 
 static const struct drm_connector_funcs intel_crt_connector_funcs = {
+        .reset = intel_crt_reset,
         .dpms = drm_helper_connector_dpms,
         .detect = intel_crt_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index bee24b1a58e8..7e42aa586504 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -642,26 +642,23 @@ static const intel_limit_t intel_limits_ironlake_display_port = {
         .find_pll = intel_find_pll_ironlake_dp,
 };
 
-static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc)
+static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
+                                                int refclk)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         const intel_limit_t *limit;
-        int refclk = 120;
 
         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-                if (dev_priv->lvds_use_ssc && dev_priv->lvds_ssc_freq == 100)
-                        refclk = 100;
-
                 if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) ==
                     LVDS_CLKB_POWER_UP) {
                         /* LVDS dual channel */
-                        if (refclk == 100)
+                        if (refclk == 100000)
                                 limit = &intel_limits_ironlake_dual_lvds_100m;
                         else
                                 limit = &intel_limits_ironlake_dual_lvds;
                 } else {
-                        if (refclk == 100)
+                        if (refclk == 100000)
                                 limit = &intel_limits_ironlake_single_lvds_100m;
                         else
                                 limit = &intel_limits_ironlake_single_lvds;
@@ -702,13 +699,13 @@ static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
         return limit;
 }
 
-static const intel_limit_t *intel_limit(struct drm_crtc *crtc)
+static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk)
 {
         struct drm_device *dev = crtc->dev;
         const intel_limit_t *limit;
 
         if (HAS_PCH_SPLIT(dev))
-                limit = intel_ironlake_limit(crtc);
+                limit = intel_ironlake_limit(crtc, refclk);
         else if (IS_G4X(dev)) {
                 limit = intel_g4x_limit(crtc);
         } else if (IS_PINEVIEW(dev)) {
@@ -773,11 +770,10 @@ bool intel_pipe_has_type(struct drm_crtc *crtc, int type)
  * the given connectors.
  */
 
-static bool intel_PLL_is_valid(struct drm_crtc *crtc, intel_clock_t *clock)
+static bool intel_PLL_is_valid(struct drm_device *dev,
+                               const intel_limit_t *limit,
+                               const intel_clock_t *clock)
 {
-        const intel_limit_t *limit = intel_limit (crtc);
-        struct drm_device *dev = crtc->dev;
-
         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
                 INTELPllInvalid ("p1 out of range\n");
         if (clock->p   < limit->p.min   || limit->p.max   < clock->p)
@@ -849,8 +845,8 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                         int this_err;
 
                                         intel_clock(dev, refclk, &clock);
-
-                                        if (!intel_PLL_is_valid(crtc, &clock))
+                                        if (!intel_PLL_is_valid(dev, limit,
+                                                                &clock))
                                                 continue;
 
                                         this_err = abs(clock.dot - target);
@@ -912,9 +908,11 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                         int this_err;
 
                                         intel_clock(dev, refclk, &clock);
-                                        if (!intel_PLL_is_valid(crtc, &clock))
+                                        if (!intel_PLL_is_valid(dev, limit,
+                                                                &clock))
                                                 continue;
-                                        this_err = abs(clock.dot - target) ;
+
+                                        this_err = abs(clock.dot - target);
                                         if (this_err < err_most) {
                                                 *best_clock = clock;
                                                 err_most = this_err;
@@ -1066,13 +1064,13 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_framebuffer *fb = crtc->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int plane, i;
         u32 fbc_ctl, fbc_ctl2;
 
         if (fb->pitch == dev_priv->cfb_pitch &&
-            obj_priv->fence_reg == dev_priv->cfb_fence &&
+            obj->fence_reg == dev_priv->cfb_fence &&
             intel_crtc->plane == dev_priv->cfb_plane &&
             I915_READ(FBC_CONTROL) & FBC_CTL_EN)
                 return;
@@ -1086,7 +1084,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 
         /* FBC_CTL wants 64B units */
         dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_fence = obj->fence_reg;
         dev_priv->cfb_plane = intel_crtc->plane;
         plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
 
@@ -1096,7 +1094,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 
         /* Set it up... */
         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
+        if (obj->tiling_mode != I915_TILING_NONE)
                 fbc_ctl2 |= FBC_CTL_CPU_FENCE;
         I915_WRITE(FBC_CONTROL2, fbc_ctl2);
         I915_WRITE(FBC_FENCE_OFF, crtc->y);
@@ -1107,7 +1105,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
         fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
         fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
-        if (obj_priv->tiling_mode != I915_TILING_NONE)
+        if (obj->tiling_mode != I915_TILING_NONE)
                 fbc_ctl |= dev_priv->cfb_fence;
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
@@ -1150,7 +1148,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_framebuffer *fb = crtc->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
         unsigned long stall_watermark = 200;
@@ -1159,7 +1157,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         dpfc_ctl = I915_READ(DPFC_CONTROL);
         if (dpfc_ctl & DPFC_CTL_EN) {
                 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
-                    dev_priv->cfb_fence == obj_priv->fence_reg &&
+                    dev_priv->cfb_fence == obj->fence_reg &&
                     dev_priv->cfb_plane == intel_crtc->plane &&
                     dev_priv->cfb_y == crtc->y)
                         return;
@@ -1170,12 +1168,12 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         }
 
         dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_fence = obj->fence_reg;
         dev_priv->cfb_plane = intel_crtc->plane;
         dev_priv->cfb_y = crtc->y;
 
         dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
-        if (obj_priv->tiling_mode != I915_TILING_NONE) {
+        if (obj->tiling_mode != I915_TILING_NONE) {
                 dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
                 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
         } else {
@@ -1215,13 +1213,33 @@ static bool g4x_fbc_enabled(struct drm_device *dev)
         return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
 }
 
+static void sandybridge_blit_fbc_update(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 blt_ecoskpd;
+
+        /* Make sure blitter notifies FBC of writes */
+        __gen6_force_wake_get(dev_priv);
+        blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
+        blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
+                GEN6_BLITTER_LOCK_SHIFT;
+        I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+        blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
+        I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+        blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
+                         GEN6_BLITTER_LOCK_SHIFT);
+        I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+        POSTING_READ(GEN6_BLITTER_ECOSKPD);
+        __gen6_force_wake_put(dev_priv);
+}
+
 static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_framebuffer *fb = crtc->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
         unsigned long stall_watermark = 200;
@@ -1230,9 +1248,9 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
         if (dpfc_ctl & DPFC_CTL_EN) {
                 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
-                    dev_priv->cfb_fence == obj_priv->fence_reg &&
+                    dev_priv->cfb_fence == obj->fence_reg &&
                     dev_priv->cfb_plane == intel_crtc->plane &&
-                    dev_priv->cfb_offset == obj_priv->gtt_offset &&
+                    dev_priv->cfb_offset == obj->gtt_offset &&
                     dev_priv->cfb_y == crtc->y)
                         return;
 
@@ -1242,14 +1260,14 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         }
 
         dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_fence = obj->fence_reg;
         dev_priv->cfb_plane = intel_crtc->plane;
-        dev_priv->cfb_offset = obj_priv->gtt_offset;
+        dev_priv->cfb_offset = obj->gtt_offset;
         dev_priv->cfb_y = crtc->y;
 
         dpfc_ctl &= DPFC_RESERVED;
         dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
-        if (obj_priv->tiling_mode != I915_TILING_NONE) {
+        if (obj->tiling_mode != I915_TILING_NONE) {
                 dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence);
                 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
         } else {
@@ -1260,10 +1278,17 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
         I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
-        I915_WRITE(ILK_FBC_RT_BASE, obj_priv->gtt_offset | ILK_FBC_RT_VALID);
+        I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
         /* enable it... */
         I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 
+        if (IS_GEN6(dev)) {
+                I915_WRITE(SNB_DPFC_CTL_SA,
+                           SNB_CPU_FENCE_ENABLE | dev_priv->cfb_fence);
+                I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
+                sandybridge_blit_fbc_update(dev);
+        }
+
         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
 }
 
@@ -1345,7 +1370,7 @@ static void intel_update_fbc(struct drm_device *dev)
         struct intel_crtc *intel_crtc;
         struct drm_framebuffer *fb;
         struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
 
         DRM_DEBUG_KMS("\n");
 
@@ -1384,9 +1409,9 @@ static void intel_update_fbc(struct drm_device *dev)
         intel_crtc = to_intel_crtc(crtc);
         fb = crtc->fb;
         intel_fb = to_intel_framebuffer(fb);
-        obj_priv = to_intel_bo(intel_fb->obj);
+        obj = intel_fb->obj;
 
-        if (intel_fb->obj->size > dev_priv->cfb_size) {
+        if (intel_fb->obj->base.size > dev_priv->cfb_size) {
                 DRM_DEBUG_KMS("framebuffer too large, disabling "
                               "compression\n");
                 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
@@ -1410,7 +1435,7 @@ static void intel_update_fbc(struct drm_device *dev)
                 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
                 goto out_disable;
         }
-        if (obj_priv->tiling_mode != I915_TILING_X) {
+        if (obj->tiling_mode != I915_TILING_X) {
                 DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n");
                 dev_priv->no_fbc_reason = FBC_NOT_TILED;
                 goto out_disable;
@@ -1433,14 +1458,13 @@ out_disable:
 
 int
 intel_pin_and_fence_fb_obj(struct drm_device *dev,
-                           struct drm_gem_object *obj,
-                           bool pipelined)
+                           struct drm_i915_gem_object *obj,
+                           struct intel_ring_buffer *pipelined)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         u32 alignment;
         int ret;
 
-        switch (obj_priv->tiling_mode) {
+        switch (obj->tiling_mode) {
         case I915_TILING_NONE:
                 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
                         alignment = 128 * 1024;
@@ -1461,7 +1485,7 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
                 BUG();
         }
 
-        ret = i915_gem_object_pin(obj, alignment);
+        ret = i915_gem_object_pin(obj, alignment, true);
         if (ret)
                 return ret;
 
@@ -1474,9 +1498,8 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
          * framebuffer compression.  For simplicity, we always install
          * a fence as the cost is not that onerous.
          */
-        if (obj_priv->fence_reg == I915_FENCE_REG_NONE &&
-            obj_priv->tiling_mode != I915_TILING_NONE) {
-                ret = i915_gem_object_get_fence_reg(obj, false);
+        if (obj->tiling_mode != I915_TILING_NONE) {
+                ret = i915_gem_object_get_fence(obj, pipelined, false);
                 if (ret)
                         goto err_unpin;
         }
@@ -1497,8 +1520,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int plane = intel_crtc->plane;
         unsigned long Start, Offset;
         u32 dspcntr;
@@ -1515,7 +1537,6 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 
         intel_fb = to_intel_framebuffer(fb);
         obj = intel_fb->obj;
-        obj_priv = to_intel_bo(obj);
 
         reg = DSPCNTR(plane);
         dspcntr = I915_READ(reg);
@@ -1540,7 +1561,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 return -EINVAL;
         }
         if (INTEL_INFO(dev)->gen >= 4) {
-                if (obj_priv->tiling_mode != I915_TILING_NONE)
+                if (obj->tiling_mode != I915_TILING_NONE)
                         dspcntr |= DISPPLANE_TILED;
                 else
                         dspcntr &= ~DISPPLANE_TILED;
@@ -1552,7 +1573,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 
         I915_WRITE(reg, dspcntr);
 
-        Start = obj_priv->gtt_offset;
+        Start = obj->gtt_offset;
         Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
 
         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
@@ -1598,7 +1619,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         mutex_lock(&dev->struct_mutex);
         ret = intel_pin_and_fence_fb_obj(dev,
                                          to_intel_framebuffer(crtc->fb)->obj,
-                                         false);
+                                         NULL);
         if (ret != 0) {
                 mutex_unlock(&dev->struct_mutex);
                 return ret;
@@ -1606,18 +1627,17 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 
         if (old_fb) {
                 struct drm_i915_private *dev_priv = dev->dev_private;
-                struct drm_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
-                struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+                struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
 
                 wait_event(dev_priv->pending_flip_queue,
-                           atomic_read(&obj_priv->pending_flip) == 0);
+                           atomic_read(&obj->pending_flip) == 0);
 
                 /* Big Hammer, we also need to ensure that any pending
                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
                  * current scanout is retired before unpinning the old
                  * framebuffer.
                  */
-                ret = i915_gem_object_flush_gpu(obj_priv, false);
+                ret = i915_gem_object_flush_gpu(obj, false);
                 if (ret) {
                         i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
                         mutex_unlock(&dev->struct_mutex);
@@ -1633,8 +1653,10 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 return ret;
         }
 
-        if (old_fb)
+        if (old_fb) {
+                intel_wait_for_vblank(dev, intel_crtc->pipe);
                 i915_gem_object_unpin(to_intel_framebuffer(old_fb)->obj);
+        }
 
         mutex_unlock(&dev->struct_mutex);
 
@@ -1996,31 +2018,31 @@ static void intel_flush_display_plane(struct drm_device *dev,
 static void intel_clear_scanline_wait(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
         u32 tmp;
 
         if (IS_GEN2(dev))
                 /* Can't break the hang on i8xx */
                 return;
 
-        tmp = I915_READ(PRB0_CTL);
-        if (tmp & RING_WAIT) {
-                I915_WRITE(PRB0_CTL, tmp);
-                POSTING_READ(PRB0_CTL);
-        }
+        ring = LP_RING(dev_priv);
+        tmp = I915_READ_CTL(ring);
+        if (tmp & RING_WAIT)
+                I915_WRITE_CTL(ring, tmp);
 }
 
 static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 {
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct drm_i915_private *dev_priv;
 
         if (crtc->fb == NULL)
                 return;
 
-        obj_priv = to_intel_bo(to_intel_framebuffer(crtc->fb)->obj);
+        obj = to_intel_framebuffer(crtc->fb)->obj;
         dev_priv = crtc->dev->dev_private;
         wait_event(dev_priv->pending_flip_queue,
-                   atomic_read(&obj_priv->pending_flip) == 0);
+                   atomic_read(&obj->pending_flip) == 0);
 }
 
 static void ironlake_crtc_enable(struct drm_crtc *crtc)
@@ -2120,9 +2142,11 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
                 reg = TRANS_DP_CTL(pipe);
                 temp = I915_READ(reg);
                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
-                          TRANS_DP_SYNC_MASK);
+                          TRANS_DP_SYNC_MASK |
+                          TRANS_DP_BPC_MASK);
                 temp |= (TRANS_DP_OUTPUT_ENABLE |
                          TRANS_DP_ENH_FRAMING);
+                temp |= TRANS_DP_8BPC;
 
                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
@@ -2712,27 +2736,19 @@ fdi_reduce_ratio(u32 *num, u32 *den)
         }
 }
 
-#define DATA_N 0x800000
-#define LINK_N 0x80000
-
 static void
 ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock,
                      int link_clock, struct fdi_m_n *m_n)
 {
-        u64 temp;
-
         m_n->tu = 64; /* default size */
 
-        temp = (u64) DATA_N * pixel_clock;
-        temp = div_u64(temp, link_clock);
-        m_n->gmch_m = div_u64(temp * bits_per_pixel, nlanes);
-        m_n->gmch_m >>= 3; /* convert to bytes_per_pixel */
-        m_n->gmch_n = DATA_N;
+        /* BUG_ON(pixel_clock > INT_MAX / 36); */
+        m_n->gmch_m = bits_per_pixel * pixel_clock;
+        m_n->gmch_n = link_clock * nlanes * 8;
         fdi_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n);
 
-        temp = (u64) LINK_N * pixel_clock;
-        m_n->link_m = div_u64(temp, link_clock);
-        m_n->link_n = LINK_N;
+        m_n->link_m = pixel_clock;
+        m_n->link_n = link_clock;
         fdi_reduce_ratio(&m_n->link_m, &m_n->link_n);
 }
 
@@ -2856,6 +2872,39 @@ static struct intel_watermark_params ironlake_cursor_srwm_info = {
         ILK_FIFO_LINE_SIZE
 };
 
+static struct intel_watermark_params sandybridge_display_wm_info = {
+        SNB_DISPLAY_FIFO,
+        SNB_DISPLAY_MAXWM,
+        SNB_DISPLAY_DFTWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+
+static struct intel_watermark_params sandybridge_cursor_wm_info = {
+        SNB_CURSOR_FIFO,
+        SNB_CURSOR_MAXWM,
+        SNB_CURSOR_DFTWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+
+static struct intel_watermark_params sandybridge_display_srwm_info = {
+        SNB_DISPLAY_SR_FIFO,
+        SNB_DISPLAY_MAX_SRWM,
+        SNB_DISPLAY_DFT_SRWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+
+static struct intel_watermark_params sandybridge_cursor_srwm_info = {
+        SNB_CURSOR_SR_FIFO,
+        SNB_CURSOR_MAX_SRWM,
+        SNB_CURSOR_DFT_SRWM,
+        2,
+        SNB_FIFO_LINE_SIZE
+};
+
+
 /**
  * intel_calculate_wm - calculate watermark level
  * @clock_in_khz: pixel clock
@@ -3389,12 +3438,17 @@ static void i830_update_wm(struct drm_device *dev, int planea_clock, int unused,
 
 static bool ironlake_compute_wm0(struct drm_device *dev,
                                  int pipe,
+                                 const struct intel_watermark_params *display,
+                                 int display_latency_ns,
+                                 const struct intel_watermark_params *cursor,
+                                 int cursor_latency_ns,
                                  int *plane_wm,
                                  int *cursor_wm)
 {
         struct drm_crtc *crtc;
-        int htotal, hdisplay, clock, pixel_size = 0;
-        int line_time_us, line_count, entries;
+        int htotal, hdisplay, clock, pixel_size;
+        int line_time_us, line_count;
+        int entries, tlb_miss;
 
         crtc = intel_get_crtc_for_pipe(dev, pipe);
         if (crtc->fb == NULL || !crtc->enabled)
@@ -3406,37 +3460,141 @@ static bool ironlake_compute_wm0(struct drm_device *dev,
         pixel_size = crtc->fb->bits_per_pixel / 8;
 
         /* Use the small buffer method to calculate plane watermark */
-        entries = ((clock * pixel_size / 1000) * ILK_LP0_PLANE_LATENCY) / 1000;
-        entries = DIV_ROUND_UP(entries,
-                               ironlake_display_wm_info.cacheline_size);
-        *plane_wm = entries + ironlake_display_wm_info.guard_size;
-        if (*plane_wm > (int)ironlake_display_wm_info.max_wm)
-                *plane_wm = ironlake_display_wm_info.max_wm;
+        entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
+        tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
+        if (tlb_miss > 0)
+                entries += tlb_miss;
+        entries = DIV_ROUND_UP(entries, display->cacheline_size);
+        *plane_wm = entries + display->guard_size;
+        if (*plane_wm > (int)display->max_wm)
+                *plane_wm = display->max_wm;
 
         /* Use the large buffer method to calculate cursor watermark */
         line_time_us = ((htotal * 1000) / clock);
-        line_count = (ILK_LP0_CURSOR_LATENCY / line_time_us + 1000) / 1000;
+        line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
         entries = line_count * 64 * pixel_size;
-        entries = DIV_ROUND_UP(entries,
-                               ironlake_cursor_wm_info.cacheline_size);
-        *cursor_wm = entries + ironlake_cursor_wm_info.guard_size;
-        if (*cursor_wm > ironlake_cursor_wm_info.max_wm)
-                *cursor_wm = ironlake_cursor_wm_info.max_wm;
+        tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
+        if (tlb_miss > 0)
+                entries += tlb_miss;
+        entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+        *cursor_wm = entries + cursor->guard_size;
+        if (*cursor_wm > (int)cursor->max_wm)
+                *cursor_wm = (int)cursor->max_wm;
+
+        return true;
+}
+
+/*
+ * Check the wm result.
+ *
+ * If any calculated watermark values is larger than the maximum value that
+ * can be programmed into the associated watermark register, that watermark
+ * must be disabled.
+ */
+static bool ironlake_check_srwm(struct drm_device *dev, int level,
+                                int fbc_wm, int display_wm, int cursor_wm,
+                                const struct intel_watermark_params *display,
+                                const struct intel_watermark_params *cursor)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
+                      " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
+
+        if (fbc_wm > SNB_FBC_MAX_SRWM) {
+                DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
+                              fbc_wm, SNB_FBC_MAX_SRWM, level);
+
+                /* fbc has it's own way to disable FBC WM */
+                I915_WRITE(DISP_ARB_CTL,
+                           I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
+                return false;
+        }
+
+        if (display_wm > display->max_wm) {
+                DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
+                              display_wm, SNB_DISPLAY_MAX_SRWM, level);
+                return false;
+        }
+
+        if (cursor_wm > cursor->max_wm) {
+                DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
+                              cursor_wm, SNB_CURSOR_MAX_SRWM, level);
+                return false;
+        }
+
+        if (!(fbc_wm || display_wm || cursor_wm)) {
+                DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
+                return false;
+        }
 
         return true;
 }
 
+/*
+ * Compute watermark values of WM[1-3],
+ */
+static bool ironlake_compute_srwm(struct drm_device *dev, int level,
+                                  int hdisplay, int htotal,
+                                  int pixel_size, int clock, int latency_ns,
+                                  const struct intel_watermark_params *display,
+                                  const struct intel_watermark_params *cursor,
+                                  int *fbc_wm, int *display_wm, int *cursor_wm)
+{
+
+        unsigned long line_time_us;
+        int line_count, line_size;
+        int small, large;
+        int entries;
+
+        if (!latency_ns) {
+                *fbc_wm = *display_wm = *cursor_wm = 0;
+                return false;
+        }
+
+        line_time_us = (htotal * 1000) / clock;
+        line_count = (latency_ns / line_time_us + 1000) / 1000;
+        line_size = hdisplay * pixel_size;
+
+        /* Use the minimum of the small and large buffer method for primary */
+        small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+        large = line_count * line_size;
+
+        entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+        *display_wm = entries + display->guard_size;
+
+        /*
+         * Spec says:
+         * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
+         */
+        *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
+
+        /* calculate the self-refresh watermark for display cursor */
+        entries = line_count * pixel_size * 64;
+        entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+        *cursor_wm = entries + cursor->guard_size;
+
+        return ironlake_check_srwm(dev, level,
+                                   *fbc_wm, *display_wm, *cursor_wm,
+                                   display, cursor);
+}
+
 static void ironlake_update_wm(struct drm_device *dev,
                                int planea_clock, int planeb_clock,
-                               int sr_hdisplay, int sr_htotal,
+                               int hdisplay, int htotal,
                                int pixel_size)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int plane_wm, cursor_wm, enabled;
-        int tmp;
+        int fbc_wm, plane_wm, cursor_wm, enabled;
+        int clock;
 
         enabled = 0;
-        if (ironlake_compute_wm0(dev, 0, &plane_wm, &cursor_wm)) {
+        if (ironlake_compute_wm0(dev, 0,
+                                 &ironlake_display_wm_info,
+                                 ILK_LP0_PLANE_LATENCY,
+                                 &ironlake_cursor_wm_info,
+                                 ILK_LP0_CURSOR_LATENCY,
+                                 &plane_wm, &cursor_wm)) {
                 I915_WRITE(WM0_PIPEA_ILK,
                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
@@ -3445,7 +3603,12 @@ static void ironlake_update_wm(struct drm_device *dev,
                 enabled++;
         }
 
-        if (ironlake_compute_wm0(dev, 1, &plane_wm, &cursor_wm)) {
+        if (ironlake_compute_wm0(dev, 1,
+                                 &ironlake_display_wm_info,
+                                 ILK_LP0_PLANE_LATENCY,
+                                 &ironlake_cursor_wm_info,
+                                 ILK_LP0_CURSOR_LATENCY,
+                                 &plane_wm, &cursor_wm)) {
                 I915_WRITE(WM0_PIPEB_ILK,
                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
@@ -3458,57 +3621,151 @@ static void ironlake_update_wm(struct drm_device *dev,
          * Calculate and update the self-refresh watermark only when one
          * display plane is used.
          */
-        tmp = 0;
-        if (enabled == 1 && /* XXX disabled due to buggy implmentation? */ 0) {
-                unsigned long line_time_us;
-                int small, large, plane_fbc;
-                int sr_clock, entries;
-                int line_count, line_size;
-                /* Read the self-refresh latency. The unit is 0.5us */
-                int ilk_sr_latency = I915_READ(MLTR_ILK) & ILK_SRLT_MASK;
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
 
-                sr_clock = planea_clock ? planea_clock : planeb_clock;
-                line_time_us = (sr_htotal * 1000) / sr_clock;
+        if (enabled != 1)
+                return;
 
-                /* Use ns/us then divide to preserve precision */
-                line_count = ((ilk_sr_latency * 500) / line_time_us + 1000)
-                        / 1000;
-                line_size = sr_hdisplay * pixel_size;
+        clock = planea_clock ? planea_clock : planeb_clock;
 
-                /* Use the minimum of the small and large buffer method for primary */
-                small = ((sr_clock * pixel_size / 1000) * (ilk_sr_latency * 500)) / 1000;
-                large = line_count * line_size;
+        /* WM1 */
+        if (!ironlake_compute_srwm(dev, 1, hdisplay, htotal, pixel_size,
+                                   clock, ILK_READ_WM1_LATENCY() * 500,
+                                   &ironlake_display_srwm_info,
+                                   &ironlake_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
 
-                entries = DIV_ROUND_UP(min(small, large),
-                                       ironlake_display_srwm_info.cacheline_size);
+        I915_WRITE(WM1_LP_ILK,
+                   WM1_LP_SR_EN |
+                   (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
+
+        /* WM2 */
+        if (!ironlake_compute_srwm(dev, 2, hdisplay, htotal, pixel_size,
+                                   clock, ILK_READ_WM2_LATENCY() * 500,
+                                   &ironlake_display_srwm_info,
+                                   &ironlake_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
 
-                plane_fbc = entries * 64;
-                plane_fbc = DIV_ROUND_UP(plane_fbc, line_size);
+        I915_WRITE(WM2_LP_ILK,
+                   WM2_LP_EN |
+                   (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
 
-                plane_wm = entries + ironlake_display_srwm_info.guard_size;
-                if (plane_wm > (int)ironlake_display_srwm_info.max_wm)
-                        plane_wm = ironlake_display_srwm_info.max_wm;
+        /*
+         * WM3 is unsupported on ILK, probably because we don't have latency
+         * data for that power state
+         */
+}
 
-                /* calculate the self-refresh watermark for display cursor */
-                entries = line_count * pixel_size * 64;
-                entries = DIV_ROUND_UP(entries,
-                                       ironlake_cursor_srwm_info.cacheline_size);
+static void sandybridge_update_wm(struct drm_device *dev,
+                               int planea_clock, int planeb_clock,
+                               int hdisplay, int htotal,
+                               int pixel_size)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int latency = SNB_READ_WM0_LATENCY() * 100;     /* In unit 0.1us */
+        int fbc_wm, plane_wm, cursor_wm, enabled;
+        int clock;
 
-                cursor_wm = entries + ironlake_cursor_srwm_info.guard_size;
-                if (cursor_wm > (int)ironlake_cursor_srwm_info.max_wm)
-                        cursor_wm = ironlake_cursor_srwm_info.max_wm;
+        enabled = 0;
+        if (ironlake_compute_wm0(dev, 0,
+                                 &sandybridge_display_wm_info, latency,
+                                 &sandybridge_cursor_wm_info, latency,
+                                 &plane_wm, &cursor_wm)) {
+                I915_WRITE(WM0_PIPEA_ILK,
+                           (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+                DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
+                              " plane %d, " "cursor: %d\n",
+                              plane_wm, cursor_wm);
+                enabled++;
+        }
 
-                /* configure watermark and enable self-refresh */
-                tmp = (WM1_LP_SR_EN |
-                       (ilk_sr_latency << WM1_LP_LATENCY_SHIFT) |
-                       (plane_fbc << WM1_LP_FBC_SHIFT) |
-                       (plane_wm << WM1_LP_SR_SHIFT) |
-                       cursor_wm);
-                DRM_DEBUG_KMS("self-refresh watermark: display plane %d, fbc lines %d,"
-                              " cursor %d\n", plane_wm, plane_fbc, cursor_wm);
+        if (ironlake_compute_wm0(dev, 1,
+                                 &sandybridge_display_wm_info, latency,
+                                 &sandybridge_cursor_wm_info, latency,
+                                 &plane_wm, &cursor_wm)) {
+                I915_WRITE(WM0_PIPEB_ILK,
+                           (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+                DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
+                              " plane %d, cursor: %d\n",
+                              plane_wm, cursor_wm);
+                enabled++;
         }
-        I915_WRITE(WM1_LP_ILK, tmp);
-        /* XXX setup WM2 and WM3 */
+
+        /*
+         * Calculate and update the self-refresh watermark only when one
+         * display plane is used.
+         *
+         * SNB support 3 levels of watermark.
+         *
+         * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
+         * and disabled in the descending order
+         *
+         */
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
+
+        if (enabled != 1)
+                return;
+
+        clock = planea_clock ? planea_clock : planeb_clock;
+
+        /* WM1 */
+        if (!ironlake_compute_srwm(dev, 1, hdisplay, htotal, pixel_size,
+                                   clock, SNB_READ_WM1_LATENCY() * 500,
+                                   &sandybridge_display_srwm_info,
+                                   &sandybridge_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
+
+        I915_WRITE(WM1_LP_ILK,
+                   WM1_LP_SR_EN |
+                   (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
+
+        /* WM2 */
+        if (!ironlake_compute_srwm(dev, 2,
+                                   hdisplay, htotal, pixel_size,
+                                   clock, SNB_READ_WM2_LATENCY() * 500,
+                                   &sandybridge_display_srwm_info,
+                                   &sandybridge_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
+
+        I915_WRITE(WM2_LP_ILK,
+                   WM2_LP_EN |
+                   (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
+
+        /* WM3 */
+        if (!ironlake_compute_srwm(dev, 3,
+                                   hdisplay, htotal, pixel_size,
+                                   clock, SNB_READ_WM3_LATENCY() * 500,
+                                   &sandybridge_display_srwm_info,
+                                   &sandybridge_cursor_srwm_info,
+                                   &fbc_wm, &plane_wm, &cursor_wm))
+                return;
+
+        I915_WRITE(WM3_LP_ILK,
+                   WM3_LP_EN |
+                   (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+                   (fbc_wm << WM1_LP_FBC_SHIFT) |
+                   (plane_wm << WM1_LP_SR_SHIFT) |
+                   cursor_wm);
 }
 
 /**
@@ -3586,6 +3843,11 @@ static void intel_update_watermarks(struct drm_device *dev)
                                     sr_hdisplay, sr_htotal, pixel_size);
 }
 
+static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
+{
+        return dev_priv->lvds_use_ssc && i915_panel_use_ssc;
+}
+
 static int intel_crtc_mode_set(struct drm_crtc *crtc,
                                struct drm_display_mode *mode,
                                struct drm_display_mode *adjusted_mode,
@@ -3648,7 +3910,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 num_connectors++;
         }
 
-        if (is_lvds && dev_priv->lvds_use_ssc && num_connectors < 2) {
+        if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
                 refclk = dev_priv->lvds_ssc_freq * 1000;
                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
                               refclk / 1000);
@@ -3666,7 +3928,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
          * refclk, or FALSE.  The returned values represent the clock equation:
          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
          */
-        limit = intel_limit(crtc);
+        limit = intel_limit(crtc, refclk);
         ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock);
         if (!ok) {
                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
@@ -3716,10 +3978,11 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
 
         /* FDI link */
         if (HAS_PCH_SPLIT(dev)) {
+                int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
                 int lane = 0, link_bw, bpp;
                 /* CPU eDP doesn't require FDI link, so just set DP M/N
                    according to current link config */
-                if (has_edp_encoder && !intel_encoder_is_pch_edp(&encoder->base)) {
+                if (has_edp_encoder && !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
                         target_clock = mode->clock;
                         intel_edp_link_config(has_edp_encoder,
                                               &lane, &link_bw);
@@ -3799,6 +4062,8 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
 
                 intel_crtc->fdi_lanes = lane;
 
+                if (pixel_multiplier > 1)
+                        link_bw *= pixel_multiplier;
                 ironlake_compute_m_n(bpp, lane, target_clock, link_bw, &m_n);
         }
 
@@ -3820,7 +4085,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 udelay(200);
 
                 if (has_edp_encoder) {
-                        if (dev_priv->lvds_use_ssc) {
+                        if (intel_panel_use_ssc(dev_priv)) {
                                 temp |= DREF_SSC1_ENABLE;
                                 I915_WRITE(PCH_DREF_CONTROL, temp);
 
@@ -3831,13 +4096,13 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
 
                         /* Enable CPU source on CPU attached eDP */
                         if (!intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
-                                if (dev_priv->lvds_use_ssc)
+                                if (intel_panel_use_ssc(dev_priv))
                                         temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
                                 else
                                         temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
                         } else {
                                 /* Enable SSC on PCH eDP if needed */
-                                if (dev_priv->lvds_use_ssc) {
+                                if (intel_panel_use_ssc(dev_priv)) {
                                         DRM_ERROR("enabling SSC on PCH\n");
                                         temp |= DREF_SUPERSPREAD_SOURCE_ENABLE;
                                 }
@@ -3860,6 +4125,22 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                                 reduced_clock.m2;
         }
 
+        /* Enable autotuning of the PLL clock (if permissible) */
+        if (HAS_PCH_SPLIT(dev)) {
+                int factor = 21;
+
+                if (is_lvds) {
+                        if ((intel_panel_use_ssc(dev_priv) &&
+                             dev_priv->lvds_ssc_freq == 100) ||
+                            (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
+                                factor = 25;
+                } else if (is_sdvo && is_tv)
+                        factor = 20;
+
+                if (clock.m1 < factor * clock.n)
+                        fp |= FP_CB_TUNE;
+        }
+
         dpll = 0;
         if (!HAS_PCH_SPLIT(dev))
                 dpll = DPLL_VGA_MODE_DIS;
@@ -3928,7 +4209,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 /* XXX: just matching BIOS for now */
                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
                 dpll |= 3;
-        else if (is_lvds && dev_priv->lvds_use_ssc && num_connectors < 2)
+        else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
         else
                 dpll |= PLL_REF_INPUT_DREFCLK;
@@ -4074,7 +4355,6 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
         }
 
         if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
-                I915_WRITE(fp_reg, fp);
                 I915_WRITE(dpll_reg, dpll);
 
                 /* Wait for the clocks to stabilize. */
@@ -4092,13 +4372,13 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                         }
                         I915_WRITE(DPLL_MD(pipe), temp);
                 } else {
-                        /* write it again -- the BIOS does, after all */
+                        /* The pixel multiplier can only be updated once the
+                         * DPLL is enabled and the clocks are stable.
+                         *
+                         * So write it again.
+                         */
                         I915_WRITE(dpll_reg, dpll);
                 }
-
-                /* Wait for the clocks to stabilize. */
-                POSTING_READ(dpll_reg);
-                udelay(150);
         }
 
         intel_crtc->lowfreq_avail = false;
@@ -4334,15 +4614,14 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 }
 
 static int intel_crtc_cursor_set(struct drm_crtc *crtc,
-                                 struct drm_file *file_priv,
+                                 struct drm_file *file,
                                  uint32_t handle,
                                  uint32_t width, uint32_t height)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_gem_object *bo;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         uint32_t addr;
         int ret;
 
@@ -4352,7 +4631,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         if (!handle) {
                 DRM_DEBUG_KMS("cursor off\n");
                 addr = 0;
-                bo = NULL;
+                obj = NULL;
                 mutex_lock(&dev->struct_mutex);
                 goto finish;
         }
@@ -4363,13 +4642,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                 return -EINVAL;
         }
 
-        bo = drm_gem_object_lookup(dev, file_priv, handle);
-        if (!bo)
+        obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
+        if (!obj)
                 return -ENOENT;
 
-        obj_priv = to_intel_bo(bo);
-
-        if (bo->size < width * height * 4) {
+        if (obj->base.size < width * height * 4) {
                 DRM_ERROR("buffer is to small\n");
                 ret = -ENOMEM;
                 goto fail;
@@ -4378,29 +4655,41 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         /* we only need to pin inside GTT if cursor is non-phy */
         mutex_lock(&dev->struct_mutex);
         if (!dev_priv->info->cursor_needs_physical) {
-                ret = i915_gem_object_pin(bo, PAGE_SIZE);
+                if (obj->tiling_mode) {
+                        DRM_ERROR("cursor cannot be tiled\n");
+                        ret = -EINVAL;
+                        goto fail_locked;
+                }
+
+                ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
                 if (ret) {
                         DRM_ERROR("failed to pin cursor bo\n");
                         goto fail_locked;
                 }
 
-                ret = i915_gem_object_set_to_gtt_domain(bo, 0);
+                ret = i915_gem_object_set_to_gtt_domain(obj, 0);
+                if (ret) {
+                        DRM_ERROR("failed to move cursor bo into the GTT\n");
+                        goto fail_unpin;
+                }
+
+                ret = i915_gem_object_put_fence(obj);
                 if (ret) {
                         DRM_ERROR("failed to move cursor bo into the GTT\n");
                         goto fail_unpin;
                 }
 
-                addr = obj_priv->gtt_offset;
+                addr = obj->gtt_offset;
         } else {
                 int align = IS_I830(dev) ? 16 * 1024 : 256;
-                ret = i915_gem_attach_phys_object(dev, bo,
+                ret = i915_gem_attach_phys_object(dev, obj,
                                                   (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
                                                   align);
                 if (ret) {
                         DRM_ERROR("failed to attach phys object\n");
                         goto fail_locked;
                 }
-                addr = obj_priv->phys_obj->handle->busaddr;
+                addr = obj->phys_obj->handle->busaddr;
         }
 
         if (IS_GEN2(dev))
@@ -4409,17 +4698,17 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
  finish:
         if (intel_crtc->cursor_bo) {
                 if (dev_priv->info->cursor_needs_physical) {
-                        if (intel_crtc->cursor_bo != bo)
+                        if (intel_crtc->cursor_bo != obj)
                                 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
                 } else
                         i915_gem_object_unpin(intel_crtc->cursor_bo);
-                drm_gem_object_unreference(intel_crtc->cursor_bo);
+                drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
         }
 
         mutex_unlock(&dev->struct_mutex);
 
         intel_crtc->cursor_addr = addr;
-        intel_crtc->cursor_bo = bo;
+        intel_crtc->cursor_bo = obj;
         intel_crtc->cursor_width = width;
         intel_crtc->cursor_height = height;
 
@@ -4427,11 +4716,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 
         return 0;
 fail_unpin:
-        i915_gem_object_unpin(bo);
+        i915_gem_object_unpin(obj);
 fail_locked:
         mutex_unlock(&dev->struct_mutex);
 fail:
-        drm_gem_object_unreference_unlocked(bo);
+        drm_gem_object_unreference_unlocked(&obj->base);
         return ret;
 }
 
@@ -4742,8 +5031,14 @@ static void intel_gpu_idle_timer(unsigned long arg)
         struct drm_device *dev = (struct drm_device *)arg;
         drm_i915_private_t *dev_priv = dev->dev_private;
 
-        dev_priv->busy = false;
+        if (!list_empty(&dev_priv->mm.active_list)) {
+                /* Still processing requests, so just re-arm the timer. */
+                mod_timer(&dev_priv->idle_timer, jiffies +
+                          msecs_to_jiffies(GPU_IDLE_TIMEOUT));
+                return;
+        }
 
+        dev_priv->busy = false;
         queue_work(dev_priv->wq, &dev_priv->idle_work);
 }
 
@@ -4754,9 +5049,17 @@ static void intel_crtc_idle_timer(unsigned long arg)
         struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
         struct drm_crtc *crtc = &intel_crtc->base;
         drm_i915_private_t *dev_priv = crtc->dev->dev_private;
+        struct intel_framebuffer *intel_fb;
 
-        intel_crtc->busy = false;
+        intel_fb = to_intel_framebuffer(crtc->fb);
+        if (intel_fb && intel_fb->obj->active) {
+                /* The framebuffer is still being accessed by the GPU. */
+                mod_timer(&intel_crtc->idle_timer, jiffies +
+                          msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
+                return;
+        }
 
+        intel_crtc->busy = false;
         queue_work(dev_priv->wq, &dev_priv->idle_work);
 }
 
@@ -4766,8 +5069,8 @@ static void intel_increase_pllclock(struct drm_crtc *crtc)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
-        int dpll = I915_READ(dpll_reg);
+        int dpll_reg = DPLL(pipe);
+        int dpll;
 
         if (HAS_PCH_SPLIT(dev))
                 return;
@@ -4775,17 +5078,19 @@ static void intel_increase_pllclock(struct drm_crtc *crtc)
         if (!dev_priv->lvds_downclock_avail)
                 return;
 
+        dpll = I915_READ(dpll_reg);
         if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) {
                 DRM_DEBUG_DRIVER("upclocking LVDS\n");
 
                 /* Unlock panel regs */
-                I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) |
-                           PANEL_UNLOCK_REGS);
+                I915_WRITE(PP_CONTROL,
+                           I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
 
                 dpll &= ~DISPLAY_RATE_SELECT_FPA1;
                 I915_WRITE(dpll_reg, dpll);
-                dpll = I915_READ(dpll_reg);
+                POSTING_READ(dpll_reg);
                 intel_wait_for_vblank(dev, pipe);
+
                 dpll = I915_READ(dpll_reg);
                 if (dpll & DISPLAY_RATE_SELECT_FPA1)
                         DRM_DEBUG_DRIVER("failed to upclock LVDS!\n");
@@ -4891,7 +5196,7 @@ static void intel_idle_update(struct work_struct *work)
  * buffer), we'll also mark the display as busy, so we know to increase its
  * clock frequency.
  */
-void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj)
+void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = NULL;
@@ -4972,8 +5277,9 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 
         mutex_lock(&work->dev->struct_mutex);
         i915_gem_object_unpin(work->old_fb_obj);
-        drm_gem_object_unreference(work->pending_flip_obj);
-        drm_gem_object_unreference(work->old_fb_obj);
+        drm_gem_object_unreference(&work->pending_flip_obj->base);
+        drm_gem_object_unreference(&work->old_fb_obj->base);
+
         mutex_unlock(&work->dev->struct_mutex);
         kfree(work);
 }
@@ -4984,15 +5290,17 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct drm_pending_vblank_event *e;
-        struct timeval now;
+        struct timeval tnow, tvbl;
         unsigned long flags;
 
         /* Ignore early vblank irqs */
         if (intel_crtc == NULL)
                 return;
 
+        do_gettimeofday(&tnow);
+
         spin_lock_irqsave(&dev->event_lock, flags);
         work = intel_crtc->unpin_work;
         if (work == NULL || !work->pending) {
@@ -5001,26 +5309,49 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
         }
 
         intel_crtc->unpin_work = NULL;
-        drm_vblank_put(dev, intel_crtc->pipe);
 
         if (work->event) {
                 e = work->event;
-                do_gettimeofday(&now);
-                e->event.sequence = drm_vblank_count(dev, intel_crtc->pipe);
-                e->event.tv_sec = now.tv_sec;
-                e->event.tv_usec = now.tv_usec;
+                e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl);
+
+                /* Called before vblank count and timestamps have
+                 * been updated for the vblank interval of flip
+                 * completion? Need to increment vblank count and
+                 * add one videorefresh duration to returned timestamp
+                 * to account for this. We assume this happened if we
+                 * get called over 0.9 frame durations after the last
+                 * timestamped vblank.
+                 *
+                 * This calculation can not be used with vrefresh rates
+                 * below 5Hz (10Hz to be on the safe side) without
+                 * promoting to 64 integers.
+                 */
+                if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) >
+                    9 * crtc->framedur_ns) {
+                        e->event.sequence++;
+                        tvbl = ns_to_timeval(timeval_to_ns(&tvbl) +
+                                             crtc->framedur_ns);
+                }
+
+                e->event.tv_sec = tvbl.tv_sec;
+                e->event.tv_usec = tvbl.tv_usec;
+
                 list_add_tail(&e->base.link,
                               &e->base.file_priv->event_list);
                 wake_up_interruptible(&e->base.file_priv->event_wait);
         }
 
+        drm_vblank_put(dev, intel_crtc->pipe);
+
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
-        obj_priv = to_intel_bo(work->old_fb_obj);
+        obj = work->old_fb_obj;
+
         atomic_clear_mask(1 << intel_crtc->plane,
-                          &obj_priv->pending_flip.counter);
-        if (atomic_read(&obj_priv->pending_flip) == 0)
+                          &obj->pending_flip.counter);
+        if (atomic_read(&obj->pending_flip) == 0)
                 wake_up(&dev_priv->pending_flip_queue);
+
         schedule_work(&work->work);
 
         trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
@@ -5066,8 +5397,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_framebuffer *intel_fb;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
         unsigned long flags, offset;
@@ -5101,13 +5431,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         obj = intel_fb->obj;
 
         mutex_lock(&dev->struct_mutex);
-        ret = intel_pin_and_fence_fb_obj(dev, obj, true);
+        ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
         if (ret)
                 goto cleanup_work;
 
         /* Reference the objects for the scheduled work. */
-        drm_gem_object_reference(work->old_fb_obj);
-        drm_gem_object_reference(obj);
+        drm_gem_object_reference(&work->old_fb_obj->base);
+        drm_gem_object_reference(&obj->base);
 
         crtc->fb = fb;
 
@@ -5115,22 +5445,16 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         if (ret)
                 goto cleanup_objs;
 
-        /* Block clients from rendering to the new back buffer until
-         * the flip occurs and the object is no longer visible.
-         */
-        atomic_add(1 << intel_crtc->plane,
-                   &to_intel_bo(work->old_fb_obj)->pending_flip);
-
-        work->pending_flip_obj = obj;
-        obj_priv = to_intel_bo(obj);
-
         if (IS_GEN3(dev) || IS_GEN2(dev)) {
                 u32 flip_mask;
 
                 /* Can't queue multiple flips, so wait for the previous
                  * one to finish before executing the next.
                  */
-                BEGIN_LP_RING(2);
+                ret = BEGIN_LP_RING(2);
+                if (ret)
+                        goto cleanup_objs;
+
                 if (intel_crtc->plane)
                         flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
                 else
@@ -5140,18 +5464,28 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                 ADVANCE_LP_RING();
         }
 
+        work->pending_flip_obj = obj;
+
         work->enable_stall_check = true;
 
         /* Offset into the new buffer for cases of shared fbs between CRTCs */
         offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
 
-        BEGIN_LP_RING(4);
-        switch(INTEL_INFO(dev)->gen) {
+        ret = BEGIN_LP_RING(4);
+        if (ret)
+                goto cleanup_objs;
+
+        /* Block clients from rendering to the new back buffer until
+         * the flip occurs and the object is no longer visible.
+         */
+        atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
+
+        switch (INTEL_INFO(dev)->gen) {
         case 2:
                 OUT_RING(MI_DISPLAY_FLIP |
                          MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
                 OUT_RING(fb->pitch);
-                OUT_RING(obj_priv->gtt_offset + offset);
+                OUT_RING(obj->gtt_offset + offset);
                 OUT_RING(MI_NOOP);
                 break;
 
@@ -5159,7 +5493,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                 OUT_RING(MI_DISPLAY_FLIP_I915 |
                          MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
                 OUT_RING(fb->pitch);
-                OUT_RING(obj_priv->gtt_offset + offset);
+                OUT_RING(obj->gtt_offset + offset);
                 OUT_RING(MI_NOOP);
                 break;
 
@@ -5172,7 +5506,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                 OUT_RING(MI_DISPLAY_FLIP |
                          MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
                 OUT_RING(fb->pitch);
-                OUT_RING(obj_priv->gtt_offset | obj_priv->tiling_mode);
+                OUT_RING(obj->gtt_offset | obj->tiling_mode);
 
                 /* XXX Enabling the panel-fitter across page-flip is so far
                  * untested on non-native modes, so ignore it for now.
@@ -5186,8 +5520,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         case 6:
                 OUT_RING(MI_DISPLAY_FLIP |
                          MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-                OUT_RING(fb->pitch | obj_priv->tiling_mode);
-                OUT_RING(obj_priv->gtt_offset);
+                OUT_RING(fb->pitch | obj->tiling_mode);
+                OUT_RING(obj->gtt_offset);
 
                 pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
                 pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff;
@@ -5203,8 +5537,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         return 0;
 
 cleanup_objs:
-        drm_gem_object_unreference(work->old_fb_obj);
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&work->old_fb_obj->base);
+        drm_gem_object_unreference(&obj->base);
 cleanup_work:
         mutex_unlock(&dev->struct_mutex);
 
@@ -5217,6 +5551,18 @@ cleanup_work:
         return ret;
 }
 
+static void intel_crtc_reset(struct drm_crtc *crtc)
+{
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+        /* Reset flags back to the 'unknown' status so that they
+         * will be correctly set on the initial modeset.
+         */
+        intel_crtc->cursor_addr = 0;
+        intel_crtc->dpms_mode = -1;
+        intel_crtc->active = true; /* force the pipe off on setup_init_config */
+}
+
 static struct drm_crtc_helper_funcs intel_helper_funcs = {
         .dpms = intel_crtc_dpms,
         .mode_fixup = intel_crtc_mode_fixup,
@@ -5228,6 +5574,7 @@ static struct drm_crtc_helper_funcs intel_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs intel_crtc_funcs = {
+        .reset = intel_crtc_reset,
         .cursor_set = intel_crtc_cursor_set,
         .cursor_move = intel_crtc_cursor_move,
         .gamma_set = intel_crtc_gamma_set,
@@ -5236,6 +5583,55 @@ static const struct drm_crtc_funcs intel_crtc_funcs = {
         .page_flip = intel_crtc_page_flip,
 };
 
+static void intel_sanitize_modesetting(struct drm_device *dev,
+                                       int pipe, int plane)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 reg, val;
+
+        if (HAS_PCH_SPLIT(dev))
+                return;
+
+        /* Who knows what state these registers were left in by the BIOS or
+         * grub?
+         *
+         * If we leave the registers in a conflicting state (e.g. with the
+         * display plane reading from the other pipe than the one we intend
+         * to use) then when we attempt to teardown the active mode, we will
+         * not disable the pipes and planes in the correct order -- leaving
+         * a plane reading from a disabled pipe and possibly leading to
+         * undefined behaviour.
+         */
+
+        reg = DSPCNTR(plane);
+        val = I915_READ(reg);
+
+        if ((val & DISPLAY_PLANE_ENABLE) == 0)
+                return;
+        if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe)
+                return;
+
+        /* This display plane is active and attached to the other CPU pipe. */
+        pipe = !pipe;
+
+        /* Disable the plane and wait for it to stop reading from the pipe. */
+        I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
+        intel_flush_display_plane(dev, plane);
+
+        if (IS_GEN2(dev))
+                intel_wait_for_vblank(dev, pipe);
+
+        if (pipe == 0 && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
+                return;
+
+        /* Switch off the pipe. */
+        reg = PIPECONF(pipe);
+        val = I915_READ(reg);
+        if (val & PIPECONF_ENABLE) {
+                I915_WRITE(reg, val & ~PIPECONF_ENABLE);
+                intel_wait_for_pipe_off(dev, pipe);
+        }
+}
 
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
@@ -5269,9 +5665,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
 
-        intel_crtc->cursor_addr = 0;
-        intel_crtc->dpms_mode = -1;
-        intel_crtc->active = true; /* force the pipe off on setup_init_config */
+        intel_crtc_reset(&intel_crtc->base);
 
         if (HAS_PCH_SPLIT(dev)) {
                 intel_helper_funcs.prepare = ironlake_crtc_prepare;
@@ -5287,10 +5681,12 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 
         setup_timer(&intel_crtc->idle_timer, intel_crtc_idle_timer,
                     (unsigned long)intel_crtc);
+
+        intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane);
 }
 
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
-                                struct drm_file *file_priv)
+                                struct drm_file *file)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
@@ -5331,19 +5727,41 @@ static int intel_encoder_clones(struct drm_device *dev, int type_mask)
         return index_mask;
 }
 
+static bool has_edp_a(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!IS_MOBILE(dev))
+                return false;
+
+        if ((I915_READ(DP_A) & DP_DETECTED) == 0)
+                return false;
+
+        if (IS_GEN5(dev) &&
+            (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE))
+                return false;
+
+        return true;
+}
+
 static void intel_setup_outputs(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_encoder *encoder;
         bool dpd_is_edp = false;
+        bool has_lvds = false;
 
         if (IS_MOBILE(dev) && !IS_I830(dev))
-                intel_lvds_init(dev);
+                has_lvds = intel_lvds_init(dev);
+        if (!has_lvds && !HAS_PCH_SPLIT(dev)) {
+                /* disable the panel fitter on everything but LVDS */
+                I915_WRITE(PFIT_CONTROL, 0);
+        }
 
         if (HAS_PCH_SPLIT(dev)) {
                 dpd_is_edp = intel_dpd_is_edp(dev);
 
-                if (IS_MOBILE(dev) && (I915_READ(DP_A) & DP_DETECTED))
+                if (has_edp_a(dev))
                         intel_dp_init(dev, DP_A);
 
                 if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
@@ -5428,6 +5846,8 @@ static void intel_setup_outputs(struct drm_device *dev)
                 encoder->base.possible_clones =
                         intel_encoder_clones(dev, encoder->clone_mask);
         }
+
+        intel_panel_setup_backlight(dev);
 }
 
 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
@@ -5435,19 +5855,19 @@ static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 
         drm_framebuffer_cleanup(fb);
-        drm_gem_object_unreference_unlocked(intel_fb->obj);
+        drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
 
         kfree(intel_fb);
 }
 
 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
-                                                struct drm_file *file_priv,
+                                                struct drm_file *file,
                                                 unsigned int *handle)
 {
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-        struct drm_gem_object *object = intel_fb->obj;
+        struct drm_i915_gem_object *obj = intel_fb->obj;
 
-        return drm_gem_handle_create(file_priv, object, handle);
+        return drm_gem_handle_create(file, &obj->base, handle);
 }
 
 static const struct drm_framebuffer_funcs intel_fb_funcs = {
@@ -5458,12 +5878,11 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
 int intel_framebuffer_init(struct drm_device *dev,
                            struct intel_framebuffer *intel_fb,
                            struct drm_mode_fb_cmd *mode_cmd,
-                           struct drm_gem_object *obj)
+                           struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         int ret;
 
-        if (obj_priv->tiling_mode == I915_TILING_Y)
+        if (obj->tiling_mode == I915_TILING_Y)
                 return -EINVAL;
 
         if (mode_cmd->pitch & 63)
@@ -5495,11 +5914,11 @@ intel_user_framebuffer_create(struct drm_device *dev,
                               struct drm_file *filp,
                               struct drm_mode_fb_cmd *mode_cmd)
 {
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         struct intel_framebuffer *intel_fb;
         int ret;
 
-        obj = drm_gem_object_lookup(dev, filp, mode_cmd->handle);
+        obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handle));
         if (!obj)
                 return ERR_PTR(-ENOENT);
 
@@ -5507,10 +5926,9 @@ intel_user_framebuffer_create(struct drm_device *dev,
         if (!intel_fb)
                 return ERR_PTR(-ENOMEM);
 
-        ret = intel_framebuffer_init(dev, intel_fb,
-                                     mode_cmd, obj);
+        ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
         if (ret) {
-                drm_gem_object_unreference_unlocked(obj);
+                drm_gem_object_unreference_unlocked(&obj->base);
                 kfree(intel_fb);
                 return ERR_PTR(ret);
         }
@@ -5523,10 +5941,10 @@ static const struct drm_mode_config_funcs intel_mode_funcs = {
         .output_poll_changed = intel_fb_output_poll_changed,
 };
 
-static struct drm_gem_object *
+static struct drm_i915_gem_object *
 intel_alloc_context_page(struct drm_device *dev)
 {
-        struct drm_gem_object *ctx;
+        struct drm_i915_gem_object *ctx;
         int ret;
 
         ctx = i915_gem_alloc_object(dev, 4096);
@@ -5536,7 +5954,7 @@ intel_alloc_context_page(struct drm_device *dev)
         }
 
         mutex_lock(&dev->struct_mutex);
-        ret = i915_gem_object_pin(ctx, 4096);
+        ret = i915_gem_object_pin(ctx, 4096, true);
         if (ret) {
                 DRM_ERROR("failed to pin power context: %d\n", ret);
                 goto err_unref;
@@ -5554,7 +5972,7 @@ intel_alloc_context_page(struct drm_device *dev)
 err_unpin:
         i915_gem_object_unpin(ctx);
 err_unref:
-        drm_gem_object_unreference(ctx);
+        drm_gem_object_unreference(&ctx->base);
         mutex_unlock(&dev->struct_mutex);
         return NULL;
 }
@@ -5666,6 +6084,25 @@ void ironlake_disable_drps(struct drm_device *dev)
 
 }
 
+void gen6_set_rps(struct drm_device *dev, u8 val)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 swreq;
+
+        swreq = (val & 0x3ff) << 25;
+        I915_WRITE(GEN6_RPNSWREQ, swreq);
+}
+
+void gen6_disable_rps(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
+        I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
+        I915_WRITE(GEN6_PMIER, 0);
+        I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
+}
+
 static unsigned long intel_pxfreq(u32 vidfreq)
 {
         unsigned long freq;
@@ -5752,7 +6189,123 @@ void intel_init_emon(struct drm_device *dev)
         dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
 }
 
-void intel_init_clock_gating(struct drm_device *dev)
+void gen6_enable_rps(struct drm_i915_private *dev_priv)
+{
+        u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+        u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
+        u32 pcu_mbox;
+        int cur_freq, min_freq, max_freq;
+        int i;
+
+        /* Here begins a magic sequence of register writes to enable
+         * auto-downclocking.
+         *
+         * Perhaps there might be some value in exposing these to
+         * userspace...
+         */
+        I915_WRITE(GEN6_RC_STATE, 0);
+        __gen6_force_wake_get(dev_priv);
+
+        /* disable the counters and set deterministic thresholds */
+        I915_WRITE(GEN6_RC_CONTROL, 0);
+
+        I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
+        I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
+        I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
+        I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
+        I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
+
+        for (i = 0; i < I915_NUM_RINGS; i++)
+                I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10);
+
+        I915_WRITE(GEN6_RC_SLEEP, 0);
+        I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
+        I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
+        I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
+        I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
+
+        I915_WRITE(GEN6_RC_CONTROL,
+                   GEN6_RC_CTL_RC6p_ENABLE |
+                   GEN6_RC_CTL_RC6_ENABLE |
+                   GEN6_RC_CTL_EI_MODE(1) |
+                   GEN6_RC_CTL_HW_ENABLE);
+
+        I915_WRITE(GEN6_RPNSWREQ,
+                   GEN6_FREQUENCY(10) |
+                   GEN6_OFFSET(0) |
+                   GEN6_AGGRESSIVE_TURBO);
+        I915_WRITE(GEN6_RC_VIDEO_FREQ,
+                   GEN6_FREQUENCY(12));
+
+        I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
+        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                   18 << 24 |
+                   6 << 16);
+        I915_WRITE(GEN6_RP_UP_THRESHOLD, 90000);
+        I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 100000);
+        I915_WRITE(GEN6_RP_UP_EI, 100000);
+        I915_WRITE(GEN6_RP_DOWN_EI, 300000);
+        I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
+        I915_WRITE(GEN6_RP_CONTROL,
+                   GEN6_RP_MEDIA_TURBO |
+                   GEN6_RP_USE_NORMAL_FREQ |
+                   GEN6_RP_MEDIA_IS_GFX |
+                   GEN6_RP_ENABLE |
+                   GEN6_RP_UP_BUSY_MAX |
+                   GEN6_RP_DOWN_BUSY_MIN);
+
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
+
+        I915_WRITE(GEN6_PCODE_DATA, 0);
+        I915_WRITE(GEN6_PCODE_MAILBOX,
+                   GEN6_PCODE_READY |
+                   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
+
+        min_freq = (rp_state_cap & 0xff0000) >> 16;
+        max_freq = rp_state_cap & 0xff;
+        cur_freq = (gt_perf_status & 0xff00) >> 8;
+
+        /* Check for overclock support */
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
+        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
+        pcu_mbox = I915_READ(GEN6_PCODE_DATA);
+        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+                     500))
+                DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
+        if (pcu_mbox & (1<<31)) { /* OC supported */
+                max_freq = pcu_mbox & 0xff;
+                DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 100);
+        }
+
+        /* In units of 100MHz */
+        dev_priv->max_delay = max_freq;
+        dev_priv->min_delay = min_freq;
+        dev_priv->cur_delay = cur_freq;
+
+        /* requires MSI enabled */
+        I915_WRITE(GEN6_PMIER,
+                   GEN6_PM_MBOX_EVENT |
+                   GEN6_PM_THERMAL_EVENT |
+                   GEN6_PM_RP_DOWN_TIMEOUT |
+                   GEN6_PM_RP_UP_THRESHOLD |
+                   GEN6_PM_RP_DOWN_THRESHOLD |
+                   GEN6_PM_RP_UP_EI_EXPIRED |
+                   GEN6_PM_RP_DOWN_EI_EXPIRED);
+        I915_WRITE(GEN6_PMIMR, 0);
+        /* enable all PM interrupts */
+        I915_WRITE(GEN6_PMINTRMSK, 0);
+
+        __gen6_force_wake_put(dev_priv);
+}
+
+void intel_enable_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -5765,13 +6318,17 @@ void intel_init_clock_gating(struct drm_device *dev)
 
                 if (IS_GEN5(dev)) {
                         /* Required for FBC */
-                        dspclk_gate |= DPFDUNIT_CLOCK_GATE_DISABLE;
+                        dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
+                                DPFCRUNIT_CLOCK_GATE_DISABLE |
+                                DPFDUNIT_CLOCK_GATE_DISABLE;
                         /* Required for CxSR */
                         dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
 
                         I915_WRITE(PCH_3DCGDIS0,
                                    MARIUNIT_CLOCK_GATE_DISABLE |
                                    SVSMUNIT_CLOCK_GATE_DISABLE);
+                        I915_WRITE(PCH_3DCGDIS1,
+                                   VFMUNIT_CLOCK_GATE_DISABLE);
                 }
 
                 I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
@@ -5800,9 +6357,9 @@ void intel_init_clock_gating(struct drm_device *dev)
                         I915_WRITE(DISP_ARB_CTL,
                                         (I915_READ(DISP_ARB_CTL) |
                                                 DISP_FBC_WM_DIS));
-                I915_WRITE(WM3_LP_ILK, 0);
-                I915_WRITE(WM2_LP_ILK, 0);
-                I915_WRITE(WM1_LP_ILK, 0);
+                        I915_WRITE(WM3_LP_ILK, 0);
+                        I915_WRITE(WM2_LP_ILK, 0);
+                        I915_WRITE(WM1_LP_ILK, 0);
                 }
                 /*
                  * Based on the document from hardware guys the following bits
@@ -5824,7 +6381,49 @@ void intel_init_clock_gating(struct drm_device *dev)
                                    ILK_DPFC_DIS2 |
                                    ILK_CLK_FBC);
                 }
-                return;
+
+                I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                           I915_READ(ILK_DISPLAY_CHICKEN2) |
+                           ILK_ELPIN_409_SELECT);
+
+                if (IS_GEN5(dev)) {
+                        I915_WRITE(_3D_CHICKEN2,
+                                   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
+                                   _3D_CHICKEN2_WM_READ_PIPELINED);
+                }
+
+                if (IS_GEN6(dev)) {
+                        I915_WRITE(WM3_LP_ILK, 0);
+                        I915_WRITE(WM2_LP_ILK, 0);
+                        I915_WRITE(WM1_LP_ILK, 0);
+
+                        /*
+                         * According to the spec the following bits should be
+                         * set in order to enable memory self-refresh and fbc:
+                         * The bit21 and bit22 of 0x42000
+                         * The bit21 and bit22 of 0x42004
+                         * The bit5 and bit7 of 0x42020
+                         * The bit14 of 0x70180
+                         * The bit14 of 0x71180
+                         */
+                        I915_WRITE(ILK_DISPLAY_CHICKEN1,
+                                   I915_READ(ILK_DISPLAY_CHICKEN1) |
+                                   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
+                        I915_WRITE(ILK_DISPLAY_CHICKEN2,
+                                   I915_READ(ILK_DISPLAY_CHICKEN2) |
+                                   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
+                        I915_WRITE(ILK_DSPCLK_GATE,
+                                   I915_READ(ILK_DSPCLK_GATE) |
+                                   ILK_DPARB_CLK_GATE  |
+                                   ILK_DPFD_CLK_GATE);
+
+                        I915_WRITE(DSPACNTR,
+                                   I915_READ(DSPACNTR) |
+                                   DISPPLANE_TRICKLE_FEED_DISABLE);
+                        I915_WRITE(DSPBCNTR,
+                                   I915_READ(DSPBCNTR) |
+                                   DISPPLANE_TRICKLE_FEED_DISABLE);
+                }
         } else if (IS_G4X(dev)) {
                 uint32_t dspclk_gate;
                 I915_WRITE(RENCLK_GATE_D1, 0);
@@ -5862,55 +6461,84 @@ void intel_init_clock_gating(struct drm_device *dev)
         } else if (IS_I830(dev)) {
                 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
         }
+}
 
-        /*
-         * GPU can automatically power down the render unit if given a page
-         * to save state.
-         */
-        if (IS_IRONLAKE_M(dev)) {
-                if (dev_priv->renderctx == NULL)
-                        dev_priv->renderctx = intel_alloc_context_page(dev);
-                if (dev_priv->renderctx) {
-                        struct drm_i915_gem_object *obj_priv;
-                        obj_priv = to_intel_bo(dev_priv->renderctx);
-                        if (obj_priv) {
-                                BEGIN_LP_RING(4);
-                                OUT_RING(MI_SET_CONTEXT);
-                                OUT_RING(obj_priv->gtt_offset |
-                                                MI_MM_SPACE_GTT |
-                                                MI_SAVE_EXT_STATE_EN |
-                                                MI_RESTORE_EXT_STATE_EN |
-                                                MI_RESTORE_INHIBIT);
-                                OUT_RING(MI_NOOP);
-                                OUT_RING(MI_FLUSH);
-                                ADVANCE_LP_RING();
-                        }
-                } else
-                        DRM_DEBUG_KMS("Failed to allocate render context."
-                                       "Disable RC6\n");
+void intel_disable_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->renderctx) {
+                struct drm_i915_gem_object *obj = dev_priv->renderctx;
+
+                I915_WRITE(CCID, 0);
+                POSTING_READ(CCID);
+
+                i915_gem_object_unpin(obj);
+                drm_gem_object_unreference(&obj->base);
+                dev_priv->renderctx = NULL;
         }
 
-        if (I915_HAS_RC6(dev) && drm_core_check_feature(dev, DRIVER_MODESET)) {
-                struct drm_i915_gem_object *obj_priv = NULL;
+        if (dev_priv->pwrctx) {
+                struct drm_i915_gem_object *obj = dev_priv->pwrctx;
 
-                if (dev_priv->pwrctx) {
-                        obj_priv = to_intel_bo(dev_priv->pwrctx);
-                } else {
-                        struct drm_gem_object *pwrctx;
+                I915_WRITE(PWRCTXA, 0);
+                POSTING_READ(PWRCTXA);
 
-                        pwrctx = intel_alloc_context_page(dev);
-                        if (pwrctx) {
-                                dev_priv->pwrctx = pwrctx;
-                                obj_priv = to_intel_bo(pwrctx);
-                        }
-                }
+                i915_gem_object_unpin(obj);
+                drm_gem_object_unreference(&obj->base);
+                dev_priv->pwrctx = NULL;
+        }
+}
 
-                if (obj_priv) {
-                        I915_WRITE(PWRCTXA, obj_priv->gtt_offset | PWRCTX_EN);
-                        I915_WRITE(MCHBAR_RENDER_STANDBY,
-                                   I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT);
-                }
+static void ironlake_disable_rc6(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
+        I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
+        wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
+                 10);
+        POSTING_READ(CCID);
+        I915_WRITE(PWRCTXA, 0);
+        POSTING_READ(PWRCTXA);
+        I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
+        POSTING_READ(RSTDBYCTL);
+        i915_gem_object_unpin(dev_priv->renderctx);
+        drm_gem_object_unreference(&dev_priv->renderctx->base);
+        dev_priv->renderctx = NULL;
+        i915_gem_object_unpin(dev_priv->pwrctx);
+        drm_gem_object_unreference(&dev_priv->pwrctx->base);
+        dev_priv->pwrctx = NULL;
+}
+
+void ironlake_enable_rc6(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        /*
+         * GPU can automatically power down the render unit if given a page
+         * to save state.
+         */
+        ret = BEGIN_LP_RING(6);
+        if (ret) {
+                ironlake_disable_rc6(dev);
+                return;
         }
+        OUT_RING(MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
+        OUT_RING(MI_SET_CONTEXT);
+        OUT_RING(dev_priv->renderctx->gtt_offset |
+                 MI_MM_SPACE_GTT |
+                 MI_SAVE_EXT_STATE_EN |
+                 MI_RESTORE_EXT_STATE_EN |
+                 MI_RESTORE_INHIBIT);
+        OUT_RING(MI_SUSPEND_FLUSH);
+        OUT_RING(MI_NOOP);
+        OUT_RING(MI_FLUSH);
+        ADVANCE_LP_RING();
+
+        I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
+        I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
 }
 
 /* Set up chip specific display functions */
@@ -5925,7 +6553,7 @@ static void intel_init_display(struct drm_device *dev)
                 dev_priv->display.dpms = i9xx_crtc_dpms;
 
         if (I915_HAS_FBC(dev)) {
-                if (IS_IRONLAKE_M(dev)) {
+                if (HAS_PCH_SPLIT(dev)) {
                         dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
                         dev_priv->display.enable_fbc = ironlake_enable_fbc;
                         dev_priv->display.disable_fbc = ironlake_disable_fbc;
@@ -5974,6 +6602,14 @@ static void intel_init_display(struct drm_device *dev)
                                               "Disable CxSR\n");
                                 dev_priv->display.update_wm = NULL;
                         }
+                } else if (IS_GEN6(dev)) {
+                        if (SNB_READ_WM0_LATENCY()) {
+                                dev_priv->display.update_wm = sandybridge_update_wm;
+                        } else {
+                                DRM_DEBUG_KMS("Failed to read display plane latency. "
+                                              "Disable CxSR\n");
+                                dev_priv->display.update_wm = NULL;
+                        }
                 } else
                         dev_priv->display.update_wm = NULL;
         } else if (IS_PINEVIEW(dev)) {
@@ -6119,12 +6755,7 @@ void intel_modeset_init(struct drm_device *dev)
                 dev->mode_config.max_width = 8192;
                 dev->mode_config.max_height = 8192;
         }
-
-        /* set memory base */
-        if (IS_GEN2(dev))
-                dev->mode_config.fb_base = pci_resource_start(dev->pdev, 0);
-        else
-                dev->mode_config.fb_base = pci_resource_start(dev->pdev, 2);
+        dev->mode_config.fb_base = dev->agp->base;
 
         if (IS_MOBILE(dev) || !IS_GEN2(dev))
                 dev_priv->num_pipe = 2;
@@ -6139,7 +6770,7 @@ void intel_modeset_init(struct drm_device *dev)
 
         intel_setup_outputs(dev);
 
-        intel_init_clock_gating(dev);
+        intel_enable_clock_gating(dev);
 
         /* Just disable it once at startup */
         i915_disable_vga(dev);
@@ -6149,6 +6780,24 @@ void intel_modeset_init(struct drm_device *dev)
                 intel_init_emon(dev);
         }
 
+        if (IS_GEN6(dev))
+                gen6_enable_rps(dev_priv);
+
+        if (IS_IRONLAKE_M(dev)) {
+                dev_priv->renderctx = intel_alloc_context_page(dev);
+                if (!dev_priv->renderctx)
+                        goto skip_rc6;
+                dev_priv->pwrctx = intel_alloc_context_page(dev);
+                if (!dev_priv->pwrctx) {
+                        i915_gem_object_unpin(dev_priv->renderctx);
+                        drm_gem_object_unreference(&dev_priv->renderctx->base);
+                        dev_priv->renderctx = NULL;
+                        goto skip_rc6;
+                }
+                ironlake_enable_rc6(dev);
+        }
+
+skip_rc6:
         INIT_WORK(&dev_priv->idle_work, intel_idle_update);
         setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
                     (unsigned long)dev);
@@ -6180,28 +6829,13 @@ void intel_modeset_cleanup(struct drm_device *dev)
         if (dev_priv->display.disable_fbc)
                 dev_priv->display.disable_fbc(dev);
 
-        if (dev_priv->renderctx) {
-                struct drm_i915_gem_object *obj_priv;
-
-                obj_priv = to_intel_bo(dev_priv->renderctx);
-                I915_WRITE(CCID, obj_priv->gtt_offset &~ CCID_EN);
-                I915_READ(CCID);
-                i915_gem_object_unpin(dev_priv->renderctx);
-                drm_gem_object_unreference(dev_priv->renderctx);
-        }
-
-        if (dev_priv->pwrctx) {
-                struct drm_i915_gem_object *obj_priv;
-
-                obj_priv = to_intel_bo(dev_priv->pwrctx);
-                I915_WRITE(PWRCTXA, obj_priv->gtt_offset &~ PWRCTX_EN);
-                I915_READ(PWRCTXA);
-                i915_gem_object_unpin(dev_priv->pwrctx);
-                drm_gem_object_unreference(dev_priv->pwrctx);
-        }
-
         if (IS_IRONLAKE_M(dev))
                 ironlake_disable_drps(dev);
+        if (IS_GEN6(dev))
+                gen6_disable_rps(dev);
+
+        if (IS_IRONLAKE_M(dev))
+                ironlake_disable_rc6(dev);
 
         mutex_unlock(&dev->struct_mutex);
 
@@ -6253,3 +6887,113 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
         pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl);
         return 0;
 }
+
+#ifdef CONFIG_DEBUG_FS
+#include <linux/seq_file.h>
+
+struct intel_display_error_state {
+        struct intel_cursor_error_state {
+                u32 control;
+                u32 position;
+                u32 base;
+                u32 size;
+        } cursor[2];
+
+        struct intel_pipe_error_state {
+                u32 conf;
+                u32 source;
+
+                u32 htotal;
+                u32 hblank;
+                u32 hsync;
+                u32 vtotal;
+                u32 vblank;
+                u32 vsync;
+        } pipe[2];
+
+        struct intel_plane_error_state {
+                u32 control;
+                u32 stride;
+                u32 size;
+                u32 pos;
+                u32 addr;
+                u32 surface;
+                u32 tile_offset;
+        } plane[2];
+};
+
+struct intel_display_error_state *
+intel_display_capture_error_state(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_display_error_state *error;
+        int i;
+
+        error = kmalloc(sizeof(*error), GFP_ATOMIC);
+        if (error == NULL)
+                return NULL;
+
+        for (i = 0; i < 2; i++) {
+                error->cursor[i].control = I915_READ(CURCNTR(i));
+                error->cursor[i].position = I915_READ(CURPOS(i));
+                error->cursor[i].base = I915_READ(CURBASE(i));
+
+                error->plane[i].control = I915_READ(DSPCNTR(i));
+                error->plane[i].stride = I915_READ(DSPSTRIDE(i));
+                error->plane[i].size = I915_READ(DSPSIZE(i));
+                error->plane[i].pos= I915_READ(DSPPOS(i));
+                error->plane[i].addr = I915_READ(DSPADDR(i));
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        error->plane[i].surface = I915_READ(DSPSURF(i));
+                        error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
+                }
+
+                error->pipe[i].conf = I915_READ(PIPECONF(i));
+                error->pipe[i].source = I915_READ(PIPESRC(i));
+                error->pipe[i].htotal = I915_READ(HTOTAL(i));
+                error->pipe[i].hblank = I915_READ(HBLANK(i));
+                error->pipe[i].hsync = I915_READ(HSYNC(i));
+                error->pipe[i].vtotal = I915_READ(VTOTAL(i));
+                error->pipe[i].vblank = I915_READ(VBLANK(i));
+                error->pipe[i].vsync = I915_READ(VSYNC(i));
+        }
+
+        return error;
+}
+
+void
+intel_display_print_error_state(struct seq_file *m,
+                                struct drm_device *dev,
+                                struct intel_display_error_state *error)
+{
+        int i;
+
+        for (i = 0; i < 2; i++) {
+                seq_printf(m, "Pipe [%d]:\n", i);
+                seq_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
+                seq_printf(m, "  SRC: %08x\n", error->pipe[i].source);
+                seq_printf(m, "  HTOTAL: %08x\n", error->pipe[i].htotal);
+                seq_printf(m, "  HBLANK: %08x\n", error->pipe[i].hblank);
+                seq_printf(m, "  HSYNC: %08x\n", error->pipe[i].hsync);
+                seq_printf(m, "  VTOTAL: %08x\n", error->pipe[i].vtotal);
+                seq_printf(m, "  VBLANK: %08x\n", error->pipe[i].vblank);
+                seq_printf(m, "  VSYNC: %08x\n", error->pipe[i].vsync);
+
+                seq_printf(m, "Plane [%d]:\n", i);
+                seq_printf(m, "  CNTR: %08x\n", error->plane[i].control);
+                seq_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
+                seq_printf(m, "  SIZE: %08x\n", error->plane[i].size);
+                seq_printf(m, "  POS: %08x\n", error->plane[i].pos);
+                seq_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        seq_printf(m, "  SURF: %08x\n", error->plane[i].surface);
+                        seq_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
+                }
+
+                seq_printf(m, "Cursor [%d]:\n", i);
+                seq_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
+                seq_printf(m, "  POS: %08x\n", error->cursor[i].position);
+                seq_printf(m, "  BASE: %08x\n", error->cursor[i].base);
+        }
+}
+#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index c8e005553310..1f4242b682c8 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -479,6 +479,7 @@ intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
         uint16_t address = algo_data->address;
         uint8_t msg[5];
         uint8_t reply[2];
+        unsigned retry;
         int msg_bytes;
         int reply_bytes;
         int ret;
@@ -513,14 +514,33 @@ intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
                 break;
         }
 
-        for (;;) {
-          ret = intel_dp_aux_ch(intel_dp,
-                                msg, msg_bytes,
-                                reply, reply_bytes);
+        for (retry = 0; retry < 5; retry++) {
+                ret = intel_dp_aux_ch(intel_dp,
+                                      msg, msg_bytes,
+                                      reply, reply_bytes);
                 if (ret < 0) {
                         DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
                         return ret;
                 }
+
+                switch (reply[0] & AUX_NATIVE_REPLY_MASK) {
+                case AUX_NATIVE_REPLY_ACK:
+                        /* I2C-over-AUX Reply field is only valid
+                         * when paired with AUX ACK.
+                         */
+                        break;
+                case AUX_NATIVE_REPLY_NACK:
+                        DRM_DEBUG_KMS("aux_ch native nack\n");
+                        return -EREMOTEIO;
+                case AUX_NATIVE_REPLY_DEFER:
+                        udelay(100);
+                        continue;
+                default:
+                        DRM_ERROR("aux_ch invalid native reply 0x%02x\n",
+                                  reply[0]);
+                        return -EREMOTEIO;
+                }
+
                 switch (reply[0] & AUX_I2C_REPLY_MASK) {
                 case AUX_I2C_REPLY_ACK:
                         if (mode == MODE_I2C_READ) {
@@ -528,17 +548,20 @@ intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
                         }
                         return reply_bytes - 1;
                 case AUX_I2C_REPLY_NACK:
-                        DRM_DEBUG_KMS("aux_ch nack\n");
+                        DRM_DEBUG_KMS("aux_i2c nack\n");
                         return -EREMOTEIO;
                 case AUX_I2C_REPLY_DEFER:
-                        DRM_DEBUG_KMS("aux_ch defer\n");
+                        DRM_DEBUG_KMS("aux_i2c defer\n");
                         udelay(100);
                         break;
                 default:
-                        DRM_ERROR("aux_ch invalid reply 0x%02x\n", reply[0]);
+                        DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]);
                         return -EREMOTEIO;
                 }
         }
+
+        DRM_ERROR("too many retries, giving up\n");
+        return -EREMOTEIO;
 }
 
 static int
@@ -584,17 +607,6 @@ intel_dp_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 mode->clock = dev_priv->panel_fixed_mode->clock;
         }
 
-        /* Just use VBT values for eDP */
-        if (is_edp(intel_dp)) {
-                intel_dp->lane_count = dev_priv->edp.lanes;
-                intel_dp->link_bw = dev_priv->edp.rate;
-                adjusted_mode->clock = intel_dp_link_clock(intel_dp->link_bw);
-                DRM_DEBUG_KMS("eDP link bw %02x lane count %d clock %d\n",
-                              intel_dp->link_bw, intel_dp->lane_count,
-                              adjusted_mode->clock);
-                return true;
-        }
-
         for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) {
                 for (clock = 0; clock <= max_clock; clock++) {
                         int link_avail = intel_dp_max_data_rate(intel_dp_link_clock(bws[clock]), lane_count);
@@ -613,6 +625,19 @@ intel_dp_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 }
         }
 
+        if (is_edp(intel_dp)) {
+                /* okay we failed just pick the highest */
+                intel_dp->lane_count = max_lane_count;
+                intel_dp->link_bw = bws[max_clock];
+                adjusted_mode->clock = intel_dp_link_clock(intel_dp->link_bw);
+                DRM_DEBUG_KMS("Force picking display port link bw %02x lane "
+                              "count %d clock %d\n",
+                              intel_dp->link_bw, intel_dp->lane_count,
+                              adjusted_mode->clock);
+
+                return true;
+        }
+
         return false;
 }
 
@@ -1087,21 +1112,11 @@ intel_get_adjust_train(struct intel_dp *intel_dp)
 }
 
 static uint32_t
-intel_dp_signal_levels(struct intel_dp *intel_dp)
+intel_dp_signal_levels(uint8_t train_set, int lane_count)
 {
-        struct drm_device *dev = intel_dp->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t signal_levels = 0;
-        u8 train_set = intel_dp->train_set[0];
-        u32 vswing = train_set & DP_TRAIN_VOLTAGE_SWING_MASK;
-        u32 preemphasis = train_set & DP_TRAIN_PRE_EMPHASIS_MASK;
+        uint32_t        signal_levels = 0;
 
-        if (is_edp(intel_dp)) {
-                vswing = dev_priv->edp.vswing;
-                preemphasis = dev_priv->edp.preemphasis;
-        }
-
-        switch (vswing) {
+        switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
         case DP_TRAIN_VOLTAGE_SWING_400:
         default:
                 signal_levels |= DP_VOLTAGE_0_4;
@@ -1116,7 +1131,7 @@ intel_dp_signal_levels(struct intel_dp *intel_dp)
                 signal_levels |= DP_VOLTAGE_1_2;
                 break;
         }
-        switch (preemphasis) {
+        switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
         case DP_TRAIN_PRE_EMPHASIS_0:
         default:
                 signal_levels |= DP_PRE_EMPHASIS_0;
@@ -1138,18 +1153,27 @@ intel_dp_signal_levels(struct intel_dp *intel_dp)
 static uint32_t
 intel_gen6_edp_signal_levels(uint8_t train_set)
 {
-        switch (train_set & (DP_TRAIN_VOLTAGE_SWING_MASK|DP_TRAIN_PRE_EMPHASIS_MASK)) {
+        int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK |
+                                         DP_TRAIN_PRE_EMPHASIS_MASK);
+        switch (signal_levels) {
         case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
-                return EDP_LINK_TRAIN_400MV_0DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
+                return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return EDP_LINK_TRAIN_400MV_3_5DB_SNB_B;
         case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
-                return EDP_LINK_TRAIN_400MV_6DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6:
+                return EDP_LINK_TRAIN_400_600MV_6DB_SNB_B;
         case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
-                return EDP_LINK_TRAIN_600MV_3_5DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B;
         case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
-                return EDP_LINK_TRAIN_800MV_0DB_SNB_B;
+        case DP_TRAIN_VOLTAGE_SWING_1200 | DP_TRAIN_PRE_EMPHASIS_0:
+                return EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B;
         default:
-                DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level\n");
-                return EDP_LINK_TRAIN_400MV_0DB_SNB_B;
+                DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:"
+                              "0x%x\n", signal_levels);
+                return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B;
         }
 }
 
@@ -1203,18 +1227,6 @@ intel_channel_eq_ok(struct intel_dp *intel_dp)
 }
 
 static bool
-intel_dp_aux_handshake_required(struct intel_dp *intel_dp)
-{
-        struct drm_device *dev = intel_dp->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (is_edp(intel_dp) && dev_priv->no_aux_handshake)
-                return false;
-
-        return true;
-}
-
-static bool
 intel_dp_set_link_train(struct intel_dp *intel_dp,
                         uint32_t dp_reg_value,
                         uint8_t dp_train_pat)
@@ -1226,9 +1238,6 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
         I915_WRITE(intel_dp->output_reg, dp_reg_value);
         POSTING_READ(intel_dp->output_reg);
 
-        if (!intel_dp_aux_handshake_required(intel_dp))
-                return true;
-
         intel_dp_aux_native_write_1(intel_dp,
                                     DP_TRAINING_PATTERN_SET,
                                     dp_train_pat);
@@ -1261,11 +1270,10 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
         POSTING_READ(intel_dp->output_reg);
         intel_wait_for_vblank(dev, intel_crtc->pipe);
 
-        if (intel_dp_aux_handshake_required(intel_dp))
-                /* Write the link configuration data */
-                intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET,
-                                          intel_dp->link_configuration,
-                                          DP_LINK_CONFIGURATION_SIZE);
+        /* Write the link configuration data */
+        intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET,
+                                  intel_dp->link_configuration,
+                                  DP_LINK_CONFIGURATION_SIZE);
 
         DP |= DP_PORT_EN;
         if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
@@ -1283,7 +1291,7 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
                         signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]);
                         DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels;
                 } else {
-                        signal_levels = intel_dp_signal_levels(intel_dp);
+                        signal_levels = intel_dp_signal_levels(intel_dp->train_set[0], intel_dp->lane_count);
                         DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels;
                 }
 
@@ -1297,37 +1305,33 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
                         break;
                 /* Set training pattern 1 */
 
-                udelay(500);
-                if (intel_dp_aux_handshake_required(intel_dp)) {
+                udelay(100);
+                if (!intel_dp_get_link_status(intel_dp))
                         break;
-                } else {
-                        if (!intel_dp_get_link_status(intel_dp))
-                                break;
 
-                        if (intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
-                                clock_recovery = true;
-                                break;
-                        }
+                if (intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
+                        clock_recovery = true;
+                        break;
+                }
 
-                        /* Check to see if we've tried the max voltage */
-                        for (i = 0; i < intel_dp->lane_count; i++)
-                                if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
-                                        break;
-                        if (i == intel_dp->lane_count)
+                /* Check to see if we've tried the max voltage */
+                for (i = 0; i < intel_dp->lane_count; i++)
+                        if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
                                 break;
+                if (i == intel_dp->lane_count)
+                        break;
 
-                        /* Check to see if we've tried the same voltage 5 times */
-                        if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
-                                ++tries;
-                                if (tries == 5)
-                                        break;
-                        } else
-                                tries = 0;
-                        voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
+                /* Check to see if we've tried the same voltage 5 times */
+                if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
+                        ++tries;
+                        if (tries == 5)
+                                break;
+                } else
+                        tries = 0;
+                voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
 
-                        /* Compute new intel_dp->train_set as requested by target */
-                        intel_get_adjust_train(intel_dp);
-                }
+                /* Compute new intel_dp->train_set as requested by target */
+                intel_get_adjust_train(intel_dp);
         }
 
         intel_dp->DP = DP;
@@ -1339,22 +1343,29 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
         struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         bool channel_eq = false;
-        int tries;
+        int tries, cr_tries;
         u32 reg;
         uint32_t DP = intel_dp->DP;
 
         /* channel equalization */
         tries = 0;
+        cr_tries = 0;
         channel_eq = false;
         for (;;) {
                 /* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */
                 uint32_t    signal_levels;
 
+                if (cr_tries > 5) {
+                        DRM_ERROR("failed to train DP, aborting\n");
+                        intel_dp_link_down(intel_dp);
+                        break;
+                }
+
                 if (IS_GEN6(dev) && is_edp(intel_dp)) {
                         signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]);
                         DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels;
                 } else {
-                        signal_levels = intel_dp_signal_levels(intel_dp);
+                        signal_levels = intel_dp_signal_levels(intel_dp->train_set[0], intel_dp->lane_count);
                         DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels;
                 }
 
@@ -1368,28 +1379,36 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
                                              DP_TRAINING_PATTERN_2))
                         break;
 
-                udelay(500);
-
-                if (!intel_dp_aux_handshake_required(intel_dp)) {
+                udelay(400);
+                if (!intel_dp_get_link_status(intel_dp))
                         break;
-                } else {
-                        if (!intel_dp_get_link_status(intel_dp))
-                                break;
 
-                        if (intel_channel_eq_ok(intel_dp)) {
-                                channel_eq = true;
-                                break;
-                        }
+                /* Make sure clock is still ok */
+                if (!intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
+                        intel_dp_start_link_train(intel_dp);
+                        cr_tries++;
+                        continue;
+                }
 
-                        /* Try 5 times */
-                        if (tries > 5)
-                                break;
+                if (intel_channel_eq_ok(intel_dp)) {
+                        channel_eq = true;
+                        break;
+                }
 
-                        /* Compute new intel_dp->train_set as requested by target */
-                        intel_get_adjust_train(intel_dp);
-                        ++tries;
+                /* Try 5 times, then try clock recovery if that fails */
+                if (tries > 5) {
+                        intel_dp_link_down(intel_dp);
+                        intel_dp_start_link_train(intel_dp);
+                        tries = 0;
+                        cr_tries++;
+                        continue;
                 }
+
+                /* Compute new intel_dp->train_set as requested by target */
+                intel_get_adjust_train(intel_dp);
+                ++tries;
         }
+
         if (HAS_PCH_CPT(dev) && !is_edp(intel_dp))
                 reg = DP | DP_LINK_TRAIN_OFF_CPT;
         else
@@ -1408,6 +1427,9 @@ intel_dp_link_down(struct intel_dp *intel_dp)
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t DP = intel_dp->DP;
 
+        if ((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)
+                return;
+
         DRM_DEBUG_KMS("\n");
 
         if (is_edp(intel_dp)) {
@@ -1430,6 +1452,27 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 
         if (is_edp(intel_dp))
                 DP |= DP_LINK_TRAIN_OFF;
+
+        if (!HAS_PCH_CPT(dev) &&
+            I915_READ(intel_dp->output_reg) & DP_PIPEB_SELECT) {
+                struct intel_crtc *intel_crtc = to_intel_crtc(intel_dp->base.base.crtc);
+                /* Hardware workaround: leaving our transcoder select
+                 * set to transcoder B while it's off will prevent the
+                 * corresponding HDMI output on transcoder A.
+                 *
+                 * Combine this with another hardware workaround:
+                 * transcoder select bit can only be cleared while the
+                 * port is enabled.
+                 */
+                DP &= ~DP_PIPEB_SELECT;
+                I915_WRITE(intel_dp->output_reg, DP);
+
+                /* Changes to enable or select take place the vblank
+                 * after being written.
+                 */
+                intel_wait_for_vblank(dev, intel_crtc->pipe);
+        }
+
         I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN);
         POSTING_READ(intel_dp->output_reg);
 }
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 21551fe74541..74db2557d644 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -127,7 +127,7 @@ intel_mode_get_pixel_multiplier(const struct drm_display_mode *mode)
 
 struct intel_framebuffer {
         struct drm_framebuffer base;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
 };
 
 struct intel_fbdev {
@@ -166,7 +166,7 @@ struct intel_crtc {
         struct intel_unpin_work *unpin_work;
         int fdi_lanes;
 
-        struct drm_gem_object *cursor_bo;
+        struct drm_i915_gem_object *cursor_bo;
         uint32_t cursor_addr;
         int16_t cursor_x, cursor_y;
         int16_t cursor_width, cursor_height;
@@ -220,8 +220,8 @@ intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
 struct intel_unpin_work {
         struct work_struct work;
         struct drm_device *dev;
-        struct drm_gem_object *old_fb_obj;
-        struct drm_gem_object *pending_flip_obj;
+        struct drm_i915_gem_object *old_fb_obj;
+        struct drm_i915_gem_object *pending_flip_obj;
         struct drm_pending_vblank_event *event;
         int pending;
         bool enable_stall_check;
@@ -236,8 +236,9 @@ void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
 extern bool intel_sdvo_init(struct drm_device *dev, int output_device);
 extern void intel_dvo_init(struct drm_device *dev);
 extern void intel_tv_init(struct drm_device *dev);
-extern void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj);
-extern void intel_lvds_init(struct drm_device *dev);
+extern void intel_mark_busy(struct drm_device *dev,
+                            struct drm_i915_gem_object *obj);
+extern bool intel_lvds_init(struct drm_device *dev);
 extern void intel_dp_init(struct drm_device *dev, int dp_reg);
 void
 intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
@@ -256,6 +257,9 @@ extern void intel_pch_panel_fitting(struct drm_device *dev,
 extern u32 intel_panel_get_max_backlight(struct drm_device *dev);
 extern u32 intel_panel_get_backlight(struct drm_device *dev);
 extern void intel_panel_set_backlight(struct drm_device *dev, u32 level);
+extern void intel_panel_setup_backlight(struct drm_device *dev);
+extern void intel_panel_enable_backlight(struct drm_device *dev);
+extern void intel_panel_disable_backlight(struct drm_device *dev);
 
 extern void intel_crtc_load_lut(struct drm_crtc *crtc);
 extern void intel_encoder_prepare (struct drm_encoder *encoder);
@@ -293,19 +297,22 @@ extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
                                     u16 blue, int regno);
 extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
                                     u16 *blue, int regno);
-extern void intel_init_clock_gating(struct drm_device *dev);
+extern void intel_enable_clock_gating(struct drm_device *dev);
+extern void intel_disable_clock_gating(struct drm_device *dev);
 extern void ironlake_enable_drps(struct drm_device *dev);
 extern void ironlake_disable_drps(struct drm_device *dev);
+extern void gen6_enable_rps(struct drm_i915_private *dev_priv);
+extern void gen6_disable_rps(struct drm_device *dev);
 extern void intel_init_emon(struct drm_device *dev);
 
 extern int intel_pin_and_fence_fb_obj(struct drm_device *dev,
-                                      struct drm_gem_object *obj,
-                                      bool pipelined);
+                                      struct drm_i915_gem_object *obj,
+                                      struct intel_ring_buffer *pipelined);
 
 extern int intel_framebuffer_init(struct drm_device *dev,
                                   struct intel_framebuffer *ifb,
                                   struct drm_mode_fb_cmd *mode_cmd,
-                                  struct drm_gem_object *obj);
+                                  struct drm_i915_gem_object *obj);
 extern int intel_fbdev_init(struct drm_device *dev);
 extern void intel_fbdev_fini(struct drm_device *dev);
 
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index af2a1dddc28e..512782728e51 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -62,13 +62,13 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
                           struct drm_fb_helper_surface_size *sizes)
 {
         struct drm_device *dev = ifbdev->helper.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct fb_info *info;
         struct drm_framebuffer *fb;
         struct drm_mode_fb_cmd mode_cmd;
-        struct drm_gem_object *fbo = NULL;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         struct device *device = &dev->pdev->dev;
-        int size, ret, mmio_bar = IS_GEN2(dev) ? 1 : 0;
+        int size, ret;
 
         /* we don't do packed 24bpp */
         if (sizes->surface_bpp == 24)
@@ -78,23 +78,22 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         mode_cmd.height = sizes->surface_height;
 
         mode_cmd.bpp = sizes->surface_bpp;
-        mode_cmd.pitch = ALIGN(mode_cmd.width * ((mode_cmd.bpp + 1) / 8), 64);
+        mode_cmd.pitch = ALIGN(mode_cmd.width * ((mode_cmd.bpp + 7) / 8), 64);
         mode_cmd.depth = sizes->surface_depth;
 
         size = mode_cmd.pitch * mode_cmd.height;
         size = ALIGN(size, PAGE_SIZE);
-        fbo = i915_gem_alloc_object(dev, size);
-        if (!fbo) {
+        obj = i915_gem_alloc_object(dev, size);
+        if (!obj) {
                 DRM_ERROR("failed to allocate framebuffer\n");
                 ret = -ENOMEM;
                 goto out;
         }
-        obj_priv = to_intel_bo(fbo);
 
         mutex_lock(&dev->struct_mutex);
 
         /* Flush everything out, we'll be doing GTT only from now on */
-        ret = intel_pin_and_fence_fb_obj(dev, fbo, false);
+        ret = intel_pin_and_fence_fb_obj(dev, obj, false);
         if (ret) {
                 DRM_ERROR("failed to pin fb: %d\n", ret);
                 goto out_unref;
@@ -108,7 +107,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
 
         info->par = ifbdev;
 
-        ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, fbo);
+        ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj);
         if (ret)
                 goto out_unpin;
 
@@ -122,6 +121,11 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
         info->fbops = &intelfb_ops;
 
+        ret = fb_alloc_cmap(&info->cmap, 256, 0);
+        if (ret) {
+                ret = -ENOMEM;
+                goto out_unpin;
+        }
         /* setup aperture base/size for vesafb takeover */
         info->apertures = alloc_apertures(1);
         if (!info->apertures) {
@@ -129,26 +133,17 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
                 goto out_unpin;
         }
         info->apertures->ranges[0].base = dev->mode_config.fb_base;
-        if (!IS_GEN2(dev))
-                info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 2);
-        else
-                info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0);
+        info->apertures->ranges[0].size =
+                dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
 
-        info->fix.smem_start = dev->mode_config.fb_base + obj_priv->gtt_offset;
+        info->fix.smem_start = dev->mode_config.fb_base + obj->gtt_offset;
         info->fix.smem_len = size;
 
-        info->screen_base = ioremap_wc(dev->agp->base + obj_priv->gtt_offset,
-                                       size);
+        info->screen_base = ioremap_wc(dev->agp->base + obj->gtt_offset, size);
         if (!info->screen_base) {
                 ret = -ENOSPC;
                 goto out_unpin;
         }
-
-        ret = fb_alloc_cmap(&info->cmap, 256, 0);
-        if (ret) {
-                ret = -ENOMEM;
-                goto out_unpin;
-        }
         info->screen_size = size;
 
 //      memset(info->screen_base, 0, size);
@@ -156,10 +151,6 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
         drm_fb_helper_fill_var(info, &ifbdev->helper, sizes->fb_width, sizes->fb_height);
 
-        /* FIXME: we really shouldn't expose mmio space at all */
-        info->fix.mmio_start = pci_resource_start(dev->pdev, mmio_bar);
-        info->fix.mmio_len = pci_resource_len(dev->pdev, mmio_bar);
-
         info->pixmap.size = 64*1024;
         info->pixmap.buf_align = 8;
         info->pixmap.access_align = 32;
@@ -168,7 +159,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
 
         DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n",
                       fb->width, fb->height,
-                      obj_priv->gtt_offset, fbo);
+                      obj->gtt_offset, obj);
 
 
         mutex_unlock(&dev->struct_mutex);
@@ -176,9 +167,9 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
         return 0;
 
 out_unpin:
-        i915_gem_object_unpin(fbo);
+        i915_gem_object_unpin(obj);
 out_unref:
-        drm_gem_object_unreference(fbo);
+        drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
 out:
         return ret;
@@ -225,7 +216,7 @@ static void intel_fbdev_destroy(struct drm_device *dev,
 
         drm_framebuffer_cleanup(&ifb->base);
         if (ifb->obj) {
-                drm_gem_object_unreference_unlocked(ifb->obj);
+                drm_gem_object_unreference_unlocked(&ifb->obj->base);
                 ifb->obj = NULL;
         }
 }
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 3dba086e7eea..58040f68ed7a 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -85,8 +85,9 @@ static u32 get_reserved(struct intel_gpio *gpio)
 
         /* On most chips, these bits must be preserved in software. */
         if (!IS_I830(dev) && !IS_845G(dev))
-                reserved = I915_READ(gpio->reg) & (GPIO_DATA_PULLUP_DISABLE |
-                                                   GPIO_CLOCK_PULLUP_DISABLE);
+                reserved = I915_READ_NOTRACE(gpio->reg) &
+                                             (GPIO_DATA_PULLUP_DISABLE |
+                                              GPIO_CLOCK_PULLUP_DISABLE);
 
         return reserved;
 }
@@ -96,9 +97,9 @@ static int get_clock(void *data)
         struct intel_gpio *gpio = data;
         struct drm_i915_private *dev_priv = gpio->dev_priv;
         u32 reserved = get_reserved(gpio);
-        I915_WRITE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
-        I915_WRITE(gpio->reg, reserved);
-        return (I915_READ(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
+        I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
+        I915_WRITE_NOTRACE(gpio->reg, reserved);
+        return (I915_READ_NOTRACE(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
 }
 
 static int get_data(void *data)
@@ -106,9 +107,9 @@ static int get_data(void *data)
         struct intel_gpio *gpio = data;
         struct drm_i915_private *dev_priv = gpio->dev_priv;
         u32 reserved = get_reserved(gpio);
-        I915_WRITE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
-        I915_WRITE(gpio->reg, reserved);
-        return (I915_READ(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
+        I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
+        I915_WRITE_NOTRACE(gpio->reg, reserved);
+        return (I915_READ_NOTRACE(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
 }
 
 static void set_clock(void *data, int state_high)
@@ -124,7 +125,7 @@ static void set_clock(void *data, int state_high)
                 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
                         GPIO_CLOCK_VAL_MASK;
 
-        I915_WRITE(gpio->reg, reserved | clock_bits);
+        I915_WRITE_NOTRACE(gpio->reg, reserved | clock_bits);
         POSTING_READ(gpio->reg);
 }
 
@@ -141,7 +142,7 @@ static void set_data(void *data, int state_high)
                 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
                         GPIO_DATA_VAL_MASK;
 
-        I915_WRITE(gpio->reg, reserved | data_bits);
+        I915_WRITE_NOTRACE(gpio->reg, reserved | data_bits);
         POSTING_READ(gpio->reg);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 4324a326f98e..ace8d5d30dd2 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -68,7 +68,7 @@ static struct intel_lvds *intel_attached_lvds(struct drm_connector *connector)
 /**
  * Sets the power state for the panel.
  */
-static void intel_lvds_set_power(struct intel_lvds *intel_lvds, bool on)
+static void intel_lvds_enable(struct intel_lvds *intel_lvds)
 {
         struct drm_device *dev = intel_lvds->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -82,26 +82,60 @@ static void intel_lvds_set_power(struct intel_lvds *intel_lvds, bool on)
                 lvds_reg = LVDS;
         }
 
-        if (on) {
-                I915_WRITE(lvds_reg, I915_READ(lvds_reg) | LVDS_PORT_EN);
-                I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON);
-                intel_panel_set_backlight(dev, dev_priv->backlight_level);
-        } else {
-                dev_priv->backlight_level = intel_panel_get_backlight(dev);
-
-                intel_panel_set_backlight(dev, 0);
-                I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
+        I915_WRITE(lvds_reg, I915_READ(lvds_reg) | LVDS_PORT_EN);
 
-                if (intel_lvds->pfit_control) {
-                        if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000))
-                                DRM_ERROR("timed out waiting for panel to power off\n");
-                        I915_WRITE(PFIT_CONTROL, 0);
-                        intel_lvds->pfit_control = 0;
+        if (intel_lvds->pfit_dirty) {
+                /*
+                 * Enable automatic panel scaling so that non-native modes
+                 * fill the screen.  The panel fitter should only be
+                 * adjusted whilst the pipe is disabled, according to
+                 * register description and PRM.
+                 */
+                DRM_DEBUG_KMS("applying panel-fitter: %x, %x\n",
+                              intel_lvds->pfit_control,
+                              intel_lvds->pfit_pgm_ratios);
+                if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000)) {
+                        DRM_ERROR("timed out waiting for panel to power off\n");
+                } else {
+                        I915_WRITE(PFIT_PGM_RATIOS, intel_lvds->pfit_pgm_ratios);
+                        I915_WRITE(PFIT_CONTROL, intel_lvds->pfit_control);
                         intel_lvds->pfit_dirty = false;
                 }
+        }
+
+        I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON);
+        POSTING_READ(lvds_reg);
+
+        intel_panel_enable_backlight(dev);
+}
 
-                I915_WRITE(lvds_reg, I915_READ(lvds_reg) & ~LVDS_PORT_EN);
+static void intel_lvds_disable(struct intel_lvds *intel_lvds)
+{
+        struct drm_device *dev = intel_lvds->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 ctl_reg, lvds_reg;
+
+        if (HAS_PCH_SPLIT(dev)) {
+                ctl_reg = PCH_PP_CONTROL;
+                lvds_reg = PCH_LVDS;
+        } else {
+                ctl_reg = PP_CONTROL;
+                lvds_reg = LVDS;
+        }
+
+        intel_panel_disable_backlight(dev);
+
+        I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
+
+        if (intel_lvds->pfit_control) {
+                if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000))
+                        DRM_ERROR("timed out waiting for panel to power off\n");
+
+                I915_WRITE(PFIT_CONTROL, 0);
+                intel_lvds->pfit_dirty = true;
         }
+
+        I915_WRITE(lvds_reg, I915_READ(lvds_reg) & ~LVDS_PORT_EN);
         POSTING_READ(lvds_reg);
 }
 
@@ -110,9 +144,9 @@ static void intel_lvds_dpms(struct drm_encoder *encoder, int mode)
         struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
         if (mode == DRM_MODE_DPMS_ON)
-                intel_lvds_set_power(intel_lvds, true);
+                intel_lvds_enable(intel_lvds);
         else
-                intel_lvds_set_power(intel_lvds, false);
+                intel_lvds_disable(intel_lvds);
 
         /* XXX: We never power down the LVDS pairs. */
 }
@@ -269,14 +303,13 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
                         u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
                         u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
 
-                        pfit_control |= PFIT_ENABLE;
                         /* 965+ is easy, it does everything in hw */
                         if (scaled_width > scaled_height)
-                                pfit_control |= PFIT_SCALING_PILLAR;
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_PILLAR;
                         else if (scaled_width < scaled_height)
-                                pfit_control |= PFIT_SCALING_LETTER;
-                        else
-                                pfit_control |= PFIT_SCALING_AUTO;
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER;
+                        else if (adjusted_mode->hdisplay != mode->hdisplay)
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
                 } else {
                         u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
                         u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
@@ -323,13 +356,17 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
                  * Full scaling, even if it changes the aspect ratio.
                  * Fortunately this is all done for us in hw.
                  */
-                pfit_control |= PFIT_ENABLE;
-                if (INTEL_INFO(dev)->gen >= 4)
-                        pfit_control |= PFIT_SCALING_AUTO;
-                else
-                        pfit_control |= (VERT_AUTO_SCALE | HORIZ_AUTO_SCALE |
-                                         VERT_INTERP_BILINEAR |
-                                         HORIZ_INTERP_BILINEAR);
+                if (mode->vdisplay != adjusted_mode->vdisplay ||
+                    mode->hdisplay != adjusted_mode->hdisplay) {
+                        pfit_control |= PFIT_ENABLE;
+                        if (INTEL_INFO(dev)->gen >= 4)
+                                pfit_control |= PFIT_SCALING_AUTO;
+                        else
+                                pfit_control |= (VERT_AUTO_SCALE |
+                                                 VERT_INTERP_BILINEAR |
+                                                 HORIZ_AUTO_SCALE |
+                                                 HORIZ_INTERP_BILINEAR);
+                }
                 break;
 
         default:
@@ -337,6 +374,10 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
         }
 
 out:
+        if ((pfit_control & PFIT_ENABLE) == 0) {
+                pfit_control = 0;
+                pfit_pgm_ratios = 0;
+        }
         if (pfit_control != intel_lvds->pfit_control ||
             pfit_pgm_ratios != intel_lvds->pfit_pgm_ratios) {
                 intel_lvds->pfit_control = pfit_control;
@@ -360,8 +401,6 @@ static void intel_lvds_prepare(struct drm_encoder *encoder)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
-        dev_priv->backlight_level = intel_panel_get_backlight(dev);
-
         /* We try to do the minimum that is necessary in order to unlock
          * the registers for mode setting.
          *
@@ -392,9 +431,6 @@ static void intel_lvds_commit(struct drm_encoder *encoder)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
-        if (dev_priv->backlight_level == 0)
-                dev_priv->backlight_level = intel_panel_get_max_backlight(dev);
-
         /* Undo any unlocking done in prepare to prevent accidental
          * adjustment of the registers.
          */
@@ -411,43 +447,18 @@ static void intel_lvds_commit(struct drm_encoder *encoder)
         /* Always do a full power on as we do not know what state
          * we were left in.
          */
-        intel_lvds_set_power(intel_lvds, true);
+        intel_lvds_enable(intel_lvds);
 }
 
 static void intel_lvds_mode_set(struct drm_encoder *encoder,
                                 struct drm_display_mode *mode,
                                 struct drm_display_mode *adjusted_mode)
 {
-        struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
-
         /*
          * The LVDS pin pair will already have been turned on in the
          * intel_crtc_mode_set since it has a large impact on the DPLL
          * settings.
          */
-
-        if (HAS_PCH_SPLIT(dev))
-                return;
-
-        if (!intel_lvds->pfit_dirty)
-                return;
-
-        /*
-         * Enable automatic panel scaling so that non-native modes fill the
-         * screen.  Should be enabled before the pipe is enabled, according to
-         * register description and PRM.
-         */
-        DRM_DEBUG_KMS("applying panel-fitter: %x, %x\n",
-                      intel_lvds->pfit_control,
-                      intel_lvds->pfit_pgm_ratios);
-        if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000))
-                DRM_ERROR("timed out waiting for panel to power off\n");
-
-        I915_WRITE(PFIT_PGM_RATIOS, intel_lvds->pfit_pgm_ratios);
-        I915_WRITE(PFIT_CONTROL, intel_lvds->pfit_control);
-        intel_lvds->pfit_dirty = false;
 }
 
 /**
@@ -693,6 +704,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
         },
         {
                 .callback = intel_no_lvds_dmi_callback,
+                .ident = "AOpen i915GMm-HFS",
+                .matches = {
+                        DMI_MATCH(DMI_BOARD_VENDOR, "AOpen"),
+                        DMI_MATCH(DMI_BOARD_NAME, "i915GMm-HFS"),
+                },
+        },
+        {
+                .callback = intel_no_lvds_dmi_callback,
                 .ident = "Aopen i945GTt-VFA",
                 .matches = {
                         DMI_MATCH(DMI_PRODUCT_VERSION, "AO00001JW"),
@@ -837,7 +856,7 @@ static bool intel_lvds_ddc_probe(struct drm_device *dev, u8 pin)
  * Create the connector, register the LVDS DDC bus, and try to figure out what
  * modes we can display on the LVDS panel (if present).
  */
-void intel_lvds_init(struct drm_device *dev)
+bool intel_lvds_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds *intel_lvds;
@@ -853,37 +872,37 @@ void intel_lvds_init(struct drm_device *dev)
 
         /* Skip init on machines we know falsely report LVDS */
         if (dmi_check_system(intel_no_lvds))
-                return;
+                return false;
 
         pin = GMBUS_PORT_PANEL;
         if (!lvds_is_present_in_vbt(dev, &pin)) {
                 DRM_DEBUG_KMS("LVDS is not present in VBT\n");
-                return;
+                return false;
         }
 
         if (HAS_PCH_SPLIT(dev)) {
                 if ((I915_READ(PCH_LVDS) & LVDS_DETECTED) == 0)
-                        return;
+                        return false;
                 if (dev_priv->edp.support) {
                         DRM_DEBUG_KMS("disable LVDS for eDP support\n");
-                        return;
+                        return false;
                 }
         }
 
         if (!intel_lvds_ddc_probe(dev, pin)) {
                 DRM_DEBUG_KMS("LVDS did not respond to DDC probe\n");
-                return;
+                return false;
         }
 
         intel_lvds = kzalloc(sizeof(struct intel_lvds), GFP_KERNEL);
         if (!intel_lvds) {
-                return;
+                return false;
         }
 
         intel_connector = kzalloc(sizeof(struct intel_connector), GFP_KERNEL);
         if (!intel_connector) {
                 kfree(intel_lvds);
-                return;
+                return false;
         }
 
         if (!HAS_PCH_SPLIT(dev)) {
@@ -904,6 +923,8 @@ void intel_lvds_init(struct drm_device *dev)
 
         intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT);
         intel_encoder->crtc_mask = (1 << 1);
+        if (INTEL_INFO(dev)->gen >= 5)
+                intel_encoder->crtc_mask |= (1 << 0);
         drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs);
         drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs);
         connector->display_info.subpixel_order = SubPixelHorizontalRGB;
@@ -1009,10 +1030,18 @@ void intel_lvds_init(struct drm_device *dev)
 out:
         if (HAS_PCH_SPLIT(dev)) {
                 u32 pwm;
-                /* make sure PWM is enabled */
+
+                pipe = (I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT) ? 1 : 0;
+
+                /* make sure PWM is enabled and locked to the LVDS pipe */
                 pwm = I915_READ(BLC_PWM_CPU_CTL2);
-                pwm |= (PWM_ENABLE | PWM_PIPE_B);
-                I915_WRITE(BLC_PWM_CPU_CTL2, pwm);
+                if (pipe == 0 && (pwm & PWM_PIPE_B))
+                        I915_WRITE(BLC_PWM_CPU_CTL2, pwm & ~PWM_ENABLE);
+                if (pipe)
+                        pwm |= PWM_PIPE_B;
+                else
+                        pwm &= ~PWM_PIPE_B;
+                I915_WRITE(BLC_PWM_CPU_CTL2, pwm | PWM_ENABLE);
 
                 pwm = I915_READ(BLC_PWM_PCH_CTL1);
                 pwm |= PWM_PCH_ENABLE;
@@ -1026,7 +1055,7 @@ out:
         /* keep the LVDS connector */
         dev_priv->int_lvds_connector = connector;
         drm_sysfs_connector_add(connector);
-        return;
+        return true;
 
 failed:
         DRM_DEBUG_KMS("No LVDS modes found, disabling.\n");
@@ -1034,4 +1063,5 @@ failed:
         drm_encoder_cleanup(encoder);
         kfree(intel_lvds);
         kfree(intel_connector);
+        return false;
 }
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 9b0d9a867aea..64fd64443ca6 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -26,6 +26,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/acpi_io.h>
 #include <acpi/video.h>
 
 #include "drmP.h"
@@ -273,14 +274,8 @@ void intel_opregion_enable_asle(struct drm_device *dev)
         struct opregion_asle *asle = dev_priv->opregion.asle;
 
         if (asle) {
-                if (IS_MOBILE(dev)) {
-                        unsigned long irqflags;
-
-                        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+                if (IS_MOBILE(dev))
                         intel_enable_asle(dev);
-                        spin_unlock_irqrestore(&dev_priv->user_irq_lock,
-                                               irqflags);
-                }
 
                 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN |
                         ASLE_PFMB_EN;
@@ -482,7 +477,7 @@ int intel_opregion_setup(struct drm_device *dev)
                 return -ENOTSUPP;
         }
 
-        base = ioremap(asls, OPREGION_SIZE);
+        base = acpi_os_ioremap(asls, OPREGION_SIZE);
         if (!base)
                 return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 02ff0a481f47..3fbb98b948d6 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -221,15 +221,16 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
         int ret;
 
         BUG_ON(overlay->last_flip_req);
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, request, &dev_priv->render_ring);
-        if (overlay->last_flip_req == 0)
-                return -ENOMEM;
-
+        ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv));
+        if (ret) {
+            kfree(request);
+            return ret;
+        }
+        overlay->last_flip_req = request->seqno;
         overlay->flip_tail = tail;
         ret = i915_do_wait_request(dev,
                                    overlay->last_flip_req, true,
-                                   &dev_priv->render_ring);
+                                   LP_RING(dev_priv));
         if (ret)
                 return ret;
 
@@ -289,6 +290,7 @@ i830_deactivate_pipe_a(struct drm_device *dev)
 static int intel_overlay_on(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_request *request;
         int pipe_a_quirk = 0;
         int ret;
@@ -308,7 +310,12 @@ static int intel_overlay_on(struct intel_overlay *overlay)
                 goto out;
         }
 
-        BEGIN_LP_RING(4);
+        ret = BEGIN_LP_RING(4);
+        if (ret) {
+                kfree(request);
+                goto out;
+        }
+
         OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON);
         OUT_RING(overlay->flip_addr | OFC_UPDATE);
         OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
@@ -332,6 +339,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
         struct drm_i915_gem_request *request;
         u32 flip_addr = overlay->flip_addr;
         u32 tmp;
+        int ret;
 
         BUG_ON(!overlay->active);
 
@@ -347,36 +355,44 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
         if (tmp & (1 << 17))
                 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
 
-        BEGIN_LP_RING(2);
+        ret = BEGIN_LP_RING(2);
+        if (ret) {
+                kfree(request);
+                return ret;
+        }
         OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
         OUT_RING(flip_addr);
         ADVANCE_LP_RING();
 
-        overlay->last_flip_req =
-                i915_add_request(dev, NULL, request, &dev_priv->render_ring);
+        ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv));
+        if (ret) {
+                kfree(request);
+                return ret;
+        }
+
+        overlay->last_flip_req = request->seqno;
         return 0;
 }
 
 static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
 {
-        struct drm_gem_object *obj = &overlay->old_vid_bo->base;
+        struct drm_i915_gem_object *obj = overlay->old_vid_bo;
 
         i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 
         overlay->old_vid_bo = NULL;
 }
 
 static void intel_overlay_off_tail(struct intel_overlay *overlay)
 {
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj = overlay->vid_bo;
 
         /* never have the overlay hw on without showing a frame */
         BUG_ON(!overlay->vid_bo);
-        obj = &overlay->vid_bo->base;
 
         i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         overlay->vid_bo = NULL;
 
         overlay->crtc->overlay = NULL;
@@ -389,8 +405,10 @@ static int intel_overlay_off(struct intel_overlay *overlay,
                              bool interruptible)
 {
         struct drm_device *dev = overlay->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 flip_addr = overlay->flip_addr;
         struct drm_i915_gem_request *request;
+        int ret;
 
         BUG_ON(!overlay->active);
 
@@ -404,7 +422,11 @@ static int intel_overlay_off(struct intel_overlay *overlay,
          * of the hw. Do it in both cases */
         flip_addr |= OFC_UPDATE;
 
-        BEGIN_LP_RING(6);
+        ret = BEGIN_LP_RING(6);
+        if (ret) {
+                kfree(request);
+                return ret;
+        }
         /* wait for overlay to go idle */
         OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
         OUT_RING(flip_addr);
@@ -432,7 +454,7 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay,
                 return 0;
 
         ret = i915_do_wait_request(dev, overlay->last_flip_req,
-                                   interruptible, &dev_priv->render_ring);
+                                   interruptible, LP_RING(dev_priv));
         if (ret)
                 return ret;
 
@@ -467,7 +489,12 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
                 if (request == NULL)
                         return -ENOMEM;
 
-                BEGIN_LP_RING(2);
+                ret = BEGIN_LP_RING(2);
+                if (ret) {
+                        kfree(request);
+                        return ret;
+                }
+
                 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
                 OUT_RING(MI_NOOP);
                 ADVANCE_LP_RING();
@@ -736,13 +763,12 @@ static u32 overlay_cmd_reg(struct put_image_params *params)
 }
 
 static int intel_overlay_do_put_image(struct intel_overlay *overlay,
-                                      struct drm_gem_object *new_bo,
+                                      struct drm_i915_gem_object *new_bo,
                                       struct put_image_params *params)
 {
         int ret, tmp_width;
         struct overlay_registers *regs;
         bool scale_changed = false;
-        struct drm_i915_gem_object *bo_priv = to_intel_bo(new_bo);
         struct drm_device *dev = overlay->dev;
 
         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -753,7 +779,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         if (ret != 0)
                 return ret;
 
-        ret = i915_gem_object_pin(new_bo, PAGE_SIZE);
+        ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true);
         if (ret != 0)
                 return ret;
 
@@ -761,6 +787,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         if (ret != 0)
                 goto out_unpin;
 
+        ret = i915_gem_object_put_fence(new_bo);
+        if (ret)
+                goto out_unpin;
+
         if (!overlay->active) {
                 regs = intel_overlay_map_regs(overlay);
                 if (!regs) {
@@ -797,7 +827,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         regs->SWIDTHSW = calc_swidthsw(overlay->dev,
                                        params->offset_Y, tmp_width);
         regs->SHEIGHT = params->src_h;
-        regs->OBUF_0Y = bo_priv->gtt_offset + params-> offset_Y;
+        regs->OBUF_0Y = new_bo->gtt_offset + params-> offset_Y;
         regs->OSTRIDE = params->stride_Y;
 
         if (params->format & I915_OVERLAY_YUV_PLANAR) {
@@ -811,8 +841,8 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
                                       params->src_w/uv_hscale);
                 regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16;
                 regs->SHEIGHT |= (params->src_h/uv_vscale) << 16;
-                regs->OBUF_0U = bo_priv->gtt_offset + params->offset_U;
-                regs->OBUF_0V = bo_priv->gtt_offset + params->offset_V;
+                regs->OBUF_0U = new_bo->gtt_offset + params->offset_U;
+                regs->OBUF_0V = new_bo->gtt_offset + params->offset_V;
                 regs->OSTRIDE |= params->stride_UV << 16;
         }
 
@@ -829,7 +859,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
                 goto out_unpin;
 
         overlay->old_vid_bo = overlay->vid_bo;
-        overlay->vid_bo = to_intel_bo(new_bo);
+        overlay->vid_bo = new_bo;
 
         return 0;
 
@@ -942,7 +972,7 @@ static int check_overlay_scaling(struct put_image_params *rec)
 
 static int check_overlay_src(struct drm_device *dev,
                              struct drm_intel_overlay_put_image *rec,
-                             struct drm_gem_object *new_bo)
+                             struct drm_i915_gem_object *new_bo)
 {
         int uv_hscale = uv_hsubsampling(rec->flags);
         int uv_vscale = uv_vsubsampling(rec->flags);
@@ -1027,7 +1057,7 @@ static int check_overlay_src(struct drm_device *dev,
                         return -EINVAL;
 
                 tmp = rec->stride_Y*rec->src_height;
-                if (rec->offset_Y + tmp > new_bo->size)
+                if (rec->offset_Y + tmp > new_bo->base.size)
                         return -EINVAL;
                 break;
 
@@ -1038,12 +1068,12 @@ static int check_overlay_src(struct drm_device *dev,
                         return -EINVAL;
 
                 tmp = rec->stride_Y * rec->src_height;
-                if (rec->offset_Y + tmp > new_bo->size)
+                if (rec->offset_Y + tmp > new_bo->base.size)
                         return -EINVAL;
 
                 tmp = rec->stride_UV * (rec->src_height / uv_vscale);
-                if (rec->offset_U + tmp > new_bo->size ||
-                    rec->offset_V + tmp > new_bo->size)
+                if (rec->offset_U + tmp > new_bo->base.size ||
+                    rec->offset_V + tmp > new_bo->base.size)
                         return -EINVAL;
                 break;
         }
@@ -1086,7 +1116,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         struct intel_overlay *overlay;
         struct drm_mode_object *drmmode_obj;
         struct intel_crtc *crtc;
-        struct drm_gem_object *new_bo;
+        struct drm_i915_gem_object *new_bo;
         struct put_image_params *params;
         int ret;
 
@@ -1125,8 +1155,8 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         }
         crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
 
-        new_bo = drm_gem_object_lookup(dev, file_priv,
-                                       put_image_rec->bo_handle);
+        new_bo = to_intel_bo(drm_gem_object_lookup(dev, file_priv,
+                                                   put_image_rec->bo_handle));
         if (!new_bo) {
                 ret = -ENOENT;
                 goto out_free;
@@ -1135,6 +1165,12 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         mutex_lock(&dev->mode_config.mutex);
         mutex_lock(&dev->struct_mutex);
 
+        if (new_bo->tiling_mode) {
+                DRM_ERROR("buffer used for overlay image can not be tiled\n");
+                ret = -EINVAL;
+                goto out_unlock;
+        }
+
         ret = intel_overlay_recover_from_interrupt(overlay, true);
         if (ret != 0)
                 goto out_unlock;
@@ -1217,7 +1253,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
         mutex_unlock(&dev->mode_config.mutex);
-        drm_gem_object_unreference_unlocked(new_bo);
+        drm_gem_object_unreference_unlocked(&new_bo->base);
 out_free:
         kfree(params);
 
@@ -1370,7 +1406,7 @@ void intel_setup_overlay(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct intel_overlay *overlay;
-        struct drm_gem_object *reg_bo;
+        struct drm_i915_gem_object *reg_bo;
         struct overlay_registers *regs;
         int ret;
 
@@ -1385,7 +1421,7 @@ void intel_setup_overlay(struct drm_device *dev)
         reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE);
         if (!reg_bo)
                 goto out_free;
-        overlay->reg_bo = to_intel_bo(reg_bo);
+        overlay->reg_bo = reg_bo;
 
         if (OVERLAY_NEEDS_PHYSICAL(dev)) {
                 ret = i915_gem_attach_phys_object(dev, reg_bo,
@@ -1395,14 +1431,14 @@ void intel_setup_overlay(struct drm_device *dev)
                         DRM_ERROR("failed to attach phys overlay regs\n");
                         goto out_free_bo;
                 }
-                overlay->flip_addr = overlay->reg_bo->phys_obj->handle->busaddr;
+                overlay->flip_addr = reg_bo->phys_obj->handle->busaddr;
         } else {
-                ret = i915_gem_object_pin(reg_bo, PAGE_SIZE);
+                ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true);
                 if (ret) {
                         DRM_ERROR("failed to pin overlay register bo\n");
                         goto out_free_bo;
                 }
-                overlay->flip_addr = overlay->reg_bo->gtt_offset;
+                overlay->flip_addr = reg_bo->gtt_offset;
 
                 ret = i915_gem_object_set_to_gtt_domain(reg_bo, true);
                 if (ret) {
@@ -1434,7 +1470,7 @@ void intel_setup_overlay(struct drm_device *dev)
 out_unpin_bo:
         i915_gem_object_unpin(reg_bo);
 out_free_bo:
-        drm_gem_object_unreference(reg_bo);
+        drm_gem_object_unreference(&reg_bo->base);
 out_free:
         kfree(overlay);
         return;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 92ff8f385278..c65992df458d 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -125,15 +125,55 @@ static int is_backlight_combination_mode(struct drm_device *dev)
         return 0;
 }
 
+static u32 i915_read_blc_pwm_ctl(struct drm_i915_private *dev_priv)
+{
+        u32 val;
+
+        /* Restore the CTL value if it lost, e.g. GPU reset */
+
+        if (HAS_PCH_SPLIT(dev_priv->dev)) {
+                val = I915_READ(BLC_PWM_PCH_CTL2);
+                if (dev_priv->saveBLC_PWM_CTL2 == 0) {
+                        dev_priv->saveBLC_PWM_CTL2 = val;
+                } else if (val == 0) {
+                        I915_WRITE(BLC_PWM_PCH_CTL2,
+                                   dev_priv->saveBLC_PWM_CTL);
+                        val = dev_priv->saveBLC_PWM_CTL;
+                }
+        } else {
+                val = I915_READ(BLC_PWM_CTL);
+                if (dev_priv->saveBLC_PWM_CTL == 0) {
+                        dev_priv->saveBLC_PWM_CTL = val;
+                        dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
+                } else if (val == 0) {
+                        I915_WRITE(BLC_PWM_CTL,
+                                   dev_priv->saveBLC_PWM_CTL);
+                        I915_WRITE(BLC_PWM_CTL2,
+                                   dev_priv->saveBLC_PWM_CTL2);
+                        val = dev_priv->saveBLC_PWM_CTL;
+                }
+        }
+
+        return val;
+}
+
 u32 intel_panel_get_max_backlight(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 max;
 
+        max = i915_read_blc_pwm_ctl(dev_priv);
+        if (max == 0) {
+                /* XXX add code here to query mode clock or hardware clock
+                 * and program max PWM appropriately.
+                 */
+                printk_once(KERN_WARNING "fixme: max PWM is zero.\n");
+                return 1;
+        }
+
         if (HAS_PCH_SPLIT(dev)) {
-                max = I915_READ(BLC_PWM_PCH_CTL2) >> 16;
+                max >>= 16;
         } else {
-                max = I915_READ(BLC_PWM_CTL);
                 if (IS_PINEVIEW(dev)) {
                         max >>= 17;
                 } else {
@@ -146,14 +186,6 @@ u32 intel_panel_get_max_backlight(struct drm_device *dev)
                         max *= 0xff;
         }
 
-        if (max == 0) {
-                /* XXX add code here to query mode clock or hardware clock
-                 * and program max PWM appropriately.
-                 */
-                DRM_ERROR("fixme: max PWM is zero.\n");
-                max = 1;
-        }
-
         DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max);
         return max;
 }
@@ -218,3 +250,34 @@ void intel_panel_set_backlight(struct drm_device *dev, u32 level)
                 tmp &= ~BACKLIGHT_DUTY_CYCLE_MASK;
         I915_WRITE(BLC_PWM_CTL, tmp | level);
 }
+
+void intel_panel_disable_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->backlight_enabled) {
+                dev_priv->backlight_level = intel_panel_get_backlight(dev);
+                dev_priv->backlight_enabled = false;
+        }
+
+        intel_panel_set_backlight(dev, 0);
+}
+
+void intel_panel_enable_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->backlight_level == 0)
+                dev_priv->backlight_level = intel_panel_get_max_backlight(dev);
+
+        intel_panel_set_backlight(dev, dev_priv->backlight_level);
+        dev_priv->backlight_enabled = true;
+}
+
+void intel_panel_setup_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        dev_priv->backlight_level = intel_panel_get_backlight(dev);
+        dev_priv->backlight_enabled = dev_priv->backlight_level != 0;
+}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index b83306f9244b..6218fa97aa1e 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -34,6 +34,14 @@
 #include "i915_trace.h"
 #include "intel_drv.h"
 
+static inline int ring_space(struct intel_ring_buffer *ring)
+{
+        int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
+        if (space < 0)
+                space += ring->size;
+        return space;
+}
+
 static u32 i915_gem_get_seqno(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
@@ -48,14 +56,15 @@ static u32 i915_gem_get_seqno(struct drm_device *dev)
         return seqno;
 }
 
-static void
-render_ring_flush(struct drm_device *dev,
-                  struct intel_ring_buffer *ring,
+static int
+render_ring_flush(struct intel_ring_buffer *ring,
                   u32   invalidate_domains,
                   u32   flush_domains)
 {
+        struct drm_device *dev = ring->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         u32 cmd;
+        int ret;
 
 #if WATCH_EXEC
         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
@@ -109,79 +118,87 @@ render_ring_flush(struct drm_device *dev,
                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
                         cmd |= MI_EXE_FLUSH;
 
+                if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
+                    (IS_G4X(dev) || IS_GEN5(dev)))
+                        cmd |= MI_INVALIDATE_ISP;
+
 #if WATCH_EXEC
                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 #endif
-                intel_ring_begin(dev, ring, 2);
-                intel_ring_emit(dev, ring, cmd);
-                intel_ring_emit(dev, ring, MI_NOOP);
-                intel_ring_advance(dev, ring);
+                ret = intel_ring_begin(ring, 2);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(ring, cmd);
+                intel_ring_emit(ring, MI_NOOP);
+                intel_ring_advance(ring);
         }
+
+        return 0;
 }
 
-static void ring_write_tail(struct drm_device *dev,
-                            struct intel_ring_buffer *ring,
+static void ring_write_tail(struct intel_ring_buffer *ring,
                             u32 value)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
         I915_WRITE_TAIL(ring, value);
 }
 
-u32 intel_ring_get_active_head(struct drm_device *dev,
-                               struct intel_ring_buffer *ring)
+u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        u32 acthd_reg = INTEL_INFO(dev)->gen >= 4 ?
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
                         RING_ACTHD(ring->mmio_base) : ACTHD;
 
         return I915_READ(acthd_reg);
 }
 
-static int init_ring_common(struct drm_device *dev,
-                            struct intel_ring_buffer *ring)
+static int init_ring_common(struct intel_ring_buffer *ring)
 {
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_gem_object *obj = ring->obj;
         u32 head;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
-        obj_priv = to_intel_bo(ring->gem_object);
 
         /* Stop the ring if it's running. */
         I915_WRITE_CTL(ring, 0);
         I915_WRITE_HEAD(ring, 0);
-        ring->write_tail(dev, ring, 0);
+        ring->write_tail(ring, 0);
 
         /* Initialize the ring. */
-        I915_WRITE_START(ring, obj_priv->gtt_offset);
+        I915_WRITE_START(ring, obj->gtt_offset);
         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 
         /* G45 ring initialization fails to reset head to zero */
         if (head != 0) {
-                DRM_ERROR("%s head not reset to zero "
-                                "ctl %08x head %08x tail %08x start %08x\n",
-                                ring->name,
-                                I915_READ_CTL(ring),
-                                I915_READ_HEAD(ring),
-                                I915_READ_TAIL(ring),
-                                I915_READ_START(ring));
+                DRM_DEBUG_KMS("%s head not reset to zero "
+                              "ctl %08x head %08x tail %08x start %08x\n",
+                              ring->name,
+                              I915_READ_CTL(ring),
+                              I915_READ_HEAD(ring),
+                              I915_READ_TAIL(ring),
+                              I915_READ_START(ring));
 
                 I915_WRITE_HEAD(ring, 0);
 
-                DRM_ERROR("%s head forced to zero "
-                                "ctl %08x head %08x tail %08x start %08x\n",
-                                ring->name,
-                                I915_READ_CTL(ring),
-                                I915_READ_HEAD(ring),
-                                I915_READ_TAIL(ring),
-                                I915_READ_START(ring));
+                if (I915_READ_HEAD(ring) & HEAD_ADDR) {
+                        DRM_ERROR("failed to set %s head to zero "
+                                  "ctl %08x head %08x tail %08x start %08x\n",
+                                  ring->name,
+                                  I915_READ_CTL(ring),
+                                  I915_READ_HEAD(ring),
+                                  I915_READ_TAIL(ring),
+                                  I915_READ_START(ring));
+                }
         }
 
         I915_WRITE_CTL(ring,
-                        ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
+                        ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
                         | RING_REPORT_64K | RING_VALID);
 
-        head = I915_READ_HEAD(ring) & HEAD_ADDR;
         /* If the head is still not zero, the ring is dead */
-        if (head != 0) {
+        if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
+            I915_READ_START(ring) != obj->gtt_offset ||
+            (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
                 DRM_ERROR("%s initialization failed "
                                 "ctl %08x head %08x tail %08x start %08x\n",
                                 ring->name,
@@ -192,344 +209,569 @@ static int init_ring_common(struct drm_device *dev,
                 return -EIO;
         }
 
-        if (!drm_core_check_feature(dev, DRIVER_MODESET))
-                i915_kernel_lost_context(dev);
+        if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
+                i915_kernel_lost_context(ring->dev);
         else {
-                ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
+                ring->head = I915_READ_HEAD(ring);
                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
+                ring->space = ring_space(ring);
         }
+
         return 0;
 }
 
-static int init_render_ring(struct drm_device *dev,
-                            struct intel_ring_buffer *ring)
+/*
+ * 965+ support PIPE_CONTROL commands, which provide finer grained control
+ * over cache flushing.
+ */
+struct pipe_control {
+        struct drm_i915_gem_object *obj;
+        volatile u32 *cpu_page;
+        u32 gtt_offset;
+};
+
+static int
+init_pipe_control(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret = init_ring_common(dev, ring);
-        int mode;
+        struct pipe_control *pc;
+        struct drm_i915_gem_object *obj;
+        int ret;
+
+        if (ring->private)
+                return 0;
+
+        pc = kmalloc(sizeof(*pc), GFP_KERNEL);
+        if (!pc)
+                return -ENOMEM;
+
+        obj = i915_gem_alloc_object(ring->dev, 4096);
+        if (obj == NULL) {
+                DRM_ERROR("Failed to allocate seqno page\n");
+                ret = -ENOMEM;
+                goto err;
+        }
+        obj->agp_type = AGP_USER_CACHED_MEMORY;
+
+        ret = i915_gem_object_pin(obj, 4096, true);
+        if (ret)
+                goto err_unref;
+
+        pc->gtt_offset = obj->gtt_offset;
+        pc->cpu_page =  kmap(obj->pages[0]);
+        if (pc->cpu_page == NULL)
+                goto err_unpin;
+
+        pc->obj = obj;
+        ring->private = pc;
+        return 0;
+
+err_unpin:
+        i915_gem_object_unpin(obj);
+err_unref:
+        drm_gem_object_unreference(&obj->base);
+err:
+        kfree(pc);
+        return ret;
+}
+
+static void
+cleanup_pipe_control(struct intel_ring_buffer *ring)
+{
+        struct pipe_control *pc = ring->private;
+        struct drm_i915_gem_object *obj;
+
+        if (!ring->private)
+                return;
+
+        obj = pc->obj;
+        kunmap(obj->pages[0]);
+        i915_gem_object_unpin(obj);
+        drm_gem_object_unreference(&obj->base);
+
+        kfree(pc);
+        ring->private = NULL;
+}
+
+static int init_render_ring(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret = init_ring_common(ring);
 
         if (INTEL_INFO(dev)->gen > 3) {
-                mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
+                int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
                 if (IS_GEN6(dev))
                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
                 I915_WRITE(MI_MODE, mode);
         }
+
+        if (INTEL_INFO(dev)->gen >= 6) {
+        } else if (IS_GEN5(dev)) {
+                ret = init_pipe_control(ring);
+                if (ret)
+                        return ret;
+        }
+
         return ret;
 }
 
-#define PIPE_CONTROL_FLUSH(addr)                                        \
+static void render_ring_cleanup(struct intel_ring_buffer *ring)
+{
+        if (!ring->private)
+                return;
+
+        cleanup_pipe_control(ring);
+}
+
+static void
+update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int id;
+
+        /*
+         * cs -> 1 = vcs, 0 = bcs
+         * vcs -> 1 = bcs, 0 = cs,
+         * bcs -> 1 = cs, 0 = vcs.
+         */
+        id = ring - dev_priv->ring;
+        id += 2 - i;
+        id %= 3;
+
+        intel_ring_emit(ring,
+                        MI_SEMAPHORE_MBOX |
+                        MI_SEMAPHORE_REGISTER |
+                        MI_SEMAPHORE_UPDATE);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring,
+                        RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
+}
+
+static int
+gen6_add_request(struct intel_ring_buffer *ring,
+                 u32 *result)
+{
+        u32 seqno;
+        int ret;
+
+        ret = intel_ring_begin(ring, 10);
+        if (ret)
+                return ret;
+
+        seqno = i915_gem_get_seqno(ring->dev);
+        update_semaphore(ring, 0, seqno);
+        update_semaphore(ring, 1, seqno);
+
+        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, MI_USER_INTERRUPT);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
+}
+
+int
+intel_ring_sync(struct intel_ring_buffer *ring,
+                struct intel_ring_buffer *to,
+                u32 seqno)
+{
+        int ret;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring,
+                        MI_SEMAPHORE_MBOX |
+                        MI_SEMAPHORE_REGISTER |
+                        intel_ring_sync_index(ring, to) << 17 |
+                        MI_SEMAPHORE_COMPARE);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+
+        return 0;
+}
+
+#define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 do {                                                                    \
-        OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
+        intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
-        OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
-        OUT_RING(0);                                                    \
-        OUT_RING(0);                                                    \
+        intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
+        intel_ring_emit(ring__, 0);                                                     \
+        intel_ring_emit(ring__, 0);                                                     \
 } while (0)
 
-/**
- * Creates a new sequence number, emitting a write of it to the status page
- * plus an interrupt, which will trigger i915_user_interrupt_handler.
- *
- * Must be called with struct_lock held.
- *
- * Returned sequence numbers are nonzero on success.
- */
-static u32
-render_ring_add_request(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        u32 flush_domains)
+static int
+pc_render_add_request(struct intel_ring_buffer *ring,
+                      u32 *result)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        u32 seqno;
+        struct drm_device *dev = ring->dev;
+        u32 seqno = i915_gem_get_seqno(dev);
+        struct pipe_control *pc = ring->private;
+        u32 scratch_addr = pc->gtt_offset + 128;
+        int ret;
 
-        seqno = i915_gem_get_seqno(dev);
-
-        if (IS_GEN6(dev)) {
-                BEGIN_LP_RING(6);
-                OUT_RING(GFX_OP_PIPE_CONTROL | 3);
-                OUT_RING(PIPE_CONTROL_QW_WRITE |
-                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
-                         PIPE_CONTROL_NOTIFY);
-                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
-                OUT_RING(seqno);
-                OUT_RING(0);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        } else if (HAS_PIPE_CONTROL(dev)) {
-                u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
+        /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
+         * incoherent with writes to memory, i.e. completely fubar,
+         * so we need to use PIPE_NOTIFY instead.
+         *
+         * However, we also need to workaround the qword write
+         * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
+         * memory before requesting an interrupt.
+         */
+        ret = intel_ring_begin(ring, 32);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
+                        PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
+        intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, 0);
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128; /* write to separate cachelines */
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        scratch_addr += 128;
+        PIPE_CONTROL_FLUSH(ring, scratch_addr);
+        intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
+                        PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
+                        PIPE_CONTROL_NOTIFY);
+        intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, 0);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
+}
 
-                /*
-                 * Workaround qword write incoherence by flushing the
-                 * PIPE_NOTIFY buffers out to memory before requesting
-                 * an interrupt.
-                 */
-                BEGIN_LP_RING(32);
-                OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
-                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
-                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
-                OUT_RING(seqno);
-                OUT_RING(0);
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128; /* write to separate cachelines */
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                scratch_addr += 128;
-                PIPE_CONTROL_FLUSH(scratch_addr);
-                OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
-                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
-                         PIPE_CONTROL_NOTIFY);
-                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
-                OUT_RING(seqno);
-                OUT_RING(0);
-                ADVANCE_LP_RING();
-        } else {
-                BEGIN_LP_RING(4);
-                OUT_RING(MI_STORE_DWORD_INDEX);
-                OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-                OUT_RING(seqno);
+static int
+render_ring_add_request(struct intel_ring_buffer *ring,
+                        u32 *result)
+{
+        struct drm_device *dev = ring->dev;
+        u32 seqno = i915_gem_get_seqno(dev);
+        int ret;
 
-                OUT_RING(MI_USER_INTERRUPT);
-                ADVANCE_LP_RING();
-        }
-        return seqno;
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, MI_USER_INTERRUPT);
+        intel_ring_advance(ring);
+
+        *result = seqno;
+        return 0;
 }
 
 static u32
-render_ring_get_seqno(struct drm_device *dev,
-                      struct intel_ring_buffer *ring)
+ring_get_seqno(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        if (HAS_PIPE_CONTROL(dev))
-                return ((volatile u32 *)(dev_priv->seqno_page))[0];
-        else
-                return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+        return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+}
+
+static u32
+pc_render_get_seqno(struct intel_ring_buffer *ring)
+{
+        struct pipe_control *pc = ring->private;
+        return pc->cpu_page[0];
+}
+
+static void
+ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->gt_irq_mask &= ~mask;
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+        POSTING_READ(GTIMR);
 }
 
 static void
-render_ring_get_user_irq(struct drm_device *dev,
-                         struct intel_ring_buffer *ring)
+ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->gt_irq_mask |= mask;
+        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+        POSTING_READ(GTIMR);
+}
+
+static void
+i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->irq_mask &= ~mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ(IMR);
+}
+
+static void
+i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
+{
+        dev_priv->irq_mask |= mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ(IMR);
+}
+
+static bool
+render_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        unsigned long irqflags;
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
+        if (!dev->irq_enabled)
+                return false;
+
+        spin_lock(&ring->irq_lock);
+        if (ring->irq_refcount++ == 0) {
                 if (HAS_PCH_SPLIT(dev))
-                        ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
+                        ironlake_enable_irq(dev_priv,
+                                            GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
                 else
                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
         }
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock(&ring->irq_lock);
+
+        return true;
 }
 
 static void
-render_ring_put_user_irq(struct drm_device *dev,
-                         struct intel_ring_buffer *ring)
+render_ring_put_irq(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        unsigned long irqflags;
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
-        if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
+        spin_lock(&ring->irq_lock);
+        if (--ring->irq_refcount == 0) {
                 if (HAS_PCH_SPLIT(dev))
-                        ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
+                        ironlake_disable_irq(dev_priv,
+                                             GT_USER_INTERRUPT |
+                                             GT_PIPE_NOTIFY);
                 else
                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
         }
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock(&ring->irq_lock);
 }
 
-void intel_ring_setup_status_page(struct drm_device *dev,
-                                  struct intel_ring_buffer *ring)
+void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        if (IS_GEN6(dev)) {
-                I915_WRITE(RING_HWS_PGA_GEN6(ring->mmio_base),
-                           ring->status_page.gfx_addr);
-                I915_READ(RING_HWS_PGA_GEN6(ring->mmio_base)); /* posting read */
-        } else {
-                I915_WRITE(RING_HWS_PGA(ring->mmio_base),
-                           ring->status_page.gfx_addr);
-                I915_READ(RING_HWS_PGA(ring->mmio_base)); /* posting read */
-        }
-
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        u32 mmio = IS_GEN6(ring->dev) ?
+                RING_HWS_PGA_GEN6(ring->mmio_base) :
+                RING_HWS_PGA(ring->mmio_base);
+        I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
+        POSTING_READ(mmio);
 }
 
-static void
-bsd_ring_flush(struct drm_device *dev,
-                struct intel_ring_buffer *ring,
-                u32     invalidate_domains,
-                u32     flush_domains)
+static int
+bsd_ring_flush(struct intel_ring_buffer *ring,
+               u32     invalidate_domains,
+               u32     flush_domains)
 {
-        intel_ring_begin(dev, ring, 2);
-        intel_ring_emit(dev, ring, MI_FLUSH);
-        intel_ring_emit(dev, ring, MI_NOOP);
-        intel_ring_advance(dev, ring);
-}
+        int ret;
 
-static int init_bsd_ring(struct drm_device *dev,
-                         struct intel_ring_buffer *ring)
-{
-        return init_ring_common(dev, ring);
+        if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
+                return 0;
+
+        ret = intel_ring_begin(ring, 2);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_FLUSH);
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+        return 0;
 }
 
-static u32
-ring_add_request(struct drm_device *dev,
-                 struct intel_ring_buffer *ring,
-                 u32 flush_domains)
+static int
+ring_add_request(struct intel_ring_buffer *ring,
+                 u32 *result)
 {
         u32 seqno;
+        int ret;
 
-        seqno = i915_gem_get_seqno(dev);
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
 
-        intel_ring_begin(dev, ring, 4);
-        intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
-        intel_ring_emit(dev, ring,
-                        I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        intel_ring_emit(dev, ring, seqno);
-        intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
-        intel_ring_advance(dev, ring);
+        seqno = i915_gem_get_seqno(ring->dev);
+
+        intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+        intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(ring, seqno);
+        intel_ring_emit(ring, MI_USER_INTERRUPT);
+        intel_ring_advance(ring);
 
         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
+        *result = seqno;
+        return 0;
+}
 
-        return seqno;
+static bool
+ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
+{
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        if (!dev->irq_enabled)
+               return false;
+
+        spin_lock(&ring->irq_lock);
+        if (ring->irq_refcount++ == 0)
+                ironlake_enable_irq(dev_priv, flag);
+        spin_unlock(&ring->irq_lock);
+
+        return true;
 }
 
 static void
-bsd_ring_get_user_irq(struct drm_device *dev,
-                      struct intel_ring_buffer *ring)
+ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
 {
-        /* do nothing */
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        spin_lock(&ring->irq_lock);
+        if (--ring->irq_refcount == 0)
+                ironlake_disable_irq(dev_priv, flag);
+        spin_unlock(&ring->irq_lock);
 }
+
+static bool
+gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
+{
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        if (!dev->irq_enabled)
+               return false;
+
+        spin_lock(&ring->irq_lock);
+        if (ring->irq_refcount++ == 0) {
+                ring->irq_mask &= ~rflag;
+                I915_WRITE_IMR(ring, ring->irq_mask);
+                ironlake_enable_irq(dev_priv, gflag);
+        }
+        spin_unlock(&ring->irq_lock);
+
+        return true;
+}
+
 static void
-bsd_ring_put_user_irq(struct drm_device *dev,
-                      struct intel_ring_buffer *ring)
+gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 {
-        /* do nothing */
+        struct drm_device *dev = ring->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        spin_lock(&ring->irq_lock);
+        if (--ring->irq_refcount == 0) {
+                ring->irq_mask |= rflag;
+                I915_WRITE_IMR(ring, ring->irq_mask);
+                ironlake_disable_irq(dev_priv, gflag);
+        }
+        spin_unlock(&ring->irq_lock);
 }
 
-static u32
-ring_status_page_get_seqno(struct drm_device *dev,
-                           struct intel_ring_buffer *ring)
+static bool
+bsd_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+        return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
+}
+static void
+bsd_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
 }
 
 static int
-ring_dispatch_gem_execbuffer(struct drm_device *dev,
-                             struct intel_ring_buffer *ring,
-                             struct drm_i915_gem_execbuffer2 *exec,
-                             struct drm_clip_rect *cliprects,
-                             uint64_t exec_offset)
+ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 {
-        uint32_t exec_start;
-        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
-        intel_ring_begin(dev, ring, 2);
-        intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
-                        (2 << 6) | MI_BATCH_NON_SECURE_I965);
-        intel_ring_emit(dev, ring, exec_start);
-        intel_ring_advance(dev, ring);
+        int ret;
+
+        ret = intel_ring_begin(ring, 2);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring,
+                        MI_BATCH_BUFFER_START | (2 << 6) |
+                        MI_BATCH_NON_SECURE_I965);
+        intel_ring_emit(ring, offset);
+        intel_ring_advance(ring);
+
         return 0;
 }
 
 static int
-render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
-                                    struct intel_ring_buffer *ring,
-                                    struct drm_i915_gem_execbuffer2 *exec,
-                                    struct drm_clip_rect *cliprects,
-                                    uint64_t exec_offset)
+render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
+                                u32 offset, u32 len)
 {
+        struct drm_device *dev = ring->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        int nbox = exec->num_cliprects;
-        int i = 0, count;
-        uint32_t exec_start, exec_len;
-        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
-        exec_len = (uint32_t) exec->batch_len;
+        int ret;
 
         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 
-        count = nbox ? nbox : 1;
+        if (IS_I830(dev) || IS_845G(dev)) {
+                ret = intel_ring_begin(ring, 4);
+                if (ret)
+                        return ret;
 
-        for (i = 0; i < count; i++) {
-                if (i < nbox) {
-                        int ret = i915_emit_box(dev, cliprects, i,
-                                                exec->DR1, exec->DR4);
-                        if (ret)
-                                return ret;
-                }
+                intel_ring_emit(ring, MI_BATCH_BUFFER);
+                intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
+                intel_ring_emit(ring, offset + len - 8);
+                intel_ring_emit(ring, 0);
+        } else {
+                ret = intel_ring_begin(ring, 2);
+                if (ret)
+                        return ret;
 
-                if (IS_I830(dev) || IS_845G(dev)) {
-                        intel_ring_begin(dev, ring, 4);
-                        intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
-                        intel_ring_emit(dev, ring,
-                                        exec_start | MI_BATCH_NON_SECURE);
-                        intel_ring_emit(dev, ring, exec_start + exec_len - 4);
-                        intel_ring_emit(dev, ring, 0);
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        intel_ring_emit(ring,
+                                        MI_BATCH_BUFFER_START | (2 << 6) |
+                                        MI_BATCH_NON_SECURE_I965);
+                        intel_ring_emit(ring, offset);
                 } else {
-                        intel_ring_begin(dev, ring, 2);
-                        if (INTEL_INFO(dev)->gen >= 4) {
-                                intel_ring_emit(dev, ring,
-                                                MI_BATCH_BUFFER_START | (2 << 6)
-                                                | MI_BATCH_NON_SECURE_I965);
-                                intel_ring_emit(dev, ring, exec_start);
-                        } else {
-                                intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
-                                                | (2 << 6));
-                                intel_ring_emit(dev, ring, exec_start |
-                                                MI_BATCH_NON_SECURE);
-                        }
+                        intel_ring_emit(ring,
+                                        MI_BATCH_BUFFER_START | (2 << 6));
+                        intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
                 }
-                intel_ring_advance(dev, ring);
-        }
-
-        if (IS_G4X(dev) || IS_GEN5(dev)) {
-                intel_ring_begin(dev, ring, 2);
-                intel_ring_emit(dev, ring, MI_FLUSH |
-                                MI_NO_WRITE_FLUSH |
-                                MI_INVALIDATE_ISP );
-                intel_ring_emit(dev, ring, MI_NOOP);
-                intel_ring_advance(dev, ring);
         }
-        /* XXX breadcrumb */
+        intel_ring_advance(ring);
 
         return 0;
 }
 
-static void cleanup_status_page(struct drm_device *dev,
-                                struct intel_ring_buffer *ring)
+static void cleanup_status_page(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_gem_object *obj;
 
         obj = ring->status_page.obj;
         if (obj == NULL)
                 return;
-        obj_priv = to_intel_bo(obj);
 
-        kunmap(obj_priv->pages[0]);
+        kunmap(obj->pages[0]);
         i915_gem_object_unpin(obj);
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
         ring->status_page.obj = NULL;
 
         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 }
 
-static int init_status_page(struct drm_device *dev,
-                            struct intel_ring_buffer *ring)
+static int init_status_page(struct intel_ring_buffer *ring)
 {
+        struct drm_device *dev = ring->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         obj = i915_gem_alloc_object(dev, 4096);
@@ -538,16 +780,15 @@ static int init_status_page(struct drm_device *dev,
                 ret = -ENOMEM;
                 goto err;
         }
-        obj_priv = to_intel_bo(obj);
-        obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
+        obj->agp_type = AGP_USER_CACHED_MEMORY;
 
-        ret = i915_gem_object_pin(obj, 4096);
+        ret = i915_gem_object_pin(obj, 4096, true);
         if (ret != 0) {
                 goto err_unref;
         }
 
-        ring->status_page.gfx_addr = obj_priv->gtt_offset;
-        ring->status_page.page_addr = kmap(obj_priv->pages[0]);
+        ring->status_page.gfx_addr = obj->gtt_offset;
+        ring->status_page.page_addr = kmap(obj->pages[0]);
         if (ring->status_page.page_addr == NULL) {
                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
                 goto err_unpin;
@@ -555,7 +796,7 @@ static int init_status_page(struct drm_device *dev,
         ring->status_page.obj = obj;
         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 
-        intel_ring_setup_status_page(dev, ring);
+        intel_ring_setup_status_page(ring);
         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
                         ring->name, ring->status_page.gfx_addr);
 
@@ -564,7 +805,7 @@ static int init_status_page(struct drm_device *dev,
 err_unpin:
         i915_gem_object_unpin(obj);
 err_unref:
-        drm_gem_object_unreference(obj);
+        drm_gem_object_unreference(&obj->base);
 err:
         return ret;
 }
@@ -572,9 +813,7 @@ err:
 int intel_init_ring_buffer(struct drm_device *dev,
                            struct intel_ring_buffer *ring)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj;
         int ret;
 
         ring->dev = dev;
@@ -582,8 +821,11 @@ int intel_init_ring_buffer(struct drm_device *dev,
         INIT_LIST_HEAD(&ring->request_list);
         INIT_LIST_HEAD(&ring->gpu_write_list);
 
+        spin_lock_init(&ring->irq_lock);
+        ring->irq_mask = ~0;
+
         if (I915_NEED_GFX_HWS(dev)) {
-                ret = init_status_page(dev, ring);
+                ret = init_status_page(ring);
                 if (ret)
                         return ret;
         }
@@ -595,15 +837,14 @@ int intel_init_ring_buffer(struct drm_device *dev,
                 goto err_hws;
         }
 
-        ring->gem_object = obj;
+        ring->obj = obj;
 
-        ret = i915_gem_object_pin(obj, PAGE_SIZE);
+        ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
         if (ret)
                 goto err_unref;
 
-        obj_priv = to_intel_bo(obj);
         ring->map.size = ring->size;
-        ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
+        ring->map.offset = dev->agp->base + obj->gtt_offset;
         ring->map.type = 0;
         ring->map.flags = 0;
         ring->map.mtrr = 0;
@@ -616,60 +857,64 @@ int intel_init_ring_buffer(struct drm_device *dev,
         }
 
         ring->virtual_start = ring->map.handle;
-        ret = ring->init(dev, ring);
+        ret = ring->init(ring);
         if (ret)
                 goto err_unmap;
 
-        if (!drm_core_check_feature(dev, DRIVER_MODESET))
-                i915_kernel_lost_context(dev);
-        else {
-                ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
-                ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
-        }
-        return ret;
+        /* Workaround an erratum on the i830 which causes a hang if
+         * the TAIL pointer points to within the last 2 cachelines
+         * of the buffer.
+         */
+        ring->effective_size = ring->size;
+        if (IS_I830(ring->dev))
+                ring->effective_size -= 128;
+
+        return 0;
 
 err_unmap:
         drm_core_ioremapfree(&ring->map, dev);
 err_unpin:
         i915_gem_object_unpin(obj);
 err_unref:
-        drm_gem_object_unreference(obj);
-        ring->gem_object = NULL;
+        drm_gem_object_unreference(&obj->base);
+        ring->obj = NULL;
 err_hws:
-        cleanup_status_page(dev, ring);
+        cleanup_status_page(ring);
         return ret;
 }
 
-void intel_cleanup_ring_buffer(struct drm_device *dev,
-                               struct intel_ring_buffer *ring)
+void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 {
-        if (ring->gem_object == NULL)
+        struct drm_i915_private *dev_priv;
+        int ret;
+
+        if (ring->obj == NULL)
                 return;
 
-        drm_core_ioremapfree(&ring->map, dev);
+        /* Disable the ring buffer. The ring must be idle at this point */
+        dev_priv = ring->dev->dev_private;
+        ret = intel_wait_ring_buffer(ring, ring->size - 8);
+        I915_WRITE_CTL(ring, 0);
+
+        drm_core_ioremapfree(&ring->map, ring->dev);
 
-        i915_gem_object_unpin(ring->gem_object);
-        drm_gem_object_unreference(ring->gem_object);
-        ring->gem_object = NULL;
+        i915_gem_object_unpin(ring->obj);
+        drm_gem_object_unreference(&ring->obj->base);
+        ring->obj = NULL;
 
         if (ring->cleanup)
                 ring->cleanup(ring);
 
-        cleanup_status_page(dev, ring);
+        cleanup_status_page(ring);
 }
 
-static int intel_wrap_ring_buffer(struct drm_device *dev,
-                                  struct intel_ring_buffer *ring)
+static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 {
         unsigned int *virt;
-        int rem;
-        rem = ring->size - ring->tail;
+        int rem = ring->size - ring->tail;
 
         if (ring->space < rem) {
-                int ret = intel_wait_ring_buffer(dev, ring, rem);
+                int ret = intel_wait_ring_buffer(ring, rem);
                 if (ret)
                         return ret;
         }
@@ -682,24 +927,25 @@ static int intel_wrap_ring_buffer(struct drm_device *dev,
         }
 
         ring->tail = 0;
-        ring->space = ring->head - 8;
+        ring->space = ring_space(ring);
 
         return 0;
 }
 
-int intel_wait_ring_buffer(struct drm_device *dev,
-                           struct intel_ring_buffer *ring, int n)
+int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 {
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long end;
-        drm_i915_private_t *dev_priv = dev->dev_private;
         u32 head;
 
+        /* If the reported head position has wrapped or hasn't advanced,
+         * fallback to the slow and accurate path.
+         */
         head = intel_read_status_page(ring, 4);
-        if (head) {
-                ring->head = head & HEAD_ADDR;
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
+        if (head > ring->head) {
+                ring->head = head;
+                ring->space = ring_space(ring);
                 if (ring->space >= n)
                         return 0;
         }
@@ -707,12 +953,10 @@ int intel_wait_ring_buffer(struct drm_device *dev,
         trace_i915_ring_wait_begin (dev);
         end = jiffies + 3 * HZ;
         do {
-                ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
-                ring->space = ring->head - (ring->tail + 8);
-                if (ring->space < 0)
-                        ring->space += ring->size;
+                ring->head = I915_READ_HEAD(ring);
+                ring->space = ring_space(ring);
                 if (ring->space >= n) {
-                        trace_i915_ring_wait_end (dev);
+                        trace_i915_ring_wait_end(dev);
                         return 0;
                 }
 
@@ -723,29 +967,39 @@ int intel_wait_ring_buffer(struct drm_device *dev,
                 }
 
                 msleep(1);
+                if (atomic_read(&dev_priv->mm.wedged))
+                        return -EAGAIN;
         } while (!time_after(jiffies, end));
         trace_i915_ring_wait_end (dev);
         return -EBUSY;
 }
 
-void intel_ring_begin(struct drm_device *dev,
-                      struct intel_ring_buffer *ring,
-                      int num_dwords)
+int intel_ring_begin(struct intel_ring_buffer *ring,
+                     int num_dwords)
 {
         int n = 4*num_dwords;
-        if (unlikely(ring->tail + n > ring->size))
-                intel_wrap_ring_buffer(dev, ring);
-        if (unlikely(ring->space < n))
-                intel_wait_ring_buffer(dev, ring, n);
+        int ret;
+
+        if (unlikely(ring->tail + n > ring->effective_size)) {
+                ret = intel_wrap_ring_buffer(ring);
+                if (unlikely(ret))
+                        return ret;
+        }
+
+        if (unlikely(ring->space < n)) {
+                ret = intel_wait_ring_buffer(ring, n);
+                if (unlikely(ret))
+                        return ret;
+        }
 
         ring->space -= n;
+        return 0;
 }
 
-void intel_ring_advance(struct drm_device *dev,
-                        struct intel_ring_buffer *ring)
+void intel_ring_advance(struct intel_ring_buffer *ring)
 {
         ring->tail &= ring->size - 1;
-        ring->write_tail(dev, ring, ring->tail);
+        ring->write_tail(ring, ring->tail);
 }
 
 static const struct intel_ring_buffer render_ring = {
@@ -757,10 +1011,11 @@ static const struct intel_ring_buffer render_ring = {
         .write_tail             = ring_write_tail,
         .flush                  = render_ring_flush,
         .add_request            = render_ring_add_request,
-        .get_seqno              = render_ring_get_seqno,
-        .user_irq_get           = render_ring_get_user_irq,
-        .user_irq_put           = render_ring_put_user_irq,
-        .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
+        .get_seqno              = ring_get_seqno,
+        .irq_get                = render_ring_get_irq,
+        .irq_put                = render_ring_put_irq,
+        .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
+       .cleanup                 = render_ring_cleanup,
 };
 
 /* ring buffer for bit-stream decoder */
@@ -770,22 +1025,21 @@ static const struct intel_ring_buffer bsd_ring = {
         .id                     = RING_BSD,
         .mmio_base              = BSD_RING_BASE,
         .size                   = 32 * PAGE_SIZE,
-        .init                   = init_bsd_ring,
+        .init                   = init_ring_common,
         .write_tail             = ring_write_tail,
         .flush                  = bsd_ring_flush,
         .add_request            = ring_add_request,
-        .get_seqno              = ring_status_page_get_seqno,
-        .user_irq_get           = bsd_ring_get_user_irq,
-        .user_irq_put           = bsd_ring_put_user_irq,
-        .dispatch_gem_execbuffer = ring_dispatch_gem_execbuffer,
+        .get_seqno              = ring_get_seqno,
+        .irq_get                = bsd_ring_get_irq,
+        .irq_put                = bsd_ring_put_irq,
+        .dispatch_execbuffer    = ring_dispatch_execbuffer,
 };
 
 
-static void gen6_bsd_ring_write_tail(struct drm_device *dev,
-                                     struct intel_ring_buffer *ring,
+static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
                                      u32 value)
 {
-       drm_i915_private_t *dev_priv = dev->dev_private;
+       drm_i915_private_t *dev_priv = ring->dev->dev_private;
 
        /* Every tail move must follow the sequence below */
        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
@@ -804,69 +1058,109 @@ static void gen6_bsd_ring_write_tail(struct drm_device *dev,
                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
 }
 
-static void gen6_ring_flush(struct drm_device *dev,
-                            struct intel_ring_buffer *ring,
-                            u32 invalidate_domains,
-                            u32 flush_domains)
+static int gen6_ring_flush(struct intel_ring_buffer *ring,
+                           u32 invalidate_domains,
+                           u32 flush_domains)
 {
-       intel_ring_begin(dev, ring, 4);
-       intel_ring_emit(dev, ring, MI_FLUSH_DW);
-       intel_ring_emit(dev, ring, 0);
-       intel_ring_emit(dev, ring, 0);
-       intel_ring_emit(dev, ring, 0);
-       intel_ring_advance(dev, ring);
+        int ret;
+
+        if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
+                return 0;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_FLUSH_DW);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_advance(ring);
+        return 0;
 }
 
 static int
-gen6_ring_dispatch_gem_execbuffer(struct drm_device *dev,
-                                  struct intel_ring_buffer *ring,
-                                  struct drm_i915_gem_execbuffer2 *exec,
-                                  struct drm_clip_rect *cliprects,
-                                  uint64_t exec_offset)
+gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
+                              u32 offset, u32 len)
 {
-       uint32_t exec_start;
+       int ret;
 
-       exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
+       ret = intel_ring_begin(ring, 2);
+       if (ret)
+               return ret;
 
-       intel_ring_begin(dev, ring, 2);
-       intel_ring_emit(dev, ring,
-                       MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
+       intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
        /* bit0-7 is the length on GEN6+ */
-       intel_ring_emit(dev, ring, exec_start);
-       intel_ring_advance(dev, ring);
+       intel_ring_emit(ring, offset);
+       intel_ring_advance(ring);
 
        return 0;
 }
 
+static bool
+gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_get_irq(ring,
+                                 GT_USER_INTERRUPT,
+                                 GEN6_RENDER_USER_INTERRUPT);
+}
+
+static void
+gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_put_irq(ring,
+                                 GT_USER_INTERRUPT,
+                                 GEN6_RENDER_USER_INTERRUPT);
+}
+
+static bool
+gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_get_irq(ring,
+                                 GT_GEN6_BSD_USER_INTERRUPT,
+                                 GEN6_BSD_USER_INTERRUPT);
+}
+
+static void
+gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        return gen6_ring_put_irq(ring,
+                                 GT_GEN6_BSD_USER_INTERRUPT,
+                                 GEN6_BSD_USER_INTERRUPT);
+}
+
 /* ring buffer for Video Codec for Gen6+ */
 static const struct intel_ring_buffer gen6_bsd_ring = {
-       .name                    = "gen6 bsd ring",
-       .id                      = RING_BSD,
-       .mmio_base               = GEN6_BSD_RING_BASE,
-       .size                    = 32 * PAGE_SIZE,
-       .init                    = init_bsd_ring,
-       .write_tail              = gen6_bsd_ring_write_tail,
-       .flush                   = gen6_ring_flush,
-       .add_request             = ring_add_request,
-       .get_seqno               = ring_status_page_get_seqno,
-       .user_irq_get            = bsd_ring_get_user_irq,
-       .user_irq_put            = bsd_ring_put_user_irq,
-       .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer,
+        .name                   = "gen6 bsd ring",
+        .id                     = RING_BSD,
+        .mmio_base              = GEN6_BSD_RING_BASE,
+        .size                   = 32 * PAGE_SIZE,
+        .init                   = init_ring_common,
+        .write_tail             = gen6_bsd_ring_write_tail,
+        .flush                  = gen6_ring_flush,
+        .add_request            = gen6_add_request,
+        .get_seqno              = ring_get_seqno,
+        .irq_get                = gen6_bsd_ring_get_irq,
+        .irq_put                = gen6_bsd_ring_put_irq,
+        .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
 };
 
 /* Blitter support (SandyBridge+) */
 
-static void
-blt_ring_get_user_irq(struct drm_device *dev,
-                      struct intel_ring_buffer *ring)
+static bool
+blt_ring_get_irq(struct intel_ring_buffer *ring)
 {
-        /* do nothing */
+        return gen6_ring_get_irq(ring,
+                                 GT_BLT_USER_INTERRUPT,
+                                 GEN6_BLITTER_USER_INTERRUPT);
 }
+
 static void
-blt_ring_put_user_irq(struct drm_device *dev,
-                      struct intel_ring_buffer *ring)
+blt_ring_put_irq(struct intel_ring_buffer *ring)
 {
-        /* do nothing */
+        gen6_ring_put_irq(ring,
+                          GT_BLT_USER_INTERRUPT,
+                          GEN6_BLITTER_USER_INTERRUPT);
 }
 
 
@@ -884,32 +1178,31 @@ to_blt_workaround(struct intel_ring_buffer *ring)
         return ring->private;
 }
 
-static int blt_ring_init(struct drm_device *dev,
-                         struct intel_ring_buffer *ring)
+static int blt_ring_init(struct intel_ring_buffer *ring)
 {
-        if (NEED_BLT_WORKAROUND(dev)) {
+        if (NEED_BLT_WORKAROUND(ring->dev)) {
                 struct drm_i915_gem_object *obj;
-                u32 __iomem *ptr;
+                u32 *ptr;
                 int ret;
 
-                obj = to_intel_bo(i915_gem_alloc_object(dev, 4096));
+                obj = i915_gem_alloc_object(ring->dev, 4096);
                 if (obj == NULL)
                         return -ENOMEM;
 
-                ret = i915_gem_object_pin(&obj->base, 4096);
+                ret = i915_gem_object_pin(obj, 4096, true);
                 if (ret) {
                         drm_gem_object_unreference(&obj->base);
                         return ret;
                 }
 
                 ptr = kmap(obj->pages[0]);
-                iowrite32(MI_BATCH_BUFFER_END, ptr);
-                iowrite32(MI_NOOP, ptr+1);
+                *ptr++ = MI_BATCH_BUFFER_END;
+                *ptr++ = MI_NOOP;
                 kunmap(obj->pages[0]);
 
-                ret = i915_gem_object_set_to_gtt_domain(&obj->base, false);
+                ret = i915_gem_object_set_to_gtt_domain(obj, false);
                 if (ret) {
-                        i915_gem_object_unpin(&obj->base);
+                        i915_gem_object_unpin(obj);
                         drm_gem_object_unreference(&obj->base);
                         return ret;
                 }
@@ -917,51 +1210,44 @@ static int blt_ring_init(struct drm_device *dev,
                 ring->private = obj;
         }
 
-        return init_ring_common(dev, ring);
+        return init_ring_common(ring);
 }
 
-static void blt_ring_begin(struct drm_device *dev,
-                           struct intel_ring_buffer *ring,
+static int blt_ring_begin(struct intel_ring_buffer *ring,
                           int num_dwords)
 {
         if (ring->private) {
-                intel_ring_begin(dev, ring, num_dwords+2);
-                intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START);
-                intel_ring_emit(dev, ring, to_blt_workaround(ring)->gtt_offset);
+                int ret = intel_ring_begin(ring, num_dwords+2);
+                if (ret)
+                        return ret;
+
+                intel_ring_emit(ring, MI_BATCH_BUFFER_START);
+                intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
+
+                return 0;
         } else
-                intel_ring_begin(dev, ring, 4);
+                return intel_ring_begin(ring, 4);
 }
 
-static void blt_ring_flush(struct drm_device *dev,
-                           struct intel_ring_buffer *ring,
+static int blt_ring_flush(struct intel_ring_buffer *ring,
                            u32 invalidate_domains,
                            u32 flush_domains)
 {
-        blt_ring_begin(dev, ring, 4);
-        intel_ring_emit(dev, ring, MI_FLUSH_DW);
-        intel_ring_emit(dev, ring, 0);
-        intel_ring_emit(dev, ring, 0);
-        intel_ring_emit(dev, ring, 0);
-        intel_ring_advance(dev, ring);
-}
+        int ret;
 
-static u32
-blt_ring_add_request(struct drm_device *dev,
-                     struct intel_ring_buffer *ring,
-                     u32 flush_domains)
-{
-        u32 seqno = i915_gem_get_seqno(dev);
+        if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
+                return 0;
 
-        blt_ring_begin(dev, ring, 4);
-        intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
-        intel_ring_emit(dev, ring,
-                        I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        intel_ring_emit(dev, ring, seqno);
-        intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
-        intel_ring_advance(dev, ring);
+        ret = blt_ring_begin(ring, 4);
+        if (ret)
+                return ret;
 
-        DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
-        return seqno;
+        intel_ring_emit(ring, MI_FLUSH_DW);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_advance(ring);
+        return 0;
 }
 
 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
@@ -982,47 +1268,98 @@ static const struct intel_ring_buffer gen6_blt_ring = {
        .init                    = blt_ring_init,
        .write_tail              = ring_write_tail,
        .flush                   = blt_ring_flush,
-       .add_request             = blt_ring_add_request,
-       .get_seqno               = ring_status_page_get_seqno,
-       .user_irq_get            = blt_ring_get_user_irq,
-       .user_irq_put            = blt_ring_put_user_irq,
-       .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer,
+       .add_request             = gen6_add_request,
+       .get_seqno               = ring_get_seqno,
+       .irq_get                 = blt_ring_get_irq,
+       .irq_put                 = blt_ring_put_irq,
+       .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
        .cleanup                 = blt_ring_cleanup,
 };
 
 int intel_init_render_ring_buffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-
-        dev_priv->render_ring = render_ring;
+        struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
+
+        *ring = render_ring;
+        if (INTEL_INFO(dev)->gen >= 6) {
+                ring->add_request = gen6_add_request;
+                ring->irq_get = gen6_render_ring_get_irq;
+                ring->irq_put = gen6_render_ring_put_irq;
+        } else if (IS_GEN5(dev)) {
+                ring->add_request = pc_render_add_request;
+                ring->get_seqno = pc_render_get_seqno;
+        }
 
         if (!I915_NEED_GFX_HWS(dev)) {
-                dev_priv->render_ring.status_page.page_addr
-                        = dev_priv->status_page_dmah->vaddr;
-                memset(dev_priv->render_ring.status_page.page_addr,
-                                0, PAGE_SIZE);
+                ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
+                memset(ring->status_page.page_addr, 0, PAGE_SIZE);
+        }
+
+        return intel_init_ring_buffer(dev, ring);
+}
+
+int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
+
+        *ring = render_ring;
+        if (INTEL_INFO(dev)->gen >= 6) {
+                ring->add_request = gen6_add_request;
+                ring->irq_get = gen6_render_ring_get_irq;
+                ring->irq_put = gen6_render_ring_put_irq;
+        } else if (IS_GEN5(dev)) {
+                ring->add_request = pc_render_add_request;
+                ring->get_seqno = pc_render_get_seqno;
         }
 
-        return intel_init_ring_buffer(dev, &dev_priv->render_ring);
+        ring->dev = dev;
+        INIT_LIST_HEAD(&ring->active_list);
+        INIT_LIST_HEAD(&ring->request_list);
+        INIT_LIST_HEAD(&ring->gpu_write_list);
+
+        ring->size = size;
+        ring->effective_size = ring->size;
+        if (IS_I830(ring->dev))
+                ring->effective_size -= 128;
+
+        ring->map.offset = start;
+        ring->map.size = size;
+        ring->map.type = 0;
+        ring->map.flags = 0;
+        ring->map.mtrr = 0;
+
+        drm_core_ioremap_wc(&ring->map, dev);
+        if (ring->map.handle == NULL) {
+                DRM_ERROR("can not ioremap virtual address for"
+                          " ring buffer\n");
+                return -ENOMEM;
+        }
+
+        ring->virtual_start = (void __force __iomem *)ring->map.handle;
+        return 0;
 }
 
 int intel_init_bsd_ring_buffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
 
         if (IS_GEN6(dev))
-                dev_priv->bsd_ring = gen6_bsd_ring;
+                *ring = gen6_bsd_ring;
         else
-                dev_priv->bsd_ring = bsd_ring;
+                *ring = bsd_ring;
 
-        return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
+        return intel_init_ring_buffer(dev, ring);
 }
 
 int intel_init_blt_ring_buffer(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
 
-        dev_priv->blt_ring = gen6_blt_ring;
+        *ring = gen6_blt_ring;
 
-        return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
+        return intel_init_ring_buffer(dev, ring);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 3126c2681983..6d6fde85a636 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -1,22 +1,40 @@
 #ifndef _INTEL_RINGBUFFER_H_
 #define _INTEL_RINGBUFFER_H_
 
+enum {
+    RCS = 0x0,
+    VCS,
+    BCS,
+    I915_NUM_RINGS,
+};
+
 struct  intel_hw_status_page {
-        void            *page_addr;
+        u32     __iomem *page_addr;
         unsigned int    gfx_addr;
-        struct          drm_gem_object *obj;
+        struct          drm_i915_gem_object *obj;
 };
 
-#define I915_READ_TAIL(ring) I915_READ(RING_TAIL(ring->mmio_base))
-#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL(ring->mmio_base), val)
-#define I915_READ_START(ring) I915_READ(RING_START(ring->mmio_base))
-#define I915_WRITE_START(ring, val) I915_WRITE(RING_START(ring->mmio_base), val)
-#define I915_READ_HEAD(ring) I915_READ(RING_HEAD(ring->mmio_base))
-#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD(ring->mmio_base), val)
-#define I915_READ_CTL(ring) I915_READ(RING_CTL(ring->mmio_base))
-#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val)
+#define I915_RING_READ(reg) i915_safe_read(dev_priv, reg)
+
+#define I915_READ_TAIL(ring) I915_RING_READ(RING_TAIL((ring)->mmio_base))
+#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
+
+#define I915_READ_START(ring) I915_RING_READ(RING_START((ring)->mmio_base))
+#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
+
+#define I915_READ_HEAD(ring)  I915_RING_READ(RING_HEAD((ring)->mmio_base))
+#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
+
+#define I915_READ_CTL(ring) I915_RING_READ(RING_CTL((ring)->mmio_base))
+#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
+
+#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
+#define I915_READ_IMR(ring) I915_RING_READ(RING_IMR((ring)->mmio_base))
+
+#define I915_READ_NOPID(ring) I915_RING_READ(RING_NOPID((ring)->mmio_base))
+#define I915_READ_SYNC_0(ring) I915_RING_READ(RING_SYNC_0((ring)->mmio_base))
+#define I915_READ_SYNC_1(ring) I915_RING_READ(RING_SYNC_1((ring)->mmio_base))
 
-struct drm_i915_gem_execbuffer2;
 struct  intel_ring_buffer {
         const char      *name;
         enum intel_ring_id {
@@ -25,44 +43,38 @@ struct  intel_ring_buffer {
                 RING_BLT = 0x4,
         } id;
         u32             mmio_base;
-        unsigned long   size;
         void            *virtual_start;
         struct          drm_device *dev;
-        struct          drm_gem_object *gem_object;
+        struct          drm_i915_gem_object *obj;
 
-        unsigned int    head;
-        unsigned int    tail;
+        u32             head;
+        u32             tail;
         int             space;
+        int             size;
+        int             effective_size;
         struct intel_hw_status_page status_page;
 
-        u32             irq_gem_seqno;          /* last seq seem at irq time */
-        u32             waiting_gem_seqno;
-        int             user_irq_refcount;
-        void            (*user_irq_get)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
-        void            (*user_irq_put)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
+        spinlock_t      irq_lock;
+        u32             irq_refcount;
+        u32             irq_mask;
+        u32             irq_seqno;              /* last seq seem at irq time */
+        u32             waiting_seqno;
+        u32             sync_seqno[I915_NUM_RINGS-1];
+        bool __must_check (*irq_get)(struct intel_ring_buffer *ring);
+        void            (*irq_put)(struct intel_ring_buffer *ring);
 
-        int             (*init)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring);
+        int             (*init)(struct intel_ring_buffer *ring);
 
-        void            (*write_tail)(struct drm_device *dev,
-                                      struct intel_ring_buffer *ring,
+        void            (*write_tail)(struct intel_ring_buffer *ring,
                                       u32 value);
-        void            (*flush)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        u32     invalidate_domains,
-                        u32     flush_domains);
-        u32             (*add_request)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        u32 flush_domains);
-        u32             (*get_seqno)(struct drm_device *dev,
-                                     struct intel_ring_buffer *ring);
-        int             (*dispatch_gem_execbuffer)(struct drm_device *dev,
-                        struct intel_ring_buffer *ring,
-                        struct drm_i915_gem_execbuffer2 *exec,
-                        struct drm_clip_rect *cliprects,
-                        uint64_t exec_offset);
+        int __must_check (*flush)(struct intel_ring_buffer *ring,
+                                  u32   invalidate_domains,
+                                  u32   flush_domains);
+        int             (*add_request)(struct intel_ring_buffer *ring,
+                                       u32 *seqno);
+        u32             (*get_seqno)(struct intel_ring_buffer *ring);
+        int             (*dispatch_execbuffer)(struct intel_ring_buffer *ring,
+                                               u32 offset, u32 length);
         void            (*cleanup)(struct intel_ring_buffer *ring);
 
         /**
@@ -95,7 +107,7 @@ struct  intel_ring_buffer {
         /**
          * Do we have some not yet emitted requests outstanding?
          */
-        bool outstanding_lazy_request;
+        u32 outstanding_lazy_request;
 
         wait_queue_head_t irq_queue;
         drm_local_map_t map;
@@ -104,44 +116,57 @@ struct  intel_ring_buffer {
 };
 
 static inline u32
+intel_ring_sync_index(struct intel_ring_buffer *ring,
+                      struct intel_ring_buffer *other)
+{
+        int idx;
+
+        /*
+         * cs -> 0 = vcs, 1 = bcs
+         * vcs -> 0 = bcs, 1 = cs,
+         * bcs -> 0 = cs, 1 = vcs.
+         */
+
+        idx = (other - ring) - 1;
+        if (idx < 0)
+                idx += I915_NUM_RINGS;
+
+        return idx;
+}
+
+static inline u32
 intel_read_status_page(struct intel_ring_buffer *ring,
-                int reg)
+                       int reg)
 {
-        u32 *regs = ring->status_page.page_addr;
-        return regs[reg];
+        return ioread32(ring->status_page.page_addr + reg);
 }
 
-int intel_init_ring_buffer(struct drm_device *dev,
-                           struct intel_ring_buffer *ring);
-void intel_cleanup_ring_buffer(struct drm_device *dev,
-                               struct intel_ring_buffer *ring);
-int intel_wait_ring_buffer(struct drm_device *dev,
-                           struct intel_ring_buffer *ring, int n);
-void intel_ring_begin(struct drm_device *dev,
-                      struct intel_ring_buffer *ring, int n);
-
-static inline void intel_ring_emit(struct drm_device *dev,
-                                   struct intel_ring_buffer *ring,
-                                   unsigned int data)
+void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring);
+int __must_check intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n);
+int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n);
+
+static inline void intel_ring_emit(struct intel_ring_buffer *ring,
+                                   u32 data)
 {
-        unsigned int *virt = ring->virtual_start + ring->tail;
-        *virt = data;
+        iowrite32(data, ring->virtual_start + ring->tail);
         ring->tail += 4;
 }
 
-void intel_ring_advance(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
+void intel_ring_advance(struct intel_ring_buffer *ring);
 
-u32 intel_ring_get_seqno(struct drm_device *dev,
-                struct intel_ring_buffer *ring);
+u32 intel_ring_get_seqno(struct intel_ring_buffer *ring);
+int intel_ring_sync(struct intel_ring_buffer *ring,
+                    struct intel_ring_buffer *to,
+                    u32 seqno);
 
 int intel_init_render_ring_buffer(struct drm_device *dev);
 int intel_init_bsd_ring_buffer(struct drm_device *dev);
 int intel_init_blt_ring_buffer(struct drm_device *dev);
 
-u32 intel_ring_get_active_head(struct drm_device *dev,
-                               struct intel_ring_buffer *ring);
-void intel_ring_setup_status_page(struct drm_device *dev,
-                                  struct intel_ring_buffer *ring);
+u32 intel_ring_get_active_head(struct intel_ring_buffer *ring);
+void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
+
+/* DRI warts */
+int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size);
 
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index de158b76bcd5..318f398e6b2e 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -107,7 +107,8 @@ struct intel_sdvo {
          * This is set if we treat the device as HDMI, instead of DVI.
          */
         bool is_hdmi;
-        bool has_audio;
+        bool has_hdmi_monitor;
+        bool has_hdmi_audio;
 
         /**
          * This is set if we detect output of sdvo device as LVDS and
@@ -472,20 +473,6 @@ static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
                 return false;
         }
 
-        i = 3;
-        while (status == SDVO_CMD_STATUS_PENDING && i--) {
-                if (!intel_sdvo_read_byte(intel_sdvo,
-                                          SDVO_I2C_CMD_STATUS,
-                                          &status))
-                        return false;
-        }
-        if (status != SDVO_CMD_STATUS_SUCCESS) {
-                DRM_DEBUG_KMS("command returns response %s [%d]\n",
-                              status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP ? cmd_status_names[status] : "???",
-                              status);
-                return false;
-        }
-
         return true;
 }
 
@@ -496,6 +483,8 @@ static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo,
         u8 status;
         int i;
 
+        DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(intel_sdvo));
+
         /*
          * The documentation states that all commands will be
          * processed within 15µs, and that we need only poll
@@ -504,14 +493,19 @@ static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo,
          *
          * Check 5 times in case the hardware failed to read the docs.
          */
-        do {
+        if (!intel_sdvo_read_byte(intel_sdvo,
+                                  SDVO_I2C_CMD_STATUS,
+                                  &status))
+                goto log_fail;
+
+        while (status == SDVO_CMD_STATUS_PENDING && retry--) {
+                udelay(15);
                 if (!intel_sdvo_read_byte(intel_sdvo,
                                           SDVO_I2C_CMD_STATUS,
                                           &status))
-                        return false;
-        } while (status == SDVO_CMD_STATUS_PENDING && --retry);
+                        goto log_fail;
+        }
 
-        DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(intel_sdvo));
         if (status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP)
                 DRM_LOG_KMS("(%s)", cmd_status_names[status]);
         else
@@ -532,7 +526,7 @@ static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo,
         return true;
 
 log_fail:
-        DRM_LOG_KMS("\n");
+        DRM_LOG_KMS("... failed\n");
         return false;
 }
 
@@ -549,6 +543,7 @@ static int intel_sdvo_get_pixel_multiplier(struct drm_display_mode *mode)
 static bool intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo,
                                               u8 ddc_bus)
 {
+        /* This must be the immediately preceding write before the i2c xfer */
         return intel_sdvo_write_cmd(intel_sdvo,
                                     SDVO_CMD_SET_CONTROL_BUS_SWITCH,
                                     &ddc_bus, 1);
@@ -556,7 +551,10 @@ static bool intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo,
 
 static bool intel_sdvo_set_value(struct intel_sdvo *intel_sdvo, u8 cmd, const void *data, int len)
 {
-        return intel_sdvo_write_cmd(intel_sdvo, cmd, data, len);
+        if (!intel_sdvo_write_cmd(intel_sdvo, cmd, data, len))
+                return false;
+
+        return intel_sdvo_read_response(intel_sdvo, NULL, 0);
 }
 
 static bool
@@ -858,18 +856,21 @@ static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo)
 
         intel_dip_infoframe_csum(&avi_if);
 
-        if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_INDEX,
+        if (!intel_sdvo_set_value(intel_sdvo,
+                                  SDVO_CMD_SET_HBUF_INDEX,
                                   set_buf_index, 2))
                 return false;
 
         for (i = 0; i < sizeof(avi_if); i += 8) {
-                if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_DATA,
+                if (!intel_sdvo_set_value(intel_sdvo,
+                                          SDVO_CMD_SET_HBUF_DATA,
                                           data, 8))
                         return false;
                 data++;
         }
 
-        return intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_TXRATE,
+        return intel_sdvo_set_value(intel_sdvo,
+                                    SDVO_CMD_SET_HBUF_TXRATE,
                                     &tx_rate, 1);
 }
 
@@ -1023,9 +1024,13 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
         if (!intel_sdvo_set_target_input(intel_sdvo))
                 return;
 
-        if (intel_sdvo->is_hdmi &&
-            !intel_sdvo_set_avi_infoframe(intel_sdvo))
-                return;
+        if (intel_sdvo->has_hdmi_monitor) {
+                intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI);
+                intel_sdvo_set_colorimetry(intel_sdvo,
+                                           SDVO_COLORIMETRY_RGB256);
+                intel_sdvo_set_avi_infoframe(intel_sdvo);
+        } else
+                intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI);
 
         if (intel_sdvo->is_tv &&
             !intel_sdvo_set_tv_format(intel_sdvo))
@@ -1044,7 +1049,9 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
 
         /* Set the SDVO control regs. */
         if (INTEL_INFO(dev)->gen >= 4) {
-                sdvox = SDVO_BORDER_ENABLE;
+                sdvox = 0;
+                if (INTEL_INFO(dev)->gen < 5)
+                        sdvox |= SDVO_BORDER_ENABLE;
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                         sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -1063,7 +1070,7 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
         }
         if (intel_crtc->pipe == 1)
                 sdvox |= SDVO_PIPE_B_SELECT;
-        if (intel_sdvo->has_audio)
+        if (intel_sdvo->has_hdmi_audio)
                 sdvox |= SDVO_AUDIO_ENABLE;
 
         if (INTEL_INFO(dev)->gen >= 4) {
@@ -1074,7 +1081,8 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
                 sdvox |= (pixel_multiplier - 1) << SDVO_PORT_MULTIPLY_SHIFT;
         }
 
-        if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL)
+        if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL &&
+            INTEL_INFO(dev)->gen < 5)
                 sdvox |= SDVO_STALL_SELECT;
         intel_sdvo_write_sdvox(intel_sdvo, sdvox);
 }
@@ -1295,55 +1303,14 @@ intel_sdvo_get_edid(struct drm_connector *connector)
         return drm_get_edid(connector, &sdvo->ddc);
 }
 
-static struct drm_connector *
-intel_find_analog_connector(struct drm_device *dev)
-{
-        struct drm_connector *connector;
-        struct intel_sdvo *encoder;
-
-        list_for_each_entry(encoder,
-                            &dev->mode_config.encoder_list,
-                            base.base.head) {
-                if (encoder->base.type == INTEL_OUTPUT_ANALOG) {
-                        list_for_each_entry(connector,
-                                            &dev->mode_config.connector_list,
-                                            head) {
-                                if (&encoder->base ==
-                                    intel_attached_encoder(connector))
-                                        return connector;
-                        }
-                }
-        }
-
-        return NULL;
-}
-
-static int
-intel_analog_is_connected(struct drm_device *dev)
-{
-        struct drm_connector *analog_connector;
-
-        analog_connector = intel_find_analog_connector(dev);
-        if (!analog_connector)
-                return false;
-
-        if (analog_connector->funcs->detect(analog_connector, false) ==
-                        connector_status_disconnected)
-                return false;
-
-        return true;
-}
-
 /* Mac mini hack -- use the same DDC as the analog connector */
 static struct edid *
 intel_sdvo_get_analog_edid(struct drm_connector *connector)
 {
         struct drm_i915_private *dev_priv = connector->dev->dev_private;
 
-        if (!intel_analog_is_connected(connector->dev))
-                return NULL;
-
-        return drm_get_edid(connector, &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
+        return drm_get_edid(connector,
+                            &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
 }
 
 enum drm_connector_status
@@ -1388,8 +1355,10 @@ intel_sdvo_hdmi_sink_detect(struct drm_connector *connector)
                 /* DDC bus is shared, match EDID to connector type */
                 if (edid->input & DRM_EDID_INPUT_DIGITAL) {
                         status = connector_status_connected;
-                        intel_sdvo->is_hdmi = drm_detect_hdmi_monitor(edid);
-                        intel_sdvo->has_audio = drm_detect_monitor_audio(edid);
+                        if (intel_sdvo->is_hdmi) {
+                                intel_sdvo->has_hdmi_monitor = drm_detect_hdmi_monitor(edid);
+                                intel_sdvo->has_hdmi_audio = drm_detect_monitor_audio(edid);
+                        }
                 }
                 connector->display_info.raw_edid = NULL;
                 kfree(edid);
@@ -1398,7 +1367,7 @@ intel_sdvo_hdmi_sink_detect(struct drm_connector *connector)
         if (status == connector_status_connected) {
                 struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
                 if (intel_sdvo_connector->force_audio)
-                        intel_sdvo->has_audio = intel_sdvo_connector->force_audio > 0;
+                        intel_sdvo->has_hdmi_audio = intel_sdvo_connector->force_audio > 0;
         }
 
         return status;
@@ -1415,10 +1384,12 @@ intel_sdvo_detect(struct drm_connector *connector, bool force)
         if (!intel_sdvo_write_cmd(intel_sdvo,
                                   SDVO_CMD_GET_ATTACHED_DISPLAYS, NULL, 0))
                 return connector_status_unknown;
-        if (intel_sdvo->is_tv) {
-                /* add 30ms delay when the output type is SDVO-TV */
+
+        /* add 30ms delay when the output type might be TV */
+        if (intel_sdvo->caps.output_flags &
+            (SDVO_OUTPUT_SVID0 | SDVO_OUTPUT_CVBS0))
                 mdelay(30);
-        }
+
         if (!intel_sdvo_read_response(intel_sdvo, &response, 2))
                 return connector_status_unknown;
 
@@ -1431,6 +1402,9 @@ intel_sdvo_detect(struct drm_connector *connector, bool force)
 
         intel_sdvo->attached_output = response;
 
+        intel_sdvo->has_hdmi_monitor = false;
+        intel_sdvo->has_hdmi_audio = false;
+
         if ((intel_sdvo_connector->output_flag & response) == 0)
                 ret = connector_status_disconnected;
         else if (response & SDVO_TMDS_MASK)
@@ -1472,8 +1446,10 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector)
                 edid = intel_sdvo_get_analog_edid(connector);
 
         if (edid != NULL) {
-                drm_mode_connector_update_edid_property(connector, edid);
-                drm_add_edid_modes(connector, edid);
+                if (edid->input & DRM_EDID_INPUT_DIGITAL) {
+                        drm_mode_connector_update_edid_property(connector, edid);
+                        drm_add_edid_modes(connector, edid);
+                }
                 connector->display_info.raw_edid = NULL;
                 kfree(edid);
         }
@@ -1484,7 +1460,7 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector)
  * Note!  This is in reply order (see loop in get_tv_modes).
  * XXX: all 60Hz refresh?
  */
-struct drm_display_mode sdvo_tv_modes[] = {
+static const struct drm_display_mode sdvo_tv_modes[] = {
         { DRM_MODE("320x200", DRM_MODE_TYPE_DRIVER, 5815, 320, 321, 384,
                    416, 0, 200, 201, 232, 233, 0,
                    DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
@@ -1713,12 +1689,12 @@ intel_sdvo_set_property(struct drm_connector *connector,
 
                 intel_sdvo_connector->force_audio = val;
 
-                if (val > 0 && intel_sdvo->has_audio)
+                if (val > 0 && intel_sdvo->has_hdmi_audio)
                         return 0;
-                if (val < 0 && !intel_sdvo->has_audio)
+                if (val < 0 && !intel_sdvo->has_hdmi_audio)
                         return 0;
 
-                intel_sdvo->has_audio = val > 0;
+                intel_sdvo->has_hdmi_audio = val > 0;
                 goto done;
         }
 
@@ -1942,28 +1918,18 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
                 speed = mapping->i2c_speed;
         }
 
-        sdvo->i2c = &dev_priv->gmbus[pin].adapter;
-        intel_gmbus_set_speed(sdvo->i2c, speed);
-        intel_gmbus_force_bit(sdvo->i2c, true);
+        if (pin < GMBUS_NUM_PORTS) {
+                sdvo->i2c = &dev_priv->gmbus[pin].adapter;
+                intel_gmbus_set_speed(sdvo->i2c, speed);
+                intel_gmbus_force_bit(sdvo->i2c, true);
+        } else
+                sdvo->i2c = &dev_priv->gmbus[GMBUS_PORT_DPB].adapter;
 }
 
 static bool
 intel_sdvo_is_hdmi_connector(struct intel_sdvo *intel_sdvo, int device)
 {
-        int is_hdmi;
-
-        if (!intel_sdvo_check_supp_encode(intel_sdvo))
-                return false;
-
-        if (!intel_sdvo_set_target_output(intel_sdvo,
-                                          device == 0 ? SDVO_OUTPUT_TMDS0 : SDVO_OUTPUT_TMDS1))
-                return false;
-
-        is_hdmi = 0;
-        if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ENCODE, &is_hdmi, 1))
-                return false;
-
-        return !!is_hdmi;
+        return intel_sdvo_check_supp_encode(intel_sdvo);
 }
 
 static u8
@@ -2065,10 +2031,6 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
         connector->connector_type = DRM_MODE_CONNECTOR_DVID;
 
         if (intel_sdvo_is_hdmi_connector(intel_sdvo, device)) {
-                /* enable hdmi encoding mode if supported */
-                intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI);
-                intel_sdvo_set_colorimetry(intel_sdvo,
-                                           SDVO_COLORIMETRY_RGB256);
                 connector->connector_type = DRM_MODE_CONNECTOR_HDMIA;
                 intel_sdvo->is_hdmi = true;
         }
@@ -2076,8 +2038,8 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
                                        (1 << INTEL_ANALOG_CLONE_BIT));
 
         intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
-
-        intel_sdvo_add_hdmi_properties(intel_sdvo_connector);
+        if (intel_sdvo->is_hdmi)
+                intel_sdvo_add_hdmi_properties(intel_sdvo_connector);
 
         return true;
 }
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 2f7681989316..93206e4eaa6f 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1245,10 +1245,11 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
         int type;
 
         /* Disable TV interrupts around load detect or we'll recurse */
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        i915_disable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE |
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        i915_disable_pipestat(dev_priv, 0,
+                              PIPE_HOTPLUG_INTERRUPT_ENABLE |
                               PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         save_tv_dac = tv_dac = I915_READ(TV_DAC);
         save_tv_ctl = tv_ctl = I915_READ(TV_CTL);
@@ -1301,10 +1302,11 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
         I915_WRITE(TV_CTL, save_tv_ctl);
 
         /* Restore interrupt config */
-        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE |
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        i915_enable_pipestat(dev_priv, 0,
+                             PIPE_HOTPLUG_INTERRUPT_ENABLE |
                              PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
-        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return type;
 }
diff --git a/drivers/gpu/drm/mga/mga_dma.c b/drivers/gpu/drm/mga/mga_dma.c
index 08868ac3048a..1e1eb1d7e971 100644
--- a/drivers/gpu/drm/mga/mga_dma.c
+++ b/drivers/gpu/drm/mga/mga_dma.c
@@ -703,7 +703,7 @@ static int mga_do_pci_dma_bootstrap(struct drm_device *dev,
 static int mga_do_dma_bootstrap(struct drm_device *dev,
                                 drm_mga_dma_bootstrap_t *dma_bs)
 {
-        const int is_agp = (dma_bs->agp_mode != 0) && drm_device_is_agp(dev);
+        const int is_agp = (dma_bs->agp_mode != 0) && drm_pci_device_is_agp(dev);
         int err;
         drm_mga_private_t *const dev_priv =
             (drm_mga_private_t *) dev->dev_private;
diff --git a/drivers/gpu/drm/mga/mga_drv.c b/drivers/gpu/drm/mga/mga_drv.c
index 0aaf5f67a436..42d31874edf2 100644
--- a/drivers/gpu/drm/mga/mga_drv.c
+++ b/drivers/gpu/drm/mga/mga_drv.c
@@ -75,10 +75,6 @@ static struct drm_driver driver = {
 #endif
                 .llseek = noop_llseek,
         },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-        },
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -88,15 +84,20 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver mga_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init mga_init(void)
 {
         driver.num_ioctls = mga_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &mga_pci_driver);
 }
 
 static void __exit mga_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &mga_pci_driver);
 }
 
 module_init(mga_init);
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index 21d6c29c2d21..de70959b9ed5 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -8,7 +8,7 @@ config DRM_NOUVEAU
         select FB_CFB_COPYAREA
         select FB_CFB_IMAGEBLIT
         select FB
-        select FRAMEBUFFER_CONSOLE if !EMBEDDED
+        select FRAMEBUFFER_CONSOLE if !EXPERT
         select FB_BACKLIGHT if DRM_NOUVEAU_BACKLIGHT
         select ACPI_VIDEO if ACPI && X86 && BACKLIGHT_CLASS_DEVICE && VIDEO_OUTPUT_CONTROL && INPUT
         help
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index 119152606e4c..a54238058dc5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -130,10 +130,15 @@ static int nouveau_dsm_init(void)
 
 static int nouveau_dsm_get_client_id(struct pci_dev *pdev)
 {
-        if (nouveau_dsm_priv.dhandle == DEVICE_ACPI_HANDLE(&pdev->dev))
+        /* easy option one - intel vendor ID means Integrated */
+        if (pdev->vendor == PCI_VENDOR_ID_INTEL)
                 return VGA_SWITCHEROO_IGD;
-        else
-                return VGA_SWITCHEROO_DIS;
+
+        /* is this device on Bus 0? - this may need improving */
+        if (pdev->bus->number == 0)
+                return VGA_SWITCHEROO_IGD;
+
+        return VGA_SWITCHEROO_DIS;
 }
 
 static struct vga_switcheroo_handler nouveau_dsm_handler = {
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index b14c81110575..d3a9c6e02477 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -59,7 +59,7 @@ static int nv40_set_intensity(struct backlight_device *bd)
         return 0;
 }
 
-static struct backlight_ops nv40_bl_ops = {
+static const struct backlight_ops nv40_bl_ops = {
         .options = BL_CORE_SUSPENDRESUME,
         .get_brightness = nv40_get_intensity,
         .update_status = nv40_set_intensity,
@@ -82,7 +82,7 @@ static int nv50_set_intensity(struct backlight_device *bd)
         return 0;
 }
 
-static struct backlight_ops nv50_bl_ops = {
+static const struct backlight_ops nv50_bl_ops = {
         .options = BL_CORE_SUSPENDRESUME,
         .get_brightness = nv50_get_intensity,
         .update_status = nv50_set_intensity,
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 6faf3cfc74b9..6bdab891c64e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -1927,7 +1927,7 @@ init_ltime(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
          * offset      (8  bit): opcode
          * offset + 1  (16 bit): time
          *
-         * Sleep for "time" miliseconds.
+         * Sleep for "time" milliseconds.
          */
 
         unsigned time = ROM16(bios->data[offset + 1]);
@@ -1935,7 +1935,7 @@ init_ltime(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
         if (!iexec->execute)
                 return 3;
 
-        BIOSLOG(bios, "0x%04X: Sleeping for 0x%04X miliseconds\n",
+        BIOSLOG(bios, "0x%04X: Sleeping for 0x%04X milliseconds\n",
                 offset, time);
 
         msleep(time);
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.c b/drivers/gpu/drm/nouveau/nouveau_drv.c
index bb170570938b..155ebdcbf06f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.c
@@ -171,6 +171,9 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
         if (pm_state.event == PM_EVENT_PRETHAW)
                 return 0;
 
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         NV_INFO(dev, "Disabling fbcon acceleration...\n");
         nouveau_fbcon_save_disable_accel(dev);
 
@@ -231,9 +234,9 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
                 pci_set_power_state(pdev, PCI_D3hot);
         }
 
-        acquire_console_sem();
+        console_lock();
         nouveau_fbcon_set_suspend(dev, 1);
-        release_console_sem();
+        console_unlock();
         nouveau_fbcon_restore_accel(dev);
         return 0;
 
@@ -254,6 +257,9 @@ nouveau_pci_resume(struct pci_dev *pdev)
         struct drm_crtc *crtc;
         int ret, i;
 
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
+
         nouveau_fbcon_save_disable_accel(dev);
 
         NV_INFO(dev, "We're back, enabling device...\n");
@@ -353,9 +359,9 @@ nouveau_pci_resume(struct pci_dev *pdev)
                 nv_crtc->lut.depth = 0;
         }
 
-        acquire_console_sem();
+        console_lock();
         nouveau_fbcon_set_suspend(dev, 0);
-        release_console_sem();
+        console_unlock();
 
         nouveau_fbcon_zfill_all(dev);
 
@@ -402,14 +408,6 @@ static struct drm_driver driver = {
 #endif
                 .llseek = noop_llseek,
         },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-                .probe = nouveau_pci_probe,
-                .remove = nouveau_pci_remove,
-                .suspend = nouveau_pci_suspend,
-                .resume = nouveau_pci_resume
-        },
 
         .gem_init_object = nouveau_gem_object_new,
         .gem_free_object = nouveau_gem_object_del,
@@ -426,6 +424,15 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver nouveau_pci_driver = {
+                .name = DRIVER_NAME,
+                .id_table = pciidlist,
+                .probe = nouveau_pci_probe,
+                .remove = nouveau_pci_remove,
+                .suspend = nouveau_pci_suspend,
+                .resume = nouveau_pci_resume
+};
+
 static int __init nouveau_init(void)
 {
         driver.num_ioctls = nouveau_max_ioctl;
@@ -443,7 +450,7 @@ static int __init nouveau_init(void)
                 return 0;
 
         nouveau_register_dsm_handler();
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &nouveau_pci_driver);
 }
 
 static void __exit nouveau_exit(void)
@@ -451,7 +458,7 @@ static void __exit nouveau_exit(void)
         if (!nouveau_modeset)
                 return;
 
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &nouveau_pci_driver);
         nouveau_unregister_dsm_handler();
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index ca707d18d06e..9821fcacc3d2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -754,6 +754,8 @@ struct drm_nouveau_private {
 
         struct nouveau_fbdev *nfbdev;
         struct apertures_struct *apertures;
+
+        bool powered_down;
 };
 
 static inline struct drm_nouveau_private *
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index f988362c112e..60769d2f9a66 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -362,10 +362,6 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
         drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
         drm_fb_helper_fill_var(info, &nfbdev->helper, sizes->fb_width, sizes->fb_height);
 
-        /* FIXME: we really shouldn't expose mmio space at all */
-        info->fix.mmio_start = pci_resource_start(pdev, 1);
-        info->fix.mmio_len = pci_resource_len(pdev, 1);
-
         /* Set aperture base/size for vesafb takeover */
         info->apertures = dev_priv->apertures;
         if (!info->apertures) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 26347b7cd872..123969dd4f56 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -480,7 +480,7 @@ nouveau_mem_gart_init(struct drm_device *dev)
         dev_priv->gart_info.type = NOUVEAU_GART_NONE;
 
 #if !defined(__powerpc__) && !defined(__ia64__)
-        if (drm_device_is_agp(dev) && dev->agp && nouveau_agpmode) {
+        if (drm_pci_device_is_agp(dev) && dev->agp && nouveau_agpmode) {
                 ret = nouveau_mem_init_agp(dev);
                 if (ret)
                         NV_ERROR(dev, "Error initialising AGP: %d\n", ret);
diff --git a/drivers/gpu/drm/nouveau/nouveau_object.c b/drivers/gpu/drm/nouveau/nouveau_object.c
index 30b6544467ca..03adfe4c7665 100644
--- a/drivers/gpu/drm/nouveau/nouveau_object.c
+++ b/drivers/gpu/drm/nouveau/nouveau_object.c
@@ -909,7 +909,7 @@ nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
         nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 
-        if (chan->ramin_heap.free_stack.next)
+        if (drm_mm_initialized(&chan->ramin_heap))
                 drm_mm_takedown(&chan->ramin_heap);
         nouveau_gpuobj_ref(NULL, &chan->ramin);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index 9a250eb53098..07b115184b87 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -12,6 +12,7 @@ struct nouveau_sgdma_be {
         struct drm_device *dev;
 
         dma_addr_t *pages;
+        bool *ttm_alloced;
         unsigned nr_pages;
 
         u64 offset;
@@ -20,7 +21,8 @@ struct nouveau_sgdma_be {
 
 static int
 nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
-                       struct page **pages, struct page *dummy_read_page)
+                       struct page **pages, struct page *dummy_read_page,
+                       dma_addr_t *dma_addrs)
 {
         struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
         struct drm_device *dev = nvbe->dev;
@@ -34,15 +36,25 @@ nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
         if (!nvbe->pages)
                 return -ENOMEM;
 
+        nvbe->ttm_alloced = kmalloc(sizeof(bool) * num_pages, GFP_KERNEL);
+        if (!nvbe->ttm_alloced)
+                return -ENOMEM;
+
         nvbe->nr_pages = 0;
         while (num_pages--) {
-                nvbe->pages[nvbe->nr_pages] =
-                        pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
+                if (dma_addrs[nvbe->nr_pages] != DMA_ERROR_CODE) {
+                        nvbe->pages[nvbe->nr_pages] =
+                                        dma_addrs[nvbe->nr_pages];
+                        nvbe->ttm_alloced[nvbe->nr_pages] = true;
+                } else {
+                        nvbe->pages[nvbe->nr_pages] =
+                                pci_map_page(dev->pdev, pages[nvbe->nr_pages], 0,
                                      PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(dev->pdev,
-                                          nvbe->pages[nvbe->nr_pages])) {
-                        be->func->clear(be);
-                        return -EFAULT;
+                        if (pci_dma_mapping_error(dev->pdev,
+                                                  nvbe->pages[nvbe->nr_pages])) {
+                                be->func->clear(be);
+                                return -EFAULT;
+                        }
                 }
 
                 nvbe->nr_pages++;
@@ -65,11 +77,14 @@ nouveau_sgdma_clear(struct ttm_backend *be)
                         be->func->unbind(be);
 
                 while (nvbe->nr_pages--) {
-                        pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
+                        if (!nvbe->ttm_alloced[nvbe->nr_pages])
+                                pci_unmap_page(dev->pdev, nvbe->pages[nvbe->nr_pages],
                                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
                 }
                 kfree(nvbe->pages);
+                kfree(nvbe->ttm_alloced);
                 nvbe->pages = NULL;
+                nvbe->ttm_alloced = NULL;
                 nvbe->nr_pages = 0;
         }
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_state.c b/drivers/gpu/drm/nouveau/nouveau_state.c
index d5b17b6ccd3a..2148d01354da 100644
--- a/drivers/gpu/drm/nouveau/nouveau_state.c
+++ b/drivers/gpu/drm/nouveau/nouveau_state.c
@@ -596,15 +596,25 @@ static void nouveau_switcheroo_set_state(struct pci_dev *pdev,
         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
         if (state == VGA_SWITCHEROO_ON) {
                 printk(KERN_ERR "VGA switcheroo: switched nouveau on\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 nouveau_pci_resume(pdev);
                 drm_kms_helper_poll_enable(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
         } else {
                 printk(KERN_ERR "VGA switcheroo: switched nouveau off\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 drm_kms_helper_poll_disable(dev);
                 nouveau_pci_suspend(pdev, pmm);
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
 
+static void nouveau_switcheroo_reprobe(struct pci_dev *pdev)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+        nouveau_fbcon_output_poll_changed(dev);
+}
+
 static bool nouveau_switcheroo_can_switch(struct pci_dev *pdev)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
@@ -625,6 +635,7 @@ nouveau_card_init(struct drm_device *dev)
 
         vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode);
         vga_switcheroo_register_client(dev->pdev, nouveau_switcheroo_set_state,
+                                       nouveau_switcheroo_reprobe,
                                        nouveau_switcheroo_can_switch);
 
         /* Initialise internal driver API hooks */
@@ -1054,6 +1065,7 @@ err_out:
 
 void nouveau_lastclose(struct drm_device *dev)
 {
+        vga_switcheroo_process_delayed_switch();
 }
 
 int nouveau_unload(struct drm_device *dev)
@@ -1091,9 +1103,9 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
                 getparam->value = dev->pci_device;
                 break;
         case NOUVEAU_GETPARAM_BUS_TYPE:
-                if (drm_device_is_agp(dev))
+                if (drm_pci_device_is_agp(dev))
                         getparam->value = NV_AGP;
-                else if (drm_device_is_pcie(dev))
+                else if (drm_pci_device_is_pcie(dev))
                         getparam->value = NV_PCIE;
                 else
                         getparam->value = NV_PCI;
diff --git a/drivers/gpu/drm/nouveau/nv17_tv.c b/drivers/gpu/drm/nouveau/nv17_tv.c
index 28119fd19d03..3900cebba560 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv.c
+++ b/drivers/gpu/drm/nouveau/nv17_tv.c
@@ -197,10 +197,12 @@ static int nv17_tv_get_ld_modes(struct drm_encoder *encoder,
                                 struct drm_connector *connector)
 {
         struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
-        struct drm_display_mode *mode, *tv_mode;
+        const struct drm_display_mode *tv_mode;
         int n = 0;
 
         for (tv_mode = nv17_tv_modes; tv_mode->hdisplay; tv_mode++) {
+                struct drm_display_mode *mode;
+
                 mode = drm_mode_duplicate(encoder->dev, tv_mode);
 
                 mode->clock = tv_norm->tv_enc_mode.vrefresh *
diff --git a/drivers/gpu/drm/nouveau/nv17_tv.h b/drivers/gpu/drm/nouveau/nv17_tv.h
index 6bf03840f9eb..622e72221682 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv.h
+++ b/drivers/gpu/drm/nouveau/nv17_tv.h
@@ -112,7 +112,7 @@ extern struct nv17_tv_norm_params {
 } nv17_tv_norms[NUM_TV_NORMS];
 #define get_tv_norm(enc) (&nv17_tv_norms[to_tv_enc(enc)->tv_norm])
 
-extern struct drm_display_mode nv17_tv_modes[];
+extern const struct drm_display_mode nv17_tv_modes[];
 
 static inline int interpolate(int y0, int y1, int y2, int x)
 {
diff --git a/drivers/gpu/drm/nouveau/nv17_tv_modes.c b/drivers/gpu/drm/nouveau/nv17_tv_modes.c
index 9d3893c50a41..4d1d29f60307 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv_modes.c
+++ b/drivers/gpu/drm/nouveau/nv17_tv_modes.c
@@ -438,7 +438,7 @@ void nv17_tv_state_load(struct drm_device *dev, struct nv17_tv_state *state)
 
 /* Timings similar to the ones the blob sets */
 
-struct drm_display_mode nv17_tv_modes[] = {
+const struct drm_display_mode nv17_tv_modes[] = {
         { DRM_MODE("320x200", DRM_MODE_TYPE_DRIVER, 0,
                    320, 344, 392, 560, 0, 200, 200, 202, 220, 0,
                    DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC
diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c
index ea0041810ae3..300285ae8e9e 100644
--- a/drivers/gpu/drm/nouveau/nv50_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv50_instmem.c
@@ -56,7 +56,7 @@ nv50_channel_del(struct nouveau_channel **pchan)
         nouveau_gpuobj_ref(NULL, &chan->ramfc);
         nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
-        if (chan->ramin_heap.free_stack.next)
+        if (drm_mm_initialized(&chan->ramin_heap))
                 drm_mm_takedown(&chan->ramin_heap);
         nouveau_gpuobj_ref(NULL, &chan->ramin);
         kfree(chan);
@@ -259,7 +259,7 @@ nv50_instmem_takedown(struct drm_device *dev)
         nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
         nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
 
-        if (dev_priv->ramin_heap.free_stack.next)
+        if (drm_mm_initialized(&dev_priv->ramin_heap))
                 drm_mm_takedown(&dev_priv->ramin_heap);
 
         dev_priv->engine.instmem.priv = NULL;
diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c
index c09091749054..82357d2df1f4 100644
--- a/drivers/gpu/drm/nouveau/nvc0_instmem.c
+++ b/drivers/gpu/drm/nouveau/nvc0_instmem.c
@@ -67,7 +67,7 @@ nvc0_channel_del(struct nouveau_channel **pchan)
                 return;
 
         nouveau_vm_ref(NULL, &chan->vm, NULL);
-        if (chan->ramin_heap.free_stack.next)
+        if (drm_mm_initialized(&chan->ramin_heap))
                 drm_mm_takedown(&chan->ramin_heap);
         nouveau_gpuobj_ref(NULL, &chan->ramin);
         kfree(chan);
diff --git a/drivers/gpu/drm/r128/r128_drv.c b/drivers/gpu/drm/r128/r128_drv.c
index 18c3c71e41b1..b9e8efd2b754 100644
--- a/drivers/gpu/drm/r128/r128_drv.c
+++ b/drivers/gpu/drm/r128/r128_drv.c
@@ -71,10 +71,7 @@ static struct drm_driver driver = {
 #endif
                 .llseek = noop_llseek,
         },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-        },
+
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -89,16 +86,21 @@ int r128_driver_load(struct drm_device *dev, unsigned long flags)
         return drm_vblank_init(dev, 1);
 }
 
+static struct pci_driver r128_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init r128_init(void)
 {
         driver.num_ioctls = r128_max_ioctl;
 
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &r128_pci_driver);
 }
 
 static void __exit r128_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &r128_pci_driver);
 }
 
 module_init(r128_init);
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index e97e6f842699..e47eecfc2df4 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -66,7 +66,7 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
         r200.o radeon_legacy_tv.o r600_cs.o r600_blit.o r600_blit_shaders.o \
         r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \
         evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \
-        radeon_trace_points.o
+        radeon_trace_points.o ni.o
 
 radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
 radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
diff --git a/drivers/gpu/drm/radeon/atom.c b/drivers/gpu/drm/radeon/atom.c
index 8e421f644a54..258fa5e7a2d9 100644
--- a/drivers/gpu/drm/radeon/atom.c
+++ b/drivers/gpu/drm/radeon/atom.c
@@ -112,6 +112,7 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
                         base += 3;
                         break;
                 case ATOM_IIO_WRITE:
+                        (void)ctx->card->ioreg_read(ctx->card, CU16(base + 1));
                         ctx->card->ioreg_write(ctx->card, CU16(base + 1), temp);
                         base += 3;
                         break;
@@ -733,16 +734,16 @@ static void atom_op_jump(atom_exec_context *ctx, int *ptr, int arg)
 static void atom_op_mask(atom_exec_context *ctx, int *ptr, int arg)
 {
         uint8_t attr = U8((*ptr)++);
-        uint32_t dst, src1, src2, saved;
+        uint32_t dst, mask, src, saved;
         int dptr = *ptr;
         SDEBUG("   dst: ");
         dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1);
-        SDEBUG("   src1: ");
-        src1 = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr);
-        SDEBUG("   src2: ");
-        src2 = atom_get_src(ctx, attr, ptr);
-        dst &= src1;
-        dst |= src2;
+        mask = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr);
+        SDEBUG("   mask: 0x%08x", mask);
+        SDEBUG("   src: ");
+        src = atom_get_src(ctx, attr, ptr);
+        dst &= mask;
+        dst |= src;
         SDEBUG("   dst: ");
         atom_put_dst(ctx, arg, attr, &dptr, dst, saved);
 }
diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h
index 58a0cd02c0a2..04b269d14a59 100644
--- a/drivers/gpu/drm/radeon/atombios.h
+++ b/drivers/gpu/drm/radeon/atombios.h
@@ -1629,7 +1629,7 @@ typedef struct _GET_ENGINE_CLOCK_PARAMETERS
 typedef struct _READ_EDID_FROM_HW_I2C_DATA_PARAMETERS
 {
   USHORT    usPrescale;         //Ratio between Engine clock and I2C clock
-  USHORT    usVRAMAddress;      //Adress in Frame Buffer where to pace raw EDID
+  USHORT    usVRAMAddress;      //Address in Frame Buffer where to pace raw EDID
   USHORT    usStatus;           //When use output: lower byte EDID checksum, high byte hardware status
                                 //WHen use input:  lower byte as 'byte to read':currently limited to 128byte or 1byte
   UCHAR     ucSlaveAddr;        //Read from which slave
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index df2b6f2b35f8..d56f08d3cbdc 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -253,7 +253,8 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode)
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
                 drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id);
-                atombios_blank_crtc(crtc, ATOM_ENABLE);
+                if (radeon_crtc->enabled)
+                        atombios_blank_crtc(crtc, ATOM_ENABLE);
                 if (ASIC_IS_DCE3(rdev))
                         atombios_enable_crtc_memreq(crtc, ATOM_DISABLE);
                 atombios_enable_crtc(crtc, ATOM_DISABLE);
@@ -402,6 +403,7 @@ union atom_enable_ss {
         ENABLE_LVDS_SS_PARAMETERS_V2 lvds_ss_2;
         ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION v1;
         ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2 v2;
+        ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3 v3;
 };
 
 static void atombios_crtc_program_ss(struct drm_crtc *crtc,
@@ -416,7 +418,30 @@ static void atombios_crtc_program_ss(struct drm_crtc *crtc,
 
         memset(&args, 0, sizeof(args));
 
-        if (ASIC_IS_DCE4(rdev)) {
+        if (ASIC_IS_DCE5(rdev)) {
+                args.v3.usSpreadSpectrumAmountFrac = 0;
+                args.v3.ucSpreadSpectrumType = ss->type;
+                switch (pll_id) {
+                case ATOM_PPLL1:
+                        args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P1PLL;
+                        args.v3.usSpreadSpectrumAmount = ss->amount;
+                        args.v3.usSpreadSpectrumStep = ss->step;
+                        break;
+                case ATOM_PPLL2:
+                        args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P2PLL;
+                        args.v3.usSpreadSpectrumAmount = ss->amount;
+                        args.v3.usSpreadSpectrumStep = ss->step;
+                        break;
+                case ATOM_DCPLL:
+                        args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_DCPLL;
+                        args.v3.usSpreadSpectrumAmount = 0;
+                        args.v3.usSpreadSpectrumStep = 0;
+                        break;
+                case ATOM_PPLL_INVALID:
+                        return;
+                }
+                args.v2.ucEnable = enable;
+        } else if (ASIC_IS_DCE4(rdev)) {
                 args.v2.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
                 args.v2.ucSpreadSpectrumType = ss->type;
                 switch (pll_id) {
@@ -530,7 +555,8 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                         dp_clock = dig_connector->dp_clock;
                                 }
                         }
-
+/* this might work properly with the new pll algo */
+#if 0 /* doesn't work properly on some laptops */
                         /* use recommended ref_div for ss */
                         if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
                                 if (ss_enabled) {
@@ -540,13 +566,18 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                         }
                                 }
                         }
-
+#endif
                         if (ASIC_IS_AVIVO(rdev)) {
                                 /* DVO wants 2x pixel clock if the DVO chip is in 12 bit mode */
                                 if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1)
                                         adjusted_clock = mode->clock * 2;
                                 if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
                                         pll->flags |= RADEON_PLL_PREFER_CLOSEST_LOWER;
+                                /* rv515 needs more testing with this option */
+                                if (rdev->family != CHIP_RV515) {
+                                        if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+                                                pll->flags |= RADEON_PLL_IS_LCD;
+                                }
                         } else {
                                 if (encoder->encoder_type != DRM_MODE_ENCODER_DAC)
                                         pll->flags |= RADEON_PLL_NO_ODD_POST_DIV;
@@ -581,14 +612,9 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 args.v1.usPixelClock = cpu_to_le16(mode->clock / 10);
                                 args.v1.ucTransmitterID = radeon_encoder->encoder_id;
                                 args.v1.ucEncodeMode = encoder_mode;
-                                if (encoder_mode == ATOM_ENCODER_MODE_DP) {
-                                        if (ss_enabled)
-                                                args.v1.ucConfig |=
-                                                        ADJUST_DISPLAY_CONFIG_SS_ENABLE;
-                                } else if (encoder_mode == ATOM_ENCODER_MODE_LVDS) {
+                                if (ss_enabled)
                                         args.v1.ucConfig |=
                                                 ADJUST_DISPLAY_CONFIG_SS_ENABLE;
-                                }
 
                                 atom_execute_table(rdev->mode_info.atom_context,
                                                    index, (uint32_t *)&args);
@@ -599,12 +625,12 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 args.v3.sInput.ucTransmitterID = radeon_encoder->encoder_id;
                                 args.v3.sInput.ucEncodeMode = encoder_mode;
                                 args.v3.sInput.ucDispPllConfig = 0;
+                                if (ss_enabled)
+                                        args.v3.sInput.ucDispPllConfig |=
+                                                DISPPLL_CONFIG_SS_ENABLE;
                                 if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) {
                                         struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
                                         if (encoder_mode == ATOM_ENCODER_MODE_DP) {
-                                                if (ss_enabled)
-                                                        args.v3.sInput.ucDispPllConfig |=
-                                                                DISPPLL_CONFIG_SS_ENABLE;
                                                 args.v3.sInput.ucDispPllConfig |=
                                                         DISPPLL_CONFIG_COHERENT_MODE;
                                                 /* 16200 or 27000 */
@@ -624,18 +650,11 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                         }
                                 } else if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
                                         if (encoder_mode == ATOM_ENCODER_MODE_DP) {
-                                                if (ss_enabled)
-                                                        args.v3.sInput.ucDispPllConfig |=
-                                                                DISPPLL_CONFIG_SS_ENABLE;
                                                 args.v3.sInput.ucDispPllConfig |=
                                                         DISPPLL_CONFIG_COHERENT_MODE;
                                                 /* 16200 or 27000 */
                                                 args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10);
-                                        } else if (encoder_mode == ATOM_ENCODER_MODE_LVDS) {
-                                                if (ss_enabled)
-                                                        args.v3.sInput.ucDispPllConfig |=
-                                                                DISPPLL_CONFIG_SS_ENABLE;
-                                        } else {
+                                        } else if (encoder_mode != ATOM_ENCODER_MODE_LVDS) {
                                                 if (mode->clock > 165000)
                                                         args.v3.sInput.ucDispPllConfig |=
                                                                 DISPPLL_CONFIG_DUAL_LINK;
@@ -672,9 +691,14 @@ union set_pixel_clock {
         PIXEL_CLOCK_PARAMETERS_V2 v2;
         PIXEL_CLOCK_PARAMETERS_V3 v3;
         PIXEL_CLOCK_PARAMETERS_V5 v5;
+        PIXEL_CLOCK_PARAMETERS_V6 v6;
 };
 
-static void atombios_crtc_set_dcpll(struct drm_crtc *crtc)
+/* on DCE5, make sure the voltage is high enough to support the
+ * required disp clk.
+ */
+static void atombios_crtc_set_dcpll(struct drm_crtc *crtc,
+                                    u32 dispclk)
 {
         struct drm_device *dev = crtc->dev;
         struct radeon_device *rdev = dev->dev_private;
@@ -697,9 +721,16 @@ static void atombios_crtc_set_dcpll(struct drm_crtc *crtc)
                          * SetPixelClock provides the dividers
                          */
                         args.v5.ucCRTC = ATOM_CRTC_INVALID;
-                        args.v5.usPixelClock = rdev->clock.default_dispclk;
+                        args.v5.usPixelClock = dispclk;
                         args.v5.ucPpll = ATOM_DCPLL;
                         break;
+                case 6:
+                        /* if the default dcpll clock is specified,
+                         * SetPixelClock provides the dividers
+                         */
+                        args.v6.ulDispEngClkFreq = dispclk;
+                        args.v6.ucPpll = ATOM_DCPLL;
+                        break;
                 default:
                         DRM_ERROR("Unknown table version %d %d\n", frev, crev);
                         return;
@@ -783,6 +814,18 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
                         args.v5.ucEncoderMode = encoder_mode;
                         args.v5.ucPpll = pll_id;
                         break;
+                case 6:
+                        args.v6.ulCrtcPclkFreq.ucCRTC = crtc_id;
+                        args.v6.ulCrtcPclkFreq.ulPixelClock = cpu_to_le32(clock / 10);
+                        args.v6.ucRefDiv = ref_div;
+                        args.v6.usFbDiv = cpu_to_le16(fb_div);
+                        args.v6.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000);
+                        args.v6.ucPostDiv = post_div;
+                        args.v6.ucMiscInfo = 0; /* HDMI depth, etc. */
+                        args.v6.ucTransmitterID = encoder_id;
+                        args.v6.ucEncoderMode = encoder_mode;
+                        args.v6.ucPpll = pll_id;
+                        break;
                 default:
                         DRM_ERROR("Unknown table version %d %d\n", frev, crev);
                         return;
@@ -914,8 +957,16 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode
         /* adjust pixel clock as needed */
         adjusted_clock = atombios_adjust_pll(crtc, mode, pll, ss_enabled, &ss);
 
-        radeon_compute_pll(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
-                           &ref_div, &post_div);
+        /* rv515 seems happier with the old algo */
+        if (rdev->family == CHIP_RV515)
+                radeon_compute_pll_legacy(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
+                                          &ref_div, &post_div);
+        else if (ASIC_IS_AVIVO(rdev))
+                radeon_compute_pll_avivo(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
+                                         &ref_div, &post_div);
+        else
+                radeon_compute_pll_legacy(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
+                                          &ref_div, &post_div);
 
         atombios_crtc_program_ss(crtc, ATOM_DISABLE, radeon_crtc->pll_id, &ss);
 
@@ -957,6 +1008,7 @@ static int evergreen_crtc_do_set_base(struct drm_crtc *crtc,
         struct radeon_bo *rbo;
         uint64_t fb_location;
         uint32_t fb_format, fb_pitch_pixels, tiling_flags;
+        u32 fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_NONE);
         int r;
 
         /* no fb bound */
@@ -978,7 +1030,7 @@ static int evergreen_crtc_do_set_base(struct drm_crtc *crtc,
          * just update base pointers
          */
         obj = radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 return r;
@@ -1008,11 +1060,17 @@ static int evergreen_crtc_do_set_base(struct drm_crtc *crtc,
         case 16:
                 fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
                              EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB565));
+#ifdef __BIG_ENDIAN
+                fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+#endif
                 break;
         case 24:
         case 32:
                 fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
                              EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB8888));
+#ifdef __BIG_ENDIAN
+                fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+#endif
                 break;
         default:
                 DRM_ERROR("Unsupported screen depth %d\n",
@@ -1057,6 +1115,7 @@ static int evergreen_crtc_do_set_base(struct drm_crtc *crtc,
         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
                (u32) fb_location & EVERGREEN_GRPH_SURFACE_ADDRESS_MASK);
         WREG32(EVERGREEN_GRPH_CONTROL + radeon_crtc->crtc_offset, fb_format);
+        WREG32(EVERGREEN_GRPH_SWAP_CONTROL + radeon_crtc->crtc_offset, fb_swap);
 
         WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_X + radeon_crtc->crtc_offset, 0);
         WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_Y + radeon_crtc->crtc_offset, 0);
@@ -1086,7 +1145,7 @@ static int evergreen_crtc_do_set_base(struct drm_crtc *crtc,
 
         if (!atomic && fb && fb != crtc->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
-                rbo = radeon_fb->obj->driver_private;
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r != 0))
                         return r;
@@ -1113,6 +1172,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
         struct drm_framebuffer *target_fb;
         uint64_t fb_location;
         uint32_t fb_format, fb_pitch_pixels, tiling_flags;
+        u32 fb_swap = R600_D1GRPH_SWAP_ENDIAN_NONE;
         int r;
 
         /* no fb bound */
@@ -1131,7 +1191,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
         }
 
         obj = radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 return r;
@@ -1166,12 +1226,18 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
                 fb_format =
                     AVIVO_D1GRPH_CONTROL_DEPTH_16BPP |
                     AVIVO_D1GRPH_CONTROL_16BPP_RGB565;
+#ifdef __BIG_ENDIAN
+                fb_swap = R600_D1GRPH_SWAP_ENDIAN_16BIT;
+#endif
                 break;
         case 24:
         case 32:
                 fb_format =
                     AVIVO_D1GRPH_CONTROL_DEPTH_32BPP |
                     AVIVO_D1GRPH_CONTROL_32BPP_ARGB8888;
+#ifdef __BIG_ENDIAN
+                fb_swap = R600_D1GRPH_SWAP_ENDIAN_32BIT;
+#endif
                 break;
         default:
                 DRM_ERROR("Unsupported screen depth %d\n",
@@ -1211,6 +1277,8 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
         WREG32(AVIVO_D1GRPH_SECONDARY_SURFACE_ADDRESS +
                radeon_crtc->crtc_offset, (u32) fb_location);
         WREG32(AVIVO_D1GRPH_CONTROL + radeon_crtc->crtc_offset, fb_format);
+        if (rdev->family >= CHIP_R600)
+                WREG32(R600_D1GRPH_SWAP_CONTROL + radeon_crtc->crtc_offset, fb_swap);
 
         WREG32(AVIVO_D1GRPH_SURFACE_OFFSET_X + radeon_crtc->crtc_offset, 0);
         WREG32(AVIVO_D1GRPH_SURFACE_OFFSET_Y + radeon_crtc->crtc_offset, 0);
@@ -1240,7 +1308,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 
         if (!atomic && fb && fb != crtc->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
-                rbo = radeon_fb->obj->driver_private;
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r != 0))
                         return r;
@@ -1376,7 +1444,8 @@ int atombios_crtc_mode_set(struct drm_crtc *crtc,
                                                                    rdev->clock.default_dispclk);
                 if (ss_enabled)
                         atombios_crtc_program_ss(crtc, ATOM_DISABLE, ATOM_DCPLL, &ss);
-                atombios_crtc_set_dcpll(crtc);
+                /* XXX: DCE5, make sure voltage, dispclk is high enough */
+                atombios_crtc_set_dcpll(crtc, rdev->clock.default_dispclk);
                 if (ss_enabled)
                         atombios_crtc_program_ss(crtc, ATOM_ENABLE, ATOM_DCPLL, &ss);
         }
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index 4e7778d44b8d..695de9a38506 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -187,9 +187,9 @@ static int dp_link_clock_for_mode_clock(u8 dpcd[DP_DPCD_SIZE], int mode_clock)
 int dp_mode_valid(u8 dpcd[DP_DPCD_SIZE], int mode_clock)
 {
         int lanes = dp_lanes_for_mode_clock(dpcd, mode_clock);
-        int bw = dp_lanes_for_mode_clock(dpcd, mode_clock);
+        int dp_clock = dp_link_clock_for_mode_clock(dpcd, mode_clock);
 
-        if ((lanes == 0) || (bw == 0))
+        if ((lanes == 0) || (dp_clock == 0))
                 return MODE_CLOCK_HIGH;
 
         return MODE_OK;
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index f7d7477daffb..ffdc8332b76e 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -39,6 +39,7 @@
 
 static void evergreen_gpu_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
+static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 
 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
 {
@@ -96,26 +97,29 @@ u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 }
 
 /* get temperature in millidegrees */
-u32 evergreen_get_temp(struct radeon_device *rdev)
+int evergreen_get_temp(struct radeon_device *rdev)
 {
         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
                 ASIC_T_SHIFT;
         u32 actual_temp = 0;
 
-        if ((temp >> 10) & 1)
-                actual_temp = 0;
-        else if ((temp >> 9) & 1)
+        if (temp & 0x400)
+                actual_temp = -256;
+        else if (temp & 0x200)
                 actual_temp = 255;
-        else
-                actual_temp = (temp >> 1) & 0xff;
+        else if (temp & 0x100) {
+                actual_temp = temp & 0x1ff;
+                actual_temp |= ~0x1ff;
+        } else
+                actual_temp = temp & 0xff;
 
-        return actual_temp * 1000;
+        return (actual_temp * 1000) / 2;
 }
 
-u32 sumo_get_temp(struct radeon_device *rdev)
+int sumo_get_temp(struct radeon_device *rdev)
 {
         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
-        u32 actual_temp = (temp >> 1) & 0xff;
+        int actual_temp = temp - 49;
 
         return actual_temp * 1000;
 }
@@ -400,16 +404,28 @@ static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
         case 0:
         case 4:
         default:
-                return 3840 * 2;
+                if (ASIC_IS_DCE5(rdev))
+                        return 4096 * 2;
+                else
+                        return 3840 * 2;
         case 1:
         case 5:
-                return 5760 * 2;
+                if (ASIC_IS_DCE5(rdev))
+                        return 6144 * 2;
+                else
+                        return 5760 * 2;
         case 2:
         case 6:
-                return 7680 * 2;
+                if (ASIC_IS_DCE5(rdev))
+                        return 8192 * 2;
+                else
+                        return 7680 * 2;
         case 3:
         case 7:
-                return 1920 * 2;
+                if (ASIC_IS_DCE5(rdev))
+                        return 2048 * 2;
+                else
+                        return 1920 * 2;
         }
 }
 
@@ -811,6 +827,8 @@ void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
         unsigned i;
         u32 tmp;
 
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+
         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
         for (i = 0; i < rdev->usec_timeout; i++) {
                 /* read MC_STATUS */
@@ -1144,7 +1162,7 @@ static void evergreen_mc_program(struct radeon_device *rdev)
         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
         WREG32(MC_VM_FB_LOCATION, tmp);
         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
-        WREG32(HDP_NONSURFACE_INFO, (2 << 7));
+        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
         if (rdev->flags & RADEON_IS_AGP) {
                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
@@ -1167,6 +1185,18 @@ static void evergreen_mc_program(struct radeon_device *rdev)
 /*
  * CP.
  */
+void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        /* set to DX10/11 mode */
+        radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
+        radeon_ring_write(rdev, 1);
+        /* FIXME: implement */
+        radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+        radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
+        radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
+        radeon_ring_write(rdev, ib->length_dw);
+}
+
 
 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
 {
@@ -1218,7 +1248,7 @@ static int evergreen_cp_start(struct radeon_device *rdev)
         cp_me = 0xff;
         WREG32(CP_ME_CNTL, cp_me);
 
-        r = radeon_ring_lock(rdev, evergreen_default_size + 15);
+        r = radeon_ring_lock(rdev, evergreen_default_size + 19);
         if (r) {
                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
                 return r;
@@ -1251,6 +1281,11 @@ static int evergreen_cp_start(struct radeon_device *rdev)
         radeon_ring_write(rdev, 0xffffffff);
         radeon_ring_write(rdev, 0xffffffff);
 
+        radeon_ring_write(rdev, 0xc0026900);
+        radeon_ring_write(rdev, 0x00000316);
+        radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        radeon_ring_write(rdev, 0x00000010); /*  */
+
         radeon_ring_unlock_commit(rdev);
 
         return 0;
@@ -1369,11 +1404,14 @@ static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         case CHIP_CEDAR:
         case CHIP_REDWOOD:
         case CHIP_PALM:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
                 force_no_swizzle = false;
                 break;
         case CHIP_CYPRESS:
         case CHIP_HEMLOCK:
         case CHIP_JUNIPER:
+        case CHIP_BARTS:
         default:
                 force_no_swizzle = true;
                 break;
@@ -1487,6 +1525,7 @@ static void evergreen_program_channel_remap(struct radeon_device *rdev)
         switch (rdev->family) {
         case CHIP_HEMLOCK:
         case CHIP_CYPRESS:
+        case CHIP_BARTS:
                 tcp_chan_steer_lo = 0x54763210;
                 tcp_chan_steer_hi = 0x0000ba98;
                 break;
@@ -1494,6 +1533,8 @@ static void evergreen_program_channel_remap(struct radeon_device *rdev)
         case CHIP_REDWOOD:
         case CHIP_CEDAR:
         case CHIP_PALM:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
         default:
                 tcp_chan_steer_lo = 0x76543210;
                 tcp_chan_steer_hi = 0x0000ba98;
@@ -1637,6 +1678,69 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
                 break;
+        case CHIP_BARTS:
+                rdev->config.evergreen.num_ses = 2;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 8;
+                rdev->config.evergreen.max_simds = 7;
+                rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 248;
+                rdev->config.evergreen.max_gs_threads = 32;
+                rdev->config.evergreen.max_stack_entries = 512;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 256;
+                rdev->config.evergreen.sx_max_export_pos_size = 64;
+                rdev->config.evergreen.sx_max_export_smx_size = 192;
+                rdev->config.evergreen.max_hw_contexts = 8;
+                rdev->config.evergreen.sq_num_cf_insts = 2;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_TURKS:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 4;
+                rdev->config.evergreen.max_simds = 6;
+                rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 248;
+                rdev->config.evergreen.max_gs_threads = 32;
+                rdev->config.evergreen.max_stack_entries = 256;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 256;
+                rdev->config.evergreen.sx_max_export_pos_size = 64;
+                rdev->config.evergreen.sx_max_export_smx_size = 192;
+                rdev->config.evergreen.max_hw_contexts = 8;
+                rdev->config.evergreen.sq_num_cf_insts = 2;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x100;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
+        case CHIP_CAICOS:
+                rdev->config.evergreen.num_ses = 1;
+                rdev->config.evergreen.max_pipes = 4;
+                rdev->config.evergreen.max_tile_pipes = 2;
+                rdev->config.evergreen.max_simds = 2;
+                rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
+                rdev->config.evergreen.max_gprs = 256;
+                rdev->config.evergreen.max_threads = 192;
+                rdev->config.evergreen.max_gs_threads = 16;
+                rdev->config.evergreen.max_stack_entries = 256;
+                rdev->config.evergreen.sx_num_of_sets = 4;
+                rdev->config.evergreen.sx_max_export_size = 128;
+                rdev->config.evergreen.sx_max_export_pos_size = 32;
+                rdev->config.evergreen.sx_max_export_smx_size = 96;
+                rdev->config.evergreen.max_hw_contexts = 4;
+                rdev->config.evergreen.sq_num_cf_insts = 1;
+
+                rdev->config.evergreen.sc_prim_fifo_size = 0x40;
+                rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
+                break;
         }
 
         /* Initialize HDP */
@@ -1778,6 +1882,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                 switch (rdev->family) {
                 case CHIP_CYPRESS:
                 case CHIP_HEMLOCK:
+                case CHIP_BARTS:
                         gb_backend_map = 0x66442200;
                         break;
                 case CHIP_JUNIPER:
@@ -1916,6 +2021,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         switch (rdev->family) {
         case CHIP_CEDAR:
         case CHIP_PALM:
+        case CHIP_CAICOS:
                 /* no vertex cache */
                 sq_config &= ~VC_ENABLE;
                 break;
@@ -1975,6 +2081,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         switch (rdev->family) {
         case CHIP_CEDAR:
         case CHIP_PALM:
+        case CHIP_CAICOS:
                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
                 break;
         default:
@@ -1985,6 +2092,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
 
         WREG32(VGT_GS_VERTEX_REUSE, 16);
+        WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
 
         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
@@ -2083,16 +2191,40 @@ int evergreen_mc_init(struct radeon_device *rdev)
 
 bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
 {
-        /* FIXME: implement for evergreen */
-        return false;
+        u32 srbm_status;
+        u32 grbm_status;
+        u32 grbm_status_se0, grbm_status_se1;
+        struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
+        int r;
+
+        srbm_status = RREG32(SRBM_STATUS);
+        grbm_status = RREG32(GRBM_STATUS);
+        grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
+        grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
+        if (!(grbm_status & GUI_ACTIVE)) {
+                r100_gpu_lockup_update(lockup, &rdev->cp);
+                return false;
+        }
+        /* force CP activities */
+        r = radeon_ring_lock(rdev, 2);
+        if (!r) {
+                /* PACKET2 NOP */
+                radeon_ring_write(rdev, 0x80000000);
+                radeon_ring_write(rdev, 0x80000000);
+                radeon_ring_unlock_commit(rdev);
+        }
+        rdev->cp.rptr = RREG32(CP_RB_RPTR);
+        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
 }
 
 static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
 {
         struct evergreen_mc_save save;
-        u32 srbm_reset = 0;
         u32 grbm_reset = 0;
 
+        if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
+                return 0;
+
         dev_info(rdev->dev, "GPU softreset \n");
         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
                 RREG32(GRBM_STATUS));
@@ -2129,16 +2261,6 @@ static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
         udelay(50);
         WREG32(GRBM_SOFT_RESET, 0);
         (void)RREG32(GRBM_SOFT_RESET);
-
-        /* reset all the system blocks */
-        srbm_reset = SRBM_SOFT_RESET_ALL_MASK;
-
-        dev_info(rdev->dev, "  SRBM_SOFT_RESET=0x%08X\n", srbm_reset);
-        WREG32(SRBM_SOFT_RESET, srbm_reset);
-        (void)RREG32(SRBM_SOFT_RESET);
-        udelay(50);
-        WREG32(SRBM_SOFT_RESET, 0);
-        (void)RREG32(SRBM_SOFT_RESET);
         /* Wait a little for things to settle down */
         udelay(50);
         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
@@ -2149,10 +2271,6 @@ static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
                 RREG32(GRBM_STATUS_SE1));
         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
                 RREG32(SRBM_STATUS));
-        /* After reset we need to reinit the asic as GPU often endup in an
-         * incoherent state.
-         */
-        atom_asic_init(rdev->mode_info.atom_context);
         evergreen_mc_resume(rdev, &save);
         return 0;
 }
@@ -2747,7 +2865,7 @@ restart_ih:
         if (wptr != rdev->ih.wptr)
                 goto restart_ih;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -2758,12 +2876,31 @@ static int evergreen_startup(struct radeon_device *rdev)
 {
         int r;
 
-        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
-                r = r600_init_microcode(rdev);
+        /* enable pcie gen2 link */
+        if (!ASIC_IS_DCE5(rdev))
+                evergreen_pcie_gen2_enable(rdev);
+
+        if (ASIC_IS_DCE5(rdev)) {
+                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
+                        r = ni_init_microcode(rdev);
+                        if (r) {
+                                DRM_ERROR("Failed to load firmware!\n");
+                                return r;
+                        }
+                }
+                r = btc_mc_load_microcode(rdev);
                 if (r) {
-                        DRM_ERROR("Failed to load firmware!\n");
+                        DRM_ERROR("Failed to load MC firmware!\n");
                         return r;
                 }
+        } else {
+                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
+                        r = r600_init_microcode(rdev);
+                        if (r) {
+                                DRM_ERROR("Failed to load firmware!\n");
+                                return r;
+                        }
+                }
         }
 
         evergreen_mc_program(rdev);
@@ -2782,6 +2919,11 @@ static int evergreen_startup(struct radeon_device *rdev)
                 rdev->asic->copy = NULL;
                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
         }
+        /* XXX: ontario has problems blitting to gart at the moment */
+        if (rdev->family == CHIP_PALM) {
+                rdev->asic->copy = NULL;
+                rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
+        }
 
         /* allocate wb buffer */
         r = radeon_wb_init(rdev);
@@ -2814,6 +2956,11 @@ int evergreen_resume(struct radeon_device *rdev)
 {
         int r;
 
+        /* reset the asic, the gfx blocks are often in a bad state
+         * after the driver is unloaded or after a resume
+         */
+        if (radeon_asic_reset(rdev))
+                dev_warn(rdev->dev, "GPU reset failed !\n");
         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
          * posting will perform necessary task to bring back GPU into good
          * shape.
@@ -2879,31 +3026,6 @@ int evergreen_copy_blit(struct radeon_device *rdev,
         return 0;
 }
 
-static bool evergreen_card_posted(struct radeon_device *rdev)
-{
-        u32 reg;
-
-        /* first check CRTCs */
-        if (rdev->flags & RADEON_IS_IGP)
-                reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
-                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
-        else
-                reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
-                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) |
-                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
-                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) |
-                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) |
-                        RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
-        if (reg & EVERGREEN_CRTC_MASTER_EN)
-                return true;
-
-        /* then check MEM_SIZE, in case the crtcs are off */
-        if (RREG32(CONFIG_MEMSIZE))
-                return true;
-
-        return false;
-}
-
 /* Plan is to move initialization in that function and use
  * helper function so that radeon_device_init pretty much
  * do nothing more than calling asic specific function. This
@@ -2934,8 +3056,13 @@ int evergreen_init(struct radeon_device *rdev)
         r = radeon_atombios_init(rdev);
         if (r)
                 return r;
+        /* reset the asic, the gfx blocks are often in a bad state
+         * after the driver is unloaded or after a resume
+         */
+        if (radeon_asic_reset(rdev))
+                dev_warn(rdev->dev, "GPU reset failed !\n");
         /* Post card if necessary */
-        if (!evergreen_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -3025,3 +3152,55 @@ void evergreen_fini(struct radeon_device *rdev)
         rdev->bios = NULL;
         radeon_dummy_page_fini(rdev);
 }
+
+static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, speed_cntl;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+}
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_kms.c b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
index 2ccd1f0545fe..2ed930e02f3a 100644
--- a/drivers/gpu/drm/radeon/evergreen_blit_kms.c
+++ b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
@@ -148,7 +148,8 @@ set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
         radeon_ring_write(rdev, SQ_TEX_VTX_VALID_BUFFER << 30);
 
         if ((rdev->family == CHIP_CEDAR) ||
-            (rdev->family == CHIP_PALM))
+            (rdev->family == CHIP_PALM) ||
+            (rdev->family == CHIP_CAICOS))
                 cp_set_surface_sync(rdev,
                                     PACKET3_TC_ACTION_ENA, 48, gpu_addr);
         else
@@ -231,7 +232,7 @@ draw_auto(struct radeon_device *rdev)
 
 }
 
-/* emits 30 */
+/* emits 36 */
 static void
 set_default_state(struct radeon_device *rdev)
 {
@@ -244,6 +245,8 @@ set_default_state(struct radeon_device *rdev)
         int num_hs_threads, num_ls_threads;
         int num_ps_stack_entries, num_vs_stack_entries, num_gs_stack_entries, num_es_stack_entries;
         int num_hs_stack_entries, num_ls_stack_entries;
+        u64 gpu_addr;
+        int dwords;
 
         switch (rdev->family) {
         case CHIP_CEDAR:
@@ -353,10 +356,74 @@ set_default_state(struct radeon_device *rdev)
                 num_hs_stack_entries = 42;
                 num_ls_stack_entries = 42;
                 break;
+        case CHIP_BARTS:
+                num_ps_gprs = 93;
+                num_vs_gprs = 46;
+                num_temp_gprs = 4;
+                num_gs_gprs = 31;
+                num_es_gprs = 31;
+                num_hs_gprs = 23;
+                num_ls_gprs = 23;
+                num_ps_threads = 128;
+                num_vs_threads = 20;
+                num_gs_threads = 20;
+                num_es_threads = 20;
+                num_hs_threads = 20;
+                num_ls_threads = 20;
+                num_ps_stack_entries = 85;
+                num_vs_stack_entries = 85;
+                num_gs_stack_entries = 85;
+                num_es_stack_entries = 85;
+                num_hs_stack_entries = 85;
+                num_ls_stack_entries = 85;
+                break;
+        case CHIP_TURKS:
+                num_ps_gprs = 93;
+                num_vs_gprs = 46;
+                num_temp_gprs = 4;
+                num_gs_gprs = 31;
+                num_es_gprs = 31;
+                num_hs_gprs = 23;
+                num_ls_gprs = 23;
+                num_ps_threads = 128;
+                num_vs_threads = 20;
+                num_gs_threads = 20;
+                num_es_threads = 20;
+                num_hs_threads = 20;
+                num_ls_threads = 20;
+                num_ps_stack_entries = 42;
+                num_vs_stack_entries = 42;
+                num_gs_stack_entries = 42;
+                num_es_stack_entries = 42;
+                num_hs_stack_entries = 42;
+                num_ls_stack_entries = 42;
+                break;
+        case CHIP_CAICOS:
+                num_ps_gprs = 93;
+                num_vs_gprs = 46;
+                num_temp_gprs = 4;
+                num_gs_gprs = 31;
+                num_es_gprs = 31;
+                num_hs_gprs = 23;
+                num_ls_gprs = 23;
+                num_ps_threads = 128;
+                num_vs_threads = 10;
+                num_gs_threads = 10;
+                num_es_threads = 10;
+                num_hs_threads = 10;
+                num_ls_threads = 10;
+                num_ps_stack_entries = 42;
+                num_vs_stack_entries = 42;
+                num_gs_stack_entries = 42;
+                num_es_stack_entries = 42;
+                num_hs_stack_entries = 42;
+                num_ls_stack_entries = 42;
+                break;
         }
 
         if ((rdev->family == CHIP_CEDAR) ||
-            (rdev->family == CHIP_PALM))
+            (rdev->family == CHIP_PALM) ||
+            (rdev->family == CHIP_CAICOS))
                 sq_config = 0;
         else
                 sq_config = VC_ENABLE;
@@ -432,6 +499,18 @@ set_default_state(struct radeon_device *rdev)
         radeon_ring_write(rdev, 0x00000000);
         radeon_ring_write(rdev, 0x00000000);
 
+        /* set to DX10/11 mode */
+        radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
+        radeon_ring_write(rdev, 1);
+
+        /* emit an IB pointing at default state */
+        dwords = ALIGN(rdev->r600_blit.state_len, 0x10);
+        gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset;
+        radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+        radeon_ring_write(rdev, gpu_addr & 0xFFFFFFFC);
+        radeon_ring_write(rdev, upper_32_bits(gpu_addr) & 0xFF);
+        radeon_ring_write(rdev, dwords);
+
 }
 
 static inline uint32_t i2f(uint32_t input)
@@ -462,8 +541,10 @@ static inline uint32_t i2f(uint32_t input)
 int evergreen_blit_init(struct radeon_device *rdev)
 {
         u32 obj_size;
-        int r;
+        int r, dwords;
         void *ptr;
+        u32 packet2s[16];
+        int num_packet2s = 0;
 
         /* pin copy shader into vram if already initialized */
         if (rdev->r600_blit.shader_obj)
@@ -471,8 +552,17 @@ int evergreen_blit_init(struct radeon_device *rdev)
 
         mutex_init(&rdev->r600_blit.mutex);
         rdev->r600_blit.state_offset = 0;
-        rdev->r600_blit.state_len = 0;
-        obj_size = 0;
+
+        rdev->r600_blit.state_len = evergreen_default_size;
+
+        dwords = rdev->r600_blit.state_len;
+        while (dwords & 0xf) {
+                packet2s[num_packet2s++] = PACKET2(0);
+                dwords++;
+        }
+
+        obj_size = dwords * 4;
+        obj_size = ALIGN(obj_size, 256);
 
         rdev->r600_blit.vs_offset = obj_size;
         obj_size += evergreen_vs_size * 4;
@@ -482,7 +572,7 @@ int evergreen_blit_init(struct radeon_device *rdev)
         obj_size += evergreen_ps_size * 4;
         obj_size = ALIGN(obj_size, 256);
 
-        r = radeon_bo_create(rdev, NULL, obj_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
+        r = radeon_bo_create(rdev, obj_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                 &rdev->r600_blit.shader_obj);
         if (r) {
                 DRM_ERROR("evergreen failed to allocate shader\n");
@@ -502,6 +592,12 @@ int evergreen_blit_init(struct radeon_device *rdev)
                 return r;
         }
 
+        memcpy_toio(ptr + rdev->r600_blit.state_offset,
+                    evergreen_default_state, rdev->r600_blit.state_len * 4);
+
+        if (num_packet2s)
+                memcpy_toio(ptr + rdev->r600_blit.state_offset + (rdev->r600_blit.state_len * 4),
+                            packet2s, num_packet2s * 4);
         memcpy(ptr + rdev->r600_blit.vs_offset, evergreen_vs, evergreen_vs_size * 4);
         memcpy(ptr + rdev->r600_blit.ps_offset, evergreen_ps, evergreen_ps_size * 4);
         radeon_bo_kunmap(rdev->r600_blit.shader_obj);
@@ -587,7 +683,7 @@ int evergreen_blit_prepare_copy(struct radeon_device *rdev, int size_bytes)
         /* calculate number of loops correctly */
         ring_size = num_loops * dwords_per_loop;
         /* set default  + shaders */
-        ring_size += 46; /* shaders + def state */
+        ring_size += 52; /* shaders + def state */
         ring_size += 10; /* fence emit for VB IB */
         ring_size += 5; /* done copy */
         ring_size += 10; /* fence emit for done copy */
@@ -595,7 +691,7 @@ int evergreen_blit_prepare_copy(struct radeon_device *rdev, int size_bytes)
         if (r)
                 return r;
 
-        set_default_state(rdev); /* 30 */
+        set_default_state(rdev); /* 36 */
         set_shaders(rdev); /* 16 */
         return 0;
 }
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 5b869ce86917..afec1aca2a73 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -176,6 +176,7 @@
 #define HDP_NONSURFACE_BASE                             0x2C04
 #define HDP_NONSURFACE_INFO                             0x2C08
 #define HDP_NONSURFACE_SIZE                             0x2C0C
+#define HDP_MEM_COHERENCY_FLUSH_CNTL                    0x5480
 #define HDP_REG_COHERENCY_FLUSH_CNTL                    0x54A0
 #define HDP_TILING_CONFIG                               0x2F3C
 
@@ -239,6 +240,7 @@
 #define         FORCE_EOV_MAX_CLK_CNT(x)                        ((x) << 0)
 #define         FORCE_EOV_MAX_REZ_CNT(x)                        ((x) << 16)
 #define PA_SC_LINE_STIPPLE                              0x28A0C
+#define PA_SU_LINE_STIPPLE_VALUE                        0x8A60
 #define PA_SC_LINE_STIPPLE_STATE                        0x8B10
 
 #define SCRATCH_REG0                                    0x8500
@@ -580,6 +582,44 @@
 #       define DC_HPDx_RX_INT_TIMER(x)                    ((x) << 16)
 #       define DC_HPDx_EN                                 (1 << 28)
 
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 /*
  * PM4
  */
@@ -609,10 +649,11 @@
 #define PACKET3_NOP                                     0x10
 #define PACKET3_SET_BASE                                0x11
 #define PACKET3_CLEAR_STATE                             0x12
-#define PACKET3_INDIRECT_BUFFER_SIZE                    0x13
+#define PACKET3_INDEX_BUFFER_SIZE                       0x13
 #define PACKET3_DISPATCH_DIRECT                         0x15
 #define PACKET3_DISPATCH_INDIRECT                       0x16
 #define PACKET3_INDIRECT_BUFFER_END                     0x17
+#define PACKET3_MODE_CONTROL                            0x18
 #define PACKET3_SET_PREDICATION                         0x20
 #define PACKET3_REG_RMW                                 0x21
 #define PACKET3_COND_EXEC                               0x22
@@ -650,14 +691,14 @@
 #              define PACKET3_CB8_DEST_BASE_ENA    (1 << 15)
 #              define PACKET3_CB9_DEST_BASE_ENA    (1 << 16)
 #              define PACKET3_CB10_DEST_BASE_ENA   (1 << 17)
-#              define PACKET3_CB11_DEST_BASE_ENA   (1 << 17)
+#              define PACKET3_CB11_DEST_BASE_ENA   (1 << 18)
 #              define PACKET3_FULL_CACHE_ENA       (1 << 20)
 #              define PACKET3_TC_ACTION_ENA        (1 << 23)
 #              define PACKET3_VC_ACTION_ENA        (1 << 24)
 #              define PACKET3_CB_ACTION_ENA        (1 << 25)
 #              define PACKET3_DB_ACTION_ENA        (1 << 26)
 #              define PACKET3_SH_ACTION_ENA        (1 << 27)
-#              define PACKET3_SMX_ACTION_ENA       (1 << 28)
+#              define PACKET3_SX_ACTION_ENA        (1 << 28)
 #define PACKET3_ME_INITIALIZE                           0x44
 #define         PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
 #define PACKET3_COND_WRITE                              0x45
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
new file mode 100644
index 000000000000..5e0bef80ad7f
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -0,0 +1,316 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#include <linux/firmware.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include "drmP.h"
+#include "radeon.h"
+#include "radeon_asic.h"
+#include "radeon_drm.h"
+#include "nid.h"
+#include "atom.h"
+#include "ni_reg.h"
+
+#define EVERGREEN_PFP_UCODE_SIZE 1120
+#define EVERGREEN_PM4_UCODE_SIZE 1376
+#define EVERGREEN_RLC_UCODE_SIZE 768
+#define BTC_MC_UCODE_SIZE 6024
+
+/* Firmware Names */
+MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
+MODULE_FIRMWARE("radeon/BARTS_me.bin");
+MODULE_FIRMWARE("radeon/BARTS_mc.bin");
+MODULE_FIRMWARE("radeon/BTC_rlc.bin");
+MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
+MODULE_FIRMWARE("radeon/TURKS_me.bin");
+MODULE_FIRMWARE("radeon/TURKS_mc.bin");
+MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
+MODULE_FIRMWARE("radeon/CAICOS_me.bin");
+MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
+
+#define BTC_IO_MC_REGS_SIZE 29
+
+static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00946a00}
+};
+
+static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00936a00}
+};
+
+static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
+        {0x00000077, 0xff010100},
+        {0x00000078, 0x00000000},
+        {0x00000079, 0x00001434},
+        {0x0000007a, 0xcc08ec08},
+        {0x0000007b, 0x00040000},
+        {0x0000007c, 0x000080c0},
+        {0x0000007d, 0x09000000},
+        {0x0000007e, 0x00210404},
+        {0x00000081, 0x08a8e800},
+        {0x00000082, 0x00030444},
+        {0x00000083, 0x00000000},
+        {0x00000085, 0x00000001},
+        {0x00000086, 0x00000002},
+        {0x00000087, 0x48490000},
+        {0x00000088, 0x20244647},
+        {0x00000089, 0x00000005},
+        {0x0000008b, 0x66030000},
+        {0x0000008c, 0x00006603},
+        {0x0000008d, 0x00000100},
+        {0x0000008f, 0x00001c0a},
+        {0x00000090, 0xff000001},
+        {0x00000094, 0x00101101},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x00000099, 0x00006000},
+        {0x0000009a, 0x00001000},
+        {0x0000009f, 0x00916a00}
+};
+
+int btc_mc_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        u32 mem_type, running, blackout = 0;
+        u32 *io_mc_regs;
+        int i;
+
+        if (!rdev->mc_fw)
+                return -EINVAL;
+
+        switch (rdev->family) {
+        case CHIP_BARTS:
+                io_mc_regs = (u32 *)&barts_io_mc_regs;
+                break;
+        case CHIP_TURKS:
+                io_mc_regs = (u32 *)&turks_io_mc_regs;
+                break;
+        case CHIP_CAICOS:
+        default:
+                io_mc_regs = (u32 *)&caicos_io_mc_regs;
+                break;
+        }
+
+        mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
+        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
+
+        if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
+                if (running) {
+                        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
+                        WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
+                }
+
+                /* reset the engine and set to writable */
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
+
+                /* load mc io regs */
+                for (i = 0; i < BTC_IO_MC_REGS_SIZE; i++) {
+                        WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
+                        WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
+                }
+                /* load the MC ucode */
+                fw_data = (const __be32 *)rdev->mc_fw->data;
+                for (i = 0; i < BTC_MC_UCODE_SIZE; i++)
+                        WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
+
+                /* put the engine back into the active state */
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
+
+                /* wait for training to complete */
+                while (!(RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD))
+                        udelay(10);
+
+                if (running)
+                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
+        }
+
+        return 0;
+}
+
+int ni_init_microcode(struct radeon_device *rdev)
+{
+        struct platform_device *pdev;
+        const char *chip_name;
+        const char *rlc_chip_name;
+        size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
+        char fw_name[30];
+        int err;
+
+        DRM_DEBUG("\n");
+
+        pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
+        err = IS_ERR(pdev);
+        if (err) {
+                printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
+                return -EINVAL;
+        }
+
+        switch (rdev->family) {
+        case CHIP_BARTS:
+                chip_name = "BARTS";
+                rlc_chip_name = "BTC";
+                break;
+        case CHIP_TURKS:
+                chip_name = "TURKS";
+                rlc_chip_name = "BTC";
+                break;
+        case CHIP_CAICOS:
+                chip_name = "CAICOS";
+                rlc_chip_name = "BTC";
+                break;
+        default: BUG();
+        }
+
+        pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
+        me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
+        rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
+        mc_req_size = BTC_MC_UCODE_SIZE * 4;
+
+        DRM_INFO("Loading %s Microcode\n", chip_name);
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
+        err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->pfp_fw->size != pfp_req_size) {
+                printk(KERN_ERR
+                       "ni_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->pfp_fw->size, fw_name);
+                err = -EINVAL;
+                goto out;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
+        err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->me_fw->size != me_req_size) {
+                printk(KERN_ERR
+                       "ni_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->me_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
+        err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->rlc_fw->size != rlc_req_size) {
+                printk(KERN_ERR
+                       "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->rlc_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
+        err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->mc_fw->size != mc_req_size) {
+                printk(KERN_ERR
+                       "ni_mc: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->mc_fw->size, fw_name);
+                err = -EINVAL;
+        }
+out:
+        platform_device_unregister(pdev);
+
+        if (err) {
+                if (err != -EINVAL)
+                        printk(KERN_ERR
+                               "ni_cp: Failed to load firmware \"%s\"\n",
+                               fw_name);
+                release_firmware(rdev->pfp_fw);
+                rdev->pfp_fw = NULL;
+                release_firmware(rdev->me_fw);
+                rdev->me_fw = NULL;
+                release_firmware(rdev->rlc_fw);
+                rdev->rlc_fw = NULL;
+                release_firmware(rdev->mc_fw);
+                rdev->mc_fw = NULL;
+        }
+        return err;
+}
+
diff --git a/drivers/gpu/drm/radeon/ni_reg.h b/drivers/gpu/drm/radeon/ni_reg.h
new file mode 100644
index 000000000000..5db7b7d6feb0
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni_reg.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef __NI_REG_H__
+#define __NI_REG_H__
+
+/* northern islands - DCE5 */
+
+#define NI_INPUT_GAMMA_CONTROL                         0x6840
+#       define NI_GRPH_INPUT_GAMMA_MODE(x)             (((x) & 0x3) << 0)
+#       define NI_INPUT_GAMMA_USE_LUT                  0
+#       define NI_INPUT_GAMMA_BYPASS                   1
+#       define NI_INPUT_GAMMA_SRGB_24                  2
+#       define NI_INPUT_GAMMA_XVYCC_222                3
+#       define NI_OVL_INPUT_GAMMA_MODE(x)              (((x) & 0x3) << 4)
+
+#define NI_PRESCALE_GRPH_CONTROL                       0x68b4
+#       define NI_GRPH_PRESCALE_BYPASS                 (1 << 4)
+
+#define NI_PRESCALE_OVL_CONTROL                        0x68c4
+#       define NI_OVL_PRESCALE_BYPASS                  (1 << 4)
+
+#define NI_INPUT_CSC_CONTROL                           0x68d4
+#       define NI_INPUT_CSC_GRPH_MODE(x)               (((x) & 0x3) << 0)
+#       define NI_INPUT_CSC_BYPASS                     0
+#       define NI_INPUT_CSC_PROG_COEFF                 1
+#       define NI_INPUT_CSC_PROG_SHARED_MATRIXA        2
+#       define NI_INPUT_CSC_OVL_MODE(x)                (((x) & 0x3) << 4)
+
+#define NI_OUTPUT_CSC_CONTROL                          0x68f0
+#       define NI_OUTPUT_CSC_GRPH_MODE(x)              (((x) & 0x7) << 0)
+#       define NI_OUTPUT_CSC_BYPASS                    0
+#       define NI_OUTPUT_CSC_TV_RGB                    1
+#       define NI_OUTPUT_CSC_YCBCR_601                 2
+#       define NI_OUTPUT_CSC_YCBCR_709                 3
+#       define NI_OUTPUT_CSC_PROG_COEFF                4
+#       define NI_OUTPUT_CSC_PROG_SHARED_MATRIXB       5
+#       define NI_OUTPUT_CSC_OVL_MODE(x)               (((x) & 0x7) << 4)
+
+#define NI_DEGAMMA_CONTROL                             0x6960
+#       define NI_GRPH_DEGAMMA_MODE(x)                 (((x) & 0x3) << 0)
+#       define NI_DEGAMMA_BYPASS                       0
+#       define NI_DEGAMMA_SRGB_24                      1
+#       define NI_DEGAMMA_XVYCC_222                    2
+#       define NI_OVL_DEGAMMA_MODE(x)                  (((x) & 0x3) << 4)
+#       define NI_ICON_DEGAMMA_MODE(x)                 (((x) & 0x3) << 8)
+#       define NI_CURSOR_DEGAMMA_MODE(x)               (((x) & 0x3) << 12)
+
+#define NI_GAMUT_REMAP_CONTROL                         0x6964
+#       define NI_GRPH_GAMUT_REMAP_MODE(x)             (((x) & 0x3) << 0)
+#       define NI_GAMUT_REMAP_BYPASS                   0
+#       define NI_GAMUT_REMAP_PROG_COEFF               1
+#       define NI_GAMUT_REMAP_PROG_SHARED_MATRIXA      2
+#       define NI_GAMUT_REMAP_PROG_SHARED_MATRIXB      3
+#       define NI_OVL_GAMUT_REMAP_MODE(x)              (((x) & 0x3) << 4)
+
+#define NI_REGAMMA_CONTROL                             0x6a80
+#       define NI_GRPH_REGAMMA_MODE(x)                 (((x) & 0x7) << 0)
+#       define NI_REGAMMA_BYPASS                       0
+#       define NI_REGAMMA_SRGB_24                      1
+#       define NI_REGAMMA_XVYCC_222                    2
+#       define NI_REGAMMA_PROG_A                       3
+#       define NI_REGAMMA_PROG_B                       4
+#       define NI_OVL_REGAMMA_MODE(x)                  (((x) & 0x7) << 4)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
new file mode 100644
index 000000000000..f7b445390e02
--- /dev/null
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef NI_H
+#define NI_H
+
+#define MC_SHARED_BLACKOUT_CNTL                         0x20ac
+#define MC_SEQ_SUP_CNTL                                 0x28c8
+#define         RUN_MASK                                (1 << 0)
+#define MC_SEQ_SUP_PGM                                  0x28cc
+#define MC_IO_PAD_CNTL_D0                               0x29d0
+#define         MEM_FALL_OUT_CMD                        (1 << 8)
+#define MC_SEQ_MISC0                                    0x2a00
+#define         MC_SEQ_MISC0_GDDR5_SHIFT                28
+#define         MC_SEQ_MISC0_GDDR5_MASK                 0xf0000000
+#define         MC_SEQ_MISC0_GDDR5_VALUE                5
+#define MC_SEQ_IO_DEBUG_INDEX                           0x2a44
+#define MC_SEQ_IO_DEBUG_DATA                            0x2a48
+
+#endif
+
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 300b4a64d8fe..5f15820efe12 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -682,7 +682,7 @@ int r100_irq_process(struct radeon_device *rdev)
         /* reset gui idle ack.  the status bit is broken */
         rdev->irq.gui_idle_acked = false;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         if (rdev->msi_enabled) {
                 switch (rdev->family) {
                 case CHIP_RS400:
@@ -1031,8 +1031,8 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
         WREG32(RADEON_CP_CSQ_MODE,
                REG_SET(RADEON_INDIRECT2_START, indirect2_start) |
                REG_SET(RADEON_INDIRECT1_START, indirect1_start));
-        WREG32(0x718, 0);
-        WREG32(0x744, 0x00004D4D);
+        WREG32(RADEON_CP_RB_WPTR_DELAY, 0);
+        WREG32(RADEON_CP_CSQ_MODE, 0x00004D4D);
         WREG32(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIBM_INDBM);
         radeon_ring_start(rdev);
         r = radeon_ring_test(rdev);
@@ -2086,12 +2086,13 @@ int r100_asic_reset(struct radeon_device *rdev)
 {
         struct r100_mc_save save;
         u32 status, tmp;
+        int ret = 0;
 
-        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(status)) {
                 return 0;
         }
+        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* stop CP */
@@ -2131,11 +2132,11 @@ int r100_asic_reset(struct radeon_device *rdev)
                 G_000E40_TAM_BUSY(status) || G_000E40_PB_BUSY(status)) {
                 dev_err(rdev->dev, "failed to reset GPU\n");
                 rdev->gpu_lockup = true;
-                return -1;
-        }
+                ret = -1;
+        } else
+                dev_info(rdev->dev, "GPU reset succeed\n");
         r100_mc_resume(rdev, &save);
-        dev_info(rdev->dev, "GPU reset succeed\n");
-        return 0;
+        return ret;
 }
 
 void r100_set_common_regs(struct radeon_device *rdev)
@@ -2346,10 +2347,10 @@ void r100_vga_set_state(struct radeon_device *rdev, bool state)
 
         temp = RREG32(RADEON_CONFIG_CNTL);
         if (state == false) {
-                temp &= ~(1<<8);
-                temp |= (1<<9);
+                temp &= ~RADEON_CFG_VGA_RAM_EN;
+                temp |= RADEON_CFG_VGA_IO_DIS;
         } else {
-                temp &= ~(1<<9);
+                temp &= ~RADEON_CFG_VGA_IO_DIS;
         }
         WREG32(RADEON_CONFIG_CNTL, temp);
 }
@@ -3521,7 +3522,7 @@ int r100_ring_test(struct radeon_device *rdev)
         if (i < rdev->usec_timeout) {
                 DRM_INFO("ring test succeeded in %d usecs\n", i);
         } else {
-                DRM_ERROR("radeon: ring test failed (sracth(0x%04X)=0x%08X)\n",
+                DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
                           scratch, tmp);
                 r = -EINVAL;
         }
diff --git a/drivers/gpu/drm/radeon/r100d.h b/drivers/gpu/drm/radeon/r100d.h
index b121b6c678d4..eab91760fae0 100644
--- a/drivers/gpu/drm/radeon/r100d.h
+++ b/drivers/gpu/drm/radeon/r100d.h
@@ -551,7 +551,7 @@
 #define   S_000360_CUR2_LOCK(x)                        (((x) & 0x1) << 31)
 #define   G_000360_CUR2_LOCK(x)                        (((x) >> 31) & 0x1)
 #define   C_000360_CUR2_LOCK                           0x7FFFFFFF
-#define R_0003C2_GENMO_WT                            0x0003C0
+#define R_0003C2_GENMO_WT                            0x0003C2
 #define   S_0003C2_GENMO_MONO_ADDRESS_B(x)             (((x) & 0x1) << 0)
 #define   G_0003C2_GENMO_MONO_ADDRESS_B(x)             (((x) >> 0) & 0x1)
 #define   C_0003C2_GENMO_MONO_ADDRESS_B                0xFE
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index cde1d3480d93..55fe5ba7def3 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -69,6 +69,9 @@ void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev)
         mb();
 }
 
+#define R300_PTE_WRITEABLE (1 << 2)
+#define R300_PTE_READABLE  (1 << 3)
+
 int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
 {
         void __iomem *ptr = (void *)rdev->gart.table.vram.ptr;
@@ -78,7 +81,7 @@ int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
         }
         addr = (lower_32_bits(addr) >> 8) |
                ((upper_32_bits(addr) & 0xff) << 24) |
-               0xc;
+               R300_PTE_WRITEABLE | R300_PTE_READABLE;
         /* on x86 we want this to be CPU endian, on powerpc
          * on powerpc without HW swappers, it'll get swapped on way
          * into VRAM - so no need for cpu_to_le32 on VRAM tables */
@@ -135,7 +138,7 @@ int rv370_pcie_gart_enable(struct radeon_device *rdev)
         WREG32_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO, rdev->mc.vram_start);
         WREG32_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_HI, 0);
         /* Clear error */
-        WREG32_PCIE(0x18, 0);
+        WREG32_PCIE(RADEON_PCIE_TX_GART_ERROR, 0);
         tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
         tmp |= RADEON_PCIE_TX_GART_EN;
         tmp |= RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD;
@@ -405,12 +408,13 @@ int r300_asic_reset(struct radeon_device *rdev)
 {
         struct r100_mc_save save;
         u32 status, tmp;
+        int ret = 0;
 
-        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(status)) {
                 return 0;
         }
+        r100_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* stop CP */
@@ -451,11 +455,11 @@ int r300_asic_reset(struct radeon_device *rdev)
         if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) {
                 dev_err(rdev->dev, "failed to reset GPU\n");
                 rdev->gpu_lockup = true;
-                return -1;
-        }
+                ret = -1;
+        } else
+                dev_info(rdev->dev, "GPU reset succeed\n");
         r100_mc_resume(rdev, &save);
-        dev_info(rdev->dev, "GPU reset succeed\n");
-        return 0;
+        return ret;
 }
 
 /*
@@ -558,10 +562,7 @@ int rv370_get_pcie_lanes(struct radeon_device *rdev)
 
         /* FIXME wait for idle */
 
-        if (rdev->family < CHIP_R600)
-                link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
-        else
-                link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+        link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
 
         switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
         case RADEON_PCIE_LC_LINK_WIDTH_X0:
@@ -745,6 +746,11 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 break;
         case 0x4E00:
                 /* RB3D_CCTL */
+                if ((idx_value & (1 << 10)) && /* CMASK_ENABLE */
+                    p->rdev->cmask_filp != p->filp) {
+                        DRM_ERROR("Invalid RB3D_CCTL: Cannot enable CMASK.\n");
+                        return -EINVAL;
+                }
                 track->num_cb = ((idx_value >> 5) & 0x3) + 1;
                 break;
         case 0x4E38:
@@ -787,6 +793,13 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 case 15:
                         track->cb[i].cpp = 2;
                         break;
+                case 5:
+                        if (p->rdev->family < CHIP_RV515) {
+                                DRM_ERROR("Invalid color buffer format (%d)!\n",
+                                          ((idx_value >> 21) & 0xF));
+                                return -EINVAL;
+                        }
+                        /* Pass through. */
                 case 6:
                         track->cb[i].cpp = 4;
                         break;
@@ -1199,6 +1212,10 @@ static int r300_packet3_check(struct radeon_cs_parser *p,
                 if (p->rdev->hyperz_filp != p->filp)
                         return -EINVAL;
                 break;
+        case PACKET3_3D_CLEAR_CMASK:
+                if (p->rdev->cmask_filp != p->filp)
+                        return -EINVAL;
+                break;
         case PACKET3_NOP:
                 break;
         default:
diff --git a/drivers/gpu/drm/radeon/r300d.h b/drivers/gpu/drm/radeon/r300d.h
index 0c036c60d9df..1f519a5ffb8c 100644
--- a/drivers/gpu/drm/radeon/r300d.h
+++ b/drivers/gpu/drm/radeon/r300d.h
@@ -54,6 +54,7 @@
 #define         PACKET3_3D_DRAW_IMMD_2          0x35
 #define         PACKET3_3D_DRAW_INDX_2          0x36
 #define         PACKET3_3D_CLEAR_HIZ            0x37
+#define         PACKET3_3D_CLEAR_CMASK          0x38
 #define         PACKET3_BITBLT_MULTI            0x9B
 
 #define PACKET0(reg, n) (CP_PACKET0 |                                   \
diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c
index c387346f93a9..0b59ed7c7d2c 100644
--- a/drivers/gpu/drm/radeon/r420.c
+++ b/drivers/gpu/drm/radeon/r420.c
@@ -96,7 +96,7 @@ void r420_pipes_init(struct radeon_device *rdev)
                        "programming pipes. Bad things might happen.\n");
         }
         /* get max number of pipes */
-        gb_pipe_select = RREG32(0x402C);
+        gb_pipe_select = RREG32(R400_GB_PIPE_SELECT);
         num_pipes = ((gb_pipe_select >> 12) & 3) + 1;
 
         /* SE chips have 1 pipe */
diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c
index 3c8677f9e385..2ce80d976568 100644
--- a/drivers/gpu/drm/radeon/r520.c
+++ b/drivers/gpu/drm/radeon/r520.c
@@ -79,8 +79,8 @@ static void r520_gpu_init(struct radeon_device *rdev)
                 WREG32(0x4128, 0xFF);
         }
         r420_pipes_init(rdev);
-        gb_pipe_select = RREG32(0x402C);
-        tmp = RREG32(0x170C);
+        gb_pipe_select = RREG32(R400_GB_PIPE_SELECT);
+        tmp = RREG32(R300_DST_PIPE_CONFIG);
         pipe_select_current = (tmp >> 2) & 3;
         tmp = (1 << pipe_select_current) |
               (((gb_pipe_select >> 8) & 0xF) << 4);
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 53bfe3afb0fa..be780a6b9b1d 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -94,14 +94,19 @@ int r600_mc_wait_for_idle(struct radeon_device *rdev);
 void r600_gpu_init(struct radeon_device *rdev);
 void r600_fini(struct radeon_device *rdev);
 void r600_irq_disable(struct radeon_device *rdev);
+static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 
 /* get temperature in millidegrees */
-u32 rv6xx_get_temp(struct radeon_device *rdev)
+int rv6xx_get_temp(struct radeon_device *rdev)
 {
         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
                 ASIC_T_SHIFT;
+        int actual_temp = temp & 0xff;
 
-        return temp * 1000;
+        if (temp & 0x100)
+                actual_temp -= 256;
+
+        return actual_temp * 1000;
 }
 
 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
@@ -881,12 +886,15 @@ void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
         u32 tmp;
 
         /* flush hdp cache so updates hit vram */
-        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740)) {
+        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
+            !(rdev->flags & RADEON_IS_AGP)) {
                 void __iomem *ptr = (void *)rdev->gart.table.vram.ptr;
                 u32 tmp;
 
                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
+                 * This seems to cause problems on some AGP cards. Just use the old
+                 * method for them.
                  */
                 WREG32(HDP_DEBUG1, 0);
                 tmp = readl((void __iomem *)ptr);
@@ -1198,8 +1206,10 @@ static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc
                                 mc->vram_end, mc->real_vram_size >> 20);
         } else {
                 u64 base = 0;
-                if (rdev->flags & RADEON_IS_IGP)
-                        base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
+                if (rdev->flags & RADEON_IS_IGP) {
+                        base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
+                        base <<= 24;
+                }
                 radeon_vram_location(rdev, &rdev->mc, base);
                 rdev->mc.gtt_base_align = 0;
                 radeon_gtt_location(rdev, mc);
@@ -1281,6 +1291,9 @@ int r600_gpu_soft_reset(struct radeon_device *rdev)
                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
         u32 tmp;
 
+        if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
+                return 0;
+
         dev_info(rdev->dev, "GPU softreset \n");
         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
                 RREG32(R_008010_GRBM_STATUS));
@@ -1340,13 +1353,19 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
         u32 srbm_status;
         u32 grbm_status;
         u32 grbm_status2;
+        struct r100_gpu_lockup *lockup;
         int r;
 
+        if (rdev->family >= CHIP_RV770)
+                lockup = &rdev->config.rv770.lockup;
+        else
+                lockup = &rdev->config.r600.lockup;
+
         srbm_status = RREG32(R_000E50_SRBM_STATUS);
         grbm_status = RREG32(R_008010_GRBM_STATUS);
         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
         if (!G_008010_GUI_ACTIVE(grbm_status)) {
-                r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
+                r100_gpu_lockup_update(lockup, &rdev->cp);
                 return false;
         }
         /* force CP activities */
@@ -1358,7 +1377,7 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
                 radeon_ring_unlock_commit(rdev);
         }
         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
-        return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
+        return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
 }
 
 int r600_asic_reset(struct radeon_device *rdev)
@@ -2346,28 +2365,13 @@ void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
         /* FIXME: implement */
 }
 
-
-bool r600_card_posted(struct radeon_device *rdev)
-{
-        uint32_t reg;
-
-        /* first check CRTCs */
-        reg = RREG32(D1CRTC_CONTROL) |
-                RREG32(D2CRTC_CONTROL);
-        if (reg & CRTC_EN)
-                return true;
-
-        /* then check MEM_SIZE, in case the crtcs are off */
-        if (RREG32(CONFIG_MEMSIZE))
-                return true;
-
-        return false;
-}
-
 int r600_startup(struct radeon_device *rdev)
 {
         int r;
 
+        /* enable pcie gen2 link */
+        r600_pcie_gen2_enable(rdev);
+
         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
                 r = r600_init_microcode(rdev);
                 if (r) {
@@ -2521,7 +2525,7 @@ int r600_init(struct radeon_device *rdev)
         if (r)
                 return r;
         /* Post card if necessary */
-        if (!r600_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -2724,7 +2728,7 @@ static int r600_ih_ring_alloc(struct radeon_device *rdev)
 
         /* Allocate ring buffer */
         if (rdev->ih.ring_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
+                r = radeon_bo_create(rdev, rdev->ih.ring_size,
                                      PAGE_SIZE, true,
                                      RADEON_GEM_DOMAIN_GTT,
                                      &rdev->ih.ring_obj);
@@ -3431,7 +3435,7 @@ restart_ih:
         if (wptr != rdev->ih.wptr)
                 goto restart_ih;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -3506,10 +3510,12 @@ int r600_debugfs_mc_info_init(struct radeon_device *rdev)
 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
 {
         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
-         * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
+         * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
+         * This seems to cause problems on some AGP cards. Just use the old
+         * method for them.
          */
         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
-            rdev->vram_scratch.ptr) {
+            rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
                 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
                 u32 tmp;
 
@@ -3518,3 +3524,222 @@ void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
         } else
                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 }
+
+void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
+{
+        u32 link_width_cntl, mask, target_reg;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* FIXME wait for idle */
+
+        switch (lanes) {
+        case 0:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
+                break;
+        case 1:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
+                break;
+        case 2:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
+                break;
+        case 4:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
+                break;
+        case 8:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
+                break;
+        case 12:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
+                break;
+        case 16:
+        default:
+                mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
+                break;
+        }
+
+        link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+
+        if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
+            (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
+                return;
+
+        if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
+                return;
+
+        link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
+                             RADEON_PCIE_LC_RECONFIG_NOW |
+                             R600_PCIE_LC_RENEGOTIATE_EN |
+                             R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
+        link_width_cntl |= mask;
+
+        WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+
+        /* some northbridges can renegotiate the link rather than requiring                                  
+         * a complete re-config.                                                                             
+         * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)                            
+         */
+        if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
+                link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
+        else
+                link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
+
+        WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
+                                                       RADEON_PCIE_LC_RECONFIG_NOW));
+
+        if (rdev->family >= CHIP_RV770)
+                target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
+        else
+                target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
+
+        /* wait for lane set to complete */
+        link_width_cntl = RREG32(target_reg);
+        while (link_width_cntl == 0xffffffff)
+                link_width_cntl = RREG32(target_reg);
+
+}
+
+int r600_get_pcie_lanes(struct radeon_device *rdev)
+{
+        u32 link_width_cntl;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return 0;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return 0;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return 0;
+
+        /* FIXME wait for idle */
+
+        link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
+
+        switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
+        case RADEON_PCIE_LC_LINK_WIDTH_X0:
+                return 0;
+        case RADEON_PCIE_LC_LINK_WIDTH_X1:
+                return 1;
+        case RADEON_PCIE_LC_LINK_WIDTH_X2:
+                return 2;
+        case RADEON_PCIE_LC_LINK_WIDTH_X4:
+                return 4;
+        case RADEON_PCIE_LC_LINK_WIDTH_X8:
+                return 8;
+        case RADEON_PCIE_LC_LINK_WIDTH_X16:
+        default:
+                return 16;
+        }
+}
+
+static void r600_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
+        u16 link_cntl2;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* only RV6xx+ chips are supported */
+        if (rdev->family <= CHIP_R600)
+                return;
+
+        /* 55 nm r6xx asics */
+        if ((rdev->family == CHIP_RV670) ||
+            (rdev->family == CHIP_RV620) ||
+            (rdev->family == CHIP_RV635)) {
+                /* advertise upconfig capability */
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
+                        lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
+                        link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
+                                             LC_RECONFIG_ARC_MISSING_ESCAPE);
+                        link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
+                        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                } else {
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+                }
+        }
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                /* 55 nm r6xx asics */
+                if ((rdev->family == CHIP_RV670) ||
+                    (rdev->family == CHIP_RV620) ||
+                    (rdev->family == CHIP_RV635)) {
+                        WREG32(MM_CFGREGS_CNTL, 0x8);
+                        link_cntl2 = RREG32(0x4088);
+                        WREG32(MM_CFGREGS_CNTL, 0);
+                        /* not supported yet */
+                        if (link_cntl2 & SELECTABLE_DEEMPHASIS)
+                                return;
+                }
+
+                speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
+                speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
+                speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
+                speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
+                speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                tmp = RREG32(0x541c);
+                WREG32(0x541c, tmp | 0x8);
+                WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
+                link_cntl2 = RREG16(0x4088);
+                link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
+                link_cntl2 |= 0x2;
+                WREG16(0x4088, link_cntl2);
+                WREG32(MM_CFGREGS_CNTL, 0);
+
+                if ((rdev->family == CHIP_RV670) ||
+                    (rdev->family == CHIP_RV620) ||
+                    (rdev->family == CHIP_RV635)) {
+                        training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
+                        training_cntl &= ~LC_POINT_7_PLUS_EN;
+                        WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
+                } else {
+                        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                        speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                        WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+                }
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+}
diff --git a/drivers/gpu/drm/radeon/r600_audio.c b/drivers/gpu/drm/radeon/r600_audio.c
index b5443fe1c1d1..846fae576399 100644
--- a/drivers/gpu/drm/radeon/r600_audio.c
+++ b/drivers/gpu/drm/radeon/r600_audio.c
@@ -26,6 +26,7 @@
 #include "drmP.h"
 #include "radeon.h"
 #include "radeon_reg.h"
+#include "radeon_asic.h"
 #include "atom.h"
 
 #define AUDIO_TIMER_INTERVALL 100 /* 1/10 sekund should be enough */
diff --git a/drivers/gpu/drm/radeon/r600_blit_kms.c b/drivers/gpu/drm/radeon/r600_blit_kms.c
index 86e5aa07f0db..16e211a614d7 100644
--- a/drivers/gpu/drm/radeon/r600_blit_kms.c
+++ b/drivers/gpu/drm/radeon/r600_blit_kms.c
@@ -501,7 +501,7 @@ int r600_blit_init(struct radeon_device *rdev)
         obj_size += r6xx_ps_size * 4;
         obj_size = ALIGN(obj_size, 256);
 
-        r = radeon_bo_create(rdev, NULL, obj_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
+        r = radeon_bo_create(rdev, obj_size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                 &rdev->r600_blit.shader_obj);
         if (r) {
                 DRM_ERROR("r600 failed to allocate shader\n");
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 9bebac1ec006..4706294f0ae0 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -71,75 +71,164 @@ struct r600_cs_track {
         u64                     db_bo_mc;
 };
 
+#define FMT_8_BIT(fmt, vc) [fmt] = { 1, 1, 1, vc }
+#define FMT_16_BIT(fmt, vc) [fmt] = { 1, 1, 2, vc }
+#define FMT_24_BIT(fmt) [fmt] = { 1, 1, 3, 0 }
+#define FMT_32_BIT(fmt, vc) [fmt] = { 1, 1, 4, vc }
+#define FMT_48_BIT(fmt) [fmt] = { 1, 1, 6, 0 }
+#define FMT_64_BIT(fmt, vc) [fmt] = { 1, 1, 8, vc }
+#define FMT_96_BIT(fmt) [fmt] = { 1, 1, 12, 0 }
+#define FMT_128_BIT(fmt, vc) [fmt] = { 1, 1, 16, vc }
+
+struct gpu_formats {
+        unsigned blockwidth;
+        unsigned blockheight;
+        unsigned blocksize;
+        unsigned valid_color;
+};
+
+static const struct gpu_formats color_formats_table[] = {
+        /* 8 bit */
+        FMT_8_BIT(V_038004_COLOR_8, 1),
+        FMT_8_BIT(V_038004_COLOR_4_4, 1),
+        FMT_8_BIT(V_038004_COLOR_3_3_2, 1),
+        FMT_8_BIT(V_038004_FMT_1, 0),
+
+        /* 16-bit */
+        FMT_16_BIT(V_038004_COLOR_16, 1),
+        FMT_16_BIT(V_038004_COLOR_16_FLOAT, 1),
+        FMT_16_BIT(V_038004_COLOR_8_8, 1),
+        FMT_16_BIT(V_038004_COLOR_5_6_5, 1),
+        FMT_16_BIT(V_038004_COLOR_6_5_5, 1),
+        FMT_16_BIT(V_038004_COLOR_1_5_5_5, 1),
+        FMT_16_BIT(V_038004_COLOR_4_4_4_4, 1),
+        FMT_16_BIT(V_038004_COLOR_5_5_5_1, 1),
+
+        /* 24-bit */
+        FMT_24_BIT(V_038004_FMT_8_8_8),
+                                               
+        /* 32-bit */
+        FMT_32_BIT(V_038004_COLOR_32, 1),
+        FMT_32_BIT(V_038004_COLOR_32_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_16_16, 1),
+        FMT_32_BIT(V_038004_COLOR_16_16_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_8_24, 1),
+        FMT_32_BIT(V_038004_COLOR_8_24_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_24_8, 1),
+        FMT_32_BIT(V_038004_COLOR_24_8_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_10_11_11, 1),
+        FMT_32_BIT(V_038004_COLOR_10_11_11_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_11_11_10, 1),
+        FMT_32_BIT(V_038004_COLOR_11_11_10_FLOAT, 1),
+        FMT_32_BIT(V_038004_COLOR_2_10_10_10, 1),
+        FMT_32_BIT(V_038004_COLOR_8_8_8_8, 1),
+        FMT_32_BIT(V_038004_COLOR_10_10_10_2, 1),
+        FMT_32_BIT(V_038004_FMT_5_9_9_9_SHAREDEXP, 0),
+        FMT_32_BIT(V_038004_FMT_32_AS_8, 0),
+        FMT_32_BIT(V_038004_FMT_32_AS_8_8, 0),
+
+        /* 48-bit */
+        FMT_48_BIT(V_038004_FMT_16_16_16),
+        FMT_48_BIT(V_038004_FMT_16_16_16_FLOAT),
+
+        /* 64-bit */
+        FMT_64_BIT(V_038004_COLOR_X24_8_32_FLOAT, 1),
+        FMT_64_BIT(V_038004_COLOR_32_32, 1),
+        FMT_64_BIT(V_038004_COLOR_32_32_FLOAT, 1),
+        FMT_64_BIT(V_038004_COLOR_16_16_16_16, 1),
+        FMT_64_BIT(V_038004_COLOR_16_16_16_16_FLOAT, 1),
+
+        FMT_96_BIT(V_038004_FMT_32_32_32),
+        FMT_96_BIT(V_038004_FMT_32_32_32_FLOAT),
+
+        /* 128-bit */
+        FMT_128_BIT(V_038004_COLOR_32_32_32_32, 1),
+        FMT_128_BIT(V_038004_COLOR_32_32_32_32_FLOAT, 1),
+
+        [V_038004_FMT_GB_GR] = { 2, 1, 4, 0 },
+        [V_038004_FMT_BG_RG] = { 2, 1, 4, 0 },
+
+        /* block compressed formats */
+        [V_038004_FMT_BC1] = { 4, 4, 8, 0 },
+        [V_038004_FMT_BC2] = { 4, 4, 16, 0 },
+        [V_038004_FMT_BC3] = { 4, 4, 16, 0 },
+        [V_038004_FMT_BC4] = { 4, 4, 8, 0 },
+        [V_038004_FMT_BC5] = { 4, 4, 16, 0},
+
+};
+
+static inline bool fmt_is_valid_color(u32 format)
+{
+        if (format > ARRAY_SIZE(color_formats_table))
+                return false;
+        
+        if (color_formats_table[format].valid_color)
+                return true;
+
+        return false;
+}
+
+static inline bool fmt_is_valid_texture(u32 format)
+{
+        if (format > ARRAY_SIZE(color_formats_table))
+                return false;
+        
+        if (color_formats_table[format].blockwidth > 0)
+                return true;
+
+        return false;
+}
+
+static inline int fmt_get_blocksize(u32 format)
+{
+        if (format > ARRAY_SIZE(color_formats_table))
+                return 0;
+
+        return color_formats_table[format].blocksize;
+}
+
+static inline int fmt_get_nblocksx(u32 format, u32 w)
+{
+        unsigned bw;
+        if (format > ARRAY_SIZE(color_formats_table))
+                return 0;
+
+        bw = color_formats_table[format].blockwidth;
+        if (bw == 0)
+                return 0;
+
+        return (w + bw - 1) / bw;
+}
+
+static inline int fmt_get_nblocksy(u32 format, u32 h)
+{
+        unsigned bh;
+        if (format > ARRAY_SIZE(color_formats_table))
+                return 0;
+
+        bh = color_formats_table[format].blockheight;
+        if (bh == 0)
+                return 0;
+
+        return (h + bh - 1) / bh;
+}
+
 static inline int r600_bpe_from_format(u32 *bpe, u32 format)
 {
-        switch (format) {
-        case V_038004_COLOR_8:
-        case V_038004_COLOR_4_4:
-        case V_038004_COLOR_3_3_2:
-        case V_038004_FMT_1:
-                *bpe = 1;
-                break;
-        case V_038004_COLOR_16:
-        case V_038004_COLOR_16_FLOAT:
-        case V_038004_COLOR_8_8:
-        case V_038004_COLOR_5_6_5:
-        case V_038004_COLOR_6_5_5:
-        case V_038004_COLOR_1_5_5_5:
-        case V_038004_COLOR_4_4_4_4:
-        case V_038004_COLOR_5_5_5_1:
-                *bpe = 2;
-                break;
-        case V_038004_FMT_8_8_8:
-                *bpe = 3;
-                break;
-        case V_038004_COLOR_32:
-        case V_038004_COLOR_32_FLOAT:
-        case V_038004_COLOR_16_16:
-        case V_038004_COLOR_16_16_FLOAT:
-        case V_038004_COLOR_8_24:
-        case V_038004_COLOR_8_24_FLOAT:
-        case V_038004_COLOR_24_8:
-        case V_038004_COLOR_24_8_FLOAT:
-        case V_038004_COLOR_10_11_11:
-        case V_038004_COLOR_10_11_11_FLOAT:
-        case V_038004_COLOR_11_11_10:
-        case V_038004_COLOR_11_11_10_FLOAT:
-        case V_038004_COLOR_2_10_10_10:
-        case V_038004_COLOR_8_8_8_8:
-        case V_038004_COLOR_10_10_10_2:
-        case V_038004_FMT_5_9_9_9_SHAREDEXP:
-        case V_038004_FMT_32_AS_8:
-        case V_038004_FMT_32_AS_8_8:
-                *bpe = 4;
-                break;
-        case V_038004_COLOR_X24_8_32_FLOAT:
-        case V_038004_COLOR_32_32:
-        case V_038004_COLOR_32_32_FLOAT:
-        case V_038004_COLOR_16_16_16_16:
-        case V_038004_COLOR_16_16_16_16_FLOAT:
-                *bpe = 8;
-                break;
-        case V_038004_FMT_16_16_16:
-        case V_038004_FMT_16_16_16_FLOAT:
-                *bpe = 6;
-                break;
-        case V_038004_FMT_32_32_32:
-        case V_038004_FMT_32_32_32_FLOAT:
-                *bpe = 12;
-                break;
-        case V_038004_COLOR_32_32_32_32:
-        case V_038004_COLOR_32_32_32_32_FLOAT:
-                *bpe = 16;
-                break;
-        case V_038004_FMT_GB_GR:
-        case V_038004_FMT_BG_RG:
-        case V_038004_COLOR_INVALID:
-        default:
-                *bpe = 16;
-                return -EINVAL;
-        }
+        unsigned res;
+        if (format > ARRAY_SIZE(color_formats_table))
+                goto fail;
+
+        res = color_formats_table[format].blocksize;
+        if (res == 0)
+                goto fail;
+
+        *bpe = res;
         return 0;
+
+fail:
+        *bpe = 16;
+        return -EINVAL;
 }
 
 struct array_mode_checker {
@@ -148,7 +237,7 @@ struct array_mode_checker {
         u32 nbanks;
         u32 npipes;
         u32 nsamples;
-        u32 bpe;
+        u32 blocksize;
 };
 
 /* returns alignment in pixels for pitch/height/depth and bytes for base */
@@ -162,7 +251,7 @@ static inline int r600_get_array_mode_alignment(struct array_mode_checker *value
         u32 tile_height = 8;
         u32 macro_tile_width = values->nbanks;
         u32 macro_tile_height = values->npipes;
-        u32 tile_bytes = tile_width * tile_height * values->bpe * values->nsamples;
+        u32 tile_bytes = tile_width * tile_height * values->blocksize * values->nsamples;
         u32 macro_tile_bytes = macro_tile_width * macro_tile_height * tile_bytes;
 
         switch (values->array_mode) {
@@ -174,7 +263,7 @@ static inline int r600_get_array_mode_alignment(struct array_mode_checker *value
                 *base_align = 1;
                 break;
         case ARRAY_LINEAR_ALIGNED:
-                *pitch_align = max((u32)64, (u32)(values->group_size / values->bpe));
+                *pitch_align = max((u32)64, (u32)(values->group_size / values->blocksize));
                 *height_align = tile_height;
                 *depth_align = 1;
                 *base_align = values->group_size;
@@ -182,7 +271,7 @@ static inline int r600_get_array_mode_alignment(struct array_mode_checker *value
         case ARRAY_1D_TILED_THIN1:
                 *pitch_align = max((u32)tile_width,
                                    (u32)(values->group_size /
-                                         (tile_height * values->bpe * values->nsamples)));
+                                         (tile_height * values->blocksize * values->nsamples)));
                 *height_align = tile_height;
                 *depth_align = 1;
                 *base_align = values->group_size;
@@ -190,12 +279,12 @@ static inline int r600_get_array_mode_alignment(struct array_mode_checker *value
         case ARRAY_2D_TILED_THIN1:
                 *pitch_align = max((u32)macro_tile_width,
                                   (u32)(((values->group_size / tile_height) /
-                                         (values->bpe * values->nsamples)) *
+                                         (values->blocksize * values->nsamples)) *
                                         values->nbanks)) * tile_width;
                 *height_align = macro_tile_height * tile_height;
                 *depth_align = 1;
                 *base_align = max(macro_tile_bytes,
-                                  (*pitch_align) * values->bpe * (*height_align) * values->nsamples);
+                                  (*pitch_align) * values->blocksize * (*height_align) * values->nsamples);
                 break;
         default:
                 return -EINVAL;
@@ -234,21 +323,22 @@ static void r600_cs_track_init(struct r600_cs_track *track)
 static inline int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i)
 {
         struct r600_cs_track *track = p->track;
-        u32 bpe = 0, slice_tile_max, size, tmp;
+        u32 slice_tile_max, size, tmp;
         u32 height, height_align, pitch, pitch_align, depth_align;
         u64 base_offset, base_align;
         struct array_mode_checker array_check;
         volatile u32 *ib = p->ib->ptr;
         unsigned array_mode;
-
+        u32 format;
         if (G_0280A0_TILE_MODE(track->cb_color_info[i])) {
                 dev_warn(p->dev, "FMASK or CMASK buffer are not supported by this kernel\n");
                 return -EINVAL;
         }
         size = radeon_bo_size(track->cb_color_bo[i]) - track->cb_color_bo_offset[i];
-        if (r600_bpe_from_format(&bpe, G_0280A0_FORMAT(track->cb_color_info[i]))) {
+        format = G_0280A0_FORMAT(track->cb_color_info[i]);
+        if (!fmt_is_valid_color(format)) {
                 dev_warn(p->dev, "%s:%d cb invalid format %d for %d (0x%08X)\n",
-                         __func__, __LINE__, G_0280A0_FORMAT(track->cb_color_info[i]),
+                         __func__, __LINE__, format,
                         i, track->cb_color_info[i]);
                 return -EINVAL;
         }
@@ -267,7 +357,7 @@ static inline int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i)
         array_check.nbanks = track->nbanks;
         array_check.npipes = track->npipes;
         array_check.nsamples = track->nsamples;
-        array_check.bpe = bpe;
+        array_check.blocksize = fmt_get_blocksize(format);
         if (r600_get_array_mode_alignment(&array_check,
                                           &pitch_align, &height_align, &depth_align, &base_align)) {
                 dev_warn(p->dev, "%s invalid tiling %d for %d (0x%08X)\n", __func__,
@@ -310,16 +400,15 @@ static inline int r600_cs_track_validate_cb(struct radeon_cs_parser *p, int i)
         }
 
         /* check offset */
-        tmp = height * pitch * bpe;
+        tmp = fmt_get_nblocksy(format, height) * fmt_get_nblocksx(format, pitch) * fmt_get_blocksize(format);
         if ((tmp + track->cb_color_bo_offset[i]) > radeon_bo_size(track->cb_color_bo[i])) {
                 if (array_mode == V_0280A0_ARRAY_LINEAR_GENERAL) {
                         /* the initial DDX does bad things with the CB size occasionally */
                         /* it rounds up height too far for slice tile max but the BO is smaller */
-                        tmp = (height - 7) * pitch * bpe;
-                        if ((tmp + track->cb_color_bo_offset[i]) > radeon_bo_size(track->cb_color_bo[i])) {
-                                dev_warn(p->dev, "%s offset[%d] %d %d %lu too big\n", __func__, i, track->cb_color_bo_offset[i], tmp, radeon_bo_size(track->cb_color_bo[i]));
-                                return -EINVAL;
-                        }
+                        /* r600c,g also seem to flush at bad times in some apps resulting in
+                         * bogus values here. So for linear just allow anything to avoid breaking
+                         * broken userspace.
+                         */
                 } else {
                         dev_warn(p->dev, "%s offset[%d] %d %d %lu too big\n", __func__, i, track->cb_color_bo_offset[i], tmp, radeon_bo_size(track->cb_color_bo[i]));
                         return -EINVAL;
@@ -433,7 +522,7 @@ static int r600_cs_track_check(struct radeon_cs_parser *p)
                         array_check.nbanks = track->nbanks;
                         array_check.npipes = track->npipes;
                         array_check.nsamples = track->nsamples;
-                        array_check.bpe = bpe;
+                        array_check.blocksize = bpe;
                         if (r600_get_array_mode_alignment(&array_check,
                                                           &pitch_align, &height_align, &depth_align, &base_align)) {
                                 dev_warn(p->dev, "%s invalid tiling %d (0x%08X)\n", __func__,
@@ -1108,39 +1197,61 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
         return 0;
 }
 
-static inline unsigned minify(unsigned size, unsigned levels)
+static inline unsigned mip_minify(unsigned size, unsigned level)
 {
-        size = size >> levels;
-        if (size < 1)
-                size = 1;
-        return size;
+        unsigned val;
+
+        val = max(1U, size >> level);
+        if (level > 0)
+                val = roundup_pow_of_two(val);
+        return val;
 }
 
-static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned nlevels,
-                              unsigned w0, unsigned h0, unsigned d0, unsigned bpe,
-                              unsigned pitch_align,
+static void r600_texture_size(unsigned nfaces, unsigned blevel, unsigned llevel,
+                              unsigned w0, unsigned h0, unsigned d0, unsigned format,
+                              unsigned block_align, unsigned height_align, unsigned base_align,
                               unsigned *l0_size, unsigned *mipmap_size)
 {
-        unsigned offset, i, level, face;
-        unsigned width, height, depth, rowstride, size;
-
-        w0 = minify(w0, 0);
-        h0 = minify(h0, 0);
-        d0 = minify(d0, 0);
+        unsigned offset, i, level;
+        unsigned width, height, depth, size;
+        unsigned blocksize;
+        unsigned nbx, nby;
+        unsigned nlevels = llevel - blevel + 1;
+
+        *l0_size = -1;
+        blocksize = fmt_get_blocksize(format);
+
+        w0 = mip_minify(w0, 0);
+        h0 = mip_minify(h0, 0);
+        d0 = mip_minify(d0, 0);
         for(i = 0, offset = 0, level = blevel; i < nlevels; i++, level++) {
-                width = minify(w0, i);
-                height = minify(h0, i);
-                depth = minify(d0, i);
-                for(face = 0; face < nfaces; face++) {
-                        rowstride = ALIGN((width * bpe), pitch_align);
-                        size = height * rowstride * depth;
-                        offset += size;
-                        offset = (offset + 0x1f) & ~0x1f;
-                }
+                width = mip_minify(w0, i);
+                nbx = fmt_get_nblocksx(format, width);
+
+                nbx = round_up(nbx, block_align);
+
+                height = mip_minify(h0, i);
+                nby = fmt_get_nblocksy(format, height);
+                nby = round_up(nby, height_align);
+
+                depth = mip_minify(d0, i);
+
+                size = nbx * nby * blocksize;
+                if (nfaces)
+                        size *= nfaces;
+                else
+                        size *= depth;
+
+                if (i == 0)
+                        *l0_size = size;
+
+                if (i == 0 || i == 1)
+                        offset = round_up(offset, base_align);
+
+                offset += size;
         }
-        *l0_size = ALIGN((w0 * bpe), pitch_align) * h0 * d0;
         *mipmap_size = offset;
-        if (!nlevels)
+        if (llevel == 0)
                 *mipmap_size = *l0_size;
         if (!blevel)
                 *mipmap_size -= *l0_size;
@@ -1164,11 +1275,13 @@ static inline int r600_check_texture_resource(struct radeon_cs_parser *p,  u32 i
                                               u32 tiling_flags)
 {
         struct r600_cs_track *track = p->track;
-        u32 nfaces, nlevels, blevel, w0, h0, d0, bpe = 0;
+        u32 nfaces, llevel, blevel, w0, h0, d0;
         u32 word0, word1, l0_size, mipmap_size;
         u32 height_align, pitch, pitch_align, depth_align;
+        u32 array, barray, larray;
         u64 base_align;
         struct array_mode_checker array_check;
+        u32 format;
 
         /* on legacy kernel we don't perform advanced check */
         if (p->rdev == NULL)
@@ -1194,19 +1307,25 @@ static inline int r600_check_texture_resource(struct radeon_cs_parser *p,  u32 i
         case V_038000_SQ_TEX_DIM_3D:
                 break;
         case V_038000_SQ_TEX_DIM_CUBEMAP:
-                nfaces = 6;
+                if (p->family >= CHIP_RV770)
+                        nfaces = 8;
+                else
+                        nfaces = 6;
                 break;
         case V_038000_SQ_TEX_DIM_1D_ARRAY:
         case V_038000_SQ_TEX_DIM_2D_ARRAY:
+                array = 1;
+                break;
         case V_038000_SQ_TEX_DIM_2D_MSAA:
         case V_038000_SQ_TEX_DIM_2D_ARRAY_MSAA:
         default:
                 dev_warn(p->dev, "this kernel doesn't support %d texture dim\n", G_038000_DIM(word0));
                 return -EINVAL;
         }
-        if (r600_bpe_from_format(&bpe,  G_038004_DATA_FORMAT(word1))) {
+        format = G_038004_DATA_FORMAT(word1);
+        if (!fmt_is_valid_texture(format)) {
                 dev_warn(p->dev, "%s:%d texture invalid format %d\n",
-                         __func__, __LINE__, G_038004_DATA_FORMAT(word1));
+                         __func__, __LINE__, format);
                 return -EINVAL;
         }
 
@@ -1217,7 +1336,7 @@ static inline int r600_check_texture_resource(struct radeon_cs_parser *p,  u32 i
         array_check.nbanks = track->nbanks;
         array_check.npipes = track->npipes;
         array_check.nsamples = 1;
-        array_check.bpe = bpe;
+        array_check.blocksize = fmt_get_blocksize(format);
         if (r600_get_array_mode_alignment(&array_check,
                                           &pitch_align, &height_align, &depth_align, &base_align)) {
                 dev_warn(p->dev, "%s:%d tex array mode (%d) invalid\n",
@@ -1246,22 +1365,29 @@ static inline int r600_check_texture_resource(struct radeon_cs_parser *p,  u32 i
         word0 = radeon_get_ib_value(p, idx + 4);
         word1 = radeon_get_ib_value(p, idx + 5);
         blevel = G_038010_BASE_LEVEL(word0);
-        nlevels = G_038014_LAST_LEVEL(word1);
-        r600_texture_size(nfaces, blevel, nlevels, w0, h0, d0, bpe,
-                          (pitch_align * bpe),
+        llevel = G_038014_LAST_LEVEL(word1);
+        if (array == 1) {
+                barray = G_038014_BASE_ARRAY(word1);
+                larray = G_038014_LAST_ARRAY(word1);
+
+                nfaces = larray - barray + 1;
+        }
+        r600_texture_size(nfaces, blevel, llevel, w0, h0, d0, format,
+                          pitch_align, height_align, base_align,
                           &l0_size, &mipmap_size);
         /* using get ib will give us the offset into the texture bo */
         word0 = radeon_get_ib_value(p, idx + 2) << 8;
         if ((l0_size + word0) > radeon_bo_size(texture)) {
                 dev_warn(p->dev, "texture bo too small (%d %d %d %d -> %d have %ld)\n",
-                        w0, h0, bpe, word0, l0_size, radeon_bo_size(texture));
+                        w0, h0, format, word0, l0_size, radeon_bo_size(texture));
+                dev_warn(p->dev, "alignments %d %d %d %lld\n", pitch, pitch_align, height_align, base_align);
                 return -EINVAL;
         }
         /* using get ib will give us the offset into the mipmap bo */
         word0 = radeon_get_ib_value(p, idx + 3) << 8;
         if ((mipmap_size + word0) > radeon_bo_size(mipmap)) {
                 /*dev_warn(p->dev, "mipmap bo too small (%d %d %d %d %d %d -> %d have %ld)\n",
-                  w0, h0, bpe, blevel, nlevels, word0, mipmap_size, radeon_bo_size(texture));*/
+                  w0, h0, format, blevel, nlevels, word0, mipmap_size, radeon_bo_size(texture));*/
         }
         return 0;
 }
diff --git a/drivers/gpu/drm/radeon/r600_hdmi.c b/drivers/gpu/drm/radeon/r600_hdmi.c
index e6a58ed48dcf..50db6d62eec2 100644
--- a/drivers/gpu/drm/radeon/r600_hdmi.c
+++ b/drivers/gpu/drm/radeon/r600_hdmi.c
@@ -26,6 +26,7 @@
 #include "drmP.h"
 #include "radeon_drm.h"
 #include "radeon.h"
+#include "radeon_asic.h"
 #include "atom.h"
 
 /*
diff --git a/drivers/gpu/drm/radeon/r600_reg.h b/drivers/gpu/drm/radeon/r600_reg.h
index d84612ae47e0..f869897c7456 100644
--- a/drivers/gpu/drm/radeon/r600_reg.h
+++ b/drivers/gpu/drm/radeon/r600_reg.h
@@ -81,11 +81,16 @@
 #define R600_MEDIUM_VID_LOWER_GPIO_CNTL                            0x720
 #define R600_LOW_VID_LOWER_GPIO_CNTL                               0x724
 
-
+#define R600_D1GRPH_SWAP_CONTROL                               0x610C
+#       define R600_D1GRPH_SWAP_ENDIAN_NONE                    (0 << 0)
+#       define R600_D1GRPH_SWAP_ENDIAN_16BIT                   (1 << 0)
+#       define R600_D1GRPH_SWAP_ENDIAN_32BIT                   (2 << 0)
+#       define R600_D1GRPH_SWAP_ENDIAN_64BIT                   (3 << 0)
 
 #define R600_HDP_NONSURFACE_BASE                                0x2c04
 
 #define R600_BUS_CNTL                                           0x5420
+#       define R600_BIOS_ROM_DIS                                (1 << 1)
 #define R600_CONFIG_CNTL                                        0x5424
 #define R600_CONFIG_MEMSIZE                                     0x5428
 #define R600_CONFIG_F0_BASE                                     0x542C
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index c89cfa8e0c05..d1f598663da7 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -737,6 +737,45 @@
 #       define DxGRPH_PFLIP_INT_MASK                      (1 << 0)
 #       define DxGRPH_PFLIP_INT_TYPE                      (1 << 8)
 
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#       define LC_POINT_7_PLUS_EN                         (1 << 6)
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 /*
  * PM4
  */
@@ -1264,6 +1303,11 @@
 #define     V_038004_FMT_16_16_16_FLOAT                0x0000002E
 #define     V_038004_FMT_32_32_32                      0x0000002F
 #define     V_038004_FMT_32_32_32_FLOAT                0x00000030
+#define     V_038004_FMT_BC1                           0x00000031
+#define     V_038004_FMT_BC2                           0x00000032
+#define     V_038004_FMT_BC3                           0x00000033
+#define     V_038004_FMT_BC4                           0x00000034
+#define     V_038004_FMT_BC5                           0x00000035
 #define R_038010_SQ_TEX_RESOURCE_WORD4_0             0x038010
 #define   S_038010_FORMAT_COMP_X(x)                    (((x) & 0x3) << 0)
 #define   G_038010_FORMAT_COMP_X(x)                    (((x) >> 0) & 0x3)
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 431d4186ddf0..55fefe763965 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -92,6 +92,7 @@ extern int radeon_tv;
 extern int radeon_audio;
 extern int radeon_disp_priority;
 extern int radeon_hw_i2c;
+extern int radeon_pcie_gen2;
 
 /*
  * Copy from radeon_drv.h so we don't have to include both and have conflicting
@@ -178,10 +179,10 @@ void radeon_combios_get_power_modes(struct radeon_device *rdev);
 void radeon_atombios_get_power_modes(struct radeon_device *rdev);
 void radeon_atom_set_voltage(struct radeon_device *rdev, u16 level);
 void rs690_pm_info(struct radeon_device *rdev);
-extern u32 rv6xx_get_temp(struct radeon_device *rdev);
-extern u32 rv770_get_temp(struct radeon_device *rdev);
-extern u32 evergreen_get_temp(struct radeon_device *rdev);
-extern u32 sumo_get_temp(struct radeon_device *rdev);
+extern int rv6xx_get_temp(struct radeon_device *rdev);
+extern int rv770_get_temp(struct radeon_device *rdev);
+extern int evergreen_get_temp(struct radeon_device *rdev);
+extern int sumo_get_temp(struct radeon_device *rdev);
 
 /*
  * Fences.
@@ -257,8 +258,9 @@ struct radeon_bo {
         int                             surface_reg;
         /* Constant after initialization */
         struct radeon_device            *rdev;
-        struct drm_gem_object           *gobj;
+        struct drm_gem_object           gem_base;
 };
+#define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base)
 
 struct radeon_bo_list {
         struct ttm_validate_buffer tv;
@@ -287,6 +289,15 @@ int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
                           uint64_t *gpu_addr);
 void radeon_gem_object_unpin(struct drm_gem_object *obj);
 
+int radeon_mode_dumb_create(struct drm_file *file_priv,
+                            struct drm_device *dev,
+                            struct drm_mode_create_dumb *args);
+int radeon_mode_dumb_mmap(struct drm_file *filp,
+                          struct drm_device *dev,
+                          uint32_t handle, uint64_t *offset_p);
+int radeon_mode_dumb_destroy(struct drm_file *file_priv,
+                             struct drm_device *dev,
+                             uint32_t handle);
 
 /*
  * GART structures, functions & helpers
@@ -318,6 +329,7 @@ struct radeon_gart {
         union radeon_gart_table         table;
         struct page                     **pages;
         dma_addr_t                      *pages_addr;
+        bool                            *ttm_alloced;
         bool                            ready;
 };
 
@@ -330,7 +342,8 @@ void radeon_gart_fini(struct radeon_device *rdev);
 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
                         int pages);
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
-                     int pages, struct page **pagelist);
+                     int pages, struct page **pagelist,
+                     dma_addr_t *dma_addr);
 
 
 /*
@@ -739,6 +752,7 @@ enum radeon_int_thermal_type {
         THERMAL_TYPE_RV770,
         THERMAL_TYPE_EVERGREEN,
         THERMAL_TYPE_SUMO,
+        THERMAL_TYPE_NI,
 };
 
 struct radeon_voltage {
@@ -810,8 +824,7 @@ struct radeon_pm {
         fixed20_12              sclk;
         fixed20_12              mclk;
         fixed20_12              needed_bandwidth;
-        /* XXX: use a define for num power modes */
-        struct radeon_power_state power_state[8];
+        struct radeon_power_state *power_state;
         /* number of valid power states */
         int                     num_power_states;
         int                     current_power_state_index;
@@ -822,6 +835,9 @@ struct radeon_pm {
         u32                     current_sclk;
         u32                     current_mclk;
         u32                     current_vddc;
+        u32                     default_sclk;
+        u32                     default_mclk;
+        u32                     default_vddc;
         struct radeon_i2c_chan *i2c_bus;
         /* selected pm method */
         enum radeon_pm_method     pm_method;
@@ -1031,6 +1047,7 @@ struct evergreen_asic {
         unsigned tiling_npipes;
         unsigned tiling_group_size;
         unsigned tile_config;
+        struct r100_gpu_lockup  lockup;
 };
 
 union radeon_asic_config {
@@ -1147,11 +1164,11 @@ struct radeon_device {
         const struct firmware *me_fw;   /* all family ME firmware */
         const struct firmware *pfp_fw;  /* r6/700 PFP firmware */
         const struct firmware *rlc_fw;  /* r6/700 RLC firmware */
+        const struct firmware *mc_fw;   /* NI MC firmware */
         struct r600_blit r600_blit;
         struct r700_vram_scratch vram_scratch;
         int msi_enabled; /* msi enabled */
         struct r600_ih ih; /* r6/700 interrupt ring */
-        struct workqueue_struct *wq;
         struct work_struct hotplug_work;
         int num_crtc; /* number of crtcs */
         struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
@@ -1166,10 +1183,10 @@ struct radeon_device {
         uint8_t                 audio_status_bits;
         uint8_t                 audio_category_code;
 
-        bool powered_down;
         struct notifier_block acpi_nb;
-        /* only one userspace can use Hyperz features at a time */
+        /* only one userspace can use Hyperz features or CMASK at a time */
         struct drm_file *hyperz_filp;
+        struct drm_file *cmask_filp;
         /* i2c buses */
         struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
 };
@@ -1181,19 +1198,6 @@ int radeon_device_init(struct radeon_device *rdev,
 void radeon_device_fini(struct radeon_device *rdev);
 int radeon_gpu_wait_for_idle(struct radeon_device *rdev);
 
-/* r600 blit */
-int r600_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
-void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
-void r600_kms_blit_copy(struct radeon_device *rdev,
-                        u64 src_gpu_addr, u64 dst_gpu_addr,
-                        int size_bytes);
-/* evergreen blit */
-int evergreen_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
-void evergreen_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
-void evergreen_kms_blit_copy(struct radeon_device *rdev,
-                             u64 src_gpu_addr, u64 dst_gpu_addr,
-                             int size_bytes);
-
 static inline uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg)
 {
         if (reg < rdev->rmmio_size)
@@ -1244,6 +1248,8 @@ static inline void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  */
 #define RREG8(reg) readb(((void __iomem *)rdev->rmmio) + (reg))
 #define WREG8(reg, v) writeb(v, ((void __iomem *)rdev->rmmio) + (reg))
+#define RREG16(reg) readw(((void __iomem *)rdev->rmmio) + (reg))
+#define WREG16(reg, v) writew(v, ((void __iomem *)rdev->rmmio) + (reg))
 #define RREG32(reg) r100_mm_rreg(rdev, (reg))
 #define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg)))
 #define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v))
@@ -1317,6 +1323,14 @@ void r100_pll_errata_after_index(struct radeon_device *rdev);
                 (rdev->family == CHIP_RV410) ||                 \
                 (rdev->family == CHIP_RS400) ||                 \
                 (rdev->family == CHIP_RS480))
+#define ASIC_IS_X2(rdev) ((rdev->ddev->pdev->device == 0x9441) || \
+                (rdev->ddev->pdev->device == 0x9443) || \
+                (rdev->ddev->pdev->device == 0x944B) || \
+                (rdev->ddev->pdev->device == 0x9506) || \
+                (rdev->ddev->pdev->device == 0x9509) || \
+                (rdev->ddev->pdev->device == 0x950F) || \
+                (rdev->ddev->pdev->device == 0x689C) || \
+                (rdev->ddev->pdev->device == 0x689D))
 #define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600))
 #define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600)  ||    \
                             (rdev->family == CHIP_RS690)  ||    \
@@ -1325,7 +1339,9 @@ void r100_pll_errata_after_index(struct radeon_device *rdev);
 #define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620))
 #define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730))
 #define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR))
-#define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM))
+#define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM) && \
+                             (rdev->flags & RADEON_IS_IGP))
+#define ASIC_IS_DCE5(rdev) ((rdev->family >= CHIP_BARTS))
 
 /*
  * BIOS helpers.
@@ -1432,118 +1448,15 @@ extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc
 extern int radeon_resume_kms(struct drm_device *dev);
 extern int radeon_suspend_kms(struct drm_device *dev, pm_message_t state);
 
-/* r100,rv100,rs100,rv200,rs200,r200,rv250,rs300,rv280 */
-extern void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, struct radeon_cp *cp);
-extern bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, struct r100_gpu_lockup *lockup, struct radeon_cp *cp);
-
-/* rv200,rv250,rv280 */
-extern void r200_set_safe_registers(struct radeon_device *rdev);
-
-/* r300,r350,rv350,rv370,rv380 */
-extern void r300_set_reg_safe(struct radeon_device *rdev);
-extern void r300_mc_program(struct radeon_device *rdev);
-extern void r300_mc_init(struct radeon_device *rdev);
-extern void r300_clock_startup(struct radeon_device *rdev);
-extern int r300_mc_wait_for_idle(struct radeon_device *rdev);
-extern int rv370_pcie_gart_init(struct radeon_device *rdev);
-extern void rv370_pcie_gart_fini(struct radeon_device *rdev);
-extern int rv370_pcie_gart_enable(struct radeon_device *rdev);
-extern void rv370_pcie_gart_disable(struct radeon_device *rdev);
-
-/* r420,r423,rv410 */
-extern u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg);
-extern void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
-extern int r420_debugfs_pipes_info_init(struct radeon_device *rdev);
-extern void r420_pipes_init(struct radeon_device *rdev);
-
-/* rv515 */
-struct rv515_mc_save {
-        u32 d1vga_control;
-        u32 d2vga_control;
-        u32 vga_render_control;
-        u32 vga_hdp_control;
-        u32 d1crtc_control;
-        u32 d2crtc_control;
-};
-extern void rv515_bandwidth_avivo_update(struct radeon_device *rdev);
-extern void rv515_vga_render_disable(struct radeon_device *rdev);
-extern void rv515_set_safe_registers(struct radeon_device *rdev);
-extern void rv515_mc_stop(struct radeon_device *rdev, struct rv515_mc_save *save);
-extern void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save);
-extern void rv515_clock_startup(struct radeon_device *rdev);
-extern void rv515_debugfs(struct radeon_device *rdev);
-extern int rv515_suspend(struct radeon_device *rdev);
-
-/* rs400 */
-extern int rs400_gart_init(struct radeon_device *rdev);
-extern int rs400_gart_enable(struct radeon_device *rdev);
-extern void rs400_gart_adjust_size(struct radeon_device *rdev);
-extern void rs400_gart_disable(struct radeon_device *rdev);
-extern void rs400_gart_fini(struct radeon_device *rdev);
-
-/* rs600 */
-extern void rs600_set_safe_registers(struct radeon_device *rdev);
-extern int rs600_irq_set(struct radeon_device *rdev);
-extern void rs600_irq_disable(struct radeon_device *rdev);
-
-/* rs690, rs740 */
-extern void rs690_line_buffer_adjust(struct radeon_device *rdev,
-                                        struct drm_display_mode *mode1,
-                                        struct drm_display_mode *mode2);
-
-/* r600, rv610, rv630, rv620, rv635, rv670, rs780, rs880 */
-extern bool r600_card_posted(struct radeon_device *rdev);
-extern void r600_cp_stop(struct radeon_device *rdev);
-extern int r600_cp_start(struct radeon_device *rdev);
-extern void r600_ring_init(struct radeon_device *rdev, unsigned ring_size);
-extern int r600_cp_resume(struct radeon_device *rdev);
-extern void r600_cp_fini(struct radeon_device *rdev);
-extern int r600_count_pipe_bits(uint32_t val);
-extern int r600_mc_wait_for_idle(struct radeon_device *rdev);
-extern int r600_pcie_gart_init(struct radeon_device *rdev);
-extern void r600_pcie_gart_tlb_flush(struct radeon_device *rdev);
-extern int r600_ib_test(struct radeon_device *rdev);
-extern int r600_ring_test(struct radeon_device *rdev);
-extern void r600_scratch_init(struct radeon_device *rdev);
-extern int r600_blit_init(struct radeon_device *rdev);
-extern void r600_blit_fini(struct radeon_device *rdev);
-extern int r600_init_microcode(struct radeon_device *rdev);
-extern int r600_asic_reset(struct radeon_device *rdev);
-/* r600 irq */
-extern int r600_irq_init(struct radeon_device *rdev);
-extern void r600_irq_fini(struct radeon_device *rdev);
-extern void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size);
-extern int r600_irq_set(struct radeon_device *rdev);
-extern void r600_irq_suspend(struct radeon_device *rdev);
-extern void r600_disable_interrupts(struct radeon_device *rdev);
-extern void r600_rlc_stop(struct radeon_device *rdev);
-/* r600 audio */
-extern int r600_audio_init(struct radeon_device *rdev);
-extern int r600_audio_tmds_index(struct drm_encoder *encoder);
-extern void r600_audio_set_clock(struct drm_encoder *encoder, int clock);
-extern int r600_audio_channels(struct radeon_device *rdev);
-extern int r600_audio_bits_per_sample(struct radeon_device *rdev);
-extern int r600_audio_rate(struct radeon_device *rdev);
-extern uint8_t r600_audio_status_bits(struct radeon_device *rdev);
-extern uint8_t r600_audio_category_code(struct radeon_device *rdev);
-extern void r600_audio_schedule_polling(struct radeon_device *rdev);
-extern void r600_audio_enable_polling(struct drm_encoder *encoder);
-extern void r600_audio_disable_polling(struct drm_encoder *encoder);
-extern void r600_audio_fini(struct radeon_device *rdev);
-extern void r600_hdmi_init(struct drm_encoder *encoder);
+/*
+ * r600 functions used by radeon_encoder.c
+ */
 extern void r600_hdmi_enable(struct drm_encoder *encoder);
 extern void r600_hdmi_disable(struct drm_encoder *encoder);
 extern void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode);
-extern int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder);
-extern void r600_hdmi_update_audio_settings(struct drm_encoder *encoder);
 
-extern void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
-extern void r700_cp_stop(struct radeon_device *rdev);
-extern void r700_cp_fini(struct radeon_device *rdev);
-extern void evergreen_disable_interrupt_state(struct radeon_device *rdev);
-extern int evergreen_irq_set(struct radeon_device *rdev);
-extern int evergreen_blit_init(struct radeon_device *rdev);
-extern void evergreen_blit_fini(struct radeon_device *rdev);
+extern int ni_init_microcode(struct radeon_device *rdev);
+extern int btc_mc_load_microcode(struct radeon_device *rdev);
 
 /* radeon_acpi.c */ 
 #if defined(CONFIG_ACPI) 
@@ -1552,14 +1465,6 @@ extern int radeon_acpi_init(struct radeon_device *rdev);
 static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; } 
 #endif 
 
-/* evergreen */
-struct evergreen_mc_save {
-        u32 vga_control[6];
-        u32 vga_render_control;
-        u32 vga_hdp_control;
-        u32 crtc_control[6];
-};
-
 #include "radeon_object.h"
 
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 3d73fe484f42..e75d63b8e21d 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -94,7 +94,7 @@ static void radeon_register_accessor_init(struct radeon_device *rdev)
                 rdev->mc_rreg = &rs600_mc_rreg;
                 rdev->mc_wreg = &rs600_mc_wreg;
         }
-        if ((rdev->family >= CHIP_R600) && (rdev->family <= CHIP_RV740)) {
+        if ((rdev->family >= CHIP_R600) && (rdev->family <= CHIP_HEMLOCK)) {
                 rdev->pciep_rreg = &r600_pciep_rreg;
                 rdev->pciep_wreg = &r600_pciep_wreg;
         }
@@ -631,8 +631,8 @@ static struct radeon_asic r600_asic = {
         .set_engine_clock = &radeon_atom_set_engine_clock,
         .get_memory_clock = &radeon_atom_get_memory_clock,
         .set_memory_clock = &radeon_atom_set_memory_clock,
-        .get_pcie_lanes = &rv370_get_pcie_lanes,
-        .set_pcie_lanes = NULL,
+        .get_pcie_lanes = &r600_get_pcie_lanes,
+        .set_pcie_lanes = &r600_set_pcie_lanes,
         .set_clock_gating = NULL,
         .set_surface_reg = r600_set_surface_reg,
         .clear_surface_reg = r600_clear_surface_reg,
@@ -725,8 +725,8 @@ static struct radeon_asic rv770_asic = {
         .set_engine_clock = &radeon_atom_set_engine_clock,
         .get_memory_clock = &radeon_atom_get_memory_clock,
         .set_memory_clock = &radeon_atom_set_memory_clock,
-        .get_pcie_lanes = &rv370_get_pcie_lanes,
-        .set_pcie_lanes = NULL,
+        .get_pcie_lanes = &r600_get_pcie_lanes,
+        .set_pcie_lanes = &r600_set_pcie_lanes,
         .set_clock_gating = &radeon_atom_set_clock_gating,
         .set_surface_reg = r600_set_surface_reg,
         .clear_surface_reg = r600_clear_surface_reg,
@@ -759,7 +759,7 @@ static struct radeon_asic evergreen_asic = {
         .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
         .gart_set_page = &rs600_gart_set_page,
         .ring_test = &r600_ring_test,
-        .ring_ib_execute = &r600_ring_ib_execute,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
         .irq_set = &evergreen_irq_set,
         .irq_process = &evergreen_irq_process,
         .get_vblank_counter = &evergreen_get_vblank_counter,
@@ -772,8 +772,8 @@ static struct radeon_asic evergreen_asic = {
         .set_engine_clock = &radeon_atom_set_engine_clock,
         .get_memory_clock = &radeon_atom_get_memory_clock,
         .set_memory_clock = &radeon_atom_set_memory_clock,
-        .get_pcie_lanes = NULL,
-        .set_pcie_lanes = NULL,
+        .get_pcie_lanes = &r600_get_pcie_lanes,
+        .set_pcie_lanes = &r600_set_pcie_lanes,
         .set_clock_gating = NULL,
         .set_surface_reg = r600_set_surface_reg,
         .clear_surface_reg = r600_clear_surface_reg,
@@ -805,7 +805,7 @@ static struct radeon_asic sumo_asic = {
         .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
         .gart_set_page = &rs600_gart_set_page,
         .ring_test = &r600_ring_test,
-        .ring_ib_execute = &r600_ring_ib_execute,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
         .irq_set = &evergreen_irq_set,
         .irq_process = &evergreen_irq_process,
         .get_vblank_counter = &evergreen_get_vblank_counter,
@@ -836,6 +836,52 @@ static struct radeon_asic sumo_asic = {
         .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
 };
 
+static struct radeon_asic btc_asic = {
+        .init = &evergreen_init,
+        .fini = &evergreen_fini,
+        .suspend = &evergreen_suspend,
+        .resume = &evergreen_resume,
+        .cp_commit = &r600_cp_commit,
+        .gpu_is_lockup = &evergreen_gpu_is_lockup,
+        .asic_reset = &evergreen_asic_reset,
+        .vga_set_state = &r600_vga_set_state,
+        .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
+        .gart_set_page = &rs600_gart_set_page,
+        .ring_test = &r600_ring_test,
+        .ring_ib_execute = &evergreen_ring_ib_execute,
+        .irq_set = &evergreen_irq_set,
+        .irq_process = &evergreen_irq_process,
+        .get_vblank_counter = &evergreen_get_vblank_counter,
+        .fence_ring_emit = &r600_fence_ring_emit,
+        .cs_parse = &evergreen_cs_parse,
+        .copy_blit = &evergreen_copy_blit,
+        .copy_dma = &evergreen_copy_blit,
+        .copy = &evergreen_copy_blit,
+        .get_engine_clock = &radeon_atom_get_engine_clock,
+        .set_engine_clock = &radeon_atom_set_engine_clock,
+        .get_memory_clock = &radeon_atom_get_memory_clock,
+        .set_memory_clock = &radeon_atom_set_memory_clock,
+        .get_pcie_lanes = NULL,
+        .set_pcie_lanes = NULL,
+        .set_clock_gating = NULL,
+        .set_surface_reg = r600_set_surface_reg,
+        .clear_surface_reg = r600_clear_surface_reg,
+        .bandwidth_update = &evergreen_bandwidth_update,
+        .hpd_init = &evergreen_hpd_init,
+        .hpd_fini = &evergreen_hpd_fini,
+        .hpd_sense = &evergreen_hpd_sense,
+        .hpd_set_polarity = &evergreen_hpd_set_polarity,
+        .gui_idle = &r600_gui_idle,
+        .pm_misc = &evergreen_pm_misc,
+        .pm_prepare = &evergreen_pm_prepare,
+        .pm_finish = &evergreen_pm_finish,
+        .pm_init_profile = &r600_pm_init_profile,
+        .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
+        .pre_page_flip = &evergreen_pre_page_flip,
+        .page_flip = &evergreen_page_flip,
+        .post_page_flip = &evergreen_post_page_flip,
+};
+
 int radeon_asic_init(struct radeon_device *rdev)
 {
         radeon_register_accessor_init(rdev);
@@ -923,6 +969,11 @@ int radeon_asic_init(struct radeon_device *rdev)
         case CHIP_PALM:
                 rdev->asic = &sumo_asic;
                 break;
+        case CHIP_BARTS:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
+                rdev->asic = &btc_asic;
+                break;
         default:
                 /* FIXME: not supported yet */
                 return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index 4970eda1bd41..1c7317e3aa8c 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -57,8 +57,6 @@ int r100_init(struct radeon_device *rdev);
 void r100_fini(struct radeon_device *rdev);
 int r100_suspend(struct radeon_device *rdev);
 int r100_resume(struct radeon_device *rdev);
-uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg);
-void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void r100_vga_set_state(struct radeon_device *rdev, bool state);
 bool r100_gpu_is_lockup(struct radeon_device *rdev);
 int r100_asic_reset(struct radeon_device *rdev);
@@ -102,6 +100,11 @@ int r100_pci_gart_enable(struct radeon_device *rdev);
 void r100_pci_gart_disable(struct radeon_device *rdev);
 int r100_debugfs_mc_info_init(struct radeon_device *rdev);
 int r100_gui_wait_for_idle(struct radeon_device *rdev);
+void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup,
+                            struct radeon_cp *cp);
+bool r100_gpu_cp_is_lockup(struct radeon_device *rdev,
+                           struct r100_gpu_lockup *lockup,
+                           struct radeon_cp *cp);
 void r100_ib_fini(struct radeon_device *rdev);
 int r100_ib_init(struct radeon_device *rdev);
 void r100_irq_disable(struct radeon_device *rdev);
@@ -138,10 +141,11 @@ extern void r100_post_page_flip(struct radeon_device *rdev, int crtc);
  * r200,rv250,rs300,rv280
  */
 extern int r200_copy_dma(struct radeon_device *rdev,
-                        uint64_t src_offset,
-                        uint64_t dst_offset,
-                        unsigned num_pages,
+                         uint64_t src_offset,
+                         uint64_t dst_offset,
+                         unsigned num_pages,
                          struct radeon_fence *fence);
+void r200_set_safe_registers(struct radeon_device *rdev);
 
 /*
  * r300,r350,rv350,rv380
@@ -158,10 +162,17 @@ extern void r300_fence_ring_emit(struct radeon_device *rdev,
 extern int r300_cs_parse(struct radeon_cs_parser *p);
 extern void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev);
 extern int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
-extern uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg);
-extern void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 extern void rv370_set_pcie_lanes(struct radeon_device *rdev, int lanes);
 extern int rv370_get_pcie_lanes(struct radeon_device *rdev);
+extern void r300_set_reg_safe(struct radeon_device *rdev);
+extern void r300_mc_program(struct radeon_device *rdev);
+extern void r300_mc_init(struct radeon_device *rdev);
+extern void r300_clock_startup(struct radeon_device *rdev);
+extern int r300_mc_wait_for_idle(struct radeon_device *rdev);
+extern int rv370_pcie_gart_init(struct radeon_device *rdev);
+extern void rv370_pcie_gart_fini(struct radeon_device *rdev);
+extern int rv370_pcie_gart_enable(struct radeon_device *rdev);
+extern void rv370_pcie_gart_disable(struct radeon_device *rdev);
 
 /*
  * r420,r423,rv410
@@ -171,6 +182,10 @@ extern void r420_fini(struct radeon_device *rdev);
 extern int r420_suspend(struct radeon_device *rdev);
 extern int r420_resume(struct radeon_device *rdev);
 extern void r420_pm_init_profile(struct radeon_device *rdev);
+extern u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg);
+extern void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
+extern int r420_debugfs_pipes_info_init(struct radeon_device *rdev);
+extern void r420_pipes_init(struct radeon_device *rdev);
 
 /*
  * rs400,rs480
@@ -183,6 +198,11 @@ void rs400_gart_tlb_flush(struct radeon_device *rdev);
 int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
 uint32_t rs400_mc_rreg(struct radeon_device *rdev, uint32_t reg);
 void rs400_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
+int rs400_gart_init(struct radeon_device *rdev);
+int rs400_gart_enable(struct radeon_device *rdev);
+void rs400_gart_adjust_size(struct radeon_device *rdev);
+void rs400_gart_disable(struct radeon_device *rdev);
+void rs400_gart_fini(struct radeon_device *rdev);
 
 /*
  * rs600.
@@ -194,6 +214,7 @@ extern int rs600_suspend(struct radeon_device *rdev);
 extern int rs600_resume(struct radeon_device *rdev);
 int rs600_irq_set(struct radeon_device *rdev);
 int rs600_irq_process(struct radeon_device *rdev);
+void rs600_irq_disable(struct radeon_device *rdev);
 u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc);
 void rs600_gart_tlb_flush(struct radeon_device *rdev);
 int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
@@ -211,6 +232,8 @@ extern void rs600_pm_finish(struct radeon_device *rdev);
 extern void rs600_pre_page_flip(struct radeon_device *rdev, int crtc);
 extern u32 rs600_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
 extern void rs600_post_page_flip(struct radeon_device *rdev, int crtc);
+void rs600_set_safe_registers(struct radeon_device *rdev);
+
 
 /*
  * rs690,rs740
@@ -222,20 +245,37 @@ int rs690_suspend(struct radeon_device *rdev);
 uint32_t rs690_mc_rreg(struct radeon_device *rdev, uint32_t reg);
 void rs690_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void rs690_bandwidth_update(struct radeon_device *rdev);
+void rs690_line_buffer_adjust(struct radeon_device *rdev,
+                                        struct drm_display_mode *mode1,
+                                        struct drm_display_mode *mode2);
 
 /*
  * rv515
  */
+struct rv515_mc_save {
+        u32 d1vga_control;
+        u32 d2vga_control;
+        u32 vga_render_control;
+        u32 vga_hdp_control;
+        u32 d1crtc_control;
+        u32 d2crtc_control;
+};
 int rv515_init(struct radeon_device *rdev);
 void rv515_fini(struct radeon_device *rdev);
 uint32_t rv515_mc_rreg(struct radeon_device *rdev, uint32_t reg);
 void rv515_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void rv515_ring_start(struct radeon_device *rdev);
-uint32_t rv515_pcie_rreg(struct radeon_device *rdev, uint32_t reg);
-void rv515_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 void rv515_bandwidth_update(struct radeon_device *rdev);
 int rv515_resume(struct radeon_device *rdev);
 int rv515_suspend(struct radeon_device *rdev);
+void rv515_bandwidth_avivo_update(struct radeon_device *rdev);
+void rv515_vga_render_disable(struct radeon_device *rdev);
+void rv515_set_safe_registers(struct radeon_device *rdev);
+void rv515_mc_stop(struct radeon_device *rdev, struct rv515_mc_save *save);
+void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save);
+void rv515_clock_startup(struct radeon_device *rdev);
+void rv515_debugfs(struct radeon_device *rdev);
+
 
 /*
  * r520,rv530,rv560,rv570,r580
@@ -260,14 +300,13 @@ void r600_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 int r600_cs_parse(struct radeon_cs_parser *p);
 void r600_fence_ring_emit(struct radeon_device *rdev,
                           struct radeon_fence *fence);
-int r600_irq_process(struct radeon_device *rdev);
-int r600_irq_set(struct radeon_device *rdev);
 bool r600_gpu_is_lockup(struct radeon_device *rdev);
 int r600_asic_reset(struct radeon_device *rdev);
 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
                          uint32_t tiling_flags, uint32_t pitch,
                          uint32_t offset, uint32_t obj_size);
 void r600_clear_surface_reg(struct radeon_device *rdev, int reg);
+int r600_ib_test(struct radeon_device *rdev);
 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
 int r600_ring_test(struct radeon_device *rdev);
 int r600_copy_blit(struct radeon_device *rdev,
@@ -284,6 +323,52 @@ extern void r600_pm_misc(struct radeon_device *rdev);
 extern void r600_pm_init_profile(struct radeon_device *rdev);
 extern void rs780_pm_init_profile(struct radeon_device *rdev);
 extern void r600_pm_get_dynpm_state(struct radeon_device *rdev);
+extern void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes);
+extern int r600_get_pcie_lanes(struct radeon_device *rdev);
+bool r600_card_posted(struct radeon_device *rdev);
+void r600_cp_stop(struct radeon_device *rdev);
+int r600_cp_start(struct radeon_device *rdev);
+void r600_ring_init(struct radeon_device *rdev, unsigned ring_size);
+int r600_cp_resume(struct radeon_device *rdev);
+void r600_cp_fini(struct radeon_device *rdev);
+int r600_count_pipe_bits(uint32_t val);
+int r600_mc_wait_for_idle(struct radeon_device *rdev);
+int r600_pcie_gart_init(struct radeon_device *rdev);
+void r600_scratch_init(struct radeon_device *rdev);
+int r600_blit_init(struct radeon_device *rdev);
+void r600_blit_fini(struct radeon_device *rdev);
+int r600_init_microcode(struct radeon_device *rdev);
+/* r600 irq */
+int r600_irq_process(struct radeon_device *rdev);
+int r600_irq_init(struct radeon_device *rdev);
+void r600_irq_fini(struct radeon_device *rdev);
+void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size);
+int r600_irq_set(struct radeon_device *rdev);
+void r600_irq_suspend(struct radeon_device *rdev);
+void r600_disable_interrupts(struct radeon_device *rdev);
+void r600_rlc_stop(struct radeon_device *rdev);
+/* r600 audio */
+int r600_audio_init(struct radeon_device *rdev);
+int r600_audio_tmds_index(struct drm_encoder *encoder);
+void r600_audio_set_clock(struct drm_encoder *encoder, int clock);
+int r600_audio_channels(struct radeon_device *rdev);
+int r600_audio_bits_per_sample(struct radeon_device *rdev);
+int r600_audio_rate(struct radeon_device *rdev);
+uint8_t r600_audio_status_bits(struct radeon_device *rdev);
+uint8_t r600_audio_category_code(struct radeon_device *rdev);
+void r600_audio_schedule_polling(struct radeon_device *rdev);
+void r600_audio_enable_polling(struct drm_encoder *encoder);
+void r600_audio_disable_polling(struct drm_encoder *encoder);
+void r600_audio_fini(struct radeon_device *rdev);
+void r600_hdmi_init(struct drm_encoder *encoder);
+int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder);
+void r600_hdmi_update_audio_settings(struct drm_encoder *encoder);
+/* r600 blit */
+int r600_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
+void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
+void r600_kms_blit_copy(struct radeon_device *rdev,
+                        u64 src_gpu_addr, u64 dst_gpu_addr,
+                        int size_bytes);
 
 /*
  * rv770,rv730,rv710,rv740
@@ -292,12 +377,21 @@ int rv770_init(struct radeon_device *rdev);
 void rv770_fini(struct radeon_device *rdev);
 int rv770_suspend(struct radeon_device *rdev);
 int rv770_resume(struct radeon_device *rdev);
-extern void rv770_pm_misc(struct radeon_device *rdev);
-extern u32 rv770_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
+void rv770_pm_misc(struct radeon_device *rdev);
+u32 rv770_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
+void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
+void r700_cp_stop(struct radeon_device *rdev);
+void r700_cp_fini(struct radeon_device *rdev);
 
 /*
  * evergreen
  */
+struct evergreen_mc_save {
+        u32 vga_control[6];
+        u32 vga_render_control;
+        u32 vga_hdp_control;
+        u32 crtc_control[6];
+};
 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev);
 int evergreen_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
@@ -306,6 +400,7 @@ int evergreen_resume(struct radeon_device *rdev);
 bool evergreen_gpu_is_lockup(struct radeon_device *rdev);
 int evergreen_asic_reset(struct radeon_device *rdev);
 void evergreen_bandwidth_update(struct radeon_device *rdev);
+void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
 int evergreen_copy_blit(struct radeon_device *rdev,
                         uint64_t src_offset, uint64_t dst_offset,
                         unsigned num_pages, struct radeon_fence *fence);
@@ -324,5 +419,15 @@ extern void evergreen_pm_finish(struct radeon_device *rdev);
 extern void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc);
 extern u32 evergreen_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
 extern void evergreen_post_page_flip(struct radeon_device *rdev, int crtc);
+void evergreen_disable_interrupt_state(struct radeon_device *rdev);
+int evergreen_blit_init(struct radeon_device *rdev);
+void evergreen_blit_fini(struct radeon_device *rdev);
+/* evergreen blit */
+int evergreen_blit_prepare_copy(struct radeon_device *rdev, int size_bytes);
+void evergreen_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence);
+void evergreen_kms_blit_copy(struct radeon_device *rdev,
+                             u64 src_gpu_addr, u64 dst_gpu_addr,
+                             int size_bytes);
+
 
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index ac882639b3ed..5c1cc7ad9a15 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -37,7 +37,7 @@ radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device,
 extern void radeon_link_encoder_connector(struct drm_device *dev);
 extern void
 radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum,
-                        uint32_t supported_device);
+                        uint32_t supported_device, u16 caps);
 
 /* from radeon_connector.c */
 extern void
@@ -98,6 +98,14 @@ static inline struct radeon_i2c_bus_rec radeon_lookup_i2c_gpio(struct radeon_dev
                                 }
                         }
 
+                        /* some DCE3 boards have bad data for this entry */
+                        if (ASIC_IS_DCE3(rdev)) {
+                                if ((i == 4) &&
+                                    (gpio->usClkMaskRegisterIndex == 0x1fda) &&
+                                    (gpio->sucI2cId.ucAccess == 0x94))
+                                        gpio->sucI2cId.ucAccess = 0x14;
+                        }
+
                         if (gpio->sucI2cId.ucAccess == id) {
                                 i2c.mask_clk_reg = le16_to_cpu(gpio->usClkMaskRegisterIndex) * 4;
                                 i2c.mask_data_reg = le16_to_cpu(gpio->usDataMaskRegisterIndex) * 4;
@@ -174,6 +182,14 @@ void radeon_atombios_i2c_init(struct radeon_device *rdev)
                                 }
                         }
 
+                        /* some DCE3 boards have bad data for this entry */
+                        if (ASIC_IS_DCE3(rdev)) {
+                                if ((i == 4) &&
+                                    (gpio->usClkMaskRegisterIndex == 0x1fda) &&
+                                    (gpio->sucI2cId.ucAccess == 0x94))
+                                        gpio->sucI2cId.ucAccess = 0x14;
+                        }
+
                         i2c.mask_clk_reg = le16_to_cpu(gpio->usClkMaskRegisterIndex) * 4;
                         i2c.mask_data_reg = le16_to_cpu(gpio->usDataMaskRegisterIndex) * 4;
                         i2c.en_clk_reg = le16_to_cpu(gpio->usClkEnRegisterIndex) * 4;
@@ -297,7 +313,6 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
                                      uint16_t *line_mux,
                                      struct radeon_hpd *hpd)
 {
-        struct radeon_device *rdev = dev->dev_private;
 
         /* Asus M2A-VM HDMI board lists the DVI port as HDMI */
         if ((dev->pdev->device == 0x791e) &&
@@ -372,6 +387,13 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
                         *line_mux = 0x90;
         }
 
+        /* mac rv630, rv730, others */
+        if ((supported_device == ATOM_DEVICE_TV1_SUPPORT) &&
+            (*connector_type == DRM_MODE_CONNECTOR_DVII)) {
+                *connector_type = DRM_MODE_CONNECTOR_9PinDIN;
+                *line_mux = CONNECTOR_7PIN_DIN_ENUM_ID1;
+        }
+
         /* ASUS HD 3600 XT board lists the DVI port as HDMI */
         if ((dev->pdev->device == 0x9598) &&
             (dev->pdev->subsystem_vendor == 0x1043) &&
@@ -409,21 +431,23 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
                 }
         }
 
-        /* Acer laptop reports DVI-D as DVI-I and hpd pins reversed */
+        /* Acer laptop (Acer TravelMate 5730G) has an HDMI port
+         * on the laptop and a DVI port on the docking station and
+         * both share the same encoder, hpd pin, and ddc line.
+         * So while the bios table is technically correct,
+         * we drop the DVI port here since xrandr has no concept of
+         * encoders and will try and drive both connectors
+         * with different crtcs which isn't possible on the hardware
+         * side and leaves no crtcs for LVDS or VGA.
+         */
         if ((dev->pdev->device == 0x95c4) &&
             (dev->pdev->subsystem_vendor == 0x1025) &&
             (dev->pdev->subsystem_device == 0x013c)) {
-                struct radeon_gpio_rec gpio;
-
                 if ((*connector_type == DRM_MODE_CONNECTOR_DVII) &&
                     (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) {
-                        gpio = radeon_lookup_gpio(rdev, 6);
-                        *hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio);
+                        /* actually it's a DVI-D port not DVI-I */
                         *connector_type = DRM_MODE_CONNECTOR_DVID;
-                } else if ((*connector_type == DRM_MODE_CONNECTOR_HDMIA) &&
-                           (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) {
-                        gpio = radeon_lookup_gpio(rdev, 7);
-                        *hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio);
+                        return false;
                 }
         }
 
@@ -509,6 +533,7 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
         u16 size, data_offset;
         u8 frev, crev;
         ATOM_CONNECTOR_OBJECT_TABLE *con_obj;
+        ATOM_ENCODER_OBJECT_TABLE *enc_obj;
         ATOM_OBJECT_TABLE *router_obj;
         ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj;
         ATOM_OBJECT_HEADER *obj_header;
@@ -533,6 +558,9 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
         con_obj = (ATOM_CONNECTOR_OBJECT_TABLE *)
             (ctx->bios + data_offset +
              le16_to_cpu(obj_header->usConnectorObjectTableOffset));
+        enc_obj = (ATOM_ENCODER_OBJECT_TABLE *)
+            (ctx->bios + data_offset +
+             le16_to_cpu(obj_header->usEncoderObjectTableOffset));
         router_obj = (ATOM_OBJECT_TABLE *)
                 (ctx->bios + data_offset +
                  le16_to_cpu(obj_header->usRouterObjectTableOffset));
@@ -638,14 +666,35 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
                                      OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT;
 
                                 if (grph_obj_type == GRAPH_OBJECT_TYPE_ENCODER) {
-                                        u16 encoder_obj = le16_to_cpu(path->usGraphicObjIds[j]);
-
-                                        radeon_add_atom_encoder(dev,
-                                                                encoder_obj,
-                                                                le16_to_cpu
-                                                                (path->
-                                                                 usDeviceTag));
+                                        for (k = 0; k < enc_obj->ucNumberOfObjects; k++) {
+                                                u16 encoder_obj = le16_to_cpu(enc_obj->asObjects[k].usObjectID);
+                                                if (le16_to_cpu(path->usGraphicObjIds[j]) == encoder_obj) {
+                                                        ATOM_COMMON_RECORD_HEADER *record = (ATOM_COMMON_RECORD_HEADER *)
+                                                                (ctx->bios + data_offset +
+                                                                 le16_to_cpu(enc_obj->asObjects[k].usRecordOffset));
+                                                        ATOM_ENCODER_CAP_RECORD *cap_record;
+                                                        u16 caps = 0;
 
+                                                        while (record->ucRecordType > 0 &&
+                                                               record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) {
+                                                                switch (record->ucRecordType) {
+                                                                case ATOM_ENCODER_CAP_RECORD_TYPE:
+                                                                        cap_record =(ATOM_ENCODER_CAP_RECORD *)
+                                                                                record;
+                                                                        caps = le16_to_cpu(cap_record->usEncoderCap);
+                                                                        break;
+                                                                }
+                                                                record = (ATOM_COMMON_RECORD_HEADER *)
+                                                                        ((char *)record + record->ucRecordSize);
+                                                        }
+                                                        radeon_add_atom_encoder(dev,
+                                                                                encoder_obj,
+                                                                                le16_to_cpu
+                                                                                (path->
+                                                                                 usDeviceTag),
+                                                                                caps);
+                                                }
+                                        }
                                 } else if (grph_obj_type == GRAPH_OBJECT_TYPE_ROUTER) {
                                         for (k = 0; k < router_obj->ucNumberOfObjects; k++) {
                                                 u16 router_obj_id = le16_to_cpu(router_obj->asObjects[k].usObjectID);
@@ -979,7 +1028,8 @@ bool radeon_get_atom_connector_info_from_supported_devices_table(struct
                                                 radeon_get_encoder_enum(dev,
                                                                       (1 << i),
                                                                       dac),
-                                                (1 << i));
+                                                (1 << i),
+                                                0);
                 else
                         radeon_add_legacy_encoder(dev,
                                                   radeon_get_encoder_enum(dev,
@@ -1058,6 +1108,7 @@ union firmware_info {
         ATOM_FIRMWARE_INFO_V1_3 info_13;
         ATOM_FIRMWARE_INFO_V1_4 info_14;
         ATOM_FIRMWARE_INFO_V2_1 info_21;
+        ATOM_FIRMWARE_INFO_V2_2 info_22;
 };
 
 bool radeon_atom_get_clock_info(struct drm_device *dev)
@@ -1112,16 +1163,6 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                                 p1pll->pll_out_min = 64800;
                         else
                                 p1pll->pll_out_min = 20000;
-                } else if (p1pll->pll_out_min > 64800) {
-                        /* Limiting the pll output range is a good thing generally as
-                         * it limits the number of possible pll combinations for a given
-                         * frequency presumably to the ones that work best on each card.
-                         * However, certain duallink DVI monitors seem to like
-                         * pll combinations that would be limited by this at least on
-                         * pre-DCE 3.0 r6xx hardware.  This might need to be adjusted per
-                         * family.
-                         */
-                        p1pll->pll_out_min = 64800;
                 }
 
                 p1pll->pll_in_min =
@@ -1132,8 +1173,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                 *p2pll = *p1pll;
 
                 /* system clock */
-                spll->reference_freq =
-                    le16_to_cpu(firmware_info->info.usReferenceClock);
+                if (ASIC_IS_DCE4(rdev))
+                        spll->reference_freq =
+                                le16_to_cpu(firmware_info->info_21.usCoreReferenceClock);
+                else
+                        spll->reference_freq =
+                                le16_to_cpu(firmware_info->info.usReferenceClock);
                 spll->reference_div = 0;
 
                 spll->pll_out_min =
@@ -1155,8 +1200,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                     le16_to_cpu(firmware_info->info.usMaxEngineClockPLL_Input);
 
                 /* memory clock */
-                mpll->reference_freq =
-                    le16_to_cpu(firmware_info->info.usReferenceClock);
+                if (ASIC_IS_DCE4(rdev))
+                        mpll->reference_freq =
+                                le16_to_cpu(firmware_info->info_21.usMemoryReferenceClock);
+                else
+                        mpll->reference_freq =
+                                le16_to_cpu(firmware_info->info.usReferenceClock);
                 mpll->reference_div = 0;
 
                 mpll->pll_out_min =
@@ -1185,8 +1234,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                 if (ASIC_IS_DCE4(rdev)) {
                         rdev->clock.default_dispclk =
                                 le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq);
-                        if (rdev->clock.default_dispclk == 0)
-                                rdev->clock.default_dispclk = 60000; /* 600 Mhz */
+                        if (rdev->clock.default_dispclk == 0) {
+                                if (ASIC_IS_DCE5(rdev))
+                                        rdev->clock.default_dispclk = 54000; /* 540 Mhz */
+                                else
+                                        rdev->clock.default_dispclk = 60000; /* 600 Mhz */
+                        }
                         rdev->clock.dp_extclk =
                                 le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq);
                 }
@@ -1500,6 +1553,9 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
                 if (misc & ATOM_DOUBLE_CLOCK_MODE)
                         lvds->native_mode.flags |= DRM_MODE_FLAG_DBLSCAN;
 
+                lvds->native_mode.width_mm = lvds_info->info.sLCDTiming.usImageHSize;
+                lvds->native_mode.height_mm = lvds_info->info.sLCDTiming.usImageVSize;
+
                 /* set crtc values */
                 drm_mode_set_crtcinfo(&lvds->native_mode, CRTC_INTERLACE_HALVE_V);
 
@@ -1512,6 +1568,59 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
                 else
                         lvds->linkb = false;
 
+                /* parse the lcd record table */
+                if (lvds_info->info.usModePatchTableOffset) {
+                        ATOM_FAKE_EDID_PATCH_RECORD *fake_edid_record;
+                        ATOM_PANEL_RESOLUTION_PATCH_RECORD *panel_res_record;
+                        bool bad_record = false;
+                        u8 *record = (u8 *)(mode_info->atom_context->bios +
+                                            data_offset +
+                                            lvds_info->info.usModePatchTableOffset);
+                        while (*record != ATOM_RECORD_END_TYPE) {
+                                switch (*record) {
+                                case LCD_MODE_PATCH_RECORD_MODE_TYPE:
+                                        record += sizeof(ATOM_PATCH_RECORD_MODE);
+                                        break;
+                                case LCD_RTS_RECORD_TYPE:
+                                        record += sizeof(ATOM_LCD_RTS_RECORD);
+                                        break;
+                                case LCD_CAP_RECORD_TYPE:
+                                        record += sizeof(ATOM_LCD_MODE_CONTROL_CAP);
+                                        break;
+                                case LCD_FAKE_EDID_PATCH_RECORD_TYPE:
+                                        fake_edid_record = (ATOM_FAKE_EDID_PATCH_RECORD *)record;
+                                        if (fake_edid_record->ucFakeEDIDLength) {
+                                                struct edid *edid;
+                                                int edid_size =
+                                                        max((int)EDID_LENGTH, (int)fake_edid_record->ucFakeEDIDLength);
+                                                edid = kmalloc(edid_size, GFP_KERNEL);
+                                                if (edid) {
+                                                        memcpy((u8 *)edid, (u8 *)&fake_edid_record->ucFakeEDIDString[0],
+                                                               fake_edid_record->ucFakeEDIDLength);
+
+                                                        if (drm_edid_is_valid(edid))
+                                                                rdev->mode_info.bios_hardcoded_edid = edid;
+                                                        else
+                                                                kfree(edid);
+                                                }
+                                        }
+                                        record += sizeof(ATOM_FAKE_EDID_PATCH_RECORD);
+                                        break;
+                                case LCD_PANEL_RESOLUTION_RECORD_TYPE:
+                                        panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record;
+                                        lvds->native_mode.width_mm = panel_res_record->usHSize;
+                                        lvds->native_mode.height_mm = panel_res_record->usVSize;
+                                        record += sizeof(ATOM_PANEL_RESOLUTION_PATCH_RECORD);
+                                        break;
+                                default:
+                                        DRM_ERROR("Bad LCD record %d\n", *record);
+                                        bad_record = true;
+                                        break;
+                                }
+                                if (bad_record)
+                                        break;
+                        }
+                }
         }
         return lvds;
 }
@@ -1768,6 +1877,7 @@ static const char *pp_lib_thermal_controller_names[] = {
         "Evergreen",
         "emc2103",
         "Sumo",
+        "Northern Islands",
 };
 
 union power_info {
@@ -1867,6 +1977,9 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev)
         num_modes = power_info->info.ucNumOfPowerModeEntries;
         if (num_modes > ATOM_MAX_NUMBEROF_POWER_BLOCK)
                 num_modes = ATOM_MAX_NUMBEROF_POWER_BLOCK;
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) * num_modes, GFP_KERNEL);
+        if (!rdev->pm.power_state)
+                return state_index;
         /* last mode is usually default, array is low to high */
         for (i = 0; i < num_modes; i++) {
                 rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
@@ -2031,6 +2144,11 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r
                                  (controller->ucFanParameters &
                                   ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
                         rdev->pm.int_thermal_type = THERMAL_TYPE_SUMO;
+                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_NISLANDS) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_NI;
                 } else if ((controller->ucType ==
                             ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) ||
                            (controller->ucType ==
@@ -2120,15 +2238,22 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde
                 rdev->pm.default_power_state_index = state_index;
                 rdev->pm.power_state[state_index].default_clock_mode =
                         &rdev->pm.power_state[state_index].clock_info[mode_index - 1];
-                /* patch the table values with the default slck/mclk from firmware info */
-                for (j = 0; j < mode_index; j++) {
-                        rdev->pm.power_state[state_index].clock_info[j].mclk =
-                                rdev->clock.default_mclk;
-                        rdev->pm.power_state[state_index].clock_info[j].sclk =
-                                rdev->clock.default_sclk;
-                        if (vddc)
-                                rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
-                                        vddc;
+                if (ASIC_IS_DCE5(rdev)) {
+                        /* NI chips post without MC ucode, so default clocks are strobe mode only */
+                        rdev->pm.default_sclk = rdev->pm.power_state[state_index].clock_info[0].sclk;
+                        rdev->pm.default_mclk = rdev->pm.power_state[state_index].clock_info[0].mclk;
+                        rdev->pm.default_vddc = rdev->pm.power_state[state_index].clock_info[0].voltage.voltage;
+                } else {
+                        /* patch the table values with the default slck/mclk from firmware info */
+                        for (j = 0; j < mode_index; j++) {
+                                rdev->pm.power_state[state_index].clock_info[j].mclk =
+                                        rdev->clock.default_mclk;
+                                rdev->pm.power_state[state_index].clock_info[j].sclk =
+                                        rdev->clock.default_sclk;
+                                if (vddc)
+                                        rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
+                                                vddc;
+                        }
                 }
         }
 }
@@ -2206,6 +2331,10 @@ static int radeon_atombios_parse_power_table_4_5(struct radeon_device *rdev)
         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
 
         radeon_atombios_add_pplib_thermal_controller(rdev, &power_info->pplib.sThermalController);
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) *
+                                       power_info->pplib.ucNumStates, GFP_KERNEL);
+        if (!rdev->pm.power_state)
+                return state_index;
         /* first mode is usually default, followed by low to high */
         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
                 mode_index = 0;
@@ -2286,6 +2415,10 @@ static int radeon_atombios_parse_power_table_6(struct radeon_device *rdev)
         non_clock_info_array = (struct NonClockInfoArray *)
                 (mode_info->atom_context->bios + data_offset +
                  power_info->pplib.usNonClockInfoArrayOffset);
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) *
+                                       state_array->ucNumEntries, GFP_KERNEL);
+        if (!rdev->pm.power_state)
+                return state_index;
         for (i = 0; i < state_array->ucNumEntries; i++) {
                 mode_index = 0;
                 power_state = (union pplib_power_state *)&state_array->states[i];
@@ -2359,19 +2492,22 @@ void radeon_atombios_get_power_modes(struct radeon_device *rdev)
                         break;
                 }
         } else {
-                /* add the default mode */
-                rdev->pm.power_state[state_index].type =
-                        POWER_STATE_TYPE_DEFAULT;
-                rdev->pm.power_state[state_index].num_clock_modes = 1;
-                rdev->pm.power_state[state_index].clock_info[0].mclk = rdev->clock.default_mclk;
-                rdev->pm.power_state[state_index].clock_info[0].sclk = rdev->clock.default_sclk;
-                rdev->pm.power_state[state_index].default_clock_mode =
-                        &rdev->pm.power_state[state_index].clock_info[0];
-                rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
-                rdev->pm.power_state[state_index].pcie_lanes = 16;
-                rdev->pm.default_power_state_index = state_index;
-                rdev->pm.power_state[state_index].flags = 0;
-                state_index++;
+                rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state), GFP_KERNEL);
+                if (rdev->pm.power_state) {
+                        /* add the default mode */
+                        rdev->pm.power_state[state_index].type =
+                                POWER_STATE_TYPE_DEFAULT;
+                        rdev->pm.power_state[state_index].num_clock_modes = 1;
+                        rdev->pm.power_state[state_index].clock_info[0].mclk = rdev->clock.default_mclk;
+                        rdev->pm.power_state[state_index].clock_info[0].sclk = rdev->clock.default_sclk;
+                        rdev->pm.power_state[state_index].default_clock_mode =
+                                &rdev->pm.power_state[state_index].clock_info[0];
+                        rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
+                        rdev->pm.power_state[state_index].pcie_lanes = 16;
+                        rdev->pm.default_power_state_index = state_index;
+                        rdev->pm.power_state[state_index].flags = 0;
+                        state_index++;
+                }
         }
 
         rdev->pm.num_power_states = state_index;
@@ -2487,7 +2623,7 @@ void radeon_atom_initialize_bios_scratch_regs(struct drm_device *dev)
         bios_2_scratch &= ~ATOM_S2_VRI_BRIGHT_ENABLE;
 
         /* tell the bios not to handle mode switching */
-        bios_6_scratch |= (ATOM_S6_ACC_BLOCK_DISPLAY_SWITCH | ATOM_S6_ACC_MODE);
+        bios_6_scratch |= ATOM_S6_ACC_BLOCK_DISPLAY_SWITCH;
 
         if (rdev->family >= CHIP_R600) {
                 WREG32(R600_BIOS_2_SCRATCH, bios_2_scratch);
@@ -2538,10 +2674,13 @@ void radeon_atom_output_lock(struct drm_encoder *encoder, bool lock)
         else
                 bios_6_scratch = RREG32(RADEON_BIOS_6_SCRATCH);
 
-        if (lock)
+        if (lock) {
                 bios_6_scratch |= ATOM_S6_CRITICAL_STATE;
-        else
+                bios_6_scratch &= ~ATOM_S6_ACC_MODE;
+        } else {
                 bios_6_scratch &= ~ATOM_S6_CRITICAL_STATE;
+                bios_6_scratch |= ATOM_S6_ACC_MODE;
+        }
 
         if (rdev->family >= CHIP_R600)
                 WREG32(R600_BIOS_6_SCRATCH, bios_6_scratch);
diff --git a/drivers/gpu/drm/radeon/radeon_benchmark.c b/drivers/gpu/drm/radeon/radeon_benchmark.c
index c558685cc637..10191d9372d8 100644
--- a/drivers/gpu/drm/radeon/radeon_benchmark.c
+++ b/drivers/gpu/drm/radeon/radeon_benchmark.c
@@ -41,7 +41,7 @@ void radeon_benchmark_move(struct radeon_device *rdev, unsigned bsize,
 
         size = bsize;
         n = 1024;
-        r = radeon_bo_create(rdev, NULL, size, PAGE_SIZE, true, sdomain, &sobj);
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true, sdomain, &sobj);
         if (r) {
                 goto out_cleanup;
         }
@@ -53,7 +53,7 @@ void radeon_benchmark_move(struct radeon_device *rdev, unsigned bsize,
         if (r) {
                 goto out_cleanup;
         }
-        r = radeon_bo_create(rdev, NULL, size, PAGE_SIZE, true, ddomain, &dobj);
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true, ddomain, &dobj);
         if (r) {
                 goto out_cleanup;
         }
diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c
index 654787ec43f4..1aba85cad1a8 100644
--- a/drivers/gpu/drm/radeon/radeon_bios.c
+++ b/drivers/gpu/drm/radeon/radeon_bios.c
@@ -130,6 +130,46 @@ static bool radeon_atrm_get_bios(struct radeon_device *rdev)
         }
         return true;
 }
+
+static bool ni_read_disabled_bios(struct radeon_device *rdev)
+{
+        u32 bus_cntl;
+        u32 d1vga_control;
+        u32 d2vga_control;
+        u32 vga_render_control;
+        u32 rom_cntl;
+        bool r;
+
+        bus_cntl = RREG32(R600_BUS_CNTL);
+        d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
+        d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
+        vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
+        rom_cntl = RREG32(R600_ROM_CNTL);
+
+        /* enable the rom */
+        WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
+        /* Disable VGA mode */
+        WREG32(AVIVO_D1VGA_CONTROL,
+               (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
+                AVIVO_DVGA_CONTROL_TIMING_SELECT)));
+        WREG32(AVIVO_D2VGA_CONTROL,
+               (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
+                AVIVO_DVGA_CONTROL_TIMING_SELECT)));
+        WREG32(AVIVO_VGA_RENDER_CONTROL,
+               (vga_render_control & ~AVIVO_VGA_VSTATUS_CNTL_MASK));
+        WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
+
+        r = radeon_read_bios(rdev);
+
+        /* restore regs */
+        WREG32(R600_BUS_CNTL, bus_cntl);
+        WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
+        WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
+        WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
+        WREG32(R600_ROM_CNTL, rom_cntl);
+        return r;
+}
+
 static bool r700_read_disabled_bios(struct radeon_device *rdev)
 {
         uint32_t viph_control;
@@ -143,7 +183,7 @@ static bool r700_read_disabled_bios(struct radeon_device *rdev)
         bool r;
 
         viph_control = RREG32(RADEON_VIPH_CONTROL);
-        bus_cntl = RREG32(RADEON_BUS_CNTL);
+        bus_cntl = RREG32(R600_BUS_CNTL);
         d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
         d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
         vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
@@ -152,7 +192,7 @@ static bool r700_read_disabled_bios(struct radeon_device *rdev)
         /* disable VIP */
         WREG32(RADEON_VIPH_CONTROL, (viph_control & ~RADEON_VIPH_EN));
         /* enable the rom */
-        WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
+        WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
         /* Disable VGA mode */
         WREG32(AVIVO_D1VGA_CONTROL,
                (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
@@ -191,7 +231,7 @@ static bool r700_read_disabled_bios(struct radeon_device *rdev)
                         cg_spll_status = RREG32(R600_CG_SPLL_STATUS);
         }
         WREG32(RADEON_VIPH_CONTROL, viph_control);
-        WREG32(RADEON_BUS_CNTL, bus_cntl);
+        WREG32(R600_BUS_CNTL, bus_cntl);
         WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
         WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
         WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
@@ -216,7 +256,7 @@ static bool r600_read_disabled_bios(struct radeon_device *rdev)
         bool r;
 
         viph_control = RREG32(RADEON_VIPH_CONTROL);
-        bus_cntl = RREG32(RADEON_BUS_CNTL);
+        bus_cntl = RREG32(R600_BUS_CNTL);
         d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
         d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
         vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
@@ -231,7 +271,7 @@ static bool r600_read_disabled_bios(struct radeon_device *rdev)
         /* disable VIP */
         WREG32(RADEON_VIPH_CONTROL, (viph_control & ~RADEON_VIPH_EN));
         /* enable the rom */
-        WREG32(RADEON_BUS_CNTL, (bus_cntl & ~RADEON_BUS_BIOS_DIS_ROM));
+        WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
         /* Disable VGA mode */
         WREG32(AVIVO_D1VGA_CONTROL,
                (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
@@ -262,7 +302,7 @@ static bool r600_read_disabled_bios(struct radeon_device *rdev)
 
         /* restore regs */
         WREG32(RADEON_VIPH_CONTROL, viph_control);
-        WREG32(RADEON_BUS_CNTL, bus_cntl);
+        WREG32(R600_BUS_CNTL, bus_cntl);
         WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
         WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
         WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
@@ -415,6 +455,8 @@ static bool radeon_read_disabled_bios(struct radeon_device *rdev)
 {
         if (rdev->flags & RADEON_IS_IGP)
                 return igp_read_bios_from_vram(rdev);
+        else if (rdev->family >= CHIP_BARTS)
+                return ni_read_disabled_bios(rdev);
         else if (rdev->family >= CHIP_RV770)
                 return r700_read_disabled_bios(rdev);
         else if (rdev->family >= CHIP_R600)
diff --git a/drivers/gpu/drm/radeon/radeon_combios.c b/drivers/gpu/drm/radeon/radeon_combios.c
index 3bddea5b5295..d27ef74590cd 100644
--- a/drivers/gpu/drm/radeon/radeon_combios.c
+++ b/drivers/gpu/drm/radeon/radeon_combios.c
@@ -471,8 +471,9 @@ bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev)
         return true;
 }
 
+/* this is used for atom LCDs as well */
 struct edid *
-radeon_combios_get_hardcoded_edid(struct radeon_device *rdev)
+radeon_bios_get_hardcoded_edid(struct radeon_device *rdev)
 {
         if (rdev->mode_info.bios_hardcoded_edid)
                 return rdev->mode_info.bios_hardcoded_edid;
@@ -729,7 +730,7 @@ void radeon_combios_i2c_init(struct radeon_device *rdev)
                                         clk = RBIOS8(offset + 3 + (i * 5) + 3);
                                         data = RBIOS8(offset + 3 + (i * 5) + 4);
                                         i2c = combios_setup_i2c_bus(rdev, DDC_MONID,
-                                                                    clk, data);
+                                                                    (1 << clk), (1 << data));
                                         rdev->i2c_bus[4] = radeon_i2c_create(dev, &i2c, "GPIOPAD_MASK");
                                         break;
                                 }
@@ -2441,6 +2442,17 @@ void radeon_combios_get_power_modes(struct radeon_device *rdev)
 
         rdev->pm.default_power_state_index = -1;
 
+        /* allocate 2 power states */
+        rdev->pm.power_state = kzalloc(sizeof(struct radeon_power_state) * 2, GFP_KERNEL);
+        if (!rdev->pm.power_state) {
+                rdev->pm.default_power_state_index = state_index;
+                rdev->pm.num_power_states = 0;
+
+                rdev->pm.current_power_state_index = rdev->pm.default_power_state_index;
+                rdev->pm.current_clock_mode_index = 0;
+                return;
+        }
+
         if (rdev->flags & RADEON_IS_MOBILITY) {
                 offset = combios_get_table_offset(dev, COMBIOS_POWERPLAY_INFO_TABLE);
                 if (offset) {
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 3bef9f6d66fd..22b7e3dc0eca 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -472,6 +472,9 @@ static int radeon_lvds_get_modes(struct drm_connector *connector)
         if (mode) {
                 ret = 1;
                 drm_mode_probed_add(connector, mode);
+                /* add the width/height from vbios tables if available */
+                connector->display_info.width_mm = mode->width_mm;
+                connector->display_info.height_mm = mode->height_mm;
                 /* add scaled modes */
                 radeon_add_common_modes(encoder, connector);
         }
@@ -1175,6 +1178,8 @@ radeon_add_atom_connector(struct drm_device *dev,
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
                 connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                connector->interlace_allowed = true;
+                connector->doublescan_allowed = true;
                 break;
         case DRM_MODE_CONNECTOR_DVIA:
                 drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
@@ -1190,6 +1195,8 @@ radeon_add_atom_connector(struct drm_device *dev,
                                               1);
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                connector->interlace_allowed = true;
+                connector->doublescan_allowed = true;
                 break;
         case DRM_MODE_CONNECTOR_DVII:
         case DRM_MODE_CONNECTOR_DVID:
@@ -1212,7 +1219,7 @@ radeon_add_atom_connector(struct drm_device *dev,
                 if (ASIC_IS_AVIVO(rdev)) {
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_property,
-                                                      UNDERSCAN_AUTO);
+                                                      UNDERSCAN_OFF);
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_hborder_property,
                                                       0);
@@ -1226,6 +1233,11 @@ radeon_add_atom_connector(struct drm_device *dev,
                                                       rdev->mode_info.load_detect_property,
                                                       1);
                 }
+                connector->interlace_allowed = true;
+                if (connector_type == DRM_MODE_CONNECTOR_DVII)
+                        connector->doublescan_allowed = true;
+                else
+                        connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_HDMIA:
         case DRM_MODE_CONNECTOR_HDMIB:
@@ -1247,7 +1259,7 @@ radeon_add_atom_connector(struct drm_device *dev,
                 if (ASIC_IS_AVIVO(rdev)) {
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_property,
-                                                      UNDERSCAN_AUTO);
+                                                      UNDERSCAN_OFF);
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_hborder_property,
                                                       0);
@@ -1256,6 +1268,11 @@ radeon_add_atom_connector(struct drm_device *dev,
                                                       0);
                 }
                 subpixel_order = SubPixelHorizontalRGB;
+                connector->interlace_allowed = true;
+                if (connector_type == DRM_MODE_CONNECTOR_HDMIB)
+                        connector->doublescan_allowed = true;
+                else
+                        connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_DisplayPort:
         case DRM_MODE_CONNECTOR_eDP:
@@ -1285,7 +1302,7 @@ radeon_add_atom_connector(struct drm_device *dev,
                 if (ASIC_IS_AVIVO(rdev)) {
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_property,
-                                                      UNDERSCAN_AUTO);
+                                                      UNDERSCAN_OFF);
                         drm_connector_attach_property(&radeon_connector->base,
                                                       rdev->mode_info.underscan_hborder_property,
                                                       0);
@@ -1293,6 +1310,9 @@ radeon_add_atom_connector(struct drm_device *dev,
                                                       rdev->mode_info.underscan_vborder_property,
                                                       0);
                 }
+                connector->interlace_allowed = true;
+                /* in theory with a DP to VGA converter... */
+                connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_SVIDEO:
         case DRM_MODE_CONNECTOR_Composite:
@@ -1308,6 +1328,8 @@ radeon_add_atom_connector(struct drm_device *dev,
                                               radeon_atombios_get_tv_info(rdev));
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                connector->interlace_allowed = false;
+                connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_LVDS:
                 radeon_dig_connector = kzalloc(sizeof(struct radeon_connector_atom_dig), GFP_KERNEL);
@@ -1326,6 +1348,8 @@ radeon_add_atom_connector(struct drm_device *dev,
                                               dev->mode_config.scaling_mode_property,
                                               DRM_MODE_SCALE_FULLSCREEN);
                 subpixel_order = SubPixelHorizontalRGB;
+                connector->interlace_allowed = false;
+                connector->doublescan_allowed = false;
                 break;
         }
 
@@ -1403,6 +1427,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
                 connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                connector->interlace_allowed = true;
+                connector->doublescan_allowed = true;
                 break;
         case DRM_MODE_CONNECTOR_DVIA:
                 drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
@@ -1418,6 +1444,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                               1);
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                connector->interlace_allowed = true;
+                connector->doublescan_allowed = true;
                 break;
         case DRM_MODE_CONNECTOR_DVII:
         case DRM_MODE_CONNECTOR_DVID:
@@ -1435,6 +1463,11 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                                       1);
                 }
                 subpixel_order = SubPixelHorizontalRGB;
+                connector->interlace_allowed = true;
+                if (connector_type == DRM_MODE_CONNECTOR_DVII)
+                        connector->doublescan_allowed = true;
+                else
+                        connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_SVIDEO:
         case DRM_MODE_CONNECTOR_Composite:
@@ -1457,6 +1490,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                               radeon_combios_get_tv_info(rdev));
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
+                connector->interlace_allowed = false;
+                connector->doublescan_allowed = false;
                 break;
         case DRM_MODE_CONNECTOR_LVDS:
                 drm_connector_init(dev, &radeon_connector->base, &radeon_lvds_connector_funcs, connector_type);
@@ -1470,6 +1505,8 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                               dev->mode_config.scaling_mode_property,
                                               DRM_MODE_SCALE_FULLSCREEN);
                 subpixel_order = SubPixelHorizontalRGB;
+                connector->interlace_allowed = false;
+                connector->doublescan_allowed = false;
                 break;
         }
 
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index eb6b9eed7349..3d599e33b9cc 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -2113,9 +2113,9 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
                 break;
         }
 
-        if (drm_device_is_agp(dev))
+        if (drm_pci_device_is_agp(dev))
                 dev_priv->flags |= RADEON_IS_AGP;
-        else if (drm_device_is_pcie(dev))
+        else if (drm_pci_device_is_pcie(dev))
                 dev_priv->flags |= RADEON_IS_PCIE;
         else
                 dev_priv->flags |= RADEON_IS_PCI;
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 35b5eb8fbe2a..8c1916941871 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -75,7 +75,7 @@ int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
                                 return -ENOENT;
                         }
                         p->relocs_ptr[i] = &p->relocs[i];
-                        p->relocs[i].robj = p->relocs[i].gobj->driver_private;
+                        p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
                         p->relocs[i].lobj.bo = p->relocs[i].robj;
                         p->relocs[i].lobj.wdomain = r->write_domain;
                         p->relocs[i].lobj.rdomain = r->read_domains;
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 3952cf3d0ee9..7c0a3f26ab5e 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -82,6 +82,9 @@ static const char radeon_family_name[][16] = {
         "CYPRESS",
         "HEMLOCK",
         "PALM",
+        "BARTS",
+        "TURKS",
+        "CAICOS",
         "LAST",
 };
 
@@ -181,7 +184,7 @@ int radeon_wb_init(struct radeon_device *rdev)
         int r;
 
         if (rdev->wb.wb_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
+                r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
                                 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
                 if (r) {
                         dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
@@ -225,6 +228,11 @@ int radeon_wb_init(struct radeon_device *rdev)
                                 rdev->wb.use_event = true;
                 }
         }
+        /* always use writeback/events on NI */
+        if (ASIC_IS_DCE5(rdev)) {
+                rdev->wb.enabled = true;
+                rdev->wb.use_event = true;
+        }
 
         dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
 
@@ -287,7 +295,7 @@ void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64
                 mc->mc_vram_size = mc->aper_size;
         }
         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
-        dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
+        dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
                         mc->mc_vram_size >> 20, mc->vram_start,
                         mc->vram_end, mc->real_vram_size >> 20);
 }
@@ -324,7 +332,7 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
         }
         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
-        dev_info(rdev->dev, "GTT: %lluM 0x%08llX - 0x%08llX\n",
+        dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 }
 
@@ -642,20 +650,20 @@ void radeon_check_arguments(struct radeon_device *rdev)
 static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
-        struct radeon_device *rdev = dev->dev_private;
         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
         if (state == VGA_SWITCHEROO_ON) {
                 printk(KERN_INFO "radeon: switched on\n");
                 /* don't suspend or resume card normally */
-                rdev->powered_down = false;
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 radeon_resume_kms(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
                 drm_kms_helper_poll_enable(dev);
         } else {
                 printk(KERN_INFO "radeon: switched off\n");
                 drm_kms_helper_poll_disable(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
                 radeon_suspend_kms(dev, pmm);
-                /* don't suspend or resume card normally */
-                rdev->powered_down = true;
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
 
@@ -710,11 +718,6 @@ int radeon_device_init(struct radeon_device *rdev,
         init_waitqueue_head(&rdev->irq.vblank_queue);
         init_waitqueue_head(&rdev->irq.idle_queue);
 
-        /* setup workqueue */
-        rdev->wq = create_workqueue("radeon");
-        if (rdev->wq == NULL)
-                return -ENOMEM;
-
         /* Set asic functions */
         r = radeon_asic_init(rdev);
         if (r)
@@ -779,6 +782,7 @@ int radeon_device_init(struct radeon_device *rdev,
         vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
         vga_switcheroo_register_client(rdev->pdev,
                                        radeon_switcheroo_set_state,
+                                       NULL,
                                        radeon_switcheroo_can_switch);
 
         r = radeon_init(rdev);
@@ -812,7 +816,6 @@ void radeon_device_fini(struct radeon_device *rdev)
         /* evict vram memory */
         radeon_bo_evict_vram(rdev);
         radeon_fini(rdev);
-        destroy_workqueue(rdev->wq);
         vga_switcheroo_unregister_client(rdev->pdev);
         vga_client_register(rdev->pdev, NULL, NULL, NULL);
         if (rdev->rio_mem)
@@ -841,7 +844,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
         }
         rdev = dev->dev_private;
 
-        if (rdev->powered_down)
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
         /* turn off display hw */
@@ -857,7 +860,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
                 if (rfb == NULL || rfb->obj == NULL) {
                         continue;
                 }
-                robj = rfb->obj->driver_private;
+                robj = gem_to_radeon_bo(rfb->obj);
                 /* don't unpin kernel fb objects */
                 if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
                         r = radeon_bo_reserve(robj, false);
@@ -888,9 +891,9 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
                 pci_disable_device(dev->pdev);
                 pci_set_power_state(dev->pdev, PCI_D3hot);
         }
-        acquire_console_sem();
+        console_lock();
         radeon_fbdev_set_suspend(rdev, 1);
-        release_console_sem();
+        console_unlock();
         return 0;
 }
 
@@ -899,14 +902,14 @@ int radeon_resume_kms(struct drm_device *dev)
         struct drm_connector *connector;
         struct radeon_device *rdev = dev->dev_private;
 
-        if (rdev->powered_down)
+        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
-        acquire_console_sem();
+        console_lock();
         pci_set_power_state(dev->pdev, PCI_D0);
         pci_restore_state(dev->pdev);
         if (pci_enable_device(dev->pdev)) {
-                release_console_sem();
+                console_unlock();
                 return -1;
         }
         pci_set_master(dev->pdev);
@@ -916,18 +919,17 @@ int radeon_resume_kms(struct drm_device *dev)
         radeon_pm_resume(rdev);
         radeon_restore_bios_scratch_regs(rdev);
 
-        /* turn on display hw */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
-        }
-
         radeon_fbdev_set_suspend(rdev, 0);
-        release_console_sem();
+        console_unlock();
 
         /* reset hpd state */
         radeon_hpd_init(rdev);
         /* blat the mode back in */
         drm_helper_resume_force_mode(dev);
+        /* turn on display hw */
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+        }
         return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 7b17e639ab32..4409975a363c 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -68,7 +68,7 @@ static void avivo_crtc_load_lut(struct drm_crtc *crtc)
         WREG32(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id);
 }
 
-static void evergreen_crtc_load_lut(struct drm_crtc *crtc)
+static void dce4_crtc_load_lut(struct drm_crtc *crtc)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
         struct drm_device *dev = crtc->dev;
@@ -98,6 +98,66 @@ static void evergreen_crtc_load_lut(struct drm_crtc *crtc)
         }
 }
 
+static void dce5_crtc_load_lut(struct drm_crtc *crtc)
+{
+        struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+        struct drm_device *dev = crtc->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        int i;
+
+        DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
+
+        WREG32(NI_INPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
+               (NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) |
+                NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS)));
+        WREG32(NI_PRESCALE_GRPH_CONTROL + radeon_crtc->crtc_offset,
+               NI_GRPH_PRESCALE_BYPASS);
+        WREG32(NI_PRESCALE_OVL_CONTROL + radeon_crtc->crtc_offset,
+               NI_OVL_PRESCALE_BYPASS);
+        WREG32(NI_INPUT_GAMMA_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT) |
+                NI_OVL_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT)));
+
+        WREG32(EVERGREEN_DC_LUT_CONTROL + radeon_crtc->crtc_offset, 0);
+
+        WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + radeon_crtc->crtc_offset, 0);
+        WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + radeon_crtc->crtc_offset, 0);
+        WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + radeon_crtc->crtc_offset, 0);
+
+        WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + radeon_crtc->crtc_offset, 0xffff);
+        WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + radeon_crtc->crtc_offset, 0xffff);
+        WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + radeon_crtc->crtc_offset, 0xffff);
+
+        WREG32(EVERGREEN_DC_LUT_RW_MODE + radeon_crtc->crtc_offset, 0);
+        WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
+
+        WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
+        for (i = 0; i < 256; i++) {
+                WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
+                       (radeon_crtc->lut_r[i] << 20) |
+                       (radeon_crtc->lut_g[i] << 10) |
+                       (radeon_crtc->lut_b[i] << 0));
+        }
+
+        WREG32(NI_DEGAMMA_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+                NI_OVL_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+                NI_ICON_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+                NI_CURSOR_DEGAMMA_MODE(NI_DEGAMMA_BYPASS)));
+        WREG32(NI_GAMUT_REMAP_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS) |
+                NI_OVL_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS)));
+        WREG32(NI_REGAMMA_CONTROL + radeon_crtc->crtc_offset,
+               (NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) |
+                NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS)));
+        WREG32(NI_OUTPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
+               (NI_OUTPUT_CSC_GRPH_MODE(NI_OUTPUT_CSC_BYPASS) |
+                NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
+        /* XXX match this to the depth of the crtc fmt block, move to modeset? */
+        WREG32(0x6940 + radeon_crtc->crtc_offset, 0);
+
+}
+
 static void legacy_crtc_load_lut(struct drm_crtc *crtc)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
@@ -130,8 +190,10 @@ void radeon_crtc_load_lut(struct drm_crtc *crtc)
         if (!crtc->enabled)
                 return;
 
-        if (ASIC_IS_DCE4(rdev))
-                evergreen_crtc_load_lut(crtc);
+        if (ASIC_IS_DCE5(rdev))
+                dce5_crtc_load_lut(crtc);
+        else if (ASIC_IS_DCE4(rdev))
+                dce4_crtc_load_lut(crtc);
         else if (ASIC_IS_AVIVO(rdev))
                 avivo_crtc_load_lut(crtc);
         else
@@ -309,7 +371,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         new_radeon_fb = to_radeon_framebuffer(fb);
         /* schedule unpin of the old buffer */
         obj = old_radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         work->old_rbo = rbo;
         INIT_WORK(&work->work, radeon_unpin_work_func);
 
@@ -329,7 +391,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
 
         /* pin the new buffer */
         obj = new_radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
 
         DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",
                          work->old_rbo, rbo);
@@ -679,9 +741,17 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
         if (!radeon_connector->edid) {
                 radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter);
         }
-        /* some servers provide a hardcoded edid in rom for KVMs */
-        if (!radeon_connector->edid)
-                radeon_connector->edid = radeon_combios_get_hardcoded_edid(rdev);
+
+        if (!radeon_connector->edid) {
+                if (rdev->is_atom_bios) {
+                        /* some laptops provide a hardcoded edid in rom for LCDs */
+                        if (((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_LVDS) ||
+                             (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)))
+                                radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev);
+                } else
+                        /* some servers provide a hardcoded edid in rom for KVMs */
+                        radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev);
+        }
         if (radeon_connector->edid) {
                 drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid);
                 ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid);
@@ -710,6 +780,115 @@ static int radeon_ddc_dump(struct drm_connector *connector)
         return ret;
 }
 
+/* avivo */
+static void avivo_get_fb_div(struct radeon_pll *pll,
+                             u32 target_clock,
+                             u32 post_div,
+                             u32 ref_div,
+                             u32 *fb_div,
+                             u32 *frac_fb_div)
+{
+        u32 tmp = post_div * ref_div;
+
+        tmp *= target_clock;
+        *fb_div = tmp / pll->reference_freq;
+        *frac_fb_div = tmp % pll->reference_freq;
+}
+
+static u32 avivo_get_post_div(struct radeon_pll *pll,
+                              u32 target_clock)
+{
+        u32 vco, post_div, tmp;
+
+        if (pll->flags & RADEON_PLL_USE_POST_DIV)
+                return pll->post_div;
+
+        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) {
+                if (pll->flags & RADEON_PLL_IS_LCD)
+                        vco = pll->lcd_pll_out_min;
+                else
+                        vco = pll->pll_out_min;
+        } else {
+                if (pll->flags & RADEON_PLL_IS_LCD)
+                        vco = pll->lcd_pll_out_max;
+                else
+                        vco = pll->pll_out_max;
+        }
+
+        post_div = vco / target_clock;
+        tmp = vco % target_clock;
+
+        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) {
+                if (tmp)
+                        post_div++;
+        } else {
+                if (!tmp)
+                        post_div--;
+        }
+
+        return post_div;
+}
+
+#define MAX_TOLERANCE 10
+
+void radeon_compute_pll_avivo(struct radeon_pll *pll,
+                              u32 freq,
+                              u32 *dot_clock_p,
+                              u32 *fb_div_p,
+                              u32 *frac_fb_div_p,
+                              u32 *ref_div_p,
+                              u32 *post_div_p)
+{
+        u32 target_clock = freq / 10;
+        u32 post_div = avivo_get_post_div(pll, target_clock);
+        u32 ref_div = pll->min_ref_div;
+        u32 fb_div = 0, frac_fb_div = 0, tmp;
+
+        if (pll->flags & RADEON_PLL_USE_REF_DIV)
+                ref_div = pll->reference_div;
+
+        if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
+                avivo_get_fb_div(pll, target_clock, post_div, ref_div, &fb_div, &frac_fb_div);
+                frac_fb_div = (100 * frac_fb_div) / pll->reference_freq;
+                if (frac_fb_div >= 5) {
+                        frac_fb_div -= 5;
+                        frac_fb_div = frac_fb_div / 10;
+                        frac_fb_div++;
+                }
+                if (frac_fb_div >= 10) {
+                        fb_div++;
+                        frac_fb_div = 0;
+                }
+        } else {
+                while (ref_div <= pll->max_ref_div) {
+                        avivo_get_fb_div(pll, target_clock, post_div, ref_div,
+                                         &fb_div, &frac_fb_div);
+                        if (frac_fb_div >= (pll->reference_freq / 2))
+                                fb_div++;
+                        frac_fb_div = 0;
+                        tmp = (pll->reference_freq * fb_div) / (post_div * ref_div);
+                        tmp = (tmp * 10000) / target_clock;
+
+                        if (tmp > (10000 + MAX_TOLERANCE))
+                                ref_div++;
+                        else if (tmp >= (10000 - MAX_TOLERANCE))
+                                break;
+                        else
+                                ref_div++;
+                }
+        }
+
+        *dot_clock_p = ((pll->reference_freq * fb_div * 10) + (pll->reference_freq * frac_fb_div)) /
+                (ref_div * post_div * 10);
+        *fb_div_p = fb_div;
+        *frac_fb_div_p = frac_fb_div;
+        *ref_div_p = ref_div;
+        *post_div_p = post_div;
+        DRM_DEBUG_KMS("%d, pll dividers - fb: %d.%d ref: %d, post %d\n",
+                      *dot_clock_p, fb_div, frac_fb_div, ref_div, post_div);
+}
+
+/* pre-avivo */
 static inline uint32_t radeon_div(uint64_t n, uint32_t d)
 {
         uint64_t mod;
@@ -720,13 +899,13 @@ static inline uint32_t radeon_div(uint64_t n, uint32_t d)
         return n;
 }
 
-void radeon_compute_pll(struct radeon_pll *pll,
-                        uint64_t freq,
-                        uint32_t *dot_clock_p,
-                        uint32_t *fb_div_p,
-                        uint32_t *frac_fb_div_p,
-                        uint32_t *ref_div_p,
-                        uint32_t *post_div_p)
+void radeon_compute_pll_legacy(struct radeon_pll *pll,
+                               uint64_t freq,
+                               uint32_t *dot_clock_p,
+                               uint32_t *fb_div_p,
+                               uint32_t *frac_fb_div_p,
+                               uint32_t *ref_div_p,
+                               uint32_t *post_div_p)
 {
         uint32_t min_ref_div = pll->min_ref_div;
         uint32_t max_ref_div = pll->max_ref_div;
@@ -756,6 +935,9 @@ void radeon_compute_pll(struct radeon_pll *pll,
                 pll_out_max = pll->pll_out_max;
         }
 
+        if (pll_out_min > 64800)
+                pll_out_min = 64800;
+
         if (pll->flags & RADEON_PLL_USE_REF_DIV)
                 min_ref_div = max_ref_div = pll->reference_div;
         else {
@@ -779,7 +961,7 @@ void radeon_compute_pll(struct radeon_pll *pll,
                 max_fractional_feed_div = pll->max_frac_feedback_div;
         }
 
-        for (post_div = max_post_div; post_div >= min_post_div; --post_div) {
+        for (post_div = min_post_div; post_div <= max_post_div; ++post_div) {
                 uint32_t ref_div;
 
                 if ((pll->flags & RADEON_PLL_NO_ODD_POST_DIV) && (post_div & 1))
@@ -895,6 +1077,10 @@ void radeon_compute_pll(struct radeon_pll *pll,
         *frac_fb_div_p = best_frac_feedback_div;
         *ref_div_p = best_ref_div;
         *post_div_p = best_post_div;
+        DRM_DEBUG_KMS("%d %d, pll dividers - fb: %d.%d ref: %d, post %d\n",
+                      freq, best_freq / 1000, best_feedback_div, best_frac_feedback_div,
+                      best_ref_div, best_post_div);
+
 }
 
 static void radeon_user_framebuffer_destroy(struct drm_framebuffer *fb)
@@ -1111,7 +1297,10 @@ int radeon_modeset_init(struct radeon_device *rdev)
 
         rdev->ddev->mode_config.funcs = (void *)&radeon_mode_funcs;
 
-        if (ASIC_IS_AVIVO(rdev)) {
+        if (ASIC_IS_DCE5(rdev)) {
+                rdev->ddev->mode_config.max_width = 16384;
+                rdev->ddev->mode_config.max_height = 16384;
+        } else if (ASIC_IS_AVIVO(rdev)) {
                 rdev->ddev->mode_config.max_width = 8192;
                 rdev->ddev->mode_config.max_height = 8192;
         } else {
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index a92d2a5cea90..8a0df3d2a4c3 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -48,7 +48,7 @@
  * - 2.5.0 - add get accel 2 to work around ddx breakage for evergreen
  * - 2.6.0 - add tiling config query (r6xx+), add initial HiZ support (r300->r500)
  *   2.7.0 - fixups for r600 2D tiling support. (no external ABI change), add eg dyn gpr regs
- *   2.8.0 - pageflip support
+ *   2.8.0 - pageflip support, r500 US_FORMAT regs. r500 ARGB2101010 colorbuf, r300->r500 CMASK, clock crystal query
  */
 #define KMS_DRIVER_MAJOR        2
 #define KMS_DRIVER_MINOR        8
@@ -84,6 +84,16 @@ extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
 extern struct drm_ioctl_desc radeon_ioctls_kms[];
 extern int radeon_max_kms_ioctl;
 int radeon_mmap(struct file *filp, struct vm_area_struct *vma);
+int radeon_mode_dumb_mmap(struct drm_file *filp,
+                          struct drm_device *dev,
+                          uint32_t handle, uint64_t *offset_p);
+int radeon_mode_dumb_create(struct drm_file *file_priv,
+                            struct drm_device *dev,
+                            struct drm_mode_create_dumb *args);
+int radeon_mode_dumb_destroy(struct drm_file *file_priv,
+                             struct drm_device *dev,
+                             uint32_t handle);
+
 #if defined(CONFIG_DEBUG_FS)
 int radeon_debugfs_init(struct drm_minor *minor);
 void radeon_debugfs_cleanup(struct drm_minor *minor);
@@ -104,6 +114,7 @@ int radeon_tv = 1;
 int radeon_audio = 1;
 int radeon_disp_priority = 0;
 int radeon_hw_i2c = 0;
+int radeon_pcie_gen2 = 0;
 
 MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers");
 module_param_named(no_wb, radeon_no_wb, int, 0444);
@@ -147,6 +158,9 @@ module_param_named(disp_priority, radeon_disp_priority, int, 0444);
 MODULE_PARM_DESC(hw_i2c, "hw i2c engine enable (0 = disable)");
 module_param_named(hw_i2c, radeon_hw_i2c, int, 0444);
 
+MODULE_PARM_DESC(pcie_gen2, "PCIE Gen2 mode (1 = enable)");
+module_param_named(pcie_gen2, radeon_pcie_gen2, int, 0444);
+
 static int radeon_suspend(struct drm_device *dev, pm_message_t state)
 {
         drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -224,11 +238,6 @@ static struct drm_driver driver_old = {
                  .llseek = noop_llseek,
         },
 
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
-
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
         .date = DRIVER_DATE,
@@ -239,9 +248,28 @@ static struct drm_driver driver_old = {
 
 static struct drm_driver kms_driver;
 
+static void radeon_kick_out_firmware_fb(struct pci_dev *pdev)
+{
+        struct apertures_struct *ap;
+        bool primary = false;
+
+        ap = alloc_apertures(1);
+        ap->ranges[0].base = pci_resource_start(pdev, 0);
+        ap->ranges[0].size = pci_resource_len(pdev, 0);
+
+#ifdef CONFIG_X86
+        primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+#endif
+        remove_conflicting_framebuffers(ap, "radeondrmfb", primary);
+        kfree(ap);
+}
+
 static int __devinit
 radeon_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
+        /* Get rid of things like offb */
+        radeon_kick_out_firmware_fb(pdev);
+
         return drm_get_pci_dev(pdev, ent, &kms_driver);
 }
 
@@ -299,6 +327,9 @@ static struct drm_driver kms_driver = {
         .gem_init_object = radeon_gem_object_init,
         .gem_free_object = radeon_gem_object_free,
         .dma_ioctl = radeon_dma_ioctl_kms,
+        .dumb_create = radeon_mode_dumb_create,
+        .dumb_map_offset = radeon_mode_dumb_mmap,
+        .dumb_destroy = radeon_mode_dumb_destroy,
         .fops = {
                  .owner = THIS_MODULE,
                  .open = drm_open,
@@ -313,15 +344,6 @@ static struct drm_driver kms_driver = {
 #endif
         },
 
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-                 .probe = radeon_pci_probe,
-                 .remove = radeon_pci_remove,
-                 .suspend = radeon_pci_suspend,
-                 .resume = radeon_pci_resume,
-        },
-
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
         .date = DRIVER_DATE,
@@ -331,15 +353,32 @@ static struct drm_driver kms_driver = {
 };
 
 static struct drm_driver *driver;
+static struct pci_driver *pdriver;
+
+static struct pci_driver radeon_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
+static struct pci_driver radeon_kms_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+        .probe = radeon_pci_probe,
+        .remove = radeon_pci_remove,
+        .suspend = radeon_pci_suspend,
+        .resume = radeon_pci_resume,
+};
 
 static int __init radeon_init(void)
 {
         driver = &driver_old;
+        pdriver = &radeon_pci_driver;
         driver->num_ioctls = radeon_max_ioctl;
 #ifdef CONFIG_VGA_CONSOLE
         if (vgacon_text_force() && radeon_modeset == -1) {
                 DRM_INFO("VGACON disable radeon kernel modesetting.\n");
                 driver = &driver_old;
+                pdriver = &radeon_pci_driver;
                 driver->driver_features &= ~DRIVER_MODESET;
                 radeon_modeset = 0;
         }
@@ -357,18 +396,19 @@ static int __init radeon_init(void)
         if (radeon_modeset == 1) {
                 DRM_INFO("radeon kernel modesetting enabled.\n");
                 driver = &kms_driver;
+                pdriver = &radeon_kms_pci_driver;
                 driver->driver_features |= DRIVER_MODESET;
                 driver->num_ioctls = radeon_max_kms_ioctl;
                 radeon_register_atpx_handler();
         }
         /* if the vga console setting is enabled still
          * let modprobe override it */
-        return drm_init(driver);
+        return drm_pci_init(driver, pdriver);
 }
 
 static void __exit radeon_exit(void)
 {
-        drm_exit(driver);
+        drm_pci_exit(driver, pdriver);
         radeon_unregister_atpx_handler();
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_encoders.c b/drivers/gpu/drm/radeon/radeon_encoders.c
index e4e64a80b58d..d4a542247618 100644
--- a/drivers/gpu/drm/radeon/radeon_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_encoders.c
@@ -641,7 +641,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
         switch (connector->connector_type) {
         case DRM_MODE_CONNECTOR_DVII:
         case DRM_MODE_CONNECTOR_HDMIB: /* HDMI-B is basically DL-DVI; analog works fine */
-                if (drm_detect_hdmi_monitor(radeon_connector->edid)) {
+                if (drm_detect_monitor_audio(radeon_connector->edid) && radeon_audio) {
                         /* fix me */
                         if (ASIC_IS_DCE4(rdev))
                                 return ATOM_ENCODER_MODE_DVI;
@@ -655,7 +655,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
         case DRM_MODE_CONNECTOR_DVID:
         case DRM_MODE_CONNECTOR_HDMIA:
         default:
-                if (drm_detect_hdmi_monitor(radeon_connector->edid)) {
+                if (drm_detect_monitor_audio(radeon_connector->edid) && radeon_audio) {
                         /* fix me */
                         if (ASIC_IS_DCE4(rdev))
                                 return ATOM_ENCODER_MODE_DVI;
@@ -673,7 +673,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
                 if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
                     (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP))
                         return ATOM_ENCODER_MODE_DP;
-                else if (drm_detect_hdmi_monitor(radeon_connector->edid)) {
+                else if (drm_detect_monitor_audio(radeon_connector->edid) && radeon_audio) {
                         /* fix me */
                         if (ASIC_IS_DCE4(rdev))
                                 return ATOM_ENCODER_MODE_DVI;
@@ -712,7 +712,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
  * - 2 DIG encoder blocks.
  * DIG1/2 can drive UNIPHY0/1/2 link A or link B
  *
- * DCE 4.0
+ * DCE 4.0/5.0
  * - 3 DIG transmitter blocks UNIPHY0/1/2 (links A and B).
  * Supports up to 6 digital outputs
  * - 6 DIG encoder blocks.
@@ -743,6 +743,7 @@ union dig_encoder_control {
         DIG_ENCODER_CONTROL_PS_ALLOCATION v1;
         DIG_ENCODER_CONTROL_PARAMETERS_V2 v2;
         DIG_ENCODER_CONTROL_PARAMETERS_V3 v3;
+        DIG_ENCODER_CONTROL_PARAMETERS_V4 v4;
 };
 
 void
@@ -758,6 +759,7 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
         uint8_t frev, crev;
         int dp_clock = 0;
         int dp_lane_count = 0;
+        int hpd_id = RADEON_HPD_NONE;
 
         if (connector) {
                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -766,6 +768,7 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
 
                 dp_clock = dig_connector->dp_clock;
                 dp_lane_count = dig_connector->dp_lane_count;
+                hpd_id = radeon_connector->hpd.hpd;
         }
 
         /* no dig encoder assigned */
@@ -790,19 +793,36 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
         args.v1.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
         args.v1.ucEncoderMode = atombios_get_encoder_mode(encoder);
 
-        if (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) {
-                if (dp_clock == 270000)
-                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
+        if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) ||
+            (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP_MST))
                 args.v1.ucLaneNum = dp_lane_count;
-        } else if (radeon_encoder->pixel_clock > 165000)
+        else if (radeon_encoder->pixel_clock > 165000)
                 args.v1.ucLaneNum = 8;
         else
                 args.v1.ucLaneNum = 4;
 
-        if (ASIC_IS_DCE4(rdev)) {
+        if (ASIC_IS_DCE5(rdev)) {
+                if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) ||
+                    (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP_MST)) {
+                        if (dp_clock == 270000)
+                                args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ;
+                        else if (dp_clock == 540000)
+                                args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ;
+                }
+                args.v4.acConfig.ucDigSel = dig->dig_encoder;
+                args.v4.ucBitPerColor = PANEL_8BIT_PER_COLOR;
+                if (hpd_id == RADEON_HPD_NONE)
+                        args.v4.ucHPD_ID = 0;
+                else
+                        args.v4.ucHPD_ID = hpd_id + 1;
+        } else if (ASIC_IS_DCE4(rdev)) {
+                if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) && (dp_clock == 270000))
+                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
                 args.v3.acConfig.ucDigSel = dig->dig_encoder;
                 args.v3.ucBitPerColor = PANEL_8BIT_PER_COLOR;
         } else {
+                if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) && (dp_clock == 270000))
+                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
                 switch (radeon_encoder->encoder_id) {
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
                         args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1;
@@ -829,6 +849,7 @@ union dig_transmitter_control {
         DIG_TRANSMITTER_CONTROL_PS_ALLOCATION v1;
         DIG_TRANSMITTER_CONTROL_PARAMETERS_V2 v2;
         DIG_TRANSMITTER_CONTROL_PARAMETERS_V3 v3;
+        DIG_TRANSMITTER_CONTROL_PARAMETERS_V4 v4;
 };
 
 void
@@ -910,15 +931,9 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                 else
                         args.v3.ucLaneNum = 4;
 
-                if (ASIC_IS_DCE41(rdev)) {
-                        args.v3.acConfig.ucEncoderSel = dig->dig_encoder;
-                        if (dig->linkb)
-                                args.v3.acConfig.ucLinkSel = 1;
-                } else {
-                        if (dig->linkb) {
-                                args.v3.acConfig.ucLinkSel = 1;
-                                args.v3.acConfig.ucEncoderSel = 1;
-                        }
+                if (dig->linkb) {
+                        args.v3.acConfig.ucLinkSel = 1;
+                        args.v3.acConfig.ucEncoderSel = 1;
                 }
 
                 /* Select the PLL for the PHY
@@ -929,10 +944,18 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
                         pll_id = radeon_crtc->pll_id;
                 }
-                if (is_dp && rdev->clock.dp_extclk)
-                        args.v3.acConfig.ucRefClkSource = 2; /* external src */
-                else
-                        args.v3.acConfig.ucRefClkSource = pll_id;
+
+                if (ASIC_IS_DCE5(rdev)) {
+                        if (is_dp && rdev->clock.dp_extclk)
+                                args.v4.acConfig.ucRefClkSource = 3; /* external src */
+                        else
+                                args.v4.acConfig.ucRefClkSource = pll_id;
+                } else {
+                        if (is_dp && rdev->clock.dp_extclk)
+                                args.v3.acConfig.ucRefClkSource = 2; /* external src */
+                        else
+                                args.v3.acConfig.ucRefClkSource = pll_id;
+                }
 
                 switch (radeon_encoder->encoder_id) {
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
@@ -1040,7 +1063,7 @@ atombios_set_edp_panel_power(struct drm_connector *connector, int action)
         if (!ASIC_IS_DCE4(rdev))
                 return;
 
-        if ((action != ATOM_TRANSMITTER_ACTION_POWER_ON) ||
+        if ((action != ATOM_TRANSMITTER_ACTION_POWER_ON) &&
             (action != ATOM_TRANSMITTER_ACTION_POWER_OFF))
                 return;
 
@@ -1204,6 +1227,8 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
         DISPLAY_DEVICE_OUTPUT_CONTROL_PS_ALLOCATION args;
         int index = 0;
         bool is_dig = false;
+        bool is_dce5_dac = false;
+        bool is_dce5_dvo = false;
 
         memset(&args, 0, sizeof(args));
 
@@ -1226,7 +1251,9 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
-                if (ASIC_IS_DCE3(rdev))
+                if (ASIC_IS_DCE5(rdev))
+                        is_dce5_dvo = true;
+                else if (ASIC_IS_DCE3(rdev))
                         is_dig = true;
                 else
                         index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
@@ -1242,12 +1269,16 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DAC1:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
-                if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
-                        index = GetIndexIntoMasterTable(COMMAND, TV1OutputControl);
-                else if (radeon_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT))
-                        index = GetIndexIntoMasterTable(COMMAND, CV1OutputControl);
-                else
-                        index = GetIndexIntoMasterTable(COMMAND, DAC1OutputControl);
+                if (ASIC_IS_DCE5(rdev))
+                        is_dce5_dac = true;
+                else {
+                        if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
+                                index = GetIndexIntoMasterTable(COMMAND, TV1OutputControl);
+                        else if (radeon_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT))
+                                index = GetIndexIntoMasterTable(COMMAND, CV1OutputControl);
+                        else
+                                index = GetIndexIntoMasterTable(COMMAND, DAC1OutputControl);
+                }
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_DAC2:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
@@ -1306,6 +1337,28 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                                 atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
                         break;
                 }
+        } else if (is_dce5_dac) {
+                switch (mode) {
+                case DRM_MODE_DPMS_ON:
+                        atombios_dac_setup(encoder, ATOM_ENABLE);
+                        break;
+                case DRM_MODE_DPMS_STANDBY:
+                case DRM_MODE_DPMS_SUSPEND:
+                case DRM_MODE_DPMS_OFF:
+                        atombios_dac_setup(encoder, ATOM_DISABLE);
+                        break;
+                }
+        } else if (is_dce5_dvo) {
+                switch (mode) {
+                case DRM_MODE_DPMS_ON:
+                        atombios_dvo_setup(encoder, ATOM_ENABLE);
+                        break;
+                case DRM_MODE_DPMS_STANDBY:
+                case DRM_MODE_DPMS_SUSPEND:
+                case DRM_MODE_DPMS_OFF:
+                        atombios_dvo_setup(encoder, ATOM_DISABLE);
+                        break;
+                }
         } else {
                 switch (mode) {
                 case DRM_MODE_DPMS_ON:
@@ -1335,7 +1388,7 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 switch (mode) {
                 case DRM_MODE_DPMS_ON:
                 default:
-                        if (ASIC_IS_DCE41(rdev) && (rdev->flags & RADEON_IS_IGP))
+                        if (ASIC_IS_DCE41(rdev))
                                 action = EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT;
                         else
                                 action = ATOM_ENABLE;
@@ -1343,7 +1396,7 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 case DRM_MODE_DPMS_STANDBY:
                 case DRM_MODE_DPMS_SUSPEND:
                 case DRM_MODE_DPMS_OFF:
-                        if (ASIC_IS_DCE41(rdev) && (rdev->flags & RADEON_IS_IGP))
+                        if (ASIC_IS_DCE41(rdev))
                                 action = EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT;
                         else
                                 action = ATOM_DISABLE;
@@ -1535,32 +1588,35 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder)
         struct radeon_encoder_atom_dig *dig;
         uint32_t dig_enc_in_use = 0;
 
-        /* on DCE41 and encoder can driver any phy so just crtc id */
-        if (ASIC_IS_DCE41(rdev)) {
-                return radeon_crtc->crtc_id;
-        }
-
+        /* DCE4/5 */
         if (ASIC_IS_DCE4(rdev)) {
                 dig = radeon_encoder->enc_priv;
-                switch (radeon_encoder->encoder_id) {
-                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+                if (ASIC_IS_DCE41(rdev)) {
                         if (dig->linkb)
                                 return 1;
                         else
                                 return 0;
-                        break;
-                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
-                        if (dig->linkb)
-                                return 3;
-                        else
-                                return 2;
-                        break;
-                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
-                        if (dig->linkb)
-                                return 5;
-                        else
-                                return 4;
-                        break;
+                } else {
+                        switch (radeon_encoder->encoder_id) {
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+                                if (dig->linkb)
+                                        return 1;
+                                else
+                                        return 0;
+                                break;
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
+                                if (dig->linkb)
+                                        return 3;
+                                else
+                                        return 2;
+                                break;
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                                if (dig->linkb)
+                                        return 5;
+                                else
+                                        return 4;
+                                break;
+                        }
                 }
         }
 
@@ -1667,7 +1723,7 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
         }
 
         if (ext_encoder) {
-                if (ASIC_IS_DCE41(rdev) && (rdev->flags & RADEON_IS_IGP)) {
+                if (ASIC_IS_DCE41(rdev)) {
                         atombios_external_encoder_setup(encoder, ext_encoder,
                                                         EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT);
                         atombios_external_encoder_setup(encoder, ext_encoder,
@@ -1990,7 +2046,10 @@ radeon_atombios_set_dig_info(struct radeon_encoder *radeon_encoder)
 }
 
 void
-radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t supported_device)
+radeon_add_atom_encoder(struct drm_device *dev,
+                        uint32_t encoder_enum,
+                        uint32_t supported_device,
+                        u16 caps)
 {
         struct radeon_device *rdev = dev->dev_private;
         struct drm_encoder *encoder;
@@ -2033,6 +2092,7 @@ radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t
         radeon_encoder->rmx_type = RMX_OFF;
         radeon_encoder->underscan_type = UNDERSCAN_OFF;
         radeon_encoder->is_ext_encoder = false;
+        radeon_encoder->caps = caps;
 
         switch (radeon_encoder->encoder_id) {
         case ENCODER_OBJECT_ID_INTERNAL_LVDS:
diff --git a/drivers/gpu/drm/radeon/radeon_family.h b/drivers/gpu/drm/radeon/radeon_family.h
index 4c222d5437d1..1ca55eb09ad3 100644
--- a/drivers/gpu/drm/radeon/radeon_family.h
+++ b/drivers/gpu/drm/radeon/radeon_family.h
@@ -81,6 +81,9 @@ enum radeon_family {
         CHIP_CYPRESS,
         CHIP_HEMLOCK,
         CHIP_PALM,
+        CHIP_BARTS,
+        CHIP_TURKS,
+        CHIP_CAICOS,
         CHIP_LAST,
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index efa211898fe6..28431e78ab56 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -64,7 +64,7 @@ static struct fb_ops radeonfb_ops = {
 };
 
 
-static int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled)
+int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled)
 {
         int aligned = width;
         int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
@@ -90,7 +90,7 @@ static int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bo
 
 static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj)
 {
-        struct radeon_bo *rbo = gobj->driver_private;
+        struct radeon_bo *rbo = gem_to_radeon_bo(gobj);
         int ret;
 
         ret = radeon_bo_reserve(rbo, false);
@@ -128,7 +128,7 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
                        aligned_size);
                 return -ENOMEM;
         }
-        rbo = gobj->driver_private;
+        rbo = gem_to_radeon_bo(gobj);
 
         if (fb_tiled)
                 tiling_flags = RADEON_TILING_MACRO;
@@ -202,7 +202,7 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev,
         mode_cmd.depth = sizes->surface_depth;
 
         ret = radeonfb_create_pinned_object(rfbdev, &mode_cmd, &gobj);
-        rbo = gobj->driver_private;
+        rbo = gem_to_radeon_bo(gobj);
 
         /* okay we have an object now allocate the framebuffer */
         info = framebuffer_alloc(0, device);
@@ -245,10 +245,8 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev,
                 goto out_unref;
         }
         info->apertures->ranges[0].base = rdev->ddev->mode_config.fb_base;
-        info->apertures->ranges[0].size = rdev->mc.real_vram_size;
+        info->apertures->ranges[0].size = rdev->mc.aper_size;
 
-        info->fix.mmio_start = 0;
-        info->fix.mmio_len = 0;
         info->pixmap.size = 64*1024;
         info->pixmap.buf_align = 8;
         info->pixmap.access_align = 32;
@@ -405,14 +403,14 @@ int radeon_fbdev_total_size(struct radeon_device *rdev)
         struct radeon_bo *robj;
         int size = 0;
 
-        robj = rdev->mode_info.rfbdev->rfb.obj->driver_private;
+        robj = gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj);
         size += radeon_bo_size(robj);
         return size;
 }
 
 bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
 {
-        if (robj == rdev->mode_info.rfbdev->rfb.obj->driver_private)
+        if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj))
                 return true;
         return false;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
index 65016117d95f..f0534ef2f331 100644
--- a/drivers/gpu/drm/radeon/radeon_gart.c
+++ b/drivers/gpu/drm/radeon/radeon_gart.c
@@ -78,7 +78,7 @@ int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
         int r;
 
         if (rdev->gart.table.vram.robj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->gart.table_size,
+                r = radeon_bo_create(rdev, rdev->gart.table_size,
                                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                      &rdev->gart.table.vram.robj);
                 if (r) {
@@ -149,8 +149,9 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
         p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
         for (i = 0; i < pages; i++, p++) {
                 if (rdev->gart.pages[p]) {
-                        pci_unmap_page(rdev->pdev, rdev->gart.pages_addr[p],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                        if (!rdev->gart.ttm_alloced[p])
+                                pci_unmap_page(rdev->pdev, rdev->gart.pages_addr[p],
+                                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
                         rdev->gart.pages[p] = NULL;
                         rdev->gart.pages_addr[p] = rdev->dummy_page.addr;
                         page_base = rdev->gart.pages_addr[p];
@@ -165,7 +166,7 @@ void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
 }
 
 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
-                     int pages, struct page **pagelist)
+                     int pages, struct page **pagelist, dma_addr_t *dma_addr)
 {
         unsigned t;
         unsigned p;
@@ -180,15 +181,22 @@ int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
         p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
 
         for (i = 0; i < pages; i++, p++) {
-                /* we need to support large memory configurations */
-                /* assume that unbind have already been call on the range */
-                rdev->gart.pages_addr[p] = pci_map_page(rdev->pdev, pagelist[i],
+                /* On TTM path, we only use the DMA API if TTM_PAGE_FLAG_DMA32
+                 * is requested. */
+                if (dma_addr[i] != DMA_ERROR_CODE) {
+                        rdev->gart.ttm_alloced[p] = true;
+                        rdev->gart.pages_addr[p] = dma_addr[i];
+                } else {
+                        /* we need to support large memory configurations */
+                        /* assume that unbind have already been call on the range */
+                        rdev->gart.pages_addr[p] = pci_map_page(rdev->pdev, pagelist[i],
                                                         0, PAGE_SIZE,
                                                         PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(rdev->pdev, rdev->gart.pages_addr[p])) {
-                        /* FIXME: failed to map page (return -ENOMEM?) */
-                        radeon_gart_unbind(rdev, offset, pages);
-                        return -ENOMEM;
+                        if (pci_dma_mapping_error(rdev->pdev, rdev->gart.pages_addr[p])) {
+                                /* FIXME: failed to map page (return -ENOMEM?) */
+                                radeon_gart_unbind(rdev, offset, pages);
+                                return -ENOMEM;
+                        }
                 }
                 rdev->gart.pages[p] = pagelist[i];
                 page_base = rdev->gart.pages_addr[p];
@@ -251,6 +259,12 @@ int radeon_gart_init(struct radeon_device *rdev)
                 radeon_gart_fini(rdev);
                 return -ENOMEM;
         }
+        rdev->gart.ttm_alloced = kzalloc(sizeof(bool) *
+                                         rdev->gart.num_cpu_pages, GFP_KERNEL);
+        if (rdev->gart.ttm_alloced == NULL) {
+                radeon_gart_fini(rdev);
+                return -ENOMEM;
+        }
         /* set GART entry to point to the dummy page by default */
         for (i = 0; i < rdev->gart.num_cpu_pages; i++) {
                 rdev->gart.pages_addr[i] = rdev->dummy_page.addr;
@@ -267,6 +281,8 @@ void radeon_gart_fini(struct radeon_device *rdev)
         rdev->gart.ready = false;
         kfree(rdev->gart.pages);
         kfree(rdev->gart.pages_addr);
+        kfree(rdev->gart.ttm_alloced);
         rdev->gart.pages = NULL;
         rdev->gart.pages_addr = NULL;
+        rdev->gart.ttm_alloced = NULL;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index df95eb83dac6..a419b67d8401 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -32,21 +32,18 @@
 
 int radeon_gem_object_init(struct drm_gem_object *obj)
 {
-        /* we do nothings here */
+        BUG();
+
         return 0;
 }
 
 void radeon_gem_object_free(struct drm_gem_object *gobj)
 {
-        struct radeon_bo *robj = gobj->driver_private;
+        struct radeon_bo *robj = gem_to_radeon_bo(gobj);
 
-        gobj->driver_private = NULL;
         if (robj) {
                 radeon_bo_unref(&robj);
         }
-
-        drm_gem_object_release(gobj);
-        kfree(gobj);
 }
 
 int radeon_gem_object_create(struct radeon_device *rdev, int size,
@@ -54,36 +51,34 @@ int radeon_gem_object_create(struct radeon_device *rdev, int size,
                                 bool discardable, bool kernel,
                                 struct drm_gem_object **obj)
 {
-        struct drm_gem_object *gobj;
         struct radeon_bo *robj;
         int r;
 
         *obj = NULL;
-        gobj = drm_gem_object_alloc(rdev->ddev, size);
-        if (!gobj) {
-                return -ENOMEM;
-        }
         /* At least align on page size */
         if (alignment < PAGE_SIZE) {
                 alignment = PAGE_SIZE;
         }
-        r = radeon_bo_create(rdev, gobj, size, alignment, kernel, initial_domain, &robj);
+        r = radeon_bo_create(rdev, size, alignment, kernel, initial_domain, &robj);
         if (r) {
                 if (r != -ERESTARTSYS)
                         DRM_ERROR("Failed to allocate GEM object (%d, %d, %u, %d)\n",
                                   size, initial_domain, alignment, r);
-                drm_gem_object_unreference_unlocked(gobj);
                 return r;
         }
-        gobj->driver_private = robj;
-        *obj = gobj;
+        *obj = &robj->gem_base;
+
+        mutex_lock(&rdev->gem.mutex);
+        list_add_tail(&robj->list, &rdev->gem.objects);
+        mutex_unlock(&rdev->gem.mutex);
+
         return 0;
 }
 
 int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
                           uint64_t *gpu_addr)
 {
-        struct radeon_bo *robj = obj->driver_private;
+        struct radeon_bo *robj = gem_to_radeon_bo(obj);
         int r;
 
         r = radeon_bo_reserve(robj, false);
@@ -96,7 +91,7 @@ int radeon_gem_object_pin(struct drm_gem_object *obj, uint32_t pin_domain,
 
 void radeon_gem_object_unpin(struct drm_gem_object *obj)
 {
-        struct radeon_bo *robj = obj->driver_private;
+        struct radeon_bo *robj = gem_to_radeon_bo(obj);
         int r;
 
         r = radeon_bo_reserve(robj, false);
@@ -114,7 +109,7 @@ int radeon_gem_set_domain(struct drm_gem_object *gobj,
         int r;
 
         /* FIXME: reeimplement */
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         /* work out where to validate the buffer to */
         domain = wdomain;
         if (!domain) {
@@ -228,7 +223,7 @@ int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
 
         r = radeon_gem_set_domain(gobj, args->read_domains, args->write_domain);
 
@@ -236,23 +231,31 @@ int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         return r;
 }
 
-int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
-                          struct drm_file *filp)
+int radeon_mode_dumb_mmap(struct drm_file *filp,
+                          struct drm_device *dev,
+                          uint32_t handle, uint64_t *offset_p)
 {
-        struct drm_radeon_gem_mmap *args = data;
         struct drm_gem_object *gobj;
         struct radeon_bo *robj;
 
-        gobj = drm_gem_object_lookup(dev, filp, args->handle);
+        gobj = drm_gem_object_lookup(dev, filp, handle);
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
-        args->addr_ptr = radeon_bo_mmap_offset(robj);
+        robj = gem_to_radeon_bo(gobj);
+        *offset_p = radeon_bo_mmap_offset(robj);
         drm_gem_object_unreference_unlocked(gobj);
         return 0;
 }
 
+int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
+                          struct drm_file *filp)
+{
+        struct drm_radeon_gem_mmap *args = data;
+
+        return radeon_mode_dumb_mmap(filp, dev, args->handle, &args->addr_ptr);
+}
+
 int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
                           struct drm_file *filp)
 {
@@ -266,7 +269,7 @@ int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_wait(robj, &cur_placement, true);
         switch (cur_placement) {
         case TTM_PL_VRAM:
@@ -296,7 +299,7 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
         if (gobj == NULL) {
                 return -ENOENT;
         }
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_wait(robj, NULL, false);
         /* callback hw specific functions if any */
         if (robj->rdev->asic->ioctl_wait_idle)
@@ -317,7 +320,7 @@ int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
         gobj = drm_gem_object_lookup(dev, filp, args->handle);
         if (gobj == NULL)
                 return -ENOENT;
-        robj = gobj->driver_private;
+        robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_set_tiling_flags(robj, args->tiling_flags, args->pitch);
         drm_gem_object_unreference_unlocked(gobj);
         return r;
@@ -335,7 +338,7 @@ int radeon_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
         gobj = drm_gem_object_lookup(dev, filp, args->handle);
         if (gobj == NULL)
                 return -ENOENT;
-        rbo = gobj->driver_private;
+        rbo = gem_to_radeon_bo(gobj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 goto out;
@@ -345,3 +348,38 @@ out:
         drm_gem_object_unreference_unlocked(gobj);
         return r;
 }
+
+int radeon_mode_dumb_create(struct drm_file *file_priv,
+                            struct drm_device *dev,
+                            struct drm_mode_create_dumb *args)
+{
+        struct radeon_device *rdev = dev->dev_private;
+        struct drm_gem_object *gobj;
+        int r;
+
+        args->pitch = radeon_align_pitch(rdev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8);
+        args->size = args->pitch * args->height;
+        args->size = ALIGN(args->size, PAGE_SIZE);
+
+        r = radeon_gem_object_create(rdev, args->size, 0,
+                                     RADEON_GEM_DOMAIN_VRAM,
+                                     false, ttm_bo_type_device,
+                                     &gobj);
+        if (r)
+                return -ENOMEM;
+
+        r = drm_gem_handle_create(file_priv, gobj, &args->handle);
+        if (r) {
+                drm_gem_object_unreference_unlocked(gobj);
+                return r;
+        }
+        drm_gem_object_handle_unreference_unlocked(gobj);
+        return 0;
+}
+
+int radeon_mode_dumb_destroy(struct drm_file *file_priv,
+                             struct drm_device *dev,
+                             uint32_t handle)
+{
+        return drm_gem_handle_delete(file_priv, handle);
+}
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index c6861bb751ad..9ec830c77af0 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -64,8 +64,6 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
         struct radeon_device *rdev = dev->dev_private;
         unsigned i;
 
-        INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
-
         /* Disable *all* interrupts */
         rdev->irq.sw_int = false;
         rdev->irq.gui_idle = false;
@@ -112,9 +110,14 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
 
 int radeon_irq_kms_init(struct radeon_device *rdev)
 {
+        int i;
         int r = 0;
 
+        INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
+
         spin_lock_init(&rdev->irq.sw_lock);
+        for (i = 0; i < rdev->num_crtc; i++)
+                spin_lock_init(&rdev->irq.pflip_lock[i]);
         r = drm_vblank_init(rdev->ddev, rdev->num_crtc);
         if (r) {
                 return r;
@@ -152,6 +155,7 @@ void radeon_irq_kms_fini(struct radeon_device *rdev)
                 if (rdev->msi_enabled)
                         pci_disable_msi(rdev->pdev);
         }
+        flush_work_sync(&rdev->hotplug_work);
 }
 
 void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 4bf423ca4c12..4057ebf5195d 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -58,9 +58,9 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
         dev->dev_private = (void *)rdev;
 
         /* update BUS flag */
-        if (drm_device_is_agp(dev)) {
+        if (drm_pci_device_is_agp(dev)) {
                 flags |= RADEON_IS_AGP;
-        } else if (drm_device_is_pcie(dev)) {
+        } else if (drm_pci_device_is_pcie(dev)) {
                 flags |= RADEON_IS_PCIE;
         } else {
                 flags |= RADEON_IS_PCI;
@@ -96,9 +96,27 @@ out:
         return r;
 }
 
+static void radeon_set_filp_rights(struct drm_device *dev,
+                                   struct drm_file **owner,
+                                   struct drm_file *applier,
+                                   uint32_t *value)
+{
+        mutex_lock(&dev->struct_mutex);
+        if (*value == 1) {
+                /* wants rights */
+                if (!*owner)
+                        *owner = applier;
+        } else if (*value == 0) {
+                /* revokes rights */
+                if (*owner == applier)
+                        *owner = NULL;
+        }
+        *value = *owner == applier ? 1 : 0;
+        mutex_unlock(&dev->struct_mutex);
+}
 
 /*
- * Userspace get informations ioctl
+ * Userspace get information ioctl
  */
 int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
@@ -173,18 +191,19 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                         DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", value);
                         return -EINVAL;
                 }
-                mutex_lock(&dev->struct_mutex);
-                if (value == 1) {
-                        /* wants hyper-z */
-                        if (!rdev->hyperz_filp)
-                                rdev->hyperz_filp = filp;
-                } else if (value == 0) {
-                        /* revokes hyper-z */
-                        if (rdev->hyperz_filp == filp)
-                                rdev->hyperz_filp = NULL;
+                radeon_set_filp_rights(dev, &rdev->hyperz_filp, filp, &value);
+                break;
+        case RADEON_INFO_WANT_CMASK:
+                /* The same logic as Hyper-Z. */
+                if (value >= 2) {
+                        DRM_DEBUG_KMS("WANT_CMASK: invalid value %d\n", value);
+                        return -EINVAL;
                 }
-                value = rdev->hyperz_filp == filp ?  1 : 0;
-                mutex_unlock(&dev->struct_mutex);
+                radeon_set_filp_rights(dev, &rdev->cmask_filp, filp, &value);
+                break;
+        case RADEON_INFO_CLOCK_CRYSTAL_FREQ:
+                /* return clock value in KHz */
+                value = rdev->clock.spll.reference_freq * 10;
                 break;
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->request);
@@ -203,10 +222,6 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  */
 int radeon_driver_firstopen_kms(struct drm_device *dev)
 {
-        struct radeon_device *rdev = dev->dev_private;
-
-        if (rdev->powered_down)
-                return -EINVAL;
         return 0;
 }
 
@@ -232,6 +247,8 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
         struct radeon_device *rdev = dev->dev_private;
         if (rdev->hyperz_filp == file_priv)
                 rdev->hyperz_filp = NULL;
+        if (rdev->cmask_filp == file_priv)
+                rdev->cmask_filp = NULL;
 }
 
 /*
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index ace2e6384d40..9ae599eb2e6d 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -415,7 +415,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 
         /* Pin framebuffer & get tilling informations */
         obj = radeon_fb->obj;
-        rbo = obj->driver_private;
+        rbo = gem_to_radeon_bo(obj);
         r = radeon_bo_reserve(rbo, false);
         if (unlikely(r != 0))
                 return r;
@@ -520,7 +520,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 
         if (!atomic && fb && fb != crtc->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
-                rbo = radeon_fb->obj->driver_private;
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r != 0))
                         return r;
@@ -778,9 +778,9 @@ static void radeon_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
         DRM_DEBUG_KMS("\n");
 
         if (!use_bios_divs) {
-                radeon_compute_pll(pll, mode->clock,
-                                   &freq, &feedback_div, &frac_fb_div,
-                                   &reference_div, &post_divider);
+                radeon_compute_pll_legacy(pll, mode->clock,
+                                          &freq, &feedback_div, &frac_fb_div,
+                                          &reference_div, &post_divider);
 
                 for (post_div = &post_divs[0]; post_div->divider; ++post_div) {
                         if (post_div->divider == post_divider)
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index f406f02bf14e..c3f23f6ff60e 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -149,6 +149,7 @@ struct radeon_tmds_pll {
 #define RADEON_PLL_PREFER_CLOSEST_LOWER (1 << 11)
 #define RADEON_PLL_USE_POST_DIV         (1 << 12)
 #define RADEON_PLL_IS_LCD               (1 << 13)
+#define RADEON_PLL_PREFER_MINM_OVER_MAXP (1 << 14)
 
 struct radeon_pll {
         /* reference frequency */
@@ -379,6 +380,7 @@ struct radeon_encoder {
         int hdmi_audio_workaround;
         int hdmi_buffer_status;
         bool is_ext_encoder;
+        u16 caps;
 };
 
 struct radeon_connector_atom_dig {
@@ -509,13 +511,21 @@ extern bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev,
                                              struct radeon_atom_ss *ss,
                                              int id, u32 clock);
 
-extern void radeon_compute_pll(struct radeon_pll *pll,
-                               uint64_t freq,
-                               uint32_t *dot_clock_p,
-                               uint32_t *fb_div_p,
-                               uint32_t *frac_fb_div_p,
-                               uint32_t *ref_div_p,
-                               uint32_t *post_div_p);
+extern void radeon_compute_pll_legacy(struct radeon_pll *pll,
+                                      uint64_t freq,
+                                      uint32_t *dot_clock_p,
+                                      uint32_t *fb_div_p,
+                                      uint32_t *frac_fb_div_p,
+                                      uint32_t *ref_div_p,
+                                      uint32_t *post_div_p);
+
+extern void radeon_compute_pll_avivo(struct radeon_pll *pll,
+                                     u32 freq,
+                                     u32 *dot_clock_p,
+                                     u32 *fb_div_p,
+                                     u32 *frac_fb_div_p,
+                                     u32 *ref_div_p,
+                                     u32 *post_div_p);
 
 extern void radeon_setup_encoder_clones(struct drm_device *dev);
 
@@ -566,7 +576,7 @@ extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
 
 extern bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev);
 extern struct edid *
-radeon_combios_get_hardcoded_edid(struct radeon_device *rdev);
+radeon_bios_get_hardcoded_edid(struct radeon_device *rdev);
 extern bool radeon_atom_get_clock_info(struct drm_device *dev);
 extern bool radeon_combios_get_clock_info(struct drm_device *dev);
 extern struct radeon_encoder_atom_dig *
@@ -665,4 +675,5 @@ void radeon_fb_output_poll_changed(struct radeon_device *rdev);
 
 void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id);
 
+int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled);
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 8bdf0ba2983a..8758d02cca1a 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -55,6 +55,7 @@ static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
         list_del_init(&bo->list);
         mutex_unlock(&bo->rdev->gem.mutex);
         radeon_bo_clear_surface_reg(bo);
+        drm_gem_object_release(&bo->gem_base);
         kfree(bo);
 }
 
@@ -70,7 +71,7 @@ void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
         u32 c = 0;
 
         rbo->placement.fpfn = 0;
-        rbo->placement.lpfn = rbo->rdev->mc.active_vram_size >> PAGE_SHIFT;
+        rbo->placement.lpfn = 0;
         rbo->placement.placement = rbo->placements;
         rbo->placement.busy_placement = rbo->placements;
         if (domain & RADEON_GEM_DOMAIN_VRAM)
@@ -86,15 +87,18 @@ void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
         rbo->placement.num_busy_placement = c;
 }
 
-int radeon_bo_create(struct radeon_device *rdev, struct drm_gem_object *gobj,
+int radeon_bo_create(struct radeon_device *rdev,
                      unsigned long size, int byte_align, bool kernel, u32 domain,
                      struct radeon_bo **bo_ptr)
 {
         struct radeon_bo *bo;
         enum ttm_bo_type type;
-        int page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
+        unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
+        unsigned long max_size = 0;
         int r;
 
+        size = ALIGN(size, PAGE_SIZE);
+
         if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
                 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
         }
@@ -105,12 +109,25 @@ int radeon_bo_create(struct radeon_device *rdev, struct drm_gem_object *gobj,
         }
         *bo_ptr = NULL;
 
+        /* maximun bo size is the minimun btw visible vram and gtt size */
+        max_size = min(rdev->mc.visible_vram_size, rdev->mc.gtt_size);
+        if ((page_align << PAGE_SHIFT) >= max_size) {
+                printk(KERN_WARNING "%s:%d alloc size %ldM bigger than %ldMb limit\n",
+                        __func__, __LINE__, page_align  >> (20 - PAGE_SHIFT), max_size >> 20);
+                return -ENOMEM;
+        }
+
 retry:
         bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);
         if (bo == NULL)
                 return -ENOMEM;
+        r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size);
+        if (unlikely(r)) {
+                kfree(bo);
+                return r;
+        }
         bo->rdev = rdev;
-        bo->gobj = gobj;
+        bo->gem_base.driver_private = NULL;
         bo->surface_reg = -1;
         INIT_LIST_HEAD(&bo->list);
         radeon_ttm_placement_from_domain(bo, domain);
@@ -133,12 +150,9 @@ retry:
                 return r;
         }
         *bo_ptr = bo;
-        if (gobj) {
-                mutex_lock(&bo->rdev->gem.mutex);
-                list_add_tail(&bo->list, &rdev->gem.objects);
-                mutex_unlock(&bo->rdev->gem.mutex);
-        }
+
         trace_radeon_bo_create(bo);
+
         return 0;
 }
 
@@ -251,7 +265,6 @@ int radeon_bo_evict_vram(struct radeon_device *rdev)
 void radeon_bo_force_delete(struct radeon_device *rdev)
 {
         struct radeon_bo *bo, *n;
-        struct drm_gem_object *gobj;
 
         if (list_empty(&rdev->gem.objects)) {
                 return;
@@ -259,16 +272,14 @@ void radeon_bo_force_delete(struct radeon_device *rdev)
         dev_err(rdev->dev, "Userspace still has active objects !\n");
         list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {
                 mutex_lock(&rdev->ddev->struct_mutex);
-                gobj = bo->gobj;
                 dev_err(rdev->dev, "%p %p %lu %lu force free\n",
-                        gobj, bo, (unsigned long)gobj->size,
-                        *((unsigned long *)&gobj->refcount));
+                        &bo->gem_base, bo, (unsigned long)bo->gem_base.size,
+                        *((unsigned long *)&bo->gem_base.refcount));
                 mutex_lock(&bo->rdev->gem.mutex);
                 list_del_init(&bo->list);
                 mutex_unlock(&bo->rdev->gem.mutex);
                 radeon_bo_unref(&bo);
-                gobj->driver_private = NULL;
-                drm_gem_object_unreference(gobj);
+                drm_gem_object_unreference(&bo->gem_base);
                 mutex_unlock(&rdev->ddev->struct_mutex);
         }
 }
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index 22d4c237dea5..7f8e778dba46 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -137,10 +137,9 @@ static inline int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type,
 }
 
 extern int radeon_bo_create(struct radeon_device *rdev,
-                            struct drm_gem_object *gobj, unsigned long size,
-                            int byte_align,
-                            bool kernel, u32 domain,
-                            struct radeon_bo **bo_ptr);
+                                unsigned long size, int byte_align,
+                                bool kernel, u32 domain,
+                                struct radeon_bo **bo_ptr);
 extern int radeon_bo_kmap(struct radeon_bo *bo, void **ptr);
 extern void radeon_bo_kunmap(struct radeon_bo *bo);
 extern void radeon_bo_unref(struct radeon_bo **bo);
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 4de7776bd1c5..2aed03bde4b2 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -167,13 +167,13 @@ static void radeon_set_power_state(struct radeon_device *rdev)
         if (radeon_gui_idle(rdev)) {
                 sclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
                         clock_info[rdev->pm.requested_clock_mode_index].sclk;
-                if (sclk > rdev->clock.default_sclk)
-                        sclk = rdev->clock.default_sclk;
+                if (sclk > rdev->pm.default_sclk)
+                        sclk = rdev->pm.default_sclk;
 
                 mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
                         clock_info[rdev->pm.requested_clock_mode_index].mclk;
-                if (mclk > rdev->clock.default_mclk)
-                        mclk = rdev->clock.default_mclk;
+                if (mclk > rdev->pm.default_mclk)
+                        mclk = rdev->pm.default_mclk;
 
                 /* upvolt before raising clocks, downvolt after lowering clocks */
                 if (sclk < rdev->pm.current_sclk)
@@ -405,20 +405,13 @@ static ssize_t radeon_set_pm_method(struct device *dev,
                 rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT;
                 mutex_unlock(&rdev->pm.mutex);
         } else if (strncmp("profile", buf, strlen("profile")) == 0) {
-                bool flush_wq = false;
-
                 mutex_lock(&rdev->pm.mutex);
-                if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
-                        cancel_delayed_work(&rdev->pm.dynpm_idle_work);
-                        flush_wq = true;
-                }
                 /* disable dynpm */
                 rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
                 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
                 rdev->pm.pm_method = PM_METHOD_PROFILE;
                 mutex_unlock(&rdev->pm.mutex);
-                if (flush_wq)
-                        flush_workqueue(rdev->wq);
+                cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
         } else {
                 DRM_ERROR("invalid power method!\n");
                 goto fail;
@@ -437,7 +430,7 @@ static ssize_t radeon_hwmon_show_temp(struct device *dev,
 {
         struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev));
         struct radeon_device *rdev = ddev->dev_private;
-        u32 temp;
+        int temp;
 
         switch (rdev->pm.int_thermal_type) {
         case THERMAL_TYPE_RV6XX:
@@ -447,6 +440,7 @@ static ssize_t radeon_hwmon_show_temp(struct device *dev,
                 temp = rv770_get_temp(rdev);
                 break;
         case THERMAL_TYPE_EVERGREEN:
+        case THERMAL_TYPE_NI:
                 temp = evergreen_get_temp(rdev);
                 break;
         case THERMAL_TYPE_SUMO:
@@ -524,34 +518,39 @@ static void radeon_hwmon_fini(struct radeon_device *rdev)
 
 void radeon_pm_suspend(struct radeon_device *rdev)
 {
-        bool flush_wq = false;
-
         mutex_lock(&rdev->pm.mutex);
         if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
-                cancel_delayed_work(&rdev->pm.dynpm_idle_work);
                 if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE)
                         rdev->pm.dynpm_state = DYNPM_STATE_SUSPENDED;
-                flush_wq = true;
         }
         mutex_unlock(&rdev->pm.mutex);
-        if (flush_wq)
-                flush_workqueue(rdev->wq);
+
+        cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
 }
 
 void radeon_pm_resume(struct radeon_device *rdev)
 {
+        /* set up the default clocks if the MC ucode is loaded */
+        if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
+                if (rdev->pm.default_vddc)
+                        radeon_atom_set_voltage(rdev, rdev->pm.default_vddc);
+                if (rdev->pm.default_sclk)
+                        radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
+                if (rdev->pm.default_mclk)
+                        radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
+        }
         /* asic init will reset the default power state */
         mutex_lock(&rdev->pm.mutex);
         rdev->pm.current_power_state_index = rdev->pm.default_power_state_index;
         rdev->pm.current_clock_mode_index = 0;
-        rdev->pm.current_sclk = rdev->clock.default_sclk;
-        rdev->pm.current_mclk = rdev->clock.default_mclk;
+        rdev->pm.current_sclk = rdev->pm.default_sclk;
+        rdev->pm.current_mclk = rdev->pm.default_mclk;
         rdev->pm.current_vddc = rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.voltage;
         if (rdev->pm.pm_method == PM_METHOD_DYNPM
             && rdev->pm.dynpm_state == DYNPM_STATE_SUSPENDED) {
                 rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE;
-                queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                        msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                      msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
         }
         mutex_unlock(&rdev->pm.mutex);
         radeon_pm_compute_clocks(rdev);
@@ -568,6 +567,8 @@ int radeon_pm_init(struct radeon_device *rdev)
         rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
         rdev->pm.dynpm_can_upclock = true;
         rdev->pm.dynpm_can_downclock = true;
+        rdev->pm.default_sclk = rdev->clock.default_sclk;
+        rdev->pm.default_mclk = rdev->clock.default_mclk;
         rdev->pm.current_sclk = rdev->clock.default_sclk;
         rdev->pm.current_mclk = rdev->clock.default_mclk;
         rdev->pm.int_thermal_type = THERMAL_TYPE_NONE;
@@ -579,12 +580,24 @@ int radeon_pm_init(struct radeon_device *rdev)
                         radeon_combios_get_power_modes(rdev);
                 radeon_pm_print_states(rdev);
                 radeon_pm_init_profile(rdev);
+                /* set up the default clocks if the MC ucode is loaded */
+                if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
+                        if (rdev->pm.default_vddc)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddc);
+                        if (rdev->pm.default_sclk)
+                                radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
+                        if (rdev->pm.default_mclk)
+                                radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
+                }
         }
 
         /* set up the internal thermal sensor if applicable */
         ret = radeon_hwmon_init(rdev);
         if (ret)
                 return ret;
+
+        INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
+
         if (rdev->pm.num_power_states > 1) {
                 /* where's the best place to put these? */
                 ret = device_create_file(rdev->dev, &dev_attr_power_profile);
@@ -598,8 +611,6 @@ int radeon_pm_init(struct radeon_device *rdev)
                 rdev->acpi_nb.notifier_call = radeon_acpi_event;
                 register_acpi_notifier(&rdev->acpi_nb);
 #endif
-                INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
-
                 if (radeon_debugfs_pm_init(rdev)) {
                         DRM_ERROR("Failed to register debugfs file for PM!\n");
                 }
@@ -613,25 +624,20 @@ int radeon_pm_init(struct radeon_device *rdev)
 void radeon_pm_fini(struct radeon_device *rdev)
 {
         if (rdev->pm.num_power_states > 1) {
-                bool flush_wq = false;
-
                 mutex_lock(&rdev->pm.mutex);
                 if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
                         rdev->pm.profile = PM_PROFILE_DEFAULT;
                         radeon_pm_update_profile(rdev);
                         radeon_pm_set_clocks(rdev);
                 } else if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
-                        /* cancel work */
-                        cancel_delayed_work(&rdev->pm.dynpm_idle_work);
-                        flush_wq = true;
                         /* reset default clocks */
                         rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
                         rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT;
                         radeon_pm_set_clocks(rdev);
                 }
                 mutex_unlock(&rdev->pm.mutex);
-                if (flush_wq)
-                        flush_workqueue(rdev->wq);
+
+                cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
 
                 device_remove_file(rdev->dev, &dev_attr_power_profile);
                 device_remove_file(rdev->dev, &dev_attr_power_method);
@@ -640,6 +646,9 @@ void radeon_pm_fini(struct radeon_device *rdev)
 #endif
         }
 
+        if (rdev->pm.power_state)
+                kfree(rdev->pm.power_state);
+
         radeon_hwmon_fini(rdev);
 }
 
@@ -690,12 +699,12 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev)
                                         radeon_pm_get_dynpm_state(rdev);
                                         radeon_pm_set_clocks(rdev);
 
-                                        queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                                           msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                                        schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                                              msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
                                 } else if (rdev->pm.dynpm_state == DYNPM_STATE_PAUSED) {
                                         rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE;
-                                        queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                                           msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                                        schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                                              msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
                                         DRM_DEBUG_DRIVER("radeon: dynamic power management activated\n");
                                 }
                         } else { /* count == 0 */
@@ -800,8 +809,8 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work)
                         radeon_pm_set_clocks(rdev);
                 }
 
-                queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work,
-                                        msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
+                schedule_delayed_work(&rdev->pm.dynpm_idle_work,
+                                      msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
         }
         mutex_unlock(&rdev->pm.mutex);
         ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);
@@ -818,9 +827,9 @@ static int radeon_debugfs_pm_info(struct seq_file *m, void *data)
         struct drm_device *dev = node->minor->dev;
         struct radeon_device *rdev = dev->dev_private;
 
-        seq_printf(m, "default engine clock: %u0 kHz\n", rdev->clock.default_sclk);
+        seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk);
         seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev));
-        seq_printf(m, "default memory clock: %u0 kHz\n", rdev->clock.default_mclk);
+        seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk);
         if (rdev->asic->get_memory_clock)
                 seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev));
         if (rdev->pm.current_vddc)
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index 0a310b7f71c3..ec93a75369e6 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -55,6 +55,7 @@
 #include "r500_reg.h"
 #include "r600_reg.h"
 #include "evergreen_reg.h"
+#include "ni_reg.h"
 
 #define RADEON_MC_AGP_LOCATION          0x014c
 #define         RADEON_MC_AGP_START_MASK        0x0000FFFF
@@ -320,6 +321,15 @@
 #       define RADEON_PCIE_LC_RECONFIG_NOW         (1 << 8)
 #       define RADEON_PCIE_LC_RECONFIG_LATER       (1 << 9)
 #       define RADEON_PCIE_LC_SHORT_RECONFIG_EN    (1 << 10)
+#       define R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE   (1 << 7)
+#       define R600_PCIE_LC_RENEGOTIATION_SUPPORT  (1 << 9)
+#       define R600_PCIE_LC_RENEGOTIATE_EN         (1 << 10)
+#       define R600_PCIE_LC_SHORT_RECONFIG_EN      (1 << 11)
+#       define R600_PCIE_LC_UPCONFIGURE_SUPPORT    (1 << 12)
+#       define R600_PCIE_LC_UPCONFIGURE_DIS        (1 << 13)
+
+#define R600_TARGET_AND_CURRENT_PROFILE_INDEX      0x70c
+#define R700_TARGET_AND_CURRENT_PROFILE_INDEX      0x66c
 
 #define RADEON_CACHE_CNTL                   0x1724
 #define RADEON_CACHE_LINE                   0x0f0c /* PCI */
@@ -365,6 +375,8 @@
 #define RADEON_CONFIG_APER_SIZE             0x0108
 #define RADEON_CONFIG_BONDS                 0x00e8
 #define RADEON_CONFIG_CNTL                  0x00e0
+#       define RADEON_CFG_VGA_RAM_EN        (1 << 8)
+#       define RADEON_CFG_VGA_IO_DIS        (1 << 9)
 #       define RADEON_CFG_ATI_REV_A11       (0   << 16)
 #       define RADEON_CFG_ATI_REV_A12       (1   << 16)
 #       define RADEON_CFG_ATI_REV_A13       (2   << 16)
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 06e79822a2bf..992d99d13fc5 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -175,7 +175,7 @@ int radeon_ib_pool_init(struct radeon_device *rdev)
                 return 0;
         INIT_LIST_HEAD(&rdev->ib_pool.bogus_ib);
         /* Allocate 1M object buffer */
-        r = radeon_bo_create(rdev, NULL,  RADEON_IB_POOL_SIZE*64*1024,
+        r = radeon_bo_create(rdev, RADEON_IB_POOL_SIZE*64*1024,
                              PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
                              &rdev->ib_pool.robj);
         if (r) {
@@ -332,7 +332,7 @@ int radeon_ring_init(struct radeon_device *rdev, unsigned ring_size)
         rdev->cp.ring_size = ring_size;
         /* Allocate ring buffer */
         if (rdev->cp.ring_obj == NULL) {
-                r = radeon_bo_create(rdev, NULL, rdev->cp.ring_size, PAGE_SIZE, true,
+                r = radeon_bo_create(rdev, rdev->cp.ring_size, PAGE_SIZE, true,
                                         RADEON_GEM_DOMAIN_GTT,
                                         &rdev->cp.ring_obj);
                 if (r) {
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index 5b44f652145c..dee4a0c1b4b2 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -52,7 +52,7 @@ void radeon_test_moves(struct radeon_device *rdev)
                 goto out_cleanup;
         }
 
-        r = radeon_bo_create(rdev, NULL, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                 &vram_obj);
         if (r) {
                 DRM_ERROR("Failed to create VRAM object\n");
@@ -71,7 +71,7 @@ void radeon_test_moves(struct radeon_device *rdev)
                 void **gtt_start, **gtt_end;
                 void **vram_start, **vram_end;
 
-                r = radeon_bo_create(rdev, NULL, size, PAGE_SIZE, true,
+                r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
                                          RADEON_GEM_DOMAIN_GTT, gtt_obj + i);
                 if (r) {
                         DRM_ERROR("Failed to create GTT object %d\n", i);
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 1272e4b6a1d4..df5734d0c4af 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -529,7 +529,7 @@ int radeon_ttm_init(struct radeon_device *rdev)
                 DRM_ERROR("Failed initializing VRAM heap.\n");
                 return r;
         }
-        r = radeon_bo_create(rdev, NULL, 256 * 1024, PAGE_SIZE, true,
+        r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true,
                                 RADEON_GEM_DOMAIN_VRAM,
                                 &rdev->stollen_vga_memory);
         if (r) {
@@ -647,6 +647,7 @@ struct radeon_ttm_backend {
         unsigned long                   num_pages;
         struct page                     **pages;
         struct page                     *dummy_read_page;
+        dma_addr_t                      *dma_addrs;
         bool                            populated;
         bool                            bound;
         unsigned                        offset;
@@ -655,12 +656,14 @@ struct radeon_ttm_backend {
 static int radeon_ttm_backend_populate(struct ttm_backend *backend,
                                        unsigned long num_pages,
                                        struct page **pages,
-                                       struct page *dummy_read_page)
+                                       struct page *dummy_read_page,
+                                       dma_addr_t *dma_addrs)
 {
         struct radeon_ttm_backend *gtt;
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
         gtt->pages = pages;
+        gtt->dma_addrs = dma_addrs;
         gtt->num_pages = num_pages;
         gtt->dummy_read_page = dummy_read_page;
         gtt->populated = true;
@@ -673,6 +676,7 @@ static void radeon_ttm_backend_clear(struct ttm_backend *backend)
 
         gtt = container_of(backend, struct radeon_ttm_backend, backend);
         gtt->pages = NULL;
+        gtt->dma_addrs = NULL;
         gtt->num_pages = 0;
         gtt->dummy_read_page = NULL;
         gtt->populated = false;
@@ -693,7 +697,7 @@ static int radeon_ttm_backend_bind(struct ttm_backend *backend,
                      gtt->num_pages, bo_mem, backend);
         }
         r = radeon_gart_bind(gtt->rdev, gtt->offset,
-                             gtt->num_pages, gtt->pages);
+                             gtt->num_pages, gtt->pages, gtt->dma_addrs);
         if (r) {
                 DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
                           gtt->num_pages, gtt->offset);
diff --git a/drivers/gpu/drm/radeon/reg_srcs/evergreen b/drivers/gpu/drm/radeon/reg_srcs/evergreen
index ac40fd39d787..9177f9191837 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/evergreen
+++ b/drivers/gpu/drm/radeon/reg_srcs/evergreen
@@ -439,7 +439,7 @@ evergreen 0x9400
 0x000286EC SPI_COMPUTE_NUM_THREAD_X
 0x000286F0 SPI_COMPUTE_NUM_THREAD_Y
 0x000286F4 SPI_COMPUTE_NUM_THREAD_Z
-0x000286F8 GDS_ADDR_SIZE
+0x00028724 GDS_ADDR_SIZE
 0x00028780 CB_BLEND0_CONTROL
 0x00028784 CB_BLEND1_CONTROL
 0x00028788 CB_BLEND2_CONTROL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/rv515 b/drivers/gpu/drm/radeon/reg_srcs/rv515
index b3f9f1d92005..ef422bbacfc1 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/rv515
+++ b/drivers/gpu/drm/radeon/reg_srcs/rv515
@@ -304,6 +304,22 @@ rv515 0x6d40
 0x4630 US_CODE_ADDR
 0x4634 US_CODE_RANGE
 0x4638 US_CODE_OFFSET
+0x4640 US_FORMAT0_0
+0x4644 US_FORMAT0_1
+0x4648 US_FORMAT0_2
+0x464C US_FORMAT0_3
+0x4650 US_FORMAT0_4
+0x4654 US_FORMAT0_5
+0x4658 US_FORMAT0_6
+0x465C US_FORMAT0_7
+0x4660 US_FORMAT0_8
+0x4664 US_FORMAT0_9
+0x4668 US_FORMAT0_10
+0x466C US_FORMAT0_11
+0x4670 US_FORMAT0_12
+0x4674 US_FORMAT0_13
+0x4678 US_FORMAT0_14
+0x467C US_FORMAT0_15
 0x46A4 US_OUT_FMT_0
 0x46A8 US_OUT_FMT_1
 0x46AC US_OUT_FMT_2
diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c
index 5512e4e5e636..c76283d9eb3d 100644
--- a/drivers/gpu/drm/radeon/rs400.c
+++ b/drivers/gpu/drm/radeon/rs400.c
@@ -203,6 +203,9 @@ void rs400_gart_fini(struct radeon_device *rdev)
         radeon_gart_table_ram_free(rdev);
 }
 
+#define RS400_PTE_WRITEABLE (1 << 2)
+#define RS400_PTE_READABLE  (1 << 3)
+
 int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
 {
         uint32_t entry;
@@ -213,7 +216,7 @@ int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
 
         entry = (lower_32_bits(addr) & PAGE_MASK) |
                 ((upper_32_bits(addr) & 0xff) << 4) |
-                0xc;
+                RS400_PTE_WRITEABLE | RS400_PTE_READABLE;
         entry = cpu_to_le32(entry);
         rdev->gart.table.ram.ptr[i] = entry;
         return 0;
@@ -226,8 +229,8 @@ int rs400_mc_wait_for_idle(struct radeon_device *rdev)
 
         for (i = 0; i < rdev->usec_timeout; i++) {
                 /* read MC_STATUS */
-                tmp = RREG32(0x0150);
-                if (tmp & (1 << 2)) {
+                tmp = RREG32(RADEON_MC_STATUS);
+                if (tmp & RADEON_MC_IDLE) {
                         return 0;
                 }
                 DRM_UDELAY(1);
@@ -241,7 +244,7 @@ void rs400_gpu_init(struct radeon_device *rdev)
         r420_pipes_init(rdev);
         if (rs400_mc_wait_for_idle(rdev)) {
                 printk(KERN_WARNING "rs400: Failed to wait MC idle while "
-                       "programming pipes. Bad things might happen. %08x\n", RREG32(0x150));
+                       "programming pipes. Bad things might happen. %08x\n", RREG32(RADEON_MC_STATUS));
         }
 }
 
@@ -300,9 +303,9 @@ static int rs400_debugfs_gart_info(struct seq_file *m, void *data)
                 seq_printf(m, "MCCFG_AGP_BASE_2 0x%08x\n", tmp);
                 tmp = RREG32_MC(RS690_MCCFG_AGP_LOCATION);
                 seq_printf(m, "MCCFG_AGP_LOCATION 0x%08x\n", tmp);
-                tmp = RREG32_MC(0x100);
+                tmp = RREG32_MC(RS690_MCCFG_FB_LOCATION);
                 seq_printf(m, "MCCFG_FB_LOCATION 0x%08x\n", tmp);
-                tmp = RREG32(0x134);
+                tmp = RREG32(RS690_HDP_FB_LOCATION);
                 seq_printf(m, "HDP_FB_LOCATION 0x%08x\n", tmp);
         } else {
                 tmp = RREG32(RADEON_AGP_BASE);
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index 9a85b1614c86..5afe294ed51f 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -339,16 +339,16 @@ void rs600_bm_disable(struct radeon_device *rdev)
 
 int rs600_asic_reset(struct radeon_device *rdev)
 {
-        u32 status, tmp;
-
         struct rv515_mc_save save;
+        u32 status, tmp;
+        int ret = 0;
 
-        /* Stops all mc clients */
-        rv515_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(status)) {
                 return 0;
         }
+        /* Stops all mc clients */
+        rv515_mc_stop(rdev, &save);
         status = RREG32(R_000E40_RBBM_STATUS);
         dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
         /* stop CP */
@@ -392,11 +392,11 @@ int rs600_asic_reset(struct radeon_device *rdev)
         if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) {
                 dev_err(rdev->dev, "failed to reset GPU\n");
                 rdev->gpu_lockup = true;
-                return -1;
-        }
+                ret = -1;
+        } else
+                dev_info(rdev->dev, "GPU reset succeed\n");
         rv515_mc_resume(rdev, &save);
-        dev_info(rdev->dev, "GPU reset succeed\n");
-        return 0;
+        return ret;
 }
 
 /*
@@ -692,7 +692,7 @@ int rs600_irq_process(struct radeon_device *rdev)
         /* reset gui idle ack.  the status bit is broken */
         rdev->irq.gui_idle_acked = false;
         if (queue_hotplug)
-                queue_work(rdev->wq, &rdev->hotplug_work);
+                schedule_work(&rdev->hotplug_work);
         if (rdev->msi_enabled) {
                 switch (rdev->family) {
                 case CHIP_RS600:
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 5d569f41f4ae..64b57af93714 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -69,13 +69,13 @@ void rv515_ring_start(struct radeon_device *rdev)
                           ISYNC_CPSCRATCH_IDLEGUI);
         radeon_ring_write(rdev, PACKET0(WAIT_UNTIL, 0));
         radeon_ring_write(rdev, WAIT_2D_IDLECLEAN | WAIT_3D_IDLECLEAN);
-        radeon_ring_write(rdev, PACKET0(0x170C, 0));
-        radeon_ring_write(rdev, 1 << 31);
+        radeon_ring_write(rdev, PACKET0(R300_DST_PIPE_CONFIG, 0));
+        radeon_ring_write(rdev, R300_PIPE_AUTO_CONFIG);
         radeon_ring_write(rdev, PACKET0(GB_SELECT, 0));
         radeon_ring_write(rdev, 0);
         radeon_ring_write(rdev, PACKET0(GB_ENABLE, 0));
         radeon_ring_write(rdev, 0);
-        radeon_ring_write(rdev, PACKET0(0x42C8, 0));
+        radeon_ring_write(rdev, PACKET0(R500_SU_REG_DEST, 0));
         radeon_ring_write(rdev, (1 << rdev->num_gb_pipes) - 1);
         radeon_ring_write(rdev, PACKET0(VAP_INDEX_OFFSET, 0));
         radeon_ring_write(rdev, 0);
@@ -153,8 +153,8 @@ void rv515_gpu_init(struct radeon_device *rdev)
         }
         rv515_vga_render_disable(rdev);
         r420_pipes_init(rdev);
-        gb_pipe_select = RREG32(0x402C);
-        tmp = RREG32(0x170C);
+        gb_pipe_select = RREG32(R400_GB_PIPE_SELECT);
+        tmp = RREG32(R300_DST_PIPE_CONFIG);
         pipe_select_current = (tmp >> 2) & 3;
         tmp = (1 << pipe_select_current) |
               (((gb_pipe_select >> 8) & 0xF) << 4);
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index 645aa1fd7611..3a95999d2fef 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -41,6 +41,7 @@
 
 static void rv770_gpu_init(struct radeon_device *rdev);
 void rv770_fini(struct radeon_device *rdev);
+static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
 
 u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 {
@@ -77,18 +78,23 @@ u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 }
 
 /* get temperature in millidegrees */
-u32 rv770_get_temp(struct radeon_device *rdev)
+int rv770_get_temp(struct radeon_device *rdev)
 {
         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
                 ASIC_T_SHIFT;
-        u32 actual_temp = 0;
-
-        if ((temp >> 9) & 1)
-                actual_temp = 0;
-        else
-                actual_temp = (temp >> 1) & 0xff;
-
-        return actual_temp * 1000;
+        int actual_temp;
+
+        if (temp & 0x400)
+                actual_temp = -256;
+        else if (temp & 0x200)
+                actual_temp = 255;
+        else if (temp & 0x100) {
+                actual_temp = temp & 0x1ff;
+                actual_temp |= ~0x1ff;
+        } else
+                actual_temp = temp & 0xff;
+
+        return (actual_temp * 1000) / 2;
 }
 
 void rv770_pm_misc(struct radeon_device *rdev)
@@ -993,7 +999,7 @@ static int rv770_vram_scratch_init(struct radeon_device *rdev)
         u64 gpu_addr;
 
         if (rdev->vram_scratch.robj == NULL) {
-                r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE,
+                r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
                                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
                                      &rdev->vram_scratch.robj);
                 if (r) {
@@ -1124,6 +1130,9 @@ static int rv770_startup(struct radeon_device *rdev)
 {
         int r;
 
+        /* enable pcie gen2 link */
+        rv770_pcie_gen2_enable(rdev);
+
         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
                 r = r600_init_microcode(rdev);
                 if (r) {
@@ -1264,7 +1273,7 @@ int rv770_init(struct radeon_device *rdev)
         if (r)
                 return r;
         /* Post card if necessary */
-        if (!r600_card_posted(rdev)) {
+        if (!radeon_card_posted(rdev)) {
                 if (!rdev->bios) {
                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
                         return -EINVAL;
@@ -1362,3 +1371,78 @@ void rv770_fini(struct radeon_device *rdev)
         rdev->bios = NULL;
         radeon_dummy_page_fini(rdev);
 }
+
+static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
+{
+        u32 link_width_cntl, lanes, speed_cntl, tmp;
+        u16 link_cntl2;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        /* x2 cards have a special sequence */
+        if (ASIC_IS_X2(rdev))
+                return;
+
+        /* advertise upconfig capability */
+        link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+        WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+        if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
+                lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
+                link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
+                                     LC_RECONFIG_ARC_MISSING_ESCAPE);
+                link_width_cntl |= lanes | LC_RECONFIG_NOW |
+                        LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        } else {
+                link_width_cntl |= LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+
+        speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+        if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+
+                tmp = RREG32(0x541c);
+                WREG32(0x541c, tmp | 0x8);
+                WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
+                link_cntl2 = RREG16(0x4088);
+                link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
+                link_cntl2 |= 0x2;
+                WREG16(0x4088, link_cntl2);
+                WREG32(MM_CFGREGS_CNTL, 0);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+                speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
+                speed_cntl |= LC_GEN2_EN_STRAP;
+                WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        } else {
+                link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
+                /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
+                if (1)
+                        link_width_cntl |= LC_UPCONFIGURE_DIS;
+                else
+                        link_width_cntl &= ~LC_UPCONFIGURE_DIS;
+                WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
+        }
+}
diff --git a/drivers/gpu/drm/radeon/rv770d.h b/drivers/gpu/drm/radeon/rv770d.h
index fc77e1e1a179..abc8cf5a3672 100644
--- a/drivers/gpu/drm/radeon/rv770d.h
+++ b/drivers/gpu/drm/radeon/rv770d.h
@@ -360,4 +360,42 @@
 #define D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH             0x691c
 #define D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH             0x611c
 
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 #endif
diff --git a/drivers/gpu/drm/savage/savage_drv.c b/drivers/gpu/drm/savage/savage_drv.c
index fa64d25d4248..6464490b240b 100644
--- a/drivers/gpu/drm/savage/savage_drv.c
+++ b/drivers/gpu/drm/savage/savage_drv.c
@@ -55,11 +55,6 @@ static struct drm_driver driver = {
                  .llseek = noop_llseek,
         },
 
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
-
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
         .date = DRIVER_DATE,
@@ -68,15 +63,20 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver savage_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init savage_init(void)
 {
         driver.num_ioctls = savage_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &savage_pci_driver);
 }
 
 static void __exit savage_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &savage_pci_driver);
 }
 
 module_init(savage_init);
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index 4caf5d01cfd3..46d5be6e97e5 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -82,10 +82,6 @@ static struct drm_driver driver = {
                  .fasync = drm_fasync,
                  .llseek = noop_llseek,
         },
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -95,15 +91,20 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver sis_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init sis_init(void)
 {
         driver.num_ioctls = sis_max_ioctl;
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &sis_pci_driver);
 }
 
 static void __exit sis_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &sis_pci_driver);
 }
 
 module_init(sis_init);
diff --git a/drivers/gpu/drm/tdfx/tdfx_drv.c b/drivers/gpu/drm/tdfx/tdfx_drv.c
index b70fa91d761a..8bf98810a8d6 100644
--- a/drivers/gpu/drm/tdfx/tdfx_drv.c
+++ b/drivers/gpu/drm/tdfx/tdfx_drv.c
@@ -52,10 +52,6 @@ static struct drm_driver driver = {
                  .fasync = drm_fasync,
                  .llseek = noop_llseek,
         },
-        .pci_driver = {
-                 .name = DRIVER_NAME,
-                 .id_table = pciidlist,
-        },
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -65,14 +61,19 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver tdfx_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init tdfx_init(void)
 {
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &tdfx_pci_driver);
 }
 
 static void __exit tdfx_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &tdfx_pci_driver);
 }
 
 module_init(tdfx_init);
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index f999e36f30b4..1c4a72f681c1 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -47,7 +47,8 @@ struct ttm_agp_backend {
 
 static int ttm_agp_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
         struct ttm_agp_backend *agp_be =
             container_of(backend, struct ttm_agp_backend, backend);
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index cf2ec562550e..0b6a55ac2f87 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -406,11 +406,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
                 }
 
                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
+                        if (bdev->driver->move_notify)
+                                bdev->driver->move_notify(bo, mem);
                         bo->mem = *mem;
                         mem->mm_node = NULL;
                         goto moved;
                 }
-
         }
 
         if (bdev->driver->move_notify)
@@ -1500,8 +1501,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
         list_del(&bdev->device_list);
         mutex_unlock(&glob->device_list_mutex);
 
-        if (!cancel_delayed_work(&bdev->wq))
-                flush_scheduled_work();
+        cancel_delayed_work_sync(&bdev->wq);
 
         while (ttm_bo_delayed_delete(bdev, true))
                 ;
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index b1e02fffd3cc..737a2a2e46a5 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -38,6 +38,7 @@
 #include <linux/mm.h>
 #include <linux/seq_file.h> /* for seq_printf */
 #include <linux/slab.h>
+#include <linux/dma-mapping.h>
 
 #include <asm/atomic.h>
 
@@ -662,7 +663,8 @@ out:
  * cached pages.
  */
 int ttm_get_pages(struct list_head *pages, int flags,
-                enum ttm_caching_state cstate, unsigned count)
+                  enum ttm_caching_state cstate, unsigned count,
+                  dma_addr_t *dma_address)
 {
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
         struct page *p = NULL;
@@ -681,14 +683,22 @@ int ttm_get_pages(struct list_head *pages, int flags,
                         gfp_flags |= GFP_HIGHUSER;
 
                 for (r = 0; r < count; ++r) {
-                        p = alloc_page(gfp_flags);
+                        if ((flags & TTM_PAGE_FLAG_DMA32) && dma_address) {
+                                void *addr;
+                                addr = dma_alloc_coherent(NULL, PAGE_SIZE,
+                                                          &dma_address[r],
+                                                          gfp_flags);
+                                if (addr == NULL)
+                                        return -ENOMEM;
+                                p = virt_to_page(addr);
+                        } else
+                                p = alloc_page(gfp_flags);
                         if (!p) {
 
                                 printk(KERN_ERR TTM_PFX
                                        "Unable to allocate page.");
                                 return -ENOMEM;
                         }
-
                         list_add(&p->lru, pages);
                 }
                 return 0;
@@ -720,7 +730,7 @@ int ttm_get_pages(struct list_head *pages, int flags,
                         printk(KERN_ERR TTM_PFX
                                "Failed to allocate extra pages "
                                "for large request.");
-                        ttm_put_pages(pages, 0, flags, cstate);
+                        ttm_put_pages(pages, 0, flags, cstate, NULL);
                         return r;
                 }
         }
@@ -731,17 +741,29 @@ int ttm_get_pages(struct list_head *pages, int flags,
 
 /* Put all pages in pages list to correct pool to wait for reuse */
 void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
-                enum ttm_caching_state cstate)
+                   enum ttm_caching_state cstate, dma_addr_t *dma_address)
 {
         unsigned long irq_flags;
         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
         struct page *p, *tmp;
+        unsigned r;
 
         if (pool == NULL) {
                 /* No pool for this memory type so free the pages */
 
+                r = page_count-1;
                 list_for_each_entry_safe(p, tmp, pages, lru) {
-                        __free_page(p);
+                        if ((flags & TTM_PAGE_FLAG_DMA32) && dma_address) {
+                                void *addr = page_address(p);
+                                WARN_ON(!addr || !dma_address[r]);
+                                if (addr)
+                                        dma_free_coherent(NULL, PAGE_SIZE,
+                                                          addr,
+                                                          dma_address[r]);
+                                dma_address[r] = 0;
+                        } else
+                                __free_page(p);
+                        r--;
                 }
                 /* Make the pages list empty */
                 INIT_LIST_HEAD(pages);
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index af789dc869b9..86d5b1745a45 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -49,12 +49,16 @@ static int ttm_tt_swapin(struct ttm_tt *ttm);
 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
 {
         ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(*ttm->pages));
+        ttm->dma_address = drm_calloc_large(ttm->num_pages,
+                                            sizeof(*ttm->dma_address));
 }
 
 static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
 {
         drm_free_large(ttm->pages);
         ttm->pages = NULL;
+        drm_free_large(ttm->dma_address);
+        ttm->dma_address = NULL;
 }
 
 static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
@@ -105,7 +109,8 @@ static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
 
                 INIT_LIST_HEAD(&h);
 
-                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1);
+                ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1,
+                                    &ttm->dma_address[index]);
 
                 if (ret != 0)
                         return NULL;
@@ -164,7 +169,7 @@ int ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         be->func->populate(be, ttm->num_pages, ttm->pages,
-                           ttm->dummy_read_page);
+                           ttm->dummy_read_page, ttm->dma_address);
         ttm->state = tt_unbound;
         return 0;
 }
@@ -298,7 +303,8 @@ static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
                         count++;
                 }
         }
-        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state);
+        ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state,
+                      ttm->dma_address);
         ttm->state = tt_unpopulated;
         ttm->first_himem_page = ttm->num_pages;
         ttm->last_lomem_page = -1;
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
index e1ff4e7a6eb0..920a55214bcf 100644
--- a/drivers/gpu/drm/via/via_drv.c
+++ b/drivers/gpu/drm/via/via_drv.c
@@ -62,10 +62,6 @@ static struct drm_driver driver = {
                 .fasync = drm_fasync,
                 .llseek = noop_llseek,
                 },
-        .pci_driver = {
-                .name = DRIVER_NAME,
-                .id_table = pciidlist,
-        },
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -75,16 +71,21 @@ static struct drm_driver driver = {
         .patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct pci_driver via_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+};
+
 static int __init via_init(void)
 {
         driver.num_ioctls = via_max_ioctl;
         via_init_command_verifier();
-        return drm_init(&driver);
+        return drm_pci_init(&driver, &via_pci_driver);
 }
 
 static void __exit via_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &via_pci_driver);
 }
 
 module_init(via_init);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 80bc37b274e7..87e43e0733bf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -102,7 +102,8 @@ struct vmw_ttm_backend {
 
 static int vmw_ttm_populate(struct ttm_backend *backend,
                             unsigned long num_pages, struct page **pages,
-                            struct page *dummy_read_page)
+                            struct page *dummy_read_page,
+                            dma_addr_t *dma_addrs)
 {
         struct vmw_ttm_backend *vmw_be =
             container_of(backend, struct vmw_ttm_backend, backend);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 10ca97ee0206..96949b93d920 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -909,15 +909,6 @@ static struct drm_driver driver = {
 #endif
                  .llseek = noop_llseek,
         },
-        .pci_driver = {
-                 .name = VMWGFX_DRIVER_NAME,
-                 .id_table = vmw_pci_id_list,
-                 .probe = vmw_probe,
-                 .remove = vmw_remove,
-                 .driver = {
-                         .pm = &vmw_pm_ops
-                 }
-         },
         .name = VMWGFX_DRIVER_NAME,
         .desc = VMWGFX_DRIVER_DESC,
         .date = VMWGFX_DRIVER_DATE,
@@ -926,6 +917,16 @@ static struct drm_driver driver = {
         .patchlevel = VMWGFX_DRIVER_PATCHLEVEL
 };
 
+static struct pci_driver vmw_pci_driver = {
+        .name = VMWGFX_DRIVER_NAME,
+        .id_table = vmw_pci_id_list,
+        .probe = vmw_probe,
+        .remove = vmw_remove,
+        .driver = {
+                .pm = &vmw_pm_ops
+        }
+};
+
 static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
         return drm_get_pci_dev(pdev, ent, &driver);
@@ -934,7 +935,7 @@ static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 static int __init vmwgfx_init(void)
 {
         int ret;
-        ret = drm_init(&driver);
+        ret = drm_pci_init(&driver, &vmw_pci_driver);
         if (ret)
                 DRM_ERROR("Failed initializing DRM.\n");
         return ret;
@@ -942,7 +943,7 @@ static int __init vmwgfx_init(void)
 
 static void __exit vmwgfx_exit(void)
 {
-        drm_exit(&driver);
+        drm_pci_exit(&driver, &vmw_pci_driver);
 }
 
 module_init(vmwgfx_init);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index 41d9a5b73c03..bfab60c938ac 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -480,9 +480,6 @@ int vmw_fb_init(struct vmw_private *vmw_priv)
         info->fix.smem_start = 0;
         info->fix.smem_len = fb_size;
 
-        info->fix.mmio_start = 0;
-        info->fix.mmio_len = 0;
-
         info->pseudo_palette = par->pseudo_palette;
         info->screen_base = par->vmalloc;
         info->screen_size = fb_size;
@@ -659,7 +656,7 @@ int vmw_fb_off(struct vmw_private *vmw_priv)
         par->dirty.active = false;
         spin_unlock_irqrestore(&par->dirty.lock, flags);
 
-        flush_scheduled_work();
+        flush_delayed_work_sync(&info->deferred_work);
 
         par->bo_ptr = NULL;
         ttm_bo_kunmap(&par->map);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 29113c9b26a8..b3a2cd5118d7 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -345,7 +345,7 @@ static enum drm_connector_status
         return connector_status_disconnected;
 }
 
-static struct drm_display_mode vmw_ldu_connector_builtin[] = {
+static const struct drm_display_mode vmw_ldu_connector_builtin[] = {
         /* 640x480@60Hz */
         { DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656,
                    752, 800, 0, 480, 489, 492, 525, 0,
@@ -429,7 +429,6 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
         struct drm_device *dev = connector->dev;
         struct vmw_private *dev_priv = vmw_priv(dev);
         struct drm_display_mode *mode = NULL;
-        struct drm_display_mode *bmode;
         struct drm_display_mode prefmode = { DRM_MODE("preferred",
                 DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -459,6 +458,8 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
         }
 
         for (i = 0; vmw_ldu_connector_builtin[i].type != 0; i++) {
+                const struct drm_display_mode *bmode;
+
                 bmode = &vmw_ldu_connector_builtin[i];
                 if (bmode->hdisplay > max_width ||
                     bmode->vdisplay > max_height)
diff --git a/drivers/gpu/stub/Kconfig b/drivers/gpu/stub/Kconfig
index 0e1edd7311ff..70e60a4bb678 100644
--- a/drivers/gpu/stub/Kconfig
+++ b/drivers/gpu/stub/Kconfig
@@ -1,12 +1,13 @@
 config STUB_POULSBO
         tristate "Intel GMA500 Stub Driver"
         depends on PCI
+        depends on NET # for THERMAL
         # Poulsbo stub depends on ACPI_VIDEO when ACPI is enabled
         # but for select to work, need to select ACPI_VIDEO's dependencies, ick
-        select VIDEO_OUTPUT_CONTROL if ACPI
         select BACKLIGHT_CLASS_DEVICE if ACPI
         select INPUT if ACPI
         select ACPI_VIDEO if ACPI
+        select THERMAL if ACPI
         help
           Choose this option if you have a system that has Intel GMA500
           (Poulsbo) integrated graphics. If M is selected, the module will
diff --git a/drivers/gpu/vga/Kconfig b/drivers/gpu/vga/Kconfig
index 8d0e31a22027..96c83a9a76bb 100644
--- a/drivers/gpu/vga/Kconfig
+++ b/drivers/gpu/vga/Kconfig
@@ -1,5 +1,5 @@
 config VGA_ARB
-        bool "VGA Arbitration" if EMBEDDED
+        bool "VGA Arbitration" if EXPERT
         default y
         depends on PCI
         help
diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c
index c8768f38511e..e01cacba685f 100644
--- a/drivers/gpu/vga/vga_switcheroo.c
+++ b/drivers/gpu/vga/vga_switcheroo.c
@@ -33,6 +33,7 @@ struct vga_switcheroo_client {
         struct fb_info *fb_info;
         int pwr_state;
         void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state);
+        void (*reprobe)(struct pci_dev *pdev);
         bool (*can_switch)(struct pci_dev *pdev);
         int id;
         bool active;
@@ -103,6 +104,7 @@ static void vga_switcheroo_enable(void)
 
 int vga_switcheroo_register_client(struct pci_dev *pdev,
                                    void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state),
+                                   void (*reprobe)(struct pci_dev *pdev),
                                    bool (*can_switch)(struct pci_dev *pdev))
 {
         int index;
@@ -117,6 +119,7 @@ int vga_switcheroo_register_client(struct pci_dev *pdev,
         vgasr_priv.clients[index].pwr_state = VGA_SWITCHEROO_ON;
         vgasr_priv.clients[index].pdev = pdev;
         vgasr_priv.clients[index].set_gpu_state = set_gpu_state;
+        vgasr_priv.clients[index].reprobe = reprobe;
         vgasr_priv.clients[index].can_switch = can_switch;
         vgasr_priv.clients[index].id = -1;
         if (pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW)
@@ -174,7 +177,8 @@ static int vga_switcheroo_show(struct seq_file *m, void *v)
         int i;
         mutex_lock(&vgasr_mutex);
         for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
-                seq_printf(m, "%d:%c:%s:%s\n", i,
+                seq_printf(m, "%d:%s:%c:%s:%s\n", i,
+                           vgasr_priv.clients[i].id == VGA_SWITCHEROO_DIS ? "DIS" : "IGD",
                            vgasr_priv.clients[i].active ? '+' : ' ',
                            vgasr_priv.clients[i].pwr_state ? "Pwr" : "Off",
                            pci_name(vgasr_priv.clients[i].pdev));
@@ -190,9 +194,8 @@ static int vga_switcheroo_debugfs_open(struct inode *inode, struct file *file)
 
 static int vga_switchon(struct vga_switcheroo_client *client)
 {
-        int ret;
-
-        ret = vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_ON);
+        if (vgasr_priv.handler->power_state)
+                vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_ON);
         /* call the driver callback to turn on device */
         client->set_gpu_state(client->pdev, VGA_SWITCHEROO_ON);
         client->pwr_state = VGA_SWITCHEROO_ON;
@@ -203,12 +206,14 @@ static int vga_switchoff(struct vga_switcheroo_client *client)
 {
         /* call the driver callback to turn off device */
         client->set_gpu_state(client->pdev, VGA_SWITCHEROO_OFF);
-        vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_OFF);
+        if (vgasr_priv.handler->power_state)
+                vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_OFF);
         client->pwr_state = VGA_SWITCHEROO_OFF;
         return 0;
 }
 
-static int vga_switchto(struct vga_switcheroo_client *new_client)
+/* stage one happens before delay */
+static int vga_switchto_stage1(struct vga_switcheroo_client *new_client)
 {
         int ret;
         int i;
@@ -235,10 +240,28 @@ static int vga_switchto(struct vga_switcheroo_client *new_client)
                 vga_switchon(new_client);
 
         /* swap shadow resource to denote boot VGA device has changed so X starts on new device */
-        active->active = false;
-
         active->pdev->resource[PCI_ROM_RESOURCE].flags &= ~IORESOURCE_ROM_SHADOW;
         new_client->pdev->resource[PCI_ROM_RESOURCE].flags |= IORESOURCE_ROM_SHADOW;
+        return 0;
+}
+
+/* post delay */
+static int vga_switchto_stage2(struct vga_switcheroo_client *new_client)
+{
+        int ret;
+        int i;
+        struct vga_switcheroo_client *active = NULL;
+
+        for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
+                if (vgasr_priv.clients[i].active == true) {
+                        active = &vgasr_priv.clients[i];
+                        break;
+                }
+        }
+        if (!active)
+                return 0;
+
+        active->active = false;
 
         if (new_client->fb_info) {
                 struct fb_event event;
@@ -250,6 +273,9 @@ static int vga_switchto(struct vga_switcheroo_client *new_client)
         if (ret)
                 return ret;
 
+        if (new_client->reprobe)
+                new_client->reprobe(new_client->pdev);
+
         if (active->pwr_state == VGA_SWITCHEROO_ON)
                 vga_switchoff(active);
 
@@ -265,6 +291,7 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
         const char *pdev_name;
         int i, ret;
         bool delay = false, can_switch;
+        bool just_mux = false;
         int client_id = -1;
         struct vga_switcheroo_client *client = NULL;
 
@@ -319,6 +346,15 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
         if (strncmp(usercmd, "DIS", 3) == 0)
                 client_id = VGA_SWITCHEROO_DIS;
 
+        if (strncmp(usercmd, "MIGD", 4) == 0) {
+                just_mux = true;
+                client_id = VGA_SWITCHEROO_IGD;
+        }
+        if (strncmp(usercmd, "MDIS", 4) == 0) {
+                just_mux = true;
+                client_id = VGA_SWITCHEROO_DIS;
+        }
+
         if (client_id == -1)
                 goto out;
 
@@ -330,6 +366,12 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
         }
 
         vgasr_priv.delayed_switch_active = false;
+
+        if (just_mux) {
+                ret = vgasr_priv.handler->switchto(client_id);
+                goto out;
+        }
+
         /* okay we want a switch - test if devices are willing to switch */
         can_switch = true;
         for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
@@ -345,18 +387,22 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
 
         if (can_switch == true) {
                 pdev_name = pci_name(client->pdev);
-                ret = vga_switchto(client);
+                ret = vga_switchto_stage1(client);
                 if (ret)
-                        printk(KERN_ERR "vga_switcheroo: switching failed %d\n", ret);
+                        printk(KERN_ERR "vga_switcheroo: switching failed stage 1 %d\n", ret);
+
+                ret = vga_switchto_stage2(client);
+                if (ret)
+                        printk(KERN_ERR "vga_switcheroo: switching failed stage 2 %d\n", ret);
+
         } else {
                 printk(KERN_INFO "vga_switcheroo: setting delayed switch to client %d\n", client->id);
                 vgasr_priv.delayed_switch_active = true;
                 vgasr_priv.delayed_client_id = client_id;
 
-                /* we should at least power up the card to
-                   make the switch faster */
-                if (client->pwr_state == VGA_SWITCHEROO_OFF)
-                        vga_switchon(client);
+                ret = vga_switchto_stage1(client);
+                if (ret)
+                        printk(KERN_ERR "vga_switcheroo: delayed switching stage 1 failed %d\n", ret);
         }
 
 out:
@@ -438,9 +484,9 @@ int vga_switcheroo_process_delayed_switch(void)
                 goto err;
 
         pdev_name = pci_name(client->pdev);
-        ret = vga_switchto(client);
+        ret = vga_switchto_stage2(client);
         if (ret)
-                printk(KERN_ERR "vga_switcheroo: delayed switching failed %d\n", ret);
+                printk(KERN_ERR "vga_switcheroo: delayed switching failed stage 2 %d\n", ret);
 
         vgasr_priv.delayed_switch_active = false;
         err = 0;
diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c
index c380c65da417..ace2b1623b21 100644
--- a/drivers/gpu/vga/vgaarb.c
+++ b/drivers/gpu/vga/vgaarb.c
@@ -636,7 +636,7 @@ int vga_client_register(struct pci_dev *pdev, void *cookie,
                         void (*irq_set_state)(void *cookie, bool state),
                         unsigned int (*set_vga_decode)(void *cookie, bool decode))
 {
-        int ret = -1;
+        int ret = -ENODEV;
         struct vga_device *vgadev;
         unsigned long flags;