Merge remote-tracking branch 'airlied/drm-next' into drm-nouveau-next

339 files changed, 69936 insertions, 7106 deletions

diff --git a/Documentation/DocBook/device-drivers.tmpl b/Documentation/DocBook/device-drivers.tmpl
index c36892c072da..f0648a8b09b6 100644
--- a/Documentation/DocBook/device-drivers.tmpl
+++ b/Documentation/DocBook/device-drivers.tmpl
@@ -126,6 +126,8 @@ X!Edrivers/base/interface.c
      </sect1>
      <sect1><title>Device Drivers DMA Management</title>
 !Edrivers/base/dma-buf.c
+!Edrivers/base/reservation.c
+!Iinclude/linux/reservation.h
 !Edrivers/base/dma-coherent.c
 !Edrivers/base/dma-mapping.c
      </sect1>
diff --git a/Documentation/DocBook/drm.tmpl b/Documentation/DocBook/drm.tmpl
index f9df3b872c16..4d54ac8b2032 100644
--- a/Documentation/DocBook/drm.tmpl
+++ b/Documentation/DocBook/drm.tmpl
@@ -186,11 +186,12 @@
           <varlistentry>
             <term>DRIVER_HAVE_IRQ</term><term>DRIVER_IRQ_SHARED</term>
             <listitem><para>
-              DRIVER_HAVE_IRQ indicates whether the driver has an IRQ handler. The
-              DRM core will automatically register an interrupt handler when the
-              flag is set. DRIVER_IRQ_SHARED indicates whether the device &amp;
-              handler support shared IRQs (note that this is required of PCI
-              drivers).
+              DRIVER_HAVE_IRQ indicates whether the driver has an IRQ handler
+              managed by the DRM Core. The core will support simple IRQ handler
+              installation when the flag is set. The installation process is
+              described in <xref linkend="drm-irq-registration"/>.</para>
+              <para>DRIVER_IRQ_SHARED indicates whether the device &amp; handler
+              support shared IRQs (note that this is required of PCI  drivers).
             </para></listitem>
           </varlistentry>
           <varlistentry>
@@ -344,50 +345,71 @@ char *date;</synopsis>
           The DRM core tries to facilitate IRQ handler registration and
           unregistration by providing <function>drm_irq_install</function> and
           <function>drm_irq_uninstall</function> functions. Those functions only
-          support a single interrupt per device.
-        </para>
-  <!--!Fdrivers/char/drm/drm_irq.c drm_irq_install-->
-        <para>
-          Both functions get the device IRQ by calling
-          <function>drm_dev_to_irq</function>. This inline function will call a
-          bus-specific operation to retrieve the IRQ number. For platform devices,
-          <function>platform_get_irq</function>(..., 0) is used to retrieve the
-          IRQ number.
-        </para>
-        <para>
-          <function>drm_irq_install</function> starts by calling the
-          <methodname>irq_preinstall</methodname> driver operation. The operation
-          is optional and must make sure that the interrupt will not get fired by
-          clearing all pending interrupt flags or disabling the interrupt.
-        </para>
-        <para>
-          The IRQ will then be requested by a call to
-          <function>request_irq</function>. If the DRIVER_IRQ_SHARED driver
-          feature flag is set, a shared (IRQF_SHARED) IRQ handler will be
-          requested.
-        </para>
-        <para>
-          The IRQ handler function must be provided as the mandatory irq_handler
-          driver operation. It will get passed directly to
-          <function>request_irq</function> and thus has the same prototype as all
-          IRQ handlers. It will get called with a pointer to the DRM device as the
-          second argument.
-        </para>
-        <para>
-          Finally the function calls the optional
-          <methodname>irq_postinstall</methodname> driver operation. The operation
-          usually enables interrupts (excluding the vblank interrupt, which is
-          enabled separately), but drivers may choose to enable/disable interrupts
-          at a different time.
-        </para>
-        <para>
-          <function>drm_irq_uninstall</function> is similarly used to uninstall an
-          IRQ handler. It starts by waking up all processes waiting on a vblank
-          interrupt to make sure they don't hang, and then calls the optional
-          <methodname>irq_uninstall</methodname> driver operation. The operation
-          must disable all hardware interrupts. Finally the function frees the IRQ
-          by calling <function>free_irq</function>.
+          support a single interrupt per device, devices that use more than one
+          IRQs need to be handled manually.
         </para>
+        <sect4>
+          <title>Managed IRQ Registration</title>
+          <para>
+            Both the <function>drm_irq_install</function> and
+            <function>drm_irq_uninstall</function> functions get the device IRQ by
+            calling <function>drm_dev_to_irq</function>. This inline function will
+            call a bus-specific operation to retrieve the IRQ number. For platform
+            devices, <function>platform_get_irq</function>(..., 0) is used to
+            retrieve the IRQ number.
+          </para>
+          <para>
+            <function>drm_irq_install</function> starts by calling the
+            <methodname>irq_preinstall</methodname> driver operation. The operation
+            is optional and must make sure that the interrupt will not get fired by
+            clearing all pending interrupt flags or disabling the interrupt.
+          </para>
+          <para>
+            The IRQ will then be requested by a call to
+            <function>request_irq</function>. If the DRIVER_IRQ_SHARED driver
+            feature flag is set, a shared (IRQF_SHARED) IRQ handler will be
+            requested.
+          </para>
+          <para>
+            The IRQ handler function must be provided as the mandatory irq_handler
+            driver operation. It will get passed directly to
+            <function>request_irq</function> and thus has the same prototype as all
+            IRQ handlers. It will get called with a pointer to the DRM device as the
+            second argument.
+          </para>
+          <para>
+            Finally the function calls the optional
+            <methodname>irq_postinstall</methodname> driver operation. The operation
+            usually enables interrupts (excluding the vblank interrupt, which is
+            enabled separately), but drivers may choose to enable/disable interrupts
+            at a different time.
+          </para>
+          <para>
+            <function>drm_irq_uninstall</function> is similarly used to uninstall an
+            IRQ handler. It starts by waking up all processes waiting on a vblank
+            interrupt to make sure they don't hang, and then calls the optional
+            <methodname>irq_uninstall</methodname> driver operation. The operation
+            must disable all hardware interrupts. Finally the function frees the IRQ
+            by calling <function>free_irq</function>.
+          </para>
+        </sect4>
+        <sect4>
+          <title>Manual IRQ Registration</title>
+          <para>
+            Drivers that require multiple interrupt handlers can't use the managed
+            IRQ registration functions. In that case IRQs must be registered and
+            unregistered manually (usually with the <function>request_irq</function>
+            and <function>free_irq</function> functions, or their devm_* equivalent).
+          </para>
+          <para>
+            When manually registering IRQs, drivers must not set the DRIVER_HAVE_IRQ
+            driver feature flag, and must not provide the
+            <methodname>irq_handler</methodname> driver operation. They must set the
+            <structname>drm_device</structname> <structfield>irq_enabled</structfield>
+            field to 1 upon registration of the IRQs, and clear it to 0 after
+            unregistering the IRQs.
+          </para>
+        </sect4>
       </sect3>
       <sect3>
         <title>Memory Manager Initialization</title>
@@ -1214,6 +1236,15 @@ int max_width, max_height;</synopsis>
           <title>Miscellaneous</title>
           <itemizedlist>
             <listitem>
+              <synopsis>void (*set_property)(struct drm_crtc *crtc,
+                     struct drm_property *property, uint64_t value);</synopsis>
+              <para>
+                Set the value of the given CRTC property to
+                <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
+                for more information about properties.
+              </para>
+            </listitem>
+            <listitem>
               <synopsis>void (*gamma_set)(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
                         uint32_t start, uint32_t size);</synopsis>
               <para>
@@ -1363,6 +1394,15 @@ int max_width, max_height;</synopsis>
               <xref linkend="drm-kms-init"/>.
             </para>
           </listitem>
+          <listitem>
+            <synopsis>void (*set_property)(struct drm_plane *plane,
+                     struct drm_property *property, uint64_t value);</synopsis>
+            <para>
+              Set the value of the given plane property to
+              <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
+              for more information about properties.
+            </para>
+          </listitem>
         </itemizedlist>
       </sect3>
     </sect2>
@@ -1572,6 +1612,15 @@ int max_width, max_height;</synopsis>
           <title>Miscellaneous</title>
           <itemizedlist>
             <listitem>
+              <synopsis>void (*set_property)(struct drm_connector *connector,
+                     struct drm_property *property, uint64_t value);</synopsis>
+              <para>
+                Set the value of the given connector property to
+                <parameter>value</parameter>. See <xref linkend="drm-kms-properties"/>
+                for more information about properties.
+              </para>
+            </listitem>
+            <listitem>
               <synopsis>void (*destroy)(struct drm_connector *connector);</synopsis>
               <para>
                 Destroy the connector when not needed anymore. See
@@ -1846,10 +1895,6 @@ void intel_crt_init(struct drm_device *dev)
           <synopsis>bool (*mode_fixup)(struct drm_encoder *encoder,
                        const struct drm_display_mode *mode,
                        struct drm_display_mode *adjusted_mode);</synopsis>
-          <note><para>
-            FIXME: The mode argument be const, but the i915 driver modifies
-            mode-&gt;clock in <function>intel_dp_mode_fixup</function>.
-          </para></note>
           <para>
             Let encoders adjust the requested mode or reject it completely. This
             operation returns true if the mode is accepted (possibly after being
@@ -2161,6 +2206,128 @@ void intel_crt_init(struct drm_device *dev)
       <title>EDID Helper Functions Reference</title>
 !Edrivers/gpu/drm/drm_edid.c
     </sect2>
+    <sect2>
+      <title>Rectangle Utilities Reference</title>
+!Pinclude/drm/drm_rect.h rect utils
+!Iinclude/drm/drm_rect.h
+!Edrivers/gpu/drm/drm_rect.c
+    </sect2>
+  </sect1>
+
+  <!-- Internals: kms properties -->
+
+  <sect1 id="drm-kms-properties">
+    <title>KMS Properties</title>
+    <para>
+      Drivers may need to expose additional parameters to applications than
+      those described in the previous sections. KMS supports attaching
+      properties to CRTCs, connectors and planes and offers a userspace API to
+      list, get and set the property values.
+    </para>
+    <para>
+      Properties are identified by a name that uniquely defines the property
+      purpose, and store an associated value. For all property types except blob
+      properties the value is a 64-bit unsigned integer.
+    </para>
+    <para>
+      KMS differentiates between properties and property instances. Drivers
+      first create properties and then create and associate individual instances
+      of those properties to objects. A property can be instantiated multiple
+      times and associated with different objects. Values are stored in property
+      instances, and all other property information are stored in the propery
+      and shared between all instances of the property.
+    </para>
+    <para>
+      Every property is created with a type that influences how the KMS core
+      handles the property. Supported property types are
+      <variablelist>
+        <varlistentry>
+          <term>DRM_MODE_PROP_RANGE</term>
+          <listitem><para>Range properties report their minimum and maximum
+            admissible values. The KMS core verifies that values set by
+            application fit in that range.</para></listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>DRM_MODE_PROP_ENUM</term>
+          <listitem><para>Enumerated properties take a numerical value that
+            ranges from 0 to the number of enumerated values defined by the
+            property minus one, and associate a free-formed string name to each
+            value. Applications can retrieve the list of defined value-name pairs
+            and use the numerical value to get and set property instance values.
+            </para></listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>DRM_MODE_PROP_BITMASK</term>
+          <listitem><para>Bitmask properties are enumeration properties that
+            additionally restrict all enumerated values to the 0..63 range.
+            Bitmask property instance values combine one or more of the
+            enumerated bits defined by the property.</para></listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>DRM_MODE_PROP_BLOB</term>
+          <listitem><para>Blob properties store a binary blob without any format
+            restriction. The binary blobs are created as KMS standalone objects,
+            and blob property instance values store the ID of their associated
+            blob object.</para>
+            <para>Blob properties are only used for the connector EDID property
+            and cannot be created by drivers.</para></listitem>
+        </varlistentry>
+      </variablelist>
+    </para>
+    <para>
+      To create a property drivers call one of the following functions depending
+      on the property type. All property creation functions take property flags
+      and name, as well as type-specific arguments.
+      <itemizedlist>
+        <listitem>
+          <synopsis>struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
+                                               const char *name,
+                                               uint64_t min, uint64_t max);</synopsis>
+          <para>Create a range property with the given minimum and maximum
+            values.</para>
+        </listitem>
+        <listitem>
+          <synopsis>struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
+                                              const char *name,
+                                              const struct drm_prop_enum_list *props,
+                                              int num_values);</synopsis>
+          <para>Create an enumerated property. The <parameter>props</parameter>
+            argument points to an array of <parameter>num_values</parameter>
+            value-name pairs.</para>
+        </listitem>
+        <listitem>
+          <synopsis>struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
+                                                 int flags, const char *name,
+                                                 const struct drm_prop_enum_list *props,
+                                                 int num_values);</synopsis>
+          <para>Create a bitmask property. The <parameter>props</parameter>
+            argument points to an array of <parameter>num_values</parameter>
+            value-name pairs.</para>
+        </listitem>
+      </itemizedlist>
+    </para>
+    <para>
+      Properties can additionally be created as immutable, in which case they
+      will be read-only for applications but can be modified by the driver. To
+      create an immutable property drivers must set the DRM_MODE_PROP_IMMUTABLE
+      flag at property creation time.
+    </para>
+    <para>
+      When no array of value-name pairs is readily available at property
+      creation time for enumerated or range properties, drivers can create
+      the property using the <function>drm_property_create</function> function
+      and manually add enumeration value-name pairs by calling the
+      <function>drm_property_add_enum</function> function. Care must be taken to
+      properly specify the property type through the <parameter>flags</parameter>
+      argument.
+    </para>
+    <para>
+      After creating properties drivers can attach property instances to CRTC,
+      connector and plane objects by calling the
+      <function>drm_object_attach_property</function>. The function takes a
+      pointer to the target object, a pointer to the previously created property
+      and an initial instance value.
+    </para>
   </sect1>
 
   <!-- Internals: vertical blanking -->
diff --git a/Documentation/devicetree/bindings/drm/tilcdc/tilcdc.txt b/Documentation/devicetree/bindings/drm/tilcdc/tilcdc.txt
index e5f130159ae1..fff10da5e927 100644
--- a/Documentation/devicetree/bindings/drm/tilcdc/tilcdc.txt
+++ b/Documentation/devicetree/bindings/drm/tilcdc/tilcdc.txt
@@ -10,6 +10,14 @@ Recommended properties:
    services interrupts for this device.
  - ti,hwmods: Name of the hwmod associated to the LCDC
 
+Optional properties:
+ - max-bandwidth: The maximum pixels per second that the memory
+   interface / lcd controller combination can sustain
+ - max-width: The maximum horizontal pixel width supported by
+   the lcd controller.
+ - max-pixelclock: The maximum pixel clock that can be supported
+   by the lcd controller in KHz.
+
 Example:
 
         fb: fb@4830e000 {
diff --git a/Documentation/devicetree/bindings/video/display-timing.txt b/Documentation/devicetree/bindings/video/display-timing.txt
index 150038552bc3..e1d4a0b59612 100644
--- a/Documentation/devicetree/bindings/video/display-timing.txt
+++ b/Documentation/devicetree/bindings/video/display-timing.txt
@@ -34,6 +34,7 @@ optional properties:
                         - ignored     = ignored
  - interlaced (bool): boolean to enable interlaced mode
  - doublescan (bool): boolean to enable doublescan mode
+ - doubleclk (bool): boolean to enable doubleclock mode
 
 All the optional properties that are not bool follow the following logic:
     <1>: high active
diff --git a/Documentation/devicetree/bindings/video/exynos_hdmi.txt b/Documentation/devicetree/bindings/video/exynos_hdmi.txt
index 589edee37394..c71d0f0b750a 100644
--- a/Documentation/devicetree/bindings/video/exynos_hdmi.txt
+++ b/Documentation/devicetree/bindings/video/exynos_hdmi.txt
@@ -1,7 +1,10 @@
 Device-Tree bindings for drm hdmi driver
 
 Required properties:
-- compatible: value should be "samsung,exynos5-hdmi".
+- compatible: value should be one among the following:
+        1) "samsung,exynos5-hdmi" <DEPRECATED>
+        2) "samsung,exynos4210-hdmi"
+        3) "samsung,exynos4212-hdmi"
 - reg: physical base address of the hdmi and length of memory mapped
         region.
 - interrupts: interrupt number to the cpu.
@@ -15,7 +18,7 @@ Required properties:
 Example:
 
         hdmi {
-                compatible = "samsung,exynos5-hdmi";
+                compatible = "samsung,exynos4212-hdmi";
                 reg = <0x14530000 0x100000>;
                 interrupts = <0 95 0>;
                 hpd-gpio = <&gpx3 7 0xf 1 3>;
diff --git a/Documentation/devicetree/bindings/video/exynos_hdmiddc.txt b/Documentation/devicetree/bindings/video/exynos_hdmiddc.txt
index fa166d945809..41eee971562b 100644
--- a/Documentation/devicetree/bindings/video/exynos_hdmiddc.txt
+++ b/Documentation/devicetree/bindings/video/exynos_hdmiddc.txt
@@ -1,12 +1,15 @@
 Device-Tree bindings for hdmiddc driver
 
 Required properties:
-- compatible: value should be "samsung,exynos5-hdmiddc".
+- compatible: value should be one of the following
+        1) "samsung,exynos5-hdmiddc" <DEPRECATED>
+        2) "samsung,exynos4210-hdmiddc"
+
 - reg: I2C address of the hdmiddc device.
 
 Example:
 
         hdmiddc {
-                compatible = "samsung,exynos5-hdmiddc";
+                compatible = "samsung,exynos4210-hdmiddc";
                 reg = <0x50>;
         };
diff --git a/Documentation/devicetree/bindings/video/exynos_hdmiphy.txt b/Documentation/devicetree/bindings/video/exynos_hdmiphy.txt
index 858f4f9b902f..162f641f7639 100644
--- a/Documentation/devicetree/bindings/video/exynos_hdmiphy.txt
+++ b/Documentation/devicetree/bindings/video/exynos_hdmiphy.txt
@@ -1,12 +1,15 @@
 Device-Tree bindings for hdmiphy driver
 
 Required properties:
-- compatible: value should be "samsung,exynos5-hdmiphy".
+- compatible: value should be one of the following:
+        1) "samsung,exynos5-hdmiphy" <DEPRECATED>
+        2) "samsung,exynos4210-hdmiphy".
+        3) "samsung,exynos4212-hdmiphy".
 - reg: I2C address of the hdmiphy device.
 
 Example:
 
         hdmiphy {
-                compatible = "samsung,exynos5-hdmiphy";
+                compatible = "samsung,exynos4210-hdmiphy";
                 reg = <0x38>;
         };
diff --git a/Documentation/devicetree/bindings/video/exynos_mixer.txt b/Documentation/devicetree/bindings/video/exynos_mixer.txt
index 9b2ea0343566..3334b0a8e343 100644
--- a/Documentation/devicetree/bindings/video/exynos_mixer.txt
+++ b/Documentation/devicetree/bindings/video/exynos_mixer.txt
@@ -1,7 +1,12 @@
 Device-Tree bindings for mixer driver
 
 Required properties:
-- compatible: value should be "samsung,exynos5-mixer".
+- compatible: value should be one of the following:
+        1) "samsung,exynos5-mixer" <DEPRECATED>
+        2) "samsung,exynos4210-mixer"
+        3) "samsung,exynos5250-mixer"
+        4) "samsung,exynos5420-mixer"
+
 - reg: physical base address of the mixer and length of memory mapped
         region.
 - interrupts: interrupt number to the cpu.
@@ -9,7 +14,7 @@ Required properties:
 Example:
 
         mixer {
-                compatible = "samsung,exynos5-mixer";
+                compatible = "samsung,exynos5250-mixer";
                 reg = <0x14450000 0x10000>;
                 interrupts = <0 94 0>;
         };
diff --git a/Documentation/fb/uvesafb.txt b/Documentation/fb/uvesafb.txt
index eefdd91d298a..f6362d88763b 100644
--- a/Documentation/fb/uvesafb.txt
+++ b/Documentation/fb/uvesafb.txt
@@ -81,17 +81,11 @@ pmipal  Use the protected mode interface for palette changes.
 
 mtrr:n  Setup memory type range registers for the framebuffer
         where n:
-              0 - disabled (equivalent to nomtrr) (default)
-              1 - uncachable
-              2 - write-back
-              3 - write-combining
-              4 - write-through
-
-        If you see the following in dmesg, choose the type that matches
-        the old one.  In this example, use "mtrr:2".
-...
-mtrr: type mismatch for e0000000,8000000 old: write-back new: write-combining
-...
+              0 - disabled (equivalent to nomtrr)
+              3 - write-combining (default)
+
+        Values other than 0 and 3 will result in a warning and will be
+        treated just like 3.
 
 nomtrr  Do not use memory type range registers.
 
diff --git a/Documentation/ww-mutex-design.txt b/Documentation/ww-mutex-design.txt
new file mode 100644
index 000000000000..8a112dc304c3
--- /dev/null
+++ b/Documentation/ww-mutex-design.txt
@@ -0,0 +1,344 @@
+Wait/Wound Deadlock-Proof Mutex Design
+======================================
+
+Please read mutex-design.txt first, as it applies to wait/wound mutexes too.
+
+Motivation for WW-Mutexes
+-------------------------
+
+GPU's do operations that commonly involve many buffers.  Those buffers
+can be shared across contexts/processes, exist in different memory
+domains (for example VRAM vs system memory), and so on.  And with
+PRIME / dmabuf, they can even be shared across devices.  So there are
+a handful of situations where the driver needs to wait for buffers to
+become ready.  If you think about this in terms of waiting on a buffer
+mutex for it to become available, this presents a problem because
+there is no way to guarantee that buffers appear in a execbuf/batch in
+the same order in all contexts.  That is directly under control of
+userspace, and a result of the sequence of GL calls that an application
+makes.  Which results in the potential for deadlock.  The problem gets
+more complex when you consider that the kernel may need to migrate the
+buffer(s) into VRAM before the GPU operates on the buffer(s), which
+may in turn require evicting some other buffers (and you don't want to
+evict other buffers which are already queued up to the GPU), but for a
+simplified understanding of the problem you can ignore this.
+
+The algorithm that the TTM graphics subsystem came up with for dealing with
+this problem is quite simple.  For each group of buffers (execbuf) that need
+to be locked, the caller would be assigned a unique reservation id/ticket,
+from a global counter.  In case of deadlock while locking all the buffers
+associated with a execbuf, the one with the lowest reservation ticket (i.e.
+the oldest task) wins, and the one with the higher reservation id (i.e. the
+younger task) unlocks all of the buffers that it has already locked, and then
+tries again.
+
+In the RDBMS literature this deadlock handling approach is called wait/wound:
+The older tasks waits until it can acquire the contended lock. The younger tasks
+needs to back off and drop all the locks it is currently holding, i.e. the
+younger task is wounded.
+
+Concepts
+--------
+
+Compared to normal mutexes two additional concepts/objects show up in the lock
+interface for w/w mutexes:
+
+Acquire context: To ensure eventual forward progress it is important the a task
+trying to acquire locks doesn't grab a new reservation id, but keeps the one it
+acquired when starting the lock acquisition. This ticket is stored in the
+acquire context. Furthermore the acquire context keeps track of debugging state
+to catch w/w mutex interface abuse.
+
+W/w class: In contrast to normal mutexes the lock class needs to be explicit for
+w/w mutexes, since it is required to initialize the acquire context.
+
+Furthermore there are three different class of w/w lock acquire functions:
+
+* Normal lock acquisition with a context, using ww_mutex_lock.
+
+* Slowpath lock acquisition on the contending lock, used by the wounded task
+  after having dropped all already acquired locks. These functions have the
+  _slow postfix.
+
+  From a simple semantics point-of-view the _slow functions are not strictly
+  required, since simply calling the normal ww_mutex_lock functions on the
+  contending lock (after having dropped all other already acquired locks) will
+  work correctly. After all if no other ww mutex has been acquired yet there's
+  no deadlock potential and hence the ww_mutex_lock call will block and not
+  prematurely return -EDEADLK. The advantage of the _slow functions is in
+  interface safety:
+  - ww_mutex_lock has a __must_check int return type, whereas ww_mutex_lock_slow
+    has a void return type. Note that since ww mutex code needs loops/retries
+    anyway the __must_check doesn't result in spurious warnings, even though the
+    very first lock operation can never fail.
+  - When full debugging is enabled ww_mutex_lock_slow checks that all acquired
+    ww mutex have been released (preventing deadlocks) and makes sure that we
+    block on the contending lock (preventing spinning through the -EDEADLK
+    slowpath until the contended lock can be acquired).
+
+* Functions to only acquire a single w/w mutex, which results in the exact same
+  semantics as a normal mutex. This is done by calling ww_mutex_lock with a NULL
+  context.
+
+  Again this is not strictly required. But often you only want to acquire a
+  single lock in which case it's pointless to set up an acquire context (and so
+  better to avoid grabbing a deadlock avoidance ticket).
+
+Of course, all the usual variants for handling wake-ups due to signals are also
+provided.
+
+Usage
+-----
+
+Three different ways to acquire locks within the same w/w class. Common
+definitions for methods #1 and #2:
+
+static DEFINE_WW_CLASS(ww_class);
+
+struct obj {
+        struct ww_mutex lock;
+        /* obj data */
+};
+
+struct obj_entry {
+        struct list_head head;
+        struct obj *obj;
+};
+
+Method 1, using a list in execbuf->buffers that's not allowed to be reordered.
+This is useful if a list of required objects is already tracked somewhere.
+Furthermore the lock helper can use propagate the -EALREADY return code back to
+the caller as a signal that an object is twice on the list. This is useful if
+the list is constructed from userspace input and the ABI requires userspace to
+not have duplicate entries (e.g. for a gpu commandbuffer submission ioctl).
+
+int lock_objs(struct list_head *list, struct ww_acquire_ctx *ctx)
+{
+        struct obj *res_obj = NULL;
+        struct obj_entry *contended_entry = NULL;
+        struct obj_entry *entry;
+
+        ww_acquire_init(ctx, &ww_class);
+
+retry:
+        list_for_each_entry (entry, list, head) {
+                if (entry->obj == res_obj) {
+                        res_obj = NULL;
+                        continue;
+                }
+                ret = ww_mutex_lock(&entry->obj->lock, ctx);
+                if (ret < 0) {
+                        contended_entry = entry;
+                        goto err;
+                }
+        }
+
+        ww_acquire_done(ctx);
+        return 0;
+
+err:
+        list_for_each_entry_continue_reverse (entry, list, head)
+                ww_mutex_unlock(&entry->obj->lock);
+
+        if (res_obj)
+                ww_mutex_unlock(&res_obj->lock);
+
+        if (ret == -EDEADLK) {
+                /* we lost out in a seqno race, lock and retry.. */
+                ww_mutex_lock_slow(&contended_entry->obj->lock, ctx);
+                res_obj = contended_entry->obj;
+                goto retry;
+        }
+        ww_acquire_fini(ctx);
+
+        return ret;
+}
+
+Method 2, using a list in execbuf->buffers that can be reordered. Same semantics
+of duplicate entry detection using -EALREADY as method 1 above. But the
+list-reordering allows for a bit more idiomatic code.
+
+int lock_objs(struct list_head *list, struct ww_acquire_ctx *ctx)
+{
+        struct obj_entry *entry, *entry2;
+
+        ww_acquire_init(ctx, &ww_class);
+
+        list_for_each_entry (entry, list, head) {
+                ret = ww_mutex_lock(&entry->obj->lock, ctx);
+                if (ret < 0) {
+                        entry2 = entry;
+
+                        list_for_each_entry_continue_reverse (entry2, list, head)
+                                ww_mutex_unlock(&entry2->obj->lock);
+
+                        if (ret != -EDEADLK) {
+                                ww_acquire_fini(ctx);
+                                return ret;
+                        }
+
+                        /* we lost out in a seqno race, lock and retry.. */
+                        ww_mutex_lock_slow(&entry->obj->lock, ctx);
+
+                        /*
+                         * Move buf to head of the list, this will point
+                         * buf->next to the first unlocked entry,
+                         * restarting the for loop.
+                         */
+                        list_del(&entry->head);
+                        list_add(&entry->head, list);
+                }
+        }
+
+        ww_acquire_done(ctx);
+        return 0;
+}
+
+Unlocking works the same way for both methods #1 and #2:
+
+void unlock_objs(struct list_head *list, struct ww_acquire_ctx *ctx)
+{
+        struct obj_entry *entry;
+
+        list_for_each_entry (entry, list, head)
+                ww_mutex_unlock(&entry->obj->lock);
+
+        ww_acquire_fini(ctx);
+}
+
+Method 3 is useful if the list of objects is constructed ad-hoc and not upfront,
+e.g. when adjusting edges in a graph where each node has its own ww_mutex lock,
+and edges can only be changed when holding the locks of all involved nodes. w/w
+mutexes are a natural fit for such a case for two reasons:
+- They can handle lock-acquisition in any order which allows us to start walking
+  a graph from a starting point and then iteratively discovering new edges and
+  locking down the nodes those edges connect to.
+- Due to the -EALREADY return code signalling that a given objects is already
+  held there's no need for additional book-keeping to break cycles in the graph
+  or keep track off which looks are already held (when using more than one node
+  as a starting point).
+
+Note that this approach differs in two important ways from the above methods:
+- Since the list of objects is dynamically constructed (and might very well be
+  different when retrying due to hitting the -EDEADLK wound condition) there's
+  no need to keep any object on a persistent list when it's not locked. We can
+  therefore move the list_head into the object itself.
+- On the other hand the dynamic object list construction also means that the -EALREADY return
+  code can't be propagated.
+
+Note also that methods #1 and #2 and method #3 can be combined, e.g. to first lock a
+list of starting nodes (passed in from userspace) using one of the above
+methods. And then lock any additional objects affected by the operations using
+method #3 below. The backoff/retry procedure will be a bit more involved, since
+when the dynamic locking step hits -EDEADLK we also need to unlock all the
+objects acquired with the fixed list. But the w/w mutex debug checks will catch
+any interface misuse for these cases.
+
+Also, method 3 can't fail the lock acquisition step since it doesn't return
+-EALREADY. Of course this would be different when using the _interruptible
+variants, but that's outside of the scope of these examples here.
+
+struct obj {
+        struct ww_mutex ww_mutex;
+        struct list_head locked_list;
+};
+
+static DEFINE_WW_CLASS(ww_class);
+
+void __unlock_objs(struct list_head *list)
+{
+        struct obj *entry, *temp;
+
+        list_for_each_entry_safe (entry, temp, list, locked_list) {
+                /* need to do that before unlocking, since only the current lock holder is
+                allowed to use object */
+                list_del(&entry->locked_list);
+                ww_mutex_unlock(entry->ww_mutex)
+        }
+}
+
+void lock_objs(struct list_head *list, struct ww_acquire_ctx *ctx)
+{
+        struct obj *obj;
+
+        ww_acquire_init(ctx, &ww_class);
+
+retry:
+        /* re-init loop start state */
+        loop {
+                /* magic code which walks over a graph and decides which objects
+                 * to lock */
+
+                ret = ww_mutex_lock(obj->ww_mutex, ctx);
+                if (ret == -EALREADY) {
+                        /* we have that one already, get to the next object */
+                        continue;
+                }
+                if (ret == -EDEADLK) {
+                        __unlock_objs(list);
+
+                        ww_mutex_lock_slow(obj, ctx);
+                        list_add(&entry->locked_list, list);
+                        goto retry;
+                }
+
+                /* locked a new object, add it to the list */
+                list_add_tail(&entry->locked_list, list);
+        }
+
+        ww_acquire_done(ctx);
+        return 0;
+}
+
+void unlock_objs(struct list_head *list, struct ww_acquire_ctx *ctx)
+{
+        __unlock_objs(list);
+        ww_acquire_fini(ctx);
+}
+
+Method 4: Only lock one single objects. In that case deadlock detection and
+prevention is obviously overkill, since with grabbing just one lock you can't
+produce a deadlock within just one class. To simplify this case the w/w mutex
+api can be used with a NULL context.
+
+Implementation Details
+----------------------
+
+Design:
+  ww_mutex currently encapsulates a struct mutex, this means no extra overhead for
+  normal mutex locks, which are far more common. As such there is only a small
+  increase in code size if wait/wound mutexes are not used.
+
+  In general, not much contention is expected. The locks are typically used to
+  serialize access to resources for devices. The only way to make wakeups
+  smarter would be at the cost of adding a field to struct mutex_waiter. This
+  would add overhead to all cases where normal mutexes are used, and
+  ww_mutexes are generally less performance sensitive.
+
+Lockdep:
+  Special care has been taken to warn for as many cases of api abuse
+  as possible. Some common api abuses will be caught with
+  CONFIG_DEBUG_MUTEXES, but CONFIG_PROVE_LOCKING is recommended.
+
+  Some of the errors which will be warned about:
+   - Forgetting to call ww_acquire_fini or ww_acquire_init.
+   - Attempting to lock more mutexes after ww_acquire_done.
+   - Attempting to lock the wrong mutex after -EDEADLK and
+     unlocking all mutexes.
+   - Attempting to lock the right mutex after -EDEADLK,
+     before unlocking all mutexes.
+
+   - Calling ww_mutex_lock_slow before -EDEADLK was returned.
+
+   - Unlocking mutexes with the wrong unlock function.
+   - Calling one of the ww_acquire_* twice on the same context.
+   - Using a different ww_class for the mutex than for the ww_acquire_ctx.
+   - Normal lockdep errors that can result in deadlocks.
+
+  Some of the lockdep errors that can result in deadlocks:
+   - Calling ww_acquire_init to initialize a second ww_acquire_ctx before
+     having called ww_acquire_fini on the first.
+   - 'normal' deadlocks that can occur.
+
+FIXME: Update this section once we have the TASK_DEADLOCK task state flag magic
+implemented.
diff --git a/MAINTAINERS b/MAINTAINERS
index 5be702cc8449..437dd12ab9cf 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2697,12 +2697,14 @@ F:	include/drm/exynos*
 F:      include/uapi/drm/exynos*
 
 DRM DRIVERS FOR NVIDIA TEGRA
-M:      Thierry Reding <thierry.reding@avionic-design.de>
+M:      Thierry Reding <thierry.reding@gmail.com>
+M:      Terje Bergström <tbergstrom@nvidia.com>
 L:      dri-devel@lists.freedesktop.org
 L:      linux-tegra@vger.kernel.org
-T:      git git://gitorious.org/thierryreding/linux.git
+T:      git git://anongit.freedesktop.org/tegra/linux.git
 S:      Maintained
-F:      drivers/gpu/drm/tegra/
+F:      drivers/gpu/host1x/
+F:      include/uapi/drm/tegra_drm.h
 F:      Documentation/devicetree/bindings/gpu/nvidia,tegra20-host1x.txt
 
 DSBR100 USB FM RADIO DRIVER
diff --git a/arch/arm/boot/dts/cros5250-common.dtsi b/arch/arm/boot/dts/cros5250-common.dtsi
index 3f0239ec1bc5..dc259e8b8a73 100644
--- a/arch/arm/boot/dts/cros5250-common.dtsi
+++ b/arch/arm/boot/dts/cros5250-common.dtsi
@@ -190,7 +190,7 @@
                 samsung,i2c-max-bus-freq = <66000>;
 
                 hdmiddc@50 {
-                        compatible = "samsung,exynos5-hdmiddc";
+                        compatible = "samsung,exynos4210-hdmiddc";
                         reg = <0x50>;
                 };
         };
@@ -224,7 +224,7 @@
                 samsung,i2c-max-bus-freq = <378000>;
 
                 hdmiphy@38 {
-                        compatible = "samsung,exynos5-hdmiphy";
+                        compatible = "samsung,exynos4212-hdmiphy";
                         reg = <0x38>;
                 };
         };
diff --git a/arch/arm/boot/dts/exynos5250-smdk5250.dts b/arch/arm/boot/dts/exynos5250-smdk5250.dts
index 3e0c792e2767..f320d7cb4174 100644
--- a/arch/arm/boot/dts/exynos5250-smdk5250.dts
+++ b/arch/arm/boot/dts/exynos5250-smdk5250.dts
@@ -72,7 +72,7 @@
                 samsung,i2c-max-bus-freq = <66000>;
 
                 hdmiddc@50 {
-                        compatible = "samsung,exynos5-hdmiddc";
+                        compatible = "samsung,exynos4210-hdmiddc";
                         reg = <0x50>;
                 };
         };
@@ -102,7 +102,7 @@
                 samsung,i2c-max-bus-freq = <66000>;
 
                 hdmiphy@38 {
-                        compatible = "samsung,exynos5-hdmiphy";
+                        compatible = "samsung,exynos4212-hdmiphy";
                         reg = <0x38>;
                 };
         };
diff --git a/arch/arm/boot/dts/exynos5250.dtsi b/arch/arm/boot/dts/exynos5250.dtsi
index fc9fb3d526e2..8b815c945949 100644
--- a/arch/arm/boot/dts/exynos5250.dtsi
+++ b/arch/arm/boot/dts/exynos5250.dtsi
@@ -601,7 +601,7 @@
         };
 
         hdmi {
-                compatible = "samsung,exynos5-hdmi";
+                compatible = "samsung,exynos4212-hdmi";
                 reg = <0x14530000 0x70000>;
                 interrupts = <0 95 0>;
                 clocks = <&clock 333>, <&clock 136>, <&clock 137>,
@@ -611,7 +611,7 @@
         };
 
         mixer {
-                compatible = "samsung,exynos5-mixer";
+                compatible = "samsung,exynos5250-mixer";
                 reg = <0x14450000 0x10000>;
                 interrupts = <0 94 0>;
         };
diff --git a/arch/ia64/include/asm/mutex.h b/arch/ia64/include/asm/mutex.h
index bed73a643a56..f41e66d65e31 100644
--- a/arch/ia64/include/asm/mutex.h
+++ b/arch/ia64/include/asm/mutex.h
@@ -29,17 +29,15 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  *                                 from 1 to a 0 value
  *  @count: pointer of type atomic_t
- *  @fail_fn: function to call if the original value was not 1
  *
- * Change the count from 1 to a value lower than 1, and call <fail_fn> if
- * it wasn't 1 originally. This function returns 0 if the fastpath succeeds,
- * or anything the slow path function returns.
+ * Change the count from 1 to a value lower than 1. This function returns 0
+ * if the fastpath succeeds, or -1 otherwise.
  */
 static inline int
-__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
+__mutex_fastpath_lock_retval(atomic_t *count)
 {
         if (unlikely(ia64_fetchadd4_acq(count, -1) != 1))
-                return fail_fn(count);
+                return -1;
         return 0;
 }
 
diff --git a/arch/powerpc/include/asm/mutex.h b/arch/powerpc/include/asm/mutex.h
index 5399f7e18102..127ab23e1f6c 100644
--- a/arch/powerpc/include/asm/mutex.h
+++ b/arch/powerpc/include/asm/mutex.h
@@ -82,17 +82,15 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  *                                 from 1 to a 0 value
  *  @count: pointer of type atomic_t
- *  @fail_fn: function to call if the original value was not 1
  *
- * Change the count from 1 to a value lower than 1, and call <fail_fn> if
- * it wasn't 1 originally. This function returns 0 if the fastpath succeeds,
- * or anything the slow path function returns.
+ * Change the count from 1 to a value lower than 1. This function returns 0
+ * if the fastpath succeeds, or -1 otherwise.
  */
 static inline int
-__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
+__mutex_fastpath_lock_retval(atomic_t *count)
 {
         if (unlikely(__mutex_dec_return_lock(count) < 0))
-                return fail_fn(count);
+                return -1;
         return 0;
 }
 
diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c
index 028ac1f71b51..46ac1ddea683 100644
--- a/arch/powerpc/sysdev/fsl_pci.c
+++ b/arch/powerpc/sysdev/fsl_pci.c
@@ -97,22 +97,14 @@ static int fsl_indirect_read_config(struct pci_bus *bus, unsigned int devfn,
         return indirect_read_config(bus, devfn, offset, len, val);
 }
 
-static struct pci_ops fsl_indirect_pci_ops =
+#if defined(CONFIG_FSL_SOC_BOOKE) || defined(CONFIG_PPC_86xx)
+
+static struct pci_ops fsl_indirect_pcie_ops =
 {
         .read = fsl_indirect_read_config,
         .write = indirect_write_config,
 };
 
-static void __init fsl_setup_indirect_pci(struct pci_controller* hose,
-                                          resource_size_t cfg_addr,
-                                          resource_size_t cfg_data, u32 flags)
-{
-        setup_indirect_pci(hose, cfg_addr, cfg_data, flags);
-        hose->ops = &fsl_indirect_pci_ops;
-}
-
-#if defined(CONFIG_FSL_SOC_BOOKE) || defined(CONFIG_PPC_86xx)
-
 #define MAX_PHYS_ADDR_BITS      40
 static u64 pci64_dma_offset = 1ull << MAX_PHYS_ADDR_BITS;
 
@@ -504,13 +496,15 @@ int __init fsl_add_bridge(struct platform_device *pdev, int is_primary)
         if (!hose->private_data)
                 goto no_bridge;
 
-        fsl_setup_indirect_pci(hose, rsrc.start, rsrc.start + 0x4,
-                               PPC_INDIRECT_TYPE_BIG_ENDIAN);
+        setup_indirect_pci(hose, rsrc.start, rsrc.start + 0x4,
+                           PPC_INDIRECT_TYPE_BIG_ENDIAN);
 
         if (in_be32(&pci->block_rev1) < PCIE_IP_REV_3_0)
                 hose->indirect_type |= PPC_INDIRECT_TYPE_FSL_CFG_REG_LINK;
 
         if (early_find_capability(hose, 0, 0, PCI_CAP_ID_EXP)) {
+                /* use fsl_indirect_read_config for PCIe */
+                hose->ops = &fsl_indirect_pcie_ops;
                 /* For PCIE read HEADER_TYPE to identify controler mode */
                 early_read_config_byte(hose, 0, 0, PCI_HEADER_TYPE, &hdr_type);
                 if ((hdr_type & 0x7f) != PCI_HEADER_TYPE_BRIDGE)
@@ -814,8 +808,8 @@ int __init mpc83xx_add_bridge(struct device_node *dev)
                 if (ret)
                         goto err0;
         } else {
-                fsl_setup_indirect_pci(hose, rsrc_cfg.start,
-                                       rsrc_cfg.start + 4, 0);
+                setup_indirect_pci(hose, rsrc_cfg.start,
+                                   rsrc_cfg.start + 4, 0);
         }
 
         printk(KERN_INFO "Found FSL PCI host bridge at 0x%016llx. "
diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h
index 886ac7d4937a..2f8c1abeb086 100644
--- a/arch/s390/include/asm/dma-mapping.h
+++ b/arch/s390/include/asm/dma-mapping.h
@@ -50,9 +50,10 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
         struct dma_map_ops *dma_ops = get_dma_ops(dev);
 
+        debug_dma_mapping_error(dev, dma_addr);
         if (dma_ops->mapping_error)
                 return dma_ops->mapping_error(dev, dma_addr);
-        return (dma_addr == 0UL);
+        return (dma_addr == DMA_ERROR_CODE);
 }
 
 static inline void *dma_alloc_coherent(struct device *dev, size_t size,
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index d8a6a385d048..feb719d3c851 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -754,9 +754,9 @@ static struct bin_attribute sys_reipl_fcp_scp_data_attr = {
         .write = reipl_fcp_scpdata_write,
 };
 
-DEFINE_IPL_ATTR_RW(reipl_fcp, wwpn, "0x%016llx\n", "%016llx\n",
+DEFINE_IPL_ATTR_RW(reipl_fcp, wwpn, "0x%016llx\n", "%llx\n",
                    reipl_block_fcp->ipl_info.fcp.wwpn);
-DEFINE_IPL_ATTR_RW(reipl_fcp, lun, "0x%016llx\n", "%016llx\n",
+DEFINE_IPL_ATTR_RW(reipl_fcp, lun, "0x%016llx\n", "%llx\n",
                    reipl_block_fcp->ipl_info.fcp.lun);
 DEFINE_IPL_ATTR_RW(reipl_fcp, bootprog, "%lld\n", "%lld\n",
                    reipl_block_fcp->ipl_info.fcp.bootprog);
@@ -1323,9 +1323,9 @@ static struct shutdown_action __refdata reipl_action = {
 
 /* FCP dump device attributes */
 
-DEFINE_IPL_ATTR_RW(dump_fcp, wwpn, "0x%016llx\n", "%016llx\n",
+DEFINE_IPL_ATTR_RW(dump_fcp, wwpn, "0x%016llx\n", "%llx\n",
                    dump_block_fcp->ipl_info.fcp.wwpn);
-DEFINE_IPL_ATTR_RW(dump_fcp, lun, "0x%016llx\n", "%016llx\n",
+DEFINE_IPL_ATTR_RW(dump_fcp, lun, "0x%016llx\n", "%llx\n",
                    dump_block_fcp->ipl_info.fcp.lun);
 DEFINE_IPL_ATTR_RW(dump_fcp, bootprog, "%lld\n", "%lld\n",
                    dump_block_fcp->ipl_info.fcp.bootprog);
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index 408e866ae548..dd3c1994b8bd 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -312,6 +312,7 @@ void measurement_alert_subclass_unregister(void)
 }
 EXPORT_SYMBOL(measurement_alert_subclass_unregister);
 
+#ifdef CONFIG_SMP
 void synchronize_irq(unsigned int irq)
 {
         /*
@@ -320,6 +321,7 @@ void synchronize_irq(unsigned int irq)
          */
 }
 EXPORT_SYMBOL_GPL(synchronize_irq);
+#endif
 
 #ifndef CONFIG_PCI
 
diff --git a/arch/s390/mm/mem_detect.c b/arch/s390/mm/mem_detect.c
index 3cbd3b8bf311..cca388253a39 100644
--- a/arch/s390/mm/mem_detect.c
+++ b/arch/s390/mm/mem_detect.c
@@ -123,7 +123,8 @@ void create_mem_hole(struct mem_chunk mem_chunk[], unsigned long addr,
                         continue;
                 } else if ((addr <= chunk->addr) &&
                            (addr + size >= chunk->addr + chunk->size)) {
-                        memset(chunk, 0 , sizeof(*chunk));
+                        memmove(chunk, chunk + 1, (MEMORY_CHUNKS-i-1) * sizeof(*chunk));
+                        memset(&mem_chunk[MEMORY_CHUNKS-1], 0, sizeof(*chunk));
                 } else if (addr + size < chunk->addr + chunk->size) {
                         chunk->size =  chunk->addr + chunk->size - addr - size;
                         chunk->addr = addr + size;
diff --git a/arch/sh/include/asm/mutex-llsc.h b/arch/sh/include/asm/mutex-llsc.h
index 090358a7e1bb..dad29b687bd3 100644
--- a/arch/sh/include/asm/mutex-llsc.h
+++ b/arch/sh/include/asm/mutex-llsc.h
@@ -37,7 +37,7 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
 }
 
 static inline int
-__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
+__mutex_fastpath_lock_retval(atomic_t *count)
 {
         int __done, __res;
 
@@ -51,7 +51,7 @@ __mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
                 : "t");
 
         if (unlikely(!__done || __res != 0))
-                __res = fail_fn(count);
+                __res = -1;
 
         return __res;
 }
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index d8e8eefbe24c..34f69cb9350a 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -345,4 +345,11 @@ extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
 
 #define IO_SPACE_LIMIT 0xffff
 
+#ifdef CONFIG_MTRR
+extern int __must_check arch_phys_wc_add(unsigned long base,
+                                         unsigned long size);
+extern void arch_phys_wc_del(int handle);
+#define arch_phys_wc_add arch_phys_wc_add
+#endif
+
 #endif /* _ASM_X86_IO_H */
diff --git a/arch/x86/include/asm/mtrr.h b/arch/x86/include/asm/mtrr.h
index e235582f9930..f768f6298419 100644
--- a/arch/x86/include/asm/mtrr.h
+++ b/arch/x86/include/asm/mtrr.h
@@ -26,7 +26,10 @@
 #include <uapi/asm/mtrr.h>
 
 
-/*  The following functions are for use by other drivers  */
+/*
+ * The following functions are for use by other drivers that cannot use
+ * arch_phys_wc_add and arch_phys_wc_del.
+ */
 # ifdef CONFIG_MTRR
 extern u8 mtrr_type_lookup(u64 addr, u64 end);
 extern void mtrr_save_fixed_ranges(void *);
@@ -45,6 +48,7 @@ extern void mtrr_aps_init(void);
 extern void mtrr_bp_restore(void);
 extern int mtrr_trim_uncached_memory(unsigned long end_pfn);
 extern int amd_special_default_mtrr(void);
+extern int phys_wc_to_mtrr_index(int handle);
 #  else
 static inline u8 mtrr_type_lookup(u64 addr, u64 end)
 {
@@ -80,6 +84,10 @@ static inline int mtrr_trim_uncached_memory(unsigned long end_pfn)
 static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
 {
 }
+static inline int phys_wc_to_mtrr_index(int handle)
+{
+        return -1;
+}
 
 #define mtrr_ap_init() do {} while (0)
 #define mtrr_bp_init() do {} while (0)
diff --git a/arch/x86/include/asm/mutex_32.h b/arch/x86/include/asm/mutex_32.h
index 03f90c8a5a7c..0208c3c2cbc6 100644
--- a/arch/x86/include/asm/mutex_32.h
+++ b/arch/x86/include/asm/mutex_32.h
@@ -42,17 +42,14 @@ do {								\
  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  *                                 from 1 to a 0 value
  *  @count: pointer of type atomic_t
- *  @fail_fn: function to call if the original value was not 1
  *
- * Change the count from 1 to a value lower than 1, and call <fail_fn> if it
- * wasn't 1 originally. This function returns 0 if the fastpath succeeds,
- * or anything the slow path function returns
+ * Change the count from 1 to a value lower than 1. This function returns 0
+ * if the fastpath succeeds, or -1 otherwise.
  */
-static inline int __mutex_fastpath_lock_retval(atomic_t *count,
-                                               int (*fail_fn)(atomic_t *))
+static inline int __mutex_fastpath_lock_retval(atomic_t *count)
 {
         if (unlikely(atomic_dec_return(count) < 0))
-                return fail_fn(count);
+                return -1;
         else
                 return 0;
 }
diff --git a/arch/x86/include/asm/mutex_64.h b/arch/x86/include/asm/mutex_64.h
index 68a87b0f8e29..2c543fff241b 100644
--- a/arch/x86/include/asm/mutex_64.h
+++ b/arch/x86/include/asm/mutex_64.h
@@ -37,17 +37,14 @@ do {								\
  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  *                                 from 1 to a 0 value
  *  @count: pointer of type atomic_t
- *  @fail_fn: function to call if the original value was not 1
  *
- * Change the count from 1 to a value lower than 1, and call <fail_fn> if
- * it wasn't 1 originally. This function returns 0 if the fastpath succeeds,
- * or anything the slow path function returns
+ * Change the count from 1 to a value lower than 1. This function returns 0
+ * if the fastpath succeeds, or -1 otherwise.
  */
-static inline int __mutex_fastpath_lock_retval(atomic_t *count,
-                                               int (*fail_fn)(atomic_t *))
+static inline int __mutex_fastpath_lock_retval(atomic_t *count)
 {
         if (unlikely(atomic_dec_return(count) < 0))
-                return fail_fn(count);
+                return -1;
         else
                 return 0;
 }
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 726bf963c227..3533d4d16f8c 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -51,9 +51,13 @@
 #include <asm/e820.h>
 #include <asm/mtrr.h>
 #include <asm/msr.h>
+#include <asm/pat.h>
 
 #include "mtrr.h"
 
+/* arch_phys_wc_add returns an MTRR register index plus this offset. */
+#define MTRR_TO_PHYS_WC_OFFSET 1000
+
 u32 num_var_ranges;
 
 unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
@@ -524,6 +528,73 @@ int mtrr_del(int reg, unsigned long base, unsigned long size)
 }
 EXPORT_SYMBOL(mtrr_del);
 
+/**
+ * arch_phys_wc_add - add a WC MTRR and handle errors if PAT is unavailable
+ * @base: Physical base address
+ * @size: Size of region
+ *
+ * If PAT is available, this does nothing.  If PAT is unavailable, it
+ * attempts to add a WC MTRR covering size bytes starting at base and
+ * logs an error if this fails.
+ *
+ * Drivers must store the return value to pass to mtrr_del_wc_if_needed,
+ * but drivers should not try to interpret that return value.
+ */
+int arch_phys_wc_add(unsigned long base, unsigned long size)
+{
+        int ret;
+
+        if (pat_enabled)
+                return 0;  /* Success!  (We don't need to do anything.) */
+
+        ret = mtrr_add(base, size, MTRR_TYPE_WRCOMB, true);
+        if (ret < 0) {
+                pr_warn("Failed to add WC MTRR for [%p-%p]; performance may suffer.",
+                        (void *)base, (void *)(base + size - 1));
+                return ret;
+        }
+        return ret + MTRR_TO_PHYS_WC_OFFSET;
+}
+EXPORT_SYMBOL(arch_phys_wc_add);
+
+/*
+ * arch_phys_wc_del - undoes arch_phys_wc_add
+ * @handle: Return value from arch_phys_wc_add
+ *
+ * This cleans up after mtrr_add_wc_if_needed.
+ *
+ * The API guarantees that mtrr_del_wc_if_needed(error code) and
+ * mtrr_del_wc_if_needed(0) do nothing.
+ */
+void arch_phys_wc_del(int handle)
+{
+        if (handle >= 1) {
+                WARN_ON(handle < MTRR_TO_PHYS_WC_OFFSET);
+                mtrr_del(handle - MTRR_TO_PHYS_WC_OFFSET, 0, 0);
+        }
+}
+EXPORT_SYMBOL(arch_phys_wc_del);
+
+/*
+ * phys_wc_to_mtrr_index - translates arch_phys_wc_add's return value
+ * @handle: Return value from arch_phys_wc_add
+ *
+ * This will turn the return value from arch_phys_wc_add into an mtrr
+ * index suitable for debugging.
+ *
+ * Note: There is no legitimate use for this function, except possibly
+ * in printk line.  Alas there is an illegitimate use in some ancient
+ * drm ioctls.
+ */
+int phys_wc_to_mtrr_index(int handle)
+{
+        if (handle < MTRR_TO_PHYS_WC_OFFSET)
+                return -1;
+        else
+                return handle - MTRR_TO_PHYS_WC_OFFSET;
+}
+EXPORT_SYMBOL_GPL(phys_wc_to_mtrr_index);
+
 /*
  * HACK ALERT!
  * These should be called implicitly, but we can't yet until all the initcall
diff --git a/drivers/base/Makefile b/drivers/base/Makefile
index 4e22ce3ed73d..48029aa477d9 100644
--- a/drivers/base/Makefile
+++ b/drivers/base/Makefile
@@ -10,7 +10,7 @@ obj-$(CONFIG_CMA) += dma-contiguous.o
 obj-y                   += power/
 obj-$(CONFIG_HAS_DMA)   += dma-mapping.o
 obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o
-obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf.o
+obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf.o reservation.o
 obj-$(CONFIG_ISA)       += isa.o
 obj-$(CONFIG_FW_LOADER) += firmware_class.o
 obj-$(CONFIG_NUMA)      += node.o
diff --git a/drivers/base/reservation.c b/drivers/base/reservation.c
new file mode 100644
index 000000000000..a73fbf3b8e56
--- /dev/null
+++ b/drivers/base/reservation.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2012-2013 Canonical Ltd
+ *
+ * Based on bo.c which bears the following copyright notice,
+ * but is dual licensed:
+ *
+ * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#include <linux/reservation.h>
+#include <linux/export.h>
+
+DEFINE_WW_CLASS(reservation_ww_class);
+EXPORT_SYMBOL(reservation_ww_class);
diff --git a/drivers/char/agp/ati-agp.c b/drivers/char/agp/ati-agp.c
index 0628d7b65c71..03c1dc1ab552 100644
--- a/drivers/char/agp/ati-agp.c
+++ b/drivers/char/agp/ati-agp.c
@@ -236,14 +236,14 @@ static int ati_configure(void)
 static int agp_ati_suspend(struct pci_dev *dev, pm_message_t state)
 {
         pci_save_state(dev);
-        pci_set_power_state(dev, 3);
+        pci_set_power_state(dev, PCI_D3hot);
 
         return 0;
 }
 
 static int agp_ati_resume(struct pci_dev *dev)
 {
-        pci_set_power_state(dev, 0);
+        pci_set_power_state(dev, PCI_D0);
         pci_restore_state(dev);
 
         return ati_configure();
diff --git a/drivers/char/agp/frontend.c b/drivers/char/agp/frontend.c
index 2e044338753c..1b192395a90c 100644
--- a/drivers/char/agp/frontend.c
+++ b/drivers/char/agp/frontend.c
@@ -603,7 +603,8 @@ static int agp_mmap(struct file *file, struct vm_area_struct *vma)
                         vma->vm_ops = kerninfo.vm_ops;
                 } else if (io_remap_pfn_range(vma, vma->vm_start,
                                 (kerninfo.aper_base + offset) >> PAGE_SHIFT,
-                                            size, vma->vm_page_prot)) {
+                                size,
+                                pgprot_writecombine(vma->vm_page_prot))) {
                         goto out_again;
                 }
                 mutex_unlock(&(agp_fe.agp_mutex));
@@ -618,8 +619,9 @@ static int agp_mmap(struct file *file, struct vm_area_struct *vma)
                 if (kerninfo.vm_ops) {
                         vma->vm_ops = kerninfo.vm_ops;
                 } else if (io_remap_pfn_range(vma, vma->vm_start,
-                                            kerninfo.aper_base >> PAGE_SHIFT,
-                                            size, vma->vm_page_prot)) {
+                                kerninfo.aper_base >> PAGE_SHIFT,
+                                size,
+                                pgprot_writecombine(vma->vm_page_prot))) {
                         goto out_again;
                 }
                 mutex_unlock(&(agp_fe.agp_mutex));
diff --git a/drivers/char/agp/nvidia-agp.c b/drivers/char/agp/nvidia-agp.c
index 62be3ec0da4b..be42a2312dc9 100644
--- a/drivers/char/agp/nvidia-agp.c
+++ b/drivers/char/agp/nvidia-agp.c
@@ -399,8 +399,8 @@ static void agp_nvidia_remove(struct pci_dev *pdev)
 #ifdef CONFIG_PM
 static int agp_nvidia_suspend(struct pci_dev *pdev, pm_message_t state)
 {
-        pci_save_state (pdev);
-        pci_set_power_state (pdev, 3);
+        pci_save_state(pdev);
+        pci_set_power_state(pdev, PCI_D3hot);
 
         return 0;
 }
@@ -408,7 +408,7 @@ static int agp_nvidia_suspend(struct pci_dev *pdev, pm_message_t state)
 static int agp_nvidia_resume(struct pci_dev *pdev)
 {
         /* set power state 0 and restore PCI space */
-        pci_set_power_state (pdev, 0);
+        pci_set_power_state(pdev, PCI_D0);
         pci_restore_state(pdev);
 
         /* reconfigure AGP hardware again */
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index b16c50ee769c..a7c54c843291 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -139,6 +139,7 @@ config DRM_I915
         select BACKLIGHT_CLASS_DEVICE if ACPI
         select VIDEO_OUTPUT_CONTROL if ACPI
         select INPUT if ACPI
+        select THERMAL if ACPI
         select ACPI_VIDEO if ACPI
         select ACPI_BUTTON if ACPI
         help
@@ -213,6 +214,8 @@ source "drivers/gpu/drm/mgag200/Kconfig"
 
 source "drivers/gpu/drm/cirrus/Kconfig"
 
+source "drivers/gpu/drm/rcar-du/Kconfig"
+
 source "drivers/gpu/drm/shmobile/Kconfig"
 
 source "drivers/gpu/drm/omapdrm/Kconfig"
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 1c9f24396002..801bcafa3028 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -12,7 +12,8 @@ drm-y       :=	drm_auth.o drm_buffer.o drm_bufs.o drm_cache.o \
                 drm_platform.o drm_sysfs.o drm_hashtab.o drm_mm.o \
                 drm_crtc.o drm_modes.o drm_edid.o \
                 drm_info.o drm_debugfs.o drm_encoder_slave.o \
-                drm_trace_points.o drm_global.o drm_prime.o
+                drm_trace_points.o drm_global.o drm_prime.o \
+                drm_rect.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o
@@ -48,6 +49,7 @@ obj-$(CONFIG_DRM_EXYNOS) +=exynos/
 obj-$(CONFIG_DRM_GMA500) += gma500/
 obj-$(CONFIG_DRM_UDL) += udl/
 obj-$(CONFIG_DRM_AST) += ast/
+obj-$(CONFIG_DRM_RCAR_DU) += rcar-du/
 obj-$(CONFIG_DRM_SHMOBILE) +=shmobile/
 obj-$(CONFIG_DRM_OMAP)  += omapdrm/
 obj-$(CONFIG_DRM_TILCDC)        += tilcdc/
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index 02e52d543e4b..622d4ae7eb9e 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -348,8 +348,24 @@ int ast_gem_create(struct drm_device *dev,
 int ast_bo_pin(struct ast_bo *bo, u32 pl_flag, u64 *gpu_addr);
 int ast_bo_unpin(struct ast_bo *bo);
 
-int ast_bo_reserve(struct ast_bo *bo, bool no_wait);
-void ast_bo_unreserve(struct ast_bo *bo);
+static inline int ast_bo_reserve(struct ast_bo *bo, bool no_wait)
+{
+        int ret;
+
+        ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
+        if (ret) {
+                if (ret != -ERESTARTSYS && ret != -EBUSY)
+                        DRM_ERROR("reserve failed %p\n", bo);
+                return ret;
+        }
+        return 0;
+}
+
+static inline void ast_bo_unreserve(struct ast_bo *bo)
+{
+        ttm_bo_unreserve(&bo->bo);
+}
+
 void ast_ttm_placement(struct ast_bo *bo, int domain);
 int ast_bo_push_sysram(struct ast_bo *bo);
 int ast_mmap(struct file *filp, struct vm_area_struct *vma);
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index fbc0823cfa18..7b33e14e44aa 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -51,7 +51,7 @@ static void ast_dirty_update(struct ast_fbdev *afbdev,
         struct ast_bo *bo;
         int src_offset, dst_offset;
         int bpp = (afbdev->afb.base.bits_per_pixel + 7)/8;
-        int ret;
+        int ret = -EBUSY;
         bool unmap = false;
         bool store_for_later = false;
         int x2, y2;
@@ -65,7 +65,8 @@ static void ast_dirty_update(struct ast_fbdev *afbdev,
          * then the BO is being moved and we should
          * store up the damage until later.
          */
-        ret = ast_bo_reserve(bo, true);
+        if (!in_interrupt())
+                ret = ast_bo_reserve(bo, true);
         if (ret) {
                 if (ret != -EBUSY)
                         return;
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 09da3393c527..98d670825a1a 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -271,26 +271,19 @@ int ast_mm_init(struct ast_private *ast)
                 return ret;
         }
 
-        ast->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 0),
-                                    pci_resource_len(dev->pdev, 0),
-                                    DRM_MTRR_WC);
+        ast->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0),
+                                        pci_resource_len(dev->pdev, 0));
 
         return 0;
 }
 
 void ast_mm_fini(struct ast_private *ast)
 {
-        struct drm_device *dev = ast->dev;
         ttm_bo_device_release(&ast->ttm.bdev);
 
         ast_ttm_global_release(ast);
 
-        if (ast->fb_mtrr >= 0) {
-                drm_mtrr_del(ast->fb_mtrr,
-                             pci_resource_start(dev->pdev, 0),
-                             pci_resource_len(dev->pdev, 0), DRM_MTRR_WC);
-                ast->fb_mtrr = -1;
-        }
+        arch_phys_wc_del(ast->fb_mtrr);
 }
 
 void ast_ttm_placement(struct ast_bo *bo, int domain)
@@ -310,24 +303,6 @@ void ast_ttm_placement(struct ast_bo *bo, int domain)
         bo->placement.num_busy_placement = c;
 }
 
-int ast_bo_reserve(struct ast_bo *bo, bool no_wait)
-{
-        int ret;
-
-        ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
-        if (ret) {
-                if (ret != -ERESTARTSYS && ret != -EBUSY)
-                        DRM_ERROR("reserve failed %p\n", bo);
-                return ret;
-        }
-        return 0;
-}
-
-void ast_bo_unreserve(struct ast_bo *bo)
-{
-        ttm_bo_unreserve(&bo->bo);
-}
-
 int ast_bo_create(struct drm_device *dev, int size, int align,
                   uint32_t flags, struct ast_bo **pastbo)
 {
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.h b/drivers/gpu/drm/cirrus/cirrus_drv.h
index 7ca059596887..bae55609e6c3 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.h
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.h
@@ -240,8 +240,25 @@ void cirrus_ttm_placement(struct cirrus_bo *bo, int domain);
 int cirrus_bo_create(struct drm_device *dev, int size, int align,
                      uint32_t flags, struct cirrus_bo **pcirrusbo);
 int cirrus_mmap(struct file *filp, struct vm_area_struct *vma);
-int cirrus_bo_reserve(struct cirrus_bo *bo, bool no_wait);
-void cirrus_bo_unreserve(struct cirrus_bo *bo);
+
+static inline int cirrus_bo_reserve(struct cirrus_bo *bo, bool no_wait)
+{
+        int ret;
+
+        ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
+        if (ret) {
+                if (ret != -ERESTARTSYS && ret != -EBUSY)
+                        DRM_ERROR("reserve failed %p\n", bo);
+                return ret;
+        }
+        return 0;
+}
+
+static inline void cirrus_bo_unreserve(struct cirrus_bo *bo)
+{
+        ttm_bo_unreserve(&bo->bo);
+}
+
 int cirrus_bo_push_sysram(struct cirrus_bo *bo);
 int cirrus_bo_pin(struct cirrus_bo *bo, u32 pl_flag, u64 *gpu_addr);
 #endif                          /* __CIRRUS_DRV_H__ */
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
index 3541b567bbd8..b27e95666fab 100644
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
@@ -25,7 +25,7 @@ static void cirrus_dirty_update(struct cirrus_fbdev *afbdev,
         struct cirrus_bo *bo;
         int src_offset, dst_offset;
         int bpp = (afbdev->gfb.base.bits_per_pixel + 7)/8;
-        int ret;
+        int ret = -EBUSY;
         bool unmap = false;
         bool store_for_later = false;
         int x2, y2;
@@ -39,7 +39,8 @@ static void cirrus_dirty_update(struct cirrus_fbdev *afbdev,
          * then the BO is being moved and we should
          * store up the damage until later.
          */
-        ret = cirrus_bo_reserve(bo, true);
+        if (!in_interrupt())
+                ret = cirrus_bo_reserve(bo, true);
         if (ret) {
                 if (ret != -EBUSY)
                         return;
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 2ed8cfc740c9..0047012045c2 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -271,9 +271,8 @@ int cirrus_mm_init(struct cirrus_device *cirrus)
                 return ret;
         }
 
-        cirrus->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 0),
-                                    pci_resource_len(dev->pdev, 0),
-                                    DRM_MTRR_WC);
+        cirrus->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0),
+                                           pci_resource_len(dev->pdev, 0));
 
         cirrus->mm_inited = true;
         return 0;
@@ -281,8 +280,6 @@ int cirrus_mm_init(struct cirrus_device *cirrus)
 
 void cirrus_mm_fini(struct cirrus_device *cirrus)
 {
-        struct drm_device *dev = cirrus->dev;
-
         if (!cirrus->mm_inited)
                 return;
 
@@ -290,12 +287,8 @@ void cirrus_mm_fini(struct cirrus_device *cirrus)
 
         cirrus_ttm_global_release(cirrus);
 
-        if (cirrus->fb_mtrr >= 0) {
-                drm_mtrr_del(cirrus->fb_mtrr,
-                             pci_resource_start(dev->pdev, 0),
-                             pci_resource_len(dev->pdev, 0), DRM_MTRR_WC);
-                cirrus->fb_mtrr = -1;
-        }
+        arch_phys_wc_del(cirrus->fb_mtrr);
+        cirrus->fb_mtrr = 0;
 }
 
 void cirrus_ttm_placement(struct cirrus_bo *bo, int domain)
@@ -315,24 +308,6 @@ void cirrus_ttm_placement(struct cirrus_bo *bo, int domain)
         bo->placement.num_busy_placement = c;
 }
 
-int cirrus_bo_reserve(struct cirrus_bo *bo, bool no_wait)
-{
-        int ret;
-
-        ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
-        if (ret) {
-                if (ret != -ERESTARTSYS && ret != -EBUSY)
-                        DRM_ERROR("reserve failed %p\n", bo);
-                return ret;
-        }
-        return 0;
-}
-
-void cirrus_bo_unreserve(struct cirrus_bo *bo)
-{
-        ttm_bo_unreserve(&bo->bo);
-}
-
 int cirrus_bo_create(struct drm_device *dev, int size, int align,
                   uint32_t flags, struct cirrus_bo **pcirrusbo)
 {
diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c
index 0128147265f3..5a4dbb410b71 100644
--- a/drivers/gpu/drm/drm_bufs.c
+++ b/drivers/gpu/drm/drm_bufs.c
@@ -210,12 +210,16 @@ static int drm_addmap_core(struct drm_device * dev, resource_size_t offset,
                 if (drm_core_has_MTRR(dev)) {
                         if (map->type == _DRM_FRAME_BUFFER ||
                             (map->flags & _DRM_WRITE_COMBINING)) {
-                                map->mtrr = mtrr_add(map->offset, map->size,
-                                                     MTRR_TYPE_WRCOMB, 1);
+                                map->mtrr =
+                                        arch_phys_wc_add(map->offset, map->size);
                         }
                 }
                 if (map->type == _DRM_REGISTERS) {
-                        map->handle = ioremap(map->offset, map->size);
+                        if (map->flags & _DRM_WRITE_COMBINING)
+                                map->handle = ioremap_wc(map->offset,
+                                                         map->size);
+                        else
+                                map->handle = ioremap(map->offset, map->size);
                         if (!map->handle) {
                                 kfree(map);
                                 return -ENOMEM;
@@ -410,6 +414,15 @@ int drm_addmap_ioctl(struct drm_device *dev, void *data,
 
         /* avoid a warning on 64-bit, this casting isn't very nice, but the API is set so too late */
         map->handle = (void *)(unsigned long)maplist->user_token;
+
+        /*
+         * It appears that there are no users of this value whatsoever --
+         * drmAddMap just discards it.  Let's not encourage its use.
+         * (Keeping drm_addmap_core's returned mtrr value would be wrong --
+         *  it's not a real mtrr index anymore.)
+         */
+        map->mtrr = -1;
+
         return 0;
 }
 
@@ -451,11 +464,8 @@ int drm_rmmap_locked(struct drm_device *dev, struct drm_local_map *map)
                 iounmap(map->handle);
                 /* FALLTHROUGH */
         case _DRM_FRAME_BUFFER:
-                if (drm_core_has_MTRR(dev) && map->mtrr >= 0) {
-                        int retcode;
-                        retcode = mtrr_del(map->mtrr, map->offset, map->size);
-                        DRM_DEBUG("mtrr_del=%d\n", retcode);
-                }
+                if (drm_core_has_MTRR(dev))
+                        arch_phys_wc_del(map->mtrr);
                 break;
         case _DRM_SHM:
                 vfree(map->handle);
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index e7e92429d10f..fc83bb9eb514 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -29,6 +29,7 @@
  *      Dave Airlie <airlied@linux.ie>
  *      Jesse Barnes <jesse.barnes@intel.com>
  */
+#include <linux/ctype.h>
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/export.h>
@@ -91,7 +92,7 @@ EXPORT_SYMBOL(drm_warn_on_modeset_not_all_locked);
 
 /* Avoid boilerplate.  I'm tired of typing. */
 #define DRM_ENUM_NAME_FN(fnname, list)                          \
-        char *fnname(int val)                                   \
+        const char *fnname(int val)                             \
         {                                                       \
                 int i;                                          \
                 for (i = 0; i < ARRAY_SIZE(list); i++) {        \
@@ -104,7 +105,7 @@ EXPORT_SYMBOL(drm_warn_on_modeset_not_all_locked);
 /*
  * Global properties
  */
-static struct drm_prop_enum_list drm_dpms_enum_list[] =
+static const struct drm_prop_enum_list drm_dpms_enum_list[] =
 {       { DRM_MODE_DPMS_ON, "On" },
         { DRM_MODE_DPMS_STANDBY, "Standby" },
         { DRM_MODE_DPMS_SUSPEND, "Suspend" },
@@ -116,7 +117,7 @@ DRM_ENUM_NAME_FN(drm_get_dpms_name, drm_dpms_enum_list)
 /*
  * Optional properties
  */
-static struct drm_prop_enum_list drm_scaling_mode_enum_list[] =
+static const struct drm_prop_enum_list drm_scaling_mode_enum_list[] =
 {
         { DRM_MODE_SCALE_NONE, "None" },
         { DRM_MODE_SCALE_FULLSCREEN, "Full" },
@@ -124,7 +125,7 @@ static struct drm_prop_enum_list drm_scaling_mode_enum_list[] =
         { DRM_MODE_SCALE_ASPECT, "Full aspect" },
 };
 
-static struct drm_prop_enum_list drm_dithering_mode_enum_list[] =
+static const struct drm_prop_enum_list drm_dithering_mode_enum_list[] =
 {
         { DRM_MODE_DITHERING_OFF, "Off" },
         { DRM_MODE_DITHERING_ON, "On" },
@@ -134,7 +135,7 @@ static struct drm_prop_enum_list drm_dithering_mode_enum_list[] =
 /*
  * Non-global properties, but "required" for certain connectors.
  */
-static struct drm_prop_enum_list drm_dvi_i_select_enum_list[] =
+static const struct drm_prop_enum_list drm_dvi_i_select_enum_list[] =
 {
         { DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
         { DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
@@ -143,7 +144,7 @@ static struct drm_prop_enum_list drm_dvi_i_select_enum_list[] =
 
 DRM_ENUM_NAME_FN(drm_get_dvi_i_select_name, drm_dvi_i_select_enum_list)
 
-static struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] =
+static const struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] =
 {
         { DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I and TV-out */
         { DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
@@ -153,7 +154,7 @@ static struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] =
 DRM_ENUM_NAME_FN(drm_get_dvi_i_subconnector_name,
                  drm_dvi_i_subconnector_enum_list)
 
-static struct drm_prop_enum_list drm_tv_select_enum_list[] =
+static const struct drm_prop_enum_list drm_tv_select_enum_list[] =
 {
         { DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
         { DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
@@ -164,7 +165,7 @@ static struct drm_prop_enum_list drm_tv_select_enum_list[] =
 
 DRM_ENUM_NAME_FN(drm_get_tv_select_name, drm_tv_select_enum_list)
 
-static struct drm_prop_enum_list drm_tv_subconnector_enum_list[] =
+static const struct drm_prop_enum_list drm_tv_subconnector_enum_list[] =
 {
         { DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I and TV-out */
         { DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
@@ -176,7 +177,7 @@ static struct drm_prop_enum_list drm_tv_subconnector_enum_list[] =
 DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
                  drm_tv_subconnector_enum_list)
 
-static struct drm_prop_enum_list drm_dirty_info_enum_list[] = {
+static const struct drm_prop_enum_list drm_dirty_info_enum_list[] = {
         { DRM_MODE_DIRTY_OFF,      "Off"      },
         { DRM_MODE_DIRTY_ON,       "On"       },
         { DRM_MODE_DIRTY_ANNOTATE, "Annotate" },
@@ -184,7 +185,7 @@ static struct drm_prop_enum_list drm_dirty_info_enum_list[] = {
 
 struct drm_conn_prop_enum_list {
         int type;
-        char *name;
+        const char *name;
         int count;
 };
 
@@ -210,7 +211,7 @@ static struct drm_conn_prop_enum_list drm_connector_enum_list[] =
         { DRM_MODE_CONNECTOR_VIRTUAL, "Virtual", 0},
 };
 
-static struct drm_prop_enum_list drm_encoder_enum_list[] =
+static const struct drm_prop_enum_list drm_encoder_enum_list[] =
 {       { DRM_MODE_ENCODER_NONE, "None" },
         { DRM_MODE_ENCODER_DAC, "DAC" },
         { DRM_MODE_ENCODER_TMDS, "TMDS" },
@@ -219,7 +220,7 @@ static struct drm_prop_enum_list drm_encoder_enum_list[] =
         { DRM_MODE_ENCODER_VIRTUAL, "Virtual" },
 };
 
-char *drm_get_encoder_name(struct drm_encoder *encoder)
+const char *drm_get_encoder_name(const struct drm_encoder *encoder)
 {
         static char buf[32];
 
@@ -230,7 +231,7 @@ char *drm_get_encoder_name(struct drm_encoder *encoder)
 }
 EXPORT_SYMBOL(drm_get_encoder_name);
 
-char *drm_get_connector_name(struct drm_connector *connector)
+const char *drm_get_connector_name(const struct drm_connector *connector)
 {
         static char buf[32];
 
@@ -241,7 +242,7 @@ char *drm_get_connector_name(struct drm_connector *connector)
 }
 EXPORT_SYMBOL(drm_get_connector_name);
 
-char *drm_get_connector_status_name(enum drm_connector_status status)
+const char *drm_get_connector_status_name(enum drm_connector_status status)
 {
         if (status == connector_status_connected)
                 return "connected";
@@ -252,6 +253,28 @@ char *drm_get_connector_status_name(enum drm_connector_status status)
 }
 EXPORT_SYMBOL(drm_get_connector_status_name);
 
+static char printable_char(int c)
+{
+        return isascii(c) && isprint(c) ? c : '?';
+}
+
+const char *drm_get_format_name(uint32_t format)
+{
+        static char buf[32];
+
+        snprintf(buf, sizeof(buf),
+                 "%c%c%c%c %s-endian (0x%08x)",
+                 printable_char(format & 0xff),
+                 printable_char((format >> 8) & 0xff),
+                 printable_char((format >> 16) & 0xff),
+                 printable_char((format >> 24) & 0x7f),
+                 format & DRM_FORMAT_BIG_ENDIAN ? "big" : "little",
+                 format);
+
+        return buf;
+}
+EXPORT_SYMBOL(drm_get_format_name);
+
 /**
  * drm_mode_object_get - allocate a new modeset identifier
  * @dev: DRM device
@@ -569,16 +592,8 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb)
                 }
 
                 list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
-                        if (plane->fb == fb) {
-                                /* should turn off the crtc */
-                                ret = plane->funcs->disable_plane(plane);
-                                if (ret)
-                                        DRM_ERROR("failed to disable plane with busy fb\n");
-                                /* disconnect the plane from the fb and crtc: */
-                                __drm_framebuffer_unreference(plane->fb);
-                                plane->fb = NULL;
-                                plane->crtc = NULL;
-                        }
+                        if (plane->fb == fb)
+                                drm_plane_force_disable(plane);
                 }
                 drm_modeset_unlock_all(dev);
         }
@@ -593,7 +608,7 @@ EXPORT_SYMBOL(drm_framebuffer_remove);
  * @crtc: CRTC object to init
  * @funcs: callbacks for the new CRTC
  *
- * Inits a new object created as base part of an driver crtc object.
+ * Inits a new object created as base part of a driver crtc object.
  *
  * RETURNS:
  * Zero on success, error code on failure.
@@ -628,11 +643,12 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 EXPORT_SYMBOL(drm_crtc_init);
 
 /**
- * drm_crtc_cleanup - Cleans up the core crtc usage.
+ * drm_crtc_cleanup - Clean up the core crtc usage
  * @crtc: CRTC to cleanup
  *
- * Cleanup @crtc. Removes from drm modesetting space
- * does NOT free object, caller does that.
+ * This function cleans up @crtc and removes it from the DRM mode setting
+ * core. Note that the function does *not* free the crtc structure itself,
+ * this is the responsibility of the caller.
  */
 void drm_crtc_cleanup(struct drm_crtc *crtc)
 {
@@ -657,7 +673,7 @@ EXPORT_SYMBOL(drm_crtc_cleanup);
 void drm_mode_probed_add(struct drm_connector *connector,
                          struct drm_display_mode *mode)
 {
-        list_add(&mode->head, &connector->probed_modes);
+        list_add_tail(&mode->head, &connector->probed_modes);
 }
 EXPORT_SYMBOL(drm_mode_probed_add);
 
@@ -803,6 +819,21 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
 }
 EXPORT_SYMBOL(drm_encoder_cleanup);
 
+/**
+ * drm_plane_init - Initialise a new plane object
+ * @dev: DRM device
+ * @plane: plane object to init
+ * @possible_crtcs: bitmask of possible CRTCs
+ * @funcs: callbacks for the new plane
+ * @formats: array of supported formats (%DRM_FORMAT_*)
+ * @format_count: number of elements in @formats
+ * @priv: plane is private (hidden from userspace)?
+ *
+ * Inits a new object created as base part of a driver plane object.
+ *
+ * RETURNS:
+ * Zero on success, error code on failure.
+ */
 int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
                    unsigned long possible_crtcs,
                    const struct drm_plane_funcs *funcs,
@@ -851,6 +882,14 @@ int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_plane_init);
 
+/**
+ * drm_plane_cleanup - Clean up the core plane usage
+ * @plane: plane to cleanup
+ *
+ * This function cleans up @plane and removes it from the DRM mode setting
+ * core. Note that the function does *not* free the plane structure itself,
+ * this is the responsibility of the caller.
+ */
 void drm_plane_cleanup(struct drm_plane *plane)
 {
         struct drm_device *dev = plane->dev;
@@ -868,6 +907,32 @@ void drm_plane_cleanup(struct drm_plane *plane)
 EXPORT_SYMBOL(drm_plane_cleanup);
 
 /**
+ * drm_plane_force_disable - Forcibly disable a plane
+ * @plane: plane to disable
+ *
+ * Forces the plane to be disabled.
+ *
+ * Used when the plane's current framebuffer is destroyed,
+ * and when restoring fbdev mode.
+ */
+void drm_plane_force_disable(struct drm_plane *plane)
+{
+        int ret;
+
+        if (!plane->fb)
+                return;
+
+        ret = plane->funcs->disable_plane(plane);
+        if (ret)
+                DRM_ERROR("failed to disable plane with busy fb\n");
+        /* disconnect the plane from the fb and crtc: */
+        __drm_framebuffer_unreference(plane->fb);
+        plane->fb = NULL;
+        plane->crtc = NULL;
+}
+EXPORT_SYMBOL(drm_plane_force_disable);
+
+/**
  * drm_mode_create - create a new display mode
  * @dev: DRM device
  *
@@ -1740,7 +1805,7 @@ int drm_mode_getplane(struct drm_device *dev, void *data,
 
         plane_resp->plane_id = plane->base.id;
         plane_resp->possible_crtcs = plane->possible_crtcs;
-        plane_resp->gamma_size = plane->gamma_size;
+        plane_resp->gamma_size = 0;
 
         /*
          * This ioctl is called twice, once to determine how much space is
@@ -1834,7 +1899,8 @@ int drm_mode_setplane(struct drm_device *dev, void *data,
                 if (fb->pixel_format == plane->format_types[i])
                         break;
         if (i == plane->format_count) {
-                DRM_DEBUG_KMS("Invalid pixel format 0x%08x\n", fb->pixel_format);
+                DRM_DEBUG_KMS("Invalid pixel format %s\n",
+                              drm_get_format_name(fb->pixel_format));
                 ret = -EINVAL;
                 goto out;
         }
@@ -1906,18 +1972,31 @@ out:
 int drm_mode_set_config_internal(struct drm_mode_set *set)
 {
         struct drm_crtc *crtc = set->crtc;
-        struct drm_framebuffer *fb, *old_fb;
+        struct drm_framebuffer *fb;
+        struct drm_crtc *tmp;
         int ret;
 
-        old_fb = crtc->fb;
+        /*
+         * NOTE: ->set_config can also disable other crtcs (if we steal all
+         * connectors from it), hence we need to refcount the fbs across all
+         * crtcs. Atomic modeset will have saner semantics ...
+         */
+        list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head)
+                tmp->old_fb = tmp->fb;
+
         fb = set->fb;
 
         ret = crtc->funcs->set_config(set);
         if (ret == 0) {
-                if (old_fb)
-                        drm_framebuffer_unreference(old_fb);
-                if (fb)
-                        drm_framebuffer_reference(fb);
+                /* crtc->fb must be updated by ->set_config, enforces this. */
+                WARN_ON(fb != crtc->fb);
+        }
+
+        list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head) {
+                if (tmp->fb)
+                        drm_framebuffer_reference(tmp->fb);
+                if (tmp->old_fb)
+                        drm_framebuffer_unreference(tmp->old_fb);
         }
 
         return ret;
@@ -2099,10 +2178,10 @@ out:
         return ret;
 }
 
-int drm_mode_cursor_ioctl(struct drm_device *dev,
-                        void *data, struct drm_file *file_priv)
+static int drm_mode_cursor_common(struct drm_device *dev,
+                                  struct drm_mode_cursor2 *req,
+                                  struct drm_file *file_priv)
 {
-        struct drm_mode_cursor *req = data;
         struct drm_mode_object *obj;
         struct drm_crtc *crtc;
         int ret = 0;
@@ -2122,13 +2201,17 @@ int drm_mode_cursor_ioctl(struct drm_device *dev,
 
         mutex_lock(&crtc->mutex);
         if (req->flags & DRM_MODE_CURSOR_BO) {
-                if (!crtc->funcs->cursor_set) {
+                if (!crtc->funcs->cursor_set && !crtc->funcs->cursor_set2) {
                         ret = -ENXIO;
                         goto out;
                 }
                 /* Turns off the cursor if handle is 0 */
-                ret = crtc->funcs->cursor_set(crtc, file_priv, req->handle,
-                                              req->width, req->height);
+                if (crtc->funcs->cursor_set2)
+                        ret = crtc->funcs->cursor_set2(crtc, file_priv, req->handle,
+                                                      req->width, req->height, req->hot_x, req->hot_y);
+                else
+                        ret = crtc->funcs->cursor_set(crtc, file_priv, req->handle,
+                                                      req->width, req->height);
         }
 
         if (req->flags & DRM_MODE_CURSOR_MOVE) {
@@ -2143,6 +2226,25 @@ out:
         mutex_unlock(&crtc->mutex);
 
         return ret;
+
+}
+int drm_mode_cursor_ioctl(struct drm_device *dev,
+                        void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_cursor *req = data;
+        struct drm_mode_cursor2 new_req;
+
+        memcpy(&new_req, req, sizeof(struct drm_mode_cursor));
+        new_req.hot_x = new_req.hot_y = 0;
+
+        return drm_mode_cursor_common(dev, &new_req, file_priv);
+}
+
+int drm_mode_cursor2_ioctl(struct drm_device *dev,
+                           void *data, struct drm_file *file_priv)
+{
+        struct drm_mode_cursor2 *req = data;
+        return drm_mode_cursor_common(dev, req, file_priv);
 }
 
 /* Original addfb only supported RGB formats, so figure out which one */
@@ -2312,7 +2414,8 @@ static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
 
         ret = format_check(r);
         if (ret) {
-                DRM_DEBUG_KMS("bad framebuffer format 0x%08x\n", r->pixel_format);
+                DRM_DEBUG_KMS("bad framebuffer format %s\n",
+                              drm_get_format_name(r->pixel_format));
                 return ret;
         }
 
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index ed1334e27c33..738a4294d820 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -189,13 +189,14 @@ prune:
         if (list_empty(&connector->modes))
                 return 0;
 
+        list_for_each_entry(mode, &connector->modes, head)
+                mode->vrefresh = drm_mode_vrefresh(mode);
+
         drm_mode_sort(&connector->modes);
 
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] probed modes :\n", connector->base.id,
                         drm_get_connector_name(connector));
         list_for_each_entry(mode, &connector->modes, head) {
-                mode->vrefresh = drm_mode_vrefresh(mode);
-
                 drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
                 drm_mode_debug_printmodeline(mode);
         }
@@ -564,14 +565,13 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 
         DRM_DEBUG_KMS("\n");
 
-        if (!set)
-                return -EINVAL;
-
-        if (!set->crtc)
-                return -EINVAL;
+        BUG_ON(!set);
+        BUG_ON(!set->crtc);
+        BUG_ON(!set->crtc->helper_private);
 
-        if (!set->crtc->helper_private)
-                return -EINVAL;
+        /* Enforce sane interface api - has been abused by the fb helper. */
+        BUG_ON(!set->mode && set->fb);
+        BUG_ON(set->fb && set->num_connectors == 0);
 
         crtc_funcs = set->crtc->helper_private;
 
@@ -645,11 +645,6 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                         mode_changed = true;
                 } else if (set->fb == NULL) {
                         mode_changed = true;
-                } else if (set->fb->depth != set->crtc->fb->depth) {
-                        mode_changed = true;
-                } else if (set->fb->bits_per_pixel !=
-                           set->crtc->fb->bits_per_pixel) {
-                        mode_changed = true;
                 } else if (set->fb->pixel_format !=
                            set->crtc->fb->pixel_format) {
                         mode_changed = true;
@@ -759,12 +754,6 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                                 ret = -EINVAL;
                                 goto fail;
                         }
-                        DRM_DEBUG_KMS("Setting connector DPMS state to on\n");
-                        for (i = 0; i < set->num_connectors; i++) {
-                                DRM_DEBUG_KMS("\t[CONNECTOR:%d:%s] set DPMS on\n", set->connectors[i]->base.id,
-                                              drm_get_connector_name(set->connectors[i]));
-                                set->connectors[i]->funcs->dpms(set->connectors[i], DRM_MODE_DPMS_ON);
-                        }
                 }
                 drm_helper_disable_unused_functions(dev);
         } else if (fb_changed) {
@@ -782,6 +771,22 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                 }
         }
 
+        /*
+         * crtc set_config helpers implicit set the crtc and all connected
+         * encoders to DPMS on for a full mode set. But for just an fb update it
+         * doesn't do that. To not confuse userspace, do an explicit DPMS_ON
+         * unconditionally. This will also ensure driver internal dpms state is
+         * consistent again.
+         */
+        if (set->crtc->enabled) {
+                DRM_DEBUG_KMS("Setting connector DPMS state to on\n");
+                for (i = 0; i < set->num_connectors; i++) {
+                        DRM_DEBUG_KMS("\t[CONNECTOR:%d:%s] set DPMS on\n", set->connectors[i]->base.id,
+                                      drm_get_connector_name(set->connectors[i]));
+                        set->connectors[i]->funcs->dpms(set->connectors[i], DRM_MODE_DPMS_ON);
+                }
+        }
+
         kfree(save_connectors);
         kfree(save_encoders);
         kfree(save_crtcs);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 9cc247f55502..99fcd7c32ea2 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -166,6 +166,7 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+        DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR2, drm_mode_cursor2_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 };
 
 #define DRM_CORE_IOCTL_COUNT    ARRAY_SIZE( drm_ioctls )
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 9e62bbedb5ad..95d6f4b6967c 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -968,6 +968,9 @@ bool drm_edid_block_valid(u8 *raw_edid, int block, bool print_bad_edid)
         u8 csum = 0;
         struct edid *edid = (struct edid *)raw_edid;
 
+        if (WARN_ON(!raw_edid))
+                return false;
+
         if (edid_fixup > 8 || edid_fixup < 0)
                 edid_fixup = 6;
 
@@ -1010,15 +1013,15 @@ bool drm_edid_block_valid(u8 *raw_edid, int block, bool print_bad_edid)
                 break;
         }
 
-        return 1;
+        return true;
 
 bad:
-        if (raw_edid && print_bad_edid) {
+        if (print_bad_edid) {
                 printk(KERN_ERR "Raw EDID:\n");
                 print_hex_dump(KERN_ERR, " \t", DUMP_PREFIX_NONE, 16, 1,
                                raw_edid, EDID_LENGTH, false);
         }
-        return 0;
+        return false;
 }
 EXPORT_SYMBOL(drm_edid_block_valid);
 
@@ -1706,11 +1709,11 @@ static struct drm_display_mode *drm_mode_detailed(struct drm_device *dev,
                 return NULL;
 
         if (pt->misc & DRM_EDID_PT_STEREO) {
-                printk(KERN_WARNING "stereo mode not supported\n");
+                DRM_DEBUG_KMS("stereo mode not supported\n");
                 return NULL;
         }
         if (!(pt->misc & DRM_EDID_PT_SEPARATE_SYNC)) {
-                printk(KERN_WARNING "composite sync not supported\n");
+                DRM_DEBUG_KMS("composite sync not supported\n");
         }
 
         /* it is incorrect if hsync/vsync width is zero */
@@ -2321,6 +2324,31 @@ u8 *drm_find_cea_extension(struct edid *edid)
 }
 EXPORT_SYMBOL(drm_find_cea_extension);
 
+/*
+ * Calculate the alternate clock for the CEA mode
+ * (60Hz vs. 59.94Hz etc.)
+ */
+static unsigned int
+cea_mode_alternate_clock(const struct drm_display_mode *cea_mode)
+{
+        unsigned int clock = cea_mode->clock;
+
+        if (cea_mode->vrefresh % 6 != 0)
+                return clock;
+
+        /*
+         * edid_cea_modes contains the 59.94Hz
+         * variant for 240 and 480 line modes,
+         * and the 60Hz variant otherwise.
+         */
+        if (cea_mode->vdisplay == 240 || cea_mode->vdisplay == 480)
+                clock = clock * 1001 / 1000;
+        else
+                clock = DIV_ROUND_UP(clock * 1000, 1001);
+
+        return clock;
+}
+
 /**
  * drm_match_cea_mode - look for a CEA mode matching given mode
  * @to_match: display mode
@@ -2339,21 +2367,9 @@ u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
                 const struct drm_display_mode *cea_mode = &edid_cea_modes[mode];
                 unsigned int clock1, clock2;
 
-                clock1 = clock2 = cea_mode->clock;
-
                 /* Check both 60Hz and 59.94Hz */
-                if (cea_mode->vrefresh % 6 == 0) {
-                        /*
-                         * edid_cea_modes contains the 59.94Hz
-                         * variant for 240 and 480 line modes,
-                         * and the 60Hz variant otherwise.
-                         */
-                        if (cea_mode->vdisplay == 240 ||
-                            cea_mode->vdisplay == 480)
-                                clock1 = clock1 * 1001 / 1000;
-                        else
-                                clock2 = DIV_ROUND_UP(clock2 * 1000, 1001);
-                }
+                clock1 = cea_mode->clock;
+                clock2 = cea_mode_alternate_clock(cea_mode);
 
                 if ((KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock1) ||
                      KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock2)) &&
@@ -2364,6 +2380,66 @@ u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 }
 EXPORT_SYMBOL(drm_match_cea_mode);
 
+static int
+add_alternate_cea_modes(struct drm_connector *connector, struct edid *edid)
+{
+        struct drm_device *dev = connector->dev;
+        struct drm_display_mode *mode, *tmp;
+        LIST_HEAD(list);
+        int modes = 0;
+
+        /* Don't add CEA modes if the CEA extension block is missing */
+        if (!drm_find_cea_extension(edid))
+                return 0;
+
+        /*
+         * Go through all probed modes and create a new mode
+         * with the alternate clock for certain CEA modes.
+         */
+        list_for_each_entry(mode, &connector->probed_modes, head) {
+                const struct drm_display_mode *cea_mode;
+                struct drm_display_mode *newmode;
+                u8 cea_mode_idx = drm_match_cea_mode(mode) - 1;
+                unsigned int clock1, clock2;
+
+                if (cea_mode_idx >= ARRAY_SIZE(edid_cea_modes))
+                        continue;
+
+                cea_mode = &edid_cea_modes[cea_mode_idx];
+
+                clock1 = cea_mode->clock;
+                clock2 = cea_mode_alternate_clock(cea_mode);
+
+                if (clock1 == clock2)
+                        continue;
+
+                if (mode->clock != clock1 && mode->clock != clock2)
+                        continue;
+
+                newmode = drm_mode_duplicate(dev, cea_mode);
+                if (!newmode)
+                        continue;
+
+                /*
+                 * The current mode could be either variant. Make
+                 * sure to pick the "other" clock for the new mode.
+                 */
+                if (mode->clock != clock1)
+                        newmode->clock = clock1;
+                else
+                        newmode->clock = clock2;
+
+                list_add_tail(&newmode->head, &list);
+        }
+
+        list_for_each_entry_safe(mode, tmp, &list, head) {
+                list_del(&mode->head);
+                drm_mode_probed_add(connector, mode);
+                modes++;
+        }
+
+        return modes;
+}
 
 static int
 do_cea_modes (struct drm_connector *connector, u8 *db, u8 len)
@@ -2946,6 +3022,7 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid)
         if (edid->features & DRM_EDID_FEATURE_DEFAULT_GTF)
                 num_modes += add_inferred_modes(connector, edid);
         num_modes += add_cea_modes(connector, edid);
+        num_modes += add_alternate_cea_modes(connector, edid);
 
         if (quirks & (EDID_QUIRK_PREFER_LARGE_60 | EDID_QUIRK_PREFER_LARGE_75))
                 edid_fixup_preferred(connector, quirks);
diff --git a/drivers/gpu/drm/drm_edid_load.c b/drivers/gpu/drm/drm_edid_load.c
index fa445dd4dc00..a4f5ce14dc1c 100644
--- a/drivers/gpu/drm/drm_edid_load.c
+++ b/drivers/gpu/drm/drm_edid_load.c
@@ -186,12 +186,11 @@ static u8 *edid_load(struct drm_connector *connector, char *name,
                 goto relfw_out;
         }
 
-        edid = kmalloc(fwsize, GFP_KERNEL);
+        edid = kmemdup(fwdata, fwsize, GFP_KERNEL);
         if (edid == NULL) {
                 err = -ENOMEM;
                 goto relfw_out;
         }
-        memcpy(edid, fwdata, fwsize);
 
         if (!drm_edid_block_valid(edid, 0, print_bad_edid)) {
                 connector->bad_edid_counter++;
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index b78cbe74dadf..3d13ca6e257f 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -168,6 +168,9 @@ static void drm_fb_helper_save_lut_atomic(struct drm_crtc *crtc, struct drm_fb_h
         uint16_t *r_base, *g_base, *b_base;
         int i;
 
+        if (helper->funcs->gamma_get == NULL)
+                return;
+
         r_base = crtc->gamma_store;
         g_base = r_base + crtc->gamma_size;
         b_base = g_base + crtc->gamma_size;
@@ -284,13 +287,27 @@ EXPORT_SYMBOL(drm_fb_helper_debug_leave);
  */
 bool drm_fb_helper_restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 {
+        struct drm_device *dev = fb_helper->dev;
+        struct drm_plane *plane;
         bool error = false;
-        int i, ret;
+        int i;
+
+        drm_warn_on_modeset_not_all_locked(dev);
 
-        drm_warn_on_modeset_not_all_locked(fb_helper->dev);
+        list_for_each_entry(plane, &dev->mode_config.plane_list, head)
+                drm_plane_force_disable(plane);
 
         for (i = 0; i < fb_helper->crtc_count; i++) {
                 struct drm_mode_set *mode_set = &fb_helper->crtc_info[i].mode_set;
+                struct drm_crtc *crtc = mode_set->crtc;
+                int ret;
+
+                if (crtc->funcs->cursor_set) {
+                        ret = crtc->funcs->cursor_set(crtc, NULL, 0, 0, 0);
+                        if (ret)
+                                error = true;
+                }
+
                 ret = drm_mode_set_config_internal(mode_set);
                 if (ret)
                         error = true;
@@ -583,6 +600,14 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
                 return 0;
         }
 
+        /*
+         * The driver really shouldn't advertise pseudo/directcolor
+         * visuals if it can't deal with the palette.
+         */
+        if (WARN_ON(!fb_helper->funcs->gamma_set ||
+                    !fb_helper->funcs->gamma_get))
+                return -EINVAL;
+
         pindex = regno;
 
         if (fb->bits_per_pixel == 16) {
@@ -626,12 +651,19 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
 int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info)
 {
         struct drm_fb_helper *fb_helper = info->par;
+        struct drm_device *dev = fb_helper->dev;
         struct drm_crtc_helper_funcs *crtc_funcs;
         u16 *red, *green, *blue, *transp;
         struct drm_crtc *crtc;
         int i, j, rc = 0;
         int start;
 
+        drm_modeset_lock_all(dev);
+        if (!drm_fb_helper_is_bound(fb_helper)) {
+                drm_modeset_unlock_all(dev);
+                return -EBUSY;
+        }
+
         for (i = 0; i < fb_helper->crtc_count; i++) {
                 crtc = fb_helper->crtc_info[i].mode_set.crtc;
                 crtc_funcs = crtc->helper_private;
@@ -654,10 +686,13 @@ int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info)
 
                         rc = setcolreg(crtc, hred, hgreen, hblue, start++, info);
                         if (rc)
-                                return rc;
+                                goto out;
                 }
-                crtc_funcs->load_lut(crtc);
+                if (crtc_funcs->load_lut)
+                        crtc_funcs->load_lut(crtc);
         }
+ out:
+        drm_modeset_unlock_all(dev);
         return rc;
 }
 EXPORT_SYMBOL(drm_fb_helper_setcmap);
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 429e07d0b0f1..3a24385e0368 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -271,6 +271,11 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
         priv->uid = current_euid();
         priv->pid = get_pid(task_pid(current));
         priv->minor = idr_find(&drm_minors_idr, minor_id);
+        if (!priv->minor) {
+                ret = -ENODEV;
+                goto out_put_pid;
+        }
+
         priv->ioctl_count = 0;
         /* for compatibility root is always authenticated */
         priv->authenticated = capable(CAP_SYS_ADMIN);
@@ -292,7 +297,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
         if (dev->driver->open) {
                 ret = dev->driver->open(dev, priv);
                 if (ret < 0)
-                        goto out_free;
+                        goto out_prime_destroy;
         }
 
 
@@ -304,7 +309,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
                 if (!priv->minor->master) {
                         mutex_unlock(&dev->struct_mutex);
                         ret = -ENOMEM;
-                        goto out_free;
+                        goto out_close;
                 }
 
                 priv->is_master = 1;
@@ -322,7 +327,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
                                 drm_master_put(&priv->minor->master);
                                 drm_master_put(&priv->master);
                                 mutex_unlock(&dev->struct_mutex);
-                                goto out_free;
+                                goto out_close;
                         }
                 }
                 mutex_lock(&dev->struct_mutex);
@@ -333,7 +338,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
                                 drm_master_put(&priv->minor->master);
                                 drm_master_put(&priv->master);
                                 mutex_unlock(&dev->struct_mutex);
-                                goto out_free;
+                                goto out_close;
                         }
                 }
                 mutex_unlock(&dev->struct_mutex);
@@ -367,7 +372,17 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
 #endif
 
         return 0;
-      out_free:
+
+out_close:
+        if (dev->driver->postclose)
+                dev->driver->postclose(dev, priv);
+out_prime_destroy:
+        if (drm_core_check_feature(dev, DRIVER_PRIME))
+                drm_prime_destroy_file_private(&priv->prime);
+        if (dev->driver->driver_features & DRIVER_GEM)
+                drm_gem_release(dev, priv);
+out_put_pid:
+        put_pid(priv->pid);
         kfree(priv);
         filp->private_data = NULL;
         return ret;
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index cf919e36e8ae..603f256152ef 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -108,12 +108,8 @@ drm_gem_init(struct drm_device *dev)
                 return -ENOMEM;
         }
 
-        if (drm_mm_init(&mm->offset_manager, DRM_FILE_PAGE_OFFSET_START,
-                        DRM_FILE_PAGE_OFFSET_SIZE)) {
-                drm_ht_remove(&mm->offset_hash);
-                kfree(mm);
-                return -ENOMEM;
-        }
+        drm_mm_init(&mm->offset_manager, DRM_FILE_PAGE_OFFSET_START,
+                    DRM_FILE_PAGE_OFFSET_SIZE);
 
         return 0;
 }
@@ -453,25 +449,21 @@ drm_gem_flink_ioctl(struct drm_device *dev, void *data,
         spin_lock(&dev->object_name_lock);
         if (!obj->name) {
                 ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_NOWAIT);
-                obj->name = ret;
-                args->name = (uint64_t) obj->name;
-                spin_unlock(&dev->object_name_lock);
-                idr_preload_end();
-
                 if (ret < 0)
                         goto err;
-                ret = 0;
+
+                obj->name = ret;
 
                 /* Allocate a reference for the name table.  */
                 drm_gem_object_reference(obj);
-        } else {
-                args->name = (uint64_t) obj->name;
-                spin_unlock(&dev->object_name_lock);
-                idr_preload_end();
-                ret = 0;
         }
 
+        args->name = (uint64_t) obj->name;
+        ret = 0;
+
 err:
+        spin_unlock(&dev->object_name_lock);
+        idr_preload_end();
         drm_gem_object_unreference_unlocked(obj);
         return ret;
 }
@@ -644,6 +636,59 @@ void drm_gem_vm_close(struct vm_area_struct *vma)
 }
 EXPORT_SYMBOL(drm_gem_vm_close);
 
+/**
+ * drm_gem_mmap_obj - memory map a GEM object
+ * @obj: the GEM object to map
+ * @obj_size: the object size to be mapped, in bytes
+ * @vma: VMA for the area to be mapped
+ *
+ * Set up the VMA to prepare mapping of the GEM object using the gem_vm_ops
+ * provided by the driver. Depending on their requirements, drivers can either
+ * provide a fault handler in their gem_vm_ops (in which case any accesses to
+ * the object will be trapped, to perform migration, GTT binding, surface
+ * register allocation, or performance monitoring), or mmap the buffer memory
+ * synchronously after calling drm_gem_mmap_obj.
+ *
+ * This function is mainly intended to implement the DMABUF mmap operation, when
+ * the GEM object is not looked up based on its fake offset. To implement the
+ * DRM mmap operation, drivers should use the drm_gem_mmap() function.
+ *
+ * NOTE: This function has to be protected with dev->struct_mutex
+ *
+ * Return 0 or success or -EINVAL if the object size is smaller than the VMA
+ * size, or if no gem_vm_ops are provided.
+ */
+int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
+                     struct vm_area_struct *vma)
+{
+        struct drm_device *dev = obj->dev;
+
+        lockdep_assert_held(&dev->struct_mutex);
+
+        /* Check for valid size. */
+        if (obj_size < vma->vm_end - vma->vm_start)
+                return -EINVAL;
+
+        if (!dev->driver->gem_vm_ops)
+                return -EINVAL;
+
+        vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
+        vma->vm_ops = dev->driver->gem_vm_ops;
+        vma->vm_private_data = obj;
+        vma->vm_page_prot =  pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+
+        /* Take a ref for this mapping of the object, so that the fault
+         * handler can dereference the mmap offset's pointer to the object.
+         * This reference is cleaned up by the corresponding vm_close
+         * (which should happen whether the vma was created by this call, or
+         * by a vm_open due to mremap or partial unmap or whatever).
+         */
+        drm_gem_object_reference(obj);
+
+        drm_vm_open_locked(dev, vma);
+        return 0;
+}
+EXPORT_SYMBOL(drm_gem_mmap_obj);
 
 /**
  * drm_gem_mmap - memory map routine for GEM objects
@@ -653,11 +698,9 @@ EXPORT_SYMBOL(drm_gem_vm_close);
  * If a driver supports GEM object mapping, mmap calls on the DRM file
  * descriptor will end up here.
  *
- * If we find the object based on the offset passed in (vma->vm_pgoff will
+ * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
  * contain the fake offset we created when the GTT map ioctl was called on
- * the object), we set up the driver fault handler so that any accesses
- * to the object can be trapped, to perform migration, GTT binding, surface
- * register allocation, or performance monitoring.
+ * the object) and map it with a call to drm_gem_mmap_obj().
  */
 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 {
@@ -665,7 +708,6 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
         struct drm_device *dev = priv->minor->dev;
         struct drm_gem_mm *mm = dev->mm_private;
         struct drm_local_map *map = NULL;
-        struct drm_gem_object *obj;
         struct drm_hash_item *hash;
         int ret = 0;
 
@@ -686,32 +728,7 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
                 goto out_unlock;
         }
 
-        /* Check for valid size. */
-        if (map->size < vma->vm_end - vma->vm_start) {
-                ret = -EINVAL;
-                goto out_unlock;
-        }
-
-        obj = map->handle;
-        if (!obj->dev->driver->gem_vm_ops) {
-                ret = -EINVAL;
-                goto out_unlock;
-        }
-
-        vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
-        vma->vm_ops = obj->dev->driver->gem_vm_ops;
-        vma->vm_private_data = map->handle;
-        vma->vm_page_prot =  pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
-
-        /* Take a ref for this mapping of the object, so that the fault
-         * handler can dereference the mmap offset's pointer to the object.
-         * This reference is cleaned up by the corresponding vm_close
-         * (which should happen whether the vma was created by this call, or
-         * by a vm_open due to mremap or partial unmap or whatever).
-         */
-        drm_gem_object_reference(obj);
-
-        drm_vm_open_locked(dev, vma);
+        ret = drm_gem_mmap_obj(map->handle, map->size, vma);
 
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
index 0a7e011509bd..ce063970d68c 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -21,6 +21,7 @@
 #include <linux/slab.h>
 #include <linux/mutex.h>
 #include <linux/export.h>
+#include <linux/dma-buf.h>
 #include <linux/dma-mapping.h>
 
 #include <drm/drmP.h>
@@ -32,11 +33,44 @@ static unsigned int get_gem_mmap_offset(struct drm_gem_object *obj)
         return (unsigned int)obj->map_list.hash.key << PAGE_SHIFT;
 }
 
-static void drm_gem_cma_buf_destroy(struct drm_device *drm,
-                struct drm_gem_cma_object *cma_obj)
+/*
+ * __drm_gem_cma_create - Create a GEM CMA object without allocating memory
+ * @drm: The drm device
+ * @size: The GEM object size
+ *
+ * This function creates and initializes a GEM CMA object of the given size, but
+ * doesn't allocate any memory to back the object.
+ *
+ * Return a struct drm_gem_cma_object* on success or ERR_PTR values on failure.
+ */
+static struct drm_gem_cma_object *
+__drm_gem_cma_create(struct drm_device *drm, unsigned int size)
 {
-        dma_free_writecombine(drm->dev, cma_obj->base.size, cma_obj->vaddr,
-                        cma_obj->paddr);
+        struct drm_gem_cma_object *cma_obj;
+        struct drm_gem_object *gem_obj;
+        int ret;
+
+        cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
+        if (!cma_obj)
+                return ERR_PTR(-ENOMEM);
+
+        gem_obj = &cma_obj->base;
+
+        ret = drm_gem_object_init(drm, gem_obj, size);
+        if (ret)
+                goto error;
+
+        ret = drm_gem_create_mmap_offset(gem_obj);
+        if (ret) {
+                drm_gem_object_release(gem_obj);
+                goto error;
+        }
+
+        return cma_obj;
+
+error:
+        kfree(cma_obj);
+        return ERR_PTR(ret);
 }
 
 /*
@@ -49,44 +83,42 @@ struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
                 unsigned int size)
 {
         struct drm_gem_cma_object *cma_obj;
-        struct drm_gem_object *gem_obj;
+        struct sg_table *sgt = NULL;
         int ret;
 
         size = round_up(size, PAGE_SIZE);
 
-        cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
-        if (!cma_obj)
-                return ERR_PTR(-ENOMEM);
+        cma_obj = __drm_gem_cma_create(drm, size);
+        if (IS_ERR(cma_obj))
+                return cma_obj;
 
         cma_obj->vaddr = dma_alloc_writecombine(drm->dev, size,
                         &cma_obj->paddr, GFP_KERNEL | __GFP_NOWARN);
         if (!cma_obj->vaddr) {
-                dev_err(drm->dev, "failed to allocate buffer with size %d\n", size);
+                dev_err(drm->dev, "failed to allocate buffer with size %d\n",
+                        size);
                 ret = -ENOMEM;
-                goto err_dma_alloc;
+                goto error;
         }
 
-        gem_obj = &cma_obj->base;
+        sgt = kzalloc(sizeof(*cma_obj->sgt), GFP_KERNEL);
+        if (sgt == NULL) {
+                ret = -ENOMEM;
+                goto error;
+        }
 
-        ret = drm_gem_object_init(drm, gem_obj, size);
-        if (ret)
-                goto err_obj_init;
+        ret = dma_get_sgtable(drm->dev, sgt, cma_obj->vaddr,
+                              cma_obj->paddr, size);
+        if (ret < 0)
+                goto error;
 
-        ret = drm_gem_create_mmap_offset(gem_obj);
-        if (ret)
-                goto err_create_mmap_offset;
+        cma_obj->sgt = sgt;
 
         return cma_obj;
 
-err_create_mmap_offset:
-        drm_gem_object_release(gem_obj);
-
-err_obj_init:
-        drm_gem_cma_buf_destroy(drm, cma_obj);
-
-err_dma_alloc:
-        kfree(cma_obj);
-
+error:
+        kfree(sgt);
+        drm_gem_cma_free_object(&cma_obj->base);
         return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(drm_gem_cma_create);
@@ -143,11 +175,20 @@ void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
         if (gem_obj->map_list.map)
                 drm_gem_free_mmap_offset(gem_obj);
 
-        drm_gem_object_release(gem_obj);
-
         cma_obj = to_drm_gem_cma_obj(gem_obj);
 
-        drm_gem_cma_buf_destroy(gem_obj->dev, cma_obj);
+        if (cma_obj->vaddr) {
+                dma_free_writecombine(gem_obj->dev->dev, cma_obj->base.size,
+                                      cma_obj->vaddr, cma_obj->paddr);
+                if (cma_obj->sgt) {
+                        sg_free_table(cma_obj->sgt);
+                        kfree(cma_obj->sgt);
+                }
+        } else if (gem_obj->import_attach) {
+                drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
+        }
+
+        drm_gem_object_release(gem_obj);
 
         kfree(cma_obj);
 }
@@ -174,10 +215,7 @@ int drm_gem_cma_dumb_create(struct drm_file *file_priv,
 
         cma_obj = drm_gem_cma_create_with_handle(file_priv, dev,
                         args->size, &args->handle);
-        if (IS_ERR(cma_obj))
-                return PTR_ERR(cma_obj);
-
-        return 0;
+        return PTR_RET(cma_obj);
 }
 EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create);
 
@@ -215,13 +253,26 @@ const struct vm_operations_struct drm_gem_cma_vm_ops = {
 };
 EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops);
 
+static int drm_gem_cma_mmap_obj(struct drm_gem_cma_object *cma_obj,
+                                struct vm_area_struct *vma)
+{
+        int ret;
+
+        ret = remap_pfn_range(vma, vma->vm_start, cma_obj->paddr >> PAGE_SHIFT,
+                        vma->vm_end - vma->vm_start, vma->vm_page_prot);
+        if (ret)
+                drm_gem_vm_close(vma);
+
+        return ret;
+}
+
 /*
  * drm_gem_cma_mmap - (struct file_operation)->mmap callback function
  */
 int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-        struct drm_gem_object *gem_obj;
         struct drm_gem_cma_object *cma_obj;
+        struct drm_gem_object *gem_obj;
         int ret;
 
         ret = drm_gem_mmap(filp, vma);
@@ -231,12 +282,7 @@ int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma)
         gem_obj = vma->vm_private_data;
         cma_obj = to_drm_gem_cma_obj(gem_obj);
 
-        ret = remap_pfn_range(vma, vma->vm_start, cma_obj->paddr >> PAGE_SHIFT,
-                        vma->vm_end - vma->vm_start, vma->vm_page_prot);
-        if (ret)
-                drm_gem_vm_close(vma);
-
-        return ret;
+        return drm_gem_cma_mmap_obj(cma_obj, vma);
 }
 EXPORT_SYMBOL_GPL(drm_gem_cma_mmap);
 
@@ -270,3 +316,289 @@ void drm_gem_cma_describe(struct drm_gem_cma_object *cma_obj, struct seq_file *m
 }
 EXPORT_SYMBOL_GPL(drm_gem_cma_describe);
 #endif
+
+/* -----------------------------------------------------------------------------
+ * DMA-BUF
+ */
+
+struct drm_gem_cma_dmabuf_attachment {
+        struct sg_table sgt;
+        enum dma_data_direction dir;
+};
+
+static int drm_gem_cma_dmabuf_attach(struct dma_buf *dmabuf, struct device *dev,
+                                     struct dma_buf_attachment *attach)
+{
+        struct drm_gem_cma_dmabuf_attachment *cma_attach;
+
+        cma_attach = kzalloc(sizeof(*cma_attach), GFP_KERNEL);
+        if (!cma_attach)
+                return -ENOMEM;
+
+        cma_attach->dir = DMA_NONE;
+        attach->priv = cma_attach;
+
+        return 0;
+}
+
+static void drm_gem_cma_dmabuf_detach(struct dma_buf *dmabuf,
+                                      struct dma_buf_attachment *attach)
+{
+        struct drm_gem_cma_dmabuf_attachment *cma_attach = attach->priv;
+        struct sg_table *sgt;
+
+        if (cma_attach == NULL)
+                return;
+
+        sgt = &cma_attach->sgt;
+
+        if (cma_attach->dir != DMA_NONE)
+                dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
+                                cma_attach->dir);
+
+        sg_free_table(sgt);
+        kfree(cma_attach);
+        attach->priv = NULL;
+}
+
+static struct sg_table *
+drm_gem_cma_dmabuf_map(struct dma_buf_attachment *attach,
+                       enum dma_data_direction dir)
+{
+        struct drm_gem_cma_dmabuf_attachment *cma_attach = attach->priv;
+        struct drm_gem_cma_object *cma_obj = attach->dmabuf->priv;
+        struct drm_device *drm = cma_obj->base.dev;
+        struct scatterlist *rd, *wr;
+        struct sg_table *sgt;
+        unsigned int i;
+        int nents, ret;
+
+        DRM_DEBUG_PRIME("\n");
+
+        if (WARN_ON(dir == DMA_NONE))
+                return ERR_PTR(-EINVAL);
+
+        /* Return the cached mapping when possible. */
+        if (cma_attach->dir == dir)
+                return &cma_attach->sgt;
+
+        /* Two mappings with different directions for the same attachment are
+         * not allowed.
+         */
+        if (WARN_ON(cma_attach->dir != DMA_NONE))
+                return ERR_PTR(-EBUSY);
+
+        sgt = &cma_attach->sgt;
+
+        ret = sg_alloc_table(sgt, cma_obj->sgt->orig_nents, GFP_KERNEL);
+        if (ret) {
+                DRM_ERROR("failed to alloc sgt.\n");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        mutex_lock(&drm->struct_mutex);
+
+        rd = cma_obj->sgt->sgl;
+        wr = sgt->sgl;
+        for (i = 0; i < sgt->orig_nents; ++i) {
+                sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
+                rd = sg_next(rd);
+                wr = sg_next(wr);
+        }
+
+        nents = dma_map_sg(attach->dev, sgt->sgl, sgt->orig_nents, dir);
+        if (!nents) {
+                DRM_ERROR("failed to map sgl with iommu.\n");
+                sg_free_table(sgt);
+                sgt = ERR_PTR(-EIO);
+                goto done;
+        }
+
+        cma_attach->dir = dir;
+        attach->priv = cma_attach;
+
+        DRM_DEBUG_PRIME("buffer size = %zu\n", cma_obj->base.size);
+
+done:
+        mutex_unlock(&drm->struct_mutex);
+        return sgt;
+}
+
+static void drm_gem_cma_dmabuf_unmap(struct dma_buf_attachment *attach,
+                                     struct sg_table *sgt,
+                                     enum dma_data_direction dir)
+{
+        /* Nothing to do. */
+}
+
+static void drm_gem_cma_dmabuf_release(struct dma_buf *dmabuf)
+{
+        struct drm_gem_cma_object *cma_obj = dmabuf->priv;
+
+        DRM_DEBUG_PRIME("%s\n", __FILE__);
+
+        /*
+         * drm_gem_cma_dmabuf_release() call means that file object's
+         * f_count is 0 and it calls drm_gem_object_handle_unreference()
+         * to drop the references that these values had been increased
+         * at drm_prime_handle_to_fd()
+         */
+        if (cma_obj->base.export_dma_buf == dmabuf) {
+                cma_obj->base.export_dma_buf = NULL;
+
+                /*
+                 * drop this gem object refcount to release allocated buffer
+                 * and resources.
+                 */
+                drm_gem_object_unreference_unlocked(&cma_obj->base);
+        }
+}
+
+static void *drm_gem_cma_dmabuf_kmap_atomic(struct dma_buf *dmabuf,
+                                            unsigned long page_num)
+{
+        /* TODO */
+
+        return NULL;
+}
+
+static void drm_gem_cma_dmabuf_kunmap_atomic(struct dma_buf *dmabuf,
+                                             unsigned long page_num, void *addr)
+{
+        /* TODO */
+}
+
+static void *drm_gem_cma_dmabuf_kmap(struct dma_buf *dmabuf,
+                                     unsigned long page_num)
+{
+        /* TODO */
+
+        return NULL;
+}
+
+static void drm_gem_cma_dmabuf_kunmap(struct dma_buf *dmabuf,
+                                      unsigned long page_num, void *addr)
+{
+        /* TODO */
+}
+
+static int drm_gem_cma_dmabuf_mmap(struct dma_buf *dmabuf,
+                                   struct vm_area_struct *vma)
+{
+        struct drm_gem_cma_object *cma_obj = dmabuf->priv;
+        struct drm_gem_object *gem_obj = &cma_obj->base;
+        struct drm_device *dev = gem_obj->dev;
+        int ret;
+
+        mutex_lock(&dev->struct_mutex);
+        ret = drm_gem_mmap_obj(gem_obj, gem_obj->size, vma);
+        mutex_unlock(&dev->struct_mutex);
+        if (ret < 0)
+                return ret;
+
+        return drm_gem_cma_mmap_obj(cma_obj, vma);
+}
+
+static void *drm_gem_cma_dmabuf_vmap(struct dma_buf *dmabuf)
+{
+        struct drm_gem_cma_object *cma_obj = dmabuf->priv;
+
+        return cma_obj->vaddr;
+}
+
+static struct dma_buf_ops drm_gem_cma_dmabuf_ops = {
+        .attach                 = drm_gem_cma_dmabuf_attach,
+        .detach                 = drm_gem_cma_dmabuf_detach,
+        .map_dma_buf            = drm_gem_cma_dmabuf_map,
+        .unmap_dma_buf          = drm_gem_cma_dmabuf_unmap,
+        .kmap                   = drm_gem_cma_dmabuf_kmap,
+        .kmap_atomic            = drm_gem_cma_dmabuf_kmap_atomic,
+        .kunmap                 = drm_gem_cma_dmabuf_kunmap,
+        .kunmap_atomic          = drm_gem_cma_dmabuf_kunmap_atomic,
+        .mmap                   = drm_gem_cma_dmabuf_mmap,
+        .vmap                   = drm_gem_cma_dmabuf_vmap,
+        .release                = drm_gem_cma_dmabuf_release,
+};
+
+struct dma_buf *drm_gem_cma_dmabuf_export(struct drm_device *drm,
+                                          struct drm_gem_object *obj, int flags)
+{
+        struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
+
+        return dma_buf_export(cma_obj, &drm_gem_cma_dmabuf_ops,
+                              cma_obj->base.size, flags);
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_dmabuf_export);
+
+struct drm_gem_object *drm_gem_cma_dmabuf_import(struct drm_device *drm,
+                                                 struct dma_buf *dma_buf)
+{
+        struct drm_gem_cma_object *cma_obj;
+        struct dma_buf_attachment *attach;
+        struct sg_table *sgt;
+        int ret;
+
+        DRM_DEBUG_PRIME("%s\n", __FILE__);
+
+        /* is this one of own objects? */
+        if (dma_buf->ops == &drm_gem_cma_dmabuf_ops) {
+                struct drm_gem_object *obj;
+
+                cma_obj = dma_buf->priv;
+                obj = &cma_obj->base;
+
+                /* is it from our device? */
+                if (obj->dev == drm) {
+                        /*
+                         * Importing dmabuf exported from out own gem increases
+                         * refcount on gem itself instead of f_count of dmabuf.
+                         */
+                        drm_gem_object_reference(obj);
+                        dma_buf_put(dma_buf);
+                        return obj;
+                }
+        }
+
+        /* Create a CMA GEM buffer. */
+        cma_obj = __drm_gem_cma_create(drm, dma_buf->size);
+        if (IS_ERR(cma_obj))
+                return ERR_PTR(PTR_ERR(cma_obj));
+
+        /* Attach to the buffer and map it. Make sure the mapping is contiguous
+         * on the device memory bus, as that's all we support.
+         */
+        attach = dma_buf_attach(dma_buf, drm->dev);
+        if (IS_ERR(attach)) {
+                ret = -EINVAL;
+                goto error_gem_free;
+        }
+
+        sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+        if (IS_ERR_OR_NULL(sgt)) {
+                ret = sgt ? PTR_ERR(sgt) : -ENOMEM;
+                goto error_buf_detach;
+        }
+
+        if (sgt->nents != 1) {
+                ret = -EINVAL;
+                goto error_buf_unmap;
+        }
+
+        cma_obj->base.import_attach = attach;
+        cma_obj->paddr = sg_dma_address(sgt->sgl);
+        cma_obj->sgt = sgt;
+
+        DRM_DEBUG_PRIME("dma_addr = 0x%x, size = %zu\n", cma_obj->paddr,
+                        dma_buf->size);
+
+        return &cma_obj->base;
+
+error_buf_unmap:
+        dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
+error_buf_detach:
+        dma_buf_detach(dma_buf, attach);
+error_gem_free:
+        drm_gem_cma_free_object(&cma_obj->base);
+        return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_dmabuf_import);
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index e77bd8b57df2..ffd7a7ba70d4 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -38,6 +38,9 @@
 
 #include <linux/pci.h>
 #include <linux/export.h>
+#ifdef CONFIG_X86
+#include <asm/mtrr.h>
+#endif
 
 /**
  * Get the bus id.
@@ -181,7 +184,17 @@ int drm_getmap(struct drm_device *dev, void *data,
         map->type = r_list->map->type;
         map->flags = r_list->map->flags;
         map->handle = (void *)(unsigned long) r_list->user_token;
-        map->mtrr = r_list->map->mtrr;
+
+#ifdef CONFIG_X86
+        /*
+         * There appears to be exactly one user of the mtrr index: dritest.
+         * It's easy enough to keep it working on non-PAT systems.
+         */
+        map->mtrr = phys_wc_to_mtrr_index(r_list->map->mtrr);
+#else
+        map->mtrr = -1;
+#endif
+
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index 07cf99cc8862..543b9b3171d3 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -669,7 +669,7 @@ int drm_mm_clean(struct drm_mm * mm)
 }
 EXPORT_SYMBOL(drm_mm_clean);
 
-int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
+void drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
 {
         INIT_LIST_HEAD(&mm->hole_stack);
         INIT_LIST_HEAD(&mm->unused_nodes);
@@ -690,8 +690,6 @@ int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
         list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
 
         mm->color_adjust = NULL;
-
-        return 0;
 }
 EXPORT_SYMBOL(drm_mm_init);
 
@@ -699,8 +697,8 @@ void drm_mm_takedown(struct drm_mm * mm)
 {
         struct drm_mm_node *entry, *next;
 
-        if (!list_empty(&mm->head_node.node_list)) {
-                DRM_ERROR("Memory manager not clean. Delaying takedown\n");
+        if (WARN(!list_empty(&mm->head_node.node_list),
+                 "Memory manager not clean. Delaying takedown\n")) {
                 return;
         }
 
@@ -716,36 +714,37 @@ void drm_mm_takedown(struct drm_mm * mm)
 }
 EXPORT_SYMBOL(drm_mm_takedown);
 
-void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
+static unsigned long drm_mm_debug_hole(struct drm_mm_node *entry,
+                                       const char *prefix)
 {
-        struct drm_mm_node *entry;
-        unsigned long total_used = 0, total_free = 0, total = 0;
         unsigned long hole_start, hole_end, hole_size;
 
-        hole_start = drm_mm_hole_node_start(&mm->head_node);
-        hole_end = drm_mm_hole_node_end(&mm->head_node);
-        hole_size = hole_end - hole_start;
-        if (hole_size)
+        if (entry->hole_follows) {
+                hole_start = drm_mm_hole_node_start(entry);
+                hole_end = drm_mm_hole_node_end(entry);
+                hole_size = hole_end - hole_start;
                 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
                         prefix, hole_start, hole_end,
                         hole_size);
-        total_free += hole_size;
+                return hole_size;
+        }
+
+        return 0;
+}
+
+void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
+{
+        struct drm_mm_node *entry;
+        unsigned long total_used = 0, total_free = 0, total = 0;
+
+        total_free += drm_mm_debug_hole(&mm->head_node, prefix);
 
         drm_mm_for_each_node(entry, mm) {
                 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: used\n",
                         prefix, entry->start, entry->start + entry->size,
                         entry->size);
                 total_used += entry->size;
-
-                if (entry->hole_follows) {
-                        hole_start = drm_mm_hole_node_start(entry);
-                        hole_end = drm_mm_hole_node_end(entry);
-                        hole_size = hole_end - hole_start;
-                        printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
-                                prefix, hole_start, hole_end,
-                                hole_size);
-                        total_free += hole_size;
-                }
+                total_free += drm_mm_debug_hole(entry, prefix);
         }
         total = total_free + total_used;
 
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index a371ff865a88..a6729bfe6860 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -535,6 +535,8 @@ int drm_display_mode_from_videomode(const struct videomode *vm,
                 dmode->flags |= DRM_MODE_FLAG_INTERLACE;
         if (vm->flags & DISPLAY_FLAGS_DOUBLESCAN)
                 dmode->flags |= DRM_MODE_FLAG_DBLSCAN;
+        if (vm->flags & DISPLAY_FLAGS_DOUBLECLK)
+                dmode->flags |= DRM_MODE_FLAG_DBLCLK;
         drm_mode_set_name(dmode);
 
         return 0;
@@ -787,16 +789,17 @@ EXPORT_SYMBOL(drm_mode_set_crtcinfo);
  * LOCKING:
  * None.
  *
- * Copy an existing mode into another mode, preserving the object id
- * of the destination mode.
+ * Copy an existing mode into another mode, preserving the object id and
+ * list head of the destination mode.
  */
 void drm_mode_copy(struct drm_display_mode *dst, const struct drm_display_mode *src)
 {
         int id = dst->base.id;
+        struct list_head head = dst->head;
 
         *dst = *src;
         dst->base.id = id;
-        INIT_LIST_HEAD(&dst->head);
+        dst->head = head;
 }
 EXPORT_SYMBOL(drm_mode_copy);
 
@@ -1017,6 +1020,11 @@ static int drm_mode_compare(void *priv, struct list_head *lh_a, struct list_head
         diff = b->hdisplay * b->vdisplay - a->hdisplay * a->vdisplay;
         if (diff)
                 return diff;
+
+        diff = b->vrefresh - a->vrefresh;
+        if (diff)
+                return diff;
+
         diff = b->clock - a->clock;
         return diff;
 }
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index 14194b6ef644..80c0b2b29801 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -278,10 +278,10 @@ static int drm_pci_agp_init(struct drm_device *dev)
                 }
                 if (drm_core_has_MTRR(dev)) {
                         if (dev->agp)
-                                dev->agp->agp_mtrr =
-                                        mtrr_add(dev->agp->agp_info.aper_base,
-                                                 dev->agp->agp_info.aper_size *
-                                                 1024 * 1024, MTRR_TYPE_WRCOMB, 1);
+                                dev->agp->agp_mtrr = arch_phys_wc_add(
+                                        dev->agp->agp_info.aper_base,
+                                        dev->agp->agp_info.aper_size *
+                                        1024 * 1024);
                 }
         }
         return 0;
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 5b7b9110254b..1e0de41f085c 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -62,20 +62,124 @@ struct drm_prime_member {
         struct dma_buf *dma_buf;
         uint32_t handle;
 };
-static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t handle);
+
+struct drm_prime_attachment {
+        struct sg_table *sgt;
+        enum dma_data_direction dir;
+};
+
+static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t handle)
+{
+        struct drm_prime_member *member;
+
+        member = kmalloc(sizeof(*member), GFP_KERNEL);
+        if (!member)
+                return -ENOMEM;
+
+        get_dma_buf(dma_buf);
+        member->dma_buf = dma_buf;
+        member->handle = handle;
+        list_add(&member->entry, &prime_fpriv->head);
+        return 0;
+}
+
+static int drm_gem_map_attach(struct dma_buf *dma_buf,
+                              struct device *target_dev,
+                              struct dma_buf_attachment *attach)
+{
+        struct drm_prime_attachment *prime_attach;
+        struct drm_gem_object *obj = dma_buf->priv;
+        struct drm_device *dev = obj->dev;
+
+        prime_attach = kzalloc(sizeof(*prime_attach), GFP_KERNEL);
+        if (!prime_attach)
+                return -ENOMEM;
+
+        prime_attach->dir = DMA_NONE;
+        attach->priv = prime_attach;
+
+        if (!dev->driver->gem_prime_pin)
+                return 0;
+
+        return dev->driver->gem_prime_pin(obj);
+}
+
+static void drm_gem_map_detach(struct dma_buf *dma_buf,
+                               struct dma_buf_attachment *attach)
+{
+        struct drm_prime_attachment *prime_attach = attach->priv;
+        struct drm_gem_object *obj = dma_buf->priv;
+        struct drm_device *dev = obj->dev;
+        struct sg_table *sgt;
+
+        if (dev->driver->gem_prime_unpin)
+                dev->driver->gem_prime_unpin(obj);
+
+        if (!prime_attach)
+                return;
+
+        sgt = prime_attach->sgt;
+
+        if (prime_attach->dir != DMA_NONE)
+                dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
+                                prime_attach->dir);
+
+        sg_free_table(sgt);
+        kfree(sgt);
+        kfree(prime_attach);
+        attach->priv = NULL;
+}
+
+static void drm_prime_remove_buf_handle_locked(
+                struct drm_prime_file_private *prime_fpriv,
+                struct dma_buf *dma_buf)
+{
+        struct drm_prime_member *member, *safe;
+
+        list_for_each_entry_safe(member, safe, &prime_fpriv->head, entry) {
+                if (member->dma_buf == dma_buf) {
+                        dma_buf_put(dma_buf);
+                        list_del(&member->entry);
+                        kfree(member);
+                }
+        }
+}
 
 static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
                 enum dma_data_direction dir)
 {
+        struct drm_prime_attachment *prime_attach = attach->priv;
         struct drm_gem_object *obj = attach->dmabuf->priv;
         struct sg_table *sgt;
 
+        if (WARN_ON(dir == DMA_NONE || !prime_attach))
+                return ERR_PTR(-EINVAL);
+
+        /* return the cached mapping when possible */
+        if (prime_attach->dir == dir)
+                return prime_attach->sgt;
+
+        /*
+         * two mappings with different directions for the same attachment are
+         * not allowed
+         */
+        if (WARN_ON(prime_attach->dir != DMA_NONE))
+                return ERR_PTR(-EBUSY);
+
         mutex_lock(&obj->dev->struct_mutex);
 
         sgt = obj->dev->driver->gem_prime_get_sg_table(obj);
 
-        if (!IS_ERR_OR_NULL(sgt))
-                dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir);
+        if (!IS_ERR(sgt)) {
+                if (!dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir)) {
+                        sg_free_table(sgt);
+                        kfree(sgt);
+                        sgt = ERR_PTR(-ENOMEM);
+                } else {
+                        prime_attach->sgt = sgt;
+                        prime_attach->dir = dir;
+                }
+        }
 
         mutex_unlock(&obj->dev->struct_mutex);
         return sgt;
@@ -84,9 +188,7 @@ static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
 static void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
                 struct sg_table *sgt, enum dma_data_direction dir)
 {
-        dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
-        sg_free_table(sgt);
-        kfree(sgt);
+        /* nothing to be done here */
 }
 
 static void drm_gem_dmabuf_release(struct dma_buf *dma_buf)
@@ -146,6 +248,8 @@ static int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf,
 }
 
 static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
+        .attach = drm_gem_map_attach,
+        .detach = drm_gem_map_detach,
         .map_dma_buf = drm_gem_map_dma_buf,
         .unmap_dma_buf = drm_gem_unmap_dma_buf,
         .release = drm_gem_dmabuf_release,
@@ -185,11 +289,6 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
 struct dma_buf *drm_gem_prime_export(struct drm_device *dev,
                                      struct drm_gem_object *obj, int flags)
 {
-        if (dev->driver->gem_prime_pin) {
-                int ret = dev->driver->gem_prime_pin(obj);
-                if (ret)
-                        return ERR_PTR(ret);
-        }
         return dma_buf_export(obj, &drm_gem_prime_dmabuf_ops, obj->size, flags);
 }
 EXPORT_SYMBOL(drm_gem_prime_export);
@@ -235,15 +334,34 @@ int drm_gem_prime_handle_to_fd(struct drm_device *dev,
         ret = drm_prime_add_buf_handle(&file_priv->prime,
                                        obj->export_dma_buf, handle);
         if (ret)
-                goto out;
+                goto fail_put_dmabuf;
+
+        ret = dma_buf_fd(buf, flags);
+        if (ret < 0)
+                goto fail_rm_handle;
 
-        *prime_fd = dma_buf_fd(buf, flags);
+        *prime_fd = ret;
         mutex_unlock(&file_priv->prime.lock);
         return 0;
 
 out_have_obj:
         get_dma_buf(dmabuf);
-        *prime_fd = dma_buf_fd(dmabuf, flags);
+        ret = dma_buf_fd(dmabuf, flags);
+        if (ret < 0) {
+                dma_buf_put(dmabuf);
+        } else {
+                *prime_fd = ret;
+                ret = 0;
+        }
+
+        goto out;
+
+fail_rm_handle:
+        drm_prime_remove_buf_handle_locked(&file_priv->prime, buf);
+fail_put_dmabuf:
+        /* clear NOT to be checked when releasing dma_buf */
+        obj->export_dma_buf = NULL;
+        dma_buf_put(buf);
 out:
         drm_gem_object_unreference_unlocked(obj);
         mutex_unlock(&file_priv->prime.lock);
@@ -276,7 +394,7 @@ struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev,
 
         attach = dma_buf_attach(dma_buf, dev->dev);
         if (IS_ERR(attach))
-                return ERR_PTR(PTR_ERR(attach));
+                return ERR_CAST(attach);
 
         get_dma_buf(dma_buf);
 
@@ -412,8 +530,10 @@ struct sg_table *drm_prime_pages_to_sg(struct page **pages, int nr_pages)
         int ret;
 
         sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
-        if (!sg)
+        if (!sg) {
+                ret = -ENOMEM;
                 goto out;
+        }
 
         ret = sg_alloc_table_from_pages(sg, pages, nr_pages, 0,
                                 nr_pages << PAGE_SHIFT, GFP_KERNEL);
@@ -423,7 +543,7 @@ struct sg_table *drm_prime_pages_to_sg(struct page **pages, int nr_pages)
         return sg;
 out:
         kfree(sg);
-        return NULL;
+        return ERR_PTR(ret);
 }
 EXPORT_SYMBOL(drm_prime_pages_to_sg);
 
@@ -492,21 +612,6 @@ void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv)
 }
 EXPORT_SYMBOL(drm_prime_destroy_file_private);
 
-static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t handle)
-{
-        struct drm_prime_member *member;
-
-        member = kmalloc(sizeof(*member), GFP_KERNEL);
-        if (!member)
-                return -ENOMEM;
-
-        get_dma_buf(dma_buf);
-        member->dma_buf = dma_buf;
-        member->handle = handle;
-        list_add(&member->entry, &prime_fpriv->head);
-        return 0;
-}
-
 int drm_prime_lookup_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t *handle)
 {
         struct drm_prime_member *member;
@@ -523,16 +628,8 @@ EXPORT_SYMBOL(drm_prime_lookup_buf_handle);
 
 void drm_prime_remove_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf)
 {
-        struct drm_prime_member *member, *safe;
-
         mutex_lock(&prime_fpriv->lock);
-        list_for_each_entry_safe(member, safe, &prime_fpriv->head, entry) {
-                if (member->dma_buf == dma_buf) {
-                        dma_buf_put(dma_buf);
-                        list_del(&member->entry);
-                        kfree(member);
-                }
-        }
+        drm_prime_remove_buf_handle_locked(prime_fpriv, dma_buf);
         mutex_unlock(&prime_fpriv->lock);
 }
 EXPORT_SYMBOL(drm_prime_remove_buf_handle);
diff --git a/drivers/gpu/drm/drm_rect.c b/drivers/gpu/drm/drm_rect.c
new file mode 100644
index 000000000000..7047ca025787
--- /dev/null
+++ b/drivers/gpu/drm/drm_rect.c
@@ -0,0 +1,295 @@
+/*
+ * Copyright (C) 2011-2013 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <drm/drmP.h>
+#include <drm/drm_rect.h>
+
+/**
+ * drm_rect_intersect - intersect two rectangles
+ * @r1: first rectangle
+ * @r2: second rectangle
+ *
+ * Calculate the intersection of rectangles @r1 and @r2.
+ * @r1 will be overwritten with the intersection.
+ *
+ * RETURNS:
+ * %true if rectangle @r1 is still visible after the operation,
+ * %false otherwise.
+ */
+bool drm_rect_intersect(struct drm_rect *r1, const struct drm_rect *r2)
+{
+        r1->x1 = max(r1->x1, r2->x1);
+        r1->y1 = max(r1->y1, r2->y1);
+        r1->x2 = min(r1->x2, r2->x2);
+        r1->y2 = min(r1->y2, r2->y2);
+
+        return drm_rect_visible(r1);
+}
+EXPORT_SYMBOL(drm_rect_intersect);
+
+/**
+ * drm_rect_clip_scaled - perform a scaled clip operation
+ * @src: source window rectangle
+ * @dst: destination window rectangle
+ * @clip: clip rectangle
+ * @hscale: horizontal scaling factor
+ * @vscale: vertical scaling factor
+ *
+ * Clip rectangle @dst by rectangle @clip. Clip rectangle @src by the
+ * same amounts multiplied by @hscale and @vscale.
+ *
+ * RETURNS:
+ * %true if rectangle @dst is still visible after being clipped,
+ * %false otherwise
+ */
+bool drm_rect_clip_scaled(struct drm_rect *src, struct drm_rect *dst,
+                          const struct drm_rect *clip,
+                          int hscale, int vscale)
+{
+        int diff;
+
+        diff = clip->x1 - dst->x1;
+        if (diff > 0) {
+                int64_t tmp = src->x1 + (int64_t) diff * hscale;
+                src->x1 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+        }
+        diff = clip->y1 - dst->y1;
+        if (diff > 0) {
+                int64_t tmp = src->y1 + (int64_t) diff * vscale;
+                src->y1 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+        }
+        diff = dst->x2 - clip->x2;
+        if (diff > 0) {
+                int64_t tmp = src->x2 - (int64_t) diff * hscale;
+                src->x2 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+        }
+        diff = dst->y2 - clip->y2;
+        if (diff > 0) {
+                int64_t tmp = src->y2 - (int64_t) diff * vscale;
+                src->y2 = clamp_t(int64_t, tmp, INT_MIN, INT_MAX);
+        }
+
+        return drm_rect_intersect(dst, clip);
+}
+EXPORT_SYMBOL(drm_rect_clip_scaled);
+
+static int drm_calc_scale(int src, int dst)
+{
+        int scale = 0;
+
+        if (src < 0 || dst < 0)
+                return -EINVAL;
+
+        if (dst == 0)
+                return 0;
+
+        scale = src / dst;
+
+        return scale;
+}
+
+/**
+ * drm_rect_calc_hscale - calculate the horizontal scaling factor
+ * @src: source window rectangle
+ * @dst: destination window rectangle
+ * @min_hscale: minimum allowed horizontal scaling factor
+ * @max_hscale: maximum allowed horizontal scaling factor
+ *
+ * Calculate the horizontal scaling factor as
+ * (@src width) / (@dst width).
+ *
+ * RETURNS:
+ * The horizontal scaling factor, or errno of out of limits.
+ */
+int drm_rect_calc_hscale(const struct drm_rect *src,
+                         const struct drm_rect *dst,
+                         int min_hscale, int max_hscale)
+{
+        int src_w = drm_rect_width(src);
+        int dst_w = drm_rect_width(dst);
+        int hscale = drm_calc_scale(src_w, dst_w);
+
+        if (hscale < 0 || dst_w == 0)
+                return hscale;
+
+        if (hscale < min_hscale || hscale > max_hscale)
+                return -ERANGE;
+
+        return hscale;
+}
+EXPORT_SYMBOL(drm_rect_calc_hscale);
+
+/**
+ * drm_rect_calc_vscale - calculate the vertical scaling factor
+ * @src: source window rectangle
+ * @dst: destination window rectangle
+ * @min_vscale: minimum allowed vertical scaling factor
+ * @max_vscale: maximum allowed vertical scaling factor
+ *
+ * Calculate the vertical scaling factor as
+ * (@src height) / (@dst height).
+ *
+ * RETURNS:
+ * The vertical scaling factor, or errno of out of limits.
+ */
+int drm_rect_calc_vscale(const struct drm_rect *src,
+                         const struct drm_rect *dst,
+                         int min_vscale, int max_vscale)
+{
+        int src_h = drm_rect_height(src);
+        int dst_h = drm_rect_height(dst);
+        int vscale = drm_calc_scale(src_h, dst_h);
+
+        if (vscale < 0 || dst_h == 0)
+                return vscale;
+
+        if (vscale < min_vscale || vscale > max_vscale)
+                return -ERANGE;
+
+        return vscale;
+}
+EXPORT_SYMBOL(drm_rect_calc_vscale);
+
+/**
+ * drm_calc_hscale_relaxed - calculate the horizontal scaling factor
+ * @src: source window rectangle
+ * @dst: destination window rectangle
+ * @min_hscale: minimum allowed horizontal scaling factor
+ * @max_hscale: maximum allowed horizontal scaling factor
+ *
+ * Calculate the horizontal scaling factor as
+ * (@src width) / (@dst width).
+ *
+ * If the calculated scaling factor is below @min_vscale,
+ * decrease the height of rectangle @dst to compensate.
+ *
+ * If the calculated scaling factor is above @max_vscale,
+ * decrease the height of rectangle @src to compensate.
+ *
+ * RETURNS:
+ * The horizontal scaling factor.
+ */
+int drm_rect_calc_hscale_relaxed(struct drm_rect *src,
+                                 struct drm_rect *dst,
+                                 int min_hscale, int max_hscale)
+{
+        int src_w = drm_rect_width(src);
+        int dst_w = drm_rect_width(dst);
+        int hscale = drm_calc_scale(src_w, dst_w);
+
+        if (hscale < 0 || dst_w == 0)
+                return hscale;
+
+        if (hscale < min_hscale) {
+                int max_dst_w = src_w / min_hscale;
+
+                drm_rect_adjust_size(dst, max_dst_w - dst_w, 0);
+
+                return min_hscale;
+        }
+
+        if (hscale > max_hscale) {
+                int max_src_w = dst_w * max_hscale;
+
+                drm_rect_adjust_size(src, max_src_w - src_w, 0);
+
+                return max_hscale;
+        }
+
+        return hscale;
+}
+EXPORT_SYMBOL(drm_rect_calc_hscale_relaxed);
+
+/**
+ * drm_rect_calc_vscale_relaxed - calculate the vertical scaling factor
+ * @src: source window rectangle
+ * @dst: destination window rectangle
+ * @min_vscale: minimum allowed vertical scaling factor
+ * @max_vscale: maximum allowed vertical scaling factor
+ *
+ * Calculate the vertical scaling factor as
+ * (@src height) / (@dst height).
+ *
+ * If the calculated scaling factor is below @min_vscale,
+ * decrease the height of rectangle @dst to compensate.
+ *
+ * If the calculated scaling factor is above @max_vscale,
+ * decrease the height of rectangle @src to compensate.
+ *
+ * RETURNS:
+ * The vertical scaling factor.
+ */
+int drm_rect_calc_vscale_relaxed(struct drm_rect *src,
+                                 struct drm_rect *dst,
+                                 int min_vscale, int max_vscale)
+{
+        int src_h = drm_rect_height(src);
+        int dst_h = drm_rect_height(dst);
+        int vscale = drm_calc_scale(src_h, dst_h);
+
+        if (vscale < 0 || dst_h == 0)
+                return vscale;
+
+        if (vscale < min_vscale) {
+                int max_dst_h = src_h / min_vscale;
+
+                drm_rect_adjust_size(dst, 0, max_dst_h - dst_h);
+
+                return min_vscale;
+        }
+
+        if (vscale > max_vscale) {
+                int max_src_h = dst_h * max_vscale;
+
+                drm_rect_adjust_size(src, 0, max_src_h - src_h);
+
+                return max_vscale;
+        }
+
+        return vscale;
+}
+EXPORT_SYMBOL(drm_rect_calc_vscale_relaxed);
+
+/**
+ * drm_rect_debug_print - print the rectangle information
+ * @r: rectangle to print
+ * @fixed_point: rectangle is in 16.16 fixed point format
+ */
+void drm_rect_debug_print(const struct drm_rect *r, bool fixed_point)
+{
+        int w = drm_rect_width(r);
+        int h = drm_rect_height(r);
+
+        if (fixed_point)
+                DRM_DEBUG_KMS("%d.%06ux%d.%06u%+d.%06u%+d.%06u\n",
+                              w >> 16, ((w & 0xffff) * 15625) >> 10,
+                              h >> 16, ((h & 0xffff) * 15625) >> 10,
+                              r->x1 >> 16, ((r->x1 & 0xffff) * 15625) >> 10,
+                              r->y1 >> 16, ((r->y1 & 0xffff) * 15625) >> 10);
+        else
+                DRM_DEBUG_KMS("%dx%d%+d%+d\n", w, h, r->x1, r->y1);
+}
+EXPORT_SYMBOL(drm_rect_debug_print);
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index 16f3ec579b3b..327ca19cda85 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -203,7 +203,7 @@ EXPORT_SYMBOL(drm_master_put);
 int drm_setmaster_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *file_priv)
 {
-        int ret;
+        int ret = 0;
 
         if (file_priv->is_master)
                 return 0;
@@ -229,7 +229,7 @@ int drm_setmaster_ioctl(struct drm_device *dev, void *data,
         }
         mutex_unlock(&dev->struct_mutex);
 
-        return 0;
+        return ret;
 }
 
 int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
@@ -451,14 +451,8 @@ void drm_put_dev(struct drm_device *dev)
 
         drm_lastclose(dev);
 
-        if (drm_core_has_MTRR(dev) && drm_core_has_AGP(dev) &&
-            dev->agp && dev->agp->agp_mtrr >= 0) {
-                int retval;
-                retval = mtrr_del(dev->agp->agp_mtrr,
-                                  dev->agp->agp_info.aper_base,
-                                  dev->agp->agp_info.aper_size * 1024 * 1024);
-                DRM_DEBUG("mtrr_del=%d\n", retval);
-        }
+        if (drm_core_has_MTRR(dev) && drm_core_has_AGP(dev) && dev->agp)
+                arch_phys_wc_del(dev->agp->agp_mtrr);
 
         if (dev->driver->unload)
                 dev->driver->unload(dev);
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 02296653a058..2290b3b73832 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -30,14 +30,14 @@ static struct device_type drm_sysfs_device_minor = {
 };
 
 /**
- * drm_class_suspend - DRM class suspend hook
+ * __drm_class_suspend - internal DRM class suspend routine
  * @dev: Linux device to suspend
  * @state: power state to enter
  *
  * Just figures out what the actual struct drm_device associated with
  * @dev is and calls its suspend hook, if present.
  */
-static int drm_class_suspend(struct device *dev, pm_message_t state)
+static int __drm_class_suspend(struct device *dev, pm_message_t state)
 {
         if (dev->type == &drm_sysfs_device_minor) {
                 struct drm_minor *drm_minor = to_drm_minor(dev);
@@ -52,6 +52,26 @@ static int drm_class_suspend(struct device *dev, pm_message_t state)
 }
 
 /**
+ * drm_class_suspend - internal DRM class suspend hook. Simply calls
+ * __drm_class_suspend() with the correct pm state.
+ * @dev: Linux device to suspend
+ */
+static int drm_class_suspend(struct device *dev)
+{
+        return __drm_class_suspend(dev, PMSG_SUSPEND);
+}
+
+/**
+ * drm_class_freeze - internal DRM class freeze hook. Simply calls
+ * __drm_class_suspend() with the correct pm state.
+ * @dev: Linux device to freeze
+ */
+static int drm_class_freeze(struct device *dev)
+{
+        return __drm_class_suspend(dev, PMSG_FREEZE);
+}
+
+/**
  * drm_class_resume - DRM class resume hook
  * @dev: Linux device to resume
  *
@@ -72,6 +92,12 @@ static int drm_class_resume(struct device *dev)
         return 0;
 }
 
+static const struct dev_pm_ops drm_class_dev_pm_ops = {
+        .suspend        = drm_class_suspend,
+        .resume         = drm_class_resume,
+        .freeze         = drm_class_freeze,
+};
+
 static char *drm_devnode(struct device *dev, umode_t *mode)
 {
         return kasprintf(GFP_KERNEL, "dri/%s", dev_name(dev));
@@ -106,8 +132,7 @@ struct class *drm_sysfs_create(struct module *owner, char *name)
                 goto err_out;
         }
 
-        class->suspend = drm_class_suspend;
-        class->resume = drm_class_resume;
+        class->pm = &drm_class_dev_pm_ops;
 
         err = class_create_file(class, &class_attr_version.attr);
         if (err)
diff --git a/drivers/gpu/drm/drm_trace.h b/drivers/gpu/drm/drm_trace.h
index 03ea964aa604..27cc95f36381 100644
--- a/drivers/gpu/drm/drm_trace.h
+++ b/drivers/gpu/drm/drm_trace.h
@@ -21,7 +21,7 @@ TRACE_EVENT(drm_vblank_event,
                     __entry->crtc = crtc;
                     __entry->seq = seq;
                     ),
-            TP_printk("crtc=%d, seq=%d", __entry->crtc, __entry->seq)
+            TP_printk("crtc=%d, seq=%u", __entry->crtc, __entry->seq)
 );
 
 TRACE_EVENT(drm_vblank_event_queued,
@@ -37,7 +37,7 @@ TRACE_EVENT(drm_vblank_event_queued,
                     __entry->crtc = crtc;
                     __entry->seq = seq;
                     ),
-            TP_printk("pid=%d, crtc=%d, seq=%d", __entry->pid, __entry->crtc, \
+            TP_printk("pid=%d, crtc=%d, seq=%u", __entry->pid, __entry->crtc, \
                       __entry->seq)
 );
 
@@ -54,7 +54,7 @@ TRACE_EVENT(drm_vblank_event_delivered,
                     __entry->crtc = crtc;
                     __entry->seq = seq;
                     ),
-            TP_printk("pid=%d, crtc=%d, seq=%d", __entry->pid, __entry->crtc, \
+            TP_printk("pid=%d, crtc=%d, seq=%u", __entry->pid, __entry->crtc, \
                       __entry->seq)
 );
 
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index 1d4f7c9fe661..feb20035b2c4 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -43,18 +43,19 @@
 static void drm_vm_open(struct vm_area_struct *vma);
 static void drm_vm_close(struct vm_area_struct *vma);
 
-static pgprot_t drm_io_prot(uint32_t map_type, struct vm_area_struct *vma)
+static pgprot_t drm_io_prot(struct drm_local_map *map,
+                            struct vm_area_struct *vma)
 {
         pgprot_t tmp = vm_get_page_prot(vma->vm_flags);
 
 #if defined(__i386__) || defined(__x86_64__)
-        if (boot_cpu_data.x86 > 3 && map_type != _DRM_AGP) {
-                pgprot_val(tmp) |= _PAGE_PCD;
-                pgprot_val(tmp) &= ~_PAGE_PWT;
-        }
+        if (map->type == _DRM_REGISTERS && !(map->flags & _DRM_WRITE_COMBINING))
+                tmp = pgprot_noncached(tmp);
+        else
+                tmp = pgprot_writecombine(tmp);
 #elif defined(__powerpc__)
         pgprot_val(tmp) |= _PAGE_NO_CACHE;
-        if (map_type == _DRM_REGISTERS)
+        if (map->type == _DRM_REGISTERS)
                 pgprot_val(tmp) |= _PAGE_GUARDED;
 #elif defined(__ia64__)
         if (efi_range_is_wc(vma->vm_start, vma->vm_end -
@@ -250,13 +251,8 @@ static void drm_vm_shm_close(struct vm_area_struct *vma)
                         switch (map->type) {
                         case _DRM_REGISTERS:
                         case _DRM_FRAME_BUFFER:
-                                if (drm_core_has_MTRR(dev) && map->mtrr >= 0) {
-                                        int retcode;
-                                        retcode = mtrr_del(map->mtrr,
-                                                           map->offset,
-                                                           map->size);
-                                        DRM_DEBUG("mtrr_del = %d\n", retcode);
-                                }
+                                if (drm_core_has_MTRR(dev))
+                                        arch_phys_wc_del(map->mtrr);
                                 iounmap(map->handle);
                                 break;
                         case _DRM_SHM:
@@ -617,8 +613,7 @@ int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
         case _DRM_FRAME_BUFFER:
         case _DRM_REGISTERS:
                 offset = drm_core_get_reg_ofs(dev);
-                vma->vm_flags |= VM_IO; /* not in core dump */
-                vma->vm_page_prot = drm_io_prot(map->type, vma);
+                vma->vm_page_prot = drm_io_prot(map, vma);
                 if (io_remap_pfn_range(vma, vma->vm_start,
                                        (map->offset + offset) >> PAGE_SHIFT,
                                        vma->vm_end - vma->vm_start,
diff --git a/drivers/gpu/drm/exynos/exynos_ddc.c b/drivers/gpu/drm/exynos/exynos_ddc.c
index 4e9b5ba8edff..95c75edef01a 100644
--- a/drivers/gpu/drm/exynos/exynos_ddc.c
+++ b/drivers/gpu/drm/exynos/exynos_ddc.c
@@ -53,6 +53,8 @@ static struct of_device_id hdmiddc_match_types[] = {
         {
                 .compatible = "samsung,exynos5-hdmiddc",
         }, {
+                .compatible = "samsung,exynos4210-hdmiddc",
+        }, {
                 /* end node */
         }
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_buf.c b/drivers/gpu/drm/exynos/exynos_drm_buf.c
index 57affae9568b..22865baa03b1 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_buf.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_buf.c
@@ -24,8 +24,6 @@ static int lowlevel_buffer_allocate(struct drm_device *dev,
         enum dma_attr attr;
         unsigned int nr_pages;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (buf->dma_addr) {
                 DRM_DEBUG_KMS("already allocated.\n");
                 return 0;
@@ -119,8 +117,6 @@ err_free_attrs:
 static void lowlevel_buffer_deallocate(struct drm_device *dev,
                 unsigned int flags, struct exynos_drm_gem_buf *buf)
 {
-        DRM_DEBUG_KMS("%s.\n", __FILE__);
-
         if (!buf->dma_addr) {
                 DRM_DEBUG_KMS("dma_addr is invalid.\n");
                 return;
@@ -151,7 +147,6 @@ struct exynos_drm_gem_buf *exynos_drm_init_buf(struct drm_device *dev,
 {
         struct exynos_drm_gem_buf *buffer;
 
-        DRM_DEBUG_KMS("%s.\n", __FILE__);
         DRM_DEBUG_KMS("desired size = 0x%x\n", size);
 
         buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
@@ -167,8 +162,6 @@ struct exynos_drm_gem_buf *exynos_drm_init_buf(struct drm_device *dev,
 void exynos_drm_fini_buf(struct drm_device *dev,
                                 struct exynos_drm_gem_buf *buffer)
 {
-        DRM_DEBUG_KMS("%s.\n", __FILE__);
-
         if (!buffer) {
                 DRM_DEBUG_KMS("buffer is null.\n");
                 return;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_connector.c b/drivers/gpu/drm/exynos/exynos_drm_connector.c
index 8bcc13ac9f73..02a8bc5226ca 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_connector.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_connector.c
@@ -34,7 +34,6 @@ convert_to_display_mode(struct drm_display_mode *mode,
                         struct exynos_drm_panel_info *panel)
 {
         struct fb_videomode *timing = &panel->timing;
-        DRM_DEBUG_KMS("%s\n", __FILE__);
 
         mode->clock = timing->pixclock / 1000;
         mode->vrefresh = timing->refresh;
@@ -58,37 +57,6 @@ convert_to_display_mode(struct drm_display_mode *mode,
                 mode->flags |= DRM_MODE_FLAG_DBLSCAN;
 }
 
-/* convert drm_display_mode to exynos_video_timings */
-static inline void
-convert_to_video_timing(struct fb_videomode *timing,
-                        struct drm_display_mode *mode)
-{
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
-        memset(timing, 0, sizeof(*timing));
-
-        timing->pixclock = mode->clock * 1000;
-        timing->refresh = drm_mode_vrefresh(mode);
-
-        timing->xres = mode->hdisplay;
-        timing->right_margin = mode->hsync_start - mode->hdisplay;
-        timing->hsync_len = mode->hsync_end - mode->hsync_start;
-        timing->left_margin = mode->htotal - mode->hsync_end;
-
-        timing->yres = mode->vdisplay;
-        timing->lower_margin = mode->vsync_start - mode->vdisplay;
-        timing->vsync_len = mode->vsync_end - mode->vsync_start;
-        timing->upper_margin = mode->vtotal - mode->vsync_end;
-
-        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-                timing->vmode = FB_VMODE_INTERLACED;
-        else
-                timing->vmode = FB_VMODE_NONINTERLACED;
-
-        if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-                timing->vmode |= FB_VMODE_DOUBLE;
-}
-
 static int exynos_drm_connector_get_modes(struct drm_connector *connector)
 {
         struct exynos_drm_connector *exynos_connector =
@@ -99,8 +67,6 @@ static int exynos_drm_connector_get_modes(struct drm_connector *connector)
         unsigned int count = 0;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!display_ops) {
                 DRM_DEBUG_KMS("display_ops is null.\n");
                 return 0;
@@ -168,15 +134,12 @@ static int exynos_drm_connector_mode_valid(struct drm_connector *connector,
                                         to_exynos_connector(connector);
         struct exynos_drm_manager *manager = exynos_connector->manager;
         struct exynos_drm_display_ops *display_ops = manager->display_ops;
-        struct fb_videomode timing;
         int ret = MODE_BAD;
 
         DRM_DEBUG_KMS("%s\n", __FILE__);
 
-        convert_to_video_timing(&timing, mode);
-
-        if (display_ops && display_ops->check_timing)
-                if (!display_ops->check_timing(manager->dev, (void *)&timing))
+        if (display_ops && display_ops->check_mode)
+                if (!display_ops->check_mode(manager->dev, mode))
                         ret = MODE_OK;
 
         return ret;
@@ -190,8 +153,6 @@ struct drm_encoder *exynos_drm_best_encoder(struct drm_connector *connector)
         struct drm_mode_object *obj;
         struct drm_encoder *encoder;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         obj = drm_mode_object_find(dev, exynos_connector->encoder_id,
                                    DRM_MODE_OBJECT_ENCODER);
         if (!obj) {
@@ -234,8 +195,6 @@ void exynos_drm_display_power(struct drm_connector *connector, int mode)
 static void exynos_drm_connector_dpms(struct drm_connector *connector,
                                         int mode)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * in case that drm_crtc_helper_set_mode() is called,
          * encoder/crtc->funcs->dpms() will be just returned
@@ -282,8 +241,6 @@ exynos_drm_connector_detect(struct drm_connector *connector, bool force)
                                         manager->display_ops;
         enum drm_connector_status status = connector_status_disconnected;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (display_ops && display_ops->is_connected) {
                 if (display_ops->is_connected(manager->dev))
                         status = connector_status_connected;
@@ -299,8 +256,6 @@ static void exynos_drm_connector_destroy(struct drm_connector *connector)
         struct exynos_drm_connector *exynos_connector =
                 to_exynos_connector(connector);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         drm_sysfs_connector_remove(connector);
         drm_connector_cleanup(connector);
         kfree(exynos_connector);
@@ -322,8 +277,6 @@ struct drm_connector *exynos_drm_connector_create(struct drm_device *dev,
         int type;
         int err;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_connector = kzalloc(sizeof(*exynos_connector), GFP_KERNEL);
         if (!exynos_connector) {
                 DRM_ERROR("failed to allocate connector\n");
diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c
index 4667c9f67acd..1bef6dc77478 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_core.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_core.c
@@ -27,8 +27,6 @@ static int exynos_drm_create_enc_conn(struct drm_device *dev,
         struct drm_connector *connector;
         int ret;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         subdrv->manager->dev = subdrv->dev;
 
         /* create and initialize a encoder for this sub driver. */
@@ -102,8 +100,6 @@ static int exynos_drm_subdrv_probe(struct drm_device *dev,
 static void exynos_drm_subdrv_remove(struct drm_device *dev,
                                       struct exynos_drm_subdrv *subdrv)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         if (subdrv->remove)
                 subdrv->remove(dev, subdrv->dev);
 }
@@ -114,8 +110,6 @@ int exynos_drm_device_register(struct drm_device *dev)
         unsigned int fine_cnt = 0;
         int err;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         if (!dev)
                 return -EINVAL;
 
@@ -158,8 +152,6 @@ int exynos_drm_device_unregister(struct drm_device *dev)
 {
         struct exynos_drm_subdrv *subdrv;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         if (!dev) {
                 WARN(1, "Unexpected drm device unregister!\n");
                 return -EINVAL;
@@ -176,8 +168,6 @@ EXPORT_SYMBOL_GPL(exynos_drm_device_unregister);
 
 int exynos_drm_subdrv_register(struct exynos_drm_subdrv *subdrv)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         if (!subdrv)
                 return -EINVAL;
 
@@ -189,8 +179,6 @@ EXPORT_SYMBOL_GPL(exynos_drm_subdrv_register);
 
 int exynos_drm_subdrv_unregister(struct exynos_drm_subdrv *subdrv)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         if (!subdrv)
                 return -EINVAL;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index c200e4d71e3d..9a35d171a6d3 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -76,8 +76,6 @@ static void exynos_drm_crtc_dpms(struct drm_crtc *crtc, int mode)
 
 static void exynos_drm_crtc_prepare(struct drm_crtc *crtc)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* drm framework doesn't check NULL. */
 }
 
@@ -85,8 +83,6 @@ static void exynos_drm_crtc_commit(struct drm_crtc *crtc)
 {
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_drm_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
         exynos_plane_commit(exynos_crtc->plane);
         exynos_plane_dpms(exynos_crtc->plane, DRM_MODE_DPMS_ON);
@@ -97,8 +93,6 @@ exynos_drm_crtc_mode_fixup(struct drm_crtc *crtc,
                             const struct drm_display_mode *mode,
                             struct drm_display_mode *adjusted_mode)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* drm framework doesn't check NULL */
         return true;
 }
@@ -115,8 +109,6 @@ exynos_drm_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
         int pipe = exynos_crtc->pipe;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * copy the mode data adjusted by mode_fixup() into crtc->mode
          * so that hardware can be seet to proper mode.
@@ -139,7 +131,7 @@ exynos_drm_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
         return 0;
 }
 
-static int exynos_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
+static int exynos_drm_crtc_mode_set_commit(struct drm_crtc *crtc, int x, int y,
                                           struct drm_framebuffer *old_fb)
 {
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
@@ -148,8 +140,6 @@ static int exynos_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         unsigned int crtc_h;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* when framebuffer changing is requested, crtc's dpms should be on */
         if (exynos_crtc->dpms > DRM_MODE_DPMS_ON) {
                 DRM_ERROR("failed framebuffer changing request.\n");
@@ -169,18 +159,16 @@ static int exynos_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         return 0;
 }
 
-static void exynos_drm_crtc_load_lut(struct drm_crtc *crtc)
+static int exynos_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
+                                          struct drm_framebuffer *old_fb)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-        /* drm framework doesn't check NULL */
+        return exynos_drm_crtc_mode_set_commit(crtc, x, y, old_fb);
 }
 
 static void exynos_drm_crtc_disable(struct drm_crtc *crtc)
 {
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_plane_dpms(exynos_crtc->plane, DRM_MODE_DPMS_OFF);
         exynos_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
 }
@@ -192,7 +180,6 @@ static struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
         .mode_fixup     = exynos_drm_crtc_mode_fixup,
         .mode_set       = exynos_drm_crtc_mode_set,
         .mode_set_base  = exynos_drm_crtc_mode_set_base,
-        .load_lut       = exynos_drm_crtc_load_lut,
         .disable        = exynos_drm_crtc_disable,
 };
 
@@ -206,8 +193,6 @@ static int exynos_drm_crtc_page_flip(struct drm_crtc *crtc,
         struct drm_framebuffer *old_fb = crtc->fb;
         int ret = -EINVAL;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* when the page flip is requested, crtc's dpms should be on */
         if (exynos_crtc->dpms > DRM_MODE_DPMS_ON) {
                 DRM_ERROR("failed page flip request.\n");
@@ -237,7 +222,7 @@ static int exynos_drm_crtc_page_flip(struct drm_crtc *crtc,
                 spin_unlock_irq(&dev->event_lock);
 
                 crtc->fb = fb;
-                ret = exynos_drm_crtc_mode_set_base(crtc, crtc->x, crtc->y,
+                ret = exynos_drm_crtc_mode_set_commit(crtc, crtc->x, crtc->y,
                                                     NULL);
                 if (ret) {
                         crtc->fb = old_fb;
@@ -260,8 +245,6 @@ static void exynos_drm_crtc_destroy(struct drm_crtc *crtc)
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
         struct exynos_drm_private *private = crtc->dev->dev_private;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         private->crtc[exynos_crtc->pipe] = NULL;
 
         drm_crtc_cleanup(crtc);
@@ -276,8 +259,6 @@ static int exynos_drm_crtc_set_property(struct drm_crtc *crtc,
         struct exynos_drm_private *dev_priv = dev->dev_private;
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (property == dev_priv->crtc_mode_property) {
                 enum exynos_crtc_mode mode = val;
 
@@ -322,8 +303,6 @@ static void exynos_drm_crtc_attach_mode_property(struct drm_crtc *crtc)
         struct exynos_drm_private *dev_priv = dev->dev_private;
         struct drm_property *prop;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         prop = dev_priv->crtc_mode_property;
         if (!prop) {
                 prop = drm_property_create_enum(dev, 0, "mode", mode_names,
@@ -343,8 +322,6 @@ int exynos_drm_crtc_create(struct drm_device *dev, unsigned int nr)
         struct exynos_drm_private *private = dev->dev_private;
         struct drm_crtc *crtc;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_crtc = kzalloc(sizeof(*exynos_crtc), GFP_KERNEL);
         if (!exynos_crtc) {
                 DRM_ERROR("failed to allocate exynos crtc\n");
@@ -379,8 +356,6 @@ int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int crtc)
         struct exynos_drm_crtc *exynos_crtc =
                 to_exynos_crtc(private->crtc[crtc]);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (exynos_crtc->dpms != DRM_MODE_DPMS_ON)
                 return -EPERM;
 
@@ -396,8 +371,6 @@ void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int crtc)
         struct exynos_drm_crtc *exynos_crtc =
                 to_exynos_crtc(private->crtc[crtc]);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (exynos_crtc->dpms != DRM_MODE_DPMS_ON)
                 return;
 
@@ -413,8 +386,6 @@ void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int crtc)
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(drm_crtc);
         unsigned long flags;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         spin_lock_irqsave(&dev->event_lock, flags);
 
         list_for_each_entry_safe(e, t, &dev_priv->pageflip_event_list,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c b/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c
index ff7f2a886a34..a0f997e0cbdf 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c
@@ -71,8 +71,6 @@ static struct sg_table *
         unsigned int i;
         int nents, ret;
 
-        DRM_DEBUG_PRIME("%s\n", __FILE__);
-
         /* just return current sgt if already requested. */
         if (exynos_attach->dir == dir && exynos_attach->is_mapped)
                 return &exynos_attach->sgt;
@@ -133,8 +131,6 @@ static void exynos_dmabuf_release(struct dma_buf *dmabuf)
 {
         struct exynos_drm_gem_obj *exynos_gem_obj = dmabuf->priv;
 
-        DRM_DEBUG_PRIME("%s\n", __FILE__);
-
         /*
          * exynos_dmabuf_release() call means that file object's
          * f_count is 0 and it calls drm_gem_object_handle_unreference()
@@ -219,8 +215,6 @@ struct drm_gem_object *exynos_dmabuf_prime_import(struct drm_device *drm_dev,
         struct exynos_drm_gem_buf *buffer;
         int ret;
 
-        DRM_DEBUG_PRIME("%s\n", __FILE__);
-
         /* is this one of own objects? */
         if (dma_buf->ops == &exynos_dmabuf_ops) {
                 struct drm_gem_object *obj;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index ba6d995e4375..276237348d1e 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -46,8 +46,6 @@ static int exynos_drm_load(struct drm_device *dev, unsigned long flags)
         int ret;
         int nr;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         private = kzalloc(sizeof(struct exynos_drm_private), GFP_KERNEL);
         if (!private) {
                 DRM_ERROR("failed to allocate private\n");
@@ -140,8 +138,6 @@ err_crtc:
 
 static int exynos_drm_unload(struct drm_device *dev)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         exynos_drm_fbdev_fini(dev);
         exynos_drm_device_unregister(dev);
         drm_vblank_cleanup(dev);
@@ -160,8 +156,6 @@ static int exynos_drm_open(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_exynos_file_private *file_priv;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
         if (!file_priv)
                 return -ENOMEM;
@@ -178,8 +172,6 @@ static void exynos_drm_preclose(struct drm_device *dev,
         struct drm_pending_vblank_event *e, *t;
         unsigned long flags;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         /* release events of current file */
         spin_lock_irqsave(&dev->event_lock, flags);
         list_for_each_entry_safe(e, t, &private->pageflip_event_list,
@@ -196,8 +188,6 @@ static void exynos_drm_preclose(struct drm_device *dev,
 
 static void exynos_drm_postclose(struct drm_device *dev, struct drm_file *file)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         if (!file->driver_priv)
                 return;
 
@@ -207,8 +197,6 @@ static void exynos_drm_postclose(struct drm_device *dev, struct drm_file *file)
 
 static void exynos_drm_lastclose(struct drm_device *dev)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         exynos_drm_fbdev_restore_mode(dev);
 }
 
@@ -292,8 +280,6 @@ static struct drm_driver exynos_drm_driver = {
 
 static int exynos_drm_platform_probe(struct platform_device *pdev)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
         exynos_drm_driver.num_ioctls = DRM_ARRAY_SIZE(exynos_ioctls);
 
@@ -302,8 +288,6 @@ static int exynos_drm_platform_probe(struct platform_device *pdev)
 
 static int exynos_drm_platform_remove(struct platform_device *pdev)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         drm_platform_exit(&exynos_drm_driver, pdev);
 
         return 0;
@@ -322,8 +306,6 @@ static int __init exynos_drm_init(void)
 {
         int ret;
 
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
 #ifdef CONFIG_DRM_EXYNOS_FIMD
         ret = platform_driver_register(&fimd_driver);
         if (ret < 0)
@@ -455,8 +437,6 @@ out_fimd:
 
 static void __exit exynos_drm_exit(void)
 {
-        DRM_DEBUG_DRIVER("%s\n", __FILE__);
-
         platform_device_unregister(exynos_drm_pdev);
 
         platform_driver_unregister(&exynos_drm_platform_driver);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index 680a7c1b9dea..eaa19668bf00 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -142,7 +142,7 @@ struct exynos_drm_overlay {
  * @is_connected: check for that display is connected or not.
  * @get_edid: get edid modes from display driver.
  * @get_panel: get panel object from display driver.
- * @check_timing: check if timing is valid or not.
+ * @check_mode: check if mode is valid or not.
  * @power_on: display device on or off.
  */
 struct exynos_drm_display_ops {
@@ -151,7 +151,7 @@ struct exynos_drm_display_ops {
         struct edid *(*get_edid)(struct device *dev,
                         struct drm_connector *connector);
         void *(*get_panel)(struct device *dev);
-        int (*check_timing)(struct device *dev, void *timing);
+        int (*check_mode)(struct device *dev, struct drm_display_mode *mode);
         int (*power_on)(struct device *dev, int mode);
 };
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_encoder.c b/drivers/gpu/drm/exynos/exynos_drm_encoder.c
index c63721f64aec..a99a033793bc 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_encoder.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_encoder.c
@@ -61,7 +61,7 @@ static void exynos_drm_encoder_dpms(struct drm_encoder *encoder, int mode)
         struct exynos_drm_manager_ops *manager_ops = manager->ops;
         struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
 
-        DRM_DEBUG_KMS("%s, encoder dpms: %d\n", __FILE__, mode);
+        DRM_DEBUG_KMS("encoder dpms: %d\n", mode);
 
         if (exynos_encoder->dpms == mode) {
                 DRM_DEBUG_KMS("desired dpms mode is same as previous one.\n");
@@ -104,8 +104,6 @@ exynos_drm_encoder_mode_fixup(struct drm_encoder *encoder,
         struct exynos_drm_manager *manager = exynos_drm_get_manager(encoder);
         struct exynos_drm_manager_ops *manager_ops = manager->ops;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 if (connector->encoder == encoder)
                         if (manager_ops && manager_ops->mode_fixup)
@@ -155,8 +153,6 @@ static void exynos_drm_encoder_mode_set(struct drm_encoder *encoder,
         struct exynos_drm_manager *manager;
         struct exynos_drm_manager_ops *manager_ops;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 if (connector->encoder == encoder) {
                         struct exynos_drm_encoder *exynos_encoder;
@@ -189,8 +185,6 @@ static void exynos_drm_encoder_mode_set(struct drm_encoder *encoder,
 
 static void exynos_drm_encoder_prepare(struct drm_encoder *encoder)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* drm framework doesn't check NULL. */
 }
 
@@ -200,8 +194,6 @@ static void exynos_drm_encoder_commit(struct drm_encoder *encoder)
         struct exynos_drm_manager *manager = exynos_encoder->manager;
         struct exynos_drm_manager_ops *manager_ops = manager->ops;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (manager_ops && manager_ops->commit)
                 manager_ops->commit(manager->dev);
 
@@ -274,8 +266,6 @@ static void exynos_drm_encoder_destroy(struct drm_encoder *encoder)
         struct exynos_drm_encoder *exynos_encoder =
                 to_exynos_encoder(encoder);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_encoder->manager->pipe = -1;
 
         drm_encoder_cleanup(encoder);
@@ -315,8 +305,6 @@ void exynos_drm_encoder_setup(struct drm_device *dev)
 {
         struct drm_encoder *encoder;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
                 encoder->possible_clones = exynos_drm_encoder_clones(encoder);
 }
@@ -329,8 +317,6 @@ exynos_drm_encoder_create(struct drm_device *dev,
         struct drm_encoder *encoder;
         struct exynos_drm_encoder *exynos_encoder;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!manager || !possible_crtcs)
                 return NULL;
 
@@ -427,8 +413,6 @@ void exynos_drm_encoder_crtc_dpms(struct drm_encoder *encoder, void *data)
         struct exynos_drm_manager_ops *manager_ops = manager->ops;
         int mode = *(int *)data;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (manager_ops && manager_ops->dpms)
                 manager_ops->dpms(manager->dev, mode);
 
@@ -449,8 +433,6 @@ void exynos_drm_encoder_crtc_pipe(struct drm_encoder *encoder, void *data)
                 to_exynos_encoder(encoder)->manager;
         int pipe = *(int *)data;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * when crtc is detached from encoder, this pipe is used
          * to select manager operation
@@ -465,8 +447,6 @@ void exynos_drm_encoder_plane_mode_set(struct drm_encoder *encoder, void *data)
         struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
         struct exynos_drm_overlay *overlay = data;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (overlay_ops && overlay_ops->mode_set)
                 overlay_ops->mode_set(manager->dev, overlay);
 }
@@ -478,8 +458,6 @@ void exynos_drm_encoder_plane_commit(struct drm_encoder *encoder, void *data)
         struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
         int zpos = DEFAULT_ZPOS;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (data)
                 zpos = *(int *)data;
 
@@ -494,8 +472,6 @@ void exynos_drm_encoder_plane_enable(struct drm_encoder *encoder, void *data)
         struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
         int zpos = DEFAULT_ZPOS;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (data)
                 zpos = *(int *)data;
 
@@ -510,8 +486,6 @@ void exynos_drm_encoder_plane_disable(struct drm_encoder *encoder, void *data)
         struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
         int zpos = DEFAULT_ZPOS;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (data)
                 zpos = *(int *)data;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 0e04f4ea441f..c2d149f0408a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -70,8 +70,6 @@ static void exynos_drm_fb_destroy(struct drm_framebuffer *fb)
         struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
         unsigned int i;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* make sure that overlay data are updated before relesing fb. */
         exynos_drm_encoder_complete_scanout(fb);
 
@@ -97,8 +95,6 @@ static int exynos_drm_fb_create_handle(struct drm_framebuffer *fb,
 {
         struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* This fb should have only one gem object. */
         if (WARN_ON(exynos_fb->buf_cnt != 1))
                 return -EINVAL;
@@ -112,8 +108,6 @@ static int exynos_drm_fb_dirty(struct drm_framebuffer *fb,
                                 unsigned color, struct drm_clip_rect *clips,
                                 unsigned num_clips)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO */
 
         return 0;
@@ -225,8 +219,6 @@ exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
         struct exynos_drm_fb *exynos_fb;
         int i, ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_fb = kzalloc(sizeof(*exynos_fb), GFP_KERNEL);
         if (!exynos_fb) {
                 DRM_ERROR("failed to allocate exynos drm framebuffer\n");
@@ -293,8 +285,6 @@ struct exynos_drm_gem_buf *exynos_drm_fb_buffer(struct drm_framebuffer *fb,
         struct exynos_drm_fb *exynos_fb = to_exynos_fb(fb);
         struct exynos_drm_gem_buf *buffer;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (index >= MAX_FB_BUFFER)
                 return NULL;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index 8f007aaeffc3..8e60bd61137f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -43,8 +43,6 @@ static int exynos_drm_fb_mmap(struct fb_info *info,
         unsigned long vm_size;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
 
         vm_size = vma->vm_end - vma->vm_start;
@@ -84,8 +82,6 @@ static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
         unsigned int size = fb->width * fb->height * (fb->bits_per_pixel >> 3);
         unsigned long offset;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->depth);
         drm_fb_helper_fill_var(fbi, helper, fb->width, fb->height);
 
@@ -148,8 +144,6 @@ static int exynos_drm_fbdev_create(struct drm_fb_helper *helper,
         unsigned long size;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         DRM_DEBUG_KMS("surface width(%d), height(%d) and bpp(%d\n",
                         sizes->surface_width, sizes->surface_height,
                         sizes->surface_bpp);
@@ -238,8 +232,6 @@ int exynos_drm_fbdev_init(struct drm_device *dev)
         unsigned int num_crtc;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!dev->mode_config.num_crtc || !dev->mode_config.num_connector)
                 return 0;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index 4a1616a18ab7..61b094f689a7 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -175,8 +175,6 @@ static void fimc_sw_reset(struct fimc_context *ctx)
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* stop dma operation */
         cfg = fimc_read(EXYNOS_CISTATUS);
         if (EXYNOS_CISTATUS_GET_ENVID_STATUS(cfg)) {
@@ -210,8 +208,6 @@ static void fimc_sw_reset(struct fimc_context *ctx)
 
 static int fimc_set_camblk_fimd0_wb(struct fimc_context *ctx)
 {
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         return regmap_update_bits(ctx->sysreg, SYSREG_CAMERA_BLK,
                                   SYSREG_FIMD0WB_DEST_MASK,
                                   ctx->id << SYSREG_FIMD0WB_DEST_SHIFT);
@@ -221,7 +217,7 @@ static void fimc_set_type_ctrl(struct fimc_context *ctx, enum fimc_wb wb)
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:wb[%d]\n", __func__, wb);
+        DRM_DEBUG_KMS("wb[%d]\n", wb);
 
         cfg = fimc_read(EXYNOS_CIGCTRL);
         cfg &= ~(EXYNOS_CIGCTRL_TESTPATTERN_MASK |
@@ -257,10 +253,10 @@ static void fimc_set_polarity(struct fimc_context *ctx,
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:inv_pclk[%d]inv_vsync[%d]\n",
-                __func__, pol->inv_pclk, pol->inv_vsync);
-        DRM_DEBUG_KMS("%s:inv_href[%d]inv_hsync[%d]\n",
-                __func__, pol->inv_href, pol->inv_hsync);
+        DRM_DEBUG_KMS("inv_pclk[%d]inv_vsync[%d]\n",
+                pol->inv_pclk, pol->inv_vsync);
+        DRM_DEBUG_KMS("inv_href[%d]inv_hsync[%d]\n",
+                pol->inv_href, pol->inv_hsync);
 
         cfg = fimc_read(EXYNOS_CIGCTRL);
         cfg &= ~(EXYNOS_CIGCTRL_INVPOLPCLK | EXYNOS_CIGCTRL_INVPOLVSYNC |
@@ -282,7 +278,7 @@ static void fimc_handle_jpeg(struct fimc_context *ctx, bool enable)
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:enable[%d]\n", __func__, enable);
+        DRM_DEBUG_KMS("enable[%d]\n", enable);
 
         cfg = fimc_read(EXYNOS_CIGCTRL);
         if (enable)
@@ -298,7 +294,7 @@ static void fimc_handle_irq(struct fimc_context *ctx, bool enable,
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:enable[%d]overflow[%d]level[%d]\n", __func__,
+        DRM_DEBUG_KMS("enable[%d]overflow[%d]level[%d]\n",
                         enable, overflow, level);
 
         cfg = fimc_read(EXYNOS_CIGCTRL);
@@ -319,8 +315,6 @@ static void fimc_clear_irq(struct fimc_context *ctx)
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         cfg = fimc_read(EXYNOS_CIGCTRL);
         cfg |= EXYNOS_CIGCTRL_IRQ_CLR;
         fimc_write(cfg, EXYNOS_CIGCTRL);
@@ -335,7 +329,7 @@ static bool fimc_check_ovf(struct fimc_context *ctx)
         flag = EXYNOS_CISTATUS_OVFIY | EXYNOS_CISTATUS_OVFICB |
                 EXYNOS_CISTATUS_OVFICR;
 
-        DRM_DEBUG_KMS("%s:flag[0x%x]\n", __func__, flag);
+        DRM_DEBUG_KMS("flag[0x%x]\n", flag);
 
         if (status & flag) {
                 cfg = fimc_read(EXYNOS_CIWDOFST);
@@ -364,7 +358,7 @@ static bool fimc_check_frame_end(struct fimc_context *ctx)
 
         cfg = fimc_read(EXYNOS_CISTATUS);
 
-        DRM_DEBUG_KMS("%s:cfg[0x%x]\n", __func__, cfg);
+        DRM_DEBUG_KMS("cfg[0x%x]\n", cfg);
 
         if (!(cfg & EXYNOS_CISTATUS_FRAMEEND))
                 return false;
@@ -380,15 +374,13 @@ static int fimc_get_buf_id(struct fimc_context *ctx)
         u32 cfg;
         int frame_cnt, buf_id;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         cfg = fimc_read(EXYNOS_CISTATUS2);
         frame_cnt = EXYNOS_CISTATUS2_GET_FRAMECOUNT_BEFORE(cfg);
 
         if (frame_cnt == 0)
                 frame_cnt = EXYNOS_CISTATUS2_GET_FRAMECOUNT_PRESENT(cfg);
 
-        DRM_DEBUG_KMS("%s:present[%d]before[%d]\n", __func__,
+        DRM_DEBUG_KMS("present[%d]before[%d]\n",
                 EXYNOS_CISTATUS2_GET_FRAMECOUNT_PRESENT(cfg),
                 EXYNOS_CISTATUS2_GET_FRAMECOUNT_BEFORE(cfg));
 
@@ -398,7 +390,7 @@ static int fimc_get_buf_id(struct fimc_context *ctx)
         }
 
         buf_id = frame_cnt - 1;
-        DRM_DEBUG_KMS("%s:buf_id[%d]\n", __func__, buf_id);
+        DRM_DEBUG_KMS("buf_id[%d]\n", buf_id);
 
         return buf_id;
 }
@@ -407,7 +399,7 @@ static void fimc_handle_lastend(struct fimc_context *ctx, bool enable)
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:enable[%d]\n", __func__, enable);
+        DRM_DEBUG_KMS("enable[%d]\n", enable);
 
         cfg = fimc_read(EXYNOS_CIOCTRL);
         if (enable)
@@ -424,7 +416,7 @@ static int fimc_src_set_fmt_order(struct fimc_context *ctx, u32 fmt)
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:fmt[0x%x]\n", __func__, fmt);
+        DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
 
         /* RGB */
         cfg = fimc_read(EXYNOS_CISCCTRL);
@@ -497,7 +489,7 @@ static int fimc_src_set_fmt(struct device *dev, u32 fmt)
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:fmt[0x%x]\n", __func__, fmt);
+        DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
 
         cfg = fimc_read(EXYNOS_MSCTRL);
         cfg &= ~EXYNOS_MSCTRL_INFORMAT_RGB;
@@ -557,8 +549,7 @@ static int fimc_src_set_transf(struct device *dev,
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg1, cfg2;
 
-        DRM_DEBUG_KMS("%s:degree[%d]flip[0x%x]\n", __func__,
-                degree, flip);
+        DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
 
         cfg1 = fimc_read(EXYNOS_MSCTRL);
         cfg1 &= ~(EXYNOS_MSCTRL_FLIP_X_MIRROR |
@@ -621,10 +612,9 @@ static int fimc_set_window(struct fimc_context *ctx,
         v1 = pos->y;
         v2 = sz->vsize - pos->h - pos->y;
 
-        DRM_DEBUG_KMS("%s:x[%d]y[%d]w[%d]h[%d]hsize[%d]vsize[%d]\n",
-        __func__, pos->x, pos->y, pos->w, pos->h, sz->hsize, sz->vsize);
-        DRM_DEBUG_KMS("%s:h1[%d]h2[%d]v1[%d]v2[%d]\n", __func__,
-                h1, h2, v1, v2);
+        DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]hsize[%d]vsize[%d]\n",
+                pos->x, pos->y, pos->w, pos->h, sz->hsize, sz->vsize);
+        DRM_DEBUG_KMS("h1[%d]h2[%d]v1[%d]v2[%d]\n", h1, h2, v1, v2);
 
         /*
          * set window offset 1, 2 size
@@ -653,8 +643,8 @@ static int fimc_src_set_size(struct device *dev, int swap,
         struct drm_exynos_sz img_sz = *sz;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:swap[%d]hsize[%d]vsize[%d]\n",
-                __func__, swap, sz->hsize, sz->vsize);
+        DRM_DEBUG_KMS("swap[%d]hsize[%d]vsize[%d]\n",
+                swap, sz->hsize, sz->vsize);
 
         /* original size */
         cfg = (EXYNOS_ORGISIZE_HORIZONTAL(img_sz.hsize) |
@@ -662,8 +652,7 @@ static int fimc_src_set_size(struct device *dev, int swap,
 
         fimc_write(cfg, EXYNOS_ORGISIZE);
 
-        DRM_DEBUG_KMS("%s:x[%d]y[%d]w[%d]h[%d]\n", __func__,
-                pos->x, pos->y, pos->w, pos->h);
+        DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", pos->x, pos->y, pos->w, pos->h);
 
         if (swap) {
                 img_pos.w = pos->h;
@@ -720,7 +709,7 @@ static int fimc_src_set_addr(struct device *dev,
 
         property = &c_node->property;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]buf_id[%d]buf_type[%d]\n", __func__,
+        DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
                 property->prop_id, buf_id, buf_type);
 
         if (buf_id > FIMC_MAX_SRC) {
@@ -772,7 +761,7 @@ static int fimc_dst_set_fmt_order(struct fimc_context *ctx, u32 fmt)
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:fmt[0x%x]\n", __func__, fmt);
+        DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
 
         /* RGB */
         cfg = fimc_read(EXYNOS_CISCCTRL);
@@ -851,7 +840,7 @@ static int fimc_dst_set_fmt(struct device *dev, u32 fmt)
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:fmt[0x%x]\n", __func__, fmt);
+        DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
 
         cfg = fimc_read(EXYNOS_CIEXTEN);
 
@@ -919,8 +908,7 @@ static int fimc_dst_set_transf(struct device *dev,
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:degree[%d]flip[0x%x]\n", __func__,
-                degree, flip);
+        DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
 
         cfg = fimc_read(EXYNOS_CITRGFMT);
         cfg &= ~EXYNOS_CITRGFMT_FLIP_MASK;
@@ -970,7 +958,7 @@ static int fimc_dst_set_transf(struct device *dev,
 
 static int fimc_get_ratio_shift(u32 src, u32 dst, u32 *ratio, u32 *shift)
 {
-        DRM_DEBUG_KMS("%s:src[%d]dst[%d]\n", __func__, src, dst);
+        DRM_DEBUG_KMS("src[%d]dst[%d]\n", src, dst);
 
         if (src >= dst * 64) {
                 DRM_ERROR("failed to make ratio and shift.\n");
@@ -1039,20 +1027,20 @@ static int fimc_set_prescaler(struct fimc_context *ctx, struct fimc_scaler *sc,
 
         pre_dst_width = src_w / pre_hratio;
         pre_dst_height = src_h / pre_vratio;
-        DRM_DEBUG_KMS("%s:pre_dst_width[%d]pre_dst_height[%d]\n", __func__,
+        DRM_DEBUG_KMS("pre_dst_width[%d]pre_dst_height[%d]\n",
                 pre_dst_width, pre_dst_height);
-        DRM_DEBUG_KMS("%s:pre_hratio[%d]hfactor[%d]pre_vratio[%d]vfactor[%d]\n",
-                __func__, pre_hratio, hfactor, pre_vratio, vfactor);
+        DRM_DEBUG_KMS("pre_hratio[%d]hfactor[%d]pre_vratio[%d]vfactor[%d]\n",
+                pre_hratio, hfactor, pre_vratio, vfactor);
 
         sc->hratio = (src_w << 14) / (dst_w << hfactor);
         sc->vratio = (src_h << 14) / (dst_h << vfactor);
         sc->up_h = (dst_w >= src_w) ? true : false;
         sc->up_v = (dst_h >= src_h) ? true : false;
-        DRM_DEBUG_KMS("%s:hratio[%d]vratio[%d]up_h[%d]up_v[%d]\n",
-        __func__, sc->hratio, sc->vratio, sc->up_h, sc->up_v);
+        DRM_DEBUG_KMS("hratio[%d]vratio[%d]up_h[%d]up_v[%d]\n",
+                sc->hratio, sc->vratio, sc->up_h, sc->up_v);
 
         shfactor = FIMC_SHFACTOR - (hfactor + vfactor);
-        DRM_DEBUG_KMS("%s:shfactor[%d]\n", __func__, shfactor);
+        DRM_DEBUG_KMS("shfactor[%d]\n", shfactor);
 
         cfg = (EXYNOS_CISCPRERATIO_SHFACTOR(shfactor) |
                 EXYNOS_CISCPRERATIO_PREHORRATIO(pre_hratio) |
@@ -1070,10 +1058,10 @@ static void fimc_set_scaler(struct fimc_context *ctx, struct fimc_scaler *sc)
 {
         u32 cfg, cfg_ext;
 
-        DRM_DEBUG_KMS("%s:range[%d]bypass[%d]up_h[%d]up_v[%d]\n",
-                __func__, sc->range, sc->bypass, sc->up_h, sc->up_v);
-        DRM_DEBUG_KMS("%s:hratio[%d]vratio[%d]\n",
-                __func__, sc->hratio, sc->vratio);
+        DRM_DEBUG_KMS("range[%d]bypass[%d]up_h[%d]up_v[%d]\n",
+                sc->range, sc->bypass, sc->up_h, sc->up_v);
+        DRM_DEBUG_KMS("hratio[%d]vratio[%d]\n",
+                sc->hratio, sc->vratio);
 
         cfg = fimc_read(EXYNOS_CISCCTRL);
         cfg &= ~(EXYNOS_CISCCTRL_SCALERBYPASS |
@@ -1113,8 +1101,8 @@ static int fimc_dst_set_size(struct device *dev, int swap,
         struct drm_exynos_sz img_sz = *sz;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:swap[%d]hsize[%d]vsize[%d]\n",
-                __func__, swap, sz->hsize, sz->vsize);
+        DRM_DEBUG_KMS("swap[%d]hsize[%d]vsize[%d]\n",
+                swap, sz->hsize, sz->vsize);
 
         /* original size */
         cfg = (EXYNOS_ORGOSIZE_HORIZONTAL(img_sz.hsize) |
@@ -1122,8 +1110,7 @@ static int fimc_dst_set_size(struct device *dev, int swap,
 
         fimc_write(cfg, EXYNOS_ORGOSIZE);
 
-        DRM_DEBUG_KMS("%s:x[%d]y[%d]w[%d]h[%d]\n",
-                __func__, pos->x, pos->y, pos->w, pos->h);
+        DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", pos->x, pos->y, pos->w, pos->h);
 
         /* CSC ITU */
         cfg = fimc_read(EXYNOS_CIGCTRL);
@@ -1180,7 +1167,7 @@ static int fimc_dst_get_buf_seq(struct fimc_context *ctx)
                 if (cfg & (mask << i))
                         buf_num++;
 
-        DRM_DEBUG_KMS("%s:buf_num[%d]\n", __func__, buf_num);
+        DRM_DEBUG_KMS("buf_num[%d]\n", buf_num);
 
         return buf_num;
 }
@@ -1194,8 +1181,7 @@ static int fimc_dst_set_buf_seq(struct fimc_context *ctx, u32 buf_id,
         u32 mask = 0x00000001 << buf_id;
         int ret = 0;
 
-        DRM_DEBUG_KMS("%s:buf_id[%d]buf_type[%d]\n", __func__,
-                buf_id, buf_type);
+        DRM_DEBUG_KMS("buf_id[%d]buf_type[%d]\n", buf_id, buf_type);
 
         mutex_lock(&ctx->lock);
 
@@ -1252,7 +1238,7 @@ static int fimc_dst_set_addr(struct device *dev,
 
         property = &c_node->property;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]buf_id[%d]buf_type[%d]\n", __func__,
+        DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
                 property->prop_id, buf_id, buf_type);
 
         if (buf_id > FIMC_MAX_DST) {
@@ -1302,7 +1288,7 @@ static struct exynos_drm_ipp_ops fimc_dst_ops = {
 
 static int fimc_clk_ctrl(struct fimc_context *ctx, bool enable)
 {
-        DRM_DEBUG_KMS("%s:enable[%d]\n", __func__, enable);
+        DRM_DEBUG_KMS("enable[%d]\n", enable);
 
         if (enable) {
                 clk_prepare_enable(ctx->clocks[FIMC_CLK_GATE]);
@@ -1326,7 +1312,7 @@ static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
                 c_node->event_work;
         int buf_id;
 
-        DRM_DEBUG_KMS("%s:fimc id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("fimc id[%d]\n", ctx->id);
 
         fimc_clear_irq(ctx);
         if (fimc_check_ovf(ctx))
@@ -1339,7 +1325,7 @@ static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
         if (buf_id < 0)
                 return IRQ_HANDLED;
 
-        DRM_DEBUG_KMS("%s:buf_id[%d]\n", __func__, buf_id);
+        DRM_DEBUG_KMS("buf_id[%d]\n", buf_id);
 
         if (fimc_dst_set_buf_seq(ctx, buf_id, IPP_BUF_DEQUEUE) < 0) {
                 DRM_ERROR("failed to dequeue.\n");
@@ -1357,8 +1343,6 @@ static int fimc_init_prop_list(struct exynos_drm_ippdrv *ippdrv)
 {
         struct drm_exynos_ipp_prop_list *prop_list;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         prop_list = devm_kzalloc(ippdrv->dev, sizeof(*prop_list), GFP_KERNEL);
         if (!prop_list) {
                 DRM_ERROR("failed to alloc property list.\n");
@@ -1402,7 +1386,7 @@ static inline bool fimc_check_drm_flip(enum drm_exynos_flip flip)
         case EXYNOS_DRM_FLIP_BOTH:
                 return true;
         default:
-                DRM_DEBUG_KMS("%s:invalid flip\n", __func__);
+                DRM_DEBUG_KMS("invalid flip\n");
                 return false;
         }
 }
@@ -1419,8 +1403,6 @@ static int fimc_ippdrv_check_property(struct device *dev,
         bool swap;
         int i;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         for_each_ipp_ops(i) {
                 if ((i == EXYNOS_DRM_OPS_SRC) &&
                         (property->cmd == IPP_CMD_WB))
@@ -1526,8 +1508,6 @@ static void fimc_clear_addr(struct fimc_context *ctx)
 {
         int i;
 
-        DRM_DEBUG_KMS("%s:\n", __func__);
-
         for (i = 0; i < FIMC_MAX_SRC; i++) {
                 fimc_write(0, EXYNOS_CIIYSA(i));
                 fimc_write(0, EXYNOS_CIICBSA(i));
@@ -1545,8 +1525,6 @@ static int fimc_ippdrv_reset(struct device *dev)
 {
         struct fimc_context *ctx = get_fimc_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* reset h/w block */
         fimc_sw_reset(ctx);
 
@@ -1570,7 +1548,7 @@ static int fimc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
         int ret, i;
         u32 cfg0, cfg1;
 
-        DRM_DEBUG_KMS("%s:cmd[%d]\n", __func__, cmd);
+        DRM_DEBUG_KMS("cmd[%d]\n", cmd);
 
         if (!c_node) {
                 DRM_ERROR("failed to get c_node.\n");
@@ -1679,7 +1657,7 @@ static void fimc_ippdrv_stop(struct device *dev, enum drm_exynos_ipp_cmd cmd)
         struct drm_exynos_ipp_set_wb set_wb = {0, 0};
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:cmd[%d]\n", __func__, cmd);
+        DRM_DEBUG_KMS("cmd[%d]\n", cmd);
 
         switch (cmd) {
         case IPP_CMD_M2M:
@@ -1869,8 +1847,7 @@ static int fimc_probe(struct platform_device *pdev)
                 goto err_put_clk;
         }
 
-        DRM_DEBUG_KMS("%s:id[%d]ippdrv[0x%x]\n", __func__, ctx->id,
-                (int)ippdrv);
+        DRM_DEBUG_KMS("id[%d]ippdrv[0x%x]\n", ctx->id, (int)ippdrv);
 
         mutex_init(&ctx->lock);
         platform_set_drvdata(pdev, ctx);
@@ -1917,7 +1894,7 @@ static int fimc_suspend(struct device *dev)
 {
         struct fimc_context *ctx = get_fimc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         if (pm_runtime_suspended(dev))
                 return 0;
@@ -1929,7 +1906,7 @@ static int fimc_resume(struct device *dev)
 {
         struct fimc_context *ctx = get_fimc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         if (!pm_runtime_suspended(dev))
                 return fimc_clk_ctrl(ctx, true);
@@ -1943,7 +1920,7 @@ static int fimc_runtime_suspend(struct device *dev)
 {
         struct fimc_context *ctx = get_fimc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         return  fimc_clk_ctrl(ctx, false);
 }
@@ -1952,7 +1929,7 @@ static int fimc_runtime_resume(struct device *dev)
 {
         struct fimc_context *ctx = get_fimc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         return  fimc_clk_ctrl(ctx, true);
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index 97c61dbffd82..3e106beca5b6 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -63,14 +63,24 @@
 
 struct fimd_driver_data {
         unsigned int timing_base;
+
+        unsigned int has_shadowcon:1;
+        unsigned int has_clksel:1;
+};
+
+static struct fimd_driver_data s3c64xx_fimd_driver_data = {
+        .timing_base = 0x0,
+        .has_clksel = 1,
 };
 
 static struct fimd_driver_data exynos4_fimd_driver_data = {
         .timing_base = 0x0,
+        .has_shadowcon = 1,
 };
 
 static struct fimd_driver_data exynos5_fimd_driver_data = {
         .timing_base = 0x20000,
+        .has_shadowcon = 1,
 };
 
 struct fimd_win_data {
@@ -107,10 +117,13 @@ struct fimd_context {
         atomic_t                        wait_vsync_event;
 
         struct exynos_drm_panel_info *panel;
+        struct fimd_driver_data *driver_data;
 };
 
 #ifdef CONFIG_OF
 static const struct of_device_id fimd_driver_dt_match[] = {
+        { .compatible = "samsung,s3c6400-fimd",
+          .data = &s3c64xx_fimd_driver_data },
         { .compatible = "samsung,exynos4210-fimd",
           .data = &exynos4_fimd_driver_data },
         { .compatible = "samsung,exynos5250-fimd",
@@ -137,8 +150,6 @@ static inline struct fimd_driver_data *drm_fimd_get_driver_data(
 
 static bool fimd_display_is_connected(struct device *dev)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO. */
 
         return true;
@@ -148,15 +159,11 @@ static void *fimd_get_panel(struct device *dev)
 {
         struct fimd_context *ctx = get_fimd_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         return ctx->panel;
 }
 
-static int fimd_check_timing(struct device *dev, void *timing)
+static int fimd_check_mode(struct device *dev, struct drm_display_mode *mode)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO. */
 
         return 0;
@@ -164,8 +171,6 @@ static int fimd_check_timing(struct device *dev, void *timing)
 
 static int fimd_display_power_on(struct device *dev, int mode)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO */
 
         return 0;
@@ -175,7 +180,7 @@ static struct exynos_drm_display_ops fimd_display_ops = {
         .type = EXYNOS_DISPLAY_TYPE_LCD,
         .is_connected = fimd_display_is_connected,
         .get_panel = fimd_get_panel,
-        .check_timing = fimd_check_timing,
+        .check_mode = fimd_check_mode,
         .power_on = fimd_display_power_on,
 };
 
@@ -183,7 +188,7 @@ static void fimd_dpms(struct device *subdrv_dev, int mode)
 {
         struct fimd_context *ctx = get_fimd_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s, %d\n", __FILE__, mode);
+        DRM_DEBUG_KMS("%d\n", mode);
 
         mutex_lock(&ctx->lock);
 
@@ -221,8 +226,6 @@ static void fimd_apply(struct device *subdrv_dev)
         struct fimd_win_data *win_data;
         int i;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         for (i = 0; i < WINDOWS_NR; i++) {
                 win_data = &ctx->win_data[i];
                 if (win_data->enabled && (ovl_ops && ovl_ops->commit))
@@ -239,15 +242,12 @@ static void fimd_commit(struct device *dev)
         struct exynos_drm_panel_info *panel = ctx->panel;
         struct fb_videomode *timing = &panel->timing;
         struct fimd_driver_data *driver_data;
-        struct platform_device *pdev = to_platform_device(dev);
         u32 val;
 
-        driver_data = drm_fimd_get_driver_data(pdev);
+        driver_data = ctx->driver_data;
         if (ctx->suspended)
                 return;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* setup polarity values from machine code. */
         writel(ctx->vidcon1, ctx->regs + driver_data->timing_base + VIDCON1);
 
@@ -274,6 +274,11 @@ static void fimd_commit(struct device *dev)
         val = ctx->vidcon0;
         val &= ~(VIDCON0_CLKVAL_F_MASK | VIDCON0_CLKDIR);
 
+        if (ctx->driver_data->has_clksel) {
+                val &= ~VIDCON0_CLKSEL_MASK;
+                val |= VIDCON0_CLKSEL_LCD;
+        }
+
         if (ctx->clkdiv > 1)
                 val |= VIDCON0_CLKVAL_F(ctx->clkdiv - 1) | VIDCON0_CLKDIR;
         else
@@ -292,8 +297,6 @@ static int fimd_enable_vblank(struct device *dev)
         struct fimd_context *ctx = get_fimd_context(dev);
         u32 val;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return -EPERM;
 
@@ -319,8 +322,6 @@ static void fimd_disable_vblank(struct device *dev)
         struct fimd_context *ctx = get_fimd_context(dev);
         u32 val;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return;
 
@@ -370,8 +371,6 @@ static void fimd_win_mode_set(struct device *dev,
         int win;
         unsigned long offset;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!overlay) {
                 dev_err(dev, "overlay is NULL\n");
                 return;
@@ -381,7 +380,7 @@ static void fimd_win_mode_set(struct device *dev,
         if (win == DEFAULT_ZPOS)
                 win = ctx->default_win;
 
-        if (win < 0 || win > WINDOWS_NR)
+        if (win < 0 || win >= WINDOWS_NR)
                 return;
 
         offset = overlay->fb_x * (overlay->bpp >> 3);
@@ -418,8 +417,6 @@ static void fimd_win_set_pixfmt(struct device *dev, unsigned int win)
         struct fimd_win_data *win_data = &ctx->win_data[win];
         unsigned long val;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         val = WINCONx_ENWIN;
 
         switch (win_data->bpp) {
@@ -478,8 +475,6 @@ static void fimd_win_set_colkey(struct device *dev, unsigned int win)
         struct fimd_context *ctx = get_fimd_context(dev);
         unsigned int keycon0 = 0, keycon1 = 0;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         keycon0 = ~(WxKEYCON0_KEYBL_EN | WxKEYCON0_KEYEN_F |
                         WxKEYCON0_DIRCON) | WxKEYCON0_COMPKEY(0);
 
@@ -489,6 +484,33 @@ static void fimd_win_set_colkey(struct device *dev, unsigned int win)
         writel(keycon1, ctx->regs + WKEYCON1_BASE(win));
 }
 
+/**
+ * shadow_protect_win() - disable updating values from shadow registers at vsync
+ *
+ * @win: window to protect registers for
+ * @protect: 1 to protect (disable updates)
+ */
+static void fimd_shadow_protect_win(struct fimd_context *ctx,
+                                                        int win, bool protect)
+{
+        u32 reg, bits, val;
+
+        if (ctx->driver_data->has_shadowcon) {
+                reg = SHADOWCON;
+                bits = SHADOWCON_WINx_PROTECT(win);
+        } else {
+                reg = PRTCON;
+                bits = PRTCON_PROTECT;
+        }
+
+        val = readl(ctx->regs + reg);
+        if (protect)
+                val |= bits;
+        else
+                val &= ~bits;
+        writel(val, ctx->regs + reg);
+}
+
 static void fimd_win_commit(struct device *dev, int zpos)
 {
         struct fimd_context *ctx = get_fimd_context(dev);
@@ -498,21 +520,19 @@ static void fimd_win_commit(struct device *dev, int zpos)
         unsigned int last_x;
         unsigned int last_y;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return;
 
         if (win == DEFAULT_ZPOS)
                 win = ctx->default_win;
 
-        if (win < 0 || win > WINDOWS_NR)
+        if (win < 0 || win >= WINDOWS_NR)
                 return;
 
         win_data = &ctx->win_data[win];
 
         /*
-         * SHADOWCON register is used for enabling timing.
+         * SHADOWCON/PRTCON register is used for enabling timing.
          *
          * for example, once only width value of a register is set,
          * if the dma is started then fimd hardware could malfunction so
@@ -522,9 +542,7 @@ static void fimd_win_commit(struct device *dev, int zpos)
          */
 
         /* protect windows */
-        val = readl(ctx->regs + SHADOWCON);
-        val |= SHADOWCON_WINx_PROTECT(win);
-        writel(val, ctx->regs + SHADOWCON);
+        fimd_shadow_protect_win(ctx, win, true);
 
         /* buffer start address */
         val = (unsigned long)win_data->dma_addr;
@@ -602,10 +620,13 @@ static void fimd_win_commit(struct device *dev, int zpos)
         writel(val, ctx->regs + WINCON(win));
 
         /* Enable DMA channel and unprotect windows */
-        val = readl(ctx->regs + SHADOWCON);
-        val |= SHADOWCON_CHx_ENABLE(win);
-        val &= ~SHADOWCON_WINx_PROTECT(win);
-        writel(val, ctx->regs + SHADOWCON);
+        fimd_shadow_protect_win(ctx, win, false);
+
+        if (ctx->driver_data->has_shadowcon) {
+                val = readl(ctx->regs + SHADOWCON);
+                val |= SHADOWCON_CHx_ENABLE(win);
+                writel(val, ctx->regs + SHADOWCON);
+        }
 
         win_data->enabled = true;
 }
@@ -617,12 +638,10 @@ static void fimd_win_disable(struct device *dev, int zpos)
         int win = zpos;
         u32 val;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (win == DEFAULT_ZPOS)
                 win = ctx->default_win;
 
-        if (win < 0 || win > WINDOWS_NR)
+        if (win < 0 || win >= WINDOWS_NR)
                 return;
 
         win_data = &ctx->win_data[win];
@@ -634,9 +653,7 @@ static void fimd_win_disable(struct device *dev, int zpos)
         }
 
         /* protect windows */
-        val = readl(ctx->regs + SHADOWCON);
-        val |= SHADOWCON_WINx_PROTECT(win);
-        writel(val, ctx->regs + SHADOWCON);
+        fimd_shadow_protect_win(ctx, win, true);
 
         /* wincon */
         val = readl(ctx->regs + WINCON(win));
@@ -644,10 +661,13 @@ static void fimd_win_disable(struct device *dev, int zpos)
         writel(val, ctx->regs + WINCON(win));
 
         /* unprotect windows */
-        val = readl(ctx->regs + SHADOWCON);
-        val &= ~SHADOWCON_CHx_ENABLE(win);
-        val &= ~SHADOWCON_WINx_PROTECT(win);
-        writel(val, ctx->regs + SHADOWCON);
+        if (ctx->driver_data->has_shadowcon) {
+                val = readl(ctx->regs + SHADOWCON);
+                val &= ~SHADOWCON_CHx_ENABLE(win);
+                writel(val, ctx->regs + SHADOWCON);
+        }
+
+        fimd_shadow_protect_win(ctx, win, false);
 
         win_data->enabled = false;
 }
@@ -697,8 +717,6 @@ out:
 
 static int fimd_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * enable drm irq mode.
          * - with irq_enabled = 1, we can use the vblank feature.
@@ -725,8 +743,6 @@ static int fimd_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
 
 static void fimd_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* detach this sub driver from iommu mapping if supported. */
         if (is_drm_iommu_supported(drm_dev))
                 drm_iommu_detach_device(drm_dev, dev);
@@ -741,8 +757,6 @@ static int fimd_calc_clkdiv(struct fimd_context *ctx,
         u32 best_framerate = 0;
         u32 framerate;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         retrace = timing->left_margin + timing->hsync_len +
                                 timing->right_margin + timing->xres;
         retrace *= timing->upper_margin + timing->vsync_len +
@@ -777,10 +791,6 @@ static int fimd_calc_clkdiv(struct fimd_context *ctx,
 
 static void fimd_clear_win(struct fimd_context *ctx, int win)
 {
-        u32 val;
-
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         writel(0, ctx->regs + WINCON(win));
         writel(0, ctx->regs + VIDOSD_A(win));
         writel(0, ctx->regs + VIDOSD_B(win));
@@ -789,15 +799,11 @@ static void fimd_clear_win(struct fimd_context *ctx, int win)
         if (win == 1 || win == 2)
                 writel(0, ctx->regs + VIDOSD_D(win));
 
-        val = readl(ctx->regs + SHADOWCON);
-        val &= ~SHADOWCON_WINx_PROTECT(win);
-        writel(val, ctx->regs + SHADOWCON);
+        fimd_shadow_protect_win(ctx, win, false);
 }
 
 static int fimd_clock(struct fimd_context *ctx, bool enable)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (enable) {
                 int ret;
 
@@ -883,8 +889,6 @@ static int fimd_probe(struct platform_device *pdev)
         int win;
         int ret = -EINVAL;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (dev->of_node) {
                 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
                 if (!pdata) {
@@ -949,6 +953,7 @@ static int fimd_probe(struct platform_device *pdev)
                 return ret;
         }
 
+        ctx->driver_data = drm_fimd_get_driver_data(pdev);
         ctx->vidcon0 = pdata->vidcon0;
         ctx->vidcon1 = pdata->vidcon1;
         ctx->default_win = pdata->default_win;
@@ -989,8 +994,6 @@ static int fimd_remove(struct platform_device *pdev)
         struct device *dev = &pdev->dev;
         struct fimd_context *ctx = platform_get_drvdata(pdev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_drm_subdrv_unregister(&ctx->subdrv);
 
         if (ctx->suspended)
@@ -1055,8 +1058,6 @@ static int fimd_runtime_suspend(struct device *dev)
 {
         struct fimd_context *ctx = get_fimd_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         return fimd_activate(ctx, false);
 }
 
@@ -1064,14 +1065,15 @@ static int fimd_runtime_resume(struct device *dev)
 {
         struct fimd_context *ctx = get_fimd_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         return fimd_activate(ctx, true);
 }
 #endif
 
 static struct platform_device_id fimd_driver_ids[] = {
         {
+                .name           = "s3c64xx-fb",
+                .driver_data    = (unsigned long)&s3c64xx_fimd_driver_data,
+        }, {
                 .name           = "exynos4-fb",
                 .driver_data    = (unsigned long)&exynos4_fimd_driver_data,
         }, {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index cf4543ffa079..c3f15e7646d5 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -132,8 +132,6 @@ void exynos_drm_gem_destroy(struct exynos_drm_gem_obj *exynos_gem_obj)
         struct drm_gem_object *obj;
         struct exynos_drm_gem_buf *buf;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         obj = &exynos_gem_obj->base;
         buf = exynos_gem_obj->buffer;
 
@@ -227,7 +225,6 @@ struct exynos_drm_gem_obj *exynos_drm_gem_create(struct drm_device *dev,
         }
 
         size = roundup_gem_size(size, flags);
-        DRM_DEBUG_KMS("%s\n", __FILE__);
 
         ret = check_gem_flags(flags);
         if (ret)
@@ -268,8 +265,6 @@ int exynos_drm_gem_create_ioctl(struct drm_device *dev, void *data,
         struct exynos_drm_gem_obj *exynos_gem_obj;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_gem_obj = exynos_drm_gem_create(dev, args->flags, args->size);
         if (IS_ERR(exynos_gem_obj))
                 return PTR_ERR(exynos_gem_obj);
@@ -331,8 +326,6 @@ int exynos_drm_gem_map_offset_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_exynos_gem_map_off *args = data;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         DRM_DEBUG_KMS("handle = 0x%x, offset = 0x%lx\n",
                         args->handle, (unsigned long)args->offset);
 
@@ -371,8 +364,6 @@ static int exynos_drm_gem_mmap_buffer(struct file *filp,
         unsigned long vm_size;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
         vma->vm_private_data = obj;
         vma->vm_ops = drm_dev->driver->gem_vm_ops;
@@ -429,9 +420,7 @@ int exynos_drm_gem_mmap_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_exynos_gem_mmap *args = data;
         struct drm_gem_object *obj;
-        unsigned int addr;
-
-        DRM_DEBUG_KMS("%s\n", __FILE__);
+        unsigned long addr;
 
         if (!(dev->driver->driver_features & DRIVER_GEM)) {
                 DRM_ERROR("does not support GEM.\n");
@@ -473,14 +462,14 @@ int exynos_drm_gem_mmap_ioctl(struct drm_device *dev, void *data,
 
         drm_gem_object_unreference(obj);
 
-        if (IS_ERR((void *)addr)) {
+        if (IS_ERR_VALUE(addr)) {
                 /* check filp->f_op, filp->private_data are restored */
                 if (file_priv->filp->f_op == &exynos_drm_gem_fops) {
                         file_priv->filp->f_op = fops_get(dev->driver->fops);
                         file_priv->filp->private_data = file_priv;
                 }
                 mutex_unlock(&dev->struct_mutex);
-                return PTR_ERR((void *)addr);
+                return (int)addr;
         }
 
         mutex_unlock(&dev->struct_mutex);
@@ -643,8 +632,6 @@ void exynos_gem_unmap_sgt_from_dma(struct drm_device *drm_dev,
 
 int exynos_drm_gem_init_object(struct drm_gem_object *obj)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         return 0;
 }
 
@@ -653,8 +640,6 @@ void exynos_drm_gem_free_object(struct drm_gem_object *obj)
         struct exynos_drm_gem_obj *exynos_gem_obj;
         struct exynos_drm_gem_buf *buf;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_gem_obj = to_exynos_gem_obj(obj);
         buf = exynos_gem_obj->buffer;
 
@@ -671,8 +656,6 @@ int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
         struct exynos_drm_gem_obj *exynos_gem_obj;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * alocate memory to be used for framebuffer.
          * - this callback would be called by user application
@@ -704,8 +687,6 @@ int exynos_drm_gem_dumb_map_offset(struct drm_file *file_priv,
         struct drm_gem_object *obj;
         int ret = 0;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         mutex_lock(&dev->struct_mutex);
 
         /*
@@ -743,8 +724,6 @@ int exynos_drm_gem_dumb_destroy(struct drm_file *file_priv,
 {
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * obj->refcount and obj->handle_count are decreased and
          * if both them are 0 then exynos_drm_gem_free_object()
@@ -788,8 +767,6 @@ int exynos_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
         struct drm_gem_object *obj;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* set vm_area_struct. */
         ret = drm_gem_mmap(filp, vma);
         if (ret < 0) {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 762f40d548b7..472e3b25e7f2 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -400,8 +400,6 @@ static int gsc_sw_reset(struct gsc_context *ctx)
         u32 cfg;
         int count = GSC_RESET_TIMEOUT;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* s/w reset */
         cfg = (GSC_SW_RESET_SRESET);
         gsc_write(cfg, GSC_SW_RESET);
@@ -441,8 +439,6 @@ static void gsc_set_gscblk_fimd_wb(struct gsc_context *ctx, bool enable)
 {
         u32 gscblk_cfg;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         gscblk_cfg = readl(SYSREG_GSCBLK_CFG1);
 
         if (enable)
@@ -460,7 +456,7 @@ static void gsc_handle_irq(struct gsc_context *ctx, bool enable,
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:enable[%d]overflow[%d]level[%d]\n", __func__,
+        DRM_DEBUG_KMS("enable[%d]overflow[%d]level[%d]\n",
                         enable, overflow, done);
 
         cfg = gsc_read(GSC_IRQ);
@@ -491,7 +487,7 @@ static int gsc_src_set_fmt(struct device *dev, u32 fmt)
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:fmt[0x%x]\n", __func__, fmt);
+        DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
 
         cfg = gsc_read(GSC_IN_CON);
         cfg &= ~(GSC_IN_RGB_TYPE_MASK | GSC_IN_YUV422_1P_ORDER_MASK |
@@ -567,8 +563,7 @@ static int gsc_src_set_transf(struct device *dev,
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:degree[%d]flip[0x%x]\n", __func__,
-                degree, flip);
+        DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
 
         cfg = gsc_read(GSC_IN_CON);
         cfg &= ~GSC_IN_ROT_MASK;
@@ -616,8 +611,8 @@ static int gsc_src_set_size(struct device *dev, int swap,
         struct gsc_scaler *sc = &ctx->sc;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:swap[%d]x[%d]y[%d]w[%d]h[%d]\n",
-                __func__, swap, pos->x, pos->y, pos->w, pos->h);
+        DRM_DEBUG_KMS("swap[%d]x[%d]y[%d]w[%d]h[%d]\n",
+                swap, pos->x, pos->y, pos->w, pos->h);
 
         if (swap) {
                 img_pos.w = pos->h;
@@ -634,8 +629,7 @@ static int gsc_src_set_size(struct device *dev, int swap,
                 GSC_CROPPED_HEIGHT(img_pos.h));
         gsc_write(cfg, GSC_CROPPED_SIZE);
 
-        DRM_DEBUG_KMS("%s:hsize[%d]vsize[%d]\n",
-                __func__, sz->hsize, sz->vsize);
+        DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", sz->hsize, sz->vsize);
 
         /* original size */
         cfg = gsc_read(GSC_SRCIMG_SIZE);
@@ -650,8 +644,7 @@ static int gsc_src_set_size(struct device *dev, int swap,
         cfg = gsc_read(GSC_IN_CON);
         cfg &= ~GSC_IN_RGB_TYPE_MASK;
 
-        DRM_DEBUG_KMS("%s:width[%d]range[%d]\n",
-                __func__, pos->w, sc->range);
+        DRM_DEBUG_KMS("width[%d]range[%d]\n", pos->w, sc->range);
 
         if (pos->w >= GSC_WIDTH_ITU_709)
                 if (sc->range)
@@ -677,8 +670,7 @@ static int gsc_src_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
         u32 cfg;
         u32 mask = 0x00000001 << buf_id;
 
-        DRM_DEBUG_KMS("%s:buf_id[%d]buf_type[%d]\n", __func__,
-                buf_id, buf_type);
+        DRM_DEBUG_KMS("buf_id[%d]buf_type[%d]\n", buf_id, buf_type);
 
         /* mask register set */
         cfg = gsc_read(GSC_IN_BASE_ADDR_Y_MASK);
@@ -721,7 +713,7 @@ static int gsc_src_set_addr(struct device *dev,
 
         property = &c_node->property;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]buf_id[%d]buf_type[%d]\n", __func__,
+        DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
                 property->prop_id, buf_id, buf_type);
 
         if (buf_id > GSC_MAX_SRC) {
@@ -765,7 +757,7 @@ static int gsc_dst_set_fmt(struct device *dev, u32 fmt)
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:fmt[0x%x]\n", __func__, fmt);
+        DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
 
         cfg = gsc_read(GSC_OUT_CON);
         cfg &= ~(GSC_OUT_RGB_TYPE_MASK | GSC_OUT_YUV422_1P_ORDER_MASK |
@@ -838,8 +830,7 @@ static int gsc_dst_set_transf(struct device *dev,
         struct exynos_drm_ippdrv *ippdrv = &ctx->ippdrv;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:degree[%d]flip[0x%x]\n", __func__,
-                degree, flip);
+        DRM_DEBUG_KMS("degree[%d]flip[0x%x]\n", degree, flip);
 
         cfg = gsc_read(GSC_IN_CON);
         cfg &= ~GSC_IN_ROT_MASK;
@@ -881,7 +872,7 @@ static int gsc_dst_set_transf(struct device *dev,
 
 static int gsc_get_ratio_shift(u32 src, u32 dst, u32 *ratio)
 {
-        DRM_DEBUG_KMS("%s:src[%d]dst[%d]\n", __func__, src, dst);
+        DRM_DEBUG_KMS("src[%d]dst[%d]\n", src, dst);
 
         if (src >= dst * 8) {
                 DRM_ERROR("failed to make ratio and shift.\n");
@@ -944,20 +935,19 @@ static int gsc_set_prescaler(struct gsc_context *ctx, struct gsc_scaler *sc,
                 return ret;
         }
 
-        DRM_DEBUG_KMS("%s:pre_hratio[%d]pre_vratio[%d]\n",
-                __func__, sc->pre_hratio, sc->pre_vratio);
+        DRM_DEBUG_KMS("pre_hratio[%d]pre_vratio[%d]\n",
+                sc->pre_hratio, sc->pre_vratio);
 
         sc->main_hratio = (src_w << 16) / dst_w;
         sc->main_vratio = (src_h << 16) / dst_h;
 
-        DRM_DEBUG_KMS("%s:main_hratio[%ld]main_vratio[%ld]\n",
-                __func__, sc->main_hratio, sc->main_vratio);
+        DRM_DEBUG_KMS("main_hratio[%ld]main_vratio[%ld]\n",
+                sc->main_hratio, sc->main_vratio);
 
         gsc_get_prescaler_shfactor(sc->pre_hratio, sc->pre_vratio,
                 &sc->pre_shfactor);
 
-        DRM_DEBUG_KMS("%s:pre_shfactor[%d]\n", __func__,
-                sc->pre_shfactor);
+        DRM_DEBUG_KMS("pre_shfactor[%d]\n", sc->pre_shfactor);
 
         cfg = (GSC_PRESC_SHFACTOR(sc->pre_shfactor) |
                 GSC_PRESC_H_RATIO(sc->pre_hratio) |
@@ -1023,8 +1013,8 @@ static void gsc_set_scaler(struct gsc_context *ctx, struct gsc_scaler *sc)
 {
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:main_hratio[%ld]main_vratio[%ld]\n",
-                __func__, sc->main_hratio, sc->main_vratio);
+        DRM_DEBUG_KMS("main_hratio[%ld]main_vratio[%ld]\n",
+                sc->main_hratio, sc->main_vratio);
 
         gsc_set_h_coef(ctx, sc->main_hratio);
         cfg = GSC_MAIN_H_RATIO_VALUE(sc->main_hratio);
@@ -1043,8 +1033,8 @@ static int gsc_dst_set_size(struct device *dev, int swap,
         struct gsc_scaler *sc = &ctx->sc;
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:swap[%d]x[%d]y[%d]w[%d]h[%d]\n",
-                __func__, swap, pos->x, pos->y, pos->w, pos->h);
+        DRM_DEBUG_KMS("swap[%d]x[%d]y[%d]w[%d]h[%d]\n",
+                swap, pos->x, pos->y, pos->w, pos->h);
 
         if (swap) {
                 img_pos.w = pos->h;
@@ -1060,8 +1050,7 @@ static int gsc_dst_set_size(struct device *dev, int swap,
         cfg = (GSC_SCALED_WIDTH(img_pos.w) | GSC_SCALED_HEIGHT(img_pos.h));
         gsc_write(cfg, GSC_SCALED_SIZE);
 
-        DRM_DEBUG_KMS("%s:hsize[%d]vsize[%d]\n",
-                __func__, sz->hsize, sz->vsize);
+        DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", sz->hsize, sz->vsize);
 
         /* original size */
         cfg = gsc_read(GSC_DSTIMG_SIZE);
@@ -1074,8 +1063,7 @@ static int gsc_dst_set_size(struct device *dev, int swap,
         cfg = gsc_read(GSC_OUT_CON);
         cfg &= ~GSC_OUT_RGB_TYPE_MASK;
 
-        DRM_DEBUG_KMS("%s:width[%d]range[%d]\n",
-                __func__, pos->w, sc->range);
+        DRM_DEBUG_KMS("width[%d]range[%d]\n", pos->w, sc->range);
 
         if (pos->w >= GSC_WIDTH_ITU_709)
                 if (sc->range)
@@ -1104,7 +1092,7 @@ static int gsc_dst_get_buf_seq(struct gsc_context *ctx)
                 if (cfg & (mask << i))
                         buf_num--;
 
-        DRM_DEBUG_KMS("%s:buf_num[%d]\n", __func__, buf_num);
+        DRM_DEBUG_KMS("buf_num[%d]\n", buf_num);
 
         return buf_num;
 }
@@ -1118,8 +1106,7 @@ static int gsc_dst_set_buf_seq(struct gsc_context *ctx, u32 buf_id,
         u32 mask = 0x00000001 << buf_id;
         int ret = 0;
 
-        DRM_DEBUG_KMS("%s:buf_id[%d]buf_type[%d]\n", __func__,
-                buf_id, buf_type);
+        DRM_DEBUG_KMS("buf_id[%d]buf_type[%d]\n", buf_id, buf_type);
 
         mutex_lock(&ctx->lock);
 
@@ -1177,7 +1164,7 @@ static int gsc_dst_set_addr(struct device *dev,
 
         property = &c_node->property;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]buf_id[%d]buf_type[%d]\n", __func__,
+        DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]buf_type[%d]\n",
                 property->prop_id, buf_id, buf_type);
 
         if (buf_id > GSC_MAX_DST) {
@@ -1217,7 +1204,7 @@ static struct exynos_drm_ipp_ops gsc_dst_ops = {
 
 static int gsc_clk_ctrl(struct gsc_context *ctx, bool enable)
 {
-        DRM_DEBUG_KMS("%s:enable[%d]\n", __func__, enable);
+        DRM_DEBUG_KMS("enable[%d]\n", enable);
 
         if (enable) {
                 clk_enable(ctx->gsc_clk);
@@ -1236,7 +1223,7 @@ static int gsc_get_src_buf_index(struct gsc_context *ctx)
         u32 buf_id = GSC_MAX_SRC;
         int ret;
 
-        DRM_DEBUG_KMS("%s:gsc id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
 
         cfg = gsc_read(GSC_IN_BASE_ADDR_Y_MASK);
         curr_index = GSC_IN_CURR_GET_INDEX(cfg);
@@ -1259,7 +1246,7 @@ static int gsc_get_src_buf_index(struct gsc_context *ctx)
                 return ret;
         }
 
-        DRM_DEBUG_KMS("%s:cfg[0x%x]curr_index[%d]buf_id[%d]\n", __func__, cfg,
+        DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
                 curr_index, buf_id);
 
         return buf_id;
@@ -1271,7 +1258,7 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
         u32 buf_id = GSC_MAX_DST;
         int ret;
 
-        DRM_DEBUG_KMS("%s:gsc id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
 
         cfg = gsc_read(GSC_OUT_BASE_ADDR_Y_MASK);
         curr_index = GSC_OUT_CURR_GET_INDEX(cfg);
@@ -1294,7 +1281,7 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
                 return ret;
         }
 
-        DRM_DEBUG_KMS("%s:cfg[0x%x]curr_index[%d]buf_id[%d]\n", __func__, cfg,
+        DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
                 curr_index, buf_id);
 
         return buf_id;
@@ -1310,7 +1297,7 @@ static irqreturn_t gsc_irq_handler(int irq, void *dev_id)
         u32 status;
         int buf_id[EXYNOS_DRM_OPS_MAX];
 
-        DRM_DEBUG_KMS("%s:gsc id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
 
         status = gsc_read(GSC_IRQ);
         if (status & GSC_IRQ_STATUS_OR_IRQ) {
@@ -1331,7 +1318,7 @@ static irqreturn_t gsc_irq_handler(int irq, void *dev_id)
                 if (buf_id[EXYNOS_DRM_OPS_DST] < 0)
                         return IRQ_HANDLED;
 
-                DRM_DEBUG_KMS("%s:buf_id_src[%d]buf_id_dst[%d]\n", __func__,
+                DRM_DEBUG_KMS("buf_id_src[%d]buf_id_dst[%d]\n",
                         buf_id[EXYNOS_DRM_OPS_SRC], buf_id[EXYNOS_DRM_OPS_DST]);
 
                 event_work->ippdrv = ippdrv;
@@ -1350,8 +1337,6 @@ static int gsc_init_prop_list(struct exynos_drm_ippdrv *ippdrv)
 {
         struct drm_exynos_ipp_prop_list *prop_list;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         prop_list = devm_kzalloc(ippdrv->dev, sizeof(*prop_list), GFP_KERNEL);
         if (!prop_list) {
                 DRM_ERROR("failed to alloc property list.\n");
@@ -1394,7 +1379,7 @@ static inline bool gsc_check_drm_flip(enum drm_exynos_flip flip)
         case EXYNOS_DRM_FLIP_BOTH:
                 return true;
         default:
-                DRM_DEBUG_KMS("%s:invalid flip\n", __func__);
+                DRM_DEBUG_KMS("invalid flip\n");
                 return false;
         }
 }
@@ -1411,8 +1396,6 @@ static int gsc_ippdrv_check_property(struct device *dev,
         bool swap;
         int i;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         for_each_ipp_ops(i) {
                 if ((i == EXYNOS_DRM_OPS_SRC) &&
                         (property->cmd == IPP_CMD_WB))
@@ -1521,8 +1504,6 @@ static int gsc_ippdrv_reset(struct device *dev)
         struct gsc_scaler *sc = &ctx->sc;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* reset h/w block */
         ret = gsc_sw_reset(ctx);
         if (ret < 0) {
@@ -1549,7 +1530,7 @@ static int gsc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
         u32 cfg;
         int ret, i;
 
-        DRM_DEBUG_KMS("%s:cmd[%d]\n", __func__, cmd);
+        DRM_DEBUG_KMS("cmd[%d]\n", cmd);
 
         if (!c_node) {
                 DRM_ERROR("failed to get c_node.\n");
@@ -1643,7 +1624,7 @@ static void gsc_ippdrv_stop(struct device *dev, enum drm_exynos_ipp_cmd cmd)
         struct drm_exynos_ipp_set_wb set_wb = {0, 0};
         u32 cfg;
 
-        DRM_DEBUG_KMS("%s:cmd[%d]\n", __func__, cmd);
+        DRM_DEBUG_KMS("cmd[%d]\n", cmd);
 
         switch (cmd) {
         case IPP_CMD_M2M:
@@ -1728,8 +1709,7 @@ static int gsc_probe(struct platform_device *pdev)
                 return ret;
         }
 
-        DRM_DEBUG_KMS("%s:id[%d]ippdrv[0x%x]\n", __func__, ctx->id,
-                (int)ippdrv);
+        DRM_DEBUG_KMS("id[%d]ippdrv[0x%x]\n", ctx->id, (int)ippdrv);
 
         mutex_init(&ctx->lock);
         platform_set_drvdata(pdev, ctx);
@@ -1772,7 +1752,7 @@ static int gsc_suspend(struct device *dev)
 {
         struct gsc_context *ctx = get_gsc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         if (pm_runtime_suspended(dev))
                 return 0;
@@ -1784,7 +1764,7 @@ static int gsc_resume(struct device *dev)
 {
         struct gsc_context *ctx = get_gsc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         if (!pm_runtime_suspended(dev))
                 return gsc_clk_ctrl(ctx, true);
@@ -1798,7 +1778,7 @@ static int gsc_runtime_suspend(struct device *dev)
 {
         struct gsc_context *ctx = get_gsc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         return  gsc_clk_ctrl(ctx, false);
 }
@@ -1807,7 +1787,7 @@ static int gsc_runtime_resume(struct device *dev)
 {
         struct gsc_context *ctx = get_gsc_context(dev);
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __FILE__, ctx->id);
+        DRM_DEBUG_KMS("id[%d]\n", ctx->id);
 
         return  gsc_clk_ctrl(ctx, true);
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_hdmi.c b/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
index 437fb947e46d..aaa550d622f0 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
@@ -88,16 +88,12 @@ void exynos_mixer_drv_attach(struct exynos_drm_hdmi_context *ctx)
 
 void exynos_hdmi_ops_register(struct exynos_hdmi_ops *ops)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ops)
                 hdmi_ops = ops;
 }
 
 void exynos_mixer_ops_register(struct exynos_mixer_ops *ops)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ops)
                 mixer_ops = ops;
 }
@@ -106,8 +102,6 @@ static bool drm_hdmi_is_connected(struct device *dev)
 {
         struct drm_hdmi_context *ctx = to_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (hdmi_ops && hdmi_ops->is_connected)
                 return hdmi_ops->is_connected(ctx->hdmi_ctx->ctx);
 
@@ -119,34 +113,31 @@ static struct edid *drm_hdmi_get_edid(struct device *dev,
 {
         struct drm_hdmi_context *ctx = to_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (hdmi_ops && hdmi_ops->get_edid)
                 return hdmi_ops->get_edid(ctx->hdmi_ctx->ctx, connector);
 
         return NULL;
 }
 
-static int drm_hdmi_check_timing(struct device *dev, void *timing)
+static int drm_hdmi_check_mode(struct device *dev,
+                struct drm_display_mode *mode)
 {
         struct drm_hdmi_context *ctx = to_context(dev);
         int ret = 0;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
         * Both, mixer and hdmi should be able to handle the requested mode.
         * If any of the two fails, return mode as BAD.
         */
 
-        if (mixer_ops && mixer_ops->check_timing)
-                ret = mixer_ops->check_timing(ctx->mixer_ctx->ctx, timing);
+        if (mixer_ops && mixer_ops->check_mode)
+                ret = mixer_ops->check_mode(ctx->mixer_ctx->ctx, mode);
 
         if (ret)
                 return ret;
 
-        if (hdmi_ops && hdmi_ops->check_timing)
-                return hdmi_ops->check_timing(ctx->hdmi_ctx->ctx, timing);
+        if (hdmi_ops && hdmi_ops->check_mode)
+                return hdmi_ops->check_mode(ctx->hdmi_ctx->ctx, mode);
 
         return 0;
 }
@@ -155,8 +146,6 @@ static int drm_hdmi_power_on(struct device *dev, int mode)
 {
         struct drm_hdmi_context *ctx = to_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (hdmi_ops && hdmi_ops->power_on)
                 return hdmi_ops->power_on(ctx->hdmi_ctx->ctx, mode);
 
@@ -167,7 +156,7 @@ static struct exynos_drm_display_ops drm_hdmi_display_ops = {
         .type = EXYNOS_DISPLAY_TYPE_HDMI,
         .is_connected = drm_hdmi_is_connected,
         .get_edid = drm_hdmi_get_edid,
-        .check_timing = drm_hdmi_check_timing,
+        .check_mode = drm_hdmi_check_mode,
         .power_on = drm_hdmi_power_on,
 };
 
@@ -177,8 +166,6 @@ static int drm_hdmi_enable_vblank(struct device *subdrv_dev)
         struct exynos_drm_subdrv *subdrv = &ctx->subdrv;
         struct exynos_drm_manager *manager = subdrv->manager;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (mixer_ops && mixer_ops->enable_vblank)
                 return mixer_ops->enable_vblank(ctx->mixer_ctx->ctx,
                                                 manager->pipe);
@@ -190,8 +177,6 @@ static void drm_hdmi_disable_vblank(struct device *subdrv_dev)
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (mixer_ops && mixer_ops->disable_vblank)
                 return mixer_ops->disable_vblank(ctx->mixer_ctx->ctx);
 }
@@ -200,8 +185,6 @@ static void drm_hdmi_wait_for_vblank(struct device *subdrv_dev)
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (mixer_ops && mixer_ops->wait_for_vblank)
                 mixer_ops->wait_for_vblank(ctx->mixer_ctx->ctx);
 }
@@ -214,11 +197,9 @@ static void drm_hdmi_mode_fixup(struct device *subdrv_dev,
         struct drm_display_mode *m;
         int mode_ok;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         drm_mode_set_crtcinfo(adjusted_mode, 0);
 
-        mode_ok = drm_hdmi_check_timing(subdrv_dev, adjusted_mode);
+        mode_ok = drm_hdmi_check_mode(subdrv_dev, adjusted_mode);
 
         /* just return if user desired mode exists. */
         if (mode_ok == 0)
@@ -229,7 +210,7 @@ static void drm_hdmi_mode_fixup(struct device *subdrv_dev,
          * to adjusted_mode.
          */
         list_for_each_entry(m, &connector->modes, head) {
-                mode_ok = drm_hdmi_check_timing(subdrv_dev, m);
+                mode_ok = drm_hdmi_check_mode(subdrv_dev, m);
 
                 if (mode_ok == 0) {
                         struct drm_mode_object base;
@@ -256,8 +237,6 @@ static void drm_hdmi_mode_set(struct device *subdrv_dev, void *mode)
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (hdmi_ops && hdmi_ops->mode_set)
                 hdmi_ops->mode_set(ctx->hdmi_ctx->ctx, mode);
 }
@@ -267,8 +246,6 @@ static void drm_hdmi_get_max_resol(struct device *subdrv_dev,
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (hdmi_ops && hdmi_ops->get_max_resol)
                 hdmi_ops->get_max_resol(ctx->hdmi_ctx->ctx, width, height);
 }
@@ -277,8 +254,6 @@ static void drm_hdmi_commit(struct device *subdrv_dev)
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (hdmi_ops && hdmi_ops->commit)
                 hdmi_ops->commit(ctx->hdmi_ctx->ctx);
 }
@@ -287,8 +262,6 @@ static void drm_hdmi_dpms(struct device *subdrv_dev, int mode)
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (mixer_ops && mixer_ops->dpms)
                 mixer_ops->dpms(ctx->mixer_ctx->ctx, mode);
 
@@ -301,8 +274,6 @@ static void drm_hdmi_apply(struct device *subdrv_dev)
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
         int i;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         for (i = 0; i < MIXER_WIN_NR; i++) {
                 if (!ctx->enabled[i])
                         continue;
@@ -331,8 +302,6 @@ static void drm_mixer_mode_set(struct device *subdrv_dev,
 {
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (mixer_ops && mixer_ops->win_mode_set)
                 mixer_ops->win_mode_set(ctx->mixer_ctx->ctx, overlay);
 }
@@ -342,9 +311,7 @@ static void drm_mixer_commit(struct device *subdrv_dev, int zpos)
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
         int win = (zpos == DEFAULT_ZPOS) ? MIXER_DEFAULT_WIN : zpos;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
-        if (win < 0 || win > MIXER_WIN_NR) {
+        if (win < 0 || win >= MIXER_WIN_NR) {
                 DRM_ERROR("mixer window[%d] is wrong\n", win);
                 return;
         }
@@ -360,9 +327,7 @@ static void drm_mixer_disable(struct device *subdrv_dev, int zpos)
         struct drm_hdmi_context *ctx = to_context(subdrv_dev);
         int win = (zpos == DEFAULT_ZPOS) ? MIXER_DEFAULT_WIN : zpos;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
-        if (win < 0 || win > MIXER_WIN_NR) {
+        if (win < 0 || win >= MIXER_WIN_NR) {
                 DRM_ERROR("mixer window[%d] is wrong\n", win);
                 return;
         }
@@ -392,8 +357,6 @@ static int hdmi_subdrv_probe(struct drm_device *drm_dev,
         struct exynos_drm_subdrv *subdrv = to_subdrv(dev);
         struct drm_hdmi_context *ctx;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!hdmi_ctx) {
                 DRM_ERROR("hdmi context not initialized.\n");
                 return -EFAULT;
@@ -440,8 +403,6 @@ static int exynos_drm_hdmi_probe(struct platform_device *pdev)
         struct exynos_drm_subdrv *subdrv;
         struct drm_hdmi_context *ctx;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
         if (!ctx) {
                 DRM_LOG_KMS("failed to alloc common hdmi context.\n");
@@ -466,8 +427,6 @@ static int exynos_drm_hdmi_remove(struct platform_device *pdev)
 {
         struct drm_hdmi_context *ctx = platform_get_drvdata(pdev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_drm_subdrv_unregister(&ctx->subdrv);
 
         return 0;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_hdmi.h b/drivers/gpu/drm/exynos/exynos_drm_hdmi.h
index 6b709440df4c..724cab181976 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_hdmi.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_hdmi.h
@@ -32,11 +32,11 @@ struct exynos_hdmi_ops {
         bool (*is_connected)(void *ctx);
         struct edid *(*get_edid)(void *ctx,
                         struct drm_connector *connector);
-        int (*check_timing)(void *ctx, struct fb_videomode *timing);
+        int (*check_mode)(void *ctx, struct drm_display_mode *mode);
         int (*power_on)(void *ctx, int mode);
 
         /* manager */
-        void (*mode_set)(void *ctx, void *mode);
+        void (*mode_set)(void *ctx, struct drm_display_mode *mode);
         void (*get_max_resol)(void *ctx, unsigned int *width,
                                 unsigned int *height);
         void (*commit)(void *ctx);
@@ -57,7 +57,7 @@ struct exynos_mixer_ops {
         void (*win_disable)(void *ctx, int zpos);
 
         /* display */
-        int (*check_timing)(void *ctx, struct fb_videomode *timing);
+        int (*check_mode)(void *ctx, struct drm_display_mode *mode);
 };
 
 void exynos_hdmi_drv_attach(struct exynos_drm_hdmi_context *ctx);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.c b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
index be1e88463466..b1ef8e7ff9c9 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_ipp.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
@@ -131,8 +131,6 @@ void exynos_platform_device_ipp_unregister(void)
 
 int exynos_drm_ippdrv_register(struct exynos_drm_ippdrv *ippdrv)
 {
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!ippdrv)
                 return -EINVAL;
 
@@ -145,8 +143,6 @@ int exynos_drm_ippdrv_register(struct exynos_drm_ippdrv *ippdrv)
 
 int exynos_drm_ippdrv_unregister(struct exynos_drm_ippdrv *ippdrv)
 {
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!ippdrv)
                 return -EINVAL;
 
@@ -162,8 +158,6 @@ static int ipp_create_id(struct idr *id_idr, struct mutex *lock, void *obj,
 {
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* do the allocation under our mutexlock */
         mutex_lock(lock);
         ret = idr_alloc(id_idr, obj, 1, 0, GFP_KERNEL);
@@ -179,7 +173,7 @@ static void *ipp_find_obj(struct idr *id_idr, struct mutex *lock, u32 id)
 {
         void *obj;
 
-        DRM_DEBUG_KMS("%s:id[%d]\n", __func__, id);
+        DRM_DEBUG_KMS("id[%d]\n", id);
 
         mutex_lock(lock);
 
@@ -216,7 +210,7 @@ static struct exynos_drm_ippdrv *ipp_find_driver(struct ipp_context *ctx,
         struct exynos_drm_ippdrv *ippdrv;
         u32 ipp_id = property->ipp_id;
 
-        DRM_DEBUG_KMS("%s:ipp_id[%d]\n", __func__, ipp_id);
+        DRM_DEBUG_KMS("ipp_id[%d]\n", ipp_id);
 
         if (ipp_id) {
                 /* find ipp driver using idr */
@@ -257,14 +251,13 @@ static struct exynos_drm_ippdrv *ipp_find_driver(struct ipp_context *ctx,
                  */
                 list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
                         if (ipp_check_dedicated(ippdrv, property->cmd)) {
-                                DRM_DEBUG_KMS("%s:used device.\n", __func__);
+                                DRM_DEBUG_KMS("used device.\n");
                                 continue;
                         }
 
                         if (ippdrv->check_property &&
                             ippdrv->check_property(ippdrv->dev, property)) {
-                                DRM_DEBUG_KMS("%s:not support property.\n",
-                                        __func__);
+                                DRM_DEBUG_KMS("not support property.\n");
                                 continue;
                         }
 
@@ -283,10 +276,10 @@ static struct exynos_drm_ippdrv *ipp_find_drv_by_handle(u32 prop_id)
         struct drm_exynos_ipp_cmd_node *c_node;
         int count = 0;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]\n", __func__, prop_id);
+        DRM_DEBUG_KMS("prop_id[%d]\n", prop_id);
 
         if (list_empty(&exynos_drm_ippdrv_list)) {
-                DRM_DEBUG_KMS("%s:ippdrv_list is empty.\n", __func__);
+                DRM_DEBUG_KMS("ippdrv_list is empty.\n");
                 return ERR_PTR(-ENODEV);
         }
 
@@ -296,8 +289,7 @@ static struct exynos_drm_ippdrv *ipp_find_drv_by_handle(u32 prop_id)
          * e.g PAUSE state, queue buf, command contro.
          */
         list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
-                DRM_DEBUG_KMS("%s:count[%d]ippdrv[0x%x]\n", __func__,
-                        count++, (int)ippdrv);
+                DRM_DEBUG_KMS("count[%d]ippdrv[0x%x]\n", count++, (int)ippdrv);
 
                 if (!list_empty(&ippdrv->cmd_list)) {
                         list_for_each_entry(c_node, &ippdrv->cmd_list, list)
@@ -320,8 +312,6 @@ int exynos_drm_ipp_get_property(struct drm_device *drm_dev, void *data,
         struct exynos_drm_ippdrv *ippdrv;
         int count = 0;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!ctx) {
                 DRM_ERROR("invalid context.\n");
                 return -EINVAL;
@@ -332,7 +322,7 @@ int exynos_drm_ipp_get_property(struct drm_device *drm_dev, void *data,
                 return -EINVAL;
         }
 
-        DRM_DEBUG_KMS("%s:ipp_id[%d]\n", __func__, prop_list->ipp_id);
+        DRM_DEBUG_KMS("ipp_id[%d]\n", prop_list->ipp_id);
 
         if (!prop_list->ipp_id) {
                 list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list)
@@ -371,11 +361,11 @@ static void ipp_print_property(struct drm_exynos_ipp_property *property,
         struct drm_exynos_pos *pos = &config->pos;
         struct drm_exynos_sz *sz = &config->sz;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]ops[%s]fmt[0x%x]\n",
-                __func__, property->prop_id, idx ? "dst" : "src", config->fmt);
+        DRM_DEBUG_KMS("prop_id[%d]ops[%s]fmt[0x%x]\n",
+                property->prop_id, idx ? "dst" : "src", config->fmt);
 
-        DRM_DEBUG_KMS("%s:pos[%d %d %d %d]sz[%d %d]f[%d]r[%d]\n",
-                __func__, pos->x, pos->y, pos->w, pos->h,
+        DRM_DEBUG_KMS("pos[%d %d %d %d]sz[%d %d]f[%d]r[%d]\n",
+                pos->x, pos->y, pos->w, pos->h,
                 sz->hsize, sz->vsize, config->flip, config->degree);
 }
 
@@ -385,7 +375,7 @@ static int ipp_find_and_set_property(struct drm_exynos_ipp_property *property)
         struct drm_exynos_ipp_cmd_node *c_node;
         u32 prop_id = property->prop_id;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]\n", __func__, prop_id);
+        DRM_DEBUG_KMS("prop_id[%d]\n", prop_id);
 
         ippdrv = ipp_find_drv_by_handle(prop_id);
         if (IS_ERR(ippdrv)) {
@@ -401,8 +391,8 @@ static int ipp_find_and_set_property(struct drm_exynos_ipp_property *property)
         list_for_each_entry(c_node, &ippdrv->cmd_list, list) {
                 if ((c_node->property.prop_id == prop_id) &&
                     (c_node->state == IPP_STATE_STOP)) {
-                        DRM_DEBUG_KMS("%s:found cmd[%d]ippdrv[0x%x]\n",
-                                __func__, property->cmd, (int)ippdrv);
+                        DRM_DEBUG_KMS("found cmd[%d]ippdrv[0x%x]\n",
+                                property->cmd, (int)ippdrv);
 
                         c_node->property = *property;
                         return 0;
@@ -418,8 +408,6 @@ static struct drm_exynos_ipp_cmd_work *ipp_create_cmd_work(void)
 {
         struct drm_exynos_ipp_cmd_work *cmd_work;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         cmd_work = kzalloc(sizeof(*cmd_work), GFP_KERNEL);
         if (!cmd_work) {
                 DRM_ERROR("failed to alloc cmd_work.\n");
@@ -435,8 +423,6 @@ static struct drm_exynos_ipp_event_work *ipp_create_event_work(void)
 {
         struct drm_exynos_ipp_event_work *event_work;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         event_work = kzalloc(sizeof(*event_work), GFP_KERNEL);
         if (!event_work) {
                 DRM_ERROR("failed to alloc event_work.\n");
@@ -460,8 +446,6 @@ int exynos_drm_ipp_set_property(struct drm_device *drm_dev, void *data,
         struct drm_exynos_ipp_cmd_node *c_node;
         int ret, i;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!ctx) {
                 DRM_ERROR("invalid context.\n");
                 return -EINVAL;
@@ -486,7 +470,7 @@ int exynos_drm_ipp_set_property(struct drm_device *drm_dev, void *data,
          * instead of allocation.
          */
         if (property->prop_id) {
-                DRM_DEBUG_KMS("%s:prop_id[%d]\n", __func__, property->prop_id);
+                DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
                 return ipp_find_and_set_property(property);
         }
 
@@ -512,8 +496,8 @@ int exynos_drm_ipp_set_property(struct drm_device *drm_dev, void *data,
                 goto err_clear;
         }
 
-        DRM_DEBUG_KMS("%s:created prop_id[%d]cmd[%d]ippdrv[0x%x]\n",
-                __func__, property->prop_id, property->cmd, (int)ippdrv);
+        DRM_DEBUG_KMS("created prop_id[%d]cmd[%d]ippdrv[0x%x]\n",
+                property->prop_id, property->cmd, (int)ippdrv);
 
         /* stored property information and ippdrv in private data */
         c_node->priv = priv;
@@ -569,8 +553,6 @@ err_clear:
 
 static void ipp_clean_cmd_node(struct drm_exynos_ipp_cmd_node *c_node)
 {
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* delete list */
         list_del(&c_node->list);
 
@@ -593,8 +575,6 @@ static int ipp_check_mem_list(struct drm_exynos_ipp_cmd_node *c_node)
         struct list_head *head;
         int ret, i, count[EXYNOS_DRM_OPS_MAX] = { 0, };
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         mutex_lock(&c_node->mem_lock);
 
         for_each_ipp_ops(i) {
@@ -602,20 +582,19 @@ static int ipp_check_mem_list(struct drm_exynos_ipp_cmd_node *c_node)
                 head = &c_node->mem_list[i];
 
                 if (list_empty(head)) {
-                        DRM_DEBUG_KMS("%s:%s memory empty.\n", __func__,
-                                i ? "dst" : "src");
+                        DRM_DEBUG_KMS("%s memory empty.\n", i ? "dst" : "src");
                         continue;
                 }
 
                 /* find memory node entry */
                 list_for_each_entry(m_node, head, list) {
-                        DRM_DEBUG_KMS("%s:%s,count[%d]m_node[0x%x]\n", __func__,
+                        DRM_DEBUG_KMS("%s,count[%d]m_node[0x%x]\n",
                                 i ? "dst" : "src", count[i], (int)m_node);
                         count[i]++;
                 }
         }
 
-        DRM_DEBUG_KMS("%s:min[%d]max[%d]\n", __func__,
+        DRM_DEBUG_KMS("min[%d]max[%d]\n",
                 min(count[EXYNOS_DRM_OPS_SRC], count[EXYNOS_DRM_OPS_DST]),
                 max(count[EXYNOS_DRM_OPS_SRC], count[EXYNOS_DRM_OPS_DST]));
 
@@ -644,15 +623,14 @@ static struct drm_exynos_ipp_mem_node
         struct list_head *head;
         int count = 0;
 
-        DRM_DEBUG_KMS("%s:buf_id[%d]\n", __func__, qbuf->buf_id);
+        DRM_DEBUG_KMS("buf_id[%d]\n", qbuf->buf_id);
 
         /* source/destination memory list */
         head = &c_node->mem_list[qbuf->ops_id];
 
         /* find memory node from memory list */
         list_for_each_entry(m_node, head, list) {
-                DRM_DEBUG_KMS("%s:count[%d]m_node[0x%x]\n",
-                        __func__, count++, (int)m_node);
+                DRM_DEBUG_KMS("count[%d]m_node[0x%x]\n", count++, (int)m_node);
 
                 /* compare buffer id */
                 if (m_node->buf_id == qbuf->buf_id)
@@ -669,7 +647,7 @@ static int ipp_set_mem_node(struct exynos_drm_ippdrv *ippdrv,
         struct exynos_drm_ipp_ops *ops = NULL;
         int ret = 0;
 
-        DRM_DEBUG_KMS("%s:node[0x%x]\n", __func__, (int)m_node);
+        DRM_DEBUG_KMS("node[0x%x]\n", (int)m_node);
 
         if (!m_node) {
                 DRM_ERROR("invalid queue node.\n");
@@ -678,7 +656,7 @@ static int ipp_set_mem_node(struct exynos_drm_ippdrv *ippdrv,
 
         mutex_lock(&c_node->mem_lock);
 
-        DRM_DEBUG_KMS("%s:ops_id[%d]\n", __func__, m_node->ops_id);
+        DRM_DEBUG_KMS("ops_id[%d]\n", m_node->ops_id);
 
         /* get operations callback */
         ops = ippdrv->ops[m_node->ops_id];
@@ -714,8 +692,6 @@ static struct drm_exynos_ipp_mem_node
         void *addr;
         int i;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         mutex_lock(&c_node->mem_lock);
 
         m_node = kzalloc(sizeof(*m_node), GFP_KERNEL);
@@ -732,14 +708,11 @@ static struct drm_exynos_ipp_mem_node
         m_node->prop_id = qbuf->prop_id;
         m_node->buf_id = qbuf->buf_id;
 
-        DRM_DEBUG_KMS("%s:m_node[0x%x]ops_id[%d]\n", __func__,
-                (int)m_node, qbuf->ops_id);
-        DRM_DEBUG_KMS("%s:prop_id[%d]buf_id[%d]\n", __func__,
-                qbuf->prop_id, m_node->buf_id);
+        DRM_DEBUG_KMS("m_node[0x%x]ops_id[%d]\n", (int)m_node, qbuf->ops_id);
+        DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]\n", qbuf->prop_id, m_node->buf_id);
 
         for_each_ipp_planar(i) {
-                DRM_DEBUG_KMS("%s:i[%d]handle[0x%x]\n", __func__,
-                        i, qbuf->handle[i]);
+                DRM_DEBUG_KMS("i[%d]handle[0x%x]\n", i, qbuf->handle[i]);
 
                 /* get dma address by handle */
                 if (qbuf->handle[i]) {
@@ -752,9 +725,8 @@ static struct drm_exynos_ipp_mem_node
 
                         buf_info.handles[i] = qbuf->handle[i];
                         buf_info.base[i] = *(dma_addr_t *) addr;
-                        DRM_DEBUG_KMS("%s:i[%d]base[0x%x]hd[0x%x]\n",
-                                __func__, i, buf_info.base[i],
-                                (int)buf_info.handles[i]);
+                        DRM_DEBUG_KMS("i[%d]base[0x%x]hd[0x%x]\n",
+                                i, buf_info.base[i], (int)buf_info.handles[i]);
                 }
         }
 
@@ -778,7 +750,7 @@ static int ipp_put_mem_node(struct drm_device *drm_dev,
 {
         int i;
 
-        DRM_DEBUG_KMS("%s:node[0x%x]\n", __func__, (int)m_node);
+        DRM_DEBUG_KMS("node[0x%x]\n", (int)m_node);
 
         if (!m_node) {
                 DRM_ERROR("invalid dequeue node.\n");
@@ -792,7 +764,7 @@ static int ipp_put_mem_node(struct drm_device *drm_dev,
 
         mutex_lock(&c_node->mem_lock);
 
-        DRM_DEBUG_KMS("%s:ops_id[%d]\n", __func__, m_node->ops_id);
+        DRM_DEBUG_KMS("ops_id[%d]\n", m_node->ops_id);
 
         /* put gem buffer */
         for_each_ipp_planar(i) {
@@ -824,8 +796,7 @@ static int ipp_get_event(struct drm_device *drm_dev,
         struct drm_exynos_ipp_send_event *e;
         unsigned long flags;
 
-        DRM_DEBUG_KMS("%s:ops_id[%d]buf_id[%d]\n", __func__,
-                qbuf->ops_id, qbuf->buf_id);
+        DRM_DEBUG_KMS("ops_id[%d]buf_id[%d]\n", qbuf->ops_id, qbuf->buf_id);
 
         e = kzalloc(sizeof(*e), GFP_KERNEL);
 
@@ -857,16 +828,13 @@ static void ipp_put_event(struct drm_exynos_ipp_cmd_node *c_node,
         struct drm_exynos_ipp_send_event *e, *te;
         int count = 0;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (list_empty(&c_node->event_list)) {
-                DRM_DEBUG_KMS("%s:event_list is empty.\n", __func__);
+                DRM_DEBUG_KMS("event_list is empty.\n");
                 return;
         }
 
         list_for_each_entry_safe(e, te, &c_node->event_list, base.link) {
-                DRM_DEBUG_KMS("%s:count[%d]e[0x%x]\n",
-                        __func__, count++, (int)e);
+                DRM_DEBUG_KMS("count[%d]e[0x%x]\n", count++, (int)e);
 
                 /*
                  * quf == NULL condition means all event deletion.
@@ -912,8 +880,6 @@ static int ipp_queue_buf_with_run(struct device *dev,
         struct exynos_drm_ipp_ops *ops;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         ippdrv = ipp_find_drv_by_handle(qbuf->prop_id);
         if (IS_ERR(ippdrv)) {
                 DRM_ERROR("failed to get ipp driver.\n");
@@ -929,12 +895,12 @@ static int ipp_queue_buf_with_run(struct device *dev,
         property = &c_node->property;
 
         if (c_node->state != IPP_STATE_START) {
-                DRM_DEBUG_KMS("%s:bypass for invalid state.\n" , __func__);
+                DRM_DEBUG_KMS("bypass for invalid state.\n");
                 return 0;
         }
 
         if (!ipp_check_mem_list(c_node)) {
-                DRM_DEBUG_KMS("%s:empty memory.\n", __func__);
+                DRM_DEBUG_KMS("empty memory.\n");
                 return 0;
         }
 
@@ -964,8 +930,6 @@ static void ipp_clean_queue_buf(struct drm_device *drm_dev,
 {
         struct drm_exynos_ipp_mem_node *m_node, *tm_node;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!list_empty(&c_node->mem_list[qbuf->ops_id])) {
                 /* delete list */
                 list_for_each_entry_safe(m_node, tm_node,
@@ -989,8 +953,6 @@ int exynos_drm_ipp_queue_buf(struct drm_device *drm_dev, void *data,
         struct drm_exynos_ipp_mem_node *m_node;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!qbuf) {
                 DRM_ERROR("invalid buf parameter.\n");
                 return -EINVAL;
@@ -1001,8 +963,8 @@ int exynos_drm_ipp_queue_buf(struct drm_device *drm_dev, void *data,
                 return -EINVAL;
         }
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]ops_id[%s]buf_id[%d]buf_type[%d]\n",
-                __func__, qbuf->prop_id, qbuf->ops_id ? "dst" : "src",
+        DRM_DEBUG_KMS("prop_id[%d]ops_id[%s]buf_id[%d]buf_type[%d]\n",
+                qbuf->prop_id, qbuf->ops_id ? "dst" : "src",
                 qbuf->buf_id, qbuf->buf_type);
 
         /* find command node */
@@ -1075,8 +1037,6 @@ err_clean_node:
 static bool exynos_drm_ipp_check_valid(struct device *dev,
                 enum drm_exynos_ipp_ctrl ctrl, enum drm_exynos_ipp_state state)
 {
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (ctrl != IPP_CTRL_PLAY) {
                 if (pm_runtime_suspended(dev)) {
                         DRM_ERROR("pm:runtime_suspended.\n");
@@ -1104,7 +1064,6 @@ static bool exynos_drm_ipp_check_valid(struct device *dev,
         default:
                 DRM_ERROR("invalid state.\n");
                 goto err_status;
-                break;
         }
 
         return true;
@@ -1126,8 +1085,6 @@ int exynos_drm_ipp_cmd_ctrl(struct drm_device *drm_dev, void *data,
         struct drm_exynos_ipp_cmd_work *cmd_work;
         struct drm_exynos_ipp_cmd_node *c_node;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!ctx) {
                 DRM_ERROR("invalid context.\n");
                 return -EINVAL;
@@ -1138,7 +1095,7 @@ int exynos_drm_ipp_cmd_ctrl(struct drm_device *drm_dev, void *data,
                 return -EINVAL;
         }
 
-        DRM_DEBUG_KMS("%s:ctrl[%d]prop_id[%d]\n", __func__,
+        DRM_DEBUG_KMS("ctrl[%d]prop_id[%d]\n",
                 cmd_ctrl->ctrl, cmd_ctrl->prop_id);
 
         ippdrv = ipp_find_drv_by_handle(cmd_ctrl->prop_id);
@@ -1213,7 +1170,7 @@ int exynos_drm_ipp_cmd_ctrl(struct drm_device *drm_dev, void *data,
                 return -EINVAL;
         }
 
-        DRM_DEBUG_KMS("%s:done ctrl[%d]prop_id[%d]\n", __func__,
+        DRM_DEBUG_KMS("done ctrl[%d]prop_id[%d]\n",
                 cmd_ctrl->ctrl, cmd_ctrl->prop_id);
 
         return 0;
@@ -1249,7 +1206,7 @@ static int ipp_set_property(struct exynos_drm_ippdrv *ippdrv,
                 return -EINVAL;
         }
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]\n", __func__, property->prop_id);
+        DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
 
         /* reset h/w block */
         if (ippdrv->reset &&
@@ -1310,13 +1267,13 @@ static int ipp_start_property(struct exynos_drm_ippdrv *ippdrv,
         struct list_head *head;
         int ret, i;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]\n", __func__, property->prop_id);
+        DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
 
         /* store command info in ippdrv */
         ippdrv->c_node = c_node;
 
         if (!ipp_check_mem_list(c_node)) {
-                DRM_DEBUG_KMS("%s:empty memory.\n", __func__);
+                DRM_DEBUG_KMS("empty memory.\n");
                 return -ENOMEM;
         }
 
@@ -1343,8 +1300,7 @@ static int ipp_start_property(struct exynos_drm_ippdrv *ippdrv,
                                 return ret;
                         }
 
-                        DRM_DEBUG_KMS("%s:m_node[0x%x]\n",
-                                __func__, (int)m_node);
+                        DRM_DEBUG_KMS("m_node[0x%x]\n", (int)m_node);
 
                         ret = ipp_set_mem_node(ippdrv, c_node, m_node);
                         if (ret) {
@@ -1382,7 +1338,7 @@ static int ipp_start_property(struct exynos_drm_ippdrv *ippdrv,
                 return -EINVAL;
         }
 
-        DRM_DEBUG_KMS("%s:cmd[%d]\n", __func__, property->cmd);
+        DRM_DEBUG_KMS("cmd[%d]\n", property->cmd);
 
         /* start operations */
         if (ippdrv->start) {
@@ -1405,7 +1361,7 @@ static int ipp_stop_property(struct drm_device *drm_dev,
         struct list_head *head;
         int ret = 0, i;
 
-        DRM_DEBUG_KMS("%s:prop_id[%d]\n", __func__, property->prop_id);
+        DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
 
         /* put event */
         ipp_put_event(c_node, NULL);
@@ -1418,8 +1374,7 @@ static int ipp_stop_property(struct drm_device *drm_dev,
                         head = &c_node->mem_list[i];
 
                         if (list_empty(head)) {
-                                DRM_DEBUG_KMS("%s:mem_list is empty.\n",
-                                        __func__);
+                                DRM_DEBUG_KMS("mem_list is empty.\n");
                                 break;
                         }
 
@@ -1439,7 +1394,7 @@ static int ipp_stop_property(struct drm_device *drm_dev,
                 head = &c_node->mem_list[EXYNOS_DRM_OPS_DST];
 
                 if (list_empty(head)) {
-                        DRM_DEBUG_KMS("%s:mem_list is empty.\n", __func__);
+                        DRM_DEBUG_KMS("mem_list is empty.\n");
                         break;
                 }
 
@@ -1456,7 +1411,7 @@ static int ipp_stop_property(struct drm_device *drm_dev,
                 head = &c_node->mem_list[EXYNOS_DRM_OPS_SRC];
 
                 if (list_empty(head)) {
-                        DRM_DEBUG_KMS("%s:mem_list is empty.\n", __func__);
+                        DRM_DEBUG_KMS("mem_list is empty.\n");
                         break;
                 }
 
@@ -1491,8 +1446,6 @@ void ipp_sched_cmd(struct work_struct *work)
         struct drm_exynos_ipp_property *property;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         ippdrv = cmd_work->ippdrv;
         if (!ippdrv) {
                 DRM_ERROR("invalid ippdrv list.\n");
@@ -1550,7 +1503,7 @@ void ipp_sched_cmd(struct work_struct *work)
                 break;
         }
 
-        DRM_DEBUG_KMS("%s:ctrl[%d] done.\n", __func__, cmd_work->ctrl);
+        DRM_DEBUG_KMS("ctrl[%d] done.\n", cmd_work->ctrl);
 
 err_unlock:
         mutex_unlock(&c_node->cmd_lock);
@@ -1571,8 +1524,7 @@ static int ipp_send_event(struct exynos_drm_ippdrv *ippdrv,
         int ret, i;
 
         for_each_ipp_ops(i)
-                DRM_DEBUG_KMS("%s:%s buf_id[%d]\n", __func__,
-                        i ? "dst" : "src", buf_id[i]);
+                DRM_DEBUG_KMS("%s buf_id[%d]\n", i ? "dst" : "src", buf_id[i]);
 
         if (!drm_dev) {
                 DRM_ERROR("failed to get drm_dev.\n");
@@ -1585,12 +1537,12 @@ static int ipp_send_event(struct exynos_drm_ippdrv *ippdrv,
         }
 
         if (list_empty(&c_node->event_list)) {
-                DRM_DEBUG_KMS("%s:event list is empty.\n", __func__);
+                DRM_DEBUG_KMS("event list is empty.\n");
                 return 0;
         }
 
         if (!ipp_check_mem_list(c_node)) {
-                DRM_DEBUG_KMS("%s:empty memory.\n", __func__);
+                DRM_DEBUG_KMS("empty memory.\n");
                 return 0;
         }
 
@@ -1609,7 +1561,7 @@ static int ipp_send_event(struct exynos_drm_ippdrv *ippdrv,
                         }
 
                         tbuf_id[i] = m_node->buf_id;
-                        DRM_DEBUG_KMS("%s:%s buf_id[%d]\n", __func__,
+                        DRM_DEBUG_KMS("%s buf_id[%d]\n",
                                 i ? "dst" : "src", tbuf_id[i]);
 
                         ret = ipp_put_mem_node(drm_dev, c_node, m_node);
@@ -1677,8 +1629,7 @@ static int ipp_send_event(struct exynos_drm_ippdrv *ippdrv,
         }
 
         do_gettimeofday(&now);
-        DRM_DEBUG_KMS("%s:tv_sec[%ld]tv_usec[%ld]\n"
-                , __func__, now.tv_sec, now.tv_usec);
+        DRM_DEBUG_KMS("tv_sec[%ld]tv_usec[%ld]\n", now.tv_sec, now.tv_usec);
         e->event.tv_sec = now.tv_sec;
         e->event.tv_usec = now.tv_usec;
         e->event.prop_id = property->prop_id;
@@ -1692,7 +1643,7 @@ static int ipp_send_event(struct exynos_drm_ippdrv *ippdrv,
         wake_up_interruptible(&e->base.file_priv->event_wait);
         spin_unlock_irqrestore(&drm_dev->event_lock, flags);
 
-        DRM_DEBUG_KMS("%s:done cmd[%d]prop_id[%d]buf_id[%d]\n", __func__,
+        DRM_DEBUG_KMS("done cmd[%d]prop_id[%d]buf_id[%d]\n",
                 property->cmd, property->prop_id, tbuf_id[EXYNOS_DRM_OPS_DST]);
 
         return 0;
@@ -1711,8 +1662,7 @@ void ipp_sched_event(struct work_struct *work)
                 return;
         }
 
-        DRM_DEBUG_KMS("%s:buf_id[%d]\n", __func__,
-                event_work->buf_id[EXYNOS_DRM_OPS_DST]);
+        DRM_DEBUG_KMS("buf_id[%d]\n", event_work->buf_id[EXYNOS_DRM_OPS_DST]);
 
         ippdrv = event_work->ippdrv;
         if (!ippdrv) {
@@ -1733,8 +1683,8 @@ void ipp_sched_event(struct work_struct *work)
          * or going out operations.
          */
         if (c_node->state != IPP_STATE_START) {
-                DRM_DEBUG_KMS("%s:bypass state[%d]prop_id[%d]\n",
-                        __func__, c_node->state, c_node->property.prop_id);
+                DRM_DEBUG_KMS("bypass state[%d]prop_id[%d]\n",
+                        c_node->state, c_node->property.prop_id);
                 goto err_completion;
         }
 
@@ -1759,8 +1709,6 @@ static int ipp_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
         struct exynos_drm_ippdrv *ippdrv;
         int ret, count = 0;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* get ipp driver entry */
         list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
                 ippdrv->drm_dev = drm_dev;
@@ -1772,7 +1720,7 @@ static int ipp_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
                         goto err_idr;
                 }
 
-                DRM_DEBUG_KMS("%s:count[%d]ippdrv[0x%x]ipp_id[%d]\n", __func__,
+                DRM_DEBUG_KMS("count[%d]ippdrv[0x%x]ipp_id[%d]\n",
                         count++, (int)ippdrv, ippdrv->ipp_id);
 
                 if (ippdrv->ipp_id == 0) {
@@ -1816,8 +1764,6 @@ static void ipp_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
 {
         struct exynos_drm_ippdrv *ippdrv;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* get ipp driver entry */
         list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
                 if (is_drm_iommu_supported(drm_dev))
@@ -1834,8 +1780,6 @@ static int ipp_subdrv_open(struct drm_device *drm_dev, struct device *dev,
         struct drm_exynos_file_private *file_priv = file->driver_priv;
         struct exynos_drm_ipp_private *priv;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
         if (!priv) {
                 DRM_ERROR("failed to allocate priv.\n");
@@ -1846,7 +1790,7 @@ static int ipp_subdrv_open(struct drm_device *drm_dev, struct device *dev,
 
         INIT_LIST_HEAD(&priv->event_list);
 
-        DRM_DEBUG_KMS("%s:done priv[0x%x]\n", __func__, (int)priv);
+        DRM_DEBUG_KMS("done priv[0x%x]\n", (int)priv);
 
         return 0;
 }
@@ -1860,10 +1804,10 @@ static void ipp_subdrv_close(struct drm_device *drm_dev, struct device *dev,
         struct drm_exynos_ipp_cmd_node *c_node, *tc_node;
         int count = 0;
 
-        DRM_DEBUG_KMS("%s:for priv[0x%x]\n", __func__, (int)priv);
+        DRM_DEBUG_KMS("for priv[0x%x]\n", (int)priv);
 
         if (list_empty(&exynos_drm_ippdrv_list)) {
-                DRM_DEBUG_KMS("%s:ippdrv_list is empty.\n", __func__);
+                DRM_DEBUG_KMS("ippdrv_list is empty.\n");
                 goto err_clear;
         }
 
@@ -1873,8 +1817,8 @@ static void ipp_subdrv_close(struct drm_device *drm_dev, struct device *dev,
 
                 list_for_each_entry_safe(c_node, tc_node,
                         &ippdrv->cmd_list, list) {
-                        DRM_DEBUG_KMS("%s:count[%d]ippdrv[0x%x]\n",
-                                __func__, count++, (int)ippdrv);
+                        DRM_DEBUG_KMS("count[%d]ippdrv[0x%x]\n",
+                                count++, (int)ippdrv);
 
                         if (c_node->priv == priv) {
                                 /*
@@ -1913,8 +1857,6 @@ static int ipp_probe(struct platform_device *pdev)
         if (!ctx)
                 return -ENOMEM;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         mutex_init(&ctx->ipp_lock);
         mutex_init(&ctx->prop_lock);
 
@@ -1978,8 +1920,6 @@ static int ipp_remove(struct platform_device *pdev)
 {
         struct ipp_context *ctx = platform_get_drvdata(pdev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         /* unregister sub driver */
         exynos_drm_subdrv_unregister(&ctx->subdrv);
 
@@ -1999,7 +1939,7 @@ static int ipp_remove(struct platform_device *pdev)
 
 static int ipp_power_ctrl(struct ipp_context *ctx, bool enable)
 {
-        DRM_DEBUG_KMS("%s:enable[%d]\n", __func__, enable);
+        DRM_DEBUG_KMS("enable[%d]\n", enable);
 
         return 0;
 }
@@ -2009,8 +1949,6 @@ static int ipp_suspend(struct device *dev)
 {
         struct ipp_context *ctx = get_ipp_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (pm_runtime_suspended(dev))
                 return 0;
 
@@ -2021,8 +1959,6 @@ static int ipp_resume(struct device *dev)
 {
         struct ipp_context *ctx = get_ipp_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!pm_runtime_suspended(dev))
                 return ipp_power_ctrl(ctx, true);
 
@@ -2035,8 +1971,6 @@ static int ipp_runtime_suspend(struct device *dev)
 {
         struct ipp_context *ctx = get_ipp_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         return ipp_power_ctrl(ctx, false);
 }
 
@@ -2044,8 +1978,6 @@ static int ipp_runtime_resume(struct device *dev)
 {
         struct ipp_context *ctx = get_ipp_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         return ipp_power_ctrl(ctx, true);
 }
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index 83efc662d65a..6ee55e68e0a2 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -81,8 +81,6 @@ int exynos_plane_mode_set(struct drm_plane *plane, struct drm_crtc *crtc,
         int nr;
         int i;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         nr = exynos_drm_fb_get_buf_cnt(fb);
         for (i = 0; i < nr; i++) {
                 struct exynos_drm_gem_buf *buffer = exynos_drm_fb_buffer(fb, i);
@@ -159,8 +157,6 @@ void exynos_plane_dpms(struct drm_plane *plane, int mode)
         struct exynos_plane *exynos_plane = to_exynos_plane(plane);
         struct exynos_drm_overlay *overlay = &exynos_plane->overlay;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (mode == DRM_MODE_DPMS_ON) {
                 if (exynos_plane->enabled)
                         return;
@@ -189,8 +185,6 @@ exynos_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 {
         int ret;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         ret = exynos_plane_mode_set(plane, crtc, fb, crtc_x, crtc_y,
                         crtc_w, crtc_h, src_x >> 16, src_y >> 16,
                         src_w >> 16, src_h >> 16);
@@ -207,8 +201,6 @@ exynos_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
 static int exynos_disable_plane(struct drm_plane *plane)
 {
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         exynos_plane_dpms(plane, DRM_MODE_DPMS_OFF);
 
         return 0;
@@ -218,8 +210,6 @@ static void exynos_plane_destroy(struct drm_plane *plane)
 {
         struct exynos_plane *exynos_plane = to_exynos_plane(plane);
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         exynos_disable_plane(plane);
         drm_plane_cleanup(plane);
         kfree(exynos_plane);
@@ -233,8 +223,6 @@ static int exynos_plane_set_property(struct drm_plane *plane,
         struct exynos_plane *exynos_plane = to_exynos_plane(plane);
         struct exynos_drm_private *dev_priv = dev->dev_private;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (property == dev_priv->plane_zpos_property) {
                 exynos_plane->overlay.zpos = val;
                 return 0;
@@ -256,8 +244,6 @@ static void exynos_plane_attach_zpos_property(struct drm_plane *plane)
         struct exynos_drm_private *dev_priv = dev->dev_private;
         struct drm_property *prop;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         prop = dev_priv->plane_zpos_property;
         if (!prop) {
                 prop = drm_property_create_range(dev, 0, "zpos", 0,
@@ -277,8 +263,6 @@ struct drm_plane *exynos_plane_init(struct drm_device *dev,
         struct exynos_plane *exynos_plane;
         int err;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         exynos_plane = kzalloc(sizeof(struct exynos_plane), GFP_KERNEL);
         if (!exynos_plane) {
                 DRM_ERROR("failed to allocate plane\n");
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 9b6c70964d71..427640aa5148 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -244,7 +244,7 @@ static int rotator_src_set_size(struct device *dev, int swap,
         /* Get format */
         fmt = rotator_reg_get_fmt(rot);
         if (!rotator_check_reg_fmt(fmt)) {
-                DRM_ERROR("%s:invalid format.\n", __func__);
+                DRM_ERROR("invalid format.\n");
                 return -EINVAL;
         }
 
@@ -287,7 +287,7 @@ static int rotator_src_set_addr(struct device *dev,
                 /* Get format */
                 fmt = rotator_reg_get_fmt(rot);
                 if (!rotator_check_reg_fmt(fmt)) {
-                        DRM_ERROR("%s:invalid format.\n", __func__);
+                        DRM_ERROR("invalid format.\n");
                         return -EINVAL;
                 }
 
@@ -381,7 +381,7 @@ static int rotator_dst_set_size(struct device *dev, int swap,
         /* Get format */
         fmt = rotator_reg_get_fmt(rot);
         if (!rotator_check_reg_fmt(fmt)) {
-                DRM_ERROR("%s:invalid format.\n", __func__);
+                DRM_ERROR("invalid format.\n");
                 return -EINVAL;
         }
 
@@ -422,7 +422,7 @@ static int rotator_dst_set_addr(struct device *dev,
                 /* Get format */
                 fmt = rotator_reg_get_fmt(rot);
                 if (!rotator_check_reg_fmt(fmt)) {
-                        DRM_ERROR("%s:invalid format.\n", __func__);
+                        DRM_ERROR("invalid format.\n");
                         return -EINVAL;
                 }
 
@@ -471,8 +471,6 @@ static int rotator_init_prop_list(struct exynos_drm_ippdrv *ippdrv)
 {
         struct drm_exynos_ipp_prop_list *prop_list;
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         prop_list = devm_kzalloc(ippdrv->dev, sizeof(*prop_list), GFP_KERNEL);
         if (!prop_list) {
                 DRM_ERROR("failed to alloc property list.\n");
@@ -502,7 +500,7 @@ static inline bool rotator_check_drm_fmt(u32 fmt)
         case DRM_FORMAT_NV12:
                 return true;
         default:
-                DRM_DEBUG_KMS("%s:not support format\n", __func__);
+                DRM_DEBUG_KMS("not support format\n");
                 return false;
         }
 }
@@ -516,7 +514,7 @@ static inline bool rotator_check_drm_flip(enum drm_exynos_flip flip)
         case EXYNOS_DRM_FLIP_BOTH:
                 return true;
         default:
-                DRM_DEBUG_KMS("%s:invalid flip\n", __func__);
+                DRM_DEBUG_KMS("invalid flip\n");
                 return false;
         }
 }
@@ -536,19 +534,18 @@ static int rotator_ippdrv_check_property(struct device *dev,
 
         /* Check format configuration */
         if (src_config->fmt != dst_config->fmt) {
-                DRM_DEBUG_KMS("%s:not support csc feature\n", __func__);
+                DRM_DEBUG_KMS("not support csc feature\n");
                 return -EINVAL;
         }
 
         if (!rotator_check_drm_fmt(dst_config->fmt)) {
-                DRM_DEBUG_KMS("%s:invalid format\n", __func__);
+                DRM_DEBUG_KMS("invalid format\n");
                 return -EINVAL;
         }
 
         /* Check transform configuration */
         if (src_config->degree != EXYNOS_DRM_DEGREE_0) {
-                DRM_DEBUG_KMS("%s:not support source-side rotation\n",
-                        __func__);
+                DRM_DEBUG_KMS("not support source-side rotation\n");
                 return -EINVAL;
         }
 
@@ -561,51 +558,47 @@ static int rotator_ippdrv_check_property(struct device *dev,
                 /* No problem */
                 break;
         default:
-                DRM_DEBUG_KMS("%s:invalid degree\n", __func__);
+                DRM_DEBUG_KMS("invalid degree\n");
                 return -EINVAL;
         }
 
         if (src_config->flip != EXYNOS_DRM_FLIP_NONE) {
-                DRM_DEBUG_KMS("%s:not support source-side flip\n", __func__);
+                DRM_DEBUG_KMS("not support source-side flip\n");
                 return -EINVAL;
         }
 
         if (!rotator_check_drm_flip(dst_config->flip)) {
-                DRM_DEBUG_KMS("%s:invalid flip\n", __func__);
+                DRM_DEBUG_KMS("invalid flip\n");
                 return -EINVAL;
         }
 
         /* Check size configuration */
         if ((src_pos->x + src_pos->w > src_sz->hsize) ||
                 (src_pos->y + src_pos->h > src_sz->vsize)) {
-                DRM_DEBUG_KMS("%s:out of source buffer bound\n", __func__);
+                DRM_DEBUG_KMS("out of source buffer bound\n");
                 return -EINVAL;
         }
 
         if (swap) {
                 if ((dst_pos->x + dst_pos->h > dst_sz->vsize) ||
                         (dst_pos->y + dst_pos->w > dst_sz->hsize)) {
-                        DRM_DEBUG_KMS("%s:out of destination buffer bound\n",
-                                __func__);
+                        DRM_DEBUG_KMS("out of destination buffer bound\n");
                         return -EINVAL;
                 }
 
                 if ((src_pos->w != dst_pos->h) || (src_pos->h != dst_pos->w)) {
-                        DRM_DEBUG_KMS("%s:not support scale feature\n",
-                                __func__);
+                        DRM_DEBUG_KMS("not support scale feature\n");
                         return -EINVAL;
                 }
         } else {
                 if ((dst_pos->x + dst_pos->w > dst_sz->hsize) ||
                         (dst_pos->y + dst_pos->h > dst_sz->vsize)) {
-                        DRM_DEBUG_KMS("%s:out of destination buffer bound\n",
-                                __func__);
+                        DRM_DEBUG_KMS("out of destination buffer bound\n");
                         return -EINVAL;
                 }
 
                 if ((src_pos->w != dst_pos->w) || (src_pos->h != dst_pos->h)) {
-                        DRM_DEBUG_KMS("%s:not support scale feature\n",
-                                __func__);
+                        DRM_DEBUG_KMS("not support scale feature\n");
                         return -EINVAL;
                 }
         }
@@ -693,7 +686,7 @@ static int rotator_probe(struct platform_device *pdev)
                 goto err_ippdrv_register;
         }
 
-        DRM_DEBUG_KMS("%s:ippdrv[0x%x]\n", __func__, (int)ippdrv);
+        DRM_DEBUG_KMS("ippdrv[0x%x]\n", (int)ippdrv);
 
         platform_set_drvdata(pdev, rot);
 
@@ -752,8 +745,6 @@ static struct platform_device_id rotator_driver_ids[] = {
 
 static int rotator_clk_crtl(struct rot_context *rot, bool enable)
 {
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (enable) {
                 clk_enable(rot->clock);
                 rot->suspended = false;
@@ -771,8 +762,6 @@ static int rotator_suspend(struct device *dev)
 {
         struct rot_context *rot = dev_get_drvdata(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (pm_runtime_suspended(dev))
                 return 0;
 
@@ -783,8 +772,6 @@ static int rotator_resume(struct device *dev)
 {
         struct rot_context *rot = dev_get_drvdata(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         if (!pm_runtime_suspended(dev))
                 return rotator_clk_crtl(rot, true);
 
@@ -797,8 +784,6 @@ static int rotator_runtime_suspend(struct device *dev)
 {
         struct rot_context *rot = dev_get_drvdata(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         return  rotator_clk_crtl(rot, false);
 }
 
@@ -806,8 +791,6 @@ static int rotator_runtime_resume(struct device *dev)
 {
         struct rot_context *rot = dev_get_drvdata(dev);
 
-        DRM_DEBUG_KMS("%s\n", __func__);
-
         return  rotator_clk_crtl(rot, true);
 }
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index 24376c194a5e..784bbce0741a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -89,8 +89,6 @@ static bool vidi_display_is_connected(struct device *dev)
 {
         struct vidi_context *ctx = get_vidi_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * connection request would come from user side
          * to do hotplug through specific ioctl.
@@ -105,8 +103,6 @@ static struct edid *vidi_get_edid(struct device *dev,
         struct edid *edid;
         int edid_len;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * the edid data comes from user side and it would be set
          * to ctx->raw_edid through specific ioctl.
@@ -128,17 +124,13 @@ static struct edid *vidi_get_edid(struct device *dev,
 
 static void *vidi_get_panel(struct device *dev)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO. */
 
         return NULL;
 }
 
-static int vidi_check_timing(struct device *dev, void *timing)
+static int vidi_check_mode(struct device *dev, struct drm_display_mode *mode)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO. */
 
         return 0;
@@ -146,8 +138,6 @@ static int vidi_check_timing(struct device *dev, void *timing)
 
 static int vidi_display_power_on(struct device *dev, int mode)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO */
 
         return 0;
@@ -158,7 +148,7 @@ static struct exynos_drm_display_ops vidi_display_ops = {
         .is_connected = vidi_display_is_connected,
         .get_edid = vidi_get_edid,
         .get_panel = vidi_get_panel,
-        .check_timing = vidi_check_timing,
+        .check_mode = vidi_check_mode,
         .power_on = vidi_display_power_on,
 };
 
@@ -166,7 +156,7 @@ static void vidi_dpms(struct device *subdrv_dev, int mode)
 {
         struct vidi_context *ctx = get_vidi_context(subdrv_dev);
 
-        DRM_DEBUG_KMS("%s, %d\n", __FILE__, mode);
+        DRM_DEBUG_KMS("%d\n", mode);
 
         mutex_lock(&ctx->lock);
 
@@ -196,8 +186,6 @@ static void vidi_apply(struct device *subdrv_dev)
         struct vidi_win_data *win_data;
         int i;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         for (i = 0; i < WINDOWS_NR; i++) {
                 win_data = &ctx->win_data[i];
                 if (win_data->enabled && (ovl_ops && ovl_ops->commit))
@@ -212,8 +200,6 @@ static void vidi_commit(struct device *dev)
 {
         struct vidi_context *ctx = get_vidi_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return;
 }
@@ -222,8 +208,6 @@ static int vidi_enable_vblank(struct device *dev)
 {
         struct vidi_context *ctx = get_vidi_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return -EPERM;
 
@@ -246,8 +230,6 @@ static void vidi_disable_vblank(struct device *dev)
 {
         struct vidi_context *ctx = get_vidi_context(dev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return;
 
@@ -271,8 +253,6 @@ static void vidi_win_mode_set(struct device *dev,
         int win;
         unsigned long offset;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!overlay) {
                 dev_err(dev, "overlay is NULL\n");
                 return;
@@ -282,7 +262,7 @@ static void vidi_win_mode_set(struct device *dev,
         if (win == DEFAULT_ZPOS)
                 win = ctx->default_win;
 
-        if (win < 0 || win > WINDOWS_NR)
+        if (win < 0 || win >= WINDOWS_NR)
                 return;
 
         offset = overlay->fb_x * (overlay->bpp >> 3);
@@ -324,15 +304,13 @@ static void vidi_win_commit(struct device *dev, int zpos)
         struct vidi_win_data *win_data;
         int win = zpos;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (ctx->suspended)
                 return;
 
         if (win == DEFAULT_ZPOS)
                 win = ctx->default_win;
 
-        if (win < 0 || win > WINDOWS_NR)
+        if (win < 0 || win >= WINDOWS_NR)
                 return;
 
         win_data = &ctx->win_data[win];
@@ -351,12 +329,10 @@ static void vidi_win_disable(struct device *dev, int zpos)
         struct vidi_win_data *win_data;
         int win = zpos;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (win == DEFAULT_ZPOS)
                 win = ctx->default_win;
 
-        if (win < 0 || win > WINDOWS_NR)
+        if (win < 0 || win >= WINDOWS_NR)
                 return;
 
         win_data = &ctx->win_data[win];
@@ -407,8 +383,6 @@ static void vidi_fake_vblank_handler(struct work_struct *work)
 
 static int vidi_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /*
          * enable drm irq mode.
          * - with irq_enabled = 1, we can use the vblank feature.
@@ -431,8 +405,6 @@ static int vidi_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
 
 static void vidi_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
 {
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         /* TODO. */
 }
 
@@ -441,8 +413,6 @@ static int vidi_power_on(struct vidi_context *ctx, bool enable)
         struct exynos_drm_subdrv *subdrv = &ctx->subdrv;
         struct device *dev = subdrv->dev;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (enable != false && enable != true)
                 return -EINVAL;
 
@@ -483,8 +453,6 @@ static int vidi_store_connection(struct device *dev,
         struct vidi_context *ctx = get_vidi_context(dev);
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         ret = kstrtoint(buf, 0, &ctx->connected);
         if (ret)
                 return ret;
@@ -522,8 +490,6 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data,
         struct drm_exynos_vidi_connection *vidi = data;
         int edid_len;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         if (!vidi) {
                 DRM_DEBUG_KMS("user data for vidi is null.\n");
                 return -EINVAL;
@@ -592,8 +558,6 @@ static int vidi_probe(struct platform_device *pdev)
         struct exynos_drm_subdrv *subdrv;
         int ret;
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
         if (!ctx)
                 return -ENOMEM;
@@ -625,8 +589,6 @@ static int vidi_remove(struct platform_device *pdev)
 {
         struct vidi_context *ctx = platform_get_drvdata(pdev);
 
-        DRM_DEBUG_KMS("%s\n", __FILE__);
-
         exynos_drm_subdrv_unregister(&ctx->subdrv);
 
         if (ctx->raw_edid != (struct edid *)fake_edid_info) {
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index fd1426dca882..62ef5971ac3c 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -83,6 +83,7 @@ struct hdmi_resources {
         struct clk                      *sclk_pixel;
         struct clk                      *sclk_hdmiphy;
         struct clk                      *hdmiphy;
+        struct clk                      *mout_hdmi;
         struct regulator_bulk_data      *regul_bulk;
         int                             regul_count;
 };
@@ -689,8 +690,6 @@ static void hdmi_reg_infoframe(struct hdmi_context *hdata,
         u32 mod;
         u32 vic;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mod = hdmi_reg_read(hdata, HDMI_MODE_SEL);
         if (hdata->dvi_mode) {
                 hdmi_reg_writeb(hdata, HDMI_VSI_CON,
@@ -755,8 +754,6 @@ static struct edid *hdmi_get_edid(void *ctx, struct drm_connector *connector)
         struct edid *raw_edid;
         struct hdmi_context *hdata = ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (!hdata->ddc_port)
                 return ERR_PTR(-ENODEV);
 
@@ -777,8 +774,6 @@ static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock)
         const struct hdmiphy_config *confs;
         int count, i;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (hdata->type == HDMI_TYPE13) {
                 confs = hdmiphy_v13_configs;
                 count = ARRAY_SIZE(hdmiphy_v13_configs);
@@ -796,18 +791,17 @@ static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock)
         return -EINVAL;
 }
 
-static int hdmi_check_timing(void *ctx, struct fb_videomode *timing)
+static int hdmi_check_mode(void *ctx, struct drm_display_mode *mode)
 {
         struct hdmi_context *hdata = ctx;
         int ret;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
+        DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n",
+                mode->hdisplay, mode->vdisplay, mode->vrefresh,
+                (mode->flags & DRM_MODE_FLAG_INTERLACE) ? true :
+                false, mode->clock * 1000);
 
-        DRM_DEBUG_KMS("[%d]x[%d] [%d]Hz [%x]\n", timing->xres,
-                        timing->yres, timing->refresh,
-                        timing->vmode);
-
-        ret = hdmi_find_phy_conf(hdata, timing->pixclock);
+        ret = hdmi_find_phy_conf(hdata, mode->clock * 1000);
         if (ret < 0)
                 return ret;
         return 0;
@@ -1042,7 +1036,7 @@ static void hdmi_conf_init(struct hdmi_context *hdata)
         }
 }
 
-static void hdmi_v13_timing_apply(struct hdmi_context *hdata)
+static void hdmi_v13_mode_apply(struct hdmi_context *hdata)
 {
         const struct hdmi_tg_regs *tg = &hdata->mode_conf.conf.v13_conf.tg;
         const struct hdmi_v13_core_regs *core =
@@ -1118,9 +1112,9 @@ static void hdmi_v13_timing_apply(struct hdmi_context *hdata)
                 hdmi_regs_dump(hdata, "timing apply");
         }
 
-        clk_disable(hdata->res.sclk_hdmi);
-        clk_set_parent(hdata->res.sclk_hdmi, hdata->res.sclk_hdmiphy);
-        clk_enable(hdata->res.sclk_hdmi);
+        clk_disable_unprepare(hdata->res.sclk_hdmi);
+        clk_set_parent(hdata->res.mout_hdmi, hdata->res.sclk_hdmiphy);
+        clk_prepare_enable(hdata->res.sclk_hdmi);
 
         /* enable HDMI and timing generator */
         hdmi_reg_writemask(hdata, HDMI_CON_0, ~0, HDMI_EN);
@@ -1131,7 +1125,7 @@ static void hdmi_v13_timing_apply(struct hdmi_context *hdata)
                 hdmi_reg_writemask(hdata, HDMI_TG_CMD, ~0, HDMI_TG_EN);
 }
 
-static void hdmi_v14_timing_apply(struct hdmi_context *hdata)
+static void hdmi_v14_mode_apply(struct hdmi_context *hdata)
 {
         const struct hdmi_tg_regs *tg = &hdata->mode_conf.conf.v14_conf.tg;
         const struct hdmi_v14_core_regs *core =
@@ -1285,9 +1279,9 @@ static void hdmi_v14_timing_apply(struct hdmi_context *hdata)
                 hdmi_regs_dump(hdata, "timing apply");
         }
 
-        clk_disable(hdata->res.sclk_hdmi);
-        clk_set_parent(hdata->res.sclk_hdmi, hdata->res.sclk_hdmiphy);
-        clk_enable(hdata->res.sclk_hdmi);
+        clk_disable_unprepare(hdata->res.sclk_hdmi);
+        clk_set_parent(hdata->res.mout_hdmi, hdata->res.sclk_hdmiphy);
+        clk_prepare_enable(hdata->res.sclk_hdmi);
 
         /* enable HDMI and timing generator */
         hdmi_reg_writemask(hdata, HDMI_CON_0, ~0, HDMI_EN);
@@ -1298,12 +1292,12 @@ static void hdmi_v14_timing_apply(struct hdmi_context *hdata)
                 hdmi_reg_writemask(hdata, HDMI_TG_CMD, ~0, HDMI_TG_EN);
 }
 
-static void hdmi_timing_apply(struct hdmi_context *hdata)
+static void hdmi_mode_apply(struct hdmi_context *hdata)
 {
         if (hdata->type == HDMI_TYPE13)
-                hdmi_v13_timing_apply(hdata);
+                hdmi_v13_mode_apply(hdata);
         else
-                hdmi_v14_timing_apply(hdata);
+                hdmi_v14_mode_apply(hdata);
 }
 
 static void hdmiphy_conf_reset(struct hdmi_context *hdata)
@@ -1311,9 +1305,9 @@ static void hdmiphy_conf_reset(struct hdmi_context *hdata)
         u8 buffer[2];
         u32 reg;
 
-        clk_disable(hdata->res.sclk_hdmi);
-        clk_set_parent(hdata->res.sclk_hdmi, hdata->res.sclk_pixel);
-        clk_enable(hdata->res.sclk_hdmi);
+        clk_disable_unprepare(hdata->res.sclk_hdmi);
+        clk_set_parent(hdata->res.mout_hdmi, hdata->res.sclk_pixel);
+        clk_prepare_enable(hdata->res.sclk_hdmi);
 
         /* operation mode */
         buffer[0] = 0x1f;
@@ -1336,8 +1330,6 @@ static void hdmiphy_conf_reset(struct hdmi_context *hdata)
 
 static void hdmiphy_poweron(struct hdmi_context *hdata)
 {
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (hdata->type == HDMI_TYPE14)
                 hdmi_reg_writemask(hdata, HDMI_PHY_CON_0, 0,
                         HDMI_PHY_POWER_OFF_EN);
@@ -1345,8 +1337,6 @@ static void hdmiphy_poweron(struct hdmi_context *hdata)
 
 static void hdmiphy_poweroff(struct hdmi_context *hdata)
 {
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (hdata->type == HDMI_TYPE14)
                 hdmi_reg_writemask(hdata, HDMI_PHY_CON_0, ~0,
                         HDMI_PHY_POWER_OFF_EN);
@@ -1410,8 +1400,6 @@ static void hdmiphy_conf_apply(struct hdmi_context *hdata)
 
 static void hdmi_conf_apply(struct hdmi_context *hdata)
 {
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         hdmiphy_conf_reset(hdata);
         hdmiphy_conf_apply(hdata);
 
@@ -1423,7 +1411,7 @@ static void hdmi_conf_apply(struct hdmi_context *hdata)
         hdmi_audio_init(hdata);
 
         /* setting core registers */
-        hdmi_timing_apply(hdata);
+        hdmi_mode_apply(hdata);
         hdmi_audio_control(hdata, true);
 
         hdmi_regs_dump(hdata, "start");
@@ -1569,8 +1557,7 @@ static void hdmi_v14_mode_set(struct hdmi_context *hdata,
                         (m->vsync_start - m->vdisplay) / 2);
                 hdmi_set_reg(core->v2_blank, 2, m->vtotal / 2);
                 hdmi_set_reg(core->v1_blank, 2, (m->vtotal - m->vdisplay) / 2);
-                hdmi_set_reg(core->v_blank_f0, 2, (m->vtotal +
-                        ((m->vsync_end - m->vsync_start) * 4) + 5) / 2);
+                hdmi_set_reg(core->v_blank_f0, 2, m->vtotal - m->vdisplay / 2);
                 hdmi_set_reg(core->v_blank_f1, 2, m->vtotal);
                 hdmi_set_reg(core->v_sync_line_aft_2, 2, (m->vtotal / 2) + 7);
                 hdmi_set_reg(core->v_sync_line_aft_1, 2, (m->vtotal / 2) + 2);
@@ -1580,7 +1567,10 @@ static void hdmi_v14_mode_set(struct hdmi_context *hdata,
                         (m->htotal / 2) + (m->hsync_start - m->hdisplay));
                 hdmi_set_reg(tg->vact_st, 2, (m->vtotal - m->vdisplay) / 2);
                 hdmi_set_reg(tg->vact_sz, 2, m->vdisplay / 2);
-                hdmi_set_reg(tg->vact_st2, 2, 0x249);/* Reset value + 1*/
+                hdmi_set_reg(tg->vact_st2, 2, m->vtotal - m->vdisplay / 2);
+                hdmi_set_reg(tg->vsync2, 2, (m->vtotal / 2) + 1);
+                hdmi_set_reg(tg->vsync_bot_hdmi, 2, (m->vtotal / 2) + 1);
+                hdmi_set_reg(tg->field_bot_hdmi, 2, (m->vtotal / 2) + 1);
                 hdmi_set_reg(tg->vact_st3, 2, 0x0);
                 hdmi_set_reg(tg->vact_st4, 2, 0x0);
         } else {
@@ -1602,6 +1592,9 @@ static void hdmi_v14_mode_set(struct hdmi_context *hdata,
                 hdmi_set_reg(tg->vact_st2, 2, 0x248); /* Reset value */
                 hdmi_set_reg(tg->vact_st3, 2, 0x47b); /* Reset value */
                 hdmi_set_reg(tg->vact_st4, 2, 0x6ae); /* Reset value */
+                hdmi_set_reg(tg->vsync2, 2, 0x233); /* Reset value */
+                hdmi_set_reg(tg->vsync_bot_hdmi, 2, 0x233); /* Reset value */
+                hdmi_set_reg(tg->field_bot_hdmi, 2, 0x233); /* Reset value */
         }
 
         /* Following values & calculations are same irrespective of mode type */
@@ -1633,22 +1626,19 @@ static void hdmi_v14_mode_set(struct hdmi_context *hdata,
         hdmi_set_reg(tg->hact_sz, 2, m->hdisplay);
         hdmi_set_reg(tg->v_fsz, 2, m->vtotal);
         hdmi_set_reg(tg->vsync, 2, 0x1);
-        hdmi_set_reg(tg->vsync2, 2, 0x233); /* Reset value */
         hdmi_set_reg(tg->field_chg, 2, 0x233); /* Reset value */
         hdmi_set_reg(tg->vsync_top_hdmi, 2, 0x1); /* Reset value */
-        hdmi_set_reg(tg->vsync_bot_hdmi, 2, 0x233); /* Reset value */
         hdmi_set_reg(tg->field_top_hdmi, 2, 0x1); /* Reset value */
-        hdmi_set_reg(tg->field_bot_hdmi, 2, 0x233); /* Reset value */
         hdmi_set_reg(tg->tg_3d, 1, 0x0);
 }
 
-static void hdmi_mode_set(void *ctx, void *mode)
+static void hdmi_mode_set(void *ctx, struct drm_display_mode *mode)
 {
         struct hdmi_context *hdata = ctx;
         struct drm_display_mode *m = mode;
 
-        DRM_DEBUG_KMS("[%s]: xres=%d, yres=%d, refresh=%d, intl=%s\n",
-                __func__, m->hdisplay, m->vdisplay,
+        DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%s\n",
+                m->hdisplay, m->vdisplay,
                 m->vrefresh, (m->flags & DRM_MODE_FLAG_INTERLACE) ?
                 "INTERLACED" : "PROGERESSIVE");
 
@@ -1661,8 +1651,6 @@ static void hdmi_mode_set(void *ctx, void *mode)
 static void hdmi_get_max_resol(void *ctx, unsigned int *width,
                                         unsigned int *height)
 {
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         *width = MAX_WIDTH;
         *height = MAX_HEIGHT;
 }
@@ -1671,8 +1659,6 @@ static void hdmi_commit(void *ctx)
 {
         struct hdmi_context *hdata = ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mutex_lock(&hdata->hdmi_mutex);
         if (!hdata->powered) {
                 mutex_unlock(&hdata->hdmi_mutex);
@@ -1687,8 +1673,6 @@ static void hdmi_poweron(struct hdmi_context *hdata)
 {
         struct hdmi_resources *res = &hdata->res;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mutex_lock(&hdata->hdmi_mutex);
         if (hdata->powered) {
                 mutex_unlock(&hdata->hdmi_mutex);
@@ -1699,10 +1683,12 @@ static void hdmi_poweron(struct hdmi_context *hdata)
 
         mutex_unlock(&hdata->hdmi_mutex);
 
-        regulator_bulk_enable(res->regul_count, res->regul_bulk);
-        clk_enable(res->hdmiphy);
-        clk_enable(res->hdmi);
-        clk_enable(res->sclk_hdmi);
+        if (regulator_bulk_enable(res->regul_count, res->regul_bulk))
+                DRM_DEBUG_KMS("failed to enable regulator bulk\n");
+
+        clk_prepare_enable(res->hdmiphy);
+        clk_prepare_enable(res->hdmi);
+        clk_prepare_enable(res->sclk_hdmi);
 
         hdmiphy_poweron(hdata);
 }
@@ -1711,8 +1697,6 @@ static void hdmi_poweroff(struct hdmi_context *hdata)
 {
         struct hdmi_resources *res = &hdata->res;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mutex_lock(&hdata->hdmi_mutex);
         if (!hdata->powered)
                 goto out;
@@ -1725,9 +1709,9 @@ static void hdmi_poweroff(struct hdmi_context *hdata)
         hdmiphy_conf_reset(hdata);
         hdmiphy_poweroff(hdata);
 
-        clk_disable(res->sclk_hdmi);
-        clk_disable(res->hdmi);
-        clk_disable(res->hdmiphy);
+        clk_disable_unprepare(res->sclk_hdmi);
+        clk_disable_unprepare(res->hdmi);
+        clk_disable_unprepare(res->hdmiphy);
         regulator_bulk_disable(res->regul_count, res->regul_bulk);
 
         mutex_lock(&hdata->hdmi_mutex);
@@ -1742,7 +1726,7 @@ static void hdmi_dpms(void *ctx, int mode)
 {
         struct hdmi_context *hdata = ctx;
 
-        DRM_DEBUG_KMS("[%d] %s mode %d\n", __LINE__, __func__, mode);
+        DRM_DEBUG_KMS("mode %d\n", mode);
 
         switch (mode) {
         case DRM_MODE_DPMS_ON:
@@ -1765,7 +1749,7 @@ static struct exynos_hdmi_ops hdmi_ops = {
         /* display */
         .is_connected   = hdmi_is_connected,
         .get_edid       = hdmi_get_edid,
-        .check_timing   = hdmi_check_timing,
+        .check_mode     = hdmi_check_mode,
 
         /* manager */
         .mode_set       = hdmi_mode_set,
@@ -1831,8 +1815,13 @@ static int hdmi_resources_init(struct hdmi_context *hdata)
                 DRM_ERROR("failed to get clock 'hdmiphy'\n");
                 goto fail;
         }
+        res->mout_hdmi = devm_clk_get(dev, "mout_hdmi");
+        if (IS_ERR(res->mout_hdmi)) {
+                DRM_ERROR("failed to get clock 'mout_hdmi'\n");
+                goto fail;
+        }
 
-        clk_set_parent(res->sclk_hdmi, res->sclk_pixel);
+        clk_set_parent(res->mout_hdmi, res->sclk_pixel);
 
         res->regul_bulk = devm_kzalloc(dev, ARRAY_SIZE(supply) *
                 sizeof(res->regul_bulk[0]), GFP_KERNEL);
@@ -1877,7 +1866,6 @@ static struct s5p_hdmi_platform_data *drm_hdmi_dt_parse_pdata
 {
         struct device_node *np = dev->of_node;
         struct s5p_hdmi_platform_data *pd;
-        enum of_gpio_flags flags;
         u32 value;
 
         pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
@@ -1891,7 +1879,7 @@ static struct s5p_hdmi_platform_data *drm_hdmi_dt_parse_pdata
                 goto err_data;
         }
 
-        pd->hpd_gpio = of_get_named_gpio_flags(np, "hpd-gpio", 0, &flags);
+        pd->hpd_gpio = of_get_named_gpio(np, "hpd-gpio", 0);
 
         return pd;
 
@@ -1930,6 +1918,9 @@ static struct of_device_id hdmi_match_types[] = {
                 .compatible = "samsung,exynos5-hdmi",
                 .data   = (void *)HDMI_TYPE14,
         }, {
+                .compatible = "samsung,exynos4212-hdmi",
+                .data   = (void *)HDMI_TYPE14,
+        }, {
                 /* end node */
         }
 };
@@ -1944,8 +1935,6 @@ static int hdmi_probe(struct platform_device *pdev)
         struct resource *res;
         int ret;
 
-        DRM_DEBUG_KMS("[%d]\n", __LINE__);
-
         if (dev->of_node) {
                 pdata = drm_hdmi_dt_parse_pdata(dev);
                 if (IS_ERR(pdata)) {
@@ -2071,8 +2060,6 @@ static int hdmi_remove(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         pm_runtime_disable(dev);
 
         /* hdmiphy i2c driver */
@@ -2089,8 +2076,6 @@ static int hdmi_suspend(struct device *dev)
         struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
         struct hdmi_context *hdata = ctx->ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         disable_irq(hdata->irq);
 
         hdata->hpd = false;
@@ -2098,7 +2083,7 @@ static int hdmi_suspend(struct device *dev)
                 drm_helper_hpd_irq_event(ctx->drm_dev);
 
         if (pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("%s : Already suspended\n", __func__);
+                DRM_DEBUG_KMS("Already suspended\n");
                 return 0;
         }
 
@@ -2112,14 +2097,12 @@ static int hdmi_resume(struct device *dev)
         struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
         struct hdmi_context *hdata = ctx->ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         hdata->hpd = gpio_get_value(hdata->hpd_gpio);
 
         enable_irq(hdata->irq);
 
         if (!pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("%s : Already resumed\n", __func__);
+                DRM_DEBUG_KMS("Already resumed\n");
                 return 0;
         }
 
@@ -2134,7 +2117,6 @@ static int hdmi_runtime_suspend(struct device *dev)
 {
         struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
         struct hdmi_context *hdata = ctx->ctx;
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
 
         hdmi_poweroff(hdata);
 
@@ -2145,7 +2127,6 @@ static int hdmi_runtime_resume(struct device *dev)
 {
         struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
         struct hdmi_context *hdata = ctx->ctx;
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
 
         hdmi_poweron(hdata);
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmiphy.c b/drivers/gpu/drm/exynos/exynos_hdmiphy.c
index ea49d132ecf6..ef04255076c7 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmiphy.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmiphy.c
@@ -51,6 +51,10 @@ static struct of_device_id hdmiphy_match_types[] = {
         {
                 .compatible = "samsung,exynos5-hdmiphy",
         }, {
+                .compatible = "samsung,exynos4210-hdmiphy",
+        }, {
+                .compatible = "samsung,exynos4212-hdmiphy",
+        }, {
                 /* end node */
         }
 };
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index 7c197d3820c5..b1280b43931c 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -78,6 +78,7 @@ struct mixer_resources {
 enum mixer_version_id {
         MXR_VER_0_0_0_16,
         MXR_VER_16_0_33_0,
+        MXR_VER_128_0_0_184,
 };
 
 struct mixer_context {
@@ -283,17 +284,19 @@ static void mixer_cfg_scan(struct mixer_context *ctx, unsigned int height)
         val = (ctx->interlace ? MXR_CFG_SCAN_INTERLACE :
                                 MXR_CFG_SCAN_PROGRASSIVE);
 
-        /* choosing between porper HD and SD mode */
-        if (height <= 480)
-                val |= MXR_CFG_SCAN_NTSC | MXR_CFG_SCAN_SD;
-        else if (height <= 576)
-                val |= MXR_CFG_SCAN_PAL | MXR_CFG_SCAN_SD;
-        else if (height <= 720)
-                val |= MXR_CFG_SCAN_HD_720 | MXR_CFG_SCAN_HD;
-        else if (height <= 1080)
-                val |= MXR_CFG_SCAN_HD_1080 | MXR_CFG_SCAN_HD;
-        else
-                val |= MXR_CFG_SCAN_HD_720 | MXR_CFG_SCAN_HD;
+        if (ctx->mxr_ver != MXR_VER_128_0_0_184) {
+                /* choosing between proper HD and SD mode */
+                if (height <= 480)
+                        val |= MXR_CFG_SCAN_NTSC | MXR_CFG_SCAN_SD;
+                else if (height <= 576)
+                        val |= MXR_CFG_SCAN_PAL | MXR_CFG_SCAN_SD;
+                else if (height <= 720)
+                        val |= MXR_CFG_SCAN_HD_720 | MXR_CFG_SCAN_HD;
+                else if (height <= 1080)
+                        val |= MXR_CFG_SCAN_HD_1080 | MXR_CFG_SCAN_HD;
+                else
+                        val |= MXR_CFG_SCAN_HD_720 | MXR_CFG_SCAN_HD;
+        }
 
         mixer_reg_writemask(res, MXR_CFG, val, MXR_CFG_SCAN_MASK);
 }
@@ -557,6 +560,14 @@ static void mixer_graph_buffer(struct mixer_context *ctx, int win)
         /* setup geometry */
         mixer_reg_write(res, MXR_GRAPHIC_SPAN(win), win_data->fb_width);
 
+        /* setup display size */
+        if (ctx->mxr_ver == MXR_VER_128_0_0_184 &&
+                win == MIXER_DEFAULT_WIN) {
+                val  = MXR_MXR_RES_HEIGHT(win_data->fb_height);
+                val |= MXR_MXR_RES_WIDTH(win_data->fb_width);
+                mixer_reg_write(res, MXR_RESOLUTION, val);
+        }
+
         val  = MXR_GRP_WH_WIDTH(win_data->crtc_width);
         val |= MXR_GRP_WH_HEIGHT(win_data->crtc_height);
         val |= MXR_GRP_WH_H_SCALE(x_ratio);
@@ -581,7 +592,8 @@ static void mixer_graph_buffer(struct mixer_context *ctx, int win)
         mixer_cfg_layer(ctx, win, true);
 
         /* layer update mandatory for mixer 16.0.33.0 */
-        if (ctx->mxr_ver == MXR_VER_16_0_33_0)
+        if (ctx->mxr_ver == MXR_VER_16_0_33_0 ||
+                ctx->mxr_ver == MXR_VER_128_0_0_184)
                 mixer_layer_update(ctx);
 
         mixer_run(ctx);
@@ -696,8 +708,6 @@ static int mixer_enable_vblank(void *ctx, int pipe)
         struct mixer_context *mixer_ctx = ctx;
         struct mixer_resources *res = &mixer_ctx->mixer_res;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mixer_ctx->pipe = pipe;
 
         /* enable vsync interrupt */
@@ -712,8 +722,6 @@ static void mixer_disable_vblank(void *ctx)
         struct mixer_context *mixer_ctx = ctx;
         struct mixer_resources *res = &mixer_ctx->mixer_res;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         /* disable vsync interrupt */
         mixer_reg_writemask(res, MXR_INT_EN, 0, MXR_INT_EN_VSYNC);
 }
@@ -725,8 +733,6 @@ static void mixer_win_mode_set(void *ctx,
         struct hdmi_win_data *win_data;
         int win;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (!overlay) {
                 DRM_ERROR("overlay is NULL\n");
                 return;
@@ -742,7 +748,7 @@ static void mixer_win_mode_set(void *ctx,
         if (win == DEFAULT_ZPOS)
                 win = MIXER_DEFAULT_WIN;
 
-        if (win < 0 || win > MIXER_WIN_NR) {
+        if (win < 0 || win >= MIXER_WIN_NR) {
                 DRM_ERROR("mixer window[%d] is wrong\n", win);
                 return;
         }
@@ -776,7 +782,7 @@ static void mixer_win_commit(void *ctx, int win)
 {
         struct mixer_context *mixer_ctx = ctx;
 
-        DRM_DEBUG_KMS("[%d] %s, win: %d\n", __LINE__, __func__, win);
+        DRM_DEBUG_KMS("win: %d\n", win);
 
         mutex_lock(&mixer_ctx->mixer_mutex);
         if (!mixer_ctx->powered) {
@@ -799,7 +805,7 @@ static void mixer_win_disable(void *ctx, int win)
         struct mixer_resources *res = &mixer_ctx->mixer_res;
         unsigned long flags;
 
-        DRM_DEBUG_KMS("[%d] %s, win: %d\n", __LINE__, __func__, win);
+        DRM_DEBUG_KMS("win: %d\n", win);
 
         mutex_lock(&mixer_ctx->mixer_mutex);
         if (!mixer_ctx->powered) {
@@ -820,17 +826,21 @@ static void mixer_win_disable(void *ctx, int win)
         mixer_ctx->win_data[win].enabled = false;
 }
 
-static int mixer_check_timing(void *ctx, struct fb_videomode *timing)
+static int mixer_check_mode(void *ctx, struct drm_display_mode *mode)
 {
+        struct mixer_context *mixer_ctx = ctx;
         u32 w, h;
 
-        w = timing->xres;
-        h = timing->yres;
+        w = mode->hdisplay;
+        h = mode->vdisplay;
 
-        DRM_DEBUG_KMS("%s : xres=%d, yres=%d, refresh=%d, intl=%d\n",
-                __func__, timing->xres, timing->yres,
-                timing->refresh, (timing->vmode &
-                FB_VMODE_INTERLACED) ? true : false);
+        DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d\n",
+                mode->hdisplay, mode->vdisplay, mode->vrefresh,
+                (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 1 : 0);
+
+        if (mixer_ctx->mxr_ver == MXR_VER_0_0_0_16 ||
+                mixer_ctx->mxr_ver == MXR_VER_128_0_0_184)
+                return 0;
 
         if ((w >= 464 && w <= 720 && h >= 261 && h <= 576) ||
                 (w >= 1024 && w <= 1280 && h >= 576 && h <= 720) ||
@@ -891,8 +901,6 @@ static void mixer_poweron(struct mixer_context *ctx)
 {
         struct mixer_resources *res = &ctx->mixer_res;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mutex_lock(&ctx->mixer_mutex);
         if (ctx->powered) {
                 mutex_unlock(&ctx->mixer_mutex);
@@ -901,10 +909,10 @@ static void mixer_poweron(struct mixer_context *ctx)
         ctx->powered = true;
         mutex_unlock(&ctx->mixer_mutex);
 
-        clk_enable(res->mixer);
+        clk_prepare_enable(res->mixer);
         if (ctx->vp_enabled) {
-                clk_enable(res->vp);
-                clk_enable(res->sclk_mixer);
+                clk_prepare_enable(res->vp);
+                clk_prepare_enable(res->sclk_mixer);
         }
 
         mixer_reg_write(res, MXR_INT_EN, ctx->int_en);
@@ -917,8 +925,6 @@ static void mixer_poweroff(struct mixer_context *ctx)
 {
         struct mixer_resources *res = &ctx->mixer_res;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mutex_lock(&ctx->mixer_mutex);
         if (!ctx->powered)
                 goto out;
@@ -928,10 +934,10 @@ static void mixer_poweroff(struct mixer_context *ctx)
 
         ctx->int_en = mixer_reg_read(res, MXR_INT_EN);
 
-        clk_disable(res->mixer);
+        clk_disable_unprepare(res->mixer);
         if (ctx->vp_enabled) {
-                clk_disable(res->vp);
-                clk_disable(res->sclk_mixer);
+                clk_disable_unprepare(res->vp);
+                clk_disable_unprepare(res->sclk_mixer);
         }
 
         mutex_lock(&ctx->mixer_mutex);
@@ -945,8 +951,6 @@ static void mixer_dpms(void *ctx, int mode)
 {
         struct mixer_context *mixer_ctx = ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         switch (mode) {
         case DRM_MODE_DPMS_ON:
                 if (pm_runtime_suspended(mixer_ctx->dev))
@@ -978,7 +982,7 @@ static struct exynos_mixer_ops mixer_ops = {
         .win_disable            = mixer_win_disable,
 
         /* display */
-        .check_timing           = mixer_check_timing,
+        .check_mode             = mixer_check_mode,
 };
 
 static irqreturn_t mixer_irq_handler(int irq, void *arg)
@@ -1128,12 +1132,17 @@ static int vp_resources_init(struct exynos_drm_hdmi_context *ctx,
         return 0;
 }
 
-static struct mixer_drv_data exynos5_mxr_drv_data = {
+static struct mixer_drv_data exynos5420_mxr_drv_data = {
+        .version = MXR_VER_128_0_0_184,
+        .is_vp_enabled = 0,
+};
+
+static struct mixer_drv_data exynos5250_mxr_drv_data = {
         .version = MXR_VER_16_0_33_0,
         .is_vp_enabled = 0,
 };
 
-static struct mixer_drv_data exynos4_mxr_drv_data = {
+static struct mixer_drv_data exynos4210_mxr_drv_data = {
         .version = MXR_VER_0_0_0_16,
         .is_vp_enabled = 1,
 };
@@ -1141,10 +1150,10 @@ static struct mixer_drv_data exynos4_mxr_drv_data = {
 static struct platform_device_id mixer_driver_types[] = {
         {
                 .name           = "s5p-mixer",
-                .driver_data    = (unsigned long)&exynos4_mxr_drv_data,
+                .driver_data    = (unsigned long)&exynos4210_mxr_drv_data,
         }, {
                 .name           = "exynos5-mixer",
-                .driver_data    = (unsigned long)&exynos5_mxr_drv_data,
+                .driver_data    = (unsigned long)&exynos5250_mxr_drv_data,
         }, {
                 /* end node */
         }
@@ -1153,7 +1162,13 @@ static struct platform_device_id mixer_driver_types[] = {
 static struct of_device_id mixer_match_types[] = {
         {
                 .compatible = "samsung,exynos5-mixer",
-                .data   = &exynos5_mxr_drv_data,
+                .data   = &exynos5250_mxr_drv_data,
+        }, {
+                .compatible = "samsung,exynos5250-mixer",
+                .data   = &exynos5250_mxr_drv_data,
+        }, {
+                .compatible = "samsung,exynos5420-mixer",
+                .data   = &exynos5420_mxr_drv_data,
         }, {
                 /* end node */
         }
@@ -1186,8 +1201,7 @@ static int mixer_probe(struct platform_device *pdev)
 
         if (dev->of_node) {
                 const struct of_device_id *match;
-                match = of_match_node(of_match_ptr(mixer_match_types),
-                                                          dev->of_node);
+                match = of_match_node(mixer_match_types, dev->of_node);
                 drv = (struct mixer_drv_data *)match->data;
         } else {
                 drv = (struct mixer_drv_data *)
@@ -1251,10 +1265,8 @@ static int mixer_suspend(struct device *dev)
         struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
         struct mixer_context *ctx = drm_hdmi_ctx->ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("%s : Already suspended\n", __func__);
+                DRM_DEBUG_KMS("Already suspended\n");
                 return 0;
         }
 
@@ -1268,10 +1280,8 @@ static int mixer_resume(struct device *dev)
         struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
         struct mixer_context *ctx = drm_hdmi_ctx->ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         if (!pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("%s : Already resumed\n", __func__);
+                DRM_DEBUG_KMS("Already resumed\n");
                 return 0;
         }
 
@@ -1287,8 +1297,6 @@ static int mixer_runtime_suspend(struct device *dev)
         struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
         struct mixer_context *ctx = drm_hdmi_ctx->ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mixer_poweroff(ctx);
 
         return 0;
@@ -1299,8 +1307,6 @@ static int mixer_runtime_resume(struct device *dev)
         struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
         struct mixer_context *ctx = drm_hdmi_ctx->ctx;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
         mixer_poweron(ctx);
 
         return 0;
diff --git a/drivers/gpu/drm/exynos/regs-mixer.h b/drivers/gpu/drm/exynos/regs-mixer.h
index 5d8dbc0301e6..4537026bc385 100644
--- a/drivers/gpu/drm/exynos/regs-mixer.h
+++ b/drivers/gpu/drm/exynos/regs-mixer.h
@@ -44,6 +44,9 @@
 #define MXR_CM_COEFF_Y                  0x0080
 #define MXR_CM_COEFF_CB                 0x0084
 #define MXR_CM_COEFF_CR                 0x0088
+#define MXR_MO                          0x0304
+#define MXR_RESOLUTION                  0x0310
+
 #define MXR_GRAPHIC0_BASE_S             0x2024
 #define MXR_GRAPHIC1_BASE_S             0x2044
 
@@ -119,6 +122,10 @@
 #define MXR_GRP_WH_WIDTH(x)             MXR_MASK_VAL(x, 26, 16)
 #define MXR_GRP_WH_HEIGHT(x)            MXR_MASK_VAL(x, 10, 0)
 
+/* bits for MXR_RESOLUTION */
+#define MXR_MXR_RES_HEIGHT(x)           MXR_MASK_VAL(x, 26, 16)
+#define MXR_MXR_RES_WIDTH(x)            MXR_MASK_VAL(x, 10, 0)
+
 /* bits for MXR_GRAPHICn_SXY */
 #define MXR_GRP_SXY_SX(x)               MXR_MASK_VAL(x, 26, 16)
 #define MXR_GRP_SXY_SY(x)               MXR_MASK_VAL(x, 10, 0)
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 91f3ac6cef35..40034ecefd3b 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -36,6 +36,7 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o \
           intel_overlay.o \
           intel_sprite.o \
           intel_opregion.o \
+          intel_sideband.o \
           dvo_ch7xxx.o \
           dvo_ch7017.o \
           dvo_ivch.o \
diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c
index 3edd981e0770..757e0fa11043 100644
--- a/drivers/gpu/drm/i915/dvo_ch7xxx.c
+++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c
@@ -32,12 +32,14 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #define CH7xxx_REG_DID          0x4b
 
 #define CH7011_VID              0x83 /* 7010 as well */
+#define CH7010B_VID             0x05
 #define CH7009A_VID             0x84
 #define CH7009B_VID             0x85
 #define CH7301_VID              0x95
 
 #define CH7xxx_VID              0x84
 #define CH7xxx_DID              0x17
+#define CH7010_DID              0x16
 
 #define CH7xxx_NUM_REGS         0x4c
 
@@ -87,11 +89,20 @@ static struct ch7xxx_id_struct {
         char *name;
 } ch7xxx_ids[] = {
         { CH7011_VID, "CH7011" },
+        { CH7010B_VID, "CH7010B" },
         { CH7009A_VID, "CH7009A" },
         { CH7009B_VID, "CH7009B" },
         { CH7301_VID, "CH7301" },
 };
 
+static struct ch7xxx_did_struct {
+        uint8_t did;
+        char *name;
+} ch7xxx_dids[] = {
+        { CH7xxx_DID, "CH7XXX" },
+        { CH7010_DID, "CH7010B" },
+};
+
 struct ch7xxx_priv {
         bool quiet;
 };
@@ -108,6 +119,18 @@ static char *ch7xxx_get_id(uint8_t vid)
         return NULL;
 }
 
+static char *ch7xxx_get_did(uint8_t did)
+{
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(ch7xxx_dids); i++) {
+                if (ch7xxx_dids[i].did == did)
+                        return ch7xxx_dids[i].name;
+        }
+
+        return NULL;
+}
+
 /** Reads an 8 bit register */
 static bool ch7xxx_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch)
 {
@@ -179,7 +202,7 @@ static bool ch7xxx_init(struct intel_dvo_device *dvo,
         /* this will detect the CH7xxx chip on the specified i2c bus */
         struct ch7xxx_priv *ch7xxx;
         uint8_t vendor, device;
-        char *name;
+        char *name, *devid;
 
         ch7xxx = kzalloc(sizeof(struct ch7xxx_priv), GFP_KERNEL);
         if (ch7xxx == NULL)
@@ -204,7 +227,8 @@ static bool ch7xxx_init(struct intel_dvo_device *dvo,
         if (!ch7xxx_readb(dvo, CH7xxx_REG_DID, &device))
                 goto out;
 
-        if (device != CH7xxx_DID) {
+        devid = ch7xxx_get_did(device);
+        if (!devid) {
                 DRM_DEBUG_KMS("ch7xxx not detected; got 0x%02x from %s "
                                 "slave %d.\n",
                           vendor, adapter->name, dvo->slave_addr);
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index e913d325d5b8..47d6c748057e 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -61,11 +61,11 @@ static int i915_capabilities(struct seq_file *m, void *data)
 
         seq_printf(m, "gen: %d\n", info->gen);
         seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev));
-#define DEV_INFO_FLAG(x) seq_printf(m, #x ": %s\n", yesno(info->x))
-#define DEV_INFO_SEP ;
-        DEV_INFO_FLAGS;
-#undef DEV_INFO_FLAG
-#undef DEV_INFO_SEP
+#define PRINT_FLAG(x)  seq_printf(m, #x ": %s\n", yesno(info->x))
+#define SEP_SEMICOLON ;
+        DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
+#undef PRINT_FLAG
+#undef SEP_SEMICOLON
 
         return 0;
 }
@@ -196,6 +196,32 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
         } \
 } while (0)
 
+struct file_stats {
+        int count;
+        size_t total, active, inactive, unbound;
+};
+
+static int per_file_stats(int id, void *ptr, void *data)
+{
+        struct drm_i915_gem_object *obj = ptr;
+        struct file_stats *stats = data;
+
+        stats->count++;
+        stats->total += obj->base.size;
+
+        if (obj->gtt_space) {
+                if (!list_empty(&obj->ring_list))
+                        stats->active += obj->base.size;
+                else
+                        stats->inactive += obj->base.size;
+        } else {
+                if (!list_empty(&obj->global_list))
+                        stats->unbound += obj->base.size;
+        }
+
+        return 0;
+}
+
 static int i915_gem_object_info(struct seq_file *m, void* data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -204,6 +230,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
         u32 count, mappable_count, purgeable_count;
         size_t size, mappable_size, purgeable_size;
         struct drm_i915_gem_object *obj;
+        struct drm_file *file;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -215,7 +242,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
                    dev_priv->mm.object_memory);
 
         size = count = mappable_size = mappable_count = 0;
-        count_objects(&dev_priv->mm.bound_list, gtt_list);
+        count_objects(&dev_priv->mm.bound_list, global_list);
         seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
                    count, mappable_count, size, mappable_size);
 
@@ -230,7 +257,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
                    count, mappable_count, size, mappable_size);
 
         size = count = purgeable_size = purgeable_count = 0;
-        list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
                 size += obj->base.size, ++count;
                 if (obj->madv == I915_MADV_DONTNEED)
                         purgeable_size += obj->base.size, ++purgeable_count;
@@ -238,7 +265,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
         seq_printf(m, "%u unbound objects, %zu bytes\n", count, size);
 
         size = count = mappable_size = mappable_count = 0;
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 if (obj->fault_mappable) {
                         size += obj->gtt_space->size;
                         ++count;
@@ -263,6 +290,21 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
                    dev_priv->gtt.total,
                    dev_priv->gtt.mappable_end - dev_priv->gtt.start);
 
+        seq_printf(m, "\n");
+        list_for_each_entry_reverse(file, &dev->filelist, lhead) {
+                struct file_stats stats;
+
+                memset(&stats, 0, sizeof(stats));
+                idr_for_each(&file->object_idr, per_file_stats, &stats);
+                seq_printf(m, "%s: %u objects, %zu bytes (%zu active, %zu inactive, %zu unbound)\n",
+                           get_pid_task(file->pid, PIDTYPE_PID)->comm,
+                           stats.count,
+                           stats.total,
+                           stats.active,
+                           stats.inactive,
+                           stats.unbound);
+        }
+
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
@@ -283,7 +325,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void* data)
                 return ret;
 
         total_obj_size = total_gtt_size = count = 0;
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 if (list == PINNED_LIST && obj->pin_count == 0)
                         continue;
 
@@ -570,6 +612,7 @@ static const char *ring_str(int ring)
         case RCS: return "render";
         case VCS: return "bsd";
         case BCS: return "blt";
+        case VECS: return "vebox";
         default: return "";
         }
 }
@@ -604,73 +647,187 @@ static const char *purgeable_flag(int purgeable)
         return purgeable ? " purgeable" : "";
 }
 
-static void print_error_buffers(struct seq_file *m,
+static bool __i915_error_ok(struct drm_i915_error_state_buf *e)
+{
+
+        if (!e->err && WARN(e->bytes > (e->size - 1), "overflow")) {
+                e->err = -ENOSPC;
+                return false;
+        }
+
+        if (e->bytes == e->size - 1 || e->err)
+                return false;
+
+        return true;
+}
+
+static bool __i915_error_seek(struct drm_i915_error_state_buf *e,
+                              unsigned len)
+{
+        if (e->pos + len <= e->start) {
+                e->pos += len;
+                return false;
+        }
+
+        /* First vsnprintf needs to fit in its entirety for memmove */
+        if (len >= e->size) {
+                e->err = -EIO;
+                return false;
+        }
+
+        return true;
+}
+
+static void __i915_error_advance(struct drm_i915_error_state_buf *e,
+                                 unsigned len)
+{
+        /* If this is first printf in this window, adjust it so that
+         * start position matches start of the buffer
+         */
+
+        if (e->pos < e->start) {
+                const size_t off = e->start - e->pos;
+
+                /* Should not happen but be paranoid */
+                if (off > len || e->bytes) {
+                        e->err = -EIO;
+                        return;
+                }
+
+                memmove(e->buf, e->buf + off, len - off);
+                e->bytes = len - off;
+                e->pos = e->start;
+                return;
+        }
+
+        e->bytes += len;
+        e->pos += len;
+}
+
+static void i915_error_vprintf(struct drm_i915_error_state_buf *e,
+                               const char *f, va_list args)
+{
+        unsigned len;
+
+        if (!__i915_error_ok(e))
+                return;
+
+        /* Seek the first printf which is hits start position */
+        if (e->pos < e->start) {
+                len = vsnprintf(NULL, 0, f, args);
+                if (!__i915_error_seek(e, len))
+                        return;
+        }
+
+        len = vsnprintf(e->buf + e->bytes, e->size - e->bytes, f, args);
+        if (len >= e->size - e->bytes)
+                len = e->size - e->bytes - 1;
+
+        __i915_error_advance(e, len);
+}
+
+static void i915_error_puts(struct drm_i915_error_state_buf *e,
+                            const char *str)
+{
+        unsigned len;
+
+        if (!__i915_error_ok(e))
+                return;
+
+        len = strlen(str);
+
+        /* Seek the first printf which is hits start position */
+        if (e->pos < e->start) {
+                if (!__i915_error_seek(e, len))
+                        return;
+        }
+
+        if (len >= e->size - e->bytes)
+                len = e->size - e->bytes - 1;
+        memcpy(e->buf + e->bytes, str, len);
+
+        __i915_error_advance(e, len);
+}
+
+void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
+{
+        va_list args;
+
+        va_start(args, f);
+        i915_error_vprintf(e, f, args);
+        va_end(args);
+}
+
+#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
+#define err_puts(e, s) i915_error_puts(e, s)
+
+static void print_error_buffers(struct drm_i915_error_state_buf *m,
                                 const char *name,
                                 struct drm_i915_error_buffer *err,
                                 int count)
 {
-        seq_printf(m, "%s [%d]:\n", name, count);
+        err_printf(m, "%s [%d]:\n", name, count);
 
         while (count--) {
-                seq_printf(m, "  %08x %8u %02x %02x %x %x%s%s%s%s%s%s%s",
+                err_printf(m, "  %08x %8u %02x %02x %x %x",
                            err->gtt_offset,
                            err->size,
                            err->read_domains,
                            err->write_domain,
-                           err->rseqno, err->wseqno,
-                           pin_flag(err->pinned),
-                           tiling_flag(err->tiling),
-                           dirty_flag(err->dirty),
-                           purgeable_flag(err->purgeable),
-                           err->ring != -1 ? " " : "",
-                           ring_str(err->ring),
-                           cache_level_str(err->cache_level));
+                           err->rseqno, err->wseqno);
+                err_puts(m, pin_flag(err->pinned));
+                err_puts(m, tiling_flag(err->tiling));
+                err_puts(m, dirty_flag(err->dirty));
+                err_puts(m, purgeable_flag(err->purgeable));
+                err_puts(m, err->ring != -1 ? " " : "");
+                err_puts(m, ring_str(err->ring));
+                err_puts(m, cache_level_str(err->cache_level));
 
                 if (err->name)
-                        seq_printf(m, " (name: %d)", err->name);
+                        err_printf(m, " (name: %d)", err->name);
                 if (err->fence_reg != I915_FENCE_REG_NONE)
-                        seq_printf(m, " (fence: %d)", err->fence_reg);
+                        err_printf(m, " (fence: %d)", err->fence_reg);
 
-                seq_printf(m, "\n");
+                err_puts(m, "\n");
                 err++;
         }
 }
 
-static void i915_ring_error_state(struct seq_file *m,
+static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
                                   struct drm_device *dev,
                                   struct drm_i915_error_state *error,
                                   unsigned ring)
 {
         BUG_ON(ring >= I915_NUM_RINGS); /* shut up confused gcc */
-        seq_printf(m, "%s command stream:\n", ring_str(ring));
-        seq_printf(m, "  HEAD: 0x%08x\n", error->head[ring]);
-        seq_printf(m, "  TAIL: 0x%08x\n", error->tail[ring]);
-        seq_printf(m, "  CTL: 0x%08x\n", error->ctl[ring]);
-        seq_printf(m, "  ACTHD: 0x%08x\n", error->acthd[ring]);
-        seq_printf(m, "  IPEIR: 0x%08x\n", error->ipeir[ring]);
-        seq_printf(m, "  IPEHR: 0x%08x\n", error->ipehr[ring]);
-        seq_printf(m, "  INSTDONE: 0x%08x\n", error->instdone[ring]);
+        err_printf(m, "%s command stream:\n", ring_str(ring));
+        err_printf(m, "  HEAD: 0x%08x\n", error->head[ring]);
+        err_printf(m, "  TAIL: 0x%08x\n", error->tail[ring]);
+        err_printf(m, "  CTL: 0x%08x\n", error->ctl[ring]);
+        err_printf(m, "  ACTHD: 0x%08x\n", error->acthd[ring]);
+        err_printf(m, "  IPEIR: 0x%08x\n", error->ipeir[ring]);
+        err_printf(m, "  IPEHR: 0x%08x\n", error->ipehr[ring]);
+        err_printf(m, "  INSTDONE: 0x%08x\n", error->instdone[ring]);
         if (ring == RCS && INTEL_INFO(dev)->gen >= 4)
-                seq_printf(m, "  BBADDR: 0x%08llx\n", error->bbaddr);
+                err_printf(m, "  BBADDR: 0x%08llx\n", error->bbaddr);
 
         if (INTEL_INFO(dev)->gen >= 4)
-                seq_printf(m, "  INSTPS: 0x%08x\n", error->instps[ring]);
-        seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm[ring]);
-        seq_printf(m, "  FADDR: 0x%08x\n", error->faddr[ring]);
+                err_printf(m, "  INSTPS: 0x%08x\n", error->instps[ring]);
+        err_printf(m, "  INSTPM: 0x%08x\n", error->instpm[ring]);
+        err_printf(m, "  FADDR: 0x%08x\n", error->faddr[ring]);
         if (INTEL_INFO(dev)->gen >= 6) {
-                seq_printf(m, "  RC PSMI: 0x%08x\n", error->rc_psmi[ring]);
-                seq_printf(m, "  FAULT_REG: 0x%08x\n", error->fault_reg[ring]);
-                seq_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
+                err_printf(m, "  RC PSMI: 0x%08x\n", error->rc_psmi[ring]);
+                err_printf(m, "  FAULT_REG: 0x%08x\n", error->fault_reg[ring]);
+                err_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
                            error->semaphore_mboxes[ring][0],
                            error->semaphore_seqno[ring][0]);
-                seq_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
+                err_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
                            error->semaphore_mboxes[ring][1],
                            error->semaphore_seqno[ring][1]);
         }
-        seq_printf(m, "  seqno: 0x%08x\n", error->seqno[ring]);
-        seq_printf(m, "  waiting: %s\n", yesno(error->waiting[ring]));
-        seq_printf(m, "  ring->head: 0x%08x\n", error->cpu_ring_head[ring]);
-        seq_printf(m, "  ring->tail: 0x%08x\n", error->cpu_ring_tail[ring]);
+        err_printf(m, "  seqno: 0x%08x\n", error->seqno[ring]);
+        err_printf(m, "  waiting: %s\n", yesno(error->waiting[ring]));
+        err_printf(m, "  ring->head: 0x%08x\n", error->cpu_ring_head[ring]);
+        err_printf(m, "  ring->tail: 0x%08x\n", error->cpu_ring_tail[ring]);
 }
 
 struct i915_error_state_file_priv {
@@ -678,9 +835,11 @@ struct i915_error_state_file_priv {
         struct drm_i915_error_state *error;
 };
 
-static int i915_error_state(struct seq_file *m, void *unused)
+
+static int i915_error_state(struct i915_error_state_file_priv *error_priv,
+                            struct drm_i915_error_state_buf *m)
+
 {
-        struct i915_error_state_file_priv *error_priv = m->private;
         struct drm_device *dev = error_priv->dev;
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_error_state *error = error_priv->error;
@@ -688,34 +847,35 @@ static int i915_error_state(struct seq_file *m, void *unused)
         int i, j, page, offset, elt;
 
         if (!error) {
-                seq_printf(m, "no error state collected\n");
+                err_printf(m, "no error state collected\n");
                 return 0;
         }
 
-        seq_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
+        err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
                    error->time.tv_usec);
-        seq_printf(m, "Kernel: " UTS_RELEASE "\n");
-        seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
-        seq_printf(m, "EIR: 0x%08x\n", error->eir);
-        seq_printf(m, "IER: 0x%08x\n", error->ier);
-        seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
-        seq_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
-        seq_printf(m, "DERRMR: 0x%08x\n", error->derrmr);
-        seq_printf(m, "CCID: 0x%08x\n", error->ccid);
+        err_printf(m, "Kernel: " UTS_RELEASE "\n");
+        err_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
+        err_printf(m, "EIR: 0x%08x\n", error->eir);
+        err_printf(m, "IER: 0x%08x\n", error->ier);
+        err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
+        err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
+        err_printf(m, "DERRMR: 0x%08x\n", error->derrmr);
+        err_printf(m, "CCID: 0x%08x\n", error->ccid);
 
         for (i = 0; i < dev_priv->num_fence_regs; i++)
-                seq_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
+                err_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
 
         for (i = 0; i < ARRAY_SIZE(error->extra_instdone); i++)
-                seq_printf(m, "  INSTDONE_%d: 0x%08x\n", i, error->extra_instdone[i]);
+                err_printf(m, "  INSTDONE_%d: 0x%08x\n", i,
+                           error->extra_instdone[i]);
 
         if (INTEL_INFO(dev)->gen >= 6) {
-                seq_printf(m, "ERROR: 0x%08x\n", error->error);
-                seq_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
+                err_printf(m, "ERROR: 0x%08x\n", error->error);
+                err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
         }
 
         if (INTEL_INFO(dev)->gen == 7)
-                seq_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
+                err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
 
         for_each_ring(ring, dev_priv, i)
                 i915_ring_error_state(m, dev, error, i);
@@ -734,24 +894,25 @@ static int i915_error_state(struct seq_file *m, void *unused)
                 struct drm_i915_error_object *obj;
 
                 if ((obj = error->ring[i].batchbuffer)) {
-                        seq_printf(m, "%s --- gtt_offset = 0x%08x\n",
+                        err_printf(m, "%s --- gtt_offset = 0x%08x\n",
                                    dev_priv->ring[i].name,
                                    obj->gtt_offset);
                         offset = 0;
                         for (page = 0; page < obj->page_count; page++) {
                                 for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-                                        seq_printf(m, "%08x :  %08x\n", offset, obj->pages[page][elt]);
+                                        err_printf(m, "%08x :  %08x\n", offset,
+                                                   obj->pages[page][elt]);
                                         offset += 4;
                                 }
                         }
                 }
 
                 if (error->ring[i].num_requests) {
-                        seq_printf(m, "%s --- %d requests\n",
+                        err_printf(m, "%s --- %d requests\n",
                                    dev_priv->ring[i].name,
                                    error->ring[i].num_requests);
                         for (j = 0; j < error->ring[i].num_requests; j++) {
-                                seq_printf(m, "  seqno 0x%08x, emitted %ld, tail 0x%08x\n",
+                                err_printf(m, "  seqno 0x%08x, emitted %ld, tail 0x%08x\n",
                                            error->ring[i].requests[j].seqno,
                                            error->ring[i].requests[j].jiffies,
                                            error->ring[i].requests[j].tail);
@@ -759,13 +920,13 @@ static int i915_error_state(struct seq_file *m, void *unused)
                 }
 
                 if ((obj = error->ring[i].ringbuffer)) {
-                        seq_printf(m, "%s --- ringbuffer = 0x%08x\n",
+                        err_printf(m, "%s --- ringbuffer = 0x%08x\n",
                                    dev_priv->ring[i].name,
                                    obj->gtt_offset);
                         offset = 0;
                         for (page = 0; page < obj->page_count; page++) {
                                 for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-                                        seq_printf(m, "%08x :  %08x\n",
+                                        err_printf(m, "%08x :  %08x\n",
                                                    offset,
                                                    obj->pages[page][elt]);
                                         offset += 4;
@@ -775,12 +936,12 @@ static int i915_error_state(struct seq_file *m, void *unused)
 
                 obj = error->ring[i].ctx;
                 if (obj) {
-                        seq_printf(m, "%s --- HW Context = 0x%08x\n",
+                        err_printf(m, "%s --- HW Context = 0x%08x\n",
                                    dev_priv->ring[i].name,
                                    obj->gtt_offset);
                         offset = 0;
                         for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-                                seq_printf(m, "[%04x] %08x %08x %08x %08x\n",
+                                err_printf(m, "[%04x] %08x %08x %08x %08x\n",
                                            offset,
                                            obj->pages[0][elt],
                                            obj->pages[0][elt+1],
@@ -806,8 +967,7 @@ i915_error_state_write(struct file *filp,
                        size_t cnt,
                        loff_t *ppos)
 {
-        struct seq_file *m = filp->private_data;
-        struct i915_error_state_file_priv *error_priv = m->private;
+        struct i915_error_state_file_priv *error_priv = filp->private_data;
         struct drm_device *dev = error_priv->dev;
         int ret;
 
@@ -842,25 +1002,81 @@ static int i915_error_state_open(struct inode *inode, struct file *file)
                 kref_get(&error_priv->error->ref);
         spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
 
-        return single_open(file, i915_error_state, error_priv);
+        file->private_data = error_priv;
+
+        return 0;
 }
 
 static int i915_error_state_release(struct inode *inode, struct file *file)
 {
-        struct seq_file *m = file->private_data;
-        struct i915_error_state_file_priv *error_priv = m->private;
+        struct i915_error_state_file_priv *error_priv = file->private_data;
 
         if (error_priv->error)
                 kref_put(&error_priv->error->ref, i915_error_state_free);
         kfree(error_priv);
 
-        return single_release(inode, file);
+        return 0;
+}
+
+static ssize_t i915_error_state_read(struct file *file, char __user *userbuf,
+                                     size_t count, loff_t *pos)
+{
+        struct i915_error_state_file_priv *error_priv = file->private_data;
+        struct drm_i915_error_state_buf error_str;
+        loff_t tmp_pos = 0;
+        ssize_t ret_count = 0;
+        int ret = 0;
+
+        memset(&error_str, 0, sizeof(error_str));
+
+        /* We need to have enough room to store any i915_error_state printf
+         * so that we can move it to start position.
+         */
+        error_str.size = count + 1 > PAGE_SIZE ? count + 1 : PAGE_SIZE;
+        error_str.buf = kmalloc(error_str.size,
+                                GFP_TEMPORARY | __GFP_NORETRY | __GFP_NOWARN);
+
+        if (error_str.buf == NULL) {
+                error_str.size = PAGE_SIZE;
+                error_str.buf = kmalloc(error_str.size, GFP_TEMPORARY);
+        }
+
+        if (error_str.buf == NULL) {
+                error_str.size = 128;
+                error_str.buf = kmalloc(error_str.size, GFP_TEMPORARY);
+        }
+
+        if (error_str.buf == NULL)
+                return -ENOMEM;
+
+        error_str.start = *pos;
+
+        ret = i915_error_state(error_priv, &error_str);
+        if (ret)
+                goto out;
+
+        if (error_str.bytes == 0 && error_str.err) {
+                ret = error_str.err;
+                goto out;
+        }
+
+        ret_count = simple_read_from_buffer(userbuf, count, &tmp_pos,
+                                            error_str.buf,
+                                            error_str.bytes);
+
+        if (ret_count < 0)
+                ret = ret_count;
+        else
+                *pos = error_str.start + ret_count;
+out:
+        kfree(error_str.buf);
+        return ret ?: ret_count;
 }
 
 static const struct file_operations i915_error_state_fops = {
         .owner = THIS_MODULE,
         .open = i915_error_state_open,
-        .read = seq_read,
+        .read = i915_error_state_read,
         .write = i915_error_state_write,
         .llseek = default_llseek,
         .release = i915_error_state_release,
@@ -941,7 +1157,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
                            MEMSTAT_VID_SHIFT);
                 seq_printf(m, "Current P-state: %d\n",
                            (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
-        } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
+        } else if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
                 u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
                 u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
                 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
@@ -1009,6 +1225,26 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
 
                 seq_printf(m, "Max overclocked frequency: %dMHz\n",
                            dev_priv->rps.hw_max * GT_FREQUENCY_MULTIPLIER);
+        } else if (IS_VALLEYVIEW(dev)) {
+                u32 freq_sts, val;
+
+                mutex_lock(&dev_priv->rps.hw_lock);
+                freq_sts = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
+                seq_printf(m, "PUNIT_REG_GPU_FREQ_STS: 0x%08x\n", freq_sts);
+                seq_printf(m, "DDR freq: %d MHz\n", dev_priv->mem_freq);
+
+                val = vlv_punit_read(dev_priv, PUNIT_FUSE_BUS1);
+                seq_printf(m, "max GPU freq: %d MHz\n",
+                           vlv_gpu_freq(dev_priv->mem_freq, val));
+
+                val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM);
+                seq_printf(m, "min GPU freq: %d MHz\n",
+                           vlv_gpu_freq(dev_priv->mem_freq, val));
+
+                seq_printf(m, "current GPU freq: %d MHz\n",
+                           vlv_gpu_freq(dev_priv->mem_freq,
+                                        (freq_sts >> 8) & 0xff));
+                mutex_unlock(&dev_priv->rps.hw_lock);
         } else {
                 seq_printf(m, "no P-state info available\n");
         }
@@ -1290,6 +1526,25 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
         return 0;
 }
 
+static int i915_ips_status(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!HAS_IPS(dev)) {
+                seq_puts(m, "not supported\n");
+                return 0;
+        }
+
+        if (I915_READ(IPS_CTL) & IPS_ENABLE)
+                seq_puts(m, "enabled\n");
+        else
+                seq_puts(m, "disabled\n");
+
+        return 0;
+}
+
 static int i915_sr_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -1642,27 +1897,27 @@ static int i915_dpio_info(struct seq_file *m, void *data)
         seq_printf(m, "DPIO_CTL: 0x%08x\n", I915_READ(DPIO_CTL));
 
         seq_printf(m, "DPIO_DIV_A: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_DIV_A));
+                   vlv_dpio_read(dev_priv, _DPIO_DIV_A));
         seq_printf(m, "DPIO_DIV_B: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_DIV_B));
+                   vlv_dpio_read(dev_priv, _DPIO_DIV_B));
 
         seq_printf(m, "DPIO_REFSFR_A: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_REFSFR_A));
+                   vlv_dpio_read(dev_priv, _DPIO_REFSFR_A));
         seq_printf(m, "DPIO_REFSFR_B: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_REFSFR_B));
+                   vlv_dpio_read(dev_priv, _DPIO_REFSFR_B));
 
         seq_printf(m, "DPIO_CORE_CLK_A: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_CORE_CLK_A));
+                   vlv_dpio_read(dev_priv, _DPIO_CORE_CLK_A));
         seq_printf(m, "DPIO_CORE_CLK_B: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_CORE_CLK_B));
+                   vlv_dpio_read(dev_priv, _DPIO_CORE_CLK_B));
 
-        seq_printf(m, "DPIO_LFP_COEFF_A: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_LFP_COEFF_A));
-        seq_printf(m, "DPIO_LFP_COEFF_B: 0x%08x\n",
-                   intel_dpio_read(dev_priv, _DPIO_LFP_COEFF_B));
+        seq_printf(m, "DPIO_LPF_COEFF_A: 0x%08x\n",
+                   vlv_dpio_read(dev_priv, _DPIO_LPF_COEFF_A));
+        seq_printf(m, "DPIO_LPF_COEFF_B: 0x%08x\n",
+                   vlv_dpio_read(dev_priv, _DPIO_LPF_COEFF_B));
 
         seq_printf(m, "DPIO_FASTCLK_DISABLE: 0x%08x\n",
-                   intel_dpio_read(dev_priv, DPIO_FASTCLK_DISABLE));
+                   vlv_dpio_read(dev_priv, DPIO_FASTCLK_DISABLE));
 
         mutex_unlock(&dev_priv->dpio_lock);
 
@@ -1780,7 +2035,8 @@ i915_drop_caches_set(void *data, u64 val)
         }
 
         if (val & DROP_UNBOUND) {
-                list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
+                list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
+                                         global_list)
                         if (obj->pages_pin_count == 0) {
                                 ret = i915_gem_object_put_pages(obj);
                                 if (ret)
@@ -1812,7 +2068,11 @@ i915_max_freq_get(void *data, u64 *val)
         if (ret)
                 return ret;
 
-        *val = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
+        if (IS_VALLEYVIEW(dev))
+                *val = vlv_gpu_freq(dev_priv->mem_freq,
+                                    dev_priv->rps.max_delay);
+        else
+                *val = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -1837,9 +2097,16 @@ i915_max_freq_set(void *data, u64 val)
         /*
          * Turbo will still be enabled, but won't go above the set value.
          */
-        do_div(val, GT_FREQUENCY_MULTIPLIER);
-        dev_priv->rps.max_delay = val;
-        gen6_set_rps(dev, val);
+        if (IS_VALLEYVIEW(dev)) {
+                val = vlv_freq_opcode(dev_priv->mem_freq, val);
+                dev_priv->rps.max_delay = val;
+                gen6_set_rps(dev, val);
+        } else {
+                do_div(val, GT_FREQUENCY_MULTIPLIER);
+                dev_priv->rps.max_delay = val;
+                gen6_set_rps(dev, val);
+        }
+
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -1863,7 +2130,11 @@ i915_min_freq_get(void *data, u64 *val)
         if (ret)
                 return ret;
 
-        *val = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
+        if (IS_VALLEYVIEW(dev))
+                *val = vlv_gpu_freq(dev_priv->mem_freq,
+                                    dev_priv->rps.min_delay);
+        else
+                *val = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -1888,9 +2159,15 @@ i915_min_freq_set(void *data, u64 val)
         /*
          * Turbo will still be enabled, but won't go below the set value.
          */
-        do_div(val, GT_FREQUENCY_MULTIPLIER);
-        dev_priv->rps.min_delay = val;
-        gen6_set_rps(dev, val);
+        if (IS_VALLEYVIEW(dev)) {
+                val = vlv_freq_opcode(dev_priv->mem_freq, val);
+                dev_priv->rps.min_delay = val;
+                valleyview_set_rps(dev, val);
+        } else {
+                do_div(val, GT_FREQUENCY_MULTIPLIER);
+                dev_priv->rps.min_delay = val;
+                gen6_set_rps(dev, val);
+        }
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -2057,6 +2334,7 @@ static struct drm_info_list i915_debugfs_list[] = {
         {"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
         {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
         {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
+        {"i915_gem_hws_vebox", i915_hws_info, 0, (void *)VECS},
         {"i915_rstdby_delays", i915_rstdby_delays, 0},
         {"i915_cur_delayinfo", i915_cur_delayinfo, 0},
         {"i915_delayfreq_table", i915_delayfreq_table, 0},
@@ -2066,6 +2344,7 @@ static struct drm_info_list i915_debugfs_list[] = {
         {"i915_ring_freq_table", i915_ring_freq_table, 0},
         {"i915_gfxec", i915_gfxec, 0},
         {"i915_fbc_status", i915_fbc_status, 0},
+        {"i915_ips_status", i915_ips_status, 0},
         {"i915_sr_status", i915_sr_status, 0},
         {"i915_opregion", i915_opregion, 0},
         {"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 3b315ba85a3e..adb319b53ecd 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -42,7 +42,6 @@
 #include <linux/vga_switcheroo.h>
 #include <linux/slab.h>
 #include <acpi/video.h>
-#include <asm/pat.h>
 
 #define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
 
@@ -956,6 +955,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
         case I915_PARAM_HAS_BLT:
                 value = intel_ring_initialized(&dev_priv->ring[BCS]);
                 break;
+        case I915_PARAM_HAS_VEBOX:
+                value = intel_ring_initialized(&dev_priv->ring[VECS]);
+                break;
         case I915_PARAM_HAS_RELAXED_FENCING:
                 value = 1;
                 break;
@@ -999,8 +1001,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
                 value = 1;
                 break;
         default:
-                DRM_DEBUG_DRIVER("Unknown parameter %d\n",
-                                 param->param);
+                DRM_DEBUG("Unknown parameter %d\n", param->param);
                 return -EINVAL;
         }
 
@@ -1359,8 +1360,10 @@ static int i915_load_modeset_init(struct drm_device *dev)
 cleanup_gem:
         mutex_lock(&dev->struct_mutex);
         i915_gem_cleanup_ringbuffer(dev);
+        i915_gem_context_fini(dev);
         mutex_unlock(&dev->struct_mutex);
         i915_gem_cleanup_aliasing_ppgtt(dev);
+        drm_mm_takedown(&dev_priv->mm.gtt_space);
 cleanup_irq:
         drm_irq_uninstall(dev);
 cleanup_gem_stolen:
@@ -1397,29 +1400,6 @@ void i915_master_destroy(struct drm_device *dev, struct drm_master *master)
         master->driver_priv = NULL;
 }
 
-static void
-i915_mtrr_setup(struct drm_i915_private *dev_priv, unsigned long base,
-                unsigned long size)
-{
-        dev_priv->mm.gtt_mtrr = -1;
-
-#if defined(CONFIG_X86_PAT)
-        if (cpu_has_pat)
-                return;
-#endif
-
-        /* Set up a WC MTRR for non-PAT systems.  This is more common than
-         * one would think, because the kernel disables PAT on first
-         * generation Core chips because WC PAT gets overridden by a UC
-         * MTRR if present.  Even if a UC MTRR isn't present.
-         */
-        dev_priv->mm.gtt_mtrr = mtrr_add(base, size, MTRR_TYPE_WRCOMB, 1);
-        if (dev_priv->mm.gtt_mtrr < 0) {
-                DRM_INFO("MTRR allocation failed.  Graphics "
-                         "performance may suffer.\n");
-        }
-}
-
 static void i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
 {
         struct apertures_struct *ap;
@@ -1431,7 +1411,7 @@ static void i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
                 return;
 
         ap->ranges[0].base = dev_priv->gtt.mappable_base;
-        ap->ranges[0].size = dev_priv->gtt.mappable_end - dev_priv->gtt.start;
+        ap->ranges[0].size = dev_priv->gtt.mappable_end;
 
         primary =
                 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
@@ -1445,15 +1425,19 @@ static void i915_dump_device_info(struct drm_i915_private *dev_priv)
 {
         const struct intel_device_info *info = dev_priv->info;
 
-#define DEV_INFO_FLAG(name) info->name ? #name "," : ""
-#define DEV_INFO_SEP ,
+#define PRINT_S(name) "%s"
+#define SEP_EMPTY
+#define PRINT_FLAG(name) info->name ? #name "," : ""
+#define SEP_COMMA ,
         DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x flags="
-                         "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
+                         DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
                          info->gen,
                          dev_priv->dev->pdev->device,
-                         DEV_INFO_FLAGS);
-#undef DEV_INFO_FLAG
-#undef DEV_INFO_SEP
+                         DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
+#undef PRINT_S
+#undef SEP_EMPTY
+#undef PRINT_FLAG
+#undef SEP_COMMA
 }
 
 /**
@@ -1468,7 +1452,7 @@ static void intel_early_sanitize_regs(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (IS_HASWELL(dev))
+        if (HAS_FPGA_DBG_UNCLAIMED(dev))
                 I915_WRITE_NOTRACE(FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 }
 
@@ -1574,8 +1558,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
                 goto out_rmmap;
         }
 
-        i915_mtrr_setup(dev_priv, dev_priv->gtt.mappable_base,
-                        aperture_size);
+        dev_priv->mm.gtt_mtrr = arch_phys_wc_add(dev_priv->gtt.mappable_base,
+                                                 aperture_size);
 
         /* The i915 workqueue is primarily used for batched retirement of
          * requests (and thus managing bo) once the task has been completed
@@ -1629,6 +1613,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         spin_lock_init(&dev_priv->irq_lock);
         spin_lock_init(&dev_priv->gpu_error.lock);
         spin_lock_init(&dev_priv->rps.lock);
+        spin_lock_init(&dev_priv->backlight.lock);
         mutex_init(&dev_priv->dpio_lock);
 
         mutex_init(&dev_priv->rps.hw_lock);
@@ -1647,6 +1632,9 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         /* Start out suspended */
         dev_priv->mm.suspended = 1;
 
+        if (HAS_POWER_WELL(dev))
+                i915_init_power_well(dev);
+
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                 ret = i915_load_modeset_init(dev);
                 if (ret < 0) {
@@ -1679,12 +1667,7 @@ out_gem_unload:
         intel_teardown_mchbar(dev);
         destroy_workqueue(dev_priv->wq);
 out_mtrrfree:
-        if (dev_priv->mm.gtt_mtrr >= 0) {
-                mtrr_del(dev_priv->mm.gtt_mtrr,
-                         dev_priv->gtt.mappable_base,
-                         aperture_size);
-                dev_priv->mm.gtt_mtrr = -1;
-        }
+        arch_phys_wc_del(dev_priv->mm.gtt_mtrr);
         io_mapping_free(dev_priv->gtt.mappable);
         dev_priv->gtt.gtt_remove(dev);
 out_rmmap:
@@ -1703,6 +1686,9 @@ int i915_driver_unload(struct drm_device *dev)
 
         intel_gpu_ips_teardown();
 
+        if (HAS_POWER_WELL(dev))
+                i915_remove_power_well(dev);
+
         i915_teardown_sysfs(dev);
 
         if (dev_priv->mm.inactive_shrinker.shrink)
@@ -1719,12 +1705,7 @@ int i915_driver_unload(struct drm_device *dev)
         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
 
         io_mapping_free(dev_priv->gtt.mappable);
-        if (dev_priv->mm.gtt_mtrr >= 0) {
-                mtrr_del(dev_priv->mm.gtt_mtrr,
-                         dev_priv->gtt.mappable_base,
-                         dev_priv->gtt.mappable_end);
-                dev_priv->mm.gtt_mtrr = -1;
-        }
+        arch_phys_wc_del(dev_priv->mm.gtt_mtrr);
 
         acpi_video_unregister();
 
@@ -1737,10 +1718,10 @@ int i915_driver_unload(struct drm_device *dev)
                  * free the memory space allocated for the child device
                  * config parsed from VBT
                  */
-                if (dev_priv->child_dev && dev_priv->child_dev_num) {
-                        kfree(dev_priv->child_dev);
-                        dev_priv->child_dev = NULL;
-                        dev_priv->child_dev_num = 0;
+                if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
+                        kfree(dev_priv->vbt.child_dev);
+                        dev_priv->vbt.child_dev = NULL;
+                        dev_priv->vbt.child_dev_num = 0;
                 }
 
                 vga_switcheroo_unregister_client(dev->pdev);
@@ -1773,6 +1754,7 @@ int i915_driver_unload(struct drm_device *dev)
                         i915_free_hws(dev);
         }
 
+        drm_mm_takedown(&dev_priv->mm.gtt_space);
         if (dev_priv->regs != NULL)
                 pci_iounmap(dev->pdev, dev_priv->regs);
 
@@ -1782,6 +1764,8 @@ int i915_driver_unload(struct drm_device *dev)
         destroy_workqueue(dev_priv->wq);
         pm_qos_remove_request(&dev_priv->pm_qos);
 
+        dev_priv->gtt.gtt_remove(dev);
+
         if (dev_priv->slab)
                 kmem_cache_destroy(dev_priv->slab);
 
@@ -1796,7 +1780,7 @@ int i915_driver_open(struct drm_device *dev, struct drm_file *file)
         struct drm_i915_file_private *file_priv;
 
         DRM_DEBUG_DRIVER("\n");
-        file_priv = kmalloc(sizeof(*file_priv), GFP_KERNEL);
+        file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
         if (!file_priv)
                 return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index a2e4953b8e8d..062cbda1bf4a 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -128,6 +128,10 @@ module_param_named(disable_power_well, i915_disable_power_well, int, 0600);
 MODULE_PARM_DESC(disable_power_well,
                  "Disable the power well when possible (default: false)");
 
+int i915_enable_ips __read_mostly = 1;
+module_param_named(enable_ips, i915_enable_ips, int, 0600);
+MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)");
+
 static struct drm_driver driver;
 extern int intel_agp_enabled;
 
@@ -280,6 +284,7 @@ static const struct intel_device_info intel_ivybridge_m_info = {
         GEN7_FEATURES,
         .is_ivybridge = 1,
         .is_mobile = 1,
+        .has_fbc = 1,
 };
 
 static const struct intel_device_info intel_ivybridge_q_info = {
@@ -308,12 +313,19 @@ static const struct intel_device_info intel_valleyview_d_info = {
 static const struct intel_device_info intel_haswell_d_info = {
         GEN7_FEATURES,
         .is_haswell = 1,
+        .has_ddi = 1,
+        .has_fpga_dbg = 1,
+        .has_vebox_ring = 1,
 };
 
 static const struct intel_device_info intel_haswell_m_info = {
         GEN7_FEATURES,
         .is_haswell = 1,
         .is_mobile = 1,
+        .has_ddi = 1,
+        .has_fpga_dbg = 1,
+        .has_fbc = 1,
+        .has_vebox_ring = 1,
 };
 
 static const struct pci_device_id pciidlist[] = {               /* aka */
@@ -445,7 +457,6 @@ void intel_detect_pch(struct drm_device *dev)
          */
         if (INTEL_INFO(dev)->num_pipes == 0) {
                 dev_priv->pch_type = PCH_NOP;
-                dev_priv->num_pch_pll = 0;
                 return;
         }
 
@@ -454,9 +465,15 @@ void intel_detect_pch(struct drm_device *dev)
          * make graphics device passthrough work easy for VMM, that only
          * need to expose ISA bridge to let driver know the real hardware
          * underneath. This is a requirement from virtualization team.
+         *
+         * In some virtualized environments (e.g. XEN), there is irrelevant
+         * ISA bridge in the system. To work reliably, we should scan trhough
+         * all the ISA bridge devices and check for the first match, instead
+         * of only checking the first one.
          */
         pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
-        if (pch) {
+        while (pch) {
+                struct pci_dev *curr = pch;
                 if (pch->vendor == PCI_VENDOR_ID_INTEL) {
                         unsigned short id;
                         id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
@@ -464,37 +481,39 @@ void intel_detect_pch(struct drm_device *dev)
 
                         if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_IBX;
-                                dev_priv->num_pch_pll = 2;
                                 DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
                                 WARN_ON(!IS_GEN5(dev));
                         } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_CPT;
-                                dev_priv->num_pch_pll = 2;
                                 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
                                 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
                         } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
                                 /* PantherPoint is CPT compatible */
                                 dev_priv->pch_type = PCH_CPT;
-                                dev_priv->num_pch_pll = 2;
                                 DRM_DEBUG_KMS("Found PatherPoint PCH\n");
                                 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
                         } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_LPT;
-                                dev_priv->num_pch_pll = 0;
                                 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
                                 WARN_ON(!IS_HASWELL(dev));
                                 WARN_ON(IS_ULT(dev));
                         } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_LPT;
-                                dev_priv->num_pch_pll = 0;
                                 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
                                 WARN_ON(!IS_HASWELL(dev));
                                 WARN_ON(!IS_ULT(dev));
+                        } else {
+                                goto check_next;
                         }
-                        BUG_ON(dev_priv->num_pch_pll > I915_NUM_PLLS);
+                        pci_dev_put(pch);
+                        break;
                 }
-                pci_dev_put(pch);
+check_next:
+                pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, curr);
+                pci_dev_put(curr);
         }
+        if (!pch)
+                DRM_DEBUG_KMS("No PCH found?\n");
 }
 
 bool i915_semaphore_is_enabled(struct drm_device *dev)
@@ -549,6 +568,8 @@ static int i915_drm_freeze(struct drm_device *dev)
                  */
                 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
                         dev_priv->display.crtc_disable(crtc);
+
+                intel_modeset_suspend_hw(dev);
         }
 
         i915_save_state(dev);
@@ -556,7 +577,7 @@ static int i915_drm_freeze(struct drm_device *dev)
         intel_opregion_fini(dev);
 
         console_lock();
-        intel_fbdev_set_suspend(dev, 1);
+        intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED);
         console_unlock();
 
         return 0;
@@ -600,7 +621,7 @@ void intel_console_resume(struct work_struct *work)
         struct drm_device *dev = dev_priv->dev;
 
         console_lock();
-        intel_fbdev_set_suspend(dev, 0);
+        intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING);
         console_unlock();
 }
 
@@ -669,7 +690,7 @@ static int __i915_drm_thaw(struct drm_device *dev)
          * path of resume if possible.
          */
         if (console_trylock()) {
-                intel_fbdev_set_suspend(dev, 0);
+                intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING);
                 console_unlock();
         } else {
                 schedule_work(&dev_priv->console_resume_work);
@@ -855,37 +876,14 @@ static int gen6_do_reset(struct drm_device *dev)
 
 int intel_gpu_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret = -ENODEV;
-
         switch (INTEL_INFO(dev)->gen) {
         case 7:
-        case 6:
-                ret = gen6_do_reset(dev);
-                break;
-        case 5:
-                ret = ironlake_do_reset(dev);
-                break;
-        case 4:
-                ret = i965_do_reset(dev);
-                break;
-        case 2:
-                ret = i8xx_do_reset(dev);
-                break;
-        }
-
-        /* Also reset the gpu hangman. */
-        if (dev_priv->gpu_error.stop_rings) {
-                DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
-                dev_priv->gpu_error.stop_rings = 0;
-                if (ret == -ENODEV) {
-                        DRM_ERROR("Reset not implemented, but ignoring "
-                                  "error for simulated gpu hangs\n");
-                        ret = 0;
-                }
+        case 6: return gen6_do_reset(dev);
+        case 5: return ironlake_do_reset(dev);
+        case 4: return i965_do_reset(dev);
+        case 2: return i8xx_do_reset(dev);
+        default: return -ENODEV;
         }
-
-        return ret;
 }
 
 /**
@@ -906,6 +904,7 @@ int intel_gpu_reset(struct drm_device *dev)
 int i915_reset(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
+        bool simulated;
         int ret;
 
         if (!i915_try_reset)
@@ -915,13 +914,26 @@ int i915_reset(struct drm_device *dev)
 
         i915_gem_reset(dev);
 
-        ret = -ENODEV;
-        if (get_seconds() - dev_priv->gpu_error.last_reset < 5)
+        simulated = dev_priv->gpu_error.stop_rings != 0;
+
+        if (!simulated && get_seconds() - dev_priv->gpu_error.last_reset < 5) {
                 DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
-        else
+                ret = -ENODEV;
+        } else {
                 ret = intel_gpu_reset(dev);
 
-        dev_priv->gpu_error.last_reset = get_seconds();
+                /* Also reset the gpu hangman. */
+                if (simulated) {
+                        DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
+                        dev_priv->gpu_error.stop_rings = 0;
+                        if (ret == -ENODEV) {
+                                DRM_ERROR("Reset not implemented, but ignoring "
+                                          "error for simulated gpu hangs\n");
+                                ret = 0;
+                        }
+                } else
+                        dev_priv->gpu_error.last_reset = get_seconds();
+        }
         if (ret) {
                 DRM_ERROR("Failed to reset chip.\n");
                 mutex_unlock(&dev->struct_mutex);
@@ -984,12 +996,6 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
         struct intel_device_info *intel_info =
                 (struct intel_device_info *) ent->driver_data;
 
-        if (intel_info->is_valleyview)
-                if(!i915_preliminary_hw_support) {
-                        DRM_ERROR("Preliminary hardware support disabled\n");
-                        return -ENODEV;
-                }
-
         /* Only bind to function 0 of the device. Early generations
          * used function 1 as a placeholder for multi-head. This causes
          * us confusion instead, especially on the systems where both
@@ -1218,16 +1224,16 @@ MODULE_LICENSE("GPL and additional rights");
 static void
 ilk_dummy_write(struct drm_i915_private *dev_priv)
 {
-        /* WaIssueDummyWriteToWakeupFromRC6: Issue a dummy write to wake up the
-         * chip from rc6 before touching it for real. MI_MODE is masked, hence
-         * harmless to write 0 into. */
+        /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
+         * the chip from rc6 before touching it for real. MI_MODE is masked,
+         * hence harmless to write 0 into. */
         I915_WRITE_NOTRACE(MI_MODE, 0);
 }
 
 static void
 hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg)
 {
-        if (IS_HASWELL(dev_priv->dev) &&
+        if (HAS_FPGA_DBG_UNCLAIMED(dev_priv->dev) &&
             (I915_READ_NOTRACE(FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
                 DRM_ERROR("Unknown unclaimed register before writing to %x\n",
                           reg);
@@ -1238,7 +1244,7 @@ hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg)
 static void
 hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg)
 {
-        if (IS_HASWELL(dev_priv->dev) &&
+        if (HAS_FPGA_DBG_UNCLAIMED(dev_priv->dev) &&
             (I915_READ_NOTRACE(FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
                 DRM_ERROR("Unclaimed write to %x\n", reg);
                 I915_WRITE_NOTRACE(FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index b9d00dcf9a2d..cc1d6056ab70 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -76,6 +76,8 @@ enum plane {
 };
 #define plane_name(p) ((p) + 'A')
 
+#define sprite_name(p, s) ((p) * dev_priv->num_plane + (s) + 'A')
+
 enum port {
         PORT_A = 0,
         PORT_B,
@@ -86,6 +88,24 @@ enum port {
 };
 #define port_name(p) ((p) + 'A')
 
+enum intel_display_power_domain {
+        POWER_DOMAIN_PIPE_A,
+        POWER_DOMAIN_PIPE_B,
+        POWER_DOMAIN_PIPE_C,
+        POWER_DOMAIN_PIPE_A_PANEL_FITTER,
+        POWER_DOMAIN_PIPE_B_PANEL_FITTER,
+        POWER_DOMAIN_PIPE_C_PANEL_FITTER,
+        POWER_DOMAIN_TRANSCODER_A,
+        POWER_DOMAIN_TRANSCODER_B,
+        POWER_DOMAIN_TRANSCODER_C,
+        POWER_DOMAIN_TRANSCODER_EDP = POWER_DOMAIN_TRANSCODER_A + 0xF,
+};
+
+#define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A)
+#define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \
+                ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER)
+#define POWER_DOMAIN_TRANSCODER(tran) ((tran) + POWER_DOMAIN_TRANSCODER_A)
+
 enum hpd_pin {
         HPD_NONE = 0,
         HPD_PORT_A = HPD_NONE, /* PORT_A is internal */
@@ -112,15 +132,38 @@ enum hpd_pin {
         list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
                 if ((intel_encoder)->base.crtc == (__crtc))
 
-struct intel_pch_pll {
+struct drm_i915_private;
+
+enum intel_dpll_id {
+        DPLL_ID_PRIVATE = -1, /* non-shared dpll in use */
+        /* real shared dpll ids must be >= 0 */
+        DPLL_ID_PCH_PLL_A,
+        DPLL_ID_PCH_PLL_B,
+};
+#define I915_NUM_PLLS 2
+
+struct intel_dpll_hw_state {
+        uint32_t dpll;
+        uint32_t fp0;
+        uint32_t fp1;
+};
+
+struct intel_shared_dpll {
         int refcount; /* count of number of CRTCs sharing this PLL */
         int active; /* count of number of active CRTCs (i.e. DPMS on) */
         bool on; /* is the PLL actually active? Disabled during modeset */
-        int pll_reg;
-        int fp0_reg;
-        int fp1_reg;
+        const char *name;
+        /* should match the index in the dev_priv->shared_dplls array */
+        enum intel_dpll_id id;
+        struct intel_dpll_hw_state hw_state;
+        void (*enable)(struct drm_i915_private *dev_priv,
+                       struct intel_shared_dpll *pll);
+        void (*disable)(struct drm_i915_private *dev_priv,
+                        struct intel_shared_dpll *pll);
+        bool (*get_hw_state)(struct drm_i915_private *dev_priv,
+                             struct intel_shared_dpll *pll,
+                             struct intel_dpll_hw_state *hw_state);
 };
-#define I915_NUM_PLLS 2
 
 /* Used by dp and fdi links */
 struct intel_link_m_n {
@@ -175,7 +218,6 @@ struct opregion_header;
 struct opregion_acpi;
 struct opregion_swsci;
 struct opregion_asle;
-struct drm_i915_private;
 
 struct intel_opregion {
         struct opregion_header __iomem *header;
@@ -286,6 +328,8 @@ struct drm_i915_error_state {
 
 struct intel_crtc_config;
 struct intel_crtc;
+struct intel_limit;
+struct dpll;
 
 struct drm_i915_display_funcs {
         bool (*fbc_enabled)(struct drm_device *dev);
@@ -293,11 +337,28 @@ struct drm_i915_display_funcs {
         void (*disable_fbc)(struct drm_device *dev);
         int (*get_display_clock_speed)(struct drm_device *dev);
         int (*get_fifo_size)(struct drm_device *dev, int plane);
+        /**
+         * find_dpll() - Find the best values for the PLL
+         * @limit: limits for the PLL
+         * @crtc: current CRTC
+         * @target: target frequency in kHz
+         * @refclk: reference clock frequency in kHz
+         * @match_clock: if provided, @best_clock P divider must
+         *               match the P divider from @match_clock
+         *               used for LVDS downclocking
+         * @best_clock: best PLL values found
+         *
+         * Returns true on success, false on failure.
+         */
+        bool (*find_dpll)(const struct intel_limit *limit,
+                          struct drm_crtc *crtc,
+                          int target, int refclk,
+                          struct dpll *match_clock,
+                          struct dpll *best_clock);
         void (*update_wm)(struct drm_device *dev);
         void (*update_sprite_wm)(struct drm_device *dev, int pipe,
-                                 uint32_t sprite_width, int pixel_size);
-        void (*update_linetime_wm)(struct drm_device *dev, int pipe,
-                                 struct drm_display_mode *mode);
+                                 uint32_t sprite_width, int pixel_size,
+                                 bool enable);
         void (*modeset_global_resources)(struct drm_device *dev);
         /* Returns the active state of the crtc, and if the crtc is active,
          * fills out the pipe-config with the hw state. */
@@ -331,68 +392,56 @@ struct drm_i915_gt_funcs {
         void (*force_wake_put)(struct drm_i915_private *dev_priv);
 };
 
-#define DEV_INFO_FLAGS \
-        DEV_INFO_FLAG(is_mobile) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_i85x) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_i915g) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_i945gm) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_g33) DEV_INFO_SEP \
-        DEV_INFO_FLAG(need_gfx_hws) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_g4x) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_pineview) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_broadwater) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_crestline) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_ivybridge) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_valleyview) DEV_INFO_SEP \
-        DEV_INFO_FLAG(is_haswell) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_force_wake) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_fbc) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_pipe_cxsr) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_hotplug) DEV_INFO_SEP \
-        DEV_INFO_FLAG(cursor_needs_physical) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_overlay) DEV_INFO_SEP \
-        DEV_INFO_FLAG(overlay_needs_physical) DEV_INFO_SEP \
-        DEV_INFO_FLAG(supports_tv) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_bsd_ring) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_blt_ring) DEV_INFO_SEP \
-        DEV_INFO_FLAG(has_llc)
+#define DEV_INFO_FOR_EACH_FLAG(func, sep) \
+        func(is_mobile) sep \
+        func(is_i85x) sep \
+        func(is_i915g) sep \
+        func(is_i945gm) sep \
+        func(is_g33) sep \
+        func(need_gfx_hws) sep \
+        func(is_g4x) sep \
+        func(is_pineview) sep \
+        func(is_broadwater) sep \
+        func(is_crestline) sep \
+        func(is_ivybridge) sep \
+        func(is_valleyview) sep \
+        func(is_haswell) sep \
+        func(has_force_wake) sep \
+        func(has_fbc) sep \
+        func(has_pipe_cxsr) sep \
+        func(has_hotplug) sep \
+        func(cursor_needs_physical) sep \
+        func(has_overlay) sep \
+        func(overlay_needs_physical) sep \
+        func(supports_tv) sep \
+        func(has_bsd_ring) sep \
+        func(has_blt_ring) sep \
+        func(has_vebox_ring) sep \
+        func(has_llc) sep \
+        func(has_ddi) sep \
+        func(has_fpga_dbg)
+
+#define DEFINE_FLAG(name) u8 name:1
+#define SEP_SEMICOLON ;
 
 struct intel_device_info {
         u32 display_mmio_offset;
         u8 num_pipes:3;
         u8 gen;
-        u8 is_mobile:1;
-        u8 is_i85x:1;
-        u8 is_i915g:1;
-        u8 is_i945gm:1;
-        u8 is_g33:1;
-        u8 need_gfx_hws:1;
-        u8 is_g4x:1;
-        u8 is_pineview:1;
-        u8 is_broadwater:1;
-        u8 is_crestline:1;
-        u8 is_ivybridge:1;
-        u8 is_valleyview:1;
-        u8 has_force_wake:1;
-        u8 is_haswell:1;
-        u8 has_fbc:1;
-        u8 has_pipe_cxsr:1;
-        u8 has_hotplug:1;
-        u8 cursor_needs_physical:1;
-        u8 has_overlay:1;
-        u8 overlay_needs_physical:1;
-        u8 supports_tv:1;
-        u8 has_bsd_ring:1;
-        u8 has_blt_ring:1;
-        u8 has_llc:1;
+        DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON);
 };
 
+#undef DEFINE_FLAG
+#undef SEP_SEMICOLON
+
 enum i915_cache_level {
         I915_CACHE_NONE = 0,
         I915_CACHE_LLC,
         I915_CACHE_LLC_MLC, /* gen6+, in docs at least! */
 };
 
+typedef uint32_t gen6_gtt_pte_t;
+
 /* The Graphics Translation Table is the way in which GEN hardware translates a
  * Graphics Virtual Address into a Physical Address. In addition to the normal
  * collateral associated with any va->pa translations GEN hardware also has a
@@ -428,6 +477,9 @@ struct i915_gtt {
                                    struct sg_table *st,
                                    unsigned int pg_start,
                                    enum i915_cache_level cache_level);
+        gen6_gtt_pte_t (*pte_encode)(struct drm_device *dev,
+                                     dma_addr_t addr,
+                                     enum i915_cache_level level);
 };
 #define gtt_total_entries(gtt) ((gtt).total >> PAGE_SHIFT)
 
@@ -449,19 +501,31 @@ struct i915_hw_ppgtt {
                                struct sg_table *st,
                                unsigned int pg_start,
                                enum i915_cache_level cache_level);
+        gen6_gtt_pte_t (*pte_encode)(struct drm_device *dev,
+                                     dma_addr_t addr,
+                                     enum i915_cache_level level);
         int (*enable)(struct drm_device *dev);
         void (*cleanup)(struct i915_hw_ppgtt *ppgtt);
 };
 
+struct i915_ctx_hang_stats {
+        /* This context had batch pending when hang was declared */
+        unsigned batch_pending;
+
+        /* This context had batch active when hang was declared */
+        unsigned batch_active;
+};
 
 /* This must match up with the value previously used for execbuf2.rsvd1. */
 #define DEFAULT_CONTEXT_ID 0
 struct i915_hw_context {
+        struct kref ref;
         int id;
         bool is_initialized;
         struct drm_i915_file_private *file_priv;
         struct intel_ring_buffer *ring;
         struct drm_i915_gem_object *obj;
+        struct i915_ctx_hang_stats hang_stats;
 };
 
 enum no_fbc_reason {
@@ -658,6 +722,7 @@ struct i915_suspend_saved_registers {
 
 struct intel_gen6_power_mgmt {
         struct work_struct work;
+        struct delayed_work vlv_work;
         u32 pm_iir;
         /* lock - irqsave spinlock that protectects the work_struct and
          * pm_iir. */
@@ -668,6 +733,7 @@ struct intel_gen6_power_mgmt {
         u8 cur_delay;
         u8 min_delay;
         u8 max_delay;
+        u8 rpe_delay;
         u8 hw_max;
 
         struct delayed_work delayed_resume_work;
@@ -704,6 +770,15 @@ struct intel_ilk_power_mgmt {
         struct drm_i915_gem_object *renderctx;
 };
 
+/* Power well structure for haswell */
+struct i915_power_well {
+        struct drm_device *device;
+        spinlock_t lock;
+        /* power well enable/disable usage count */
+        int count;
+        int i915_request;
+};
+
 struct i915_dri1_state {
         unsigned allow_batchbuffer : 1;
         u32 __iomem *gfx_hws_cpu_addr;
@@ -812,14 +887,20 @@ struct i915_gem_mm {
         u32 object_count;
 };
 
+struct drm_i915_error_state_buf {
+        unsigned bytes;
+        unsigned size;
+        int err;
+        u8 *buf;
+        loff_t start;
+        loff_t pos;
+};
+
 struct i915_gpu_error {
         /* For hangcheck timer */
 #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
 #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
         struct timer_list hangcheck_timer;
-        int hangcheck_count;
-        uint32_t last_acthd[I915_NUM_RINGS];
-        uint32_t prev_instdone[I915_NUM_INSTDONE_REG];
 
         /* For reset and error_state handling. */
         spinlock_t lock;
@@ -875,6 +956,37 @@ enum modeset_restore {
         MODESET_SUSPENDED,
 };
 
+struct intel_vbt_data {
+        struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */
+        struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */
+
+        /* Feature bits */
+        unsigned int int_tv_support:1;
+        unsigned int lvds_dither:1;
+        unsigned int lvds_vbt:1;
+        unsigned int int_crt_support:1;
+        unsigned int lvds_use_ssc:1;
+        unsigned int display_clock_mode:1;
+        unsigned int fdi_rx_polarity_inverted:1;
+        int lvds_ssc_freq;
+        unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */
+
+        /* eDP */
+        int edp_rate;
+        int edp_lanes;
+        int edp_preemphasis;
+        int edp_vswing;
+        bool edp_initialized;
+        bool edp_support;
+        int edp_bpp;
+        struct edp_power_seq edp_pps;
+
+        int crt_ddc_pin;
+
+        int child_dev_num;
+        struct child_device_config *child_dev;
+};
+
 typedef struct drm_i915_private {
         struct drm_device *dev;
         struct kmem_cache *slab;
@@ -941,9 +1053,9 @@ typedef struct drm_i915_private {
                         HPD_MARK_DISABLED = 2
                 } hpd_mark;
         } hpd_stats[HPD_NUM_PINS];
+        u32 hpd_event_bits;
         struct timer_list hotplug_reenable_timer;
 
-        int num_pch_pll;
         int num_plane;
 
         unsigned long cfb_size;
@@ -953,6 +1065,7 @@ typedef struct drm_i915_private {
         struct intel_fbc_work *fbc_work;
 
         struct intel_opregion opregion;
+        struct intel_vbt_data vbt;
 
         /* overlay */
         struct intel_overlay *overlay;
@@ -962,37 +1075,15 @@ typedef struct drm_i915_private {
         struct {
                 int level;
                 bool enabled;
+                spinlock_t lock; /* bl registers and the above bl fields */
                 struct backlight_device *device;
         } backlight;
 
         /* LVDS info */
         struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */
         struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */
-
-        /* Feature bits from the VBIOS */
-        unsigned int int_tv_support:1;
-        unsigned int lvds_dither:1;
-        unsigned int lvds_vbt:1;
-        unsigned int int_crt_support:1;
-        unsigned int lvds_use_ssc:1;
-        unsigned int display_clock_mode:1;
-        unsigned int fdi_rx_polarity_inverted:1;
-        int lvds_ssc_freq;
-        unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */
-        struct {
-                int rate;
-                int lanes;
-                int preemphasis;
-                int vswing;
-
-                bool initialized;
-                bool support;
-                int bpp;
-                struct edp_power_seq pps;
-        } edp;
         bool no_aux_handshake;
 
-        int crt_ddc_pin;
         struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */
         int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
         int num_fence_regs; /* 8 on pre-965, 16 otherwise */
@@ -1020,16 +1111,13 @@ typedef struct drm_i915_private {
         /* Kernel Modesetting */
 
         struct sdvo_device_mapping sdvo_mappings[2];
-        /* indicate whether the LVDS_BORDER should be enabled or not */
-        unsigned int lvds_border_bits;
-        /* Panel fitter placement and size for Ironlake+ */
-        u32 pch_pf_pos, pch_pf_size;
 
         struct drm_crtc *plane_to_crtc_mapping[3];
         struct drm_crtc *pipe_to_crtc_mapping[3];
         wait_queue_head_t pending_flip_queue;
 
-        struct intel_pch_pll pch_plls[I915_NUM_PLLS];
+        int num_shared_dpll;
+        struct intel_shared_dpll shared_dplls[I915_NUM_PLLS];
         struct intel_ddi_plls ddi_plls;
 
         /* Reclocking support */
@@ -1038,8 +1126,6 @@ typedef struct drm_i915_private {
         /* indicates the reduced downclock for LVDS*/
         int lvds_downclock;
         u16 orig_clock;
-        int child_dev_num;
-        struct child_device_config *child_dev;
 
         bool mchbar_need_disable;
 
@@ -1052,6 +1138,9 @@ typedef struct drm_i915_private {
          * mchdev_lock in intel_pm.c */
         struct intel_ilk_power_mgmt ips;
 
+        /* Haswell power well */
+        struct i915_power_well power_well;
+
         enum no_fbc_reason no_fbc_reason;
 
         struct drm_mm_node *compressed_fb;
@@ -1059,6 +1148,8 @@ typedef struct drm_i915_private {
 
         struct i915_gpu_error gpu_error;
 
+        struct drm_i915_gem_object *vlv_pctx;
+
         /* list of fbdev register on this device */
         struct intel_fbdev *fbdev;
 
@@ -1124,7 +1215,7 @@ struct drm_i915_gem_object {
         struct drm_mm_node *gtt_space;
         /** Stolen memory for this object, instead of being backed by shmem. */
         struct drm_mm_node *stolen;
-        struct list_head gtt_list;
+        struct list_head global_list;
 
         /** This object's place on the active/inactive lists */
         struct list_head ring_list;
@@ -1271,9 +1362,18 @@ struct drm_i915_gem_request {
         /** GEM sequence number associated with this request. */
         uint32_t seqno;
 
-        /** Postion in the ringbuffer of the end of the request */
+        /** Position in the ringbuffer of the start of the request */
+        u32 head;
+
+        /** Position in the ringbuffer of the end of the request */
         u32 tail;
 
+        /** Context related to this request */
+        struct i915_hw_context *ctx;
+
+        /** Batch buffer related to this request if any */
+        struct drm_i915_gem_object *batch_obj;
+
         /** Time at which this request was emitted, in jiffies. */
         unsigned long emitted_jiffies;
 
@@ -1291,6 +1391,8 @@ struct drm_i915_file_private {
                 struct list_head request_list;
         } mm;
         struct idr context_idr;
+
+        struct i915_ctx_hang_stats hang_stats;
 };
 
 #define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
@@ -1341,6 +1443,7 @@ struct drm_i915_file_private {
 
 #define HAS_BSD(dev)            (INTEL_INFO(dev)->has_bsd_ring)
 #define HAS_BLT(dev)            (INTEL_INFO(dev)->has_blt_ring)
+#define HAS_VEBOX(dev)          (INTEL_INFO(dev)->has_vebox_ring)
 #define HAS_LLC(dev)            (INTEL_INFO(dev)->has_llc)
 #define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
 
@@ -1371,10 +1474,13 @@ struct drm_i915_file_private {
 #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
 #define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
 
+#define HAS_IPS(dev)            (IS_ULT(dev))
+
 #define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5)
 
-#define HAS_DDI(dev)            (IS_HASWELL(dev))
+#define HAS_DDI(dev)            (INTEL_INFO(dev)->has_ddi)
 #define HAS_POWER_WELL(dev)     (IS_HASWELL(dev))
+#define HAS_FPGA_DBG_UNCLAIMED(dev)     (INTEL_INFO(dev)->has_fpga_dbg)
 
 #define INTEL_PCH_DEVICE_ID_MASK                0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE            0x3b00
@@ -1435,6 +1541,7 @@ extern bool i915_enable_hangcheck __read_mostly;
 extern int i915_enable_ppgtt __read_mostly;
 extern unsigned int i915_preliminary_hw_support __read_mostly;
 extern int i915_disable_power_well __read_mostly;
+extern int i915_enable_ips __read_mostly;
 
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
@@ -1486,8 +1593,6 @@ i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 void
 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 
-void intel_enable_asle(struct drm_device *dev);
-
 #ifdef CONFIG_DEBUG_FS
 extern void i915_destroy_error_state(struct drm_device *dev);
 #else
@@ -1626,6 +1731,7 @@ i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
 {
         if (obj->fence_reg != I915_FENCE_REG_NONE) {
                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+                WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
                 dev_priv->fence_regs[obj->fence_reg].pin_count--;
         }
 }
@@ -1658,9 +1764,12 @@ void i915_gem_init_swizzling(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
 int __must_check i915_gpu_idle(struct drm_device *dev);
 int __must_check i915_gem_idle(struct drm_device *dev);
-int i915_add_request(struct intel_ring_buffer *ring,
-                     struct drm_file *file,
-                     u32 *seqno);
+int __i915_add_request(struct intel_ring_buffer *ring,
+                       struct drm_file *file,
+                       struct drm_i915_gem_object *batch_obj,
+                       u32 *seqno);
+#define i915_add_request(ring, seqno) \
+        __i915_add_request(ring, NULL, NULL, seqno)
 int __must_check i915_wait_seqno(struct intel_ring_buffer *ring,
                                  uint32_t seqno);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
@@ -1703,6 +1812,21 @@ void i915_gem_context_fini(struct drm_device *dev);
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
 int i915_switch_context(struct intel_ring_buffer *ring,
                         struct drm_file *file, int to_id);
+void i915_gem_context_free(struct kref *ctx_ref);
+static inline void i915_gem_context_reference(struct i915_hw_context *ctx)
+{
+        kref_get(&ctx->ref);
+}
+
+static inline void i915_gem_context_unreference(struct i915_hw_context *ctx)
+{
+        kref_put(&ctx->ref, i915_gem_context_free);
+}
+
+struct i915_ctx_hang_stats * __must_check
+i915_gem_context_get_hang_stats(struct intel_ring_buffer *ring,
+                                struct drm_file *file,
+                                u32 id);
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
                                   struct drm_file *file);
 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
@@ -1784,6 +1908,8 @@ void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
 /* i915_debugfs.c */
 int i915_debugfs_init(struct drm_minor *minor);
 void i915_debugfs_cleanup(struct drm_minor *minor);
+__printf(2, 3)
+void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
 
 /* i915_suspend.c */
 extern int i915_save_state(struct drm_device *dev);
@@ -1800,7 +1926,7 @@ void i915_teardown_sysfs(struct drm_device *dev_priv);
 /* intel_i2c.c */
 extern int intel_setup_gmbus(struct drm_device *dev);
 extern void intel_teardown_gmbus(struct drm_device *dev);
-extern inline bool intel_gmbus_is_port_valid(unsigned port)
+static inline bool intel_gmbus_is_port_valid(unsigned port)
 {
         return (port >= GMBUS_PORT_SSC && port <= GMBUS_PORT_DPD);
 }
@@ -1809,7 +1935,7 @@ extern struct i2c_adapter *intel_gmbus_get_adapter(
                 struct drm_i915_private *dev_priv, unsigned port);
 extern void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed);
 extern void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit);
-extern inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
+static inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
 {
         return container_of(adapter, struct intel_gmbus, adapter)->force_bit;
 }
@@ -1821,14 +1947,10 @@ extern int intel_opregion_setup(struct drm_device *dev);
 extern void intel_opregion_init(struct drm_device *dev);
 extern void intel_opregion_fini(struct drm_device *dev);
 extern void intel_opregion_asle_intr(struct drm_device *dev);
-extern void intel_opregion_gse_intr(struct drm_device *dev);
-extern void intel_opregion_enable_asle(struct drm_device *dev);
 #else
 static inline void intel_opregion_init(struct drm_device *dev) { return; }
 static inline void intel_opregion_fini(struct drm_device *dev) { return; }
 static inline void intel_opregion_asle_intr(struct drm_device *dev) { return; }
-static inline void intel_opregion_gse_intr(struct drm_device *dev) { return; }
-static inline void intel_opregion_enable_asle(struct drm_device *dev) { return; }
 #endif
 
 /* intel_acpi.c */
@@ -1842,6 +1964,7 @@ static inline void intel_unregister_dsm_handler(void) { return; }
 
 /* modesetting */
 extern void intel_modeset_init_hw(struct drm_device *dev);
+extern void intel_modeset_suspend_hw(struct drm_device *dev);
 extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
@@ -1854,6 +1977,9 @@ extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
 extern void intel_init_pch_refclk(struct drm_device *dev);
 extern void gen6_set_rps(struct drm_device *dev, u8 val);
+extern void valleyview_set_rps(struct drm_device *dev, u8 val);
+extern int valleyview_rps_max_freq(struct drm_i915_private *dev_priv);
+extern int valleyview_rps_min_freq(struct drm_i915_private *dev_priv);
 extern void intel_detect_pch(struct drm_device *dev);
 extern int intel_trans_dp_port_sel(struct drm_crtc *crtc);
 extern int intel_enable_rc6(const struct drm_device *dev);
@@ -1865,10 +1991,11 @@ int i915_reg_read_ioctl(struct drm_device *dev, void *data,
 /* overlay */
 #ifdef CONFIG_DEBUG_FS
 extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
-extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error);
+extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e,
+                                            struct intel_overlay_error_state *error);
 
 extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
-extern void intel_display_print_error_state(struct seq_file *m,
+extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
                                             struct drm_device *dev,
                                             struct intel_display_error_state *error);
 #endif
@@ -1883,8 +2010,20 @@ int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv);
 
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val);
 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val);
-int valleyview_punit_read(struct drm_i915_private *dev_priv, u8 addr, u32 *val);
-int valleyview_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val);
+
+/* intel_sideband.c */
+u32 vlv_punit_read(struct drm_i915_private *dev_priv, u8 addr);
+void vlv_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val);
+u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr);
+u32 vlv_dpio_read(struct drm_i915_private *dev_priv, int reg);
+void vlv_dpio_write(struct drm_i915_private *dev_priv, int reg, u32 val);
+u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
+                   enum intel_sbi_destination destination);
+void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
+                     enum intel_sbi_destination destination);
+
+int vlv_gpu_freq(int ddr_freq, int val);
+int vlv_freq_opcode(int ddr_freq, int val);
 
 #define __i915_read(x, y) \
         u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 970ad17c99ab..769f75262feb 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -176,7 +176,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 
         pinned = 0;
         mutex_lock(&dev->struct_mutex);
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
                 if (obj->pin_count)
                         pinned += obj->gtt_space->size;
         mutex_unlock(&dev->struct_mutex);
@@ -956,7 +956,7 @@ i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
 
         ret = 0;
         if (seqno == ring->outstanding_lazy_request)
-                ret = i915_add_request(ring, NULL, NULL);
+                ret = i915_add_request(ring, NULL);
 
         return ret;
 }
@@ -1087,6 +1087,25 @@ i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
                             interruptible, NULL);
 }
 
+static int
+i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
+                                     struct intel_ring_buffer *ring)
+{
+        i915_gem_retire_requests_ring(ring);
+
+        /* Manually manage the write flush as we may have not yet
+         * retired the buffer.
+         *
+         * Note that the last_write_seqno is always the earlier of
+         * the two (read/write) seqno, so if we haved successfully waited,
+         * we know we have passed the last write.
+         */
+        obj->last_write_seqno = 0;
+        obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
+
+        return 0;
+}
+
 /**
  * Ensures that all rendering to the object has completed and the object is
  * safe to unbind from the GTT or access from the CPU.
@@ -1107,18 +1126,7 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
         if (ret)
                 return ret;
 
-        i915_gem_retire_requests_ring(ring);
-
-        /* Manually manage the write flush as we may have not yet
-         * retired the buffer.
-         */
-        if (obj->last_write_seqno &&
-            i915_seqno_passed(seqno, obj->last_write_seqno)) {
-                obj->last_write_seqno = 0;
-                obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
-        }
-
-        return 0;
+        return i915_gem_object_wait_rendering__tail(obj, ring);
 }
 
 /* A nonblocking variant of the above wait. This is a highly dangerous routine
@@ -1154,19 +1162,10 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
         mutex_unlock(&dev->struct_mutex);
         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
         mutex_lock(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
-        i915_gem_retire_requests_ring(ring);
-
-        /* Manually manage the write flush as we may have not yet
-         * retired the buffer.
-         */
-        if (obj->last_write_seqno &&
-            i915_seqno_passed(seqno, obj->last_write_seqno)) {
-                obj->last_write_seqno = 0;
-                obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
-        }
-
-        return ret;
+        return i915_gem_object_wait_rendering__tail(obj, ring);
 }
 
 /**
@@ -1676,7 +1675,7 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
         /* ->put_pages might need to allocate memory for the bit17 swizzle
          * array, hence protect them from being reaped by removing them from gtt
          * lists early. */
-        list_del(&obj->gtt_list);
+        list_del(&obj->global_list);
 
         ops->put_pages(obj);
         obj->pages = NULL;
@@ -1696,7 +1695,7 @@ __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
 
         list_for_each_entry_safe(obj, next,
                                  &dev_priv->mm.unbound_list,
-                                 gtt_list) {
+                                 global_list) {
                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
                     i915_gem_object_put_pages(obj) == 0) {
                         count += obj->base.size >> PAGE_SHIFT;
@@ -1733,7 +1732,8 @@ i915_gem_shrink_all(struct drm_i915_private *dev_priv)
 
         i915_gem_evict_everything(dev_priv->dev);
 
-        list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
+        list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
+                                 global_list)
                 i915_gem_object_put_pages(obj);
 }
 
@@ -1801,7 +1801,14 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
                         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
                         gfp &= ~(__GFP_IO | __GFP_WAIT);
                 }
-
+#ifdef CONFIG_SWIOTLB
+                if (swiotlb_nr_tbl()) {
+                        st->nents++;
+                        sg_set_page(sg, page, PAGE_SIZE, 0);
+                        sg = sg_next(sg);
+                        continue;
+                }
+#endif
                 if (!i || page_to_pfn(page) != last_pfn + 1) {
                         if (i)
                                 sg = sg_next(sg);
@@ -1812,8 +1819,10 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
                 }
                 last_pfn = page_to_pfn(page);
         }
-
-        sg_mark_end(sg);
+#ifdef CONFIG_SWIOTLB
+        if (!swiotlb_nr_tbl())
+#endif
+                sg_mark_end(sg);
         obj->pages = st;
 
         if (i915_gem_object_needs_bit17_swizzle(obj))
@@ -1858,7 +1867,7 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
         if (ret)
                 return ret;
 
-        list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
+        list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
         return 0;
 }
 
@@ -1996,17 +2005,18 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
         return 0;
 }
 
-int
-i915_add_request(struct intel_ring_buffer *ring,
-                 struct drm_file *file,
-                 u32 *out_seqno)
+int __i915_add_request(struct intel_ring_buffer *ring,
+                       struct drm_file *file,
+                       struct drm_i915_gem_object *obj,
+                       u32 *out_seqno)
 {
         drm_i915_private_t *dev_priv = ring->dev->dev_private;
         struct drm_i915_gem_request *request;
-        u32 request_ring_position;
+        u32 request_ring_position, request_start;
         int was_empty;
         int ret;
 
+        request_start = intel_ring_get_tail(ring);
         /*
          * Emit any outstanding flushes - execbuf can fail to emit the flush
          * after having emitted the batchbuffer command. Hence we need to fix
@@ -2038,7 +2048,21 @@ i915_add_request(struct intel_ring_buffer *ring,
 
         request->seqno = intel_ring_get_seqno(ring);
         request->ring = ring;
+        request->head = request_start;
         request->tail = request_ring_position;
+        request->ctx = ring->last_context;
+        request->batch_obj = obj;
+
+        /* Whilst this request exists, batch_obj will be on the
+         * active_list, and so will hold the active reference. Only when this
+         * request is retired will the the batch_obj be moved onto the
+         * inactive_list and lose its active reference. Hence we do not need
+         * to explicitly hold another reference here.
+         */
+
+        if (request->ctx)
+                i915_gem_context_reference(request->ctx);
+
         request->emitted_jiffies = jiffies;
         was_empty = list_empty(&ring->request_list);
         list_add_tail(&request->list, &ring->request_list);
@@ -2091,9 +2115,114 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
         spin_unlock(&file_priv->mm.lock);
 }
 
+static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
+{
+        if (acthd >= obj->gtt_offset &&
+            acthd < obj->gtt_offset + obj->base.size)
+                return true;
+
+        return false;
+}
+
+static bool i915_head_inside_request(const u32 acthd_unmasked,
+                                     const u32 request_start,
+                                     const u32 request_end)
+{
+        const u32 acthd = acthd_unmasked & HEAD_ADDR;
+
+        if (request_start < request_end) {
+                if (acthd >= request_start && acthd < request_end)
+                        return true;
+        } else if (request_start > request_end) {
+                if (acthd >= request_start || acthd < request_end)
+                        return true;
+        }
+
+        return false;
+}
+
+static bool i915_request_guilty(struct drm_i915_gem_request *request,
+                                const u32 acthd, bool *inside)
+{
+        /* There is a possibility that unmasked head address
+         * pointing inside the ring, matches the batch_obj address range.
+         * However this is extremely unlikely.
+         */
+
+        if (request->batch_obj) {
+                if (i915_head_inside_object(acthd, request->batch_obj)) {
+                        *inside = true;
+                        return true;
+                }
+        }
+
+        if (i915_head_inside_request(acthd, request->head, request->tail)) {
+                *inside = false;
+                return true;
+        }
+
+        return false;
+}
+
+static void i915_set_reset_status(struct intel_ring_buffer *ring,
+                                  struct drm_i915_gem_request *request,
+                                  u32 acthd)
+{
+        struct i915_ctx_hang_stats *hs = NULL;
+        bool inside, guilty;
+
+        /* Innocent until proven guilty */
+        guilty = false;
+
+        if (ring->hangcheck.action != wait &&
+            i915_request_guilty(request, acthd, &inside)) {
+                DRM_ERROR("%s hung %s bo (0x%x ctx %d) at 0x%x\n",
+                          ring->name,
+                          inside ? "inside" : "flushing",
+                          request->batch_obj ?
+                          request->batch_obj->gtt_offset : 0,
+                          request->ctx ? request->ctx->id : 0,
+                          acthd);
+
+                guilty = true;
+        }
+
+        /* If contexts are disabled or this is the default context, use
+         * file_priv->reset_state
+         */
+        if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
+                hs = &request->ctx->hang_stats;
+        else if (request->file_priv)
+                hs = &request->file_priv->hang_stats;
+
+        if (hs) {
+                if (guilty)
+                        hs->batch_active++;
+                else
+                        hs->batch_pending++;
+        }
+}
+
+static void i915_gem_free_request(struct drm_i915_gem_request *request)
+{
+        list_del(&request->list);
+        i915_gem_request_remove_from_client(request);
+
+        if (request->ctx)
+                i915_gem_context_unreference(request->ctx);
+
+        kfree(request);
+}
+
 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
                                       struct intel_ring_buffer *ring)
 {
+        u32 completed_seqno;
+        u32 acthd;
+
+        acthd = intel_ring_get_active_head(ring);
+        completed_seqno = ring->get_seqno(ring, false);
+
         while (!list_empty(&ring->request_list)) {
                 struct drm_i915_gem_request *request;
 
@@ -2101,9 +2230,10 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
                                            struct drm_i915_gem_request,
                                            list);
 
-                list_del(&request->list);
-                i915_gem_request_remove_from_client(request);
-                kfree(request);
+                if (request->seqno > completed_seqno)
+                        i915_set_reset_status(ring, request, acthd);
+
+                i915_gem_free_request(request);
         }
 
         while (!list_empty(&ring->active_list)) {
@@ -2195,9 +2325,7 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
                  */
                 ring->last_retired_head = request->tail;
 
-                list_del(&request->list);
-                i915_gem_request_remove_from_client(request);
-                kfree(request);
+                i915_gem_free_request(request);
         }
 
         /* Move any buffers on the active list that are no longer referenced
@@ -2264,7 +2392,7 @@ i915_gem_retire_work_handler(struct work_struct *work)
         idle = true;
         for_each_ring(ring, dev_priv, i) {
                 if (ring->gpu_caches_dirty)
-                        i915_add_request(ring, NULL, NULL);
+                        i915_add_request(ring, NULL);
 
                 idle &= list_empty(&ring->request_list);
         }
@@ -2496,9 +2624,10 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
                 obj->has_aliasing_ppgtt_mapping = 0;
         }
         i915_gem_gtt_finish_object(obj);
+        i915_gem_object_unpin_pages(obj);
 
         list_del(&obj->mm_list);
-        list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
+        list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
         /* Avoid an unnecessary call to unbind on rebind. */
         obj->map_and_fenceable = true;
 
@@ -2678,18 +2807,33 @@ static inline int fence_number(struct drm_i915_private *dev_priv,
         return fence - dev_priv->fence_regs;
 }
 
+struct write_fence {
+        struct drm_device *dev;
+        struct drm_i915_gem_object *obj;
+        int fence;
+};
+
 static void i915_gem_write_fence__ipi(void *data)
 {
+        struct write_fence *args = data;
+
+        /* Required for SNB+ with LLC */
         wbinvd();
+
+        /* Required for VLV */
+        i915_gem_write_fence(args->dev, args->fence, args->obj);
 }
 
 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
                                          struct drm_i915_fence_reg *fence,
                                          bool enable)
 {
-        struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int fence_reg = fence_number(dev_priv, fence);
+        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct write_fence args = {
+                .dev = obj->base.dev,
+                .fence = fence_number(dev_priv, fence),
+                .obj = enable ? obj : NULL,
+        };
 
         /* In order to fully serialize access to the fenced region and
          * the update to the fence register we need to take extreme
@@ -2700,13 +2844,19 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
          * SNB+ we need to take a step further and emit an explicit wbinvd()
          * on each processor in order to manually flush all memory
          * transactions before updating the fence register.
+         *
+         * However, Valleyview complicates matter. There the wbinvd is
+         * insufficient and unlike SNB/IVB requires the serialising
+         * register write. (Note that that register write by itself is
+         * conversely not sufficient for SNB+.) To compromise, we do both.
          */
-        if (HAS_LLC(obj->base.dev))
-                on_each_cpu(i915_gem_write_fence__ipi, NULL, 1);
-        i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL);
+        if (INTEL_INFO(args.dev)->gen >= 6)
+                on_each_cpu(i915_gem_write_fence__ipi, &args, 1);
+        else
+                i915_gem_write_fence(args.dev, args.fence, args.obj);
 
         if (enable) {
-                obj->fence_reg = fence_reg;
+                obj->fence_reg = args.fence;
                 fence->obj = obj;
                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
         } else {
@@ -2885,7 +3035,7 @@ static void i915_gem_verify_gtt(struct drm_device *dev)
         struct drm_i915_gem_object *obj;
         int err = 0;
 
-        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
                 if (obj->gtt_space == NULL) {
                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
                         err++;
@@ -2932,6 +3082,8 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
         struct drm_mm_node *node;
         u32 size, fence_size, fence_alignment, unfenced_alignment;
         bool mappable, fenceable;
+        size_t gtt_max = map_and_fenceable ?
+                dev_priv->gtt.mappable_end : dev_priv->gtt.total;
         int ret;
 
         fence_size = i915_gem_get_gtt_size(dev,
@@ -2958,9 +3110,11 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
         /* If the object is bigger than the entire aperture, reject it early
          * before evicting everything in a vain attempt to find space.
          */
-        if (obj->base.size >
-            (map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) {
-                DRM_ERROR("Attempting to bind an object larger than the aperture\n");
+        if (obj->base.size > gtt_max) {
+                DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
+                          obj->base.size,
+                          map_and_fenceable ? "mappable" : "total",
+                          gtt_max);
                 return -E2BIG;
         }
 
@@ -2976,14 +3130,10 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                 return -ENOMEM;
         }
 
- search_free:
-        if (map_and_fenceable)
-                ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
-                                                          size, alignment, obj->cache_level,
-                                                          0, dev_priv->gtt.mappable_end);
-        else
-                ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
-                                                 size, alignment, obj->cache_level);
+search_free:
+        ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
+                                                  size, alignment,
+                                                  obj->cache_level, 0, gtt_max);
         if (ret) {
                 ret = i915_gem_evict_something(dev, size, alignment,
                                                obj->cache_level,
@@ -3009,7 +3159,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                 return ret;
         }
 
-        list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
+        list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
         obj->gtt_space = node;
@@ -3024,7 +3174,6 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 
         obj->map_and_fenceable = mappable && fenceable;
 
-        i915_gem_object_unpin_pages(obj);
         trace_i915_gem_object_bind(obj, map_and_fenceable);
         i915_gem_verify_gtt(dev);
         return 0;
@@ -3724,7 +3873,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
                           const struct drm_i915_gem_object_ops *ops)
 {
         INIT_LIST_HEAD(&obj->mm_list);
-        INIT_LIST_HEAD(&obj->gtt_list);
+        INIT_LIST_HEAD(&obj->global_list);
         INIT_LIST_HEAD(&obj->ring_list);
         INIT_LIST_HEAD(&obj->exec_list);
 
@@ -3824,7 +3973,13 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
                 dev_priv->mm.interruptible = was_interruptible;
         }
 
-        obj->pages_pin_count = 0;
+        /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
+         * before progressing. */
+        if (obj->stolen)
+                i915_gem_object_unpin_pages(obj);
+
+        if (WARN_ON(obj->pages_pin_count))
+                obj->pages_pin_count = 0;
         i915_gem_object_put_pages(obj);
         i915_gem_object_free_mmap_offset(obj);
         i915_gem_object_release_stolen(obj);
@@ -3977,12 +4132,21 @@ static int i915_gem_init_rings(struct drm_device *dev)
                         goto cleanup_bsd_ring;
         }
 
+        if (HAS_VEBOX(dev)) {
+                ret = intel_init_vebox_ring_buffer(dev);
+                if (ret)
+                        goto cleanup_blt_ring;
+        }
+
+
         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
         if (ret)
-                goto cleanup_blt_ring;
+                goto cleanup_vebox_ring;
 
         return 0;
 
+cleanup_vebox_ring:
+        intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
 cleanup_blt_ring:
         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
 cleanup_bsd_ring:
@@ -4456,10 +4620,10 @@ i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
         }
 
         cnt = 0;
-        list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
                 if (obj->pages_pin_count == 0)
                         cnt += obj->base.size >> PAGE_SHIFT;
-        list_for_each_entry(obj, &dev_priv->mm.inactive_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.inactive_list, global_list)
                 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
                         cnt += obj->base.size >> PAGE_SHIFT;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index a1e8ecb6adf6..51b7a2171cae 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -113,7 +113,7 @@ static int get_context_size(struct drm_device *dev)
         case 7:
                 reg = I915_READ(GEN7_CXT_SIZE);
                 if (IS_HASWELL(dev))
-                        ret = HSW_CXT_TOTAL_SIZE(reg) * 64;
+                        ret = HSW_CXT_TOTAL_SIZE;
                 else
                         ret = GEN7_CXT_TOTAL_SIZE(reg) * 64;
                 break;
@@ -124,10 +124,10 @@ static int get_context_size(struct drm_device *dev)
         return ret;
 }
 
-static void do_destroy(struct i915_hw_context *ctx)
+void i915_gem_context_free(struct kref *ctx_ref)
 {
-        if (ctx->file_priv)
-                idr_remove(&ctx->file_priv->context_idr, ctx->id);
+        struct i915_hw_context *ctx = container_of(ctx_ref,
+                                                   typeof(*ctx), ref);
 
         drm_gem_object_unreference(&ctx->obj->base);
         kfree(ctx);
@@ -145,6 +145,7 @@ create_hw_context(struct drm_device *dev,
         if (ctx == NULL)
                 return ERR_PTR(-ENOMEM);
 
+        kref_init(&ctx->ref);
         ctx->obj = i915_gem_alloc_object(dev, dev_priv->hw_context_size);
         if (ctx->obj == NULL) {
                 kfree(ctx);
@@ -155,7 +156,8 @@ create_hw_context(struct drm_device *dev,
         if (INTEL_INFO(dev)->gen >= 7) {
                 ret = i915_gem_object_set_cache_level(ctx->obj,
                                                       I915_CACHE_LLC_MLC);
-                if (ret)
+                /* Failure shouldn't ever happen this early */
+                if (WARN_ON(ret))
                         goto err_out;
         }
 
@@ -169,18 +171,18 @@ create_hw_context(struct drm_device *dev,
         if (file_priv == NULL)
                 return ctx;
 
-        ctx->file_priv = file_priv;
-
         ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_ID + 1, 0,
                         GFP_KERNEL);
         if (ret < 0)
                 goto err_out;
+
+        ctx->file_priv = file_priv;
         ctx->id = ret;
 
         return ctx;
 
 err_out:
-        do_destroy(ctx);
+        i915_gem_context_unreference(ctx);
         return ERR_PTR(ret);
 }
 
@@ -213,12 +215,16 @@ static int create_default_context(struct drm_i915_private *dev_priv)
          */
         dev_priv->ring[RCS].default_context = ctx;
         ret = i915_gem_object_pin(ctx->obj, CONTEXT_ALIGN, false, false);
-        if (ret)
+        if (ret) {
+                DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret);
                 goto err_destroy;
+        }
 
         ret = do_switch(ctx);
-        if (ret)
+        if (ret) {
+                DRM_DEBUG_DRIVER("Switch failed %d\n", ret);
                 goto err_unpin;
+        }
 
         DRM_DEBUG_DRIVER("Default HW context loaded\n");
         return 0;
@@ -226,7 +232,7 @@ static int create_default_context(struct drm_i915_private *dev_priv)
 err_unpin:
         i915_gem_object_unpin(ctx->obj);
 err_destroy:
-        do_destroy(ctx);
+        i915_gem_context_unreference(ctx);
         return ret;
 }
 
@@ -236,6 +242,7 @@ void i915_gem_context_init(struct drm_device *dev)
 
         if (!HAS_HW_CONTEXTS(dev)) {
                 dev_priv->hw_contexts_disabled = true;
+                DRM_DEBUG_DRIVER("Disabling HW Contexts; old hardware\n");
                 return;
         }
 
@@ -248,11 +255,13 @@ void i915_gem_context_init(struct drm_device *dev)
 
         if (dev_priv->hw_context_size > (1<<20)) {
                 dev_priv->hw_contexts_disabled = true;
+                DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size\n");
                 return;
         }
 
         if (create_default_context(dev_priv)) {
                 dev_priv->hw_contexts_disabled = true;
+                DRM_DEBUG_DRIVER("Disabling HW Contexts; create failed\n");
                 return;
         }
 
@@ -262,6 +271,7 @@ void i915_gem_context_init(struct drm_device *dev)
 void i915_gem_context_fini(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct i915_hw_context *dctx = dev_priv->ring[RCS].default_context;
 
         if (dev_priv->hw_contexts_disabled)
                 return;
@@ -271,9 +281,16 @@ void i915_gem_context_fini(struct drm_device *dev)
          * other code, leading to spurious errors. */
         intel_gpu_reset(dev);
 
-        i915_gem_object_unpin(dev_priv->ring[RCS].default_context->obj);
+        i915_gem_object_unpin(dctx->obj);
 
-        do_destroy(dev_priv->ring[RCS].default_context);
+        /* When default context is created and switched to, base object refcount
+         * will be 2 (+1 from object creation and +1 from do_switch()).
+         * i915_gem_context_fini() will be called after gpu_idle() has switched
+         * to default context. So we need to unreference the base object once
+         * to offset the do_switch part, so that i915_gem_context_unreference()
+         * can then free the base object correctly. */
+        drm_gem_object_unreference(&dctx->obj->base);
+        i915_gem_context_unreference(dctx);
 }
 
 static int context_idr_cleanup(int id, void *p, void *data)
@@ -282,11 +299,38 @@ static int context_idr_cleanup(int id, void *p, void *data)
 
         BUG_ON(id == DEFAULT_CONTEXT_ID);
 
-        do_destroy(ctx);
-
+        i915_gem_context_unreference(ctx);
         return 0;
 }
 
+struct i915_ctx_hang_stats *
+i915_gem_context_get_hang_stats(struct intel_ring_buffer *ring,
+                                struct drm_file *file,
+                                u32 id)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        struct drm_i915_file_private *file_priv = file->driver_priv;
+        struct i915_hw_context *to;
+
+        if (dev_priv->hw_contexts_disabled)
+                return ERR_PTR(-ENOENT);
+
+        if (ring->id != RCS)
+                return ERR_PTR(-EINVAL);
+
+        if (file == NULL)
+                return ERR_PTR(-EINVAL);
+
+        if (id == DEFAULT_CONTEXT_ID)
+                return &file_priv->hang_stats;
+
+        to = i915_gem_context_get(file->driver_priv, id);
+        if (to == NULL)
+                return ERR_PTR(-ENOENT);
+
+        return &to->hang_stats;
+}
+
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
@@ -325,6 +369,7 @@ mi_set_context(struct intel_ring_buffer *ring,
         if (ret)
                 return ret;
 
+        /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw */
         if (IS_GEN7(ring->dev))
                 intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE);
         else
@@ -353,13 +398,13 @@ mi_set_context(struct intel_ring_buffer *ring,
 static int do_switch(struct i915_hw_context *to)
 {
         struct intel_ring_buffer *ring = to->ring;
-        struct drm_i915_gem_object *from_obj = ring->last_context_obj;
+        struct i915_hw_context *from = ring->last_context;
         u32 hw_flags = 0;
         int ret;
 
-        BUG_ON(from_obj != NULL && from_obj->pin_count == 0);
+        BUG_ON(from != NULL && from->obj != NULL && from->obj->pin_count == 0);
 
-        if (from_obj == to->obj)
+        if (from == to)
                 return 0;
 
         ret = i915_gem_object_pin(to->obj, CONTEXT_ALIGN, false, false);
@@ -382,7 +427,7 @@ static int do_switch(struct i915_hw_context *to)
 
         if (!to->is_initialized || is_default_context(to))
                 hw_flags |= MI_RESTORE_INHIBIT;
-        else if (WARN_ON_ONCE(from_obj == to->obj)) /* not yet expected */
+        else if (WARN_ON_ONCE(from == to)) /* not yet expected */
                 hw_flags |= MI_FORCE_RESTORE;
 
         ret = mi_set_context(ring, to, hw_flags);
@@ -397,9 +442,9 @@ static int do_switch(struct i915_hw_context *to)
          * is a bit suboptimal because the retiring can occur simply after the
          * MI_SET_CONTEXT instead of when the next seqno has completed.
          */
-        if (from_obj != NULL) {
-                from_obj->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
-                i915_gem_object_move_to_active(from_obj, ring);
+        if (from != NULL) {
+                from->obj->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
+                i915_gem_object_move_to_active(from->obj, ring);
                 /* As long as MI_SET_CONTEXT is serializing, ie. it flushes the
                  * whole damn pipeline, we don't need to explicitly mark the
                  * object dirty. The only exception is that the context must be
@@ -407,15 +452,26 @@ static int do_switch(struct i915_hw_context *to)
                  * able to defer doing this until we know the object would be
                  * swapped, but there is no way to do that yet.
                  */
-                from_obj->dirty = 1;
-                BUG_ON(from_obj->ring != ring);
-                i915_gem_object_unpin(from_obj);
+                from->obj->dirty = 1;
+                BUG_ON(from->obj->ring != ring);
+
+                ret = i915_add_request(ring, NULL);
+                if (ret) {
+                        /* Too late, we've already scheduled a context switch.
+                         * Try to undo the change so that the hw state is
+                         * consistent with out tracking. In case of emergency,
+                         * scream.
+                         */
+                        WARN_ON(mi_set_context(ring, from, MI_RESTORE_INHIBIT));
+                        return ret;
+                }
 
-                drm_gem_object_unreference(&from_obj->base);
+                i915_gem_object_unpin(from->obj);
+                i915_gem_context_unreference(from);
         }
 
-        drm_gem_object_reference(&to->obj->base);
-        ring->last_context_obj = to->obj;
+        i915_gem_context_reference(to);
+        ring->last_context = to;
         to->is_initialized = true;
 
         return 0;
@@ -444,6 +500,8 @@ int i915_switch_context(struct intel_ring_buffer *ring,
         if (dev_priv->hw_contexts_disabled)
                 return 0;
 
+        WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+
         if (ring != &dev_priv->ring[RCS])
                 return 0;
 
@@ -512,8 +570,8 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
                 return -ENOENT;
         }
 
-        do_destroy(ctx);
-
+        idr_remove(&ctx->file_priv->context_idr, ctx->id);
+        i915_gem_context_unreference(ctx);
         mutex_unlock(&dev->struct_mutex);
 
         DRM_DEBUG_DRIVER("HW context %d destroyed\n", args->ctx_id);
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 117ce3813681..87a3227e5179 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -786,7 +786,7 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
                         obj->dirty = 1;
                         obj->last_write_seqno = intel_ring_get_seqno(ring);
                         if (obj->pin_count) /* check for potential scanout */
-                                intel_mark_fb_busy(obj);
+                                intel_mark_fb_busy(obj, ring);
                 }
 
                 trace_i915_gem_object_change_domain(obj, old_read, old_write);
@@ -796,13 +796,14 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
 static void
 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
                                     struct drm_file *file,
-                                    struct intel_ring_buffer *ring)
+                                    struct intel_ring_buffer *ring,
+                                    struct drm_i915_gem_object *obj)
 {
         /* Unconditionally force add_request to emit a full flush. */
         ring->gpu_caches_dirty = true;
 
         /* Add a breadcrumb for the completion of the batch buffer */
-        (void)i915_add_request(ring, file, NULL);
+        (void)__i915_add_request(ring, file, obj, NULL);
 }
 
 static int
@@ -885,6 +886,15 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                         return -EPERM;
                 }
                 break;
+        case I915_EXEC_VEBOX:
+                ring = &dev_priv->ring[VECS];
+                if (ctx_id != 0) {
+                        DRM_DEBUG("Ring %s doesn't support contexts\n",
+                                  ring->name);
+                        return -EPERM;
+                }
+                break;
+
         default:
                 DRM_DEBUG("execbuf with unknown ring: %d\n",
                           (int)(args->flags & I915_EXEC_RING_MASK));
@@ -1074,7 +1084,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
 
         i915_gem_execbuffer_move_to_active(&eb->objects, ring);
-        i915_gem_execbuffer_retire_commands(dev, file, ring);
+        i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
 
 err:
         eb_destroy(eb);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index bdb0d7717bc7..5101ab6869b4 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -28,8 +28,6 @@
 #include "i915_trace.h"
 #include "intel_drv.h"
 
-typedef uint32_t gen6_gtt_pte_t;
-
 /* PPGTT stuff */
 #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
 
@@ -44,29 +42,22 @@ typedef uint32_t gen6_gtt_pte_t;
 #define GEN6_PTE_CACHE_LLC_MLC          (3 << 1)
 #define GEN6_PTE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
 
-static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
-                                             dma_addr_t addr,
-                                             enum i915_cache_level level)
+static gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
+                                      dma_addr_t addr,
+                                      enum i915_cache_level level)
 {
         gen6_gtt_pte_t pte = GEN6_PTE_VALID;
         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
         switch (level) {
         case I915_CACHE_LLC_MLC:
-                /* Haswell doesn't set L3 this way */
-                if (IS_HASWELL(dev))
-                        pte |= GEN6_PTE_CACHE_LLC;
-                else
-                        pte |= GEN6_PTE_CACHE_LLC_MLC;
+                pte |= GEN6_PTE_CACHE_LLC_MLC;
                 break;
         case I915_CACHE_LLC:
                 pte |= GEN6_PTE_CACHE_LLC;
                 break;
         case I915_CACHE_NONE:
-                if (IS_HASWELL(dev))
-                        pte |= HSW_PTE_UNCACHED;
-                else
-                        pte |= GEN6_PTE_UNCACHED;
+                pte |= GEN6_PTE_UNCACHED;
                 break;
         default:
                 BUG();
@@ -75,16 +66,48 @@ static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
         return pte;
 }
 
-static int gen6_ppgtt_enable(struct drm_device *dev)
+#define BYT_PTE_WRITEABLE               (1 << 1)
+#define BYT_PTE_SNOOPED_BY_CPU_CACHES   (1 << 2)
+
+static gen6_gtt_pte_t byt_pte_encode(struct drm_device *dev,
+                                     dma_addr_t addr,
+                                     enum i915_cache_level level)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        uint32_t pd_offset;
-        struct intel_ring_buffer *ring;
-        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+        gen6_gtt_pte_t pte = GEN6_PTE_VALID;
+        pte |= GEN6_PTE_ADDR_ENCODE(addr);
+
+        /* Mark the page as writeable.  Other platforms don't have a
+         * setting for read-only/writable, so this matches that behavior.
+         */
+        pte |= BYT_PTE_WRITEABLE;
+
+        if (level != I915_CACHE_NONE)
+                pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
+
+        return pte;
+}
+
+static gen6_gtt_pte_t hsw_pte_encode(struct drm_device *dev,
+                                     dma_addr_t addr,
+                                     enum i915_cache_level level)
+{
+        gen6_gtt_pte_t pte = GEN6_PTE_VALID;
+        pte |= GEN6_PTE_ADDR_ENCODE(addr);
+
+        if (level != I915_CACHE_NONE)
+                pte |= GEN6_PTE_CACHE_LLC;
+
+        return pte;
+}
+
+static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_i915_private *dev_priv = ppgtt->dev->dev_private;
         gen6_gtt_pte_t __iomem *pd_addr;
         uint32_t pd_entry;
         int i;
 
+        WARN_ON(ppgtt->pd_offset & 0x3f);
         pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
         for (i = 0; i < ppgtt->num_pd_entries; i++) {
@@ -97,6 +120,19 @@ static int gen6_ppgtt_enable(struct drm_device *dev)
                 writel(pd_entry, pd_addr + i);
         }
         readl(pd_addr);
+}
+
+static int gen6_ppgtt_enable(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        uint32_t pd_offset;
+        struct intel_ring_buffer *ring;
+        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+        int i;
+
+        BUG_ON(ppgtt->pd_offset & 0x3f);
+
+        gen6_write_pdes(ppgtt);
 
         pd_offset = ppgtt->pd_offset;
         pd_offset /= 64; /* in cachelines, */
@@ -154,9 +190,9 @@ static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
         unsigned last_pte, i;
 
-        scratch_pte = gen6_pte_encode(ppgtt->dev,
-                                      ppgtt->scratch_page_dma_addr,
-                                      I915_CACHE_LLC);
+        scratch_pte = ppgtt->pte_encode(ppgtt->dev,
+                                        ppgtt->scratch_page_dma_addr,
+                                        I915_CACHE_LLC);
 
         while (num_entries) {
                 last_pte = first_pte + num_entries;
@@ -191,8 +227,8 @@ static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
                 dma_addr_t page_addr;
 
                 page_addr = sg_page_iter_dma_address(&sg_iter);
-                pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr,
-                                                    cache_level);
+                pt_vaddr[act_pte] = ppgtt->pte_encode(ppgtt->dev, page_addr,
+                                                      cache_level);
                 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
                         kunmap_atomic(pt_vaddr);
                         act_pt++;
@@ -233,8 +269,15 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
          * entries. For aliasing ppgtt support we just steal them at the end for
          * now. */
-       first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
+        first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
 
+        if (IS_HASWELL(dev)) {
+                ppgtt->pte_encode = hsw_pte_encode;
+        } else if (IS_VALLEYVIEW(dev)) {
+                ppgtt->pte_encode = byt_pte_encode;
+        } else {
+                ppgtt->pte_encode = gen6_pte_encode;
+        }
         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
         ppgtt->enable = gen6_ppgtt_enable;
         ppgtt->clear_range = gen6_ppgtt_clear_range;
@@ -396,7 +439,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
         dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
                                       dev_priv->gtt.total / PAGE_SIZE);
 
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 i915_gem_clflush_object(obj);
                 i915_gem_gtt_bind_object(obj, obj->cache_level);
         }
@@ -437,7 +480,8 @@ static void gen6_ggtt_insert_entries(struct drm_device *dev,
 
         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
                 addr = sg_page_iter_dma_address(&sg_iter);
-                iowrite32(gen6_pte_encode(dev, addr, level), &gtt_entries[i]);
+                iowrite32(dev_priv->gtt.pte_encode(dev, addr, level),
+                          &gtt_entries[i]);
                 i++;
         }
 
@@ -449,7 +493,7 @@ static void gen6_ggtt_insert_entries(struct drm_device *dev,
          */
         if (i != 0)
                 WARN_ON(readl(&gtt_entries[i-1])
-                        != gen6_pte_encode(dev, addr, level));
+                        != dev_priv->gtt.pte_encode(dev, addr, level));
 
         /* This next bit makes the above posting read even more important. We
          * want to flush the TLBs only after we're certain all the PTE updates
@@ -474,8 +518,9 @@ static void gen6_ggtt_clear_range(struct drm_device *dev,
                  first_entry, num_entries, max_entries))
                 num_entries = max_entries;
 
-        scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma,
-                                      I915_CACHE_LLC);
+        scratch_pte = dev_priv->gtt.pte_encode(dev,
+                                               dev_priv->gtt.scratch_page_dma,
+                                               I915_CACHE_LLC);
         for (i = 0; i < num_entries; i++)
                 iowrite32(scratch_pte, &gtt_base[i]);
         readl(gtt_base);
@@ -586,7 +631,7 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
                 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
 
         /* Mark any preallocated objects as occupied */
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
                               obj->gtt_offset, obj->base.size);
 
@@ -809,6 +854,13 @@ int i915_gem_gtt_init(struct drm_device *dev)
         } else {
                 dev_priv->gtt.gtt_probe = gen6_gmch_probe;
                 dev_priv->gtt.gtt_remove = gen6_gmch_remove;
+                if (IS_HASWELL(dev)) {
+                        dev_priv->gtt.pte_encode = hsw_pte_encode;
+                } else if (IS_VALLEYVIEW(dev)) {
+                        dev_priv->gtt.pte_encode = byt_pte_encode;
+                } else {
+                        dev_priv->gtt.pte_encode = gen6_pte_encode;
+                }
         }
 
         ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 130d1db27e28..982d4732cecf 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -62,7 +62,10 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
          * its value of TOLUD.
          */
         base = 0;
-        if (INTEL_INFO(dev)->gen >= 6) {
+        if (IS_VALLEYVIEW(dev)) {
+                pci_read_config_dword(dev->pdev, 0x5c, &base);
+                base &= ~((1<<20) - 1);
+        } else if (INTEL_INFO(dev)->gen >= 6) {
                 /* Read Base Data of Stolen Memory Register (BDSM) directly.
                  * Note that there is also a MCHBAR miror at 0x1080c0 or
                  * we could use device 2:0x5c instead.
@@ -136,6 +139,7 @@ static int i915_setup_compression(struct drm_device *dev, int size)
 err_fb:
         drm_mm_put_block(compressed_fb);
 err:
+        pr_info_once("drm: not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
         return -ENOSPC;
 }
 
@@ -143,7 +147,7 @@ int i915_gem_stolen_setup_compression(struct drm_device *dev, int size)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (dev_priv->mm.stolen_base == 0)
+        if (!drm_mm_initialized(&dev_priv->mm.stolen))
                 return -ENODEV;
 
         if (size < dev_priv->cfb_size)
@@ -175,6 +179,9 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        if (!drm_mm_initialized(&dev_priv->mm.stolen))
+                return;
+
         i915_gem_stolen_cleanup_compression(dev);
         drm_mm_takedown(&dev_priv->mm.stolen);
 }
@@ -182,6 +189,7 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
 int i915_gem_init_stolen(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        int bios_reserved = 0;
 
         dev_priv->mm.stolen_base = i915_stolen_to_physical(dev);
         if (dev_priv->mm.stolen_base == 0)
@@ -190,8 +198,12 @@ int i915_gem_init_stolen(struct drm_device *dev)
         DRM_DEBUG_KMS("found %zd bytes of stolen memory at %08lx\n",
                       dev_priv->gtt.stolen_size, dev_priv->mm.stolen_base);
 
+        if (IS_VALLEYVIEW(dev))
+                bios_reserved = 1024*1024; /* top 1M on VLV/BYT */
+
         /* Basic memrange allocator for stolen space */
-        drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_size);
+        drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_size -
+                    bios_reserved);
 
         return 0;
 }
@@ -270,7 +282,7 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
                 goto cleanup;
 
         obj->has_dma_mapping = true;
-        obj->pages_pin_count = 1;
+        i915_gem_object_pin_pages(obj);
         obj->stolen = stolen;
 
         obj->base.write_domain = I915_GEM_DOMAIN_GTT;
@@ -291,7 +303,7 @@ i915_gem_object_create_stolen(struct drm_device *dev, u32 size)
         struct drm_i915_gem_object *obj;
         struct drm_mm_node *stolen;
 
-        if (dev_priv->mm.stolen_base == 0)
+        if (!drm_mm_initialized(&dev_priv->mm.stolen))
                 return NULL;
 
         DRM_DEBUG_KMS("creating stolen object: size=%x\n", size);
@@ -322,7 +334,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
         struct drm_i915_gem_object *obj;
         struct drm_mm_node *stolen;
 
-        if (dev_priv->mm.stolen_base == 0)
+        if (!drm_mm_initialized(&dev_priv->mm.stolen))
                 return NULL;
 
         DRM_DEBUG_KMS("creating preallocated stolen object: stolen_offset=%x, gtt_offset=%x, size=%x\n",
@@ -330,7 +342,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 
         /* KISS and expect everything to be page-aligned */
         BUG_ON(stolen_offset & 4095);
-        BUG_ON(gtt_offset & 4095);
         BUG_ON(size & 4095);
 
         if (WARN_ON(size == 0))
@@ -351,6 +362,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
                 return NULL;
         }
 
+        /* Some objects just need physical mem from stolen space */
+        if (gtt_offset == -1)
+                return obj;
+
         /* To simplify the initialisation sequence between KMS and GTT,
          * we allow construction of the stolen object prior to
          * setting up the GTT space. The actual reservation will occur
@@ -371,7 +386,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
         obj->gtt_offset = gtt_offset;
         obj->has_global_gtt_mapping = 1;
 
-        list_add_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
+        list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
         return obj;
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 0aa2ef0d2ae0..3d92a7cef154 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -70,15 +70,6 @@ static const u32 hpd_status_gen4[] = {
         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
 };
 
-static const u32 hpd_status_i965[] = {
-         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
-         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I965,
-         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I965,
-         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
-         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
-         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
-};
-
 static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
@@ -88,13 +79,12 @@ static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
 };
 
-static void ibx_hpd_irq_setup(struct drm_device *dev);
-static void i915_hpd_irq_setup(struct drm_device *dev);
-
 /* For display hotplug interrupt */
 static void
 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
+        assert_spin_locked(&dev_priv->irq_lock);
+
         if ((dev_priv->irq_mask & mask) != 0) {
                 dev_priv->irq_mask &= ~mask;
                 I915_WRITE(DEIMR, dev_priv->irq_mask);
@@ -105,6 +95,8 @@ ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 static void
 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
+        assert_spin_locked(&dev_priv->irq_lock);
+
         if ((dev_priv->irq_mask & mask) != mask) {
                 dev_priv->irq_mask |= mask;
                 I915_WRITE(DEIMR, dev_priv->irq_mask);
@@ -112,6 +104,215 @@ ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
         }
 }
 
+static bool ivb_can_enable_err_int(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *crtc;
+        enum pipe pipe;
+
+        assert_spin_locked(&dev_priv->irq_lock);
+
+        for_each_pipe(pipe) {
+                crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+
+                if (crtc->cpu_fifo_underrun_disabled)
+                        return false;
+        }
+
+        return true;
+}
+
+static bool cpt_can_enable_serr_int(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe pipe;
+        struct intel_crtc *crtc;
+
+        for_each_pipe(pipe) {
+                crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+
+                if (crtc->pch_fifo_underrun_disabled)
+                        return false;
+        }
+
+        return true;
+}
+
+static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
+                                                 enum pipe pipe, bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
+                                          DE_PIPEB_FIFO_UNDERRUN;
+
+        if (enable)
+                ironlake_enable_display_irq(dev_priv, bit);
+        else
+                ironlake_disable_display_irq(dev_priv, bit);
+}
+
+static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
+                                                  bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (enable) {
+                if (!ivb_can_enable_err_int(dev))
+                        return;
+
+                I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN_A |
+                                         ERR_INT_FIFO_UNDERRUN_B |
+                                         ERR_INT_FIFO_UNDERRUN_C);
+
+                ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
+        } else {
+                ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
+        }
+}
+
+static void ibx_set_fifo_underrun_reporting(struct intel_crtc *crtc,
+                                            bool enable)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t bit = (crtc->pipe == PIPE_A) ? SDE_TRANSA_FIFO_UNDER :
+                                                SDE_TRANSB_FIFO_UNDER;
+
+        if (enable)
+                I915_WRITE(SDEIMR, I915_READ(SDEIMR) & ~bit);
+        else
+                I915_WRITE(SDEIMR, I915_READ(SDEIMR) | bit);
+
+        POSTING_READ(SDEIMR);
+}
+
+static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
+                                            enum transcoder pch_transcoder,
+                                            bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (enable) {
+                if (!cpt_can_enable_serr_int(dev))
+                        return;
+
+                I915_WRITE(SERR_INT, SERR_INT_TRANS_A_FIFO_UNDERRUN |
+                                     SERR_INT_TRANS_B_FIFO_UNDERRUN |
+                                     SERR_INT_TRANS_C_FIFO_UNDERRUN);
+
+                I915_WRITE(SDEIMR, I915_READ(SDEIMR) & ~SDE_ERROR_CPT);
+        } else {
+                I915_WRITE(SDEIMR, I915_READ(SDEIMR) | SDE_ERROR_CPT);
+        }
+
+        POSTING_READ(SDEIMR);
+}
+
+/**
+ * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
+ * @dev: drm device
+ * @pipe: pipe
+ * @enable: true if we want to report FIFO underrun errors, false otherwise
+ *
+ * This function makes us disable or enable CPU fifo underruns for a specific
+ * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
+ * reporting for one pipe may also disable all the other CPU error interruts for
+ * the other pipes, due to the fact that there's just one interrupt mask/enable
+ * bit for all the pipes.
+ *
+ * Returns the previous state of underrun reporting.
+ */
+bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
+                                           enum pipe pipe, bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        unsigned long flags;
+        bool ret;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+
+        ret = !intel_crtc->cpu_fifo_underrun_disabled;
+
+        if (enable == ret)
+                goto done;
+
+        intel_crtc->cpu_fifo_underrun_disabled = !enable;
+
+        if (IS_GEN5(dev) || IS_GEN6(dev))
+                ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
+        else if (IS_GEN7(dev))
+                ivybridge_set_fifo_underrun_reporting(dev, enable);
+
+done:
+        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        return ret;
+}
+
+/**
+ * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
+ * @dev: drm device
+ * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
+ * @enable: true if we want to report FIFO underrun errors, false otherwise
+ *
+ * This function makes us disable or enable PCH fifo underruns for a specific
+ * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
+ * underrun reporting for one transcoder may also disable all the other PCH
+ * error interruts for the other transcoders, due to the fact that there's just
+ * one interrupt mask/enable bit for all the transcoders.
+ *
+ * Returns the previous state of underrun reporting.
+ */
+bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
+                                           enum transcoder pch_transcoder,
+                                           bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe p;
+        struct drm_crtc *crtc;
+        struct intel_crtc *intel_crtc;
+        unsigned long flags;
+        bool ret;
+
+        if (HAS_PCH_LPT(dev)) {
+                crtc = NULL;
+                for_each_pipe(p) {
+                        struct drm_crtc *c = dev_priv->pipe_to_crtc_mapping[p];
+                        if (intel_pipe_has_type(c, INTEL_OUTPUT_ANALOG)) {
+                                crtc = c;
+                                break;
+                        }
+                }
+                if (!crtc) {
+                        DRM_ERROR("PCH FIFO underrun, but no CRTC using the PCH found\n");
+                        return false;
+                }
+        } else {
+                crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
+        }
+        intel_crtc = to_intel_crtc(crtc);
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+
+        ret = !intel_crtc->pch_fifo_underrun_disabled;
+
+        if (enable == ret)
+                goto done;
+
+        intel_crtc->pch_fifo_underrun_disabled = !enable;
+
+        if (HAS_PCH_IBX(dev))
+                ibx_set_fifo_underrun_reporting(intel_crtc, enable);
+        else
+                cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
+
+done:
+        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        return ret;
+}
+
+
 void
 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 {
@@ -142,28 +343,21 @@ i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 }
 
 /**
- * intel_enable_asle - enable ASLE interrupt for OpRegion
+ * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
  */
-void intel_enable_asle(struct drm_device *dev)
+static void i915_enable_asle_pipestat(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
-        /* FIXME: opregion/asle for VLV */
-        if (IS_VALLEYVIEW(dev))
+        if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
                 return;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 
-        if (HAS_PCH_SPLIT(dev))
-                ironlake_enable_display_irq(dev_priv, DE_GSE);
-        else {
-                i915_enable_pipestat(dev_priv, 1,
-                                     PIPE_LEGACY_BLC_EVENT_ENABLE);
-                if (INTEL_INFO(dev)->gen >= 4)
-                        i915_enable_pipestat(dev_priv, 0,
-                                             PIPE_LEGACY_BLC_EVENT_ENABLE);
-        }
+        i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
+        if (INTEL_INFO(dev)->gen >= 4)
+                i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
 
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
@@ -181,10 +375,16 @@ static int
 i915_pipe_enabled(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
-                                                                      pipe);
 
-        return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_ENABLE;
+        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+                /* Locking is horribly broken here, but whatever. */
+                struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+                return intel_crtc->active;
+        } else {
+                return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
+        }
 }
 
 /* Called from drm generic code, passed a 'crtc', which
@@ -334,6 +534,21 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
                                                      crtc);
 }
 
+static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
+{
+        enum drm_connector_status old_status;
+
+        WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
+        old_status = connector->status;
+
+        connector->status = connector->funcs->detect(connector, false);
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
+                      connector->base.id,
+                      drm_get_connector_name(connector),
+                      old_status, connector->status);
+        return (old_status != connector->status);
+}
+
 /*
  * Handle hotplug events outside the interrupt handler proper.
  */
@@ -350,6 +565,8 @@ static void i915_hotplug_work_func(struct work_struct *work)
         struct drm_connector *connector;
         unsigned long irqflags;
         bool hpd_disabled = false;
+        bool changed = false;
+        u32 hpd_event_bits;
 
         /* HPD irq before everything is fully set up. */
         if (!dev_priv->enable_hotplug_processing)
@@ -359,6 +576,9 @@ static void i915_hotplug_work_func(struct work_struct *work)
         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+
+        hpd_event_bits = dev_priv->hpd_event_bits;
+        dev_priv->hpd_event_bits = 0;
         list_for_each_entry(connector, &mode_config->connector_list, head) {
                 intel_connector = to_intel_connector(connector);
                 intel_encoder = intel_connector->encoder;
@@ -373,6 +593,10 @@ static void i915_hotplug_work_func(struct work_struct *work)
                                 | DRM_CONNECTOR_POLL_DISCONNECT;
                         hpd_disabled = true;
                 }
+                if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
+                        DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
+                                      drm_get_connector_name(connector), intel_encoder->hpd_pin);
+                }
         }
          /* if there were no outputs to poll, poll was disabled,
           * therefore make sure it's enabled when disabling HPD on
@@ -385,14 +609,20 @@ static void i915_hotplug_work_func(struct work_struct *work)
 
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
-        list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
-                if (intel_encoder->hot_plug)
-                        intel_encoder->hot_plug(intel_encoder);
-
+        list_for_each_entry(connector, &mode_config->connector_list, head) {
+                intel_connector = to_intel_connector(connector);
+                intel_encoder = intel_connector->encoder;
+                if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
+                        if (intel_encoder->hot_plug)
+                                intel_encoder->hot_plug(intel_encoder);
+                        if (intel_hpd_irq_event(dev, connector))
+                                changed = true;
+                }
+        }
         mutex_unlock(&mode_config->mutex);
 
-        /* Just fire off a uevent and let userspace tell us what to do */
-        drm_helper_hpd_irq_event(dev);
+        if (changed)
+                drm_kms_helper_hotplug_event(dev);
 }
 
 static void ironlake_handle_rps_change(struct drm_device *dev)
@@ -447,7 +677,6 @@ static void notify_ring(struct drm_device *dev,
 
         wake_up_all(&ring->irq_queue);
         if (i915_enable_hangcheck) {
-                dev_priv->gpu_error.hangcheck_count = 0;
                 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
                           round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
         }
@@ -464,25 +693,48 @@ static void gen6_pm_rps_work(struct work_struct *work)
         pm_iir = dev_priv->rps.pm_iir;
         dev_priv->rps.pm_iir = 0;
         pm_imr = I915_READ(GEN6_PMIMR);
-        I915_WRITE(GEN6_PMIMR, 0);
+        /* Make sure not to corrupt PMIMR state used by ringbuffer code */
+        I915_WRITE(GEN6_PMIMR, pm_imr & ~GEN6_PM_RPS_EVENTS);
         spin_unlock_irq(&dev_priv->rps.lock);
 
-        if ((pm_iir & GEN6_PM_DEFERRED_EVENTS) == 0)
+        if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
                 return;
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
-        if (pm_iir & GEN6_PM_RP_UP_THRESHOLD)
+        if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
                 new_delay = dev_priv->rps.cur_delay + 1;
-        else
+
+                /*
+                 * For better performance, jump directly
+                 * to RPe if we're below it.
+                 */
+                if (IS_VALLEYVIEW(dev_priv->dev) &&
+                    dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
+                        new_delay = dev_priv->rps.rpe_delay;
+        } else
                 new_delay = dev_priv->rps.cur_delay - 1;
 
         /* sysfs frequency interfaces may have snuck in while servicing the
          * interrupt
          */
-        if (!(new_delay > dev_priv->rps.max_delay ||
-              new_delay < dev_priv->rps.min_delay)) {
-                gen6_set_rps(dev_priv->dev, new_delay);
+        if (new_delay >= dev_priv->rps.min_delay &&
+            new_delay <= dev_priv->rps.max_delay) {
+                if (IS_VALLEYVIEW(dev_priv->dev))
+                        valleyview_set_rps(dev_priv->dev, new_delay);
+                else
+                        gen6_set_rps(dev_priv->dev, new_delay);
+        }
+
+        if (IS_VALLEYVIEW(dev_priv->dev)) {
+                /*
+                 * On VLV, when we enter RC6 we may not be at the minimum
+                 * voltage level, so arm a timer to check.  It should only
+                 * fire when there's activity or once after we've entered
+                 * RC6, and then won't be re-armed until the next RPS interrupt.
+                 */
+                mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
+                                 msecs_to_jiffies(100));
         }
 
         mutex_unlock(&dev_priv->rps.hw_lock);
@@ -529,7 +781,7 @@ static void ivybridge_parity_work(struct work_struct *work)
         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        dev_priv->gt_irq_mask &= ~GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
+        dev_priv->gt_irq_mask &= ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 
@@ -561,7 +813,7 @@ static void ivybridge_handle_parity_error(struct drm_device *dev)
                 return;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        dev_priv->gt_irq_mask |= GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
+        dev_priv->gt_irq_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 
@@ -573,25 +825,26 @@ static void snb_gt_irq_handler(struct drm_device *dev,
                                u32 gt_iir)
 {
 
-        if (gt_iir & (GEN6_RENDER_USER_INTERRUPT |
-                      GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT))
+        if (gt_iir &
+            (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
                 notify_ring(dev, &dev_priv->ring[RCS]);
-        if (gt_iir & GEN6_BSD_USER_INTERRUPT)
+        if (gt_iir & GT_BSD_USER_INTERRUPT)
                 notify_ring(dev, &dev_priv->ring[VCS]);
-        if (gt_iir & GEN6_BLITTER_USER_INTERRUPT)
+        if (gt_iir & GT_BLT_USER_INTERRUPT)
                 notify_ring(dev, &dev_priv->ring[BCS]);
 
-        if (gt_iir & (GT_GEN6_BLT_CS_ERROR_INTERRUPT |
-                      GT_GEN6_BSD_CS_ERROR_INTERRUPT |
-                      GT_RENDER_CS_ERROR_INTERRUPT)) {
+        if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
+                      GT_BSD_CS_ERROR_INTERRUPT |
+                      GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
                 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
                 i915_handle_error(dev, false);
         }
 
-        if (gt_iir & GT_GEN7_L3_PARITY_ERROR_INTERRUPT)
+        if (gt_iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
                 ivybridge_handle_parity_error(dev);
 }
 
+/* Legacy way of handling PM interrupts */
 static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
                                 u32 pm_iir)
 {
@@ -619,23 +872,25 @@ static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
 #define HPD_STORM_DETECT_PERIOD 1000
 #define HPD_STORM_THRESHOLD 5
 
-static inline bool hotplug_irq_storm_detect(struct drm_device *dev,
-                                            u32 hotplug_trigger,
-                                            const u32 *hpd)
+static inline void intel_hpd_irq_handler(struct drm_device *dev,
+                                         u32 hotplug_trigger,
+                                         const u32 *hpd)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        unsigned long irqflags;
         int i;
-        bool ret = false;
+        bool storm_detected = false;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        if (!hotplug_trigger)
+                return;
 
+        spin_lock(&dev_priv->irq_lock);
         for (i = 1; i < HPD_NUM_PINS; i++) {
 
                 if (!(hpd[i] & hotplug_trigger) ||
                     dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
                         continue;
 
+                dev_priv->hpd_event_bits |= (1 << i);
                 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
                                    dev_priv->hpd_stats[i].hpd_last_jiffies
                                    + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
@@ -643,16 +898,20 @@ static inline bool hotplug_irq_storm_detect(struct drm_device *dev,
                         dev_priv->hpd_stats[i].hpd_cnt = 0;
                 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
                         dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
+                        dev_priv->hpd_event_bits &= ~(1 << i);
                         DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
-                        ret = true;
+                        storm_detected = true;
                 } else {
                         dev_priv->hpd_stats[i].hpd_cnt++;
                 }
         }
 
-        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+        if (storm_detected)
+                dev_priv->display.hpd_irq_setup(dev);
+        spin_unlock(&dev_priv->irq_lock);
 
-        return ret;
+        queue_work(dev_priv->wq,
+                   &dev_priv->hotplug_work);
 }
 
 static void gmbus_irq_handler(struct drm_device *dev)
@@ -669,6 +928,38 @@ static void dp_aux_irq_handler(struct drm_device *dev)
         wake_up_all(&dev_priv->gmbus_wait_queue);
 }
 
+/* Unlike gen6_queue_rps_work() from which this function is originally derived,
+ * we must be able to deal with other PM interrupts. This is complicated because
+ * of the way in which we use the masks to defer the RPS work (which for
+ * posterity is necessary because of forcewake).
+ */
+static void hsw_pm_irq_handler(struct drm_i915_private *dev_priv,
+                               u32 pm_iir)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->rps.lock, flags);
+        dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
+        if (dev_priv->rps.pm_iir) {
+                I915_WRITE(GEN6_PMIMR, dev_priv->rps.pm_iir);
+                /* never want to mask useful interrupts. (also posting read) */
+                WARN_ON(I915_READ_NOTRACE(GEN6_PMIMR) & ~GEN6_PM_RPS_EVENTS);
+                /* TODO: if queue_work is slow, move it out of the spinlock */
+                queue_work(dev_priv->wq, &dev_priv->rps.work);
+        }
+        spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
+
+        if (pm_iir & ~GEN6_PM_RPS_EVENTS) {
+                if (pm_iir & PM_VEBOX_USER_INTERRUPT)
+                        notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
+
+                if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
+                        DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
+                        i915_handle_error(dev_priv->dev, false);
+                }
+        }
+}
+
 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = (struct drm_device *) arg;
@@ -727,12 +1018,9 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 
                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
                                          hotplug_status);
-                        if (hotplug_trigger) {
-                                if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_status_i915))
-                                        i915_hpd_irq_setup(dev);
-                                queue_work(dev_priv->wq,
-                                           &dev_priv->hotplug_work);
-                        }
+
+                        intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
+
                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
                         I915_READ(PORT_HOTPLUG_STAT);
                 }
@@ -740,7 +1028,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
                         gmbus_irq_handler(dev);
 
-                if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
+                if (pm_iir & GEN6_PM_RPS_EVENTS)
                         gen6_queue_rps_work(dev_priv, pm_iir);
 
                 I915_WRITE(GTIIR, gt_iir);
@@ -758,15 +1046,14 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
         int pipe;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
 
-        if (hotplug_trigger) {
-                if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_ibx))
-                        ibx_hpd_irq_setup(dev);
-                queue_work(dev_priv->wq, &dev_priv->hotplug_work);
-        }
-        if (pch_iir & SDE_AUDIO_POWER_MASK)
+        intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
+
+        if (pch_iir & SDE_AUDIO_POWER_MASK) {
+                int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
+                               SDE_AUDIO_POWER_SHIFT);
                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
-                                 (pch_iir & SDE_AUDIO_POWER_MASK) >>
-                                 SDE_AUDIO_POWER_SHIFT);
+                                 port_name(port));
+        }
 
         if (pch_iir & SDE_AUX_MASK)
                 dp_aux_irq_handler(dev);
@@ -795,10 +1082,64 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
 
-        if (pch_iir & SDE_TRANSB_FIFO_UNDER)
-                DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n");
         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
-                DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n");
+                if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
+                                                          false))
+                        DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
+
+        if (pch_iir & SDE_TRANSB_FIFO_UNDER)
+                if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
+                                                          false))
+                        DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
+}
+
+static void ivb_err_int_handler(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 err_int = I915_READ(GEN7_ERR_INT);
+
+        if (err_int & ERR_INT_POISON)
+                DRM_ERROR("Poison interrupt\n");
+
+        if (err_int & ERR_INT_FIFO_UNDERRUN_A)
+                if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
+                        DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
+
+        if (err_int & ERR_INT_FIFO_UNDERRUN_B)
+                if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
+                        DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
+
+        if (err_int & ERR_INT_FIFO_UNDERRUN_C)
+                if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
+                        DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
+
+        I915_WRITE(GEN7_ERR_INT, err_int);
+}
+
+static void cpt_serr_int_handler(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 serr_int = I915_READ(SERR_INT);
+
+        if (serr_int & SERR_INT_POISON)
+                DRM_ERROR("PCH poison interrupt\n");
+
+        if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
+                if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
+                                                          false))
+                        DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
+
+        if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
+                if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
+                                                          false))
+                        DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
+
+        if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
+                if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
+                                                          false))
+                        DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
+
+        I915_WRITE(SERR_INT, serr_int);
 }
 
 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
@@ -807,15 +1148,14 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
         int pipe;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
 
-        if (hotplug_trigger) {
-                if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_cpt))
-                        ibx_hpd_irq_setup(dev);
-                queue_work(dev_priv->wq, &dev_priv->hotplug_work);
+        intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
+
+        if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
+                int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
+                               SDE_AUDIO_POWER_SHIFT_CPT);
+                DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
+                                 port_name(port));
         }
-        if (pch_iir & SDE_AUDIO_POWER_MASK_CPT)
-                DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
-                                 (pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
-                                 SDE_AUDIO_POWER_SHIFT_CPT);
 
         if (pch_iir & SDE_AUX_MASK_CPT)
                 dp_aux_irq_handler(dev);
@@ -834,6 +1174,9 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
                                          pipe_name(pipe),
                                          I915_READ(FDI_RX_IIR(pipe)));
+
+        if (pch_iir & SDE_ERROR_CPT)
+                cpt_serr_int_handler(dev);
 }
 
 static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
@@ -846,6 +1189,14 @@ static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
 
         atomic_inc(&dev_priv->irq_received);
 
+        /* We get interrupts on unclaimed registers, so check for this before we
+         * do any I915_{READ,WRITE}. */
+        if (IS_HASWELL(dev) &&
+            (I915_READ_NOTRACE(FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
+                DRM_ERROR("Unclaimed register before interrupt\n");
+                I915_WRITE_NOTRACE(FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
+        }
+
         /* disable master interrupt before clearing iir  */
         de_ier = I915_READ(DEIER);
         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
@@ -861,6 +1212,15 @@ static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
                 POSTING_READ(SDEIER);
         }
 
+        /* On Haswell, also mask ERR_INT because we don't want to risk
+         * generating "unclaimed register" interrupts from inside the interrupt
+         * handler. */
+        if (IS_HASWELL(dev)) {
+                spin_lock(&dev_priv->irq_lock);
+                ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
+                spin_unlock(&dev_priv->irq_lock);
+        }
+
         gt_iir = I915_READ(GTIIR);
         if (gt_iir) {
                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
@@ -870,11 +1230,14 @@ static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
 
         de_iir = I915_READ(DEIIR);
         if (de_iir) {
+                if (de_iir & DE_ERR_INT_IVB)
+                        ivb_err_int_handler(dev);
+
                 if (de_iir & DE_AUX_CHANNEL_A_IVB)
                         dp_aux_irq_handler(dev);
 
                 if (de_iir & DE_GSE_IVB)
-                        intel_opregion_gse_intr(dev);
+                        intel_opregion_asle_intr(dev);
 
                 for (i = 0; i < 3; i++) {
                         if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
@@ -901,12 +1264,21 @@ static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
 
         pm_iir = I915_READ(GEN6_PMIIR);
         if (pm_iir) {
-                if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
+                if (IS_HASWELL(dev))
+                        hsw_pm_irq_handler(dev_priv, pm_iir);
+                else if (pm_iir & GEN6_PM_RPS_EVENTS)
                         gen6_queue_rps_work(dev_priv, pm_iir);
                 I915_WRITE(GEN6_PMIIR, pm_iir);
                 ret = IRQ_HANDLED;
         }
 
+        if (IS_HASWELL(dev)) {
+                spin_lock(&dev_priv->irq_lock);
+                if (ivb_can_enable_err_int(dev))
+                        ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
+                spin_unlock(&dev_priv->irq_lock);
+        }
+
         I915_WRITE(DEIER, de_ier);
         POSTING_READ(DEIER);
         if (!HAS_PCH_NOP(dev)) {
@@ -921,9 +1293,10 @@ static void ilk_gt_irq_handler(struct drm_device *dev,
                                struct drm_i915_private *dev_priv,
                                u32 gt_iir)
 {
-        if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
+        if (gt_iir &
+            (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
                 notify_ring(dev, &dev_priv->ring[RCS]);
-        if (gt_iir & GT_BSD_USER_INTERRUPT)
+        if (gt_iir & ILK_BSD_USER_INTERRUPT)
                 notify_ring(dev, &dev_priv->ring[VCS]);
 }
 
@@ -968,7 +1341,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
                 dp_aux_irq_handler(dev);
 
         if (de_iir & DE_GSE)
-                intel_opregion_gse_intr(dev);
+                intel_opregion_asle_intr(dev);
 
         if (de_iir & DE_PIPEA_VBLANK)
                 drm_handle_vblank(dev, 0);
@@ -976,6 +1349,17 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         if (de_iir & DE_PIPEB_VBLANK)
                 drm_handle_vblank(dev, 1);
 
+        if (de_iir & DE_POISON)
+                DRM_ERROR("Poison interrupt\n");
+
+        if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
+                if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
+                        DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
+
+        if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
+                if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
+                        DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
+
         if (de_iir & DE_PLANEA_FLIP_DONE) {
                 intel_prepare_page_flip(dev, 0);
                 intel_finish_page_flip_plane(dev, 0);
@@ -1002,7 +1386,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         if (IS_GEN5(dev) &&  de_iir & DE_PCU_EVENT)
                 ironlake_handle_rps_change(dev);
 
-        if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS)
+        if (IS_GEN6(dev) && pm_iir & GEN6_PM_RPS_EVENTS)
                 gen6_queue_rps_work(dev_priv, pm_iir);
 
         I915_WRITE(GTIIR, gt_iir);
@@ -1222,11 +1606,13 @@ i915_error_state_free(struct kref *error_ref)
         for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
                 i915_error_object_free(error->ring[i].batchbuffer);
                 i915_error_object_free(error->ring[i].ringbuffer);
+                i915_error_object_free(error->ring[i].ctx);
                 kfree(error->ring[i].requests);
         }
 
         kfree(error->active_bo);
         kfree(error->overlay);
+        kfree(error->display);
         kfree(error);
 }
 static void capture_bo(struct drm_i915_error_buffer *err,
@@ -1273,7 +1659,7 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
         struct drm_i915_gem_object *obj;
         int i = 0;
 
-        list_for_each_entry(obj, head, gtt_list) {
+        list_for_each_entry(obj, head, global_list) {
                 if (obj->pin_count == 0)
                         continue;
 
@@ -1415,7 +1801,7 @@ static void i915_gem_record_active_context(struct intel_ring_buffer *ring,
         if (ring->id != RCS || !error->ccid)
                 return;
 
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 if ((error->ccid & PAGE_MASK) == obj->gtt_offset) {
                         ering->ctx = i915_error_object_create_sized(dev_priv,
                                                                     obj, 1);
@@ -1552,7 +1938,7 @@ static void i915_capture_error_state(struct drm_device *dev)
         list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
                 i++;
         error->active_bo_count = i;
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
                 if (obj->pin_count)
                         i++;
         error->pinned_bo_count = i - error->active_bo_count;
@@ -1932,38 +2318,28 @@ ring_last_seqno(struct intel_ring_buffer *ring)
                           struct drm_i915_gem_request, list)->seqno;
 }
 
-static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
+static bool
+ring_idle(struct intel_ring_buffer *ring, u32 seqno)
 {
-        if (list_empty(&ring->request_list) ||
-            i915_seqno_passed(ring->get_seqno(ring, false),
-                              ring_last_seqno(ring))) {
-                /* Issue a wake-up to catch stuck h/w. */
-                if (waitqueue_active(&ring->irq_queue)) {
-                        DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
-                                  ring->name);
-                        wake_up_all(&ring->irq_queue);
-                        *err = true;
-                }
-                return true;
-        }
-        return false;
+        return (list_empty(&ring->request_list) ||
+                i915_seqno_passed(seqno, ring_last_seqno(ring)));
 }
 
-static bool semaphore_passed(struct intel_ring_buffer *ring)
+static struct intel_ring_buffer *
+semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
-        u32 acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
-        struct intel_ring_buffer *signaller;
-        u32 cmd, ipehr, acthd_min;
+        u32 cmd, ipehr, acthd, acthd_min;
 
         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
         if ((ipehr & ~(0x3 << 16)) !=
             (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
-                return false;
+                return NULL;
 
         /* ACTHD is likely pointing to the dword after the actual command,
          * so scan backwards until we find the MBOX.
          */
+        acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
         acthd_min = max((int)acthd - 3 * 4, 0);
         do {
                 cmd = ioread32(ring->virtual_start + acthd);
@@ -1972,124 +2348,216 @@ static bool semaphore_passed(struct intel_ring_buffer *ring)
 
                 acthd -= 4;
                 if (acthd < acthd_min)
-                        return false;
+                        return NULL;
         } while (1);
 
-        signaller = &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
-        return i915_seqno_passed(signaller->get_seqno(signaller, false),
-                                 ioread32(ring->virtual_start+acthd+4)+1);
+        *seqno = ioread32(ring->virtual_start+acthd+4)+1;
+        return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
 }
 
-static bool kick_ring(struct intel_ring_buffer *ring)
+static int semaphore_passed(struct intel_ring_buffer *ring)
 {
-        struct drm_device *dev = ring->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 tmp = I915_READ_CTL(ring);
-        if (tmp & RING_WAIT) {
-                DRM_ERROR("Kicking stuck wait on %s\n",
-                          ring->name);
-                I915_WRITE_CTL(ring, tmp);
-                return true;
-        }
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        struct intel_ring_buffer *signaller;
+        u32 seqno, ctl;
 
-        if (INTEL_INFO(dev)->gen >= 6 &&
-            tmp & RING_WAIT_SEMAPHORE &&
-            semaphore_passed(ring)) {
-                DRM_ERROR("Kicking stuck semaphore on %s\n",
-                          ring->name);
-                I915_WRITE_CTL(ring, tmp);
-                return true;
-        }
-        return false;
+        ring->hangcheck.deadlock = true;
+
+        signaller = semaphore_waits_for(ring, &seqno);
+        if (signaller == NULL || signaller->hangcheck.deadlock)
+                return -1;
+
+        /* cursory check for an unkickable deadlock */
+        ctl = I915_READ_CTL(signaller);
+        if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
+                return -1;
+
+        return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
 }
 
-static bool i915_hangcheck_hung(struct drm_device *dev)
+static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-
-        if (dev_priv->gpu_error.hangcheck_count++ > 1) {
-                bool hung = true;
+        struct intel_ring_buffer *ring;
+        int i;
 
-                DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
-                i915_handle_error(dev, true);
+        for_each_ring(ring, dev_priv, i)
+                ring->hangcheck.deadlock = false;
+}
 
-                if (!IS_GEN2(dev)) {
-                        struct intel_ring_buffer *ring;
-                        int i;
+static enum intel_ring_hangcheck_action
+ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp;
 
-                        /* Is the chip hanging on a WAIT_FOR_EVENT?
-                         * If so we can simply poke the RB_WAIT bit
-                         * and break the hang. This should work on
-                         * all but the second generation chipsets.
-                         */
-                        for_each_ring(ring, dev_priv, i)
-                                hung &= !kick_ring(ring);
-                }
+        if (ring->hangcheck.acthd != acthd)
+                return active;
 
+        if (IS_GEN2(dev))
                 return hung;
+
+        /* Is the chip hanging on a WAIT_FOR_EVENT?
+         * If so we can simply poke the RB_WAIT bit
+         * and break the hang. This should work on
+         * all but the second generation chipsets.
+         */
+        tmp = I915_READ_CTL(ring);
+        if (tmp & RING_WAIT) {
+                DRM_ERROR("Kicking stuck wait on %s\n",
+                          ring->name);
+                I915_WRITE_CTL(ring, tmp);
+                return kick;
         }
 
-        return false;
+        if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
+                switch (semaphore_passed(ring)) {
+                default:
+                        return hung;
+                case 1:
+                        DRM_ERROR("Kicking stuck semaphore on %s\n",
+                                  ring->name);
+                        I915_WRITE_CTL(ring, tmp);
+                        return kick;
+                case 0:
+                        return wait;
+                }
+        }
+
+        return hung;
 }
 
 /**
  * This is called when the chip hasn't reported back with completed
- * batchbuffers in a long time. The first time this is called we simply record
- * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses
- * again, we assume the chip is wedged and try to fix it.
+ * batchbuffers in a long time. We keep track per ring seqno progress and
+ * if there are no progress, hangcheck score for that ring is increased.
+ * Further, acthd is inspected to see if the ring is stuck. On stuck case
+ * we kick the ring. If we see no progress on three subsequent calls
+ * we assume chip is wedged and try to fix it by resetting the chip.
  */
 void i915_hangcheck_elapsed(unsigned long data)
 {
         struct drm_device *dev = (struct drm_device *)data;
         drm_i915_private_t *dev_priv = dev->dev_private;
-        uint32_t acthd[I915_NUM_RINGS], instdone[I915_NUM_INSTDONE_REG];
         struct intel_ring_buffer *ring;
-        bool err = false, idle;
         int i;
+        int busy_count = 0, rings_hung = 0;
+        bool stuck[I915_NUM_RINGS] = { 0 };
+#define BUSY 1
+#define KICK 5
+#define HUNG 20
+#define FIRE 30
 
         if (!i915_enable_hangcheck)
                 return;
 
-        memset(acthd, 0, sizeof(acthd));
-        idle = true;
         for_each_ring(ring, dev_priv, i) {
-            idle &= i915_hangcheck_ring_idle(ring, &err);
-            acthd[i] = intel_ring_get_active_head(ring);
-        }
+                u32 seqno, acthd;
+                bool busy = true;
+
+                semaphore_clear_deadlocks(dev_priv);
+
+                seqno = ring->get_seqno(ring, false);
+                acthd = intel_ring_get_active_head(ring);
+
+                if (ring->hangcheck.seqno == seqno) {
+                        if (ring_idle(ring, seqno)) {
+                                if (waitqueue_active(&ring->irq_queue)) {
+                                        /* Issue a wake-up to catch stuck h/w. */
+                                        DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
+                                                  ring->name);
+                                        wake_up_all(&ring->irq_queue);
+                                        ring->hangcheck.score += HUNG;
+                                } else
+                                        busy = false;
+                        } else {
+                                int score;
+
+                                /* We always increment the hangcheck score
+                                 * if the ring is busy and still processing
+                                 * the same request, so that no single request
+                                 * can run indefinitely (such as a chain of
+                                 * batches). The only time we do not increment
+                                 * the hangcheck score on this ring, if this
+                                 * ring is in a legitimate wait for another
+                                 * ring. In that case the waiting ring is a
+                                 * victim and we want to be sure we catch the
+                                 * right culprit. Then every time we do kick
+                                 * the ring, add a small increment to the
+                                 * score so that we can catch a batch that is
+                                 * being repeatedly kicked and so responsible
+                                 * for stalling the machine.
+                                 */
+                                ring->hangcheck.action = ring_stuck(ring,
+                                                                    acthd);
+
+                                switch (ring->hangcheck.action) {
+                                case wait:
+                                        score = 0;
+                                        break;
+                                case active:
+                                        score = BUSY;
+                                        break;
+                                case kick:
+                                        score = KICK;
+                                        break;
+                                case hung:
+                                        score = HUNG;
+                                        stuck[i] = true;
+                                        break;
+                                }
+                                ring->hangcheck.score += score;
+                        }
+                } else {
+                        /* Gradually reduce the count so that we catch DoS
+                         * attempts across multiple batches.
+                         */
+                        if (ring->hangcheck.score > 0)
+                                ring->hangcheck.score--;
+                }
 
-        /* If all work is done then ACTHD clearly hasn't advanced. */
-        if (idle) {
-                if (err) {
-                        if (i915_hangcheck_hung(dev))
-                                return;
+                ring->hangcheck.seqno = seqno;
+                ring->hangcheck.acthd = acthd;
+                busy_count += busy;
+        }
 
-                        goto repeat;
+        for_each_ring(ring, dev_priv, i) {
+                if (ring->hangcheck.score > FIRE) {
+                        DRM_ERROR("%s on %s\n",
+                                  stuck[i] ? "stuck" : "no progress",
+                                  ring->name);
+                        rings_hung++;
                 }
-
-                dev_priv->gpu_error.hangcheck_count = 0;
-                return;
         }
 
-        i915_get_extra_instdone(dev, instdone);
-        if (memcmp(dev_priv->gpu_error.last_acthd, acthd,
-                   sizeof(acthd)) == 0 &&
-            memcmp(dev_priv->gpu_error.prev_instdone, instdone,
-                   sizeof(instdone)) == 0) {
-                if (i915_hangcheck_hung(dev))
-                        return;
-        } else {
-                dev_priv->gpu_error.hangcheck_count = 0;
+        if (rings_hung)
+                return i915_handle_error(dev, true);
 
-                memcpy(dev_priv->gpu_error.last_acthd, acthd,
-                       sizeof(acthd));
-                memcpy(dev_priv->gpu_error.prev_instdone, instdone,
-                       sizeof(instdone));
-        }
+        if (busy_count)
+                /* Reset timer case chip hangs without another request
+                 * being added */
+                mod_timer(&dev_priv->gpu_error.hangcheck_timer,
+                          round_jiffies_up(jiffies +
+                                           DRM_I915_HANGCHECK_JIFFIES));
+}
+
+static void ibx_irq_preinstall(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (HAS_PCH_NOP(dev))
+                return;
 
-repeat:
-        /* Reset timer case chip hangs without another request being added */
-        mod_timer(&dev_priv->gpu_error.hangcheck_timer,
-                  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
+        /* south display irq */
+        I915_WRITE(SDEIMR, 0xffffffff);
+        /*
+         * SDEIER is also touched by the interrupt handler to work around missed
+         * PCH interrupts. Hence we can't update it after the interrupt handler
+         * is enabled - instead we unconditionally enable all PCH interrupt
+         * sources here, but then only unmask them as needed with SDEIMR.
+         */
+        I915_WRITE(SDEIER, 0xffffffff);
+        POSTING_READ(SDEIER);
 }
 
 /* drm_dma.h hooks
@@ -2113,19 +2581,34 @@ static void ironlake_irq_preinstall(struct drm_device *dev)
         I915_WRITE(GTIER, 0x0);
         POSTING_READ(GTIER);
 
-        if (HAS_PCH_NOP(dev))
-                return;
+        ibx_irq_preinstall(dev);
+}
 
-        /* south display irq */
-        I915_WRITE(SDEIMR, 0xffffffff);
-        /*
-         * SDEIER is also touched by the interrupt handler to work around missed
-         * PCH interrupts. Hence we can't update it after the interrupt handler
-         * is enabled - instead we unconditionally enable all PCH interrupt
-         * sources here, but then only unmask them as needed with SDEIMR.
-         */
-        I915_WRITE(SDEIER, 0xffffffff);
-        POSTING_READ(SDEIER);
+static void ivybridge_irq_preinstall(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+
+        atomic_set(&dev_priv->irq_received, 0);
+
+        I915_WRITE(HWSTAM, 0xeffe);
+
+        /* XXX hotplug from PCH */
+
+        I915_WRITE(DEIMR, 0xffffffff);
+        I915_WRITE(DEIER, 0x0);
+        POSTING_READ(DEIER);
+
+        /* and GT */
+        I915_WRITE(GTIMR, 0xffffffff);
+        I915_WRITE(GTIER, 0x0);
+        POSTING_READ(GTIER);
+
+        /* Power management */
+        I915_WRITE(GEN6_PMIMR, 0xffffffff);
+        I915_WRITE(GEN6_PMIER, 0x0);
+        POSTING_READ(GEN6_PMIER);
+
+        ibx_irq_preinstall(dev);
 }
 
 static void valleyview_irq_preinstall(struct drm_device *dev)
@@ -2201,33 +2684,41 @@ static void ibx_irq_postinstall(struct drm_device *dev)
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         u32 mask;
 
-        if (HAS_PCH_IBX(dev))
-                mask = SDE_GMBUS | SDE_AUX_MASK;
-        else
-                mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
-
         if (HAS_PCH_NOP(dev))
                 return;
 
+        if (HAS_PCH_IBX(dev)) {
+                mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
+                       SDE_TRANSA_FIFO_UNDER | SDE_POISON;
+        } else {
+                mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
+
+                I915_WRITE(SERR_INT, I915_READ(SERR_INT));
+        }
+
         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
         I915_WRITE(SDEIMR, ~mask);
 }
 
 static int ironlake_irq_postinstall(struct drm_device *dev)
 {
+        unsigned long irqflags;
+
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
         /* enable kind of interrupts always enabled */
         u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
                            DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
-                           DE_AUX_CHANNEL_A;
-        u32 render_irqs;
+                           DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
+                           DE_PIPEA_FIFO_UNDERRUN | DE_POISON;
+        u32 gt_irqs;
 
         dev_priv->irq_mask = ~display_mask;
 
         /* should always can generate irq */
         I915_WRITE(DEIIR, I915_READ(DEIIR));
         I915_WRITE(DEIMR, dev_priv->irq_mask);
-        I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
+        I915_WRITE(DEIER, display_mask |
+                          DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT);
         POSTING_READ(DEIER);
 
         dev_priv->gt_irq_mask = ~0;
@@ -2235,26 +2726,28 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
         I915_WRITE(GTIIR, I915_READ(GTIIR));
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 
+        gt_irqs = GT_RENDER_USER_INTERRUPT;
+
         if (IS_GEN6(dev))
-                render_irqs =
-                        GT_USER_INTERRUPT |
-                        GEN6_BSD_USER_INTERRUPT |
-                        GEN6_BLITTER_USER_INTERRUPT;
+                gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
         else
-                render_irqs =
-                        GT_USER_INTERRUPT |
-                        GT_PIPE_NOTIFY |
-                        GT_BSD_USER_INTERRUPT;
-        I915_WRITE(GTIER, render_irqs);
+                gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
+                           ILK_BSD_USER_INTERRUPT;
+
+        I915_WRITE(GTIER, gt_irqs);
         POSTING_READ(GTIER);
 
         ibx_irq_postinstall(dev);
 
         if (IS_IRONLAKE_M(dev)) {
-                /* Clear & enable PCU event interrupts */
-                I915_WRITE(DEIIR, DE_PCU_EVENT);
-                I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT);
+                /* Enable PCU event interrupts
+                 *
+                 * spinlocking not required here for correctness since interrupt
+                 * setup is guaranteed to run in single-threaded context. But we
+                 * need it to make the assert_spin_locked happy. */
+                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
+                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
         }
 
         return 0;
@@ -2269,12 +2762,15 @@ static int ivybridge_irq_postinstall(struct drm_device *dev)
                 DE_PLANEC_FLIP_DONE_IVB |
                 DE_PLANEB_FLIP_DONE_IVB |
                 DE_PLANEA_FLIP_DONE_IVB |
-                DE_AUX_CHANNEL_A_IVB;
-        u32 render_irqs;
+                DE_AUX_CHANNEL_A_IVB |
+                DE_ERR_INT_IVB;
+        u32 pm_irqs = GEN6_PM_RPS_EVENTS;
+        u32 gt_irqs;
 
         dev_priv->irq_mask = ~display_mask;
 
         /* should always can generate irq */
+        I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
         I915_WRITE(DEIIR, I915_READ(DEIIR));
         I915_WRITE(DEIMR, dev_priv->irq_mask);
         I915_WRITE(DEIER,
@@ -2284,16 +2780,32 @@ static int ivybridge_irq_postinstall(struct drm_device *dev)
                    DE_PIPEA_VBLANK_IVB);
         POSTING_READ(DEIER);
 
-        dev_priv->gt_irq_mask = ~GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
+        dev_priv->gt_irq_mask = ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
         I915_WRITE(GTIIR, I915_READ(GTIIR));
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 
-        render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
-                GEN6_BLITTER_USER_INTERRUPT | GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
-        I915_WRITE(GTIER, render_irqs);
+        gt_irqs = GT_RENDER_USER_INTERRUPT | GT_BSD_USER_INTERRUPT |
+                  GT_BLT_USER_INTERRUPT | GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
+        I915_WRITE(GTIER, gt_irqs);
         POSTING_READ(GTIER);
 
+        I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
+        if (HAS_VEBOX(dev))
+                pm_irqs |= PM_VEBOX_USER_INTERRUPT |
+                        PM_VEBOX_CS_ERROR_INTERRUPT;
+
+        /* Our enable/disable rps functions may touch these registers so
+         * make sure to set a known state for only the non-RPS bits.
+         * The RMW is extra paranoia since this should be called after being set
+         * to a known state in preinstall.
+         * */
+        I915_WRITE(GEN6_PMIMR,
+                   (I915_READ(GEN6_PMIMR) | ~GEN6_PM_RPS_EVENTS) & ~pm_irqs);
+        I915_WRITE(GEN6_PMIER,
+                   (I915_READ(GEN6_PMIER) & GEN6_PM_RPS_EVENTS) | pm_irqs);
+        POSTING_READ(GEN6_PMIER);
+
         ibx_irq_postinstall(dev);
 
         return 0;
@@ -2302,10 +2814,9 @@ static int ivybridge_irq_postinstall(struct drm_device *dev)
 static int valleyview_irq_postinstall(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        u32 gt_irqs;
         u32 enable_mask;
         u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
-        u32 render_irqs;
-        u16 msid;
 
         enable_mask = I915_DISPLAY_PORT_INTERRUPT;
         enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
@@ -2321,13 +2832,6 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
 
-        /* Hack for broken MSIs on VLV */
-        pci_write_config_dword(dev_priv->dev->pdev, 0x94, 0xfee00000);
-        pci_read_config_word(dev->pdev, 0x98, &msid);
-        msid &= 0xff; /* mask out delivery bits */
-        msid |= (1<<14);
-        pci_write_config_word(dev_priv->dev->pdev, 0x98, msid);
-
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         POSTING_READ(PORT_HOTPLUG_EN);
 
@@ -2348,9 +2852,9 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
         I915_WRITE(GTIIR, I915_READ(GTIIR));
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 
-        render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
-                GEN6_BLITTER_USER_INTERRUPT;
-        I915_WRITE(GTIER, render_irqs);
+        gt_irqs = GT_RENDER_USER_INTERRUPT | GT_BSD_USER_INTERRUPT |
+                GT_BLT_USER_INTERRUPT;
+        I915_WRITE(GTIER, gt_irqs);
         POSTING_READ(GTIER);
 
         /* ack & enable invalid PTE error interrupts */
@@ -2402,6 +2906,8 @@ static void ironlake_irq_uninstall(struct drm_device *dev)
         I915_WRITE(DEIMR, 0xffffffff);
         I915_WRITE(DEIER, 0x0);
         I915_WRITE(DEIIR, I915_READ(DEIIR));
+        if (IS_GEN7(dev))
+                I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
 
         I915_WRITE(GTIMR, 0xffffffff);
         I915_WRITE(GTIER, 0x0);
@@ -2413,6 +2919,8 @@ static void ironlake_irq_uninstall(struct drm_device *dev)
         I915_WRITE(SDEIMR, 0xffffffff);
         I915_WRITE(SDEIER, 0x0);
         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
+        if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
+                I915_WRITE(SERR_INT, I915_READ(SERR_INT));
 }
 
 static void i8xx_irq_preinstall(struct drm_device * dev)
@@ -2626,7 +3134,7 @@ static int i915_irq_postinstall(struct drm_device *dev)
         I915_WRITE(IER, enable_mask);
         POSTING_READ(IER);
 
-        intel_opregion_enable_asle(dev);
+        i915_enable_asle_pipestat(dev);
 
         return 0;
 }
@@ -2715,12 +3223,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
                                   hotplug_status);
-                        if (hotplug_trigger) {
-                                if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_status_i915))
-                                        i915_hpd_irq_setup(dev);
-                                queue_work(dev_priv->wq,
-                                           &dev_priv->hotplug_work);
-                        }
+
+                        intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
+
                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
                         POSTING_READ(PORT_HOTPLUG_STAT);
                 }
@@ -2860,7 +3365,7 @@ static int i965_irq_postinstall(struct drm_device *dev)
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         POSTING_READ(PORT_HOTPLUG_EN);
 
-        intel_opregion_enable_asle(dev);
+        i915_enable_asle_pipestat(dev);
 
         return 0;
 }
@@ -2872,6 +3377,8 @@ static void i915_hpd_irq_setup(struct drm_device *dev)
         struct intel_encoder *intel_encoder;
         u32 hotplug_en;
 
+        assert_spin_locked(&dev_priv->irq_lock);
+
         if (I915_HAS_HOTPLUG(dev)) {
                 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
                 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
@@ -2952,17 +3459,14 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
                         u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
                                                                   HOTPLUG_INT_STATUS_G4X :
-                                                                  HOTPLUG_INT_STATUS_I965);
+                                                                  HOTPLUG_INT_STATUS_I915);
 
                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
                                   hotplug_status);
-                        if (hotplug_trigger) {
-                                if (hotplug_irq_storm_detect(dev, hotplug_trigger,
-                                                            IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i965))
-                                        i915_hpd_irq_setup(dev);
-                                queue_work(dev_priv->wq,
-                                           &dev_priv->hotplug_work);
-                        }
+
+                        intel_hpd_irq_handler(dev, hotplug_trigger,
+                                              IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
+
                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
                         I915_READ(PORT_HOTPLUG_STAT);
                 }
@@ -3113,9 +3617,9 @@ void intel_irq_init(struct drm_device *dev)
                 dev->driver->disable_vblank = valleyview_disable_vblank;
                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
         } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
-                /* Share pre & uninstall handlers with ILK/SNB */
+                /* Share uninstall handlers with ILK/SNB */
                 dev->driver->irq_handler = ivybridge_irq_handler;
-                dev->driver->irq_preinstall = ironlake_irq_preinstall;
+                dev->driver->irq_preinstall = ivybridge_irq_preinstall;
                 dev->driver->irq_postinstall = ivybridge_irq_postinstall;
                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
                 dev->driver->enable_vblank = ivybridge_enable_vblank;
@@ -3158,6 +3662,7 @@ void intel_hpd_init(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct drm_connector *connector;
+        unsigned long irqflags;
         int i;
 
         for (i = 1; i < HPD_NUM_PINS; i++) {
@@ -3170,6 +3675,11 @@ void intel_hpd_init(struct drm_device *dev)
                 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
                         connector->polled = DRM_CONNECTOR_POLL_HPD;
         }
+
+        /* Interrupt setup is already guaranteed to be single-threaded, this is
+         * just to make the assert_spin_locked checks happy. */
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
         if (dev_priv->display.hpd_irq_setup)
                 dev_priv->display.hpd_irq_setup(dev);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 2d6b62e42daf..f2326fc60ac9 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -147,15 +147,9 @@
 #define   VGA_MSR_MEM_EN (1<<1)
 #define   VGA_MSR_CGA_MODE (1<<0)
 
-/*
- * SR01 is the only VGA register touched on non-UMS setups.
- * VLV doesn't do UMS, so the sequencer index/data registers
- * are the only VGA registers which need to include
- * display_mmio_offset.
- */
-#define VGA_SR_INDEX (dev_priv->info->display_mmio_offset + 0x3c4)
+#define VGA_SR_INDEX 0x3c4
 #define SR01                    1
-#define VGA_SR_DATA (dev_priv->info->display_mmio_offset + 0x3c5)
+#define VGA_SR_DATA 0x3c5
 
 #define VGA_AR_INDEX 0x3c0
 #define   VGA_AR_VID_EN (1<<5)
@@ -265,13 +259,19 @@
 #define  MI_SEMAPHORE_UPDATE        (1<<21)
 #define  MI_SEMAPHORE_COMPARE       (1<<20)
 #define  MI_SEMAPHORE_REGISTER      (1<<18)
-#define  MI_SEMAPHORE_SYNC_RV       (2<<16)
-#define  MI_SEMAPHORE_SYNC_RB       (0<<16)
-#define  MI_SEMAPHORE_SYNC_VR       (0<<16)
-#define  MI_SEMAPHORE_SYNC_VB       (2<<16)
-#define  MI_SEMAPHORE_SYNC_BR       (2<<16)
-#define  MI_SEMAPHORE_SYNC_BV       (0<<16)
-#define  MI_SEMAPHORE_SYNC_INVALID  (1<<0)
+#define  MI_SEMAPHORE_SYNC_VR       (0<<16) /* RCS  wait for VCS  (RVSYNC) */
+#define  MI_SEMAPHORE_SYNC_VER      (1<<16) /* RCS  wait for VECS (RVESYNC) */
+#define  MI_SEMAPHORE_SYNC_BR       (2<<16) /* RCS  wait for BCS  (RBSYNC) */
+#define  MI_SEMAPHORE_SYNC_BV       (0<<16) /* VCS  wait for BCS  (VBSYNC) */
+#define  MI_SEMAPHORE_SYNC_VEV      (1<<16) /* VCS  wait for VECS (VVESYNC) */
+#define  MI_SEMAPHORE_SYNC_RV       (2<<16) /* VCS  wait for RCS  (VRSYNC) */
+#define  MI_SEMAPHORE_SYNC_RB       (0<<16) /* BCS  wait for RCS  (BRSYNC) */
+#define  MI_SEMAPHORE_SYNC_VEB      (1<<16) /* BCS  wait for VECS (BVESYNC) */
+#define  MI_SEMAPHORE_SYNC_VB       (2<<16) /* BCS  wait for VCS  (BVSYNC) */
+#define  MI_SEMAPHORE_SYNC_BVE      (0<<16) /* VECS wait for BCS  (VEBSYNC) */
+#define  MI_SEMAPHORE_SYNC_VVE      (1<<16) /* VECS wait for VCS  (VEVSYNC) */
+#define  MI_SEMAPHORE_SYNC_RVE      (2<<16) /* VECS wait for RCS  (VERSYNC) */
+#define  MI_SEMAPHORE_SYNC_INVALID  (3<<16)
 /*
  * 3D instructions used by the kernel
  */
@@ -342,33 +342,74 @@
 #define  DEBUG_RESET_DISPLAY            (1<<9)
 
 /*
- * DPIO - a special bus for various display related registers to hide behind:
- *  0x800c: m1, m2, n, p1, p2, k dividers
- *  0x8014: REF and SFR select
- *  0x8014: N divider, VCO select
- *  0x801c/3c: core clock bits
- *  0x8048/68: low pass filter coefficients
- *  0x8100: fast clock controls
+ * IOSF sideband
+ */
+#define VLV_IOSF_DOORBELL_REQ                   (VLV_DISPLAY_BASE + 0x2100)
+#define   IOSF_DEVFN_SHIFT                      24
+#define   IOSF_OPCODE_SHIFT                     16
+#define   IOSF_PORT_SHIFT                       8
+#define   IOSF_BYTE_ENABLES_SHIFT               4
+#define   IOSF_BAR_SHIFT                        1
+#define   IOSF_SB_BUSY                          (1<<0)
+#define   IOSF_PORT_PUNIT                       0x4
+#define   IOSF_PORT_NC                          0x11
+#define   IOSF_PORT_DPIO                        0x12
+#define VLV_IOSF_DATA                           (VLV_DISPLAY_BASE + 0x2104)
+#define VLV_IOSF_ADDR                           (VLV_DISPLAY_BASE + 0x2108)
+
+#define PUNIT_OPCODE_REG_READ                   6
+#define PUNIT_OPCODE_REG_WRITE                  7
+
+#define PUNIT_REG_GPU_LFM                       0xd3
+#define PUNIT_REG_GPU_FREQ_REQ                  0xd4
+#define PUNIT_REG_GPU_FREQ_STS                  0xd8
+#define PUNIT_REG_MEDIA_TURBO_FREQ_REQ          0xdc
+
+#define PUNIT_FUSE_BUS2                         0xf6 /* bits 47:40 */
+#define PUNIT_FUSE_BUS1                         0xf5 /* bits 55:48 */
+
+#define IOSF_NC_FB_GFX_FREQ_FUSE                0x1c
+#define   FB_GFX_MAX_FREQ_FUSE_SHIFT            3
+#define   FB_GFX_MAX_FREQ_FUSE_MASK             0x000007f8
+#define   FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT    11
+#define   FB_GFX_FGUARANTEED_FREQ_FUSE_MASK     0x0007f800
+#define IOSF_NC_FB_GFX_FMAX_FUSE_HI             0x34
+#define   FB_FMAX_VMIN_FREQ_HI_MASK             0x00000007
+#define IOSF_NC_FB_GFX_FMAX_FUSE_LO             0x30
+#define   FB_FMAX_VMIN_FREQ_LO_SHIFT            27
+#define   FB_FMAX_VMIN_FREQ_LO_MASK             0xf8000000
+
+/*
+ * DPIO - a special bus for various display related registers to hide behind
  *
  * DPIO is VLV only.
+ *
+ * Note: digital port B is DDI0, digital pot C is DDI1
  */
-#define DPIO_PKT                        (VLV_DISPLAY_BASE + 0x2100)
-#define  DPIO_RID                       (0<<24)
-#define  DPIO_OP_WRITE                  (1<<16)
-#define  DPIO_OP_READ                   (0<<16)
-#define  DPIO_PORTID                    (0x12<<8)
-#define  DPIO_BYTE                      (0xf<<4)
-#define  DPIO_BUSY                      (1<<0) /* status only */
-#define DPIO_DATA                       (VLV_DISPLAY_BASE + 0x2104)
-#define DPIO_REG                        (VLV_DISPLAY_BASE + 0x2108)
+#define DPIO_DEVFN                      0
+#define DPIO_OPCODE_REG_WRITE           1
+#define DPIO_OPCODE_REG_READ            0
+
 #define DPIO_CTL                        (VLV_DISPLAY_BASE + 0x2110)
 #define  DPIO_MODSEL1                   (1<<3) /* if ref clk b == 27 */
 #define  DPIO_MODSEL0                   (1<<2) /* if ref clk a == 27 */
 #define  DPIO_SFR_BYPASS                (1<<1)
 #define  DPIO_RESET                     (1<<0)
 
+#define _DPIO_TX3_SWING_CTL4_A          0x690
+#define _DPIO_TX3_SWING_CTL4_B          0x2a90
+#define DPIO_TX3_SWING_CTL4(pipe) _PIPE(pipe, _DPIO_TX_SWING_CTL4_A, \
+                                        _DPIO_TX3_SWING_CTL4_B)
+
+/*
+ * Per pipe/PLL DPIO regs
+ */
 #define _DPIO_DIV_A                     0x800c
 #define   DPIO_POST_DIV_SHIFT           (28) /* 3 bits */
+#define   DPIO_POST_DIV_DAC             0
+#define   DPIO_POST_DIV_HDMIDP          1 /* DAC 225-400M rate */
+#define   DPIO_POST_DIV_LVDS1           2
+#define   DPIO_POST_DIV_LVDS2           3
 #define   DPIO_K_SHIFT                  (24) /* 4 bits */
 #define   DPIO_P1_SHIFT                 (21) /* 3 bits */
 #define   DPIO_P2_SHIFT                 (16) /* 5 bits */
@@ -394,14 +435,111 @@
 #define _DPIO_CORE_CLK_B                0x803c
 #define DPIO_CORE_CLK(pipe) _PIPE(pipe, _DPIO_CORE_CLK_A, _DPIO_CORE_CLK_B)
 
-#define _DPIO_LFP_COEFF_A               0x8048
-#define _DPIO_LFP_COEFF_B               0x8068
-#define DPIO_LFP_COEFF(pipe) _PIPE(pipe, _DPIO_LFP_COEFF_A, _DPIO_LFP_COEFF_B)
+#define _DPIO_IREF_CTL_A                0x8040
+#define _DPIO_IREF_CTL_B                0x8060
+#define DPIO_IREF_CTL(pipe) _PIPE(pipe, _DPIO_IREF_CTL_A, _DPIO_IREF_CTL_B)
+
+#define DPIO_IREF_BCAST                 0xc044
+#define _DPIO_IREF_A                    0x8044
+#define _DPIO_IREF_B                    0x8064
+#define DPIO_IREF(pipe) _PIPE(pipe, _DPIO_IREF_A, _DPIO_IREF_B)
+
+#define _DPIO_PLL_CML_A                 0x804c
+#define _DPIO_PLL_CML_B                 0x806c
+#define DPIO_PLL_CML(pipe) _PIPE(pipe, _DPIO_PLL_CML_A, _DPIO_PLL_CML_B)
+
+#define _DPIO_LPF_COEFF_A               0x8048
+#define _DPIO_LPF_COEFF_B               0x8068
+#define DPIO_LPF_COEFF(pipe) _PIPE(pipe, _DPIO_LPF_COEFF_A, _DPIO_LPF_COEFF_B)
+
+#define DPIO_CALIBRATION                0x80ac
 
 #define DPIO_FASTCLK_DISABLE            0x8100
 
-#define DPIO_DATA_CHANNEL1              0x8220
-#define DPIO_DATA_CHANNEL2              0x8420
+/*
+ * Per DDI channel DPIO regs
+ */
+
+#define _DPIO_PCS_TX_0                  0x8200
+#define _DPIO_PCS_TX_1                  0x8400
+#define   DPIO_PCS_TX_LANE2_RESET       (1<<16)
+#define   DPIO_PCS_TX_LANE1_RESET       (1<<7)
+#define DPIO_PCS_TX(port) _PORT(port, _DPIO_PCS_TX_0, _DPIO_PCS_TX_1)
+
+#define _DPIO_PCS_CLK_0                 0x8204
+#define _DPIO_PCS_CLK_1                 0x8404
+#define   DPIO_PCS_CLK_CRI_RXEB_EIOS_EN (1<<22)
+#define   DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN (1<<21)
+#define   DPIO_PCS_CLK_DATAWIDTH_SHIFT  (6)
+#define   DPIO_PCS_CLK_SOFT_RESET       (1<<5)
+#define DPIO_PCS_CLK(port) _PORT(port, _DPIO_PCS_CLK_0, _DPIO_PCS_CLK_1)
+
+#define _DPIO_PCS_CTL_OVR1_A            0x8224
+#define _DPIO_PCS_CTL_OVR1_B            0x8424
+#define DPIO_PCS_CTL_OVER1(port) _PORT(port, _DPIO_PCS_CTL_OVR1_A, \
+                                       _DPIO_PCS_CTL_OVR1_B)
+
+#define _DPIO_PCS_STAGGER0_A            0x822c
+#define _DPIO_PCS_STAGGER0_B            0x842c
+#define DPIO_PCS_STAGGER0(port) _PORT(port, _DPIO_PCS_STAGGER0_A, \
+                                      _DPIO_PCS_STAGGER0_B)
+
+#define _DPIO_PCS_STAGGER1_A            0x8230
+#define _DPIO_PCS_STAGGER1_B            0x8430
+#define DPIO_PCS_STAGGER1(port) _PORT(port, _DPIO_PCS_STAGGER1_A, \
+                                      _DPIO_PCS_STAGGER1_B)
+
+#define _DPIO_PCS_CLOCKBUF0_A           0x8238
+#define _DPIO_PCS_CLOCKBUF0_B           0x8438
+#define DPIO_PCS_CLOCKBUF0(port) _PORT(port, _DPIO_PCS_CLOCKBUF0_A, \
+                                       _DPIO_PCS_CLOCKBUF0_B)
+
+#define _DPIO_PCS_CLOCKBUF8_A           0x825c
+#define _DPIO_PCS_CLOCKBUF8_B           0x845c
+#define DPIO_PCS_CLOCKBUF8(port) _PORT(port, _DPIO_PCS_CLOCKBUF8_A, \
+                                       _DPIO_PCS_CLOCKBUF8_B)
+
+#define _DPIO_TX_SWING_CTL2_A           0x8288
+#define _DPIO_TX_SWING_CTL2_B           0x8488
+#define DPIO_TX_SWING_CTL2(port) _PORT(port, _DPIO_TX_SWING_CTL2_A, \
+                                       _DPIO_TX_SWING_CTL2_B)
+
+#define _DPIO_TX_SWING_CTL3_A           0x828c
+#define _DPIO_TX_SWING_CTL3_B           0x848c
+#define DPIO_TX_SWING_CTL3(port) _PORT(port, _DPIO_TX_SWING_CTL3_A, \
+                                       _DPIO_TX_SWING_CTL3_B)
+
+#define _DPIO_TX_SWING_CTL4_A           0x8290
+#define _DPIO_TX_SWING_CTL4_B           0x8490
+#define DPIO_TX_SWING_CTL4(port) _PORT(port, _DPIO_TX_SWING_CTL4_A, \
+                                       _DPIO_TX_SWING_CTL4_B)
+
+#define _DPIO_TX_OCALINIT_0             0x8294
+#define _DPIO_TX_OCALINIT_1             0x8494
+#define   DPIO_TX_OCALINIT_EN           (1<<31)
+#define DPIO_TX_OCALINIT(port) _PORT(port, _DPIO_TX_OCALINIT_0, \
+                                     _DPIO_TX_OCALINIT_1)
+
+#define _DPIO_TX_CTL_0                  0x82ac
+#define _DPIO_TX_CTL_1                  0x84ac
+#define DPIO_TX_CTL(port) _PORT(port, _DPIO_TX_CTL_0, _DPIO_TX_CTL_1)
+
+#define _DPIO_TX_LANE_0                 0x82b8
+#define _DPIO_TX_LANE_1                 0x84b8
+#define DPIO_TX_LANE(port) _PORT(port, _DPIO_TX_LANE_0, _DPIO_TX_LANE_1)
+
+#define _DPIO_DATA_CHANNEL1             0x8220
+#define _DPIO_DATA_CHANNEL2             0x8420
+#define DPIO_DATA_CHANNEL(port) _PORT(port, _DPIO_DATA_CHANNEL1, _DPIO_DATA_CHANNEL2)
+
+#define _DPIO_PORT0_PCS0                0x0220
+#define _DPIO_PORT0_PCS1                0x0420
+#define _DPIO_PORT1_PCS2                0x2620
+#define _DPIO_PORT1_PCS3                0x2820
+#define DPIO_DATA_LANE_A(port) _PORT(port, _DPIO_PORT0_PCS0, _DPIO_PORT1_PCS2)
+#define DPIO_DATA_LANE_B(port) _PORT(port, _DPIO_PORT0_PCS1, _DPIO_PORT1_PCS3)
+#define DPIO_DATA_CHANNEL1              0x8220
+#define DPIO_DATA_CHANNEL2              0x8420
 
 /*
  * Fence registers
@@ -443,6 +581,7 @@
 #define RENDER_RING_BASE        0x02000
 #define BSD_RING_BASE           0x04000
 #define GEN6_BSD_RING_BASE      0x12000
+#define VEBOX_RING_BASE         0x1a000
 #define BLT_RING_BASE           0x22000
 #define RING_TAIL(base)         ((base)+0x30)
 #define RING_HEAD(base)         ((base)+0x34)
@@ -450,12 +589,20 @@
 #define RING_CTL(base)          ((base)+0x3c)
 #define RING_SYNC_0(base)       ((base)+0x40)
 #define RING_SYNC_1(base)       ((base)+0x44)
-#define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE))
-#define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE))
-#define GEN6_VRSYNC (RING_SYNC_1(GEN6_BSD_RING_BASE))
-#define GEN6_VBSYNC (RING_SYNC_0(GEN6_BSD_RING_BASE))
-#define GEN6_BRSYNC (RING_SYNC_0(BLT_RING_BASE))
-#define GEN6_BVSYNC (RING_SYNC_1(BLT_RING_BASE))
+#define RING_SYNC_2(base)       ((base)+0x48)
+#define GEN6_RVSYNC     (RING_SYNC_0(RENDER_RING_BASE))
+#define GEN6_RBSYNC     (RING_SYNC_1(RENDER_RING_BASE))
+#define GEN6_RVESYNC    (RING_SYNC_2(RENDER_RING_BASE))
+#define GEN6_VBSYNC     (RING_SYNC_0(GEN6_BSD_RING_BASE))
+#define GEN6_VRSYNC     (RING_SYNC_1(GEN6_BSD_RING_BASE))
+#define GEN6_VVESYNC    (RING_SYNC_2(GEN6_BSD_RING_BASE))
+#define GEN6_BRSYNC     (RING_SYNC_0(BLT_RING_BASE))
+#define GEN6_BVSYNC     (RING_SYNC_1(BLT_RING_BASE))
+#define GEN6_BVESYNC    (RING_SYNC_2(BLT_RING_BASE))
+#define GEN6_VEBSYNC    (RING_SYNC_0(VEBOX_RING_BASE))
+#define GEN6_VERSYNC    (RING_SYNC_1(VEBOX_RING_BASE))
+#define GEN6_VEVSYNC    (RING_SYNC_2(VEBOX_RING_BASE))
+#define GEN6_NOSYNC 0
 #define RING_MAX_IDLE(base)     ((base)+0x54)
 #define RING_HWS_PGA(base)      ((base)+0x80)
 #define RING_HWS_PGA_GEN6(base) ((base)+0x2080)
@@ -467,6 +614,7 @@
 #define DONE_REG                0x40b0
 #define BSD_HWS_PGA_GEN7        (0x04180)
 #define BLT_HWS_PGA_GEN7        (0x04280)
+#define VEBOX_HWS_PGA_GEN7      (0x04380)
 #define RING_ACTHD(base)        ((base)+0x74)
 #define RING_NOPID(base)        ((base)+0x94)
 #define RING_IMR(base)          ((base)+0xa8)
@@ -527,7 +675,11 @@
 
 #define ERROR_GEN6      0x040a0
 #define GEN7_ERR_INT    0x44040
-#define   ERR_INT_MMIO_UNCLAIMED (1<<13)
+#define   ERR_INT_POISON                (1<<31)
+#define   ERR_INT_MMIO_UNCLAIMED        (1<<13)
+#define   ERR_INT_FIFO_UNDERRUN_C       (1<<6)
+#define   ERR_INT_FIFO_UNDERRUN_B       (1<<3)
+#define   ERR_INT_FIFO_UNDERRUN_A       (1<<0)
 
 #define FPGA_DBG                0x42300
 #define   FPGA_DBG_RM_NOCLAIM   (1<<31)
@@ -583,24 +735,7 @@
 #define VLV_IIR         (VLV_DISPLAY_BASE + 0x20a4)
 #define VLV_IMR         (VLV_DISPLAY_BASE + 0x20a8)
 #define VLV_ISR         (VLV_DISPLAY_BASE + 0x20ac)
-#define   I915_PIPE_CONTROL_NOTIFY_INTERRUPT            (1<<18)
-#define   I915_DISPLAY_PORT_INTERRUPT                   (1<<17)
-#define   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT    (1<<15)
-#define   I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT      (1<<14) /* p-state */
-#define   I915_HWB_OOM_INTERRUPT                        (1<<13)
-#define   I915_SYNC_STATUS_INTERRUPT                    (1<<12)
-#define   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT   (1<<11)
-#define   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT   (1<<10)
-#define   I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT     (1<<9)
-#define   I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT   (1<<8)
-#define   I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT          (1<<7)
-#define   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT           (1<<6)
-#define   I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT          (1<<5)
-#define   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT           (1<<4)
-#define   I915_DEBUG_INTERRUPT                          (1<<2)
-#define   I915_USER_INTERRUPT                           (1<<1)
-#define   I915_ASLE_INTERRUPT                           (1<<0)
-#define   I915_BSD_USER_INTERRUPT                      (1<<25)
+#define VLV_PCBR        (VLV_DISPLAY_BASE + 0x2120)
 #define   DISPLAY_PLANE_FLIP_PENDING(plane) (1<<(11-(plane))) /* A and B only */
 #define EIR             0x020b0
 #define EMR             0x020b4
@@ -712,28 +847,6 @@
 #define CACHE_MODE_1            0x7004 /* IVB+ */
 #define   PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1<<6)
 
-/* GEN6 interrupt control
- * Note that the per-ring interrupt bits do alias with the global interrupt bits
- * in GTIMR. */
-#define GEN6_RENDER_HWSTAM      0x2098
-#define GEN6_RENDER_IMR         0x20a8
-#define   GEN6_RENDER_CONTEXT_SWITCH_INTERRUPT          (1 << 8)
-#define   GEN6_RENDER_PPGTT_PAGE_FAULT                  (1 << 7)
-#define   GEN6_RENDER_TIMEOUT_COUNTER_EXPIRED           (1 << 6)
-#define   GEN6_RENDER_L3_PARITY_ERROR                   (1 << 5)
-#define   GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT     (1 << 4)
-#define   GEN6_RENDER_COMMAND_PARSER_MASTER_ERROR       (1 << 3)
-#define   GEN6_RENDER_SYNC_STATUS                       (1 << 2)
-#define   GEN6_RENDER_DEBUG_INTERRUPT                   (1 << 1)
-#define   GEN6_RENDER_USER_INTERRUPT                    (1 << 0)
-
-#define GEN6_BLITTER_HWSTAM     0x22098
-#define GEN6_BLITTER_IMR        0x220a8
-#define   GEN6_BLITTER_MI_FLUSH_DW_NOTIFY_INTERRUPT     (1 << 26)
-#define   GEN6_BLITTER_COMMAND_PARSER_MASTER_ERROR      (1 << 25)
-#define   GEN6_BLITTER_SYNC_STATUS                      (1 << 24)
-#define   GEN6_BLITTER_USER_INTERRUPT                   (1 << 22)
-
 #define GEN6_BLITTER_ECOSKPD    0x221d0
 #define   GEN6_BLITTER_LOCK_SHIFT                       16
 #define   GEN6_BLITTER_FBC_NOTIFY                       (1<<3)
@@ -744,9 +857,52 @@
 #define   GEN6_BSD_SLEEP_INDICATOR      (1 << 3)
 #define   GEN6_BSD_GO_INDICATOR         (1 << 4)
 
-#define GEN6_BSD_HWSTAM                 0x12098
-#define GEN6_BSD_IMR                    0x120a8
-#define   GEN6_BSD_USER_INTERRUPT       (1 << 12)
+/* On modern GEN architectures interrupt control consists of two sets
+ * of registers. The first set pertains to the ring generating the
+ * interrupt. The second control is for the functional block generating the
+ * interrupt. These are PM, GT, DE, etc.
+ *
+ * Luckily *knocks on wood* all the ring interrupt bits match up with the
+ * GT interrupt bits, so we don't need to duplicate the defines.
+ *
+ * These defines should cover us well from SNB->HSW with minor exceptions
+ * it can also work on ILK.
+ */
+#define GT_BLT_FLUSHDW_NOTIFY_INTERRUPT         (1 << 26)
+#define GT_BLT_CS_ERROR_INTERRUPT               (1 << 25)
+#define GT_BLT_USER_INTERRUPT                   (1 << 22)
+#define GT_BSD_CS_ERROR_INTERRUPT               (1 << 15)
+#define GT_BSD_USER_INTERRUPT                   (1 << 12)
+#define GT_RENDER_L3_PARITY_ERROR_INTERRUPT     (1 <<  5) /* !snb */
+#define GT_RENDER_PIPECTL_NOTIFY_INTERRUPT      (1 <<  4)
+#define GT_RENDER_CS_MASTER_ERROR_INTERRUPT     (1 <<  3)
+#define GT_RENDER_SYNC_STATUS_INTERRUPT         (1 <<  2)
+#define GT_RENDER_DEBUG_INTERRUPT               (1 <<  1)
+#define GT_RENDER_USER_INTERRUPT                (1 <<  0)
+
+#define PM_VEBOX_CS_ERROR_INTERRUPT             (1 << 12) /* hsw+ */
+#define PM_VEBOX_USER_INTERRUPT                 (1 << 10) /* hsw+ */
+
+/* These are all the "old" interrupts */
+#define ILK_BSD_USER_INTERRUPT                          (1<<5)
+#define I915_PIPE_CONTROL_NOTIFY_INTERRUPT              (1<<18)
+#define I915_DISPLAY_PORT_INTERRUPT                     (1<<17)
+#define I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT      (1<<15)
+#define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT        (1<<14) /* p-state */
+#define I915_HWB_OOM_INTERRUPT                          (1<<13)
+#define I915_SYNC_STATUS_INTERRUPT                      (1<<12)
+#define I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT     (1<<11)
+#define I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT     (1<<10)
+#define I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT       (1<<9)
+#define I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT     (1<<8)
+#define I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT            (1<<7)
+#define I915_DISPLAY_PIPE_A_EVENT_INTERRUPT             (1<<6)
+#define I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT            (1<<5)
+#define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT             (1<<4)
+#define I915_DEBUG_INTERRUPT                            (1<<2)
+#define I915_USER_INTERRUPT                             (1<<1)
+#define I915_ASLE_INTERRUPT                             (1<<0)
+#define I915_BSD_USER_INTERRUPT                         (1 << 25)
 
 #define GEN6_BSD_RNCID                  0x12198
 
@@ -807,7 +963,9 @@
 #define   DPFC_CTL_EN           (1<<31)
 #define   DPFC_CTL_PLANEA       (0<<30)
 #define   DPFC_CTL_PLANEB       (1<<30)
+#define   IVB_DPFC_CTL_PLANE_SHIFT      (29)
 #define   DPFC_CTL_FENCE_EN     (1<<29)
+#define   IVB_DPFC_CTL_FENCE_EN (1<<28)
 #define   DPFC_CTL_PERSISTENT_MODE      (1<<25)
 #define   DPFC_SR_EN            (1<<10)
 #define   DPFC_CTL_LIMIT_1X     (0<<6)
@@ -840,6 +998,7 @@
 #define ILK_DPFC_CHICKEN        0x43224
 #define ILK_FBC_RT_BASE         0x2128
 #define   ILK_FBC_RT_VALID      (1<<0)
+#define   SNB_FBC_FRONT_BUFFER  (1<<1)
 
 #define ILK_DISPLAY_CHICKEN1    0x42000
 #define   ILK_FBCQ_DIS          (1<<22)
@@ -855,6 +1014,25 @@
 #define   SNB_CPU_FENCE_ENABLE  (1<<29)
 #define DPFC_CPU_FENCE_OFFSET   0x100104
 
+/* Framebuffer compression for Ivybridge */
+#define IVB_FBC_RT_BASE                 0x7020
+
+#define IPS_CTL         0x43408
+#define   IPS_ENABLE    (1 << 31)
+
+#define MSG_FBC_REND_STATE      0x50380
+#define   FBC_REND_NUKE         (1<<2)
+#define   FBC_REND_CACHE_CLEAN  (1<<1)
+
+#define _HSW_PIPE_SLICE_CHICKEN_1_A     0x420B0
+#define _HSW_PIPE_SLICE_CHICKEN_1_B     0x420B4
+#define   HSW_BYPASS_FBC_QUEUE          (1<<22)
+#define HSW_PIPE_SLICE_CHICKEN_1(pipe) _PIPE(pipe, + \
+                                             _HSW_PIPE_SLICE_CHICKEN_1_A, + \
+                                             _HSW_PIPE_SLICE_CHICKEN_1_B)
+
+#define HSW_CLKGATE_DISABLE_PART_1      0x46500
+#define   HSW_DPFC_GATING_DISABLE       (1<<23)
 
 /*
  * GPIO regs
@@ -963,7 +1141,10 @@
 #define   DPLL_FPA01_P1_POST_DIV_MASK   0x00ff0000 /* i915 */
 #define   DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW  0x00ff8000 /* Pineview */
 #define   DPLL_LOCK_VLV                 (1<<15)
+#define   DPLL_INTEGRATED_CRI_CLK_VLV   (1<<14)
 #define   DPLL_INTEGRATED_CLOCK_VLV     (1<<13)
+#define   DPLL_PORTC_READY_MASK         (0xf << 4)
+#define   DPLL_PORTB_READY_MASK         (0xf)
 
 #define   DPLL_FPA01_P1_POST_DIV_MASK_I830      0x001f0000
 /*
@@ -1073,7 +1254,7 @@
 #define  DSTATE_PLL_D3_OFF                      (1<<3)
 #define  DSTATE_GFX_CLOCK_GATING                (1<<1)
 #define  DSTATE_DOT_CLOCK_GATING                (1<<0)
-#define DSPCLK_GATE_D           0x6200
+#define DSPCLK_GATE_D   (dev_priv->info->display_mmio_offset + 0x6200)
 # define DPUNIT_B_CLOCK_GATE_DISABLE            (1 << 30) /* 965 */
 # define VSUNIT_CLOCK_GATE_DISABLE              (1 << 29) /* 965 */
 # define VRHUNIT_CLOCK_GATE_DISABLE             (1 << 28) /* 965 */
@@ -1186,6 +1367,8 @@
 #define FW_BLC_SELF_VLV         (VLV_DISPLAY_BASE + 0x6500)
 #define  FW_CSPWRDWNEN          (1<<15)
 
+#define MI_ARB_VLV              (VLV_DISPLAY_BASE + 0x6504)
+
 /*
  * Palette regs
  */
@@ -1535,14 +1718,13 @@
                                          GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \
                                          GEN7_CXT_GT1_SIZE(ctx_reg) + \
                                          GEN7_CXT_VFSTATE_SIZE(ctx_reg))
-#define HSW_CXT_POWER_SIZE(ctx_reg)     ((ctx_reg >> 26) & 0x3f)
-#define HSW_CXT_RING_SIZE(ctx_reg)      ((ctx_reg >> 23) & 0x7)
-#define HSW_CXT_RENDER_SIZE(ctx_reg)    ((ctx_reg >> 15) & 0xff)
-#define HSW_CXT_TOTAL_SIZE(ctx_reg)     (HSW_CXT_POWER_SIZE(ctx_reg) + \
-                                         HSW_CXT_RING_SIZE(ctx_reg) + \
-                                         HSW_CXT_RENDER_SIZE(ctx_reg) + \
-                                         GEN7_CXT_VFSTATE_SIZE(ctx_reg))
-
+/* Haswell does have the CXT_SIZE register however it does not appear to be
+ * valid. Now, docs explain in dwords what is in the context object. The full
+ * size is 70720 bytes, however, the power context and execlist context will
+ * never be saved (power context is stored elsewhere, and execlists don't work
+ * on HSW) - so the final size is 66944 bytes, which rounds to 17 pages.
+ */
+#define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
 
 /*
  * Overlay regs
@@ -1691,6 +1873,12 @@
 /* SDVO is different across gen3/4 */
 #define   SDVOC_HOTPLUG_INT_STATUS_G4X          (1 << 3)
 #define   SDVOB_HOTPLUG_INT_STATUS_G4X          (1 << 2)
+/*
+ * Bspec seems to be seriously misleaded about the SDVO hpd bits on i965g/gm,
+ * since reality corrobates that they're the same as on gen3. But keep these
+ * bits here (and the comment!) to help any other lost wanderers back onto the
+ * right tracks.
+ */
 #define   SDVOC_HOTPLUG_INT_STATUS_I965         (3 << 4)
 #define   SDVOB_HOTPLUG_INT_STATUS_I965         (3 << 2)
 #define   SDVOC_HOTPLUG_INT_STATUS_I915         (1 << 7)
@@ -1702,13 +1890,6 @@
                                                  PORTC_HOTPLUG_INT_STATUS | \
                                                  PORTD_HOTPLUG_INT_STATUS)
 
-#define HOTPLUG_INT_STATUS_I965                 (CRT_HOTPLUG_INT_STATUS | \
-                                                 SDVOB_HOTPLUG_INT_STATUS_I965 | \
-                                                 SDVOC_HOTPLUG_INT_STATUS_I965 | \
-                                                 PORTB_HOTPLUG_INT_STATUS | \
-                                                 PORTC_HOTPLUG_INT_STATUS | \
-                                                 PORTD_HOTPLUG_INT_STATUS)
-
 #define HOTPLUG_INT_STATUS_I915                 (CRT_HOTPLUG_INT_STATUS | \
                                                  SDVOB_HOTPLUG_INT_STATUS_I915 | \
                                                  SDVOC_HOTPLUG_INT_STATUS_I915 | \
@@ -1967,6 +2148,10 @@
 #define   BLM_PIPE_A                    (0 << 29)
 #define   BLM_PIPE_B                    (1 << 29)
 #define   BLM_PIPE_C                    (2 << 29) /* ivb + */
+#define   BLM_TRANSCODER_A              BLM_PIPE_A /* hsw */
+#define   BLM_TRANSCODER_B              BLM_PIPE_B
+#define   BLM_TRANSCODER_C              BLM_PIPE_C
+#define   BLM_TRANSCODER_EDP            (3 << 29)
 #define   BLM_PIPE(pipe)                ((pipe) << 29)
 #define   BLM_POLARITY_I965             (1 << 28) /* gen4 only */
 #define   BLM_PHASE_IN_INTERUPT_STATUS  (1 << 26)
@@ -2540,9 +2725,7 @@
 #define   DP_PRE_EMPHASIS_SHIFT         22
 
 /* How many wires to use. I guess 3 was too hard */
-#define   DP_PORT_WIDTH_1               (0 << 19)
-#define   DP_PORT_WIDTH_2               (1 << 19)
-#define   DP_PORT_WIDTH_4               (3 << 19)
+#define   DP_PORT_WIDTH(width)          (((width) - 1) << 19)
 #define   DP_PORT_WIDTH_MASK            (7 << 19)
 
 /* Mystic DPCD version 1.1 special mode */
@@ -2646,18 +2829,20 @@
  * which is after the LUTs, so we want the bytes for our color format.
  * For our current usage, this is always 3, one byte for R, G and B.
  */
-#define _PIPEA_GMCH_DATA_M                      0x70050
-#define _PIPEB_GMCH_DATA_M                      0x71050
+#define _PIPEA_DATA_M_G4X       0x70050
+#define _PIPEB_DATA_M_G4X       0x71050
 
 /* Transfer unit size for display port - 1, default is 0x3f (for TU size 64) */
 #define  TU_SIZE(x)             (((x)-1) << 25) /* default size 64 */
+#define  TU_SIZE_SHIFT          25
 #define  TU_SIZE_MASK           (0x3f << 25)
 
 #define  DATA_LINK_M_N_MASK     (0xffffff)
 #define  DATA_LINK_N_MAX        (0x800000)
 
-#define _PIPEA_GMCH_DATA_N                      0x70054
-#define _PIPEB_GMCH_DATA_N                      0x71054
+#define _PIPEA_DATA_N_G4X       0x70054
+#define _PIPEB_DATA_N_G4X       0x71054
+#define   PIPE_GMCH_DATA_N_MASK                 (0xffffff)
 
 /*
  * Computing Link M and N values for the Display Port link
@@ -2670,16 +2855,18 @@
  * Attributes and VB-ID.
  */
 
-#define _PIPEA_DP_LINK_M                                0x70060
-#define _PIPEB_DP_LINK_M                                0x71060
+#define _PIPEA_LINK_M_G4X       0x70060
+#define _PIPEB_LINK_M_G4X       0x71060
+#define   PIPEA_DP_LINK_M_MASK                  (0xffffff)
 
-#define _PIPEA_DP_LINK_N                                0x70064
-#define _PIPEB_DP_LINK_N                                0x71064
+#define _PIPEA_LINK_N_G4X       0x70064
+#define _PIPEB_LINK_N_G4X       0x71064
+#define   PIPEA_DP_LINK_N_MASK                  (0xffffff)
 
-#define PIPE_GMCH_DATA_M(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_M, _PIPEB_GMCH_DATA_M)
-#define PIPE_GMCH_DATA_N(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_N, _PIPEB_GMCH_DATA_N)
-#define PIPE_DP_LINK_M(pipe) _PIPE(pipe, _PIPEA_DP_LINK_M, _PIPEB_DP_LINK_M)
-#define PIPE_DP_LINK_N(pipe) _PIPE(pipe, _PIPEA_DP_LINK_N, _PIPEB_DP_LINK_N)
+#define PIPE_DATA_M_G4X(pipe) _PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X)
+#define PIPE_DATA_N_G4X(pipe) _PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X)
+#define PIPE_LINK_M_G4X(pipe) _PIPE(pipe, _PIPEA_LINK_M_G4X, _PIPEB_LINK_M_G4X)
+#define PIPE_LINK_N_G4X(pipe) _PIPE(pipe, _PIPEA_LINK_N_G4X, _PIPEB_LINK_N_G4X)
 
 /* Display & cursor control */
 
@@ -2715,6 +2902,7 @@
 #define   PIPECONF_INTERLACED_ILK               (3 << 21)
 #define   PIPECONF_INTERLACED_DBL_ILK           (4 << 21) /* ilk/snb only */
 #define   PIPECONF_PFIT_PF_INTERLACED_DBL_ILK   (5 << 21) /* ilk/snb only */
+#define   PIPECONF_INTERLACE_MODE_MASK          (7 << 21)
 #define   PIPECONF_CXSR_DOWNCLOCK       (1<<16)
 #define   PIPECONF_COLOR_RANGE_SELECT   (1 << 13)
 #define   PIPECONF_BPC_MASK     (0x7 << 5)
@@ -2915,6 +3103,10 @@
 #define WM3S_LP_IVB             0x45128
 #define  WM1S_LP_EN             (1<<31)
 
+#define HSW_WM_LP_VAL(lat, fbc, pri, cur) \
+        (WM3_LP_EN | ((lat) << WM1_LP_LATENCY_SHIFT) | \
+         ((fbc) << WM1_LP_FBC_SHIFT) | ((pri) << WM1_LP_SR_SHIFT) | (cur))
+
 /* Memory latency timer register */
 #define MLTR_ILK                0x11222
 #define  MLTR_WM1_SHIFT         0
@@ -3294,7 +3486,7 @@
 #define SPRGAMC(pipe) _PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC)
 #define SPRSURFLIVE(pipe) _PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE)
 
-#define _SPACNTR                0x72180
+#define _SPACNTR                (VLV_DISPLAY_BASE + 0x72180)
 #define   SP_ENABLE                     (1<<31)
 #define   SP_GEAMMA_ENABLE              (1<<30)
 #define   SP_PIXFORMAT_MASK             (0xf<<26)
@@ -3313,30 +3505,30 @@
 #define   SP_YUV_ORDER_YVYU             (2<<16)
 #define   SP_YUV_ORDER_VYUY             (3<<16)
 #define   SP_TILED                      (1<<10)
-#define _SPALINOFF              0x72184
-#define _SPASTRIDE              0x72188
-#define _SPAPOS                 0x7218c
-#define _SPASIZE                0x72190
-#define _SPAKEYMINVAL           0x72194
-#define _SPAKEYMSK              0x72198
-#define _SPASURF                0x7219c
-#define _SPAKEYMAXVAL           0x721a0
-#define _SPATILEOFF             0x721a4
-#define _SPACONSTALPHA          0x721a8
-#define _SPAGAMC                0x721f4
-
-#define _SPBCNTR                0x72280
-#define _SPBLINOFF              0x72284
-#define _SPBSTRIDE              0x72288
-#define _SPBPOS                 0x7228c
-#define _SPBSIZE                0x72290
-#define _SPBKEYMINVAL           0x72294
-#define _SPBKEYMSK              0x72298
-#define _SPBSURF                0x7229c
-#define _SPBKEYMAXVAL           0x722a0
-#define _SPBTILEOFF             0x722a4
-#define _SPBCONSTALPHA          0x722a8
-#define _SPBGAMC                0x722f4
+#define _SPALINOFF              (VLV_DISPLAY_BASE + 0x72184)
+#define _SPASTRIDE              (VLV_DISPLAY_BASE + 0x72188)
+#define _SPAPOS                 (VLV_DISPLAY_BASE + 0x7218c)
+#define _SPASIZE                (VLV_DISPLAY_BASE + 0x72190)
+#define _SPAKEYMINVAL           (VLV_DISPLAY_BASE + 0x72194)
+#define _SPAKEYMSK              (VLV_DISPLAY_BASE + 0x72198)
+#define _SPASURF                (VLV_DISPLAY_BASE + 0x7219c)
+#define _SPAKEYMAXVAL           (VLV_DISPLAY_BASE + 0x721a0)
+#define _SPATILEOFF             (VLV_DISPLAY_BASE + 0x721a4)
+#define _SPACONSTALPHA          (VLV_DISPLAY_BASE + 0x721a8)
+#define _SPAGAMC                (VLV_DISPLAY_BASE + 0x721f4)
+
+#define _SPBCNTR                (VLV_DISPLAY_BASE + 0x72280)
+#define _SPBLINOFF              (VLV_DISPLAY_BASE + 0x72284)
+#define _SPBSTRIDE              (VLV_DISPLAY_BASE + 0x72288)
+#define _SPBPOS                 (VLV_DISPLAY_BASE + 0x7228c)
+#define _SPBSIZE                (VLV_DISPLAY_BASE + 0x72290)
+#define _SPBKEYMINVAL           (VLV_DISPLAY_BASE + 0x72294)
+#define _SPBKEYMSK              (VLV_DISPLAY_BASE + 0x72298)
+#define _SPBSURF                (VLV_DISPLAY_BASE + 0x7229c)
+#define _SPBKEYMAXVAL           (VLV_DISPLAY_BASE + 0x722a0)
+#define _SPBTILEOFF             (VLV_DISPLAY_BASE + 0x722a4)
+#define _SPBCONSTALPHA          (VLV_DISPLAY_BASE + 0x722a8)
+#define _SPBGAMC                (VLV_DISPLAY_BASE + 0x722f4)
 
 #define SPCNTR(pipe, plane) _PIPE(pipe * 2 + plane, _SPACNTR, _SPBCNTR)
 #define SPLINOFF(pipe, plane) _PIPE(pipe * 2 + plane, _SPALINOFF, _SPBLINOFF)
@@ -3474,6 +3666,15 @@
 #define _LGC_PALETTE_B           0x4a800
 #define LGC_PALETTE(pipe) _PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B)
 
+#define _GAMMA_MODE_A           0x4a480
+#define _GAMMA_MODE_B           0x4ac80
+#define GAMMA_MODE(pipe) _PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B)
+#define GAMMA_MODE_MODE_MASK    (3 << 0)
+#define GAMMA_MODE_MODE_8BIT    (0 << 0)
+#define GAMMA_MODE_MODE_10BIT   (1 << 0)
+#define GAMMA_MODE_MODE_12BIT   (2 << 0)
+#define GAMMA_MODE_MODE_SPLIT   (3 << 0)
+
 /* interrupts */
 #define DE_MASTER_IRQ_CONTROL   (1 << 31)
 #define DE_SPRITEB_FLIP_DONE    (1 << 29)
@@ -3502,7 +3703,7 @@
 #define DE_PIPEA_FIFO_UNDERRUN  (1 << 0)
 
 /* More Ivybridge lolz */
-#define DE_ERR_DEBUG_IVB                (1<<30)
+#define DE_ERR_INT_IVB                  (1<<30)
 #define DE_GSE_IVB                      (1<<29)
 #define DE_PCH_EVENT_IVB                (1<<28)
 #define DE_DP_A_HOTPLUG_IVB             (1<<27)
@@ -3525,21 +3726,6 @@
 #define DEIIR   0x44008
 #define DEIER   0x4400c
 
-/* GT interrupt.
- * Note that for gen6+ the ring-specific interrupt bits do alias with the
- * corresponding bits in the per-ring interrupt control registers. */
-#define GT_GEN6_BLT_FLUSHDW_NOTIFY_INTERRUPT    (1 << 26)
-#define GT_GEN6_BLT_CS_ERROR_INTERRUPT          (1 << 25)
-#define GT_GEN6_BLT_USER_INTERRUPT              (1 << 22)
-#define GT_GEN6_BSD_CS_ERROR_INTERRUPT          (1 << 15)
-#define GT_GEN6_BSD_USER_INTERRUPT              (1 << 12)
-#define GT_BSD_USER_INTERRUPT                   (1 << 5) /* ilk only */
-#define GT_GEN7_L3_PARITY_ERROR_INTERRUPT       (1 << 5)
-#define GT_PIPE_NOTIFY                          (1 << 4)
-#define GT_RENDER_CS_ERROR_INTERRUPT            (1 << 3)
-#define GT_SYNC_STATUS                          (1 << 2)
-#define GT_USER_INTERRUPT                       (1 << 0)
-
 #define GTISR   0x44010
 #define GTIMR   0x44014
 #define GTIIR   0x44018
@@ -3569,6 +3755,9 @@
 # define CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE      (1 << 5)
 # define CHICKEN3_DGMG_DONE_FIX_DISABLE         (1 << 2)
 
+#define CHICKEN_PAR1_1          0x42080
+#define  FORCE_ARB_IDLE_PLANES  (1 << 14)
+
 #define DISP_ARB_CTL    0x45000
 #define  DISP_TILE_SURFACE_SWIZZLING    (1<<13)
 #define  DISP_FBC_WM_DIS                (1<<15)
@@ -3661,6 +3850,7 @@
                                  SDE_PORTC_HOTPLUG_CPT |        \
                                  SDE_PORTB_HOTPLUG_CPT)
 #define SDE_GMBUS_CPT           (1 << 17)
+#define SDE_ERROR_CPT           (1 << 16)
 #define SDE_AUDIO_CP_REQ_C_CPT  (1 << 10)
 #define SDE_AUDIO_CP_CHG_C_CPT  (1 << 9)
 #define SDE_FDI_RXC_CPT         (1 << 8)
@@ -3685,6 +3875,12 @@
 #define SDEIIR  0xc4008
 #define SDEIER  0xc400c
 
+#define SERR_INT                        0xc4040
+#define  SERR_INT_POISON                (1<<31)
+#define  SERR_INT_TRANS_C_FIFO_UNDERRUN (1<<6)
+#define  SERR_INT_TRANS_B_FIFO_UNDERRUN (1<<3)
+#define  SERR_INT_TRANS_A_FIFO_UNDERRUN (1<<0)
+
 /* digital port hotplug */
 #define PCH_PORT_HOTPLUG        0xc4030         /* SHOTPLUG_CTL */
 #define PORTD_HOTPLUG_ENABLE            (1 << 20)
@@ -3734,15 +3930,15 @@
 
 #define _PCH_DPLL_A              0xc6014
 #define _PCH_DPLL_B              0xc6018
-#define _PCH_DPLL(pll) (pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B)
+#define PCH_DPLL(pll) (pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B)
 
 #define _PCH_FPA0                0xc6040
 #define  FP_CB_TUNE             (0x3<<22)
 #define _PCH_FPA1                0xc6044
 #define _PCH_FPB0                0xc6048
 #define _PCH_FPB1                0xc604c
-#define _PCH_FP0(pll) (pll == 0 ? _PCH_FPA0 : _PCH_FPB0)
-#define _PCH_FP1(pll) (pll == 0 ? _PCH_FPA1 : _PCH_FPB1)
+#define PCH_FP0(pll) (pll == 0 ? _PCH_FPA0 : _PCH_FPB0)
+#define PCH_FP1(pll) (pll == 0 ? _PCH_FPA1 : _PCH_FPB1)
 
 #define PCH_DPLL_TEST           0xc606c
 
@@ -3782,46 +3978,40 @@
 #define PCH_SSC4_AUX_PARMS      0xc6214
 
 #define PCH_DPLL_SEL            0xc7000
-#define  TRANSA_DPLL_ENABLE     (1<<3)
-#define  TRANSA_DPLLB_SEL       (1<<0)
-#define  TRANSA_DPLLA_SEL       0
-#define  TRANSB_DPLL_ENABLE     (1<<7)
-#define  TRANSB_DPLLB_SEL       (1<<4)
-#define  TRANSB_DPLLA_SEL       (0)
-#define  TRANSC_DPLL_ENABLE     (1<<11)
-#define  TRANSC_DPLLB_SEL       (1<<8)
-#define  TRANSC_DPLLA_SEL       (0)
+#define  TRANS_DPLLB_SEL(pipe)          (1 << (pipe * 4))
+#define  TRANS_DPLLA_SEL(pipe)          0
+#define  TRANS_DPLL_ENABLE(pipe)        (1 << (pipe * 4 + 3))
 
 /* transcoder */
 
-#define _TRANS_HTOTAL_A          0xe0000
-#define  TRANS_HTOTAL_SHIFT     16
-#define  TRANS_HACTIVE_SHIFT    0
-#define _TRANS_HBLANK_A          0xe0004
-#define  TRANS_HBLANK_END_SHIFT 16
-#define  TRANS_HBLANK_START_SHIFT 0
-#define _TRANS_HSYNC_A           0xe0008
-#define  TRANS_HSYNC_END_SHIFT  16
-#define  TRANS_HSYNC_START_SHIFT 0
-#define _TRANS_VTOTAL_A          0xe000c
-#define  TRANS_VTOTAL_SHIFT     16
-#define  TRANS_VACTIVE_SHIFT    0
-#define _TRANS_VBLANK_A          0xe0010
-#define  TRANS_VBLANK_END_SHIFT 16
-#define  TRANS_VBLANK_START_SHIFT 0
-#define _TRANS_VSYNC_A           0xe0014
-#define  TRANS_VSYNC_END_SHIFT  16
-#define  TRANS_VSYNC_START_SHIFT 0
-#define _TRANS_VSYNCSHIFT_A     0xe0028
-
-#define _TRANSA_DATA_M1          0xe0030
-#define _TRANSA_DATA_N1          0xe0034
-#define _TRANSA_DATA_M2          0xe0038
-#define _TRANSA_DATA_N2          0xe003c
-#define _TRANSA_DP_LINK_M1       0xe0040
-#define _TRANSA_DP_LINK_N1       0xe0044
-#define _TRANSA_DP_LINK_M2       0xe0048
-#define _TRANSA_DP_LINK_N2       0xe004c
+#define _PCH_TRANS_HTOTAL_A             0xe0000
+#define  TRANS_HTOTAL_SHIFT             16
+#define  TRANS_HACTIVE_SHIFT            0
+#define _PCH_TRANS_HBLANK_A             0xe0004
+#define  TRANS_HBLANK_END_SHIFT         16
+#define  TRANS_HBLANK_START_SHIFT       0
+#define _PCH_TRANS_HSYNC_A              0xe0008
+#define  TRANS_HSYNC_END_SHIFT          16
+#define  TRANS_HSYNC_START_SHIFT        0
+#define _PCH_TRANS_VTOTAL_A             0xe000c
+#define  TRANS_VTOTAL_SHIFT             16
+#define  TRANS_VACTIVE_SHIFT            0
+#define _PCH_TRANS_VBLANK_A             0xe0010
+#define  TRANS_VBLANK_END_SHIFT         16
+#define  TRANS_VBLANK_START_SHIFT       0
+#define _PCH_TRANS_VSYNC_A              0xe0014
+#define  TRANS_VSYNC_END_SHIFT          16
+#define  TRANS_VSYNC_START_SHIFT        0
+#define _PCH_TRANS_VSYNCSHIFT_A         0xe0028
+
+#define _PCH_TRANSA_DATA_M1     0xe0030
+#define _PCH_TRANSA_DATA_N1     0xe0034
+#define _PCH_TRANSA_DATA_M2     0xe0038
+#define _PCH_TRANSA_DATA_N2     0xe003c
+#define _PCH_TRANSA_LINK_M1     0xe0040
+#define _PCH_TRANSA_LINK_N1     0xe0044
+#define _PCH_TRANSA_LINK_M2     0xe0048
+#define _PCH_TRANSA_LINK_N2     0xe004c
 
 /* Per-transcoder DIP controls */
 
@@ -3890,44 +4080,45 @@
 #define HSW_TVIDEO_DIP_VSC_DATA(trans) \
          _TRANSCODER(trans, HSW_VIDEO_DIP_VSC_DATA_A, HSW_VIDEO_DIP_VSC_DATA_B)
 
-#define _TRANS_HTOTAL_B          0xe1000
-#define _TRANS_HBLANK_B          0xe1004
-#define _TRANS_HSYNC_B           0xe1008
-#define _TRANS_VTOTAL_B          0xe100c
-#define _TRANS_VBLANK_B          0xe1010
-#define _TRANS_VSYNC_B           0xe1014
-#define _TRANS_VSYNCSHIFT_B      0xe1028
-
-#define TRANS_HTOTAL(pipe) _PIPE(pipe, _TRANS_HTOTAL_A, _TRANS_HTOTAL_B)
-#define TRANS_HBLANK(pipe) _PIPE(pipe, _TRANS_HBLANK_A, _TRANS_HBLANK_B)
-#define TRANS_HSYNC(pipe) _PIPE(pipe, _TRANS_HSYNC_A, _TRANS_HSYNC_B)
-#define TRANS_VTOTAL(pipe) _PIPE(pipe, _TRANS_VTOTAL_A, _TRANS_VTOTAL_B)
-#define TRANS_VBLANK(pipe) _PIPE(pipe, _TRANS_VBLANK_A, _TRANS_VBLANK_B)
-#define TRANS_VSYNC(pipe) _PIPE(pipe, _TRANS_VSYNC_A, _TRANS_VSYNC_B)
-#define TRANS_VSYNCSHIFT(pipe) _PIPE(pipe, _TRANS_VSYNCSHIFT_A, \
-                                     _TRANS_VSYNCSHIFT_B)
-
-#define _TRANSB_DATA_M1          0xe1030
-#define _TRANSB_DATA_N1          0xe1034
-#define _TRANSB_DATA_M2          0xe1038
-#define _TRANSB_DATA_N2          0xe103c
-#define _TRANSB_DP_LINK_M1       0xe1040
-#define _TRANSB_DP_LINK_N1       0xe1044
-#define _TRANSB_DP_LINK_M2       0xe1048
-#define _TRANSB_DP_LINK_N2       0xe104c
-
-#define TRANSDATA_M1(pipe) _PIPE(pipe, _TRANSA_DATA_M1, _TRANSB_DATA_M1)
-#define TRANSDATA_N1(pipe) _PIPE(pipe, _TRANSA_DATA_N1, _TRANSB_DATA_N1)
-#define TRANSDATA_M2(pipe) _PIPE(pipe, _TRANSA_DATA_M2, _TRANSB_DATA_M2)
-#define TRANSDATA_N2(pipe) _PIPE(pipe, _TRANSA_DATA_N2, _TRANSB_DATA_N2)
-#define TRANSDPLINK_M1(pipe) _PIPE(pipe, _TRANSA_DP_LINK_M1, _TRANSB_DP_LINK_M1)
-#define TRANSDPLINK_N1(pipe) _PIPE(pipe, _TRANSA_DP_LINK_N1, _TRANSB_DP_LINK_N1)
-#define TRANSDPLINK_M2(pipe) _PIPE(pipe, _TRANSA_DP_LINK_M2, _TRANSB_DP_LINK_M2)
-#define TRANSDPLINK_N2(pipe) _PIPE(pipe, _TRANSA_DP_LINK_N2, _TRANSB_DP_LINK_N2)
-
-#define _TRANSACONF              0xf0008
-#define _TRANSBCONF              0xf1008
-#define TRANSCONF(plane) _PIPE(plane, _TRANSACONF, _TRANSBCONF)
+#define _PCH_TRANS_HTOTAL_B          0xe1000
+#define _PCH_TRANS_HBLANK_B          0xe1004
+#define _PCH_TRANS_HSYNC_B           0xe1008
+#define _PCH_TRANS_VTOTAL_B          0xe100c
+#define _PCH_TRANS_VBLANK_B          0xe1010
+#define _PCH_TRANS_VSYNC_B           0xe1014
+#define _PCH_TRANS_VSYNCSHIFT_B  0xe1028
+
+#define PCH_TRANS_HTOTAL(pipe) _PIPE(pipe, _PCH_TRANS_HTOTAL_A, _PCH_TRANS_HTOTAL_B)
+#define PCH_TRANS_HBLANK(pipe) _PIPE(pipe, _PCH_TRANS_HBLANK_A, _PCH_TRANS_HBLANK_B)
+#define PCH_TRANS_HSYNC(pipe) _PIPE(pipe, _PCH_TRANS_HSYNC_A, _PCH_TRANS_HSYNC_B)
+#define PCH_TRANS_VTOTAL(pipe) _PIPE(pipe, _PCH_TRANS_VTOTAL_A, _PCH_TRANS_VTOTAL_B)
+#define PCH_TRANS_VBLANK(pipe) _PIPE(pipe, _PCH_TRANS_VBLANK_A, _PCH_TRANS_VBLANK_B)
+#define PCH_TRANS_VSYNC(pipe) _PIPE(pipe, _PCH_TRANS_VSYNC_A, _PCH_TRANS_VSYNC_B)
+#define PCH_TRANS_VSYNCSHIFT(pipe) _PIPE(pipe, _PCH_TRANS_VSYNCSHIFT_A, \
+                                         _PCH_TRANS_VSYNCSHIFT_B)
+
+#define _PCH_TRANSB_DATA_M1     0xe1030
+#define _PCH_TRANSB_DATA_N1     0xe1034
+#define _PCH_TRANSB_DATA_M2     0xe1038
+#define _PCH_TRANSB_DATA_N2     0xe103c
+#define _PCH_TRANSB_LINK_M1     0xe1040
+#define _PCH_TRANSB_LINK_N1     0xe1044
+#define _PCH_TRANSB_LINK_M2     0xe1048
+#define _PCH_TRANSB_LINK_N2     0xe104c
+
+#define PCH_TRANS_DATA_M1(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_M1, _PCH_TRANSB_DATA_M1)
+#define PCH_TRANS_DATA_N1(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_N1, _PCH_TRANSB_DATA_N1)
+#define PCH_TRANS_DATA_M2(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_M2, _PCH_TRANSB_DATA_M2)
+#define PCH_TRANS_DATA_N2(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_N2, _PCH_TRANSB_DATA_N2)
+#define PCH_TRANS_LINK_M1(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_M1, _PCH_TRANSB_LINK_M1)
+#define PCH_TRANS_LINK_N1(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_N1, _PCH_TRANSB_LINK_N1)
+#define PCH_TRANS_LINK_M2(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_M2, _PCH_TRANSB_LINK_M2)
+#define PCH_TRANS_LINK_N2(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_N2, _PCH_TRANSB_LINK_N2)
+
+#define _PCH_TRANSACONF              0xf0008
+#define _PCH_TRANSBCONF              0xf1008
+#define PCH_TRANSCONF(pipe) _PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF)
+#define LPT_TRANSCONF           _PCH_TRANSACONF /* lpt has only one transcoder */
 #define  TRANS_DISABLE          (0<<31)
 #define  TRANS_ENABLE           (1<<31)
 #define  TRANS_STATE_MASK       (1<<30)
@@ -4011,10 +4202,9 @@
 #define  FDI_LINK_TRAIN_600MV_3_5DB_SNB_B       (0x39<<22)
 #define  FDI_LINK_TRAIN_800MV_0DB_SNB_B         (0x38<<22)
 #define  FDI_LINK_TRAIN_VOL_EMP_MASK            (0x3f<<22)
-#define  FDI_DP_PORT_WIDTH_X1           (0<<19)
-#define  FDI_DP_PORT_WIDTH_X2           (1<<19)
-#define  FDI_DP_PORT_WIDTH_X3           (2<<19)
-#define  FDI_DP_PORT_WIDTH_X4           (3<<19)
+#define  FDI_DP_PORT_WIDTH_SHIFT                19
+#define  FDI_DP_PORT_WIDTH_MASK                 (7 << FDI_DP_PORT_WIDTH_SHIFT)
+#define  FDI_DP_PORT_WIDTH(width)           (((width) - 1) << FDI_DP_PORT_WIDTH_SHIFT)
 #define  FDI_TX_ENHANCE_FRAME_ENABLE    (1<<18)
 /* Ironlake: hardwired to 1 */
 #define  FDI_TX_PLL_ENABLE              (1<<14)
@@ -4039,7 +4229,6 @@
 /* train, dp width same as FDI_TX */
 #define  FDI_FS_ERRC_ENABLE             (1<<27)
 #define  FDI_FE_ERRC_ENABLE             (1<<26)
-#define  FDI_DP_PORT_WIDTH_X8           (7<<19)
 #define  FDI_RX_POLARITY_REVERSED_LPT   (1<<16)
 #define  FDI_8BPC                       (0<<16)
 #define  FDI_10BPC                      (1<<16)
@@ -4061,9 +4250,6 @@
 #define  FDI_LINK_TRAIN_PATTERN_IDLE_CPT        (2<<8)
 #define  FDI_LINK_TRAIN_NORMAL_CPT              (3<<8)
 #define  FDI_LINK_TRAIN_PATTERN_MASK_CPT        (3<<8)
-/* LPT */
-#define  FDI_PORT_WIDTH_2X_LPT                  (1<<19)
-#define  FDI_PORT_WIDTH_1X_LPT                  (0<<19)
 
 #define _FDI_RXA_MISC                   0xf0010
 #define _FDI_RXB_MISC                   0xf1010
@@ -4309,6 +4495,7 @@
 #define   GEN6_RC_CTL_RC6_ENABLE                (1<<18)
 #define   GEN6_RC_CTL_RC1e_ENABLE               (1<<20)
 #define   GEN6_RC_CTL_RC7_ENABLE                (1<<22)
+#define   GEN7_RC_CTL_TO_MODE                   (1<<28)
 #define   GEN6_RC_CTL_EI_MODE(x)                ((x)<<27)
 #define   GEN6_RC_CTL_HW_ENABLE                 (1<<31)
 #define GEN6_RP_DOWN_TIMEOUT                    0xA010
@@ -4370,7 +4557,7 @@
 #define  GEN6_PM_RP_DOWN_THRESHOLD              (1<<4)
 #define  GEN6_PM_RP_UP_EI_EXPIRED               (1<<2)
 #define  GEN6_PM_RP_DOWN_EI_EXPIRED             (1<<1)
-#define  GEN6_PM_DEFERRED_EVENTS                (GEN6_PM_RP_UP_THRESHOLD | \
+#define  GEN6_PM_RPS_EVENTS                     (GEN6_PM_RP_UP_THRESHOLD | \
                                                  GEN6_PM_RP_DOWN_THRESHOLD | \
                                                  GEN6_PM_RP_DOWN_TIMEOUT)
 
@@ -4392,20 +4579,6 @@
 #define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT        8
 #define   GEN6_PCODE_FREQ_RING_RATIO_SHIFT      16
 
-#define VLV_IOSF_DOORBELL_REQ                   0x182100
-#define   IOSF_DEVFN_SHIFT                      24
-#define   IOSF_OPCODE_SHIFT                     16
-#define   IOSF_PORT_SHIFT                       8
-#define   IOSF_BYTE_ENABLES_SHIFT               4
-#define   IOSF_BAR_SHIFT                        1
-#define   IOSF_SB_BUSY                          (1<<0)
-#define   IOSF_PORT_PUNIT                       0x4
-#define VLV_IOSF_DATA                           0x182104
-#define VLV_IOSF_ADDR                           0x182108
-
-#define PUNIT_OPCODE_REG_READ                   6
-#define PUNIT_OPCODE_REG_WRITE                  7
-
 #define GEN6_GT_CORE_STATUS             0x138060
 #define   GEN6_CORE_CPD_STATE_MASK      (7<<4)
 #define   GEN6_RCn_MASK                 7
@@ -4602,9 +4775,6 @@
 #define  TRANS_DDI_EDP_INPUT_B_ONOFF    (5<<12)
 #define  TRANS_DDI_EDP_INPUT_C_ONOFF    (6<<12)
 #define  TRANS_DDI_BFI_ENABLE           (1<<4)
-#define  TRANS_DDI_PORT_WIDTH_X1        (0<<1)
-#define  TRANS_DDI_PORT_WIDTH_X2        (1<<1)
-#define  TRANS_DDI_PORT_WIDTH_X4        (3<<1)
 
 /* DisplayPort Transport Control */
 #define DP_TP_CTL_A                     0x64040
@@ -4648,9 +4818,7 @@
 #define  DDI_BUF_PORT_REVERSAL                  (1<<16)
 #define  DDI_BUF_IS_IDLE                        (1<<7)
 #define  DDI_A_4_LANES                          (1<<4)
-#define  DDI_PORT_WIDTH_X1                      (0<<1)
-#define  DDI_PORT_WIDTH_X2                      (1<<1)
-#define  DDI_PORT_WIDTH_X4                      (3<<1)
+#define  DDI_PORT_WIDTH(width)                  (((width) - 1) << 1)
 #define  DDI_INIT_DISPLAY_DETECTED              (1<<0)
 
 /* DDI Buffer Translations */
@@ -4774,6 +4942,9 @@
 #define  SFUSE_STRAP_DDIC_DETECTED      (1<<1)
 #define  SFUSE_STRAP_DDID_DETECTED      (1<<0)
 
+#define WM_MISC                         0x45260
+#define  WM_MISC_DATA_PARTITION_5_6     (1 << 0)
+
 #define WM_DBG                          0x45280
 #define  WM_DBG_DISALLOW_MULTIPLE_LP    (1<<0)
 #define  WM_DBG_DISALLOW_MAXFIFO        (1<<1)
@@ -4787,6 +4958,9 @@
 #define _PIPE_A_CSC_COEFF_RV_GV 0x49020
 #define _PIPE_A_CSC_COEFF_BV    0x49024
 #define _PIPE_A_CSC_MODE        0x49028
+#define   CSC_BLACK_SCREEN_OFFSET       (1 << 2)
+#define   CSC_POSITION_BEFORE_GAMMA     (1 << 1)
+#define   CSC_MODE_YUV_TO_RGB           (1 << 0)
 #define _PIPE_A_CSC_PREOFF_HI   0x49030
 #define _PIPE_A_CSC_PREOFF_ME   0x49034
 #define _PIPE_A_CSC_PREOFF_LO   0x49038
@@ -4808,10 +4982,6 @@
 #define _PIPE_B_CSC_POSTOFF_ME  0x49144
 #define _PIPE_B_CSC_POSTOFF_LO  0x49148
 
-#define CSC_BLACK_SCREEN_OFFSET (1 << 2)
-#define CSC_POSITION_BEFORE_GAMMA (1 << 1)
-#define CSC_MODE_YUV_TO_RGB (1 << 0)
-
 #define PIPE_CSC_COEFF_RY_GY(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RY_GY, _PIPE_B_CSC_COEFF_RY_GY)
 #define PIPE_CSC_COEFF_BY(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BY, _PIPE_B_CSC_COEFF_BY)
 #define PIPE_CSC_COEFF_RU_GU(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RU_GU, _PIPE_B_CSC_COEFF_RU_GU)
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 41f0fdecfbdc..88b9a663944f 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -192,6 +192,7 @@ static void i915_restore_vga(struct drm_device *dev)
 static void i915_save_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
 
         /* Display arbitration control */
         if (INTEL_INFO(dev)->gen <= 4)
@@ -202,6 +203,8 @@ static void i915_save_display(struct drm_device *dev)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_save_display_reg(dev);
 
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+
         /* LVDS state */
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->regfile.savePP_CONTROL = I915_READ(PCH_PP_CONTROL);
@@ -222,6 +225,8 @@ static void i915_save_display(struct drm_device *dev)
                         dev_priv->regfile.saveLVDS = I915_READ(LVDS);
         }
 
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+
         if (!IS_I830(dev) && !IS_845G(dev) && !HAS_PCH_SPLIT(dev))
                 dev_priv->regfile.savePFIT_CONTROL = I915_READ(PFIT_CONTROL);
 
@@ -257,6 +262,7 @@ static void i915_restore_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 mask = 0xffffffff;
+        unsigned long flags;
 
         /* Display arbitration */
         if (INTEL_INFO(dev)->gen <= 4)
@@ -265,6 +271,8 @@ static void i915_restore_display(struct drm_device *dev)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_restore_display_reg(dev);
 
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+
         /* LVDS state */
         if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev))
                 I915_WRITE(BLC_PWM_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2);
@@ -304,6 +312,8 @@ static void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(PP_CONTROL, dev_priv->regfile.savePP_CONTROL);
         }
 
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+
         /* only restore FBC info on the platform that supports FBC*/
         intel_disable_fbc(dev);
         if (I915_HAS_FBC(dev)) {
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index d5e1890678f9..6875b5654c63 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -212,7 +212,13 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
         int ret;
 
         mutex_lock(&dev_priv->rps.hw_lock);
-        ret = dev_priv->rps.cur_delay * GT_FREQUENCY_MULTIPLIER;
+        if (IS_VALLEYVIEW(dev_priv->dev)) {
+                u32 freq;
+                freq = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
+                ret = vlv_gpu_freq(dev_priv->mem_freq, (freq >> 8) & 0xff);
+        } else {
+                ret = dev_priv->rps.cur_delay * GT_FREQUENCY_MULTIPLIER;
+        }
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", ret);
@@ -226,7 +232,10 @@ static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute
         int ret;
 
         mutex_lock(&dev_priv->rps.hw_lock);
-        ret = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
+        if (IS_VALLEYVIEW(dev_priv->dev))
+                ret = vlv_gpu_freq(dev_priv->mem_freq, dev_priv->rps.max_delay);
+        else
+                ret = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", ret);
@@ -246,16 +255,25 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
         if (ret)
                 return ret;
 
-        val /= GT_FREQUENCY_MULTIPLIER;
-
         mutex_lock(&dev_priv->rps.hw_lock);
 
-        rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
-        hw_max = dev_priv->rps.hw_max;
-        non_oc_max = (rp_state_cap & 0xff);
-        hw_min = ((rp_state_cap & 0xff0000) >> 16);
+        if (IS_VALLEYVIEW(dev_priv->dev)) {
+                val = vlv_freq_opcode(dev_priv->mem_freq, val);
+
+                hw_max = valleyview_rps_max_freq(dev_priv);
+                hw_min = valleyview_rps_min_freq(dev_priv);
+                non_oc_max = hw_max;
+        } else {
+                val /= GT_FREQUENCY_MULTIPLIER;
+
+                rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+                hw_max = dev_priv->rps.hw_max;
+                non_oc_max = (rp_state_cap & 0xff);
+                hw_min = ((rp_state_cap & 0xff0000) >> 16);
+        }
 
-        if (val < hw_min || val > hw_max || val < dev_priv->rps.min_delay) {
+        if (val < hw_min || val > hw_max ||
+            val < dev_priv->rps.min_delay) {
                 mutex_unlock(&dev_priv->rps.hw_lock);
                 return -EINVAL;
         }
@@ -264,8 +282,12 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
                 DRM_DEBUG("User requested overclocking to %d\n",
                           val * GT_FREQUENCY_MULTIPLIER);
 
-        if (dev_priv->rps.cur_delay > val)
-                gen6_set_rps(dev_priv->dev, val);
+        if (dev_priv->rps.cur_delay > val) {
+                if (IS_VALLEYVIEW(dev_priv->dev))
+                        valleyview_set_rps(dev_priv->dev, val);
+                else
+                        gen6_set_rps(dev_priv->dev, val);
+        }
 
         dev_priv->rps.max_delay = val;
 
@@ -282,7 +304,10 @@ static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute
         int ret;
 
         mutex_lock(&dev_priv->rps.hw_lock);
-        ret = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
+        if (IS_VALLEYVIEW(dev_priv->dev))
+                ret = vlv_gpu_freq(dev_priv->mem_freq, dev_priv->rps.min_delay);
+        else
+                ret = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", ret);
@@ -302,21 +327,32 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
         if (ret)
                 return ret;
 
-        val /= GT_FREQUENCY_MULTIPLIER;
-
         mutex_lock(&dev_priv->rps.hw_lock);
 
-        rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
-        hw_max = dev_priv->rps.hw_max;
-        hw_min = ((rp_state_cap & 0xff0000) >> 16);
+        if (IS_VALLEYVIEW(dev)) {
+                val = vlv_freq_opcode(dev_priv->mem_freq, val);
+
+                hw_max = valleyview_rps_max_freq(dev_priv);
+                hw_min = valleyview_rps_min_freq(dev_priv);
+        } else {
+                val /= GT_FREQUENCY_MULTIPLIER;
+
+                rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+                hw_max = dev_priv->rps.hw_max;
+                hw_min = ((rp_state_cap & 0xff0000) >> 16);
+        }
 
         if (val < hw_min || val > hw_max || val > dev_priv->rps.max_delay) {
                 mutex_unlock(&dev_priv->rps.hw_lock);
                 return -EINVAL;
         }
 
-        if (dev_priv->rps.cur_delay < val)
-                gen6_set_rps(dev_priv->dev, val);
+        if (dev_priv->rps.cur_delay < val) {
+                if (IS_VALLEYVIEW(dev))
+                        valleyview_set_rps(dev, val);
+                else
+                        gen6_set_rps(dev_priv->dev, val);
+        }
 
         dev_priv->rps.min_delay = val;
 
diff --git a/drivers/gpu/drm/i915/i915_ums.c b/drivers/gpu/drm/i915/i915_ums.c
index 985a09716237..967da4772c44 100644
--- a/drivers/gpu/drm/i915/i915_ums.c
+++ b/drivers/gpu/drm/i915/i915_ums.c
@@ -41,7 +41,7 @@ static bool i915_pipe_enabled(struct drm_device *dev, enum pipe pipe)
                 return false;
 
         if (HAS_PCH_SPLIT(dev))
-                dpll_reg = _PCH_DPLL(pipe);
+                dpll_reg = PCH_DPLL(pipe);
         else
                 dpll_reg = (pipe == PIPE_A) ? _DPLL_A : _DPLL_B;
 
@@ -148,13 +148,13 @@ void i915_save_display_reg(struct drm_device *dev)
                 dev_priv->regfile.savePFA_WIN_SZ = I915_READ(_PFA_WIN_SZ);
                 dev_priv->regfile.savePFA_WIN_POS = I915_READ(_PFA_WIN_POS);
 
-                dev_priv->regfile.saveTRANSACONF = I915_READ(_TRANSACONF);
-                dev_priv->regfile.saveTRANS_HTOTAL_A = I915_READ(_TRANS_HTOTAL_A);
-                dev_priv->regfile.saveTRANS_HBLANK_A = I915_READ(_TRANS_HBLANK_A);
-                dev_priv->regfile.saveTRANS_HSYNC_A = I915_READ(_TRANS_HSYNC_A);
-                dev_priv->regfile.saveTRANS_VTOTAL_A = I915_READ(_TRANS_VTOTAL_A);
-                dev_priv->regfile.saveTRANS_VBLANK_A = I915_READ(_TRANS_VBLANK_A);
-                dev_priv->regfile.saveTRANS_VSYNC_A = I915_READ(_TRANS_VSYNC_A);
+                dev_priv->regfile.saveTRANSACONF = I915_READ(_PCH_TRANSACONF);
+                dev_priv->regfile.saveTRANS_HTOTAL_A = I915_READ(_PCH_TRANS_HTOTAL_A);
+                dev_priv->regfile.saveTRANS_HBLANK_A = I915_READ(_PCH_TRANS_HBLANK_A);
+                dev_priv->regfile.saveTRANS_HSYNC_A = I915_READ(_PCH_TRANS_HSYNC_A);
+                dev_priv->regfile.saveTRANS_VTOTAL_A = I915_READ(_PCH_TRANS_VTOTAL_A);
+                dev_priv->regfile.saveTRANS_VBLANK_A = I915_READ(_PCH_TRANS_VBLANK_A);
+                dev_priv->regfile.saveTRANS_VSYNC_A = I915_READ(_PCH_TRANS_VSYNC_A);
         }
 
         dev_priv->regfile.saveDSPACNTR = I915_READ(_DSPACNTR);
@@ -205,13 +205,13 @@ void i915_save_display_reg(struct drm_device *dev)
                 dev_priv->regfile.savePFB_WIN_SZ = I915_READ(_PFB_WIN_SZ);
                 dev_priv->regfile.savePFB_WIN_POS = I915_READ(_PFB_WIN_POS);
 
-                dev_priv->regfile.saveTRANSBCONF = I915_READ(_TRANSBCONF);
-                dev_priv->regfile.saveTRANS_HTOTAL_B = I915_READ(_TRANS_HTOTAL_B);
-                dev_priv->regfile.saveTRANS_HBLANK_B = I915_READ(_TRANS_HBLANK_B);
-                dev_priv->regfile.saveTRANS_HSYNC_B = I915_READ(_TRANS_HSYNC_B);
-                dev_priv->regfile.saveTRANS_VTOTAL_B = I915_READ(_TRANS_VTOTAL_B);
-                dev_priv->regfile.saveTRANS_VBLANK_B = I915_READ(_TRANS_VBLANK_B);
-                dev_priv->regfile.saveTRANS_VSYNC_B = I915_READ(_TRANS_VSYNC_B);
+                dev_priv->regfile.saveTRANSBCONF = I915_READ(_PCH_TRANSBCONF);
+                dev_priv->regfile.saveTRANS_HTOTAL_B = I915_READ(_PCH_TRANS_HTOTAL_B);
+                dev_priv->regfile.saveTRANS_HBLANK_B = I915_READ(_PCH_TRANS_HBLANK_B);
+                dev_priv->regfile.saveTRANS_HSYNC_B = I915_READ(_PCH_TRANS_HSYNC_B);
+                dev_priv->regfile.saveTRANS_VTOTAL_B = I915_READ(_PCH_TRANS_VTOTAL_B);
+                dev_priv->regfile.saveTRANS_VBLANK_B = I915_READ(_PCH_TRANS_VBLANK_B);
+                dev_priv->regfile.saveTRANS_VSYNC_B = I915_READ(_PCH_TRANS_VSYNC_B);
         }
 
         dev_priv->regfile.saveDSPBCNTR = I915_READ(_DSPBCNTR);
@@ -259,14 +259,14 @@ void i915_save_display_reg(struct drm_device *dev)
                 dev_priv->regfile.saveDP_B = I915_READ(DP_B);
                 dev_priv->regfile.saveDP_C = I915_READ(DP_C);
                 dev_priv->regfile.saveDP_D = I915_READ(DP_D);
-                dev_priv->regfile.savePIPEA_GMCH_DATA_M = I915_READ(_PIPEA_GMCH_DATA_M);
-                dev_priv->regfile.savePIPEB_GMCH_DATA_M = I915_READ(_PIPEB_GMCH_DATA_M);
-                dev_priv->regfile.savePIPEA_GMCH_DATA_N = I915_READ(_PIPEA_GMCH_DATA_N);
-                dev_priv->regfile.savePIPEB_GMCH_DATA_N = I915_READ(_PIPEB_GMCH_DATA_N);
-                dev_priv->regfile.savePIPEA_DP_LINK_M = I915_READ(_PIPEA_DP_LINK_M);
-                dev_priv->regfile.savePIPEB_DP_LINK_M = I915_READ(_PIPEB_DP_LINK_M);
-                dev_priv->regfile.savePIPEA_DP_LINK_N = I915_READ(_PIPEA_DP_LINK_N);
-                dev_priv->regfile.savePIPEB_DP_LINK_N = I915_READ(_PIPEB_DP_LINK_N);
+                dev_priv->regfile.savePIPEA_GMCH_DATA_M = I915_READ(_PIPEA_DATA_M_G4X);
+                dev_priv->regfile.savePIPEB_GMCH_DATA_M = I915_READ(_PIPEB_DATA_M_G4X);
+                dev_priv->regfile.savePIPEA_GMCH_DATA_N = I915_READ(_PIPEA_DATA_N_G4X);
+                dev_priv->regfile.savePIPEB_GMCH_DATA_N = I915_READ(_PIPEB_DATA_N_G4X);
+                dev_priv->regfile.savePIPEA_DP_LINK_M = I915_READ(_PIPEA_LINK_M_G4X);
+                dev_priv->regfile.savePIPEB_DP_LINK_M = I915_READ(_PIPEB_LINK_M_G4X);
+                dev_priv->regfile.savePIPEA_DP_LINK_N = I915_READ(_PIPEA_LINK_N_G4X);
+                dev_priv->regfile.savePIPEB_DP_LINK_N = I915_READ(_PIPEB_LINK_N_G4X);
         }
         /* FIXME: regfile.save TV & SDVO state */
 
@@ -282,14 +282,14 @@ void i915_restore_display_reg(struct drm_device *dev)
 
         /* Display port ratios (must be done before clock is set) */
         if (SUPPORTS_INTEGRATED_DP(dev)) {
-                I915_WRITE(_PIPEA_GMCH_DATA_M, dev_priv->regfile.savePIPEA_GMCH_DATA_M);
-                I915_WRITE(_PIPEB_GMCH_DATA_M, dev_priv->regfile.savePIPEB_GMCH_DATA_M);
-                I915_WRITE(_PIPEA_GMCH_DATA_N, dev_priv->regfile.savePIPEA_GMCH_DATA_N);
-                I915_WRITE(_PIPEB_GMCH_DATA_N, dev_priv->regfile.savePIPEB_GMCH_DATA_N);
-                I915_WRITE(_PIPEA_DP_LINK_M, dev_priv->regfile.savePIPEA_DP_LINK_M);
-                I915_WRITE(_PIPEB_DP_LINK_M, dev_priv->regfile.savePIPEB_DP_LINK_M);
-                I915_WRITE(_PIPEA_DP_LINK_N, dev_priv->regfile.savePIPEA_DP_LINK_N);
-                I915_WRITE(_PIPEB_DP_LINK_N, dev_priv->regfile.savePIPEB_DP_LINK_N);
+                I915_WRITE(_PIPEA_DATA_M_G4X, dev_priv->regfile.savePIPEA_GMCH_DATA_M);
+                I915_WRITE(_PIPEB_DATA_M_G4X, dev_priv->regfile.savePIPEB_GMCH_DATA_M);
+                I915_WRITE(_PIPEA_DATA_N_G4X, dev_priv->regfile.savePIPEA_GMCH_DATA_N);
+                I915_WRITE(_PIPEB_DATA_N_G4X, dev_priv->regfile.savePIPEB_GMCH_DATA_N);
+                I915_WRITE(_PIPEA_LINK_M_G4X, dev_priv->regfile.savePIPEA_DP_LINK_M);
+                I915_WRITE(_PIPEB_LINK_M_G4X, dev_priv->regfile.savePIPEB_DP_LINK_M);
+                I915_WRITE(_PIPEA_LINK_N_G4X, dev_priv->regfile.savePIPEA_DP_LINK_N);
+                I915_WRITE(_PIPEB_LINK_N_G4X, dev_priv->regfile.savePIPEB_DP_LINK_N);
         }
 
         /* Fences */
@@ -379,13 +379,13 @@ void i915_restore_display_reg(struct drm_device *dev)
                 I915_WRITE(_PFA_WIN_SZ, dev_priv->regfile.savePFA_WIN_SZ);
                 I915_WRITE(_PFA_WIN_POS, dev_priv->regfile.savePFA_WIN_POS);
 
-                I915_WRITE(_TRANSACONF, dev_priv->regfile.saveTRANSACONF);
-                I915_WRITE(_TRANS_HTOTAL_A, dev_priv->regfile.saveTRANS_HTOTAL_A);
-                I915_WRITE(_TRANS_HBLANK_A, dev_priv->regfile.saveTRANS_HBLANK_A);
-                I915_WRITE(_TRANS_HSYNC_A, dev_priv->regfile.saveTRANS_HSYNC_A);
-                I915_WRITE(_TRANS_VTOTAL_A, dev_priv->regfile.saveTRANS_VTOTAL_A);
-                I915_WRITE(_TRANS_VBLANK_A, dev_priv->regfile.saveTRANS_VBLANK_A);
-                I915_WRITE(_TRANS_VSYNC_A, dev_priv->regfile.saveTRANS_VSYNC_A);
+                I915_WRITE(_PCH_TRANSACONF, dev_priv->regfile.saveTRANSACONF);
+                I915_WRITE(_PCH_TRANS_HTOTAL_A, dev_priv->regfile.saveTRANS_HTOTAL_A);
+                I915_WRITE(_PCH_TRANS_HBLANK_A, dev_priv->regfile.saveTRANS_HBLANK_A);
+                I915_WRITE(_PCH_TRANS_HSYNC_A, dev_priv->regfile.saveTRANS_HSYNC_A);
+                I915_WRITE(_PCH_TRANS_VTOTAL_A, dev_priv->regfile.saveTRANS_VTOTAL_A);
+                I915_WRITE(_PCH_TRANS_VBLANK_A, dev_priv->regfile.saveTRANS_VBLANK_A);
+                I915_WRITE(_PCH_TRANS_VSYNC_A, dev_priv->regfile.saveTRANS_VSYNC_A);
         }
 
         /* Restore plane info */
@@ -448,13 +448,13 @@ void i915_restore_display_reg(struct drm_device *dev)
                 I915_WRITE(_PFB_WIN_SZ, dev_priv->regfile.savePFB_WIN_SZ);
                 I915_WRITE(_PFB_WIN_POS, dev_priv->regfile.savePFB_WIN_POS);
 
-                I915_WRITE(_TRANSBCONF, dev_priv->regfile.saveTRANSBCONF);
-                I915_WRITE(_TRANS_HTOTAL_B, dev_priv->regfile.saveTRANS_HTOTAL_B);
-                I915_WRITE(_TRANS_HBLANK_B, dev_priv->regfile.saveTRANS_HBLANK_B);
-                I915_WRITE(_TRANS_HSYNC_B, dev_priv->regfile.saveTRANS_HSYNC_B);
-                I915_WRITE(_TRANS_VTOTAL_B, dev_priv->regfile.saveTRANS_VTOTAL_B);
-                I915_WRITE(_TRANS_VBLANK_B, dev_priv->regfile.saveTRANS_VBLANK_B);
-                I915_WRITE(_TRANS_VSYNC_B, dev_priv->regfile.saveTRANS_VSYNC_B);
+                I915_WRITE(_PCH_TRANSBCONF, dev_priv->regfile.saveTRANSBCONF);
+                I915_WRITE(_PCH_TRANS_HTOTAL_B, dev_priv->regfile.saveTRANS_HTOTAL_B);
+                I915_WRITE(_PCH_TRANS_HBLANK_B, dev_priv->regfile.saveTRANS_HBLANK_B);
+                I915_WRITE(_PCH_TRANS_HSYNC_B, dev_priv->regfile.saveTRANS_HSYNC_B);
+                I915_WRITE(_PCH_TRANS_VTOTAL_B, dev_priv->regfile.saveTRANS_VTOTAL_B);
+                I915_WRITE(_PCH_TRANS_VBLANK_B, dev_priv->regfile.saveTRANS_VBLANK_B);
+                I915_WRITE(_PCH_TRANS_VSYNC_B, dev_priv->regfile.saveTRANS_VSYNC_B);
         }
 
         /* Restore plane info */
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 95070b2124c6..53f2bed8bc5f 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -212,7 +212,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
         if (!lvds_options)
                 return;
 
-        dev_priv->lvds_dither = lvds_options->pixel_dither;
+        dev_priv->vbt.lvds_dither = lvds_options->pixel_dither;
         if (lvds_options->panel_type == 0xff)
                 return;
 
@@ -226,7 +226,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
         if (!lvds_lfp_data_ptrs)
                 return;
 
-        dev_priv->lvds_vbt = 1;
+        dev_priv->vbt.lvds_vbt = 1;
 
         panel_dvo_timing = get_lvds_dvo_timing(lvds_lfp_data,
                                                lvds_lfp_data_ptrs,
@@ -238,7 +238,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
 
         fill_detail_timing_data(panel_fixed_mode, panel_dvo_timing);
 
-        dev_priv->lfp_lvds_vbt_mode = panel_fixed_mode;
+        dev_priv->vbt.lfp_lvds_vbt_mode = panel_fixed_mode;
 
         DRM_DEBUG_KMS("Found panel mode in BIOS VBT tables:\n");
         drm_mode_debug_printmodeline(panel_fixed_mode);
@@ -274,9 +274,9 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
                 /* check the resolution, just to be sure */
                 if (fp_timing->x_res == panel_fixed_mode->hdisplay &&
                     fp_timing->y_res == panel_fixed_mode->vdisplay) {
-                        dev_priv->bios_lvds_val = fp_timing->lvds_reg_val;
+                        dev_priv->vbt.bios_lvds_val = fp_timing->lvds_reg_val;
                         DRM_DEBUG_KMS("VBT initial LVDS value %x\n",
-                                      dev_priv->bios_lvds_val);
+                                      dev_priv->vbt.bios_lvds_val);
                 }
         }
 }
@@ -316,7 +316,7 @@ parse_sdvo_panel_data(struct drm_i915_private *dev_priv,
 
         fill_detail_timing_data(panel_fixed_mode, dvo_timing + index);
 
-        dev_priv->sdvo_lvds_vbt_mode = panel_fixed_mode;
+        dev_priv->vbt.sdvo_lvds_vbt_mode = panel_fixed_mode;
 
         DRM_DEBUG_KMS("Found SDVO panel mode in BIOS VBT tables:\n");
         drm_mode_debug_printmodeline(panel_fixed_mode);
@@ -345,20 +345,20 @@ parse_general_features(struct drm_i915_private *dev_priv,
 
         general = find_section(bdb, BDB_GENERAL_FEATURES);
         if (general) {
-                dev_priv->int_tv_support = general->int_tv_support;
-                dev_priv->int_crt_support = general->int_crt_support;
-                dev_priv->lvds_use_ssc = general->enable_ssc;
-                dev_priv->lvds_ssc_freq =
+                dev_priv->vbt.int_tv_support = general->int_tv_support;
+                dev_priv->vbt.int_crt_support = general->int_crt_support;
+                dev_priv->vbt.lvds_use_ssc = general->enable_ssc;
+                dev_priv->vbt.lvds_ssc_freq =
                         intel_bios_ssc_frequency(dev, general->ssc_freq);
-                dev_priv->display_clock_mode = general->display_clock_mode;
-                dev_priv->fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted;
+                dev_priv->vbt.display_clock_mode = general->display_clock_mode;
+                dev_priv->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted;
                 DRM_DEBUG_KMS("BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n",
-                              dev_priv->int_tv_support,
-                              dev_priv->int_crt_support,
-                              dev_priv->lvds_use_ssc,
-                              dev_priv->lvds_ssc_freq,
-                              dev_priv->display_clock_mode,
-                              dev_priv->fdi_rx_polarity_inverted);
+                              dev_priv->vbt.int_tv_support,
+                              dev_priv->vbt.int_crt_support,
+                              dev_priv->vbt.lvds_use_ssc,
+                              dev_priv->vbt.lvds_ssc_freq,
+                              dev_priv->vbt.display_clock_mode,
+                              dev_priv->vbt.fdi_rx_polarity_inverted);
         }
 }
 
@@ -375,7 +375,7 @@ parse_general_definitions(struct drm_i915_private *dev_priv,
                         int bus_pin = general->crt_ddc_gmbus_pin;
                         DRM_DEBUG_KMS("crt_ddc_bus_pin: %d\n", bus_pin);
                         if (intel_gmbus_is_port_valid(bus_pin))
-                                dev_priv->crt_ddc_pin = bus_pin;
+                                dev_priv->vbt.crt_ddc_pin = bus_pin;
                 } else {
                         DRM_DEBUG_KMS("BDB_GD too small (%d). Invalid.\n",
                                       block_size);
@@ -486,7 +486,7 @@ parse_driver_features(struct drm_i915_private *dev_priv,
 
         if (SUPPORTS_EDP(dev) &&
             driver->lvds_config == BDB_DRIVER_FEATURE_EDP)
-                dev_priv->edp.support = 1;
+                dev_priv->vbt.edp_support = 1;
 
         if (driver->dual_frequency)
                 dev_priv->render_reclock_avail = true;
@@ -501,20 +501,20 @@ parse_edp(struct drm_i915_private *dev_priv, struct bdb_header *bdb)
 
         edp = find_section(bdb, BDB_EDP);
         if (!edp) {
-                if (SUPPORTS_EDP(dev_priv->dev) && dev_priv->edp.support)
+                if (SUPPORTS_EDP(dev_priv->dev) && dev_priv->vbt.edp_support)
                         DRM_DEBUG_KMS("No eDP BDB found but eDP panel supported.\n");
                 return;
         }
 
         switch ((edp->color_depth >> (panel_type * 2)) & 3) {
         case EDP_18BPP:
-                dev_priv->edp.bpp = 18;
+                dev_priv->vbt.edp_bpp = 18;
                 break;
         case EDP_24BPP:
-                dev_priv->edp.bpp = 24;
+                dev_priv->vbt.edp_bpp = 24;
                 break;
         case EDP_30BPP:
-                dev_priv->edp.bpp = 30;
+                dev_priv->vbt.edp_bpp = 30;
                 break;
         }
 
@@ -522,48 +522,48 @@ parse_edp(struct drm_i915_private *dev_priv, struct bdb_header *bdb)
         edp_pps = &edp->power_seqs[panel_type];
         edp_link_params = &edp->link_params[panel_type];
 
-        dev_priv->edp.pps = *edp_pps;
+        dev_priv->vbt.edp_pps = *edp_pps;
 
-        dev_priv->edp.rate = edp_link_params->rate ? DP_LINK_BW_2_7 :
+        dev_priv->vbt.edp_rate = edp_link_params->rate ? DP_LINK_BW_2_7 :
                 DP_LINK_BW_1_62;
         switch (edp_link_params->lanes) {
         case 0:
-                dev_priv->edp.lanes = 1;
+                dev_priv->vbt.edp_lanes = 1;
                 break;
         case 1:
-                dev_priv->edp.lanes = 2;
+                dev_priv->vbt.edp_lanes = 2;
                 break;
         case 3:
         default:
-                dev_priv->edp.lanes = 4;
+                dev_priv->vbt.edp_lanes = 4;
                 break;
         }
         switch (edp_link_params->preemphasis) {
         case 0:
-                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_0;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_0;
                 break;
         case 1:
-                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_3_5;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_3_5;
                 break;
         case 2:
-                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_6;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_6;
                 break;
         case 3:
-                dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_9_5;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_9_5;
                 break;
         }
         switch (edp_link_params->vswing) {
         case 0:
-                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_400;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_400;
                 break;
         case 1:
-                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_600;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_600;
                 break;
         case 2:
-                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_800;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_800;
                 break;
         case 3:
-                dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_1200;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_1200;
                 break;
         }
 }
@@ -611,13 +611,13 @@ parse_device_mapping(struct drm_i915_private *dev_priv,
                 DRM_DEBUG_KMS("no child dev is parsed from VBT\n");
                 return;
         }
-        dev_priv->child_dev = kcalloc(count, sizeof(*p_child), GFP_KERNEL);
-        if (!dev_priv->child_dev) {
+        dev_priv->vbt.child_dev = kcalloc(count, sizeof(*p_child), GFP_KERNEL);
+        if (!dev_priv->vbt.child_dev) {
                 DRM_DEBUG_KMS("No memory space for child device\n");
                 return;
         }
 
-        dev_priv->child_dev_num = count;
+        dev_priv->vbt.child_dev_num = count;
         count = 0;
         for (i = 0; i < child_device_num; i++) {
                 p_child = &(p_defs->devices[i]);
@@ -625,7 +625,7 @@ parse_device_mapping(struct drm_i915_private *dev_priv,
                         /* skip the device block if device type is invalid */
                         continue;
                 }
-                child_dev_ptr = dev_priv->child_dev + count;
+                child_dev_ptr = dev_priv->vbt.child_dev + count;
                 count++;
                 memcpy((void *)child_dev_ptr, (void *)p_child,
                                         sizeof(*p_child));
@@ -638,23 +638,23 @@ init_vbt_defaults(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
 
-        dev_priv->crt_ddc_pin = GMBUS_PORT_VGADDC;
+        dev_priv->vbt.crt_ddc_pin = GMBUS_PORT_VGADDC;
 
         /* LFP panel data */
-        dev_priv->lvds_dither = 1;
-        dev_priv->lvds_vbt = 0;
+        dev_priv->vbt.lvds_dither = 1;
+        dev_priv->vbt.lvds_vbt = 0;
 
         /* SDVO panel data */
-        dev_priv->sdvo_lvds_vbt_mode = NULL;
+        dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
 
         /* general features */
-        dev_priv->int_tv_support = 1;
-        dev_priv->int_crt_support = 1;
+        dev_priv->vbt.int_tv_support = 1;
+        dev_priv->vbt.int_crt_support = 1;
 
         /* Default to using SSC */
-        dev_priv->lvds_use_ssc = 1;
-        dev_priv->lvds_ssc_freq = intel_bios_ssc_frequency(dev, 1);
-        DRM_DEBUG_KMS("Set default to SSC at %dMHz\n", dev_priv->lvds_ssc_freq);
+        dev_priv->vbt.lvds_use_ssc = 1;
+        dev_priv->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(dev, 1);
+        DRM_DEBUG_KMS("Set default to SSC at %dMHz\n", dev_priv->vbt.lvds_ssc_freq);
 }
 
 static int __init intel_no_opregion_vbt_callback(const struct dmi_system_id *id)
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 58b4a53715cd..3acec8c48166 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -84,6 +84,28 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
         return true;
 }
 
+static void intel_crt_get_config(struct intel_encoder *encoder,
+                                 struct intel_crtc_config *pipe_config)
+{
+        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct intel_crt *crt = intel_encoder_to_crt(encoder);
+        u32 tmp, flags = 0;
+
+        tmp = I915_READ(crt->adpa_reg);
+
+        if (tmp & ADPA_HSYNC_ACTIVE_HIGH)
+                flags |= DRM_MODE_FLAG_PHSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NHSYNC;
+
+        if (tmp & ADPA_VSYNC_ACTIVE_HIGH)
+                flags |= DRM_MODE_FLAG_PVSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NVSYNC;
+
+        pipe_config->adjusted_mode.flags |= flags;
+}
+
 /* Note: The caller is required to filter out dpms modes not supported by the
  * platform. */
 static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode)
@@ -127,7 +149,7 @@ static void intel_enable_crt(struct intel_encoder *encoder)
         intel_crt_set_dpms(encoder, crt->connector->base.dpms);
 }
 
-
+/* Special dpms function to support cloning between dvo/sdvo/crt. */
 static void intel_crt_dpms(struct drm_connector *connector, int mode)
 {
         struct drm_device *dev = connector->dev;
@@ -158,6 +180,8 @@ static void intel_crt_dpms(struct drm_connector *connector, int mode)
         else
                 encoder->connectors_active = true;
 
+        /* We call connector dpms manually below in case pipe dpms doesn't
+         * change due to cloning. */
         if (mode < old_dpms) {
                 /* From off to on, enable the pipe first. */
                 intel_crtc_update_dpms(crtc);
@@ -207,6 +231,10 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder,
         if (HAS_PCH_SPLIT(dev))
                 pipe_config->has_pch_encoder = true;
 
+        /* LPT FDI RX only supports 8bpc. */
+        if (HAS_PCH_LPT(dev))
+                pipe_config->pipe_bpp = 24;
+
         return true;
 }
 
@@ -431,7 +459,7 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector)
 
         BUG_ON(crt->base.type != INTEL_OUTPUT_ANALOG);
 
-        i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->crt_ddc_pin);
+        i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
         edid = intel_crt_get_edid(connector, i2c);
 
         if (edid) {
@@ -637,7 +665,7 @@ static int intel_crt_get_modes(struct drm_connector *connector)
         int ret;
         struct i2c_adapter *i2c;
 
-        i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->crt_ddc_pin);
+        i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
         ret = intel_crt_ddc_get_modes(connector, i2c);
         if (ret || !IS_G4X(dev))
                 return ret;
@@ -774,6 +802,7 @@ void intel_crt_init(struct drm_device *dev)
         crt->base.compute_config = intel_crt_compute_config;
         crt->base.disable = intel_disable_crt;
         crt->base.enable = intel_enable_crt;
+        crt->base.get_config = intel_crt_get_config;
         if (I915_HAS_HOTPLUG(dev))
                 crt->base.hpd_pin = HPD_CRT;
         if (HAS_DDI(dev))
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index fb961bb81903..324211ac9c55 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -174,6 +174,8 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
          * mode set "sequence for CRT port" document:
          * - TP1 to TP2 time with the default value
          * - FDI delay to 90h
+         *
+         * WaFDIAutoLinkSetTimingOverrride:hsw
          */
         I915_WRITE(_FDI_RXA_MISC, FDI_RX_PWRDN_LANE1_VAL(2) |
                                   FDI_RX_PWRDN_LANE0_VAL(2) |
@@ -181,7 +183,8 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
 
         /* Enable the PCH Receiver FDI PLL */
         rx_ctl_val = dev_priv->fdi_rx_config | FDI_RX_ENHANCE_FRAME_ENABLE |
-                     FDI_RX_PLL_ENABLE | ((intel_crtc->fdi_lanes - 1) << 19);
+                     FDI_RX_PLL_ENABLE |
+                     FDI_DP_PORT_WIDTH(intel_crtc->config.fdi_lanes);
         I915_WRITE(_FDI_RXA_CTL, rx_ctl_val);
         POSTING_READ(_FDI_RXA_CTL);
         udelay(220);
@@ -209,7 +212,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
                  * port reversal bit */
                 I915_WRITE(DDI_BUF_CTL(PORT_E),
                            DDI_BUF_CTL_ENABLE |
-                           ((intel_crtc->fdi_lanes - 1) << 1) |
+                           ((intel_crtc->config.fdi_lanes - 1) << 1) |
                            hsw_ddi_buf_ctl_values[i / 2]);
                 POSTING_READ(DDI_BUF_CTL(PORT_E));
 
@@ -278,392 +281,6 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
         DRM_ERROR("FDI link training failed!\n");
 }
 
-/* WRPLL clock dividers */
-struct wrpll_tmds_clock {
-        u32 clock;
-        u16 p;          /* Post divider */
-        u16 n2;         /* Feedback divider */
-        u16 r2;         /* Reference divider */
-};
-
-/* Table of matching values for WRPLL clocks programming for each frequency.
- * The code assumes this table is sorted. */
-static const struct wrpll_tmds_clock wrpll_tmds_clock_table[] = {
-        {19750, 38,     25,     18},
-        {20000, 48,     32,     18},
-        {21000, 36,     21,     15},
-        {21912, 42,     29,     17},
-        {22000, 36,     22,     15},
-        {23000, 36,     23,     15},
-        {23500, 40,     40,     23},
-        {23750, 26,     16,     14},
-        {24000, 36,     24,     15},
-        {25000, 36,     25,     15},
-        {25175, 26,     40,     33},
-        {25200, 30,     21,     15},
-        {26000, 36,     26,     15},
-        {27000, 30,     21,     14},
-        {27027, 18,     100,    111},
-        {27500, 30,     29,     19},
-        {28000, 34,     30,     17},
-        {28320, 26,     30,     22},
-        {28322, 32,     42,     25},
-        {28750, 24,     23,     18},
-        {29000, 30,     29,     18},
-        {29750, 32,     30,     17},
-        {30000, 30,     25,     15},
-        {30750, 30,     41,     24},
-        {31000, 30,     31,     18},
-        {31500, 30,     28,     16},
-        {32000, 30,     32,     18},
-        {32500, 28,     32,     19},
-        {33000, 24,     22,     15},
-        {34000, 28,     30,     17},
-        {35000, 26,     32,     19},
-        {35500, 24,     30,     19},
-        {36000, 26,     26,     15},
-        {36750, 26,     46,     26},
-        {37000, 24,     23,     14},
-        {37762, 22,     40,     26},
-        {37800, 20,     21,     15},
-        {38000, 24,     27,     16},
-        {38250, 24,     34,     20},
-        {39000, 24,     26,     15},
-        {40000, 24,     32,     18},
-        {40500, 20,     21,     14},
-        {40541, 22,     147,    89},
-        {40750, 18,     19,     14},
-        {41000, 16,     17,     14},
-        {41500, 22,     44,     26},
-        {41540, 22,     44,     26},
-        {42000, 18,     21,     15},
-        {42500, 22,     45,     26},
-        {43000, 20,     43,     27},
-        {43163, 20,     24,     15},
-        {44000, 18,     22,     15},
-        {44900, 20,     108,    65},
-        {45000, 20,     25,     15},
-        {45250, 20,     52,     31},
-        {46000, 18,     23,     15},
-        {46750, 20,     45,     26},
-        {47000, 20,     40,     23},
-        {48000, 18,     24,     15},
-        {49000, 18,     49,     30},
-        {49500, 16,     22,     15},
-        {50000, 18,     25,     15},
-        {50500, 18,     32,     19},
-        {51000, 18,     34,     20},
-        {52000, 18,     26,     15},
-        {52406, 14,     34,     25},
-        {53000, 16,     22,     14},
-        {54000, 16,     24,     15},
-        {54054, 16,     173,    108},
-        {54500, 14,     24,     17},
-        {55000, 12,     22,     18},
-        {56000, 14,     45,     31},
-        {56250, 16,     25,     15},
-        {56750, 14,     25,     17},
-        {57000, 16,     27,     16},
-        {58000, 16,     43,     25},
-        {58250, 16,     38,     22},
-        {58750, 16,     40,     23},
-        {59000, 14,     26,     17},
-        {59341, 14,     40,     26},
-        {59400, 16,     44,     25},
-        {60000, 16,     32,     18},
-        {60500, 12,     39,     29},
-        {61000, 14,     49,     31},
-        {62000, 14,     37,     23},
-        {62250, 14,     42,     26},
-        {63000, 12,     21,     15},
-        {63500, 14,     28,     17},
-        {64000, 12,     27,     19},
-        {65000, 14,     32,     19},
-        {65250, 12,     29,     20},
-        {65500, 12,     32,     22},
-        {66000, 12,     22,     15},
-        {66667, 14,     38,     22},
-        {66750, 10,     21,     17},
-        {67000, 14,     33,     19},
-        {67750, 14,     58,     33},
-        {68000, 14,     30,     17},
-        {68179, 14,     46,     26},
-        {68250, 14,     46,     26},
-        {69000, 12,     23,     15},
-        {70000, 12,     28,     18},
-        {71000, 12,     30,     19},
-        {72000, 12,     24,     15},
-        {73000, 10,     23,     17},
-        {74000, 12,     23,     14},
-        {74176, 8,      100,    91},
-        {74250, 10,     22,     16},
-        {74481, 12,     43,     26},
-        {74500, 10,     29,     21},
-        {75000, 12,     25,     15},
-        {75250, 10,     39,     28},
-        {76000, 12,     27,     16},
-        {77000, 12,     53,     31},
-        {78000, 12,     26,     15},
-        {78750, 12,     28,     16},
-        {79000, 10,     38,     26},
-        {79500, 10,     28,     19},
-        {80000, 12,     32,     18},
-        {81000, 10,     21,     14},
-        {81081, 6,      100,    111},
-        {81624, 8,      29,     24},
-        {82000, 8,      17,     14},
-        {83000, 10,     40,     26},
-        {83950, 10,     28,     18},
-        {84000, 10,     28,     18},
-        {84750, 6,      16,     17},
-        {85000, 6,      17,     18},
-        {85250, 10,     30,     19},
-        {85750, 10,     27,     17},
-        {86000, 10,     43,     27},
-        {87000, 10,     29,     18},
-        {88000, 10,     44,     27},
-        {88500, 10,     41,     25},
-        {89000, 10,     28,     17},
-        {89012, 6,      90,     91},
-        {89100, 10,     33,     20},
-        {90000, 10,     25,     15},
-        {91000, 10,     32,     19},
-        {92000, 10,     46,     27},
-        {93000, 10,     31,     18},
-        {94000, 10,     40,     23},
-        {94500, 10,     28,     16},
-        {95000, 10,     44,     25},
-        {95654, 10,     39,     22},
-        {95750, 10,     39,     22},
-        {96000, 10,     32,     18},
-        {97000, 8,      23,     16},
-        {97750, 8,      42,     29},
-        {98000, 8,      45,     31},
-        {99000, 8,      22,     15},
-        {99750, 8,      34,     23},
-        {100000,        6,      20,     18},
-        {100500,        6,      19,     17},
-        {101000,        6,      37,     33},
-        {101250,        8,      21,     14},
-        {102000,        6,      17,     15},
-        {102250,        6,      25,     22},
-        {103000,        8,      29,     19},
-        {104000,        8,      37,     24},
-        {105000,        8,      28,     18},
-        {106000,        8,      22,     14},
-        {107000,        8,      46,     29},
-        {107214,        8,      27,     17},
-        {108000,        8,      24,     15},
-        {108108,        8,      173,    108},
-        {109000,        6,      23,     19},
-        {110000,        6,      22,     18},
-        {110013,        6,      22,     18},
-        {110250,        8,      49,     30},
-        {110500,        8,      36,     22},
-        {111000,        8,      23,     14},
-        {111264,        8,      150,    91},
-        {111375,        8,      33,     20},
-        {112000,        8,      63,     38},
-        {112500,        8,      25,     15},
-        {113100,        8,      57,     34},
-        {113309,        8,      42,     25},
-        {114000,        8,      27,     16},
-        {115000,        6,      23,     18},
-        {116000,        8,      43,     25},
-        {117000,        8,      26,     15},
-        {117500,        8,      40,     23},
-        {118000,        6,      38,     29},
-        {119000,        8,      30,     17},
-        {119500,        8,      46,     26},
-        {119651,        8,      39,     22},
-        {120000,        8,      32,     18},
-        {121000,        6,      39,     29},
-        {121250,        6,      31,     23},
-        {121750,        6,      23,     17},
-        {122000,        6,      42,     31},
-        {122614,        6,      30,     22},
-        {123000,        6,      41,     30},
-        {123379,        6,      37,     27},
-        {124000,        6,      51,     37},
-        {125000,        6,      25,     18},
-        {125250,        4,      13,     14},
-        {125750,        4,      27,     29},
-        {126000,        6,      21,     15},
-        {127000,        6,      24,     17},
-        {127250,        6,      41,     29},
-        {128000,        6,      27,     19},
-        {129000,        6,      43,     30},
-        {129859,        4,      25,     26},
-        {130000,        6,      26,     18},
-        {130250,        6,      42,     29},
-        {131000,        6,      32,     22},
-        {131500,        6,      38,     26},
-        {131850,        6,      41,     28},
-        {132000,        6,      22,     15},
-        {132750,        6,      28,     19},
-        {133000,        6,      34,     23},
-        {133330,        6,      37,     25},
-        {134000,        6,      61,     41},
-        {135000,        6,      21,     14},
-        {135250,        6,      167,    111},
-        {136000,        6,      62,     41},
-        {137000,        6,      35,     23},
-        {138000,        6,      23,     15},
-        {138500,        6,      40,     26},
-        {138750,        6,      37,     24},
-        {139000,        6,      34,     22},
-        {139050,        6,      34,     22},
-        {139054,        6,      34,     22},
-        {140000,        6,      28,     18},
-        {141000,        6,      36,     23},
-        {141500,        6,      22,     14},
-        {142000,        6,      30,     19},
-        {143000,        6,      27,     17},
-        {143472,        4,      17,     16},
-        {144000,        6,      24,     15},
-        {145000,        6,      29,     18},
-        {146000,        6,      47,     29},
-        {146250,        6,      26,     16},
-        {147000,        6,      49,     30},
-        {147891,        6,      23,     14},
-        {148000,        6,      23,     14},
-        {148250,        6,      28,     17},
-        {148352,        4,      100,    91},
-        {148500,        6,      33,     20},
-        {149000,        6,      48,     29},
-        {150000,        6,      25,     15},
-        {151000,        4,      19,     17},
-        {152000,        6,      27,     16},
-        {152280,        6,      44,     26},
-        {153000,        6,      34,     20},
-        {154000,        6,      53,     31},
-        {155000,        6,      31,     18},
-        {155250,        6,      50,     29},
-        {155750,        6,      45,     26},
-        {156000,        6,      26,     15},
-        {157000,        6,      61,     35},
-        {157500,        6,      28,     16},
-        {158000,        6,      65,     37},
-        {158250,        6,      44,     25},
-        {159000,        6,      53,     30},
-        {159500,        6,      39,     22},
-        {160000,        6,      32,     18},
-        {161000,        4,      31,     26},
-        {162000,        4,      18,     15},
-        {162162,        4,      131,    109},
-        {162500,        4,      53,     44},
-        {163000,        4,      29,     24},
-        {164000,        4,      17,     14},
-        {165000,        4,      22,     18},
-        {166000,        4,      32,     26},
-        {167000,        4,      26,     21},
-        {168000,        4,      46,     37},
-        {169000,        4,      104,    83},
-        {169128,        4,      64,     51},
-        {169500,        4,      39,     31},
-        {170000,        4,      34,     27},
-        {171000,        4,      19,     15},
-        {172000,        4,      51,     40},
-        {172750,        4,      32,     25},
-        {172800,        4,      32,     25},
-        {173000,        4,      41,     32},
-        {174000,        4,      49,     38},
-        {174787,        4,      22,     17},
-        {175000,        4,      35,     27},
-        {176000,        4,      30,     23},
-        {177000,        4,      38,     29},
-        {178000,        4,      29,     22},
-        {178500,        4,      37,     28},
-        {179000,        4,      53,     40},
-        {179500,        4,      73,     55},
-        {180000,        4,      20,     15},
-        {181000,        4,      55,     41},
-        {182000,        4,      31,     23},
-        {183000,        4,      42,     31},
-        {184000,        4,      30,     22},
-        {184750,        4,      26,     19},
-        {185000,        4,      37,     27},
-        {186000,        4,      51,     37},
-        {187000,        4,      36,     26},
-        {188000,        4,      32,     23},
-        {189000,        4,      21,     15},
-        {190000,        4,      38,     27},
-        {190960,        4,      41,     29},
-        {191000,        4,      41,     29},
-        {192000,        4,      27,     19},
-        {192250,        4,      37,     26},
-        {193000,        4,      20,     14},
-        {193250,        4,      53,     37},
-        {194000,        4,      23,     16},
-        {194208,        4,      23,     16},
-        {195000,        4,      26,     18},
-        {196000,        4,      45,     31},
-        {197000,        4,      35,     24},
-        {197750,        4,      41,     28},
-        {198000,        4,      22,     15},
-        {198500,        4,      25,     17},
-        {199000,        4,      28,     19},
-        {200000,        4,      37,     25},
-        {201000,        4,      61,     41},
-        {202000,        4,      112,    75},
-        {202500,        4,      21,     14},
-        {203000,        4,      146,    97},
-        {204000,        4,      62,     41},
-        {204750,        4,      44,     29},
-        {205000,        4,      38,     25},
-        {206000,        4,      29,     19},
-        {207000,        4,      23,     15},
-        {207500,        4,      40,     26},
-        {208000,        4,      37,     24},
-        {208900,        4,      48,     31},
-        {209000,        4,      48,     31},
-        {209250,        4,      31,     20},
-        {210000,        4,      28,     18},
-        {211000,        4,      25,     16},
-        {212000,        4,      22,     14},
-        {213000,        4,      30,     19},
-        {213750,        4,      38,     24},
-        {214000,        4,      46,     29},
-        {214750,        4,      35,     22},
-        {215000,        4,      43,     27},
-        {216000,        4,      24,     15},
-        {217000,        4,      37,     23},
-        {218000,        4,      42,     26},
-        {218250,        4,      42,     26},
-        {218750,        4,      34,     21},
-        {219000,        4,      47,     29},
-        {220000,        4,      44,     27},
-        {220640,        4,      49,     30},
-        {220750,        4,      36,     22},
-        {221000,        4,      36,     22},
-        {222000,        4,      23,     14},
-        {222525,        4,      28,     17},
-        {222750,        4,      33,     20},
-        {227000,        4,      37,     22},
-        {230250,        4,      29,     17},
-        {233500,        4,      38,     22},
-        {235000,        4,      40,     23},
-        {238000,        4,      30,     17},
-        {241500,        2,      17,     19},
-        {245250,        2,      20,     22},
-        {247750,        2,      22,     24},
-        {253250,        2,      15,     16},
-        {256250,        2,      18,     19},
-        {262500,        2,      31,     32},
-        {267250,        2,      66,     67},
-        {268500,        2,      94,     95},
-        {270000,        2,      14,     14},
-        {272500,        2,      77,     76},
-        {273750,        2,      57,     56},
-        {280750,        2,      24,     23},
-        {281250,        2,      23,     22},
-        {286000,        2,      17,     16},
-        {291750,        2,      26,     24},
-        {296703,        2,      56,     51},
-        {297000,        2,      22,     20},
-        {298000,        2,      21,     19},
-};
-
 static void intel_ddi_mode_set(struct drm_encoder *encoder,
                                struct drm_display_mode *mode,
                                struct drm_display_mode *adjusted_mode)
@@ -675,7 +292,7 @@ static void intel_ddi_mode_set(struct drm_encoder *encoder,
         int pipe = intel_crtc->pipe;
         int type = intel_encoder->type;
 
-        DRM_DEBUG_KMS("Preparing DDI mode for Haswell on port %c, pipe %c\n",
+        DRM_DEBUG_KMS("Preparing DDI mode on port %c, pipe %c\n",
                       port_name(port), pipe_name(pipe));
 
         intel_crtc->eld_vld = false;
@@ -686,22 +303,7 @@ static void intel_ddi_mode_set(struct drm_encoder *encoder,
 
                 intel_dp->DP = intel_dig_port->port_reversal |
                                DDI_BUF_CTL_ENABLE | DDI_BUF_EMP_400MV_0DB_HSW;
-                switch (intel_dp->lane_count) {
-                case 1:
-                        intel_dp->DP |= DDI_PORT_WIDTH_X1;
-                        break;
-                case 2:
-                        intel_dp->DP |= DDI_PORT_WIDTH_X2;
-                        break;
-                case 4:
-                        intel_dp->DP |= DDI_PORT_WIDTH_X4;
-                        break;
-                default:
-                        intel_dp->DP |= DDI_PORT_WIDTH_X4;
-                        WARN(1, "Unexpected DP lane count %d\n",
-                             intel_dp->lane_count);
-                        break;
-                }
+                intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count);
 
                 if (intel_dp->has_audio) {
                         DRM_DEBUG_DRIVER("DP audio on pipe %c on DDI\n",
@@ -748,8 +350,8 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc)
         }
 
         if (num_encoders != 1)
-                WARN(1, "%d encoders on crtc for pipe %d\n", num_encoders,
-                     intel_crtc->pipe);
+                WARN(1, "%d encoders on crtc for pipe %c\n", num_encoders,
+                     pipe_name(intel_crtc->pipe));
 
         BUG_ON(ret == NULL);
         return ret;
@@ -802,30 +404,227 @@ void intel_ddi_put_crtc_pll(struct drm_crtc *crtc)
         intel_crtc->ddi_pll_sel = PORT_CLK_SEL_NONE;
 }
 
-static void intel_ddi_calculate_wrpll(int clock, int *p, int *n2, int *r2)
+#define LC_FREQ 2700
+#define LC_FREQ_2K (LC_FREQ * 2000)
+
+#define P_MIN 2
+#define P_MAX 64
+#define P_INC 2
+
+/* Constraints for PLL good behavior */
+#define REF_MIN 48
+#define REF_MAX 400
+#define VCO_MIN 2400
+#define VCO_MAX 4800
+
+#define ABS_DIFF(a, b) ((a > b) ? (a - b) : (b - a))
+
+struct wrpll_rnp {
+        unsigned p, n2, r2;
+};
+
+static unsigned wrpll_get_budget_for_freq(int clock)
+{
+        unsigned budget;
+
+        switch (clock) {
+        case 25175000:
+        case 25200000:
+        case 27000000:
+        case 27027000:
+        case 37762500:
+        case 37800000:
+        case 40500000:
+        case 40541000:
+        case 54000000:
+        case 54054000:
+        case 59341000:
+        case 59400000:
+        case 72000000:
+        case 74176000:
+        case 74250000:
+        case 81000000:
+        case 81081000:
+        case 89012000:
+        case 89100000:
+        case 108000000:
+        case 108108000:
+        case 111264000:
+        case 111375000:
+        case 148352000:
+        case 148500000:
+        case 162000000:
+        case 162162000:
+        case 222525000:
+        case 222750000:
+        case 296703000:
+        case 297000000:
+                budget = 0;
+                break;
+        case 233500000:
+        case 245250000:
+        case 247750000:
+        case 253250000:
+        case 298000000:
+                budget = 1500;
+                break;
+        case 169128000:
+        case 169500000:
+        case 179500000:
+        case 202000000:
+                budget = 2000;
+                break;
+        case 256250000:
+        case 262500000:
+        case 270000000:
+        case 272500000:
+        case 273750000:
+        case 280750000:
+        case 281250000:
+        case 286000000:
+        case 291750000:
+                budget = 4000;
+                break;
+        case 267250000:
+        case 268500000:
+                budget = 5000;
+                break;
+        default:
+                budget = 1000;
+                break;
+        }
+
+        return budget;
+}
+
+static void wrpll_update_rnp(uint64_t freq2k, unsigned budget,
+                             unsigned r2, unsigned n2, unsigned p,
+                             struct wrpll_rnp *best)
 {
-        u32 i;
+        uint64_t a, b, c, d, diff, diff_best;
 
-        for (i = 0; i < ARRAY_SIZE(wrpll_tmds_clock_table); i++)
-                if (clock <= wrpll_tmds_clock_table[i].clock)
-                        break;
+        /* No best (r,n,p) yet */
+        if (best->p == 0) {
+                best->p = p;
+                best->n2 = n2;
+                best->r2 = r2;
+                return;
+        }
+
+        /*
+         * Output clock is (LC_FREQ_2K / 2000) * N / (P * R), which compares to
+         * freq2k.
+         *
+         * delta = 1e6 *
+         *         abs(freq2k - (LC_FREQ_2K * n2/(p * r2))) /
+         *         freq2k;
+         *
+         * and we would like delta <= budget.
+         *
+         * If the discrepancy is above the PPM-based budget, always prefer to
+         * improve upon the previous solution.  However, if you're within the
+         * budget, try to maximize Ref * VCO, that is N / (P * R^2).
+         */
+        a = freq2k * budget * p * r2;
+        b = freq2k * budget * best->p * best->r2;
+        diff = ABS_DIFF((freq2k * p * r2), (LC_FREQ_2K * n2));
+        diff_best = ABS_DIFF((freq2k * best->p * best->r2),
+                             (LC_FREQ_2K * best->n2));
+        c = 1000000 * diff;
+        d = 1000000 * diff_best;
+
+        if (a < c && b < d) {
+                /* If both are above the budget, pick the closer */
+                if (best->p * best->r2 * diff < p * r2 * diff_best) {
+                        best->p = p;
+                        best->n2 = n2;
+                        best->r2 = r2;
+                }
+        } else if (a >= c && b < d) {
+                /* If A is below the threshold but B is above it?  Update. */
+                best->p = p;
+                best->n2 = n2;
+                best->r2 = r2;
+        } else if (a >= c && b >= d) {
+                /* Both are below the limit, so pick the higher n2/(r2*r2) */
+                if (n2 * best->r2 * best->r2 > best->n2 * r2 * r2) {
+                        best->p = p;
+                        best->n2 = n2;
+                        best->r2 = r2;
+                }
+        }
+        /* Otherwise a < c && b >= d, do nothing */
+}
+
+static void
+intel_ddi_calculate_wrpll(int clock /* in Hz */,
+                          unsigned *r2_out, unsigned *n2_out, unsigned *p_out)
+{
+        uint64_t freq2k;
+        unsigned p, n2, r2;
+        struct wrpll_rnp best = { 0, 0, 0 };
+        unsigned budget;
 
-        if (i == ARRAY_SIZE(wrpll_tmds_clock_table))
-                i--;
+        freq2k = clock / 100;
 
-        *p = wrpll_tmds_clock_table[i].p;
-        *n2 = wrpll_tmds_clock_table[i].n2;
-        *r2 = wrpll_tmds_clock_table[i].r2;
+        budget = wrpll_get_budget_for_freq(clock);
 
-        if (wrpll_tmds_clock_table[i].clock != clock)
-                DRM_INFO("WRPLL: using settings for %dKHz on %dKHz mode\n",
-                         wrpll_tmds_clock_table[i].clock, clock);
+        /* Special case handling for 540 pixel clock: bypass WR PLL entirely
+         * and directly pass the LC PLL to it. */
+        if (freq2k == 5400000) {
+                *n2_out = 2;
+                *p_out = 1;
+                *r2_out = 2;
+                return;
+        }
+
+        /*
+         * Ref = LC_FREQ / R, where Ref is the actual reference input seen by
+         * the WR PLL.
+         *
+         * We want R so that REF_MIN <= Ref <= REF_MAX.
+         * Injecting R2 = 2 * R gives:
+         *   REF_MAX * r2 > LC_FREQ * 2 and
+         *   REF_MIN * r2 < LC_FREQ * 2
+         *
+         * Which means the desired boundaries for r2 are:
+         *  LC_FREQ * 2 / REF_MAX < r2 < LC_FREQ * 2 / REF_MIN
+         *
+         */
+        for (r2 = LC_FREQ * 2 / REF_MAX + 1;
+             r2 <= LC_FREQ * 2 / REF_MIN;
+             r2++) {
+
+                /*
+                 * VCO = N * Ref, that is: VCO = N * LC_FREQ / R
+                 *
+                 * Once again we want VCO_MIN <= VCO <= VCO_MAX.
+                 * Injecting R2 = 2 * R and N2 = 2 * N, we get:
+                 *   VCO_MAX * r2 > n2 * LC_FREQ and
+                 *   VCO_MIN * r2 < n2 * LC_FREQ)
+                 *
+                 * Which means the desired boundaries for n2 are:
+                 * VCO_MIN * r2 / LC_FREQ < n2 < VCO_MAX * r2 / LC_FREQ
+                 */
+                for (n2 = VCO_MIN * r2 / LC_FREQ + 1;
+                     n2 <= VCO_MAX * r2 / LC_FREQ;
+                     n2++) {
 
-        DRM_DEBUG_KMS("WRPLL: %dKHz refresh rate with p=%d, n2=%d r2=%d\n",
-                      clock, *p, *n2, *r2);
+                        for (p = P_MIN; p <= P_MAX; p += P_INC)
+                                wrpll_update_rnp(freq2k, budget,
+                                                 r2, n2, p, &best);
+                }
+        }
+
+        *n2_out = best.n2;
+        *p_out = best.p;
+        *r2_out = best.r2;
+
+        DRM_DEBUG_KMS("WRPLL: %dHz refresh rate with p=%d, n2=%d r2=%d\n",
+                      clock, *p_out, *n2_out, *r2_out);
 }
 
-bool intel_ddi_pll_mode_set(struct drm_crtc *crtc, int clock)
+bool intel_ddi_pll_mode_set(struct drm_crtc *crtc)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
@@ -835,6 +634,7 @@ bool intel_ddi_pll_mode_set(struct drm_crtc *crtc, int clock)
         int type = intel_encoder->type;
         enum pipe pipe = intel_crtc->pipe;
         uint32_t reg, val;
+        int clock = intel_crtc->config.port_clock;
 
         /* TODO: reuse PLLs when possible (compare values) */
 
@@ -863,7 +663,7 @@ bool intel_ddi_pll_mode_set(struct drm_crtc *crtc, int clock)
                 return true;
 
         } else if (type == INTEL_OUTPUT_HDMI) {
-                int p, n2, r2;
+                unsigned p, n2, r2;
 
                 if (plls->wrpll1_refcount == 0) {
                         DRM_DEBUG_KMS("Using WRPLL 1 on pipe %c\n",
@@ -885,7 +685,7 @@ bool intel_ddi_pll_mode_set(struct drm_crtc *crtc, int clock)
                 WARN(I915_READ(reg) & WRPLL_PLL_ENABLE,
                      "WRPLL already enabled\n");
 
-                intel_ddi_calculate_wrpll(clock, &p, &n2, &r2);
+                intel_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
 
                 val = WRPLL_PLL_ENABLE | WRPLL_PLL_SELECT_LCPLL_2700 |
                       WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
@@ -995,7 +795,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
                         /* Can only use the always-on power well for eDP when
                          * not using the panel fitter, and when not using motion
                           * blur mitigation (which we don't support). */
-                        if (dev_priv->pch_pf_size)
+                        if (intel_crtc->config.pch_pfit.size)
                                 temp |= TRANS_DDI_EDP_INPUT_A_ONOFF;
                         else
                                 temp |= TRANS_DDI_EDP_INPUT_A_ON;
@@ -1022,7 +822,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
 
         } else if (type == INTEL_OUTPUT_ANALOG) {
                 temp |= TRANS_DDI_MODE_SELECT_FDI;
-                temp |= (intel_crtc->fdi_lanes - 1) << 1;
+                temp |= (intel_crtc->config.fdi_lanes - 1) << 1;
 
         } else if (type == INTEL_OUTPUT_DISPLAYPORT ||
                    type == INTEL_OUTPUT_EDP) {
@@ -1030,25 +830,10 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
 
                 temp |= TRANS_DDI_MODE_SELECT_DP_SST;
 
-                switch (intel_dp->lane_count) {
-                case 1:
-                        temp |= TRANS_DDI_PORT_WIDTH_X1;
-                        break;
-                case 2:
-                        temp |= TRANS_DDI_PORT_WIDTH_X2;
-                        break;
-                case 4:
-                        temp |= TRANS_DDI_PORT_WIDTH_X4;
-                        break;
-                default:
-                        temp |= TRANS_DDI_PORT_WIDTH_X4;
-                        WARN(1, "Unsupported lane count %d\n",
-                             intel_dp->lane_count);
-                }
-
+                temp |= DDI_PORT_WIDTH(intel_dp->lane_count);
         } else {
-                WARN(1, "Invalid encoder type %d for pipe %d\n",
-                     intel_encoder->type, pipe);
+                WARN(1, "Invalid encoder type %d for pipe %c\n",
+                     intel_encoder->type, pipe_name(pipe));
         }
 
         I915_WRITE(TRANS_DDI_FUNC_CTL(cpu_transcoder), temp);
@@ -1148,7 +933,7 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
                 }
         }
 
-        DRM_DEBUG_KMS("No pipe for ddi port %i found\n", port);
+        DRM_DEBUG_KMS("No pipe for ddi port %c found\n", port_name(port));
 
         return false;
 }
@@ -1334,7 +1119,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder)
                 ironlake_edp_backlight_on(intel_dp);
         }
 
-        if (intel_crtc->eld_vld) {
+        if (intel_crtc->eld_vld && type != INTEL_OUTPUT_EDP) {
                 tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
                 tmp |= ((AUDIO_OUTPUT_ENABLE_A | AUDIO_ELD_VALID_A) << (pipe * 4));
                 I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
@@ -1352,9 +1137,12 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t tmp;
 
-        tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
-        tmp &= ~((AUDIO_OUTPUT_ENABLE_A | AUDIO_ELD_VALID_A) << (pipe * 4));
-        I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
+        if (intel_crtc->eld_vld && type != INTEL_OUTPUT_EDP) {
+                tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
+                tmp &= ~((AUDIO_OUTPUT_ENABLE_A | AUDIO_ELD_VALID_A) <<
+                         (pipe * 4));
+                I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
+        }
 
         if (type == INTEL_OUTPUT_EDP) {
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -1366,14 +1154,14 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
 int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
 {
         if (I915_READ(HSW_FUSE_STRAP) & HSW_CDCLK_LIMIT)
-                return 450;
+                return 450000;
         else if ((I915_READ(LCPLL_CTL) & LCPLL_CLK_FREQ_MASK) ==
                  LCPLL_CLK_FREQ_450)
-                return 450;
+                return 450000;
         else if (IS_ULT(dev_priv->dev))
-                return 338;
+                return 337500;
         else
-                return 540;
+                return 540000;
 }
 
 void intel_ddi_pll_init(struct drm_device *dev)
@@ -1386,7 +1174,7 @@ void intel_ddi_pll_init(struct drm_device *dev)
          * Don't even try to turn it on.
          */
 
-        DRM_DEBUG_KMS("CDCLK running at %dMHz\n",
+        DRM_DEBUG_KMS("CDCLK running at %dKHz\n",
                       intel_ddi_get_cdclk_freq(dev_priv));
 
         if (val & LCPLL_CD_SOURCE_FCLK)
@@ -1472,6 +1260,27 @@ static void intel_ddi_hot_plug(struct intel_encoder *intel_encoder)
                 intel_dp_check_link_status(intel_dp);
 }
 
+static void intel_ddi_get_config(struct intel_encoder *encoder,
+                                 struct intel_crtc_config *pipe_config)
+{
+        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
+        u32 temp, flags = 0;
+
+        temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
+        if (temp & TRANS_DDI_PHSYNC)
+                flags |= DRM_MODE_FLAG_PHSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NHSYNC;
+        if (temp & TRANS_DDI_PVSYNC)
+                flags |= DRM_MODE_FLAG_PVSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NVSYNC;
+
+        pipe_config->adjusted_mode.flags |= flags;
+}
+
 static void intel_ddi_destroy(struct drm_encoder *encoder)
 {
         /* HDMI has nothing special to destroy, so we can go with this. */
@@ -1482,9 +1291,13 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder,
                                      struct intel_crtc_config *pipe_config)
 {
         int type = encoder->type;
+        int port = intel_ddi_get_encoder_port(encoder);
 
         WARN(type == INTEL_OUTPUT_UNKNOWN, "compute_config() on unknown output!\n");
 
+        if (port == PORT_A)
+                pipe_config->cpu_transcoder = TRANSCODER_EDP;
+
         if (type == INTEL_OUTPUT_HDMI)
                 return intel_hdmi_compute_config(encoder, pipe_config);
         else
@@ -1518,16 +1331,6 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
                 return;
         }
 
-        if (port != PORT_A) {
-                hdmi_connector = kzalloc(sizeof(struct intel_connector),
-                                         GFP_KERNEL);
-                if (!hdmi_connector) {
-                        kfree(dp_connector);
-                        kfree(intel_dig_port);
-                        return;
-                }
-        }
-
         intel_encoder = &intel_dig_port->base;
         encoder = &intel_encoder->base;
 
@@ -1541,12 +1344,11 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
         intel_encoder->disable = intel_disable_ddi;
         intel_encoder->post_disable = intel_ddi_post_disable;
         intel_encoder->get_hw_state = intel_ddi_get_hw_state;
+        intel_encoder->get_config = intel_ddi_get_config;
 
         intel_dig_port->port = port;
         intel_dig_port->port_reversal = I915_READ(DDI_BUF_CTL(port)) &
                                         DDI_BUF_PORT_REVERSAL;
-        if (hdmi_connector)
-                intel_dig_port->hdmi.hdmi_reg = DDI_BUF_CTL(port);
         intel_dig_port->dp.output_reg = DDI_BUF_CTL(port);
 
         intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
@@ -1554,7 +1356,21 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
         intel_encoder->cloneable = false;
         intel_encoder->hot_plug = intel_ddi_hot_plug;
 
-        if (hdmi_connector)
+        if (!intel_dp_init_connector(intel_dig_port, dp_connector)) {
+                drm_encoder_cleanup(encoder);
+                kfree(intel_dig_port);
+                kfree(dp_connector);
+                return;
+        }
+
+        if (intel_encoder->type != INTEL_OUTPUT_EDP) {
+                hdmi_connector = kzalloc(sizeof(struct intel_connector),
+                                         GFP_KERNEL);
+                if (!hdmi_connector) {
+                        return;
+                }
+
+                intel_dig_port->hdmi.hdmi_reg = DDI_BUF_CTL(port);
                 intel_hdmi_init_connector(intel_dig_port, hdmi_connector);
-        intel_dp_init_connector(intel_dig_port, dp_connector);
+        }
 }
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 56746dcac40f..85f3eb74d2b7 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -46,18 +46,6 @@ static void intel_increase_pllclock(struct drm_crtc *crtc);
 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
 
 typedef struct {
-        /* given values */
-        int n;
-        int m1, m2;
-        int p1, p2;
-        /* derived values */
-        int     dot;
-        int     vco;
-        int     m;
-        int     p;
-} intel_clock_t;
-
-typedef struct {
         int     min, max;
 } intel_range_t;
 
@@ -71,24 +59,6 @@ typedef struct intel_limit intel_limit_t;
 struct intel_limit {
         intel_range_t   dot, vco, n, m, m1, m2, p, p1;
         intel_p2_t          p2;
-        /**
-         * find_pll() - Find the best values for the PLL
-         * @limit: limits for the PLL
-         * @crtc: current CRTC
-         * @target: target frequency in kHz
-         * @refclk: reference clock frequency in kHz
-         * @match_clock: if provided, @best_clock P divider must
-         *               match the P divider from @match_clock
-         *               used for LVDS downclocking
-         * @best_clock: best PLL values found
-         *
-         * Returns true on success, false on failure.
-         */
-        bool (*find_pll)(const intel_limit_t *limit,
-                         struct drm_crtc *crtc,
-                         int target, int refclk,
-                         intel_clock_t *match_clock,
-                         intel_clock_t *best_clock);
 };
 
 /* FDI */
@@ -104,29 +74,6 @@ intel_pch_rawclk(struct drm_device *dev)
         return I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK;
 }
 
-static bool
-intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
-                    int target, int refclk, intel_clock_t *match_clock,
-                    intel_clock_t *best_clock);
-static bool
-intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
-                        int target, int refclk, intel_clock_t *match_clock,
-                        intel_clock_t *best_clock);
-
-static bool
-intel_find_pll_g4x_dp(const intel_limit_t *, struct drm_crtc *crtc,
-                      int target, int refclk, intel_clock_t *match_clock,
-                      intel_clock_t *best_clock);
-static bool
-intel_find_pll_ironlake_dp(const intel_limit_t *, struct drm_crtc *crtc,
-                           int target, int refclk, intel_clock_t *match_clock,
-                           intel_clock_t *best_clock);
-
-static bool
-intel_vlv_find_best_pll(const intel_limit_t *limit, struct drm_crtc *crtc,
-                        int target, int refclk, intel_clock_t *match_clock,
-                        intel_clock_t *best_clock);
-
 static inline u32 /* units of 100MHz */
 intel_fdi_link_freq(struct drm_device *dev)
 {
@@ -148,7 +95,6 @@ static const intel_limit_t intel_limits_i8xx_dvo = {
         .p1 = { .min = 2, .max = 33 },
         .p2 = { .dot_limit = 165000,
                 .p2_slow = 4, .p2_fast = 2 },
-        .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_i8xx_lvds = {
@@ -162,7 +108,6 @@ static const intel_limit_t intel_limits_i8xx_lvds = {
         .p1 = { .min = 1, .max = 6 },
         .p2 = { .dot_limit = 165000,
                 .p2_slow = 14, .p2_fast = 7 },
-        .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_i9xx_sdvo = {
@@ -176,7 +121,6 @@ static const intel_limit_t intel_limits_i9xx_sdvo = {
         .p1 = { .min = 1, .max = 8 },
         .p2 = { .dot_limit = 200000,
                 .p2_slow = 10, .p2_fast = 5 },
-        .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_i9xx_lvds = {
@@ -190,7 +134,6 @@ static const intel_limit_t intel_limits_i9xx_lvds = {
         .p1 = { .min = 1, .max = 8 },
         .p2 = { .dot_limit = 112000,
                 .p2_slow = 14, .p2_fast = 7 },
-        .find_pll = intel_find_best_PLL,
 };
 
 
@@ -207,7 +150,6 @@ static const intel_limit_t intel_limits_g4x_sdvo = {
                 .p2_slow = 10,
                 .p2_fast = 10
         },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_hdmi = {
@@ -221,7 +163,6 @@ static const intel_limit_t intel_limits_g4x_hdmi = {
         .p1 = { .min = 1, .max = 8},
         .p2 = { .dot_limit = 165000,
                 .p2_slow = 10, .p2_fast = 5 },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
@@ -236,7 +177,6 @@ static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
         .p2 = { .dot_limit = 0,
                 .p2_slow = 14, .p2_fast = 14
         },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
@@ -251,21 +191,6 @@ static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
         .p2 = { .dot_limit = 0,
                 .p2_slow = 7, .p2_fast = 7
         },
-        .find_pll = intel_g4x_find_best_PLL,
-};
-
-static const intel_limit_t intel_limits_g4x_display_port = {
-        .dot = { .min = 161670, .max = 227000 },
-        .vco = { .min = 1750000, .max = 3500000},
-        .n = { .min = 1, .max = 2 },
-        .m = { .min = 97, .max = 108 },
-        .m1 = { .min = 0x10, .max = 0x12 },
-        .m2 = { .min = 0x05, .max = 0x06 },
-        .p = { .min = 10, .max = 20 },
-        .p1 = { .min = 1, .max = 2},
-        .p2 = { .dot_limit = 0,
-                .p2_slow = 10, .p2_fast = 10 },
-        .find_pll = intel_find_pll_g4x_dp,
 };
 
 static const intel_limit_t intel_limits_pineview_sdvo = {
@@ -281,7 +206,6 @@ static const intel_limit_t intel_limits_pineview_sdvo = {
         .p1 = { .min = 1, .max = 8 },
         .p2 = { .dot_limit = 200000,
                 .p2_slow = 10, .p2_fast = 5 },
-        .find_pll = intel_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_pineview_lvds = {
@@ -295,7 +219,6 @@ static const intel_limit_t intel_limits_pineview_lvds = {
         .p1 = { .min = 1, .max = 8 },
         .p2 = { .dot_limit = 112000,
                 .p2_slow = 14, .p2_fast = 14 },
-        .find_pll = intel_find_best_PLL,
 };
 
 /* Ironlake / Sandybridge
@@ -314,7 +237,6 @@ static const intel_limit_t intel_limits_ironlake_dac = {
         .p1 = { .min = 1, .max = 8 },
         .p2 = { .dot_limit = 225000,
                 .p2_slow = 10, .p2_fast = 5 },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_single_lvds = {
@@ -328,7 +250,6 @@ static const intel_limit_t intel_limits_ironlake_single_lvds = {
         .p1 = { .min = 2, .max = 8 },
         .p2 = { .dot_limit = 225000,
                 .p2_slow = 14, .p2_fast = 14 },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
@@ -342,7 +263,6 @@ static const intel_limit_t intel_limits_ironlake_dual_lvds = {
         .p1 = { .min = 2, .max = 8 },
         .p2 = { .dot_limit = 225000,
                 .p2_slow = 7, .p2_fast = 7 },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 /* LVDS 100mhz refclk limits. */
@@ -357,7 +277,6 @@ static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
         .p1 = { .min = 2, .max = 8 },
         .p2 = { .dot_limit = 225000,
                 .p2_slow = 14, .p2_fast = 14 },
-        .find_pll = intel_g4x_find_best_PLL,
 };
 
 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
@@ -371,21 +290,6 @@ static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
         .p1 = { .min = 2, .max = 6 },
         .p2 = { .dot_limit = 225000,
                 .p2_slow = 7, .p2_fast = 7 },
-        .find_pll = intel_g4x_find_best_PLL,
-};
-
-static const intel_limit_t intel_limits_ironlake_display_port = {
-        .dot = { .min = 25000, .max = 350000 },
-        .vco = { .min = 1760000, .max = 3510000},
-        .n = { .min = 1, .max = 2 },
-        .m = { .min = 81, .max = 90 },
-        .m1 = { .min = 12, .max = 22 },
-        .m2 = { .min = 5, .max = 9 },
-        .p = { .min = 10, .max = 20 },
-        .p1 = { .min = 1, .max = 2},
-        .p2 = { .dot_limit = 0,
-                .p2_slow = 10, .p2_fast = 10 },
-        .find_pll = intel_find_pll_ironlake_dp,
 };
 
 static const intel_limit_t intel_limits_vlv_dac = {
@@ -396,15 +300,14 @@ static const intel_limit_t intel_limits_vlv_dac = {
         .m1 = { .min = 2, .max = 3 },
         .m2 = { .min = 11, .max = 156 },
         .p = { .min = 10, .max = 30 },
-        .p1 = { .min = 2, .max = 3 },
+        .p1 = { .min = 1, .max = 3 },
         .p2 = { .dot_limit = 270000,
                 .p2_slow = 2, .p2_fast = 20 },
-        .find_pll = intel_vlv_find_best_pll,
 };
 
 static const intel_limit_t intel_limits_vlv_hdmi = {
-        .dot = { .min = 20000, .max = 165000 },
-        .vco = { .min = 4000000, .max = 5994000},
+        .dot = { .min = 25000, .max = 270000 },
+        .vco = { .min = 4000000, .max = 6000000 },
         .n = { .min = 1, .max = 7 },
         .m = { .min = 60, .max = 300 }, /* guess */
         .m1 = { .min = 2, .max = 3 },
@@ -413,7 +316,6 @@ static const intel_limit_t intel_limits_vlv_hdmi = {
         .p1 = { .min = 2, .max = 3 },
         .p2 = { .dot_limit = 270000,
                 .p2_slow = 2, .p2_fast = 20 },
-        .find_pll = intel_vlv_find_best_pll,
 };
 
 static const intel_limit_t intel_limits_vlv_dp = {
@@ -424,61 +326,11 @@ static const intel_limit_t intel_limits_vlv_dp = {
         .m1 = { .min = 2, .max = 3 },
         .m2 = { .min = 11, .max = 156 },
         .p = { .min = 10, .max = 30 },
-        .p1 = { .min = 2, .max = 3 },
+        .p1 = { .min = 1, .max = 3 },
         .p2 = { .dot_limit = 270000,
                 .p2_slow = 2, .p2_fast = 20 },
-        .find_pll = intel_vlv_find_best_pll,
 };
 
-u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg)
-{
-        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
-
-        if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
-                DRM_ERROR("DPIO idle wait timed out\n");
-                return 0;
-        }
-
-        I915_WRITE(DPIO_REG, reg);
-        I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_READ | DPIO_PORTID |
-                   DPIO_BYTE);
-        if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
-                DRM_ERROR("DPIO read wait timed out\n");
-                return 0;
-        }
-
-        return I915_READ(DPIO_DATA);
-}
-
-static void intel_dpio_write(struct drm_i915_private *dev_priv, int reg,
-                             u32 val)
-{
-        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
-
-        if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
-                DRM_ERROR("DPIO idle wait timed out\n");
-                return;
-        }
-
-        I915_WRITE(DPIO_DATA, val);
-        I915_WRITE(DPIO_REG, reg);
-        I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_WRITE | DPIO_PORTID |
-                   DPIO_BYTE);
-        if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100))
-                DRM_ERROR("DPIO write wait timed out\n");
-}
-
-static void vlv_init_dpio(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        /* Reset the DPIO config */
-        I915_WRITE(DPIO_CTL, 0);
-        POSTING_READ(DPIO_CTL);
-        I915_WRITE(DPIO_CTL, 1);
-        POSTING_READ(DPIO_CTL);
-}
-
 static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
                                                 int refclk)
 {
@@ -497,10 +349,7 @@ static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
                         else
                                 limit = &intel_limits_ironlake_single_lvds;
                 }
-        } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
-                   intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
-                limit = &intel_limits_ironlake_display_port;
-        else
+        } else
                 limit = &intel_limits_ironlake_dac;
 
         return limit;
@@ -521,8 +370,6 @@ static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
                 limit = &intel_limits_g4x_hdmi;
         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) {
                 limit = &intel_limits_g4x_sdvo;
-        } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
-                limit = &intel_limits_g4x_display_port;
         } else /* The option is for other outputs */
                 limit = &intel_limits_i9xx_sdvo;
 
@@ -573,13 +420,14 @@ static void pineview_clock(int refclk, intel_clock_t *clock)
         clock->dot = clock->vco / clock->p;
 }
 
-static void intel_clock(struct drm_device *dev, int refclk, intel_clock_t *clock)
+static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 {
-        if (IS_PINEVIEW(dev)) {
-                pineview_clock(refclk, clock);
-                return;
-        }
-        clock->m = 5 * (clock->m1 + 2) + (clock->m2 + 2);
+        return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
+}
+
+static void i9xx_clock(int refclk, intel_clock_t *clock)
+{
+        clock->m = i9xx_dpll_compute_m(clock);
         clock->p = clock->p1 * clock->p2;
         clock->vco = refclk * clock->m / (clock->n + 2);
         clock->dot = clock->vco / clock->p;
@@ -636,10 +484,9 @@ static bool intel_PLL_is_valid(struct drm_device *dev,
 }
 
 static bool
-intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
+i9xx_find_best_dpll(const intel_limit_t *limit, struct drm_crtc *crtc,
                     int target, int refclk, intel_clock_t *match_clock,
                     intel_clock_t *best_clock)
-
 {
         struct drm_device *dev = crtc->dev;
         intel_clock_t clock;
@@ -668,8 +515,7 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
              clock.m1++) {
                 for (clock.m2 = limit->m2.min;
                      clock.m2 <= limit->m2.max; clock.m2++) {
-                        /* m1 is always 0 in Pineview */
-                        if (clock.m2 >= clock.m1 && !IS_PINEVIEW(dev))
+                        if (clock.m2 >= clock.m1)
                                 break;
                         for (clock.n = limit->n.min;
                              clock.n <= limit->n.max; clock.n++) {
@@ -677,7 +523,66 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                         clock.p1 <= limit->p1.max; clock.p1++) {
                                         int this_err;
 
-                                        intel_clock(dev, refclk, &clock);
+                                        i9xx_clock(refclk, &clock);
+                                        if (!intel_PLL_is_valid(dev, limit,
+                                                                &clock))
+                                                continue;
+                                        if (match_clock &&
+                                            clock.p != match_clock->p)
+                                                continue;
+
+                                        this_err = abs(clock.dot - target);
+                                        if (this_err < err) {
+                                                *best_clock = clock;
+                                                err = this_err;
+                                        }
+                                }
+                        }
+                }
+        }
+
+        return (err != target);
+}
+
+static bool
+pnv_find_best_dpll(const intel_limit_t *limit, struct drm_crtc *crtc,
+                   int target, int refclk, intel_clock_t *match_clock,
+                   intel_clock_t *best_clock)
+{
+        struct drm_device *dev = crtc->dev;
+        intel_clock_t clock;
+        int err = target;
+
+        if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
+                /*
+                 * For LVDS just rely on its current settings for dual-channel.
+                 * We haven't figured out how to reliably set up different
+                 * single/dual channel state, if we even can.
+                 */
+                if (intel_is_dual_link_lvds(dev))
+                        clock.p2 = limit->p2.p2_fast;
+                else
+                        clock.p2 = limit->p2.p2_slow;
+        } else {
+                if (target < limit->p2.dot_limit)
+                        clock.p2 = limit->p2.p2_slow;
+                else
+                        clock.p2 = limit->p2.p2_fast;
+        }
+
+        memset(best_clock, 0, sizeof(*best_clock));
+
+        for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
+             clock.m1++) {
+                for (clock.m2 = limit->m2.min;
+                     clock.m2 <= limit->m2.max; clock.m2++) {
+                        for (clock.n = limit->n.min;
+                             clock.n <= limit->n.max; clock.n++) {
+                                for (clock.p1 = limit->p1.min;
+                                        clock.p1 <= limit->p1.max; clock.p1++) {
+                                        int this_err;
+
+                                        pineview_clock(refclk, &clock);
                                         if (!intel_PLL_is_valid(dev, limit,
                                                                 &clock))
                                                 continue;
@@ -699,9 +604,9 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 }
 
 static bool
-intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
-                        int target, int refclk, intel_clock_t *match_clock,
-                        intel_clock_t *best_clock)
+g4x_find_best_dpll(const intel_limit_t *limit, struct drm_crtc *crtc,
+                   int target, int refclk, intel_clock_t *match_clock,
+                   intel_clock_t *best_clock)
 {
         struct drm_device *dev = crtc->dev;
         intel_clock_t clock;
@@ -712,12 +617,6 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
         found = false;
 
         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-                int lvds_reg;
-
-                if (HAS_PCH_SPLIT(dev))
-                        lvds_reg = PCH_LVDS;
-                else
-                        lvds_reg = LVDS;
                 if (intel_is_dual_link_lvds(dev))
                         clock.p2 = limit->p2.p2_fast;
                 else
@@ -742,13 +641,10 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                      clock.p1 >= limit->p1.min; clock.p1--) {
                                         int this_err;
 
-                                        intel_clock(dev, refclk, &clock);
+                                        i9xx_clock(refclk, &clock);
                                         if (!intel_PLL_is_valid(dev, limit,
                                                                 &clock))
                                                 continue;
-                                        if (match_clock &&
-                                            clock.p != match_clock->p)
-                                                continue;
 
                                         this_err = abs(clock.dot - target);
                                         if (this_err < err_most) {
@@ -765,62 +661,9 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 }
 
 static bool
-intel_find_pll_ironlake_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
-                           int target, int refclk, intel_clock_t *match_clock,
-                           intel_clock_t *best_clock)
-{
-        struct drm_device *dev = crtc->dev;
-        intel_clock_t clock;
-
-        if (target < 200000) {
-                clock.n = 1;
-                clock.p1 = 2;
-                clock.p2 = 10;
-                clock.m1 = 12;
-                clock.m2 = 9;
-        } else {
-                clock.n = 2;
-                clock.p1 = 1;
-                clock.p2 = 10;
-                clock.m1 = 14;
-                clock.m2 = 8;
-        }
-        intel_clock(dev, refclk, &clock);
-        memcpy(best_clock, &clock, sizeof(intel_clock_t));
-        return true;
-}
-
-/* DisplayPort has only two frequencies, 162MHz and 270MHz */
-static bool
-intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
-                      int target, int refclk, intel_clock_t *match_clock,
-                      intel_clock_t *best_clock)
-{
-        intel_clock_t clock;
-        if (target < 200000) {
-                clock.p1 = 2;
-                clock.p2 = 10;
-                clock.n = 2;
-                clock.m1 = 23;
-                clock.m2 = 8;
-        } else {
-                clock.p1 = 1;
-                clock.p2 = 10;
-                clock.n = 1;
-                clock.m1 = 14;
-                clock.m2 = 2;
-        }
-        clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2);
-        clock.p = (clock.p1 * clock.p2);
-        clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p;
-        clock.vco = 0;
-        memcpy(best_clock, &clock, sizeof(intel_clock_t));
-        return true;
-}
-static bool
-intel_vlv_find_best_pll(const intel_limit_t *limit, struct drm_crtc *crtc,
-                        int target, int refclk, intel_clock_t *match_clock,
-                        intel_clock_t *best_clock)
+vlv_find_best_dpll(const intel_limit_t *limit, struct drm_crtc *crtc,
+                   int target, int refclk, intel_clock_t *match_clock,
+                   intel_clock_t *best_clock)
 {
         u32 p1, p2, m1, m2, vco, bestn, bestm1, bestm2, bestp1, bestp2;
         u32 m, n, fastclk;
@@ -1066,14 +909,24 @@ static void assert_pll(struct drm_i915_private *dev_priv,
 #define assert_pll_enabled(d, p) assert_pll(d, p, true)
 #define assert_pll_disabled(d, p) assert_pll(d, p, false)
 
+static struct intel_shared_dpll *
+intel_crtc_to_shared_dpll(struct intel_crtc *crtc)
+{
+        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+
+        if (crtc->config.shared_dpll < 0)
+                return NULL;
+
+        return &dev_priv->shared_dplls[crtc->config.shared_dpll];
+}
+
 /* For ILK+ */
-static void assert_pch_pll(struct drm_i915_private *dev_priv,
-                           struct intel_pch_pll *pll,
-                           struct intel_crtc *crtc,
-                           bool state)
+static void assert_shared_dpll(struct drm_i915_private *dev_priv,
+                               struct intel_shared_dpll *pll,
+                               bool state)
 {
-        u32 val;
         bool cur_state;
+        struct intel_dpll_hw_state hw_state;
 
         if (HAS_PCH_LPT(dev_priv->dev)) {
                 DRM_DEBUG_DRIVER("LPT detected: skipping PCH PLL test\n");
@@ -1081,36 +934,16 @@ static void assert_pch_pll(struct drm_i915_private *dev_priv,
         }
 
         if (WARN (!pll,
-                  "asserting PCH PLL %s with no PLL\n", state_string(state)))
+                  "asserting DPLL %s with no DPLL\n", state_string(state)))
                 return;
 
-        val = I915_READ(pll->pll_reg);
-        cur_state = !!(val & DPLL_VCO_ENABLE);
+        cur_state = pll->get_hw_state(dev_priv, pll, &hw_state);
         WARN(cur_state != state,
-             "PCH PLL state for reg %x assertion failure (expected %s, current %s), val=%08x\n",
-             pll->pll_reg, state_string(state), state_string(cur_state), val);
-
-        /* Make sure the selected PLL is correctly attached to the transcoder */
-        if (crtc && HAS_PCH_CPT(dev_priv->dev)) {
-                u32 pch_dpll;
-
-                pch_dpll = I915_READ(PCH_DPLL_SEL);
-                cur_state = pll->pll_reg == _PCH_DPLL_B;
-                if (!WARN(((pch_dpll >> (4 * crtc->pipe)) & 1) != cur_state,
-                          "PLL[%d] not attached to this transcoder %d: %08x\n",
-                          cur_state, crtc->pipe, pch_dpll)) {
-                        cur_state = !!(val >> (4*crtc->pipe + 3));
-                        WARN(cur_state != state,
-                             "PLL[%d] not %s on this transcoder %d: %08x\n",
-                             pll->pll_reg == _PCH_DPLL_B,
-                             state_string(state),
-                             crtc->pipe,
-                             val);
-                }
-        }
+             "%s assertion failure (expected %s, current %s)\n",
+             pll->name, state_string(state), state_string(cur_state));
 }
-#define assert_pch_pll_enabled(d, p, c) assert_pch_pll(d, p, c, true)
-#define assert_pch_pll_disabled(d, p, c) assert_pch_pll(d, p, c, false)
+#define assert_shared_dpll_enabled(d, p) assert_shared_dpll(d, p, true)
+#define assert_shared_dpll_disabled(d, p) assert_shared_dpll(d, p, false)
 
 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
                           enum pipe pipe, bool state)
@@ -1227,8 +1060,8 @@ void assert_pipe(struct drm_i915_private *dev_priv,
         if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
                 state = true;
 
-        if (!intel_using_power_well(dev_priv->dev) &&
-            cpu_transcoder != TRANSCODER_EDP) {
+        if (!intel_display_power_enabled(dev_priv->dev,
+                                POWER_DOMAIN_TRANSCODER(cpu_transcoder))) {
                 cur_state = false;
         } else {
                 reg = PIPECONF(cpu_transcoder);
@@ -1262,12 +1095,13 @@ static void assert_plane(struct drm_i915_private *dev_priv,
 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
                                    enum pipe pipe)
 {
+        struct drm_device *dev = dev_priv->dev;
         int reg, i;
         u32 val;
         int cur_pipe;
 
-        /* Planes are fixed to pipes on ILK+ */
-        if (HAS_PCH_SPLIT(dev_priv->dev) || IS_VALLEYVIEW(dev_priv->dev)) {
+        /* Primary planes are fixed to pipes on gen4+ */
+        if (INTEL_INFO(dev)->gen >= 4) {
                 reg = DSPCNTR(pipe);
                 val = I915_READ(reg);
                 WARN((val & DISPLAY_PLANE_ENABLE),
@@ -1277,7 +1111,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
         }
 
         /* Need to check both planes against the pipe */
-        for (i = 0; i < 2; i++) {
+        for (i = 0; i < INTEL_INFO(dev)->num_pipes; i++) {
                 reg = DSPCNTR(i);
                 val = I915_READ(reg);
                 cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
@@ -1291,19 +1125,30 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
                                     enum pipe pipe)
 {
+        struct drm_device *dev = dev_priv->dev;
         int reg, i;
         u32 val;
 
-        if (!IS_VALLEYVIEW(dev_priv->dev))
-                return;
-
-        /* Need to check both planes against the pipe */
-        for (i = 0; i < dev_priv->num_plane; i++) {
-                reg = SPCNTR(pipe, i);
+        if (IS_VALLEYVIEW(dev)) {
+                for (i = 0; i < dev_priv->num_plane; i++) {
+                        reg = SPCNTR(pipe, i);
+                        val = I915_READ(reg);
+                        WARN((val & SP_ENABLE),
+                             "sprite %c assertion failure, should be off on pipe %c but is still active\n",
+                             sprite_name(pipe, i), pipe_name(pipe));
+                }
+        } else if (INTEL_INFO(dev)->gen >= 7) {
+                reg = SPRCTL(pipe);
                 val = I915_READ(reg);
-                WARN((val & SP_ENABLE),
-                     "sprite %d assertion failure, should be off on pipe %c but is still active\n",
-                     pipe * 2 + i, pipe_name(pipe));
+                WARN((val & SPRITE_ENABLE),
+                     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
+                     plane_name(pipe), pipe_name(pipe));
+        } else if (INTEL_INFO(dev)->gen >= 5) {
+                reg = DVSCNTR(pipe);
+                val = I915_READ(reg);
+                WARN((val & DVS_ENABLE),
+                     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
+                     plane_name(pipe), pipe_name(pipe));
         }
 }
 
@@ -1323,14 +1168,14 @@ static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
         WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
 }
 
-static void assert_transcoder_disabled(struct drm_i915_private *dev_priv,
-                                       enum pipe pipe)
+static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
+                                           enum pipe pipe)
 {
         int reg;
         u32 val;
         bool enabled;
 
-        reg = TRANSCONF(pipe);
+        reg = PCH_TRANSCONF(pipe);
         val = I915_READ(reg);
         enabled = !!(val & TRANS_ENABLE);
         WARN(enabled,
@@ -1474,6 +1319,8 @@ static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
         int reg;
         u32 val;
 
+        assert_pipe_disabled(dev_priv, pipe);
+
         /* No really, not for ILK+ */
         BUG_ON(!IS_VALLEYVIEW(dev_priv->dev) && dev_priv->info->gen >= 5);
 
@@ -1525,156 +1372,86 @@ static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
         POSTING_READ(reg);
 }
 
-/* SBI access */
-static void
-intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
-                enum intel_sbi_destination destination)
+void vlv_wait_port_ready(struct drm_i915_private *dev_priv, int port)
 {
-        u32 tmp;
+        u32 port_mask;
 
-        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
-
-        if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0,
-                                100)) {
-                DRM_ERROR("timeout waiting for SBI to become ready\n");
-                return;
-        }
-
-        I915_WRITE(SBI_ADDR, (reg << 16));
-        I915_WRITE(SBI_DATA, value);
-
-        if (destination == SBI_ICLK)
-                tmp = SBI_CTL_DEST_ICLK | SBI_CTL_OP_CRWR;
+        if (!port)
+                port_mask = DPLL_PORTB_READY_MASK;
         else
-                tmp = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IOWR;
-        I915_WRITE(SBI_CTL_STAT, SBI_BUSY | tmp);
+                port_mask = DPLL_PORTC_READY_MASK;
 
-        if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0,
-                                100)) {
-                DRM_ERROR("timeout waiting for SBI to complete write transaction\n");
-                return;
-        }
-}
-
-static u32
-intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
-               enum intel_sbi_destination destination)
-{
-        u32 value = 0;
-        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
-
-        if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0,
-                                100)) {
-                DRM_ERROR("timeout waiting for SBI to become ready\n");
-                return 0;
-        }
-
-        I915_WRITE(SBI_ADDR, (reg << 16));
-
-        if (destination == SBI_ICLK)
-                value = SBI_CTL_DEST_ICLK | SBI_CTL_OP_CRRD;
-        else
-                value = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD;
-        I915_WRITE(SBI_CTL_STAT, value | SBI_BUSY);
-
-        if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0,
-                                100)) {
-                DRM_ERROR("timeout waiting for SBI to complete read transaction\n");
-                return 0;
-        }
-
-        return I915_READ(SBI_DATA);
+        if (wait_for((I915_READ(DPLL(0)) & port_mask) == 0, 1000))
+                WARN(1, "timed out waiting for port %c ready: 0x%08x\n",
+                     'B' + port, I915_READ(DPLL(0)));
 }
 
 /**
- * ironlake_enable_pch_pll - enable PCH PLL
+ * ironlake_enable_shared_dpll - enable PCH PLL
  * @dev_priv: i915 private structure
  * @pipe: pipe PLL to enable
  *
  * The PCH PLL needs to be enabled before the PCH transcoder, since it
  * drives the transcoder clock.
  */
-static void ironlake_enable_pch_pll(struct intel_crtc *intel_crtc)
+static void ironlake_enable_shared_dpll(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
-        struct intel_pch_pll *pll;
-        int reg;
-        u32 val;
+        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
 
         /* PCH PLLs only available on ILK, SNB and IVB */
         BUG_ON(dev_priv->info->gen < 5);
-        pll = intel_crtc->pch_pll;
-        if (pll == NULL)
+        if (WARN_ON(pll == NULL))
                 return;
 
         if (WARN_ON(pll->refcount == 0))
                 return;
 
-        DRM_DEBUG_KMS("enable PCH PLL %x (active %d, on? %d)for crtc %d\n",
-                      pll->pll_reg, pll->active, pll->on,
-                      intel_crtc->base.base.id);
-
-        /* PCH refclock must be enabled first */
-        assert_pch_refclk_enabled(dev_priv);
+        DRM_DEBUG_KMS("enable %s (active %d, on? %d)for crtc %d\n",
+                      pll->name, pll->active, pll->on,
+                      crtc->base.base.id);
 
-        if (pll->active++ && pll->on) {
-                assert_pch_pll_enabled(dev_priv, pll, NULL);
+        if (pll->active++) {
+                WARN_ON(!pll->on);
+                assert_shared_dpll_enabled(dev_priv, pll);
                 return;
         }
+        WARN_ON(pll->on);
 
-        DRM_DEBUG_KMS("enabling PCH PLL %x\n", pll->pll_reg);
-
-        reg = pll->pll_reg;
-        val = I915_READ(reg);
-        val |= DPLL_VCO_ENABLE;
-        I915_WRITE(reg, val);
-        POSTING_READ(reg);
-        udelay(200);
-
+        DRM_DEBUG_KMS("enabling %s\n", pll->name);
+        pll->enable(dev_priv, pll);
         pll->on = true;
 }
 
-static void intel_disable_pch_pll(struct intel_crtc *intel_crtc)
+static void intel_disable_shared_dpll(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
-        struct intel_pch_pll *pll = intel_crtc->pch_pll;
-        int reg;
-        u32 val;
+        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
 
         /* PCH only available on ILK+ */
         BUG_ON(dev_priv->info->gen < 5);
-        if (pll == NULL)
+        if (WARN_ON(pll == NULL))
                return;
 
         if (WARN_ON(pll->refcount == 0))
                 return;
 
-        DRM_DEBUG_KMS("disable PCH PLL %x (active %d, on? %d) for crtc %d\n",
-                      pll->pll_reg, pll->active, pll->on,
-                      intel_crtc->base.base.id);
+        DRM_DEBUG_KMS("disable %s (active %d, on? %d) for crtc %d\n",
+                      pll->name, pll->active, pll->on,
+                      crtc->base.base.id);
 
         if (WARN_ON(pll->active == 0)) {
-                assert_pch_pll_disabled(dev_priv, pll, NULL);
+                assert_shared_dpll_disabled(dev_priv, pll);
                 return;
         }
 
-        if (--pll->active) {
-                assert_pch_pll_enabled(dev_priv, pll, NULL);
+        assert_shared_dpll_enabled(dev_priv, pll);
+        WARN_ON(!pll->on);
+        if (--pll->active)
                 return;
-        }
-
-        DRM_DEBUG_KMS("disabling PCH PLL %x\n", pll->pll_reg);
-
-        /* Make sure transcoder isn't still depending on us */
-        assert_transcoder_disabled(dev_priv, intel_crtc->pipe);
-
-        reg = pll->pll_reg;
-        val = I915_READ(reg);
-        val &= ~DPLL_VCO_ENABLE;
-        I915_WRITE(reg, val);
-        POSTING_READ(reg);
-        udelay(200);
 
+        DRM_DEBUG_KMS("disabling %s\n", pll->name);
+        pll->disable(dev_priv, pll);
         pll->on = false;
 }
 
@@ -1683,15 +1460,15 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 {
         struct drm_device *dev = dev_priv->dev;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         uint32_t reg, val, pipeconf_val;
 
         /* PCH only available on ILK+ */
         BUG_ON(dev_priv->info->gen < 5);
 
         /* Make sure PCH DPLL is enabled */
-        assert_pch_pll_enabled(dev_priv,
-                               to_intel_crtc(crtc)->pch_pll,
-                               to_intel_crtc(crtc));
+        assert_shared_dpll_enabled(dev_priv,
+                                   intel_crtc_to_shared_dpll(intel_crtc));
 
         /* FDI must be feeding us bits for PCH ports */
         assert_fdi_tx_enabled(dev_priv, pipe);
@@ -1706,7 +1483,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
                 I915_WRITE(reg, val);
         }
 
-        reg = TRANSCONF(pipe);
+        reg = PCH_TRANSCONF(pipe);
         val = I915_READ(reg);
         pipeconf_val = I915_READ(PIPECONF(pipe));
 
@@ -1731,7 +1508,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 
         I915_WRITE(reg, val | TRANS_ENABLE);
         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
-                DRM_ERROR("failed to enable transcoder %d\n", pipe);
+                DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
 }
 
 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
@@ -1760,8 +1537,8 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         else
                 val |= TRANS_PROGRESSIVE;
 
-        I915_WRITE(TRANSCONF(TRANSCODER_A), val);
-        if (wait_for(I915_READ(_TRANSACONF) & TRANS_STATE_ENABLE, 100))
+        I915_WRITE(LPT_TRANSCONF, val);
+        if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100))
                 DRM_ERROR("Failed to enable PCH transcoder\n");
 }
 
@@ -1778,13 +1555,13 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
         /* Ports must be off as well */
         assert_pch_ports_disabled(dev_priv, pipe);
 
-        reg = TRANSCONF(pipe);
+        reg = PCH_TRANSCONF(pipe);
         val = I915_READ(reg);
         val &= ~TRANS_ENABLE;
         I915_WRITE(reg, val);
         /* wait for PCH transcoder off, transcoder state */
         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
-                DRM_ERROR("failed to disable transcoder %d\n", pipe);
+                DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
 
         if (!HAS_PCH_IBX(dev)) {
                 /* Workaround: Clear the timing override chicken bit again. */
@@ -1799,11 +1576,11 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 {
         u32 val;
 
-        val = I915_READ(_TRANSACONF);
+        val = I915_READ(LPT_TRANSCONF);
         val &= ~TRANS_ENABLE;
-        I915_WRITE(_TRANSACONF, val);
+        I915_WRITE(LPT_TRANSCONF, val);
         /* wait for PCH transcoder off, transcoder state */
-        if (wait_for((I915_READ(_TRANSACONF) & TRANS_STATE_ENABLE) == 0, 50))
+        if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50))
                 DRM_ERROR("Failed to disable PCH transcoder\n");
 
         /* Workaround: clear timing override bit. */
@@ -1835,6 +1612,9 @@ static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
         int reg;
         u32 val;
 
+        assert_planes_disabled(dev_priv, pipe);
+        assert_sprites_disabled(dev_priv, pipe);
+
         if (HAS_PCH_LPT(dev_priv->dev))
                 pch_transcoder = TRANSCODER_A;
         else
@@ -2096,7 +1876,7 @@ static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         case 1:
                 break;
         default:
-                DRM_ERROR("Can't update plane %d in SAREA\n", plane);
+                DRM_ERROR("Can't update plane %c in SAREA\n", plane_name(plane));
                 return -EINVAL;
         }
 
@@ -2145,6 +1925,9 @@ static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                         dspcntr &= ~DISPPLANE_TILED;
         }
 
+        if (IS_G4X(dev))
+                dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
+
         I915_WRITE(reg, dspcntr);
 
         linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
@@ -2193,7 +1976,7 @@ static int ironlake_update_plane(struct drm_crtc *crtc,
         case 2:
                 break;
         default:
-                DRM_ERROR("Can't update plane %d in SAREA\n", plane);
+                DRM_ERROR("Can't update plane %c in SAREA\n", plane_name(plane));
                 return -EINVAL;
         }
 
@@ -2384,9 +2167,9 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         }
 
         if (intel_crtc->plane > INTEL_INFO(dev)->num_pipes) {
-                DRM_ERROR("no plane for crtc: plane %d, num_pipes %d\n",
-                                intel_crtc->plane,
-                                INTEL_INFO(dev)->num_pipes);
+                DRM_ERROR("no plane for crtc: plane %c, num_pipes %d\n",
+                          plane_name(intel_crtc->plane),
+                          INTEL_INFO(dev)->num_pipes);
                 return -EINVAL;
         }
 
@@ -2414,7 +2197,8 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         crtc->y = y;
 
         if (old_fb) {
-                intel_wait_for_vblank(dev, intel_crtc->pipe);
+                if (intel_crtc->active && old_fb != fb)
+                        intel_wait_for_vblank(dev, intel_crtc->pipe);
                 intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
         }
 
@@ -2467,6 +2251,11 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
                            FDI_FE_ERRC_ENABLE);
 }
 
+static bool pipe_has_enabled_pch(struct intel_crtc *intel_crtc)
+{
+        return intel_crtc->base.enabled && intel_crtc->config.has_pch_encoder;
+}
+
 static void ivb_modeset_global_resources(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2476,10 +2265,13 @@ static void ivb_modeset_global_resources(struct drm_device *dev)
                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_C]);
         uint32_t temp;
 
-        /* When everything is off disable fdi C so that we could enable fdi B
-         * with all lanes. XXX: This misses the case where a pipe is not using
-         * any pch resources and so doesn't need any fdi lanes. */
-        if (!pipe_B_crtc->base.enabled && !pipe_C_crtc->base.enabled) {
+        /*
+         * When everything is off disable fdi C so that we could enable fdi B
+         * with all lanes. Note that we don't care about enabled pipes without
+         * an enabled pch encoder.
+         */
+        if (!pipe_has_enabled_pch(pipe_B_crtc) &&
+            !pipe_has_enabled_pch(pipe_C_crtc)) {
                 WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
                 WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
 
@@ -2517,8 +2309,8 @@ static void ironlake_fdi_link_train(struct drm_crtc *crtc)
         /* enable CPU FDI TX and PCH FDI RX */
         reg = FDI_TX_CTL(pipe);
         temp = I915_READ(reg);
-        temp &= ~(7 << 19);
-        temp |= (intel_crtc->fdi_lanes - 1) << 19;
+        temp &= ~FDI_DP_PORT_WIDTH_MASK;
+        temp |= FDI_DP_PORT_WIDTH(intel_crtc->config.fdi_lanes);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_1;
         I915_WRITE(reg, temp | FDI_TX_ENABLE);
@@ -2615,8 +2407,8 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
         /* enable CPU FDI TX and PCH FDI RX */
         reg = FDI_TX_CTL(pipe);
         temp = I915_READ(reg);
-        temp &= ~(7 << 19);
-        temp |= (intel_crtc->fdi_lanes - 1) << 19;
+        temp &= ~FDI_DP_PORT_WIDTH_MASK;
+        temp |= FDI_DP_PORT_WIDTH(intel_crtc->config.fdi_lanes);
         temp &= ~FDI_LINK_TRAIN_NONE;
         temp |= FDI_LINK_TRAIN_PATTERN_1;
         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
@@ -2750,8 +2542,8 @@ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
         /* enable CPU FDI TX and PCH FDI RX */
         reg = FDI_TX_CTL(pipe);
         temp = I915_READ(reg);
-        temp &= ~(7 << 19);
-        temp |= (intel_crtc->fdi_lanes - 1) << 19;
+        temp &= ~FDI_DP_PORT_WIDTH_MASK;
+        temp |= FDI_DP_PORT_WIDTH(intel_crtc->config.fdi_lanes);
         temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
         temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
@@ -2852,8 +2644,8 @@ static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
         reg = FDI_RX_CTL(pipe);
         temp = I915_READ(reg);
-        temp &= ~((0x7 << 19) | (0x7 << 16));
-        temp |= (intel_crtc->fdi_lanes - 1) << 19;
+        temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
+        temp |= FDI_DP_PORT_WIDTH(intel_crtc->config.fdi_lanes);
         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
 
@@ -3085,6 +2877,30 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
         mutex_unlock(&dev_priv->dpio_lock);
 }
 
+static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
+                                                enum pipe pch_transcoder)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum transcoder cpu_transcoder = crtc->config.cpu_transcoder;
+
+        I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
+                   I915_READ(HTOTAL(cpu_transcoder)));
+        I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
+                   I915_READ(HBLANK(cpu_transcoder)));
+        I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
+                   I915_READ(HSYNC(cpu_transcoder)));
+
+        I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
+                   I915_READ(VTOTAL(cpu_transcoder)));
+        I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
+                   I915_READ(VBLANK(cpu_transcoder)));
+        I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
+                   I915_READ(VSYNC(cpu_transcoder)));
+        I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
+                   I915_READ(VSYNCSHIFT(cpu_transcoder)));
+}
+
 /*
  * Enable PCH resources required for PCH ports:
  *   - PCH PLLs
@@ -3101,7 +2917,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
         int pipe = intel_crtc->pipe;
         u32 reg, temp;
 
-        assert_transcoder_disabled(dev_priv, pipe);
+        assert_pch_transcoder_disabled(dev_priv, pipe);
 
         /* Write the TU size bits before fdi link training, so that error
          * detection works. */
@@ -3115,31 +2931,18 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
          * transcoder, and we actually should do this to not upset any PCH
          * transcoder that already use the clock when we share it.
          *
-         * Note that enable_pch_pll tries to do the right thing, but get_pch_pll
-         * unconditionally resets the pll - we need that to have the right LVDS
-         * enable sequence. */
-        ironlake_enable_pch_pll(intel_crtc);
+         * Note that enable_shared_dpll tries to do the right thing, but
+         * get_shared_dpll unconditionally resets the pll - we need that to have
+         * the right LVDS enable sequence. */
+        ironlake_enable_shared_dpll(intel_crtc);
 
         if (HAS_PCH_CPT(dev)) {
                 u32 sel;
 
                 temp = I915_READ(PCH_DPLL_SEL);
-                switch (pipe) {
-                default:
-                case 0:
-                        temp |= TRANSA_DPLL_ENABLE;
-                        sel = TRANSA_DPLLB_SEL;
-                        break;
-                case 1:
-                        temp |= TRANSB_DPLL_ENABLE;
-                        sel = TRANSB_DPLLB_SEL;
-                        break;
-                case 2:
-                        temp |= TRANSC_DPLL_ENABLE;
-                        sel = TRANSC_DPLLB_SEL;
-                        break;
-                }
-                if (intel_crtc->pch_pll->pll_reg == _PCH_DPLL_B)
+                temp |= TRANS_DPLL_ENABLE(pipe);
+                sel = TRANS_DPLLB_SEL(pipe);
+                if (intel_crtc->config.shared_dpll == DPLL_ID_PCH_PLL_B)
                         temp |= sel;
                 else
                         temp &= ~sel;
@@ -3148,14 +2951,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 
         /* set transcoder timing, panel must allow it */
         assert_panel_unlocked(dev_priv, pipe);
-        I915_WRITE(TRANS_HTOTAL(pipe), I915_READ(HTOTAL(pipe)));
-        I915_WRITE(TRANS_HBLANK(pipe), I915_READ(HBLANK(pipe)));
-        I915_WRITE(TRANS_HSYNC(pipe),  I915_READ(HSYNC(pipe)));
-
-        I915_WRITE(TRANS_VTOTAL(pipe), I915_READ(VTOTAL(pipe)));
-        I915_WRITE(TRANS_VBLANK(pipe), I915_READ(VBLANK(pipe)));
-        I915_WRITE(TRANS_VSYNC(pipe),  I915_READ(VSYNC(pipe)));
-        I915_WRITE(TRANS_VSYNCSHIFT(pipe),  I915_READ(VSYNCSHIFT(pipe)));
+        ironlake_pch_transcoder_set_timings(intel_crtc, pipe);
 
         intel_fdi_normal_train(crtc);
 
@@ -3205,86 +3001,82 @@ static void lpt_pch_enable(struct drm_crtc *crtc)
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
 
-        assert_transcoder_disabled(dev_priv, TRANSCODER_A);
+        assert_pch_transcoder_disabled(dev_priv, TRANSCODER_A);
 
         lpt_program_iclkip(crtc);
 
         /* Set transcoder timing. */
-        I915_WRITE(_TRANS_HTOTAL_A, I915_READ(HTOTAL(cpu_transcoder)));
-        I915_WRITE(_TRANS_HBLANK_A, I915_READ(HBLANK(cpu_transcoder)));
-        I915_WRITE(_TRANS_HSYNC_A,  I915_READ(HSYNC(cpu_transcoder)));
-
-        I915_WRITE(_TRANS_VTOTAL_A, I915_READ(VTOTAL(cpu_transcoder)));
-        I915_WRITE(_TRANS_VBLANK_A, I915_READ(VBLANK(cpu_transcoder)));
-        I915_WRITE(_TRANS_VSYNC_A,  I915_READ(VSYNC(cpu_transcoder)));
-        I915_WRITE(_TRANS_VSYNCSHIFT_A, I915_READ(VSYNCSHIFT(cpu_transcoder)));
+        ironlake_pch_transcoder_set_timings(intel_crtc, PIPE_A);
 
         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
 }
 
-static void intel_put_pch_pll(struct intel_crtc *intel_crtc)
+static void intel_put_shared_dpll(struct intel_crtc *crtc)
 {
-        struct intel_pch_pll *pll = intel_crtc->pch_pll;
+        struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
 
         if (pll == NULL)
                 return;
 
         if (pll->refcount == 0) {
-                WARN(1, "bad PCH PLL refcount\n");
+                WARN(1, "bad %s refcount\n", pll->name);
                 return;
         }
 
-        --pll->refcount;
-        intel_crtc->pch_pll = NULL;
+        if (--pll->refcount == 0) {
+                WARN_ON(pll->on);
+                WARN_ON(pll->active);
+        }
+
+        crtc->config.shared_dpll = DPLL_ID_PRIVATE;
 }
 
-static struct intel_pch_pll *intel_get_pch_pll(struct intel_crtc *intel_crtc, u32 dpll, u32 fp)
+static struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, u32 dpll, u32 fp)
 {
-        struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
-        struct intel_pch_pll *pll;
-        int i;
+        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
+        enum intel_dpll_id i;
 
-        pll = intel_crtc->pch_pll;
         if (pll) {
-                DRM_DEBUG_KMS("CRTC:%d reusing existing PCH PLL %x\n",
-                              intel_crtc->base.base.id, pll->pll_reg);
-                goto prepare;
+                DRM_DEBUG_KMS("CRTC:%d dropping existing %s\n",
+                              crtc->base.base.id, pll->name);
+                intel_put_shared_dpll(crtc);
         }
 
         if (HAS_PCH_IBX(dev_priv->dev)) {
                 /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
-                i = intel_crtc->pipe;
-                pll = &dev_priv->pch_plls[i];
+                i = crtc->pipe;
+                pll = &dev_priv->shared_dplls[i];
 
-                DRM_DEBUG_KMS("CRTC:%d using pre-allocated PCH PLL %x\n",
-                              intel_crtc->base.base.id, pll->pll_reg);
+                DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
+                              crtc->base.base.id, pll->name);
 
                 goto found;
         }
 
-        for (i = 0; i < dev_priv->num_pch_pll; i++) {
-                pll = &dev_priv->pch_plls[i];
+        for (i = 0; i < dev_priv->num_shared_dpll; i++) {
+                pll = &dev_priv->shared_dplls[i];
 
                 /* Only want to check enabled timings first */
                 if (pll->refcount == 0)
                         continue;
 
-                if (dpll == (I915_READ(pll->pll_reg) & 0x7fffffff) &&
-                    fp == I915_READ(pll->fp0_reg)) {
-                        DRM_DEBUG_KMS("CRTC:%d sharing existing PCH PLL %x (refcount %d, ative %d)\n",
-                                      intel_crtc->base.base.id,
-                                      pll->pll_reg, pll->refcount, pll->active);
+                if (dpll == (I915_READ(PCH_DPLL(pll->id)) & 0x7fffffff) &&
+                    fp == I915_READ(PCH_FP0(pll->id))) {
+                        DRM_DEBUG_KMS("CRTC:%d sharing existing %s (refcount %d, ative %d)\n",
+                                      crtc->base.base.id,
+                                      pll->name, pll->refcount, pll->active);
 
                         goto found;
                 }
         }
 
         /* Ok no matching timings, maybe there's a free one? */
-        for (i = 0; i < dev_priv->num_pch_pll; i++) {
-                pll = &dev_priv->pch_plls[i];
+        for (i = 0; i < dev_priv->num_shared_dpll; i++) {
+                pll = &dev_priv->shared_dplls[i];
                 if (pll->refcount == 0) {
-                        DRM_DEBUG_KMS("CRTC:%d allocated PCH PLL %x\n",
-                                      intel_crtc->base.base.id, pll->pll_reg);
+                        DRM_DEBUG_KMS("CRTC:%d allocated %s\n",
+                                      crtc->base.base.id, pll->name);
                         goto found;
                 }
         }
@@ -3292,24 +3084,32 @@ static struct intel_pch_pll *intel_get_pch_pll(struct intel_crtc *intel_crtc, u3
         return NULL;
 
 found:
-        intel_crtc->pch_pll = pll;
-        pll->refcount++;
-        DRM_DEBUG_DRIVER("using pll %d for pipe %d\n", i, intel_crtc->pipe);
-prepare: /* separate function? */
-        DRM_DEBUG_DRIVER("switching PLL %x off\n", pll->pll_reg);
+        crtc->config.shared_dpll = i;
+        DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name,
+                         pipe_name(crtc->pipe));
 
-        /* Wait for the clocks to stabilize before rewriting the regs */
-        I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE);
-        POSTING_READ(pll->pll_reg);
-        udelay(150);
+        if (pll->active == 0) {
+                memcpy(&pll->hw_state, &crtc->config.dpll_hw_state,
+                       sizeof(pll->hw_state));
+
+                DRM_DEBUG_DRIVER("setting up %s\n", pll->name);
+                WARN_ON(pll->on);
+                assert_shared_dpll_disabled(dev_priv, pll);
+
+                /* Wait for the clocks to stabilize before rewriting the regs */
+                I915_WRITE(PCH_DPLL(pll->id), dpll & ~DPLL_VCO_ENABLE);
+                POSTING_READ(PCH_DPLL(pll->id));
+                udelay(150);
+
+                I915_WRITE(PCH_FP0(pll->id), fp);
+                I915_WRITE(PCH_DPLL(pll->id), dpll & ~DPLL_VCO_ENABLE);
+        }
+        pll->refcount++;
 
-        I915_WRITE(pll->fp0_reg, fp);
-        I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE);
-        pll->on = false;
         return pll;
 }
 
-void intel_cpt_verify_modeset(struct drm_device *dev, int pipe)
+static void cpt_verify_modeset(struct drm_device *dev, int pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int dslreg = PIPEDSL(pipe);
@@ -3319,10 +3119,53 @@ void intel_cpt_verify_modeset(struct drm_device *dev, int pipe)
         udelay(500);
         if (wait_for(I915_READ(dslreg) != temp, 5)) {
                 if (wait_for(I915_READ(dslreg) != temp, 5))
-                        DRM_ERROR("mode set failed: pipe %d stuck\n", pipe);
+                        DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
+        }
+}
+
+static void ironlake_pfit_enable(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe = crtc->pipe;
+
+        if (crtc->config.pch_pfit.size) {
+                /* Force use of hard-coded filter coefficients
+                 * as some pre-programmed values are broken,
+                 * e.g. x201.
+                 */
+                if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+                        I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
+                                                 PF_PIPE_SEL_IVB(pipe));
+                else
+                        I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
+                I915_WRITE(PF_WIN_POS(pipe), crtc->config.pch_pfit.pos);
+                I915_WRITE(PF_WIN_SZ(pipe), crtc->config.pch_pfit.size);
         }
 }
 
+static void intel_enable_planes(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        enum pipe pipe = to_intel_crtc(crtc)->pipe;
+        struct intel_plane *intel_plane;
+
+        list_for_each_entry(intel_plane, &dev->mode_config.plane_list, base.head)
+                if (intel_plane->pipe == pipe)
+                        intel_plane_restore(&intel_plane->base);
+}
+
+static void intel_disable_planes(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        enum pipe pipe = to_intel_crtc(crtc)->pipe;
+        struct intel_plane *intel_plane;
+
+        list_for_each_entry(intel_plane, &dev->mode_config.plane_list, base.head)
+                if (intel_plane->pipe == pipe)
+                        intel_plane_disable(&intel_plane->base);
+}
+
 static void ironlake_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
@@ -3339,6 +3182,10 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
                 return;
 
         intel_crtc->active = true;
+
+        intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
+        intel_set_pch_fifo_underrun_reporting(dev, pipe, true);
+
         intel_update_watermarks(dev);
 
         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
@@ -3362,22 +3209,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
                 if (encoder->pre_enable)
                         encoder->pre_enable(encoder);
 
-        /* Enable panel fitting for LVDS */
-        if (dev_priv->pch_pf_size &&
-            (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) ||
-             intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) {
-                /* Force use of hard-coded filter coefficients
-                 * as some pre-programmed values are broken,
-                 * e.g. x201.
-                 */
-                if (IS_IVYBRIDGE(dev))
-                        I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
-                                                 PF_PIPE_SEL_IVB(pipe));
-                else
-                        I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
-                I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos);
-                I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
-        }
+        ironlake_pfit_enable(intel_crtc);
 
         /*
          * On ILK+ LUT must be loaded before the pipe is running but with
@@ -3388,6 +3220,8 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
         intel_enable_pipe(dev_priv, pipe,
                           intel_crtc->config.has_pch_encoder);
         intel_enable_plane(dev_priv, plane, pipe);
+        intel_enable_planes(crtc);
+        intel_crtc_update_cursor(crtc, true);
 
         if (intel_crtc->config.has_pch_encoder)
                 ironlake_pch_enable(crtc);
@@ -3396,13 +3230,11 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
         intel_update_fbc(dev);
         mutex_unlock(&dev->struct_mutex);
 
-        intel_crtc_update_cursor(crtc, true);
-
         for_each_encoder_on_crtc(dev, crtc, encoder)
                 encoder->enable(encoder);
 
         if (HAS_PCH_CPT(dev))
-                intel_cpt_verify_modeset(dev, intel_crtc->pipe);
+                cpt_verify_modeset(dev, intel_crtc->pipe);
 
         /*
          * There seems to be a race in PCH platform hw (at least on some
@@ -3415,6 +3247,42 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
         intel_wait_for_vblank(dev, intel_crtc->pipe);
 }
 
+/* IPS only exists on ULT machines and is tied to pipe A. */
+static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
+{
+        return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
+}
+
+static void hsw_enable_ips(struct intel_crtc *crtc)
+{
+        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+
+        if (!crtc->config.ips_enabled)
+                return;
+
+        /* We can only enable IPS after we enable a plane and wait for a vblank.
+         * We guarantee that the plane is enabled by calling intel_enable_ips
+         * only after intel_enable_plane. And intel_enable_plane already waits
+         * for a vblank, so all we need to do here is to enable the IPS bit. */
+        assert_plane_enabled(dev_priv, crtc->plane);
+        I915_WRITE(IPS_CTL, IPS_ENABLE);
+}
+
+static void hsw_disable_ips(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!crtc->config.ips_enabled)
+                return;
+
+        assert_plane_enabled(dev_priv, crtc->plane);
+        I915_WRITE(IPS_CTL, 0);
+
+        /* We need to wait for a vblank before we can disable the plane. */
+        intel_wait_for_vblank(dev, crtc->pipe);
+}
+
 static void haswell_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
@@ -3430,6 +3298,11 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
                 return;
 
         intel_crtc->active = true;
+
+        intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
+        if (intel_crtc->config.has_pch_encoder)
+                intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, true);
+
         intel_update_watermarks(dev);
 
         if (intel_crtc->config.has_pch_encoder)
@@ -3441,18 +3314,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 
         intel_ddi_enable_pipe_clock(intel_crtc);
 
-        /* Enable panel fitting for eDP */
-        if (dev_priv->pch_pf_size &&
-            intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP)) {
-                /* Force use of hard-coded filter coefficients
-                 * as some pre-programmed values are broken,
-                 * e.g. x201.
-                 */
-                I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
-                                         PF_PIPE_SEL_IVB(pipe));
-                I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos);
-                I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
-        }
+        ironlake_pfit_enable(intel_crtc);
 
         /*
          * On ILK+ LUT must be loaded before the pipe is running but with
@@ -3466,6 +3328,10 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         intel_enable_pipe(dev_priv, pipe,
                           intel_crtc->config.has_pch_encoder);
         intel_enable_plane(dev_priv, plane, pipe);
+        intel_enable_planes(crtc);
+        intel_crtc_update_cursor(crtc, true);
+
+        hsw_enable_ips(intel_crtc);
 
         if (intel_crtc->config.has_pch_encoder)
                 lpt_pch_enable(crtc);
@@ -3474,8 +3340,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         intel_update_fbc(dev);
         mutex_unlock(&dev->struct_mutex);
 
-        intel_crtc_update_cursor(crtc, true);
-
         for_each_encoder_on_crtc(dev, crtc, encoder)
                 encoder->enable(encoder);
 
@@ -3490,6 +3354,21 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         intel_wait_for_vblank(dev, intel_crtc->pipe);
 }
 
+static void ironlake_pfit_disable(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe = crtc->pipe;
+
+        /* To avoid upsetting the power well on haswell only disable the pfit if
+         * it's in use. The hw state code will make sure we get this right. */
+        if (crtc->config.pch_pfit.size) {
+                I915_WRITE(PF_CTL(pipe), 0);
+                I915_WRITE(PF_WIN_POS(pipe), 0);
+                I915_WRITE(PF_WIN_SZ(pipe), 0);
+        }
+}
+
 static void ironlake_crtc_disable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
@@ -3509,58 +3388,51 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 
         intel_crtc_wait_for_pending_flips(crtc);
         drm_vblank_off(dev, pipe);
-        intel_crtc_update_cursor(crtc, false);
-
-        intel_disable_plane(dev_priv, plane, pipe);
 
         if (dev_priv->cfb_plane == plane)
                 intel_disable_fbc(dev);
 
+        intel_crtc_update_cursor(crtc, false);
+        intel_disable_planes(crtc);
+        intel_disable_plane(dev_priv, plane, pipe);
+
+        if (intel_crtc->config.has_pch_encoder)
+                intel_set_pch_fifo_underrun_reporting(dev, pipe, false);
+
         intel_disable_pipe(dev_priv, pipe);
 
-        /* Disable PF */
-        I915_WRITE(PF_CTL(pipe), 0);
-        I915_WRITE(PF_WIN_SZ(pipe), 0);
+        ironlake_pfit_disable(intel_crtc);
 
         for_each_encoder_on_crtc(dev, crtc, encoder)
                 if (encoder->post_disable)
                         encoder->post_disable(encoder);
 
-        ironlake_fdi_disable(crtc);
+        if (intel_crtc->config.has_pch_encoder) {
+                ironlake_fdi_disable(crtc);
 
-        ironlake_disable_pch_transcoder(dev_priv, pipe);
+                ironlake_disable_pch_transcoder(dev_priv, pipe);
+                intel_set_pch_fifo_underrun_reporting(dev, pipe, true);
 
-        if (HAS_PCH_CPT(dev)) {
-                /* disable TRANS_DP_CTL */
-                reg = TRANS_DP_CTL(pipe);
-                temp = I915_READ(reg);
-                temp &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK);
-                temp |= TRANS_DP_PORT_SEL_NONE;
-                I915_WRITE(reg, temp);
-
-                /* disable DPLL_SEL */
-                temp = I915_READ(PCH_DPLL_SEL);
-                switch (pipe) {
-                case 0:
-                        temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLB_SEL);
-                        break;
-                case 1:
-                        temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
-                        break;
-                case 2:
-                        /* C shares PLL A or B */
-                        temp &= ~(TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL);
-                        break;
-                default:
-                        BUG(); /* wtf */
+                if (HAS_PCH_CPT(dev)) {
+                        /* disable TRANS_DP_CTL */
+                        reg = TRANS_DP_CTL(pipe);
+                        temp = I915_READ(reg);
+                        temp &= ~(TRANS_DP_OUTPUT_ENABLE |
+                                  TRANS_DP_PORT_SEL_MASK);
+                        temp |= TRANS_DP_PORT_SEL_NONE;
+                        I915_WRITE(reg, temp);
+
+                        /* disable DPLL_SEL */
+                        temp = I915_READ(PCH_DPLL_SEL);
+                        temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
+                        I915_WRITE(PCH_DPLL_SEL, temp);
                 }
-                I915_WRITE(PCH_DPLL_SEL, temp);
-        }
 
-        /* disable PCH DPLL */
-        intel_disable_pch_pll(intel_crtc);
+                /* disable PCH DPLL */
+                intel_disable_shared_dpll(intel_crtc);
 
-        ironlake_fdi_pll_disable(intel_crtc);
+                ironlake_fdi_pll_disable(intel_crtc);
+        }
 
         intel_crtc->active = false;
         intel_update_watermarks(dev);
@@ -3588,24 +3460,24 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 
         intel_crtc_wait_for_pending_flips(crtc);
         drm_vblank_off(dev, pipe);
-        intel_crtc_update_cursor(crtc, false);
-
-        intel_disable_plane(dev_priv, plane, pipe);
 
+        /* FBC must be disabled before disabling the plane on HSW. */
         if (dev_priv->cfb_plane == plane)
                 intel_disable_fbc(dev);
 
+        hsw_disable_ips(intel_crtc);
+
+        intel_crtc_update_cursor(crtc, false);
+        intel_disable_planes(crtc);
+        intel_disable_plane(dev_priv, plane, pipe);
+
+        if (intel_crtc->config.has_pch_encoder)
+                intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, false);
         intel_disable_pipe(dev_priv, pipe);
 
         intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
 
-        /* XXX: Once we have proper panel fitter state tracking implemented with
-         * hardware state read/check support we should switch to only disable
-         * the panel fitter when we know it's used. */
-        if (intel_using_power_well(dev)) {
-                I915_WRITE(PF_CTL(pipe), 0);
-                I915_WRITE(PF_WIN_SZ(pipe), 0);
-        }
+        ironlake_pfit_disable(intel_crtc);
 
         intel_ddi_disable_pipe_clock(intel_crtc);
 
@@ -3615,6 +3487,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 
         if (intel_crtc->config.has_pch_encoder) {
                 lpt_disable_pch_transcoder(dev_priv);
+                intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, true);
                 intel_ddi_fdi_disable(crtc);
         }
 
@@ -3629,17 +3502,11 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 static void ironlake_crtc_off(struct drm_crtc *crtc)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        intel_put_pch_pll(intel_crtc);
+        intel_put_shared_dpll(intel_crtc);
 }
 
 static void haswell_crtc_off(struct drm_crtc *crtc)
 {
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
-        /* Stop saying we're using TRANSCODER_EDP because some other CRTC might
-         * start using it. */
-        intel_crtc->config.cpu_transcoder = (enum transcoder) intel_crtc->pipe;
-
         intel_ddi_put_crtc_pll(crtc);
 }
 
@@ -3685,6 +3552,77 @@ g4x_fixup_plane(struct drm_i915_private *dev_priv, enum pipe pipe)
         }
 }
 
+static void i9xx_pfit_enable(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc_config *pipe_config = &crtc->config;
+
+        if (!crtc->config.gmch_pfit.control)
+                return;
+
+        /*
+         * The panel fitter should only be adjusted whilst the pipe is disabled,
+         * according to register description and PRM.
+         */
+        WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
+        assert_pipe_disabled(dev_priv, crtc->pipe);
+
+        I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios);
+        I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control);
+
+        /* Border color in case we don't scale up to the full screen. Black by
+         * default, change to something else for debugging. */
+        I915_WRITE(BCLRPAT(crtc->pipe), 0);
+}
+
+static void valleyview_crtc_enable(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_encoder *encoder;
+        int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
+
+        WARN_ON(!crtc->enabled);
+
+        if (intel_crtc->active)
+                return;
+
+        intel_crtc->active = true;
+        intel_update_watermarks(dev);
+
+        mutex_lock(&dev_priv->dpio_lock);
+
+        for_each_encoder_on_crtc(dev, crtc, encoder)
+                if (encoder->pre_pll_enable)
+                        encoder->pre_pll_enable(encoder);
+
+        intel_enable_pll(dev_priv, pipe);
+
+        for_each_encoder_on_crtc(dev, crtc, encoder)
+                if (encoder->pre_enable)
+                        encoder->pre_enable(encoder);
+
+        /* VLV wants encoder enabling _before_ the pipe is up. */
+        for_each_encoder_on_crtc(dev, crtc, encoder)
+                encoder->enable(encoder);
+
+        i9xx_pfit_enable(intel_crtc);
+
+        intel_crtc_load_lut(crtc);
+
+        intel_enable_pipe(dev_priv, pipe, false);
+        intel_enable_plane(dev_priv, plane, pipe);
+        intel_enable_planes(crtc);
+        intel_crtc_update_cursor(crtc, true);
+
+        intel_update_fbc(dev);
+
+        mutex_unlock(&dev_priv->dpio_lock);
+}
+
 static void i9xx_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
@@ -3708,17 +3646,22 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
                 if (encoder->pre_enable)
                         encoder->pre_enable(encoder);
 
+        i9xx_pfit_enable(intel_crtc);
+
+        intel_crtc_load_lut(crtc);
+
         intel_enable_pipe(dev_priv, pipe, false);
         intel_enable_plane(dev_priv, plane, pipe);
+        intel_enable_planes(crtc);
+        /* The fixup needs to happen before cursor is enabled */
         if (IS_G4X(dev))
                 g4x_fixup_plane(dev_priv, pipe);
-
-        intel_crtc_load_lut(crtc);
-        intel_update_fbc(dev);
+        intel_crtc_update_cursor(crtc, true);
 
         /* Give the overlay scaler a chance to enable if it's on this pipe */
         intel_crtc_dpms_overlay(intel_crtc, true);
-        intel_crtc_update_cursor(crtc, true);
+
+        intel_update_fbc(dev);
 
         for_each_encoder_on_crtc(dev, crtc, encoder)
                 encoder->enable(encoder);
@@ -3728,20 +3671,15 @@ static void i9xx_pfit_disable(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum pipe pipe;
-        uint32_t pctl = I915_READ(PFIT_CONTROL);
 
-        assert_pipe_disabled(dev_priv, crtc->pipe);
+        if (!crtc->config.gmch_pfit.control)
+                return;
 
-        if (INTEL_INFO(dev)->gen >= 4)
-                pipe = (pctl & PFIT_PIPE_MASK) >> PFIT_PIPE_SHIFT;
-        else
-                pipe = PIPE_B;
+        assert_pipe_disabled(dev_priv, crtc->pipe);
 
-        if (pipe == crtc->pipe) {
-                DRM_DEBUG_DRIVER("disabling pfit, current: 0x%08x\n", pctl);
-                I915_WRITE(PFIT_CONTROL, 0);
-        }
+        DRM_DEBUG_DRIVER("disabling pfit, current: 0x%08x\n",
+                         I915_READ(PFIT_CONTROL));
+        I915_WRITE(PFIT_CONTROL, 0);
 }
 
 static void i9xx_crtc_disable(struct drm_crtc *crtc)
@@ -3762,17 +3700,23 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
         /* Give the overlay scaler a chance to disable if it's on this pipe */
         intel_crtc_wait_for_pending_flips(crtc);
         drm_vblank_off(dev, pipe);
-        intel_crtc_dpms_overlay(intel_crtc, false);
-        intel_crtc_update_cursor(crtc, false);
 
         if (dev_priv->cfb_plane == plane)
                 intel_disable_fbc(dev);
 
+        intel_crtc_dpms_overlay(intel_crtc, false);
+        intel_crtc_update_cursor(crtc, false);
+        intel_disable_planes(crtc);
         intel_disable_plane(dev_priv, plane, pipe);
+
         intel_disable_pipe(dev_priv, pipe);
 
         i9xx_pfit_disable(intel_crtc);
 
+        for_each_encoder_on_crtc(dev, crtc, encoder)
+                if (encoder->post_disable)
+                        encoder->post_disable(encoder);
+
         intel_disable_pll(dev_priv, pipe);
 
         intel_crtc->active = false;
@@ -3845,8 +3789,8 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
         /* crtc should still be enabled when we disable it. */
         WARN_ON(!crtc->enabled);
 
-        intel_crtc->eld_vld = false;
         dev_priv->display.crtc_disable(crtc);
+        intel_crtc->eld_vld = false;
         intel_crtc_update_sarea(crtc, false);
         dev_priv->display.off(crtc);
 
@@ -3977,17 +3921,131 @@ bool intel_connector_get_hw_state(struct intel_connector *connector)
         return encoder->get_hw_state(encoder, &pipe);
 }
 
-static bool intel_crtc_compute_config(struct drm_crtc *crtc,
-                                      struct intel_crtc_config *pipe_config)
+static bool ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
+                                     struct intel_crtc_config *pipe_config)
 {
-        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *pipe_B_crtc =
+                to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_B]);
+
+        DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
+                      pipe_name(pipe), pipe_config->fdi_lanes);
+        if (pipe_config->fdi_lanes > 4) {
+                DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
+                              pipe_name(pipe), pipe_config->fdi_lanes);
+                return false;
+        }
+
+        if (IS_HASWELL(dev)) {
+                if (pipe_config->fdi_lanes > 2) {
+                        DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
+                                      pipe_config->fdi_lanes);
+                        return false;
+                } else {
+                        return true;
+                }
+        }
+
+        if (INTEL_INFO(dev)->num_pipes == 2)
+                return true;
+
+        /* Ivybridge 3 pipe is really complicated */
+        switch (pipe) {
+        case PIPE_A:
+                return true;
+        case PIPE_B:
+                if (dev_priv->pipe_to_crtc_mapping[PIPE_C]->enabled &&
+                    pipe_config->fdi_lanes > 2) {
+                        DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
+                                      pipe_name(pipe), pipe_config->fdi_lanes);
+                        return false;
+                }
+                return true;
+        case PIPE_C:
+                if (!pipe_has_enabled_pch(pipe_B_crtc) ||
+                    pipe_B_crtc->config.fdi_lanes <= 2) {
+                        if (pipe_config->fdi_lanes > 2) {
+                                DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
+                                              pipe_name(pipe), pipe_config->fdi_lanes);
+                                return false;
+                        }
+                } else {
+                        DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
+                        return false;
+                }
+                return true;
+        default:
+                BUG();
+        }
+}
+
+#define RETRY 1
+static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
+                                       struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = intel_crtc->base.dev;
+        struct drm_display_mode *adjusted_mode = &pipe_config->adjusted_mode;
+        int lane, link_bw, fdi_dotclock;
+        bool setup_ok, needs_recompute = false;
+
+retry:
+        /* FDI is a binary signal running at ~2.7GHz, encoding
+         * each output octet as 10 bits. The actual frequency
+         * is stored as a divider into a 100MHz clock, and the
+         * mode pixel clock is stored in units of 1KHz.
+         * Hence the bw of each lane in terms of the mode signal
+         * is:
+         */
+        link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
+
+        fdi_dotclock = adjusted_mode->clock;
+        fdi_dotclock /= pipe_config->pixel_multiplier;
+
+        lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
+                                           pipe_config->pipe_bpp);
+
+        pipe_config->fdi_lanes = lane;
+
+        intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
+                               link_bw, &pipe_config->fdi_m_n);
+
+        setup_ok = ironlake_check_fdi_lanes(intel_crtc->base.dev,
+                                            intel_crtc->pipe, pipe_config);
+        if (!setup_ok && pipe_config->pipe_bpp > 6*3) {
+                pipe_config->pipe_bpp -= 2*3;
+                DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
+                              pipe_config->pipe_bpp);
+                needs_recompute = true;
+                pipe_config->bw_constrained = true;
+
+                goto retry;
+        }
+
+        if (needs_recompute)
+                return RETRY;
+
+        return setup_ok ? 0 : -EINVAL;
+}
+
+static void hsw_compute_ips_config(struct intel_crtc *crtc,
+                                   struct intel_crtc_config *pipe_config)
+{
+        pipe_config->ips_enabled = i915_enable_ips &&
+                                   hsw_crtc_supports_ips(crtc) &&
+                                   pipe_config->pipe_bpp == 24;
+}
+
+static int intel_crtc_compute_config(struct intel_crtc *crtc,
+                                     struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = crtc->base.dev;
         struct drm_display_mode *adjusted_mode = &pipe_config->adjusted_mode;
 
         if (HAS_PCH_SPLIT(dev)) {
                 /* FDI link clock is fixed at 2.7G */
                 if (pipe_config->requested_mode.clock * 3
                     > IRONLAKE_FDI_FREQ * 4)
-                        return false;
+                        return -EINVAL;
         }
 
         /* All interlaced capable intel hw wants timings in frames. Note though
@@ -3996,12 +4054,12 @@ static bool intel_crtc_compute_config(struct drm_crtc *crtc,
         if (!pipe_config->timings_set)
                 drm_mode_set_crtcinfo(adjusted_mode, 0);
 
-        /* WaPruneModeWithIncorrectHsyncOffset: Cantiga+ cannot handle modes
-         * with a hsync front porch of 0.
+        /* Cantiga+ cannot handle modes with a hsync front porch of 0.
+         * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
          */
         if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
                 adjusted_mode->hsync_start == adjusted_mode->hdisplay)
-                return false;
+                return -EINVAL;
 
         if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) && pipe_config->pipe_bpp > 10*3) {
                 pipe_config->pipe_bpp = 10*3; /* 12bpc is gen5+ */
@@ -4011,7 +4069,18 @@ static bool intel_crtc_compute_config(struct drm_crtc *crtc,
                 pipe_config->pipe_bpp = 8*3;
         }
 
-        return true;
+        if (HAS_IPS(dev))
+                hsw_compute_ips_config(crtc, pipe_config);
+
+        /* XXX: PCH clock sharing is done in ->mode_set, so make sure the old
+         * clock survives for now. */
+        if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+                pipe_config->shared_dpll = crtc->config.shared_dpll;
+
+        if (pipe_config->has_pch_encoder)
+                return ironlake_fdi_compute_config(crtc, pipe_config);
+
+        return 0;
 }
 
 static int valleyview_get_display_clock_speed(struct drm_device *dev)
@@ -4120,7 +4189,7 @@ static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
 {
         if (i915_panel_use_ssc >= 0)
                 return i915_panel_use_ssc != 0;
-        return dev_priv->lvds_use_ssc
+        return dev_priv->vbt.lvds_use_ssc
                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
 }
 
@@ -4156,7 +4225,7 @@ static int i9xx_get_refclk(struct drm_crtc *crtc, int num_connectors)
                 refclk = vlv_get_refclk(crtc);
         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
             intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
-                refclk = dev_priv->lvds_ssc_freq * 1000;
+                refclk = dev_priv->vbt.lvds_ssc_freq * 1000;
                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
                               refclk / 1000);
         } else if (!IS_GEN2(dev)) {
@@ -4168,28 +4237,14 @@ static int i9xx_get_refclk(struct drm_crtc *crtc, int num_connectors)
         return refclk;
 }
 
-static void i9xx_adjust_sdvo_tv_clock(struct intel_crtc *crtc)
+static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
 {
-        unsigned dotclock = crtc->config.adjusted_mode.clock;
-        struct dpll *clock = &crtc->config.dpll;
-
-        /* SDVO TV has fixed PLL values depend on its clock range,
-           this mirrors vbios setting. */
-        if (dotclock >= 100000 && dotclock < 140500) {
-                clock->p1 = 2;
-                clock->p2 = 10;
-                clock->n = 3;
-                clock->m1 = 16;
-                clock->m2 = 8;
-        } else if (dotclock >= 140500 && dotclock <= 200000) {
-                clock->p1 = 1;
-                clock->p2 = 10;
-                clock->n = 6;
-                clock->m1 = 12;
-                clock->m2 = 8;
-        }
+        return (1 << dpll->n) << 16 | dpll->m2;
+}
 
-        crtc->config.clock_set = true;
+static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
+{
+        return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
 }
 
 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
@@ -4199,18 +4254,15 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe = crtc->pipe;
         u32 fp, fp2 = 0;
-        struct dpll *clock = &crtc->config.dpll;
 
         if (IS_PINEVIEW(dev)) {
-                fp = (1 << clock->n) << 16 | clock->m1 << 8 | clock->m2;
+                fp = pnv_dpll_compute_fp(&crtc->config.dpll);
                 if (reduced_clock)
-                        fp2 = (1 << reduced_clock->n) << 16 |
-                                reduced_clock->m1 << 8 | reduced_clock->m2;
+                        fp2 = pnv_dpll_compute_fp(reduced_clock);
         } else {
-                fp = clock->n << 16 | clock->m1 << 8 | clock->m2;
+                fp = i9xx_dpll_compute_fp(&crtc->config.dpll);
                 if (reduced_clock)
-                        fp2 = reduced_clock->n << 16 | reduced_clock->m1 << 8 |
-                                reduced_clock->m2;
+                        fp2 = i9xx_dpll_compute_fp(reduced_clock);
         }
 
         I915_WRITE(FP0(pipe), fp);
@@ -4225,6 +4277,68 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
         }
 }
 
+static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv)
+{
+        u32 reg_val;
+
+        /*
+         * PLLB opamp always calibrates to max value of 0x3f, force enable it
+         * and set it to a reasonable value instead.
+         */
+        reg_val = vlv_dpio_read(dev_priv, DPIO_IREF(1));
+        reg_val &= 0xffffff00;
+        reg_val |= 0x00000030;
+        vlv_dpio_write(dev_priv, DPIO_IREF(1), reg_val);
+
+        reg_val = vlv_dpio_read(dev_priv, DPIO_CALIBRATION);
+        reg_val &= 0x8cffffff;
+        reg_val = 0x8c000000;
+        vlv_dpio_write(dev_priv, DPIO_CALIBRATION, reg_val);
+
+        reg_val = vlv_dpio_read(dev_priv, DPIO_IREF(1));
+        reg_val &= 0xffffff00;
+        vlv_dpio_write(dev_priv, DPIO_IREF(1), reg_val);
+
+        reg_val = vlv_dpio_read(dev_priv, DPIO_CALIBRATION);
+        reg_val &= 0x00ffffff;
+        reg_val |= 0xb0000000;
+        vlv_dpio_write(dev_priv, DPIO_CALIBRATION, reg_val);
+}
+
+static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
+                                         struct intel_link_m_n *m_n)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe = crtc->pipe;
+
+        I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
+        I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
+        I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
+        I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
+}
+
+static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
+                                         struct intel_link_m_n *m_n)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe = crtc->pipe;
+        enum transcoder transcoder = crtc->config.cpu_transcoder;
+
+        if (INTEL_INFO(dev)->gen >= 5) {
+                I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
+                I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
+                I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
+                I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
+        } else {
+                I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
+                I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
+                I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
+                I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
+        }
+}
+
 static void intel_dp_set_m_n(struct intel_crtc *crtc)
 {
         if (crtc->config.has_pch_encoder)
@@ -4237,24 +4351,16 @@ static void vlv_update_pll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_encoder *encoder;
         int pipe = crtc->pipe;
-        u32 dpll, mdiv, pdiv;
+        u32 dpll, mdiv;
         u32 bestn, bestm1, bestm2, bestp1, bestp2;
-        bool is_sdvo;
-        u32 temp;
+        bool is_hdmi;
+        u32 coreclk, reg_val, dpll_md;
 
         mutex_lock(&dev_priv->dpio_lock);
 
-        is_sdvo = intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_SDVO) ||
-                intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_HDMI);
-
-        dpll = DPLL_VGA_MODE_DIS;
-        dpll |= DPLL_EXT_BUFFER_ENABLE_VLV;
-        dpll |= DPLL_REFA_CLK_ENABLE_VLV;
-        dpll |= DPLL_INTEGRATED_CLOCK_VLV;
-
-        I915_WRITE(DPLL(pipe), dpll);
-        POSTING_READ(DPLL(pipe));
+        is_hdmi = intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_HDMI);
 
         bestn = crtc->config.dpll.n;
         bestm1 = crtc->config.dpll.m1;
@@ -4262,72 +4368,104 @@ static void vlv_update_pll(struct intel_crtc *crtc)
         bestp1 = crtc->config.dpll.p1;
         bestp2 = crtc->config.dpll.p2;
 
-        /*
-         * In Valleyview PLL and program lane counter registers are exposed
-         * through DPIO interface
-         */
+        /* See eDP HDMI DPIO driver vbios notes doc */
+
+        /* PLL B needs special handling */
+        if (pipe)
+                vlv_pllb_recal_opamp(dev_priv);
+
+        /* Set up Tx target for periodic Rcomp update */
+        vlv_dpio_write(dev_priv, DPIO_IREF_BCAST, 0x0100000f);
+
+        /* Disable target IRef on PLL */
+        reg_val = vlv_dpio_read(dev_priv, DPIO_IREF_CTL(pipe));
+        reg_val &= 0x00ffffff;
+        vlv_dpio_write(dev_priv, DPIO_IREF_CTL(pipe), reg_val);
+
+        /* Disable fast lock */
+        vlv_dpio_write(dev_priv, DPIO_FASTCLK_DISABLE, 0x610);
+
+        /* Set idtafcrecal before PLL is enabled */
         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
         mdiv |= ((bestn << DPIO_N_SHIFT));
-        mdiv |= (1 << DPIO_POST_DIV_SHIFT);
         mdiv |= (1 << DPIO_K_SHIFT);
+
+        /*
+         * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
+         * but we don't support that).
+         * Note: don't use the DAC post divider as it seems unstable.
+         */
+        mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
+        vlv_dpio_write(dev_priv, DPIO_DIV(pipe), mdiv);
+
         mdiv |= DPIO_ENABLE_CALIBRATION;
-        intel_dpio_write(dev_priv, DPIO_DIV(pipe), mdiv);
+        vlv_dpio_write(dev_priv, DPIO_DIV(pipe), mdiv);
+
+        /* Set HBR and RBR LPF coefficients */
+        if (crtc->config.port_clock == 162000 ||
+            intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_ANALOG) ||
+            intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_HDMI))
+                vlv_dpio_write(dev_priv, DPIO_LPF_COEFF(pipe),
+                                 0x005f0021);
+        else
+                vlv_dpio_write(dev_priv, DPIO_LPF_COEFF(pipe),
+                                 0x00d0000f);
+
+        if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_EDP) ||
+            intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DISPLAYPORT)) {
+                /* Use SSC source */
+                if (!pipe)
+                        vlv_dpio_write(dev_priv, DPIO_REFSFR(pipe),
+                                         0x0df40000);
+                else
+                        vlv_dpio_write(dev_priv, DPIO_REFSFR(pipe),
+                                         0x0df70000);
+        } else { /* HDMI or VGA */
+                /* Use bend source */
+                if (!pipe)
+                        vlv_dpio_write(dev_priv, DPIO_REFSFR(pipe),
+                                         0x0df70000);
+                else
+                        vlv_dpio_write(dev_priv, DPIO_REFSFR(pipe),
+                                         0x0df40000);
+        }
 
-        intel_dpio_write(dev_priv, DPIO_CORE_CLK(pipe), 0x01000000);
+        coreclk = vlv_dpio_read(dev_priv, DPIO_CORE_CLK(pipe));
+        coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
+        if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DISPLAYPORT) ||
+            intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_EDP))
+                coreclk |= 0x01000000;
+        vlv_dpio_write(dev_priv, DPIO_CORE_CLK(pipe), coreclk);
 
-        pdiv = (1 << DPIO_REFSEL_OVERRIDE) | (5 << DPIO_PLL_MODESEL_SHIFT) |
-                (3 << DPIO_BIAS_CURRENT_CTL_SHIFT) | (1<<20) |
-                (7 << DPIO_PLL_REFCLK_SEL_SHIFT) | (8 << DPIO_DRIVER_CTL_SHIFT) |
-                (5 << DPIO_CLK_BIAS_CTL_SHIFT);
-        intel_dpio_write(dev_priv, DPIO_REFSFR(pipe), pdiv);
+        vlv_dpio_write(dev_priv, DPIO_PLL_CML(pipe), 0x87871000);
 
-        intel_dpio_write(dev_priv, DPIO_LFP_COEFF(pipe), 0x005f003b);
+        for_each_encoder_on_crtc(dev, &crtc->base, encoder)
+                if (encoder->pre_pll_enable)
+                        encoder->pre_pll_enable(encoder);
+
+        /* Enable DPIO clock input */
+        dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REFA_CLK_ENABLE_VLV |
+                DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_CLOCK_VLV;
+        if (pipe)
+                dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
 
         dpll |= DPLL_VCO_ENABLE;
         I915_WRITE(DPLL(pipe), dpll);
         POSTING_READ(DPLL(pipe));
+        udelay(150);
+
         if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
 
-        intel_dpio_write(dev_priv, DPIO_FASTCLK_DISABLE, 0x620);
+        dpll_md = (crtc->config.pixel_multiplier - 1)
+                << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+        I915_WRITE(DPLL_MD(pipe), dpll_md);
+        POSTING_READ(DPLL_MD(pipe));
 
         if (crtc->config.has_dp_encoder)
                 intel_dp_set_m_n(crtc);
 
-        I915_WRITE(DPLL(pipe), dpll);
-
-        /* Wait for the clocks to stabilize. */
-        POSTING_READ(DPLL(pipe));
-        udelay(150);
-
-        temp = 0;
-        if (is_sdvo) {
-                temp = 0;
-                if (crtc->config.pixel_multiplier > 1) {
-                        temp = (crtc->config.pixel_multiplier - 1)
-                                << DPLL_MD_UDI_MULTIPLIER_SHIFT;
-                }
-        }
-        I915_WRITE(DPLL_MD(pipe), temp);
-        POSTING_READ(DPLL_MD(pipe));
-
-        /* Now program lane control registers */
-        if(intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DISPLAYPORT)
-           || intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_HDMI)) {
-                temp = 0x1000C4;
-                if(pipe == 1)
-                        temp |= (1 << 21);
-                intel_dpio_write(dev_priv, DPIO_DATA_CHANNEL1, temp);
-        }
-
-        if(intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_EDP)) {
-                temp = 0x1000C4;
-                if(pipe == 1)
-                        temp |= (1 << 21);
-                intel_dpio_write(dev_priv, DPIO_DATA_CHANNEL2, temp);
-        }
-
         mutex_unlock(&dev_priv->dpio_lock);
 }
 
@@ -4355,14 +4493,14 @@ static void i9xx_update_pll(struct intel_crtc *crtc,
         else
                 dpll |= DPLLB_MODE_DAC_SERIAL;
 
-        if (is_sdvo) {
-                if ((crtc->config.pixel_multiplier > 1) &&
-                    (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))) {
-                        dpll |= (crtc->config.pixel_multiplier - 1)
-                                << SDVO_MULTIPLIER_SHIFT_HIRES;
-                }
-                dpll |= DPLL_DVO_HIGH_SPEED;
+        if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+                dpll |= (crtc->config.pixel_multiplier - 1)
+                        << SDVO_MULTIPLIER_SHIFT_HIRES;
         }
+
+        if (is_sdvo)
+                dpll |= DPLL_DVO_HIGH_SPEED;
+
         if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DISPLAYPORT))
                 dpll |= DPLL_DVO_HIGH_SPEED;
 
@@ -4391,12 +4529,8 @@ static void i9xx_update_pll(struct intel_crtc *crtc,
         if (INTEL_INFO(dev)->gen >= 4)
                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
 
-        if (is_sdvo && intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_TVOUT))
+        if (crtc->config.sdvo_tv_clock)
                 dpll |= PLL_REF_INPUT_TVCLKINBC;
-        else if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_TVOUT))
-                /* XXX: just matching BIOS for now */
-                /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
-                dpll |= 3;
         else if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) &&
                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
@@ -4422,15 +4556,9 @@ static void i9xx_update_pll(struct intel_crtc *crtc,
         udelay(150);
 
         if (INTEL_INFO(dev)->gen >= 4) {
-                u32 temp = 0;
-                if (is_sdvo) {
-                        temp = 0;
-                        if (crtc->config.pixel_multiplier > 1) {
-                                temp = (crtc->config.pixel_multiplier - 1)
-                                        << DPLL_MD_UDI_MULTIPLIER_SHIFT;
-                        }
-                }
-                I915_WRITE(DPLL_MD(pipe), temp);
+                u32 dpll_md = (crtc->config.pixel_multiplier - 1)
+                        << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+                I915_WRITE(DPLL_MD(pipe), dpll_md);
         } else {
                 /* The pixel multiplier can only be updated once the
                  * DPLL is enabled and the clocks are stable.
@@ -4442,7 +4570,6 @@ static void i9xx_update_pll(struct intel_crtc *crtc,
 }
 
 static void i8xx_update_pll(struct intel_crtc *crtc,
-                            struct drm_display_mode *adjusted_mode,
                             intel_clock_t *reduced_clock,
                             int num_connectors)
 {
@@ -4497,20 +4624,26 @@ static void i8xx_update_pll(struct intel_crtc *crtc,
         I915_WRITE(DPLL(pipe), dpll);
 }
 
-static void intel_set_pipe_timings(struct intel_crtc *intel_crtc,
-                                   struct drm_display_mode *mode,
-                                   struct drm_display_mode *adjusted_mode)
+static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe = intel_crtc->pipe;
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
-        uint32_t vsyncshift;
+        struct drm_display_mode *adjusted_mode =
+                &intel_crtc->config.adjusted_mode;
+        struct drm_display_mode *mode = &intel_crtc->config.requested_mode;
+        uint32_t vsyncshift, crtc_vtotal, crtc_vblank_end;
+
+        /* We need to be careful not to changed the adjusted mode, for otherwise
+         * the hw state checker will get angry at the mismatch. */
+        crtc_vtotal = adjusted_mode->crtc_vtotal;
+        crtc_vblank_end = adjusted_mode->crtc_vblank_end;
 
         if (!IS_GEN2(dev) && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
                 /* the chip adds 2 halflines automatically */
-                adjusted_mode->crtc_vtotal -= 1;
-                adjusted_mode->crtc_vblank_end -= 1;
+                crtc_vtotal -= 1;
+                crtc_vblank_end -= 1;
                 vsyncshift = adjusted_mode->crtc_hsync_start
                              - adjusted_mode->crtc_htotal / 2;
         } else {
@@ -4532,10 +4665,10 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc,
 
         I915_WRITE(VTOTAL(cpu_transcoder),
                    (adjusted_mode->crtc_vdisplay - 1) |
-                   ((adjusted_mode->crtc_vtotal - 1) << 16));
+                   ((crtc_vtotal - 1) << 16));
         I915_WRITE(VBLANK(cpu_transcoder),
                    (adjusted_mode->crtc_vblank_start - 1) |
-                   ((adjusted_mode->crtc_vblank_end - 1) << 16));
+                   ((crtc_vblank_end - 1) << 16));
         I915_WRITE(VSYNC(cpu_transcoder),
                    (adjusted_mode->crtc_vsync_start - 1) |
                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
@@ -4555,13 +4688,52 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc,
                    ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
 }
 
+static void intel_get_pipe_timings(struct intel_crtc *crtc,
+                                   struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
+        uint32_t tmp;
+
+        tmp = I915_READ(HTOTAL(cpu_transcoder));
+        pipe_config->adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
+        pipe_config->adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
+        tmp = I915_READ(HBLANK(cpu_transcoder));
+        pipe_config->adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
+        pipe_config->adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
+        tmp = I915_READ(HSYNC(cpu_transcoder));
+        pipe_config->adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
+        pipe_config->adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
+
+        tmp = I915_READ(VTOTAL(cpu_transcoder));
+        pipe_config->adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
+        pipe_config->adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
+        tmp = I915_READ(VBLANK(cpu_transcoder));
+        pipe_config->adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
+        pipe_config->adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
+        tmp = I915_READ(VSYNC(cpu_transcoder));
+        pipe_config->adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
+        pipe_config->adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
+
+        if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
+                pipe_config->adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
+                pipe_config->adjusted_mode.crtc_vtotal += 1;
+                pipe_config->adjusted_mode.crtc_vblank_end += 1;
+        }
+
+        tmp = I915_READ(PIPESRC(crtc->pipe));
+        pipe_config->requested_mode.vdisplay = (tmp & 0xffff) + 1;
+        pipe_config->requested_mode.hdisplay = ((tmp >> 16) & 0xffff) + 1;
+}
+
 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t pipeconf;
 
-        pipeconf = I915_READ(PIPECONF(intel_crtc->pipe));
+        pipeconf = 0;
 
         if (intel_crtc->pipe == 0 && INTEL_INFO(dev)->gen < 4) {
                 /* Enable pixel doubling when the dot clock is > 90% of the (display)
@@ -4573,26 +4745,28 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
                 if (intel_crtc->config.requested_mode.clock >
                     dev_priv->display.get_display_clock_speed(dev) * 9 / 10)
                         pipeconf |= PIPECONF_DOUBLE_WIDE;
-                else
-                        pipeconf &= ~PIPECONF_DOUBLE_WIDE;
         }
 
-        /* default to 8bpc */
-        pipeconf &= ~(PIPECONF_BPC_MASK | PIPECONF_DITHER_EN);
-        if (intel_crtc->config.has_dp_encoder) {
-                if (intel_crtc->config.dither) {
-                        pipeconf |= PIPECONF_6BPC |
-                                    PIPECONF_DITHER_EN |
+        /* only g4x and later have fancy bpc/dither controls */
+        if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
+                /* Bspec claims that we can't use dithering for 30bpp pipes. */
+                if (intel_crtc->config.dither && intel_crtc->config.pipe_bpp != 30)
+                        pipeconf |= PIPECONF_DITHER_EN |
                                     PIPECONF_DITHER_TYPE_SP;
-                }
-        }
 
-        if (IS_VALLEYVIEW(dev) && intel_pipe_has_type(&intel_crtc->base,
-                                                      INTEL_OUTPUT_EDP)) {
-                if (intel_crtc->config.dither) {
-                        pipeconf |= PIPECONF_6BPC |
-                                        PIPECONF_ENABLE |
-                                        I965_PIPECONF_ACTIVE;
+                switch (intel_crtc->config.pipe_bpp) {
+                case 18:
+                        pipeconf |= PIPECONF_6BPC;
+                        break;
+                case 24:
+                        pipeconf |= PIPECONF_8BPC;
+                        break;
+                case 30:
+                        pipeconf |= PIPECONF_10BPC;
+                        break;
+                default:
+                        /* Case prevented by intel_choose_pipe_bpp_dither. */
+                        BUG();
                 }
         }
 
@@ -4602,23 +4776,17 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
                 } else {
                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
-                        pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
                 }
         }
 
-        pipeconf &= ~PIPECONF_INTERLACE_MASK;
         if (!IS_GEN2(dev) &&
             intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
                 pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
         else
                 pipeconf |= PIPECONF_PROGRESSIVE;
 
-        if (IS_VALLEYVIEW(dev)) {
-                if (intel_crtc->config.limited_color_range)
-                        pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
-                else
-                        pipeconf &= ~PIPECONF_COLOR_RANGE_SELECT;
-        }
+        if (IS_VALLEYVIEW(dev) && intel_crtc->config.limited_color_range)
+                pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
 
         I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
         POSTING_READ(PIPECONF(intel_crtc->pipe));
@@ -4631,16 +4799,14 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_display_mode *adjusted_mode =
-                &intel_crtc->config.adjusted_mode;
         struct drm_display_mode *mode = &intel_crtc->config.requested_mode;
         int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
         int refclk, num_connectors = 0;
         intel_clock_t clock, reduced_clock;
         u32 dspcntr;
-        bool ok, has_reduced_clock = false, is_sdvo = false;
-        bool is_lvds = false, is_tv = false;
+        bool ok, has_reduced_clock = false;
+        bool is_lvds = false;
         struct intel_encoder *encoder;
         const intel_limit_t *limit;
         int ret;
@@ -4650,15 +4816,6 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
                 case INTEL_OUTPUT_LVDS:
                         is_lvds = true;
                         break;
-                case INTEL_OUTPUT_SDVO:
-                case INTEL_OUTPUT_HDMI:
-                        is_sdvo = true;
-                        if (encoder->needs_tv_clock)
-                                is_tv = true;
-                        break;
-                case INTEL_OUTPUT_TVOUT:
-                        is_tv = true;
-                        break;
                 }
 
                 num_connectors++;
@@ -4672,9 +4829,10 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
          */
         limit = intel_limit(crtc, refclk);
-        ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
-                             &clock);
-        if (!ok) {
+        ok = dev_priv->display.find_dpll(limit, crtc,
+                                         intel_crtc->config.port_clock,
+                                         refclk, NULL, &clock);
+        if (!ok && !intel_crtc->config.clock_set) {
                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
                 return -EINVAL;
         }
@@ -4689,10 +4847,10 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
                  * by using the FP0/FP1. In such case we will disable the LVDS
                  * downclock feature.
                 */
-                has_reduced_clock = limit->find_pll(limit, crtc,
+                has_reduced_clock =
+                        dev_priv->display.find_dpll(limit, crtc,
                                                     dev_priv->lvds_downclock,
-                                                    refclk,
-                                                    &clock,
+                                                    refclk, &clock,
                                                     &reduced_clock);
         }
         /* Compat-code for transition, will disappear. */
@@ -4704,11 +4862,8 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
                 intel_crtc->config.dpll.p2 = clock.p2;
         }
 
-        if (is_sdvo && is_tv)
-                i9xx_adjust_sdvo_tv_clock(intel_crtc);
-
         if (IS_GEN2(dev))
-                i8xx_update_pll(intel_crtc, adjusted_mode,
+                i8xx_update_pll(intel_crtc,
                                 has_reduced_clock ? &reduced_clock : NULL,
                                 num_connectors);
         else if (IS_VALLEYVIEW(dev))
@@ -4716,7 +4871,7 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
         else
                 i9xx_update_pll(intel_crtc,
                                 has_reduced_clock ? &reduced_clock : NULL,
-                                num_connectors);
+                                num_connectors);
 
         /* Set up the display plane register */
         dspcntr = DISPPLANE_GAMMA_ENABLE;
@@ -4728,10 +4883,7 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
                         dspcntr |= DISPPLANE_SEL_PIPE_B;
         }
 
-        DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
-        drm_mode_debug_printmodeline(mode);
-
-        intel_set_pipe_timings(intel_crtc, mode, adjusted_mode);
+        intel_set_pipe_timings(intel_crtc);
 
         /* pipesrc and dspsize control the size that is scaled from,
          * which should always be the user's requested size.
@@ -4743,10 +4895,6 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 
         i9xx_set_pipeconf(intel_crtc);
 
-        intel_enable_pipe(dev_priv, pipe, false);
-
-        intel_wait_for_vblank(dev, pipe);
-
         I915_WRITE(DSPCNTR(plane), dspcntr);
         POSTING_READ(DSPCNTR(plane));
 
@@ -4757,6 +4905,36 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
         return ret;
 }
 
+static void i9xx_get_pfit_config(struct intel_crtc *crtc,
+                                 struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t tmp;
+
+        tmp = I915_READ(PFIT_CONTROL);
+
+        if (INTEL_INFO(dev)->gen < 4) {
+                if (crtc->pipe != PIPE_B)
+                        return;
+
+                /* gen2/3 store dither state in pfit control, needs to match */
+                pipe_config->gmch_pfit.control = tmp & PANEL_8TO6_DITHER_ENABLE;
+        } else {
+                if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
+                        return;
+        }
+
+        if (!(tmp & PFIT_ENABLE))
+                return;
+
+        pipe_config->gmch_pfit.control = I915_READ(PFIT_CONTROL);
+        pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
+        if (INTEL_INFO(dev)->gen < 5)
+                pipe_config->gmch_pfit.lvds_border_bits =
+                        I915_READ(LVDS) & LVDS_BORDER_ENABLE;
+}
+
 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                                  struct intel_crtc_config *pipe_config)
 {
@@ -4764,10 +4942,34 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t tmp;
 
+        pipe_config->cpu_transcoder = crtc->pipe;
+        pipe_config->shared_dpll = DPLL_ID_PRIVATE;
+
         tmp = I915_READ(PIPECONF(crtc->pipe));
         if (!(tmp & PIPECONF_ENABLE))
                 return false;
 
+        intel_get_pipe_timings(crtc, pipe_config);
+
+        i9xx_get_pfit_config(crtc, pipe_config);
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                tmp = I915_READ(DPLL_MD(crtc->pipe));
+                pipe_config->pixel_multiplier =
+                        ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
+                         >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
+        } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+                tmp = I915_READ(DPLL(crtc->pipe));
+                pipe_config->pixel_multiplier =
+                        ((tmp & SDVO_MULTIPLIER_MASK)
+                         >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
+        } else {
+                /* Note that on i915G/GM the pixel multiplier is in the sdvo
+                 * port and will be fixed up in the encoder->get_config
+                 * function. */
+                pipe_config->pixel_multiplier = 1;
+        }
+
         return true;
 }
 
@@ -4779,7 +4981,6 @@ static void ironlake_init_pch_refclk(struct drm_device *dev)
         u32 val, final;
         bool has_lvds = false;
         bool has_cpu_edp = false;
-        bool has_pch_edp = false;
         bool has_panel = false;
         bool has_ck505 = false;
         bool can_ssc = false;
@@ -4794,25 +4995,22 @@ static void ironlake_init_pch_refclk(struct drm_device *dev)
                         break;
                 case INTEL_OUTPUT_EDP:
                         has_panel = true;
-                        if (intel_encoder_is_pch_edp(&encoder->base))
-                                has_pch_edp = true;
-                        else
+                        if (enc_to_dig_port(&encoder->base)->port == PORT_A)
                                 has_cpu_edp = true;
                         break;
                 }
         }
 
         if (HAS_PCH_IBX(dev)) {
-                has_ck505 = dev_priv->display_clock_mode;
+                has_ck505 = dev_priv->vbt.display_clock_mode;
                 can_ssc = has_ck505;
         } else {
                 has_ck505 = false;
                 can_ssc = true;
         }
 
-        DRM_DEBUG_KMS("has_panel %d has_lvds %d has_pch_edp %d has_cpu_edp %d has_ck505 %d\n",
-                      has_panel, has_lvds, has_pch_edp, has_cpu_edp,
-                      has_ck505);
+        DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d\n",
+                      has_panel, has_lvds, has_ck505);
 
         /* Ironlake: try to setup display ref clock before DPLL
          * enabling. This is only under driver's control after
@@ -5102,7 +5300,6 @@ static int ironlake_get_refclk(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_encoder *encoder;
-        struct intel_encoder *edp_encoder = NULL;
         int num_connectors = 0;
         bool is_lvds = false;
 
@@ -5111,34 +5308,28 @@ static int ironlake_get_refclk(struct drm_crtc *crtc)
                 case INTEL_OUTPUT_LVDS:
                         is_lvds = true;
                         break;
-                case INTEL_OUTPUT_EDP:
-                        edp_encoder = encoder;
-                        break;
                 }
                 num_connectors++;
         }
 
         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
-                              dev_priv->lvds_ssc_freq);
-                return dev_priv->lvds_ssc_freq * 1000;
+                              dev_priv->vbt.lvds_ssc_freq);
+                return dev_priv->vbt.lvds_ssc_freq * 1000;
         }
 
         return 120000;
 }
 
-static void ironlake_set_pipeconf(struct drm_crtc *crtc,
-                                  struct drm_display_mode *adjusted_mode,
-                                  bool dither)
+static void ironlake_set_pipeconf(struct drm_crtc *crtc)
 {
         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         uint32_t val;
 
-        val = I915_READ(PIPECONF(pipe));
+        val = 0;
 
-        val &= ~PIPECONF_BPC_MASK;
         switch (intel_crtc->config.pipe_bpp) {
         case 18:
                 val |= PIPECONF_6BPC;
@@ -5157,20 +5348,16 @@ static void ironlake_set_pipeconf(struct drm_crtc *crtc,
                 BUG();
         }
 
-        val &= ~(PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_MASK);
-        if (dither)
+        if (intel_crtc->config.dither)
                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
 
-        val &= ~PIPECONF_INTERLACE_MASK;
-        if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
+        if (intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
                 val |= PIPECONF_INTERLACED_ILK;
         else
                 val |= PIPECONF_PROGRESSIVE;
 
         if (intel_crtc->config.limited_color_range)
                 val |= PIPECONF_COLOR_RANGE_SELECT;
-        else
-                val &= ~PIPECONF_COLOR_RANGE_SELECT;
 
         I915_WRITE(PIPECONF(pipe), val);
         POSTING_READ(PIPECONF(pipe));
@@ -5240,33 +5427,31 @@ static void intel_set_pipe_csc(struct drm_crtc *crtc)
         }
 }
 
-static void haswell_set_pipeconf(struct drm_crtc *crtc,
-                                 struct drm_display_mode *adjusted_mode,
-                                 bool dither)
+static void haswell_set_pipeconf(struct drm_crtc *crtc)
 {
         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
         uint32_t val;
 
-        val = I915_READ(PIPECONF(cpu_transcoder));
+        val = 0;
 
-        val &= ~(PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_MASK);
-        if (dither)
+        if (intel_crtc->config.dither)
                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
 
-        val &= ~PIPECONF_INTERLACE_MASK_HSW;
-        if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
+        if (intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
                 val |= PIPECONF_INTERLACED_ILK;
         else
                 val |= PIPECONF_PROGRESSIVE;
 
         I915_WRITE(PIPECONF(cpu_transcoder), val);
         POSTING_READ(PIPECONF(cpu_transcoder));
+
+        I915_WRITE(GAMMA_MODE(intel_crtc->pipe), GAMMA_MODE_MODE_8BIT);
+        POSTING_READ(GAMMA_MODE(intel_crtc->pipe));
 }
 
 static bool ironlake_compute_clocks(struct drm_crtc *crtc,
-                                    struct drm_display_mode *adjusted_mode,
                                     intel_clock_t *clock,
                                     bool *has_reduced_clock,
                                     intel_clock_t *reduced_clock)
@@ -5276,22 +5461,13 @@ static bool ironlake_compute_clocks(struct drm_crtc *crtc,
         struct intel_encoder *intel_encoder;
         int refclk;
         const intel_limit_t *limit;
-        bool ret, is_sdvo = false, is_tv = false, is_lvds = false;
+        bool ret, is_lvds = false;
 
         for_each_encoder_on_crtc(dev, crtc, intel_encoder) {
                 switch (intel_encoder->type) {
                 case INTEL_OUTPUT_LVDS:
                         is_lvds = true;
                         break;
-                case INTEL_OUTPUT_SDVO:
-                case INTEL_OUTPUT_HDMI:
-                        is_sdvo = true;
-                        if (intel_encoder->needs_tv_clock)
-                                is_tv = true;
-                        break;
-                case INTEL_OUTPUT_TVOUT:
-                        is_tv = true;
-                        break;
                 }
         }
 
@@ -5303,8 +5479,9 @@ static bool ironlake_compute_clocks(struct drm_crtc *crtc,
          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
          */
         limit = intel_limit(crtc, refclk);
-        ret = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
-                              clock);
+        ret = dev_priv->display.find_dpll(limit, crtc,
+                                          to_intel_crtc(crtc)->config.port_clock,
+                                          refclk, NULL, clock);
         if (!ret)
                 return false;
 
@@ -5315,16 +5492,13 @@ static bool ironlake_compute_clocks(struct drm_crtc *crtc,
                  * by using the FP0/FP1. In such case we will disable the LVDS
                  * downclock feature.
                 */
-                *has_reduced_clock = limit->find_pll(limit, crtc,
-                                                     dev_priv->lvds_downclock,
-                                                     refclk,
-                                                     clock,
-                                                     reduced_clock);
+                *has_reduced_clock =
+                        dev_priv->display.find_dpll(limit, crtc,
+                                                    dev_priv->lvds_downclock,
+                                                    refclk, clock,
+                                                    reduced_clock);
         }
 
-        if (is_sdvo && is_tv)
-                i9xx_adjust_sdvo_tv_clock(to_intel_crtc(crtc));
-
         return true;
 }
 
@@ -5346,65 +5520,25 @@ static void cpt_enable_fdi_bc_bifurcation(struct drm_device *dev)
         POSTING_READ(SOUTH_CHICKEN1);
 }
 
-static bool ironlake_check_fdi_lanes(struct intel_crtc *intel_crtc)
+static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *pipe_B_crtc =
-                to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_B]);
-
-        DRM_DEBUG_KMS("checking fdi config on pipe %i, lanes %i\n",
-                      intel_crtc->pipe, intel_crtc->fdi_lanes);
-        if (intel_crtc->fdi_lanes > 4) {
-                DRM_DEBUG_KMS("invalid fdi lane config on pipe %i: %i lanes\n",
-                              intel_crtc->pipe, intel_crtc->fdi_lanes);
-                /* Clamp lanes to avoid programming the hw with bogus values. */
-                intel_crtc->fdi_lanes = 4;
-
-                return false;
-        }
-
-        if (INTEL_INFO(dev)->num_pipes == 2)
-                return true;
 
         switch (intel_crtc->pipe) {
         case PIPE_A:
-                return true;
+                break;
         case PIPE_B:
-                if (dev_priv->pipe_to_crtc_mapping[PIPE_C]->enabled &&
-                    intel_crtc->fdi_lanes > 2) {
-                        DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %i: %i lanes\n",
-                                      intel_crtc->pipe, intel_crtc->fdi_lanes);
-                        /* Clamp lanes to avoid programming the hw with bogus values. */
-                        intel_crtc->fdi_lanes = 2;
-
-                        return false;
-                }
-
-                if (intel_crtc->fdi_lanes > 2)
+                if (intel_crtc->config.fdi_lanes > 2)
                         WARN_ON(I915_READ(SOUTH_CHICKEN1) & FDI_BC_BIFURCATION_SELECT);
                 else
                         cpt_enable_fdi_bc_bifurcation(dev);
 
-                return true;
+                break;
         case PIPE_C:
-                if (!pipe_B_crtc->base.enabled || pipe_B_crtc->fdi_lanes <= 2) {
-                        if (intel_crtc->fdi_lanes > 2) {
-                                DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %i: %i lanes\n",
-                                              intel_crtc->pipe, intel_crtc->fdi_lanes);
-                                /* Clamp lanes to avoid programming the hw with bogus values. */
-                                intel_crtc->fdi_lanes = 2;
-
-                                return false;
-                        }
-                } else {
-                        DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
-                        return false;
-                }
-
                 cpt_enable_fdi_bc_bifurcation(dev);
 
-                return true;
+                break;
         default:
                 BUG();
         }
@@ -5421,78 +5555,13 @@ int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
         return bps / (link_bw * 8) + 1;
 }
 
-void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
-                                  struct intel_link_m_n *m_n)
+static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
 {
-        struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe = crtc->pipe;
-
-        I915_WRITE(TRANSDATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
-        I915_WRITE(TRANSDATA_N1(pipe), m_n->gmch_n);
-        I915_WRITE(TRANSDPLINK_M1(pipe), m_n->link_m);
-        I915_WRITE(TRANSDPLINK_N1(pipe), m_n->link_n);
-}
-
-void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
-                                  struct intel_link_m_n *m_n)
-{
-        struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe = crtc->pipe;
-        enum transcoder transcoder = crtc->config.cpu_transcoder;
-
-        if (INTEL_INFO(dev)->gen >= 5) {
-                I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
-                I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
-                I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
-                I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
-        } else {
-                I915_WRITE(PIPE_GMCH_DATA_M(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
-                I915_WRITE(PIPE_GMCH_DATA_N(pipe), m_n->gmch_n);
-                I915_WRITE(PIPE_DP_LINK_M(pipe), m_n->link_m);
-                I915_WRITE(PIPE_DP_LINK_N(pipe), m_n->link_n);
-        }
-}
-
-static void ironlake_fdi_set_m_n(struct drm_crtc *crtc)
-{
-        struct drm_device *dev = crtc->dev;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_display_mode *adjusted_mode =
-                &intel_crtc->config.adjusted_mode;
-        struct intel_link_m_n m_n = {0};
-        int target_clock, lane, link_bw;
-
-        /* FDI is a binary signal running at ~2.7GHz, encoding
-         * each output octet as 10 bits. The actual frequency
-         * is stored as a divider into a 100MHz clock, and the
-         * mode pixel clock is stored in units of 1KHz.
-         * Hence the bw of each lane in terms of the mode signal
-         * is:
-         */
-        link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
-
-        if (intel_crtc->config.pixel_target_clock)
-                target_clock = intel_crtc->config.pixel_target_clock;
-        else
-                target_clock = adjusted_mode->clock;
-
-        lane = ironlake_get_lanes_required(target_clock, link_bw,
-                                           intel_crtc->config.pipe_bpp);
-
-        intel_crtc->fdi_lanes = lane;
-
-        if (intel_crtc->config.pixel_multiplier > 1)
-                link_bw *= intel_crtc->config.pixel_multiplier;
-        intel_link_compute_m_n(intel_crtc->config.pipe_bpp, lane, target_clock,
-                               link_bw, &m_n);
-
-        intel_cpu_transcoder_set_m_n(intel_crtc, &m_n);
+        return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
 }
 
 static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
-                                      intel_clock_t *clock, u32 *fp,
+                                      u32 *fp,
                                       intel_clock_t *reduced_clock, u32 *fp2)
 {
         struct drm_crtc *crtc = &intel_crtc->base;
@@ -5501,7 +5570,7 @@ static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
         struct intel_encoder *intel_encoder;
         uint32_t dpll;
         int factor, num_connectors = 0;
-        bool is_lvds = false, is_sdvo = false, is_tv = false;
+        bool is_lvds = false, is_sdvo = false;
 
         for_each_encoder_on_crtc(dev, crtc, intel_encoder) {
                 switch (intel_encoder->type) {
@@ -5511,11 +5580,6 @@ static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
                 case INTEL_OUTPUT_SDVO:
                 case INTEL_OUTPUT_HDMI:
                         is_sdvo = true;
-                        if (intel_encoder->needs_tv_clock)
-                                is_tv = true;
-                        break;
-                case INTEL_OUTPUT_TVOUT:
-                        is_tv = true;
                         break;
                 }
 
@@ -5526,13 +5590,13 @@ static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
         factor = 21;
         if (is_lvds) {
                 if ((intel_panel_use_ssc(dev_priv) &&
-                     dev_priv->lvds_ssc_freq == 100) ||
+                     dev_priv->vbt.lvds_ssc_freq == 100) ||
                     (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
                         factor = 25;
-        } else if (is_sdvo && is_tv)
+        } else if (intel_crtc->config.sdvo_tv_clock)
                 factor = 20;
 
-        if (clock->m < factor * clock->n)
+        if (ironlake_needs_fb_cb_tune(&intel_crtc->config.dpll, factor))
                 *fp |= FP_CB_TUNE;
 
         if (fp2 && (reduced_clock->m < factor * reduced_clock->n))
@@ -5544,23 +5608,21 @@ static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
                 dpll |= DPLLB_MODE_LVDS;
         else
                 dpll |= DPLLB_MODE_DAC_SERIAL;
-        if (is_sdvo) {
-                if (intel_crtc->config.pixel_multiplier > 1) {
-                        dpll |= (intel_crtc->config.pixel_multiplier - 1)
-                                << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
-                }
+
+        dpll |= (intel_crtc->config.pixel_multiplier - 1)
+                << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
+
+        if (is_sdvo)
                 dpll |= DPLL_DVO_HIGH_SPEED;
-        }
-        if (intel_crtc->config.has_dp_encoder &&
-            intel_crtc->config.has_pch_encoder)
+        if (intel_crtc->config.has_dp_encoder)
                 dpll |= DPLL_DVO_HIGH_SPEED;
 
         /* compute bitmask from p1 value */
-        dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
+        dpll |= (1 << (intel_crtc->config.dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
         /* also FPA1 */
-        dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
+        dpll |= (1 << (intel_crtc->config.dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
 
-        switch (clock->p2) {
+        switch (intel_crtc->config.dpll.p2) {
         case 5:
                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
                 break;
@@ -5575,18 +5637,12 @@ static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
                 break;
         }
 
-        if (is_sdvo && is_tv)
-                dpll |= PLL_REF_INPUT_TVCLKINBC;
-        else if (is_tv)
-                /* XXX: just matching BIOS for now */
-                /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
-                dpll |= 3;
-        else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
+        if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
         else
                 dpll |= PLL_REF_INPUT_DREFCLK;
 
-        return dpll;
+        return dpll | DPLL_VCO_ENABLE;
 }
 
 static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
@@ -5596,19 +5652,16 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_display_mode *adjusted_mode =
-                &intel_crtc->config.adjusted_mode;
-        struct drm_display_mode *mode = &intel_crtc->config.requested_mode;
         int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
         int num_connectors = 0;
         intel_clock_t clock, reduced_clock;
-        u32 dpll, fp = 0, fp2 = 0;
+        u32 dpll = 0, fp = 0, fp2 = 0;
         bool ok, has_reduced_clock = false;
         bool is_lvds = false;
         struct intel_encoder *encoder;
+        struct intel_shared_dpll *pll;
         int ret;
-        bool dither, fdi_config_ok;
 
         for_each_encoder_on_crtc(dev, crtc, encoder) {
                 switch (encoder->type) {
@@ -5623,11 +5676,9 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)),
              "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev));
 
-        intel_crtc->config.cpu_transcoder = pipe;
-
-        ok = ironlake_compute_clocks(crtc, adjusted_mode, &clock,
+        ok = ironlake_compute_clocks(crtc, &clock,
                                      &has_reduced_clock, &reduced_clock);
-        if (!ok) {
+        if (!ok && !intel_crtc->config.clock_set) {
                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
                 return -EINVAL;
         }
@@ -5643,34 +5694,31 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         /* Ensure that the cursor is valid for the new mode before changing... */
         intel_crtc_update_cursor(crtc, true);
 
-        /* determine panel color depth */
-        dither = intel_crtc->config.dither;
-        if (is_lvds && dev_priv->lvds_dither)
-                dither = true;
-
-        fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
-        if (has_reduced_clock)
-                fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 |
-                        reduced_clock.m2;
-
-        dpll = ironlake_compute_dpll(intel_crtc, &clock, &fp, &reduced_clock,
-                                     has_reduced_clock ? &fp2 : NULL);
-
-        DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe);
-        drm_mode_debug_printmodeline(mode);
-
         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
         if (intel_crtc->config.has_pch_encoder) {
-                struct intel_pch_pll *pll;
+                fp = i9xx_dpll_compute_fp(&intel_crtc->config.dpll);
+                if (has_reduced_clock)
+                        fp2 = i9xx_dpll_compute_fp(&reduced_clock);
+
+                dpll = ironlake_compute_dpll(intel_crtc,
+                                             &fp, &reduced_clock,
+                                             has_reduced_clock ? &fp2 : NULL);
+
+                intel_crtc->config.dpll_hw_state.dpll = dpll;
+                intel_crtc->config.dpll_hw_state.fp0 = fp;
+                if (has_reduced_clock)
+                        intel_crtc->config.dpll_hw_state.fp1 = fp2;
+                else
+                        intel_crtc->config.dpll_hw_state.fp1 = fp;
 
-                pll = intel_get_pch_pll(intel_crtc, dpll, fp);
+                pll = intel_get_shared_dpll(intel_crtc, dpll, fp);
                 if (pll == NULL) {
-                        DRM_DEBUG_DRIVER("failed to find PLL for pipe %d\n",
-                                         pipe);
+                        DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n",
+                                         pipe_name(pipe));
                         return -EINVAL;
                 }
         } else
-                intel_put_pch_pll(intel_crtc);
+                intel_put_shared_dpll(intel_crtc);
 
         if (intel_crtc->config.has_dp_encoder)
                 intel_dp_set_m_n(intel_crtc);
@@ -5679,11 +5727,18 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
                 if (encoder->pre_pll_enable)
                         encoder->pre_pll_enable(encoder);
 
-        if (intel_crtc->pch_pll) {
-                I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll);
+        if (is_lvds && has_reduced_clock && i915_powersave)
+                intel_crtc->lowfreq_avail = true;
+        else
+                intel_crtc->lowfreq_avail = false;
+
+        if (intel_crtc->config.has_pch_encoder) {
+                pll = intel_crtc_to_shared_dpll(intel_crtc);
+
+                I915_WRITE(PCH_DPLL(pll->id), dpll);
 
                 /* Wait for the clocks to stabilize. */
-                POSTING_READ(intel_crtc->pch_pll->pll_reg);
+                POSTING_READ(PCH_DPLL(pll->id));
                 udelay(150);
 
                 /* The pixel multiplier can only be updated once the
@@ -5691,32 +5746,25 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
                  *
                  * So write it again.
                  */
-                I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll);
-        }
+                I915_WRITE(PCH_DPLL(pll->id), dpll);
 
-        intel_crtc->lowfreq_avail = false;
-        if (intel_crtc->pch_pll) {
-                if (is_lvds && has_reduced_clock && i915_powersave) {
-                        I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp2);
-                        intel_crtc->lowfreq_avail = true;
-                } else {
-                        I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp);
-                }
+                if (has_reduced_clock)
+                        I915_WRITE(PCH_FP1(pll->id), fp2);
+                else
+                        I915_WRITE(PCH_FP1(pll->id), fp);
         }
 
-        intel_set_pipe_timings(intel_crtc, mode, adjusted_mode);
+        intel_set_pipe_timings(intel_crtc);
 
-        /* Note, this also computes intel_crtc->fdi_lanes which is used below in
-         * ironlake_check_fdi_lanes. */
-        intel_crtc->fdi_lanes = 0;
-        if (intel_crtc->config.has_pch_encoder)
-                ironlake_fdi_set_m_n(crtc);
-
-        fdi_config_ok = ironlake_check_fdi_lanes(intel_crtc);
+        if (intel_crtc->config.has_pch_encoder) {
+                intel_cpu_transcoder_set_m_n(intel_crtc,
+                                             &intel_crtc->config.fdi_m_n);
+        }
 
-        ironlake_set_pipeconf(crtc, adjusted_mode, dither);
+        if (IS_IVYBRIDGE(dev))
+                ivybridge_update_fdi_bc_bifurcation(intel_crtc);
 
-        intel_wait_for_vblank(dev, pipe);
+        ironlake_set_pipeconf(crtc);
 
         /* Set up the display plane register */
         I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE);
@@ -5726,9 +5774,46 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 
         intel_update_watermarks(dev);
 
-        intel_update_linetime_watermarks(dev, pipe, adjusted_mode);
+        return ret;
+}
 
-        return fdi_config_ok ? ret : -EINVAL;
+static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
+                                        struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum transcoder transcoder = pipe_config->cpu_transcoder;
+
+        pipe_config->fdi_m_n.link_m = I915_READ(PIPE_LINK_M1(transcoder));
+        pipe_config->fdi_m_n.link_n = I915_READ(PIPE_LINK_N1(transcoder));
+        pipe_config->fdi_m_n.gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
+                                        & ~TU_SIZE_MASK;
+        pipe_config->fdi_m_n.gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
+        pipe_config->fdi_m_n.tu = ((I915_READ(PIPE_DATA_M1(transcoder))
+                                   & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
+}
+
+static void ironlake_get_pfit_config(struct intel_crtc *crtc,
+                                     struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t tmp;
+
+        tmp = I915_READ(PF_CTL(crtc->pipe));
+
+        if (tmp & PF_ENABLE) {
+                pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
+                pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
+
+                /* We currently do not free assignements of panel fitters on
+                 * ivb/hsw (since we don't use the higher upscaling modes which
+                 * differentiates them) so just WARN about this case for now. */
+                if (IS_GEN7(dev)) {
+                        WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
+                                PF_PIPE_SEL_IVB(crtc->pipe));
+                }
+        }
 }
 
 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
@@ -5738,42 +5823,67 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t tmp;
 
+        pipe_config->cpu_transcoder = crtc->pipe;
+        pipe_config->shared_dpll = DPLL_ID_PRIVATE;
+
         tmp = I915_READ(PIPECONF(crtc->pipe));
         if (!(tmp & PIPECONF_ENABLE))
                 return false;
 
-        if (I915_READ(TRANSCONF(crtc->pipe)) & TRANS_ENABLE)
+        if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
+                struct intel_shared_dpll *pll;
+
                 pipe_config->has_pch_encoder = true;
 
+                tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
+                pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
+                                          FDI_DP_PORT_WIDTH_SHIFT) + 1;
+
+                ironlake_get_fdi_m_n_config(crtc, pipe_config);
+
+                /* XXX: Can't properly read out the pch dpll pixel multiplier
+                 * since we don't have state tracking for pch clocks yet. */
+                pipe_config->pixel_multiplier = 1;
+
+                if (HAS_PCH_IBX(dev_priv->dev)) {
+                        pipe_config->shared_dpll = crtc->pipe;
+                } else {
+                        tmp = I915_READ(PCH_DPLL_SEL);
+                        if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
+                                pipe_config->shared_dpll = DPLL_ID_PCH_PLL_B;
+                        else
+                                pipe_config->shared_dpll = DPLL_ID_PCH_PLL_A;
+                }
+
+                pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
+
+                WARN_ON(!pll->get_hw_state(dev_priv, pll,
+                                           &pipe_config->dpll_hw_state));
+        } else {
+                pipe_config->pixel_multiplier = 1;
+        }
+
+        intel_get_pipe_timings(crtc, pipe_config);
+
+        ironlake_get_pfit_config(crtc, pipe_config);
+
         return true;
 }
 
 static void haswell_modeset_global_resources(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         bool enable = false;
         struct intel_crtc *crtc;
-        struct intel_encoder *encoder;
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
-                if (crtc->pipe != PIPE_A && crtc->base.enabled)
-                        enable = true;
-                /* XXX: Should check for edp transcoder here, but thanks to init
-                 * sequence that's not yet available. Just in case desktop eDP
-                 * on PORT D is possible on haswell, too. */
-        }
+                if (!crtc->base.enabled)
+                        continue;
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
-                if (encoder->type != INTEL_OUTPUT_EDP &&
-                    encoder->connectors_active)
+                if (crtc->pipe != PIPE_A || crtc->config.pch_pfit.size ||
+                    crtc->config.cpu_transcoder != TRANSCODER_EDP)
                         enable = true;
         }
 
-        /* Even the eDP panel fitter is outside the always-on well. */
-        if (dev_priv->pch_pf_size)
-                enable = true;
-
         intel_set_power_well(dev, enable);
 }
 
@@ -5784,68 +5894,28 @@ static int haswell_crtc_mode_set(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_display_mode *adjusted_mode =
-                &intel_crtc->config.adjusted_mode;
-        struct drm_display_mode *mode = &intel_crtc->config.requested_mode;
-        int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
-        int num_connectors = 0;
-        bool is_cpu_edp = false;
-        struct intel_encoder *encoder;
         int ret;
-        bool dither;
-
-        for_each_encoder_on_crtc(dev, crtc, encoder) {
-                switch (encoder->type) {
-                case INTEL_OUTPUT_EDP:
-                        if (!intel_encoder_is_pch_edp(&encoder->base))
-                                is_cpu_edp = true;
-                        break;
-                }
-
-                num_connectors++;
-        }
-
-        if (is_cpu_edp)
-                intel_crtc->config.cpu_transcoder = TRANSCODER_EDP;
-        else
-                intel_crtc->config.cpu_transcoder = pipe;
-
-        /* We are not sure yet this won't happen. */
-        WARN(!HAS_PCH_LPT(dev), "Unexpected PCH type %d\n",
-             INTEL_PCH_TYPE(dev));
 
-        WARN(num_connectors != 1, "%d connectors attached to pipe %c\n",
-             num_connectors, pipe_name(pipe));
-
-        WARN_ON(I915_READ(PIPECONF(intel_crtc->config.cpu_transcoder)) &
-                (PIPECONF_ENABLE | I965_PIPECONF_ACTIVE));
-
-        WARN_ON(I915_READ(DSPCNTR(plane)) & DISPLAY_PLANE_ENABLE);
-
-        if (!intel_ddi_pll_mode_set(crtc, adjusted_mode->clock))
+        if (!intel_ddi_pll_mode_set(crtc))
                 return -EINVAL;
 
         /* Ensure that the cursor is valid for the new mode before changing... */
         intel_crtc_update_cursor(crtc, true);
 
-        /* determine panel color depth */
-        dither = intel_crtc->config.dither;
-
-        DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe);
-        drm_mode_debug_printmodeline(mode);
-
         if (intel_crtc->config.has_dp_encoder)
                 intel_dp_set_m_n(intel_crtc);
 
         intel_crtc->lowfreq_avail = false;
 
-        intel_set_pipe_timings(intel_crtc, mode, adjusted_mode);
+        intel_set_pipe_timings(intel_crtc);
 
-        if (intel_crtc->config.has_pch_encoder)
-                ironlake_fdi_set_m_n(crtc);
+        if (intel_crtc->config.has_pch_encoder) {
+                intel_cpu_transcoder_set_m_n(intel_crtc,
+                                             &intel_crtc->config.fdi_m_n);
+        }
 
-        haswell_set_pipeconf(crtc, adjusted_mode, dither);
+        haswell_set_pipeconf(crtc);
 
         intel_set_pipe_csc(crtc);
 
@@ -5857,8 +5927,6 @@ static int haswell_crtc_mode_set(struct drm_crtc *crtc,
 
         intel_update_watermarks(dev);
 
-        intel_update_linetime_watermarks(dev, pipe, adjusted_mode);
-
         return ret;
 }
 
@@ -5867,22 +5935,69 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        enum intel_display_power_domain pfit_domain;
         uint32_t tmp;
 
-        tmp = I915_READ(PIPECONF(crtc->config.cpu_transcoder));
+        pipe_config->cpu_transcoder = crtc->pipe;
+        pipe_config->shared_dpll = DPLL_ID_PRIVATE;
+
+        tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
+        if (tmp & TRANS_DDI_FUNC_ENABLE) {
+                enum pipe trans_edp_pipe;
+                switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
+                default:
+                        WARN(1, "unknown pipe linked to edp transcoder\n");
+                case TRANS_DDI_EDP_INPUT_A_ONOFF:
+                case TRANS_DDI_EDP_INPUT_A_ON:
+                        trans_edp_pipe = PIPE_A;
+                        break;
+                case TRANS_DDI_EDP_INPUT_B_ONOFF:
+                        trans_edp_pipe = PIPE_B;
+                        break;
+                case TRANS_DDI_EDP_INPUT_C_ONOFF:
+                        trans_edp_pipe = PIPE_C;
+                        break;
+                }
+
+                if (trans_edp_pipe == crtc->pipe)
+                        pipe_config->cpu_transcoder = TRANSCODER_EDP;
+        }
+
+        if (!intel_display_power_enabled(dev,
+                        POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
+                return false;
+
+        tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
         if (!(tmp & PIPECONF_ENABLE))
                 return false;
 
         /*
-         * aswell has only FDI/PCH transcoder A. It is which is connected to
+         * Haswell has only FDI/PCH transcoder A. It is which is connected to
          * DDI E. So just check whether this pipe is wired to DDI E and whether
          * the PCH transcoder is on.
          */
-        tmp = I915_READ(TRANS_DDI_FUNC_CTL(crtc->pipe));
+        tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
         if ((tmp & TRANS_DDI_PORT_MASK) == TRANS_DDI_SELECT_PORT(PORT_E) &&
-            I915_READ(TRANSCONF(PIPE_A)) & TRANS_ENABLE)
+            I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
                 pipe_config->has_pch_encoder = true;
 
+                tmp = I915_READ(FDI_RX_CTL(PIPE_A));
+                pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
+                                          FDI_DP_PORT_WIDTH_SHIFT) + 1;
+
+                ironlake_get_fdi_m_n_config(crtc, pipe_config);
+        }
+
+        intel_get_pipe_timings(crtc, pipe_config);
+
+        pfit_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
+        if (intel_display_power_enabled(dev, pfit_domain))
+                ironlake_get_pfit_config(crtc, pipe_config);
+
+        pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
+                                   (I915_READ(IPS_CTL) & IPS_ENABLE);
+
+        pipe_config->pixel_multiplier = 1;
 
         return true;
 }
@@ -6120,7 +6235,7 @@ static void ironlake_write_eld(struct drm_connector *connector,
                 eldv |= IBX_ELD_VALIDB << 4;
                 eldv |= IBX_ELD_VALIDB << 8;
         } else {
-                DRM_DEBUG_DRIVER("ELD on port %c\n", 'A' + i);
+                DRM_DEBUG_DRIVER("ELD on port %c\n", port_name(i));
                 eldv = IBX_ELD_VALIDB << ((i - 1) * 4);
         }
 
@@ -6188,16 +6303,31 @@ void intel_crtc_load_lut(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int palreg = PALETTE(intel_crtc->pipe);
+        enum pipe pipe = intel_crtc->pipe;
+        int palreg = PALETTE(pipe);
         int i;
+        bool reenable_ips = false;
 
         /* The clocks have to be on to load the palette. */
         if (!crtc->enabled || !intel_crtc->active)
                 return;
 
+        if (!HAS_PCH_SPLIT(dev_priv->dev))
+                assert_pll_enabled(dev_priv, pipe);
+
         /* use legacy palette for Ironlake */
         if (HAS_PCH_SPLIT(dev))
-                palreg = LGC_PALETTE(intel_crtc->pipe);
+                palreg = LGC_PALETTE(pipe);
+
+        /* Workaround : Do not read or write the pipe palette/gamma data while
+         * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
+         */
+        if (intel_crtc->config.ips_enabled &&
+            ((I915_READ(GAMMA_MODE(pipe)) & GAMMA_MODE_MODE_MASK) ==
+             GAMMA_MODE_MODE_SPLIT)) {
+                hsw_disable_ips(intel_crtc);
+                reenable_ips = true;
+        }
 
         for (i = 0; i < 256; i++) {
                 I915_WRITE(palreg + 4 * i,
@@ -6205,6 +6335,9 @@ void intel_crtc_load_lut(struct drm_crtc *crtc)
                            (intel_crtc->lut_g[i] << 8) |
                            intel_crtc->lut_b[i]);
         }
+
+        if (reenable_ips)
+                hsw_enable_ips(intel_crtc);
 }
 
 static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
@@ -6451,7 +6584,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         intel_crtc->cursor_width = width;
         intel_crtc->cursor_height = height;
 
-        intel_crtc_update_cursor(crtc, true);
+        intel_crtc_update_cursor(crtc, intel_crtc->cursor_bo != NULL);
 
         return 0;
 fail_unpin:
@@ -6470,7 +6603,7 @@ static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
         intel_crtc->cursor_x = x;
         intel_crtc->cursor_y = y;
 
-        intel_crtc_update_cursor(crtc, true);
+        intel_crtc_update_cursor(crtc, intel_crtc->cursor_bo != NULL);
 
         return 0;
 }
@@ -6791,8 +6924,10 @@ static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
                         return 0;
                 }
 
-                /* XXX: Handle the 100Mhz refclk */
-                intel_clock(dev, 96000, &clock);
+                if (IS_PINEVIEW(dev))
+                        pineview_clock(96000, &clock);
+                else
+                        i9xx_clock(96000, &clock);
         } else {
                 bool is_lvds = (pipe == 1) && (I915_READ(LVDS) & LVDS_PORT_EN);
 
@@ -6804,9 +6939,9 @@ static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
                         if ((dpll & PLL_REF_INPUT_MASK) ==
                             PLLB_REF_INPUT_SPREADSPECTRUMIN) {
                                 /* XXX: might not be 66MHz */
-                                intel_clock(dev, 66000, &clock);
+                                i9xx_clock(66000, &clock);
                         } else
-                                intel_clock(dev, 48000, &clock);
+                                i9xx_clock(48000, &clock);
                 } else {
                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
                                 clock.p1 = 2;
@@ -6819,7 +6954,7 @@ static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
                         else
                                 clock.p2 = 2;
 
-                        intel_clock(dev, 48000, &clock);
+                        i9xx_clock(48000, &clock);
                 }
         }
 
@@ -6950,7 +7085,8 @@ void intel_mark_idle(struct drm_device *dev)
         }
 }
 
-void intel_mark_fb_busy(struct drm_i915_gem_object *obj)
+void intel_mark_fb_busy(struct drm_i915_gem_object *obj,
+                        struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = obj->base.dev;
         struct drm_crtc *crtc;
@@ -6962,8 +7098,12 @@ void intel_mark_fb_busy(struct drm_i915_gem_object *obj)
                 if (!crtc->fb)
                         continue;
 
-                if (to_intel_framebuffer(crtc->fb)->obj == obj)
-                        intel_increase_pllclock(crtc);
+                if (to_intel_framebuffer(crtc->fb)->obj != obj)
+                        continue;
+
+                intel_increase_pllclock(crtc);
+                if (ring && intel_fbc_enabled(dev))
+                        ring->fbc_dirty = true;
         }
 }
 
@@ -6984,6 +7124,8 @@ static void intel_crtc_destroy(struct drm_crtc *crtc)
                 kfree(work);
         }
 
+        intel_crtc_cursor_set(crtc, NULL, 0, 0, 0);
+
         drm_crtc_cleanup(crtc);
 
         kfree(intel_crtc);
@@ -7411,7 +7553,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                 goto cleanup_pending;
 
         intel_disable_fbc(dev);
-        intel_mark_fb_busy(obj);
+        intel_mark_fb_busy(obj, NULL);
         mutex_unlock(&dev->struct_mutex);
 
         trace_i915_flip_request(intel_crtc->plane, obj);
@@ -7442,28 +7584,6 @@ static struct drm_crtc_helper_funcs intel_helper_funcs = {
         .load_lut = intel_crtc_load_lut,
 };
 
-bool intel_encoder_check_is_cloned(struct intel_encoder *encoder)
-{
-        struct intel_encoder *other_encoder;
-        struct drm_crtc *crtc = &encoder->new_crtc->base;
-
-        if (WARN_ON(!crtc))
-                return false;
-
-        list_for_each_entry(other_encoder,
-                            &crtc->dev->mode_config.encoder_list,
-                            base.head) {
-
-                if (&other_encoder->new_crtc->base != crtc ||
-                    encoder == other_encoder)
-                        continue;
-                else
-                        return true;
-        }
-
-        return false;
-}
-
 static bool intel_encoder_crtc_ok(struct drm_encoder *encoder,
                                   struct drm_crtc *crtc)
 {
@@ -7531,13 +7651,39 @@ static void intel_modeset_commit_output_state(struct drm_device *dev)
         }
 }
 
+static void
+connected_sink_compute_bpp(struct intel_connector * connector,
+                           struct intel_crtc_config *pipe_config)
+{
+        int bpp = pipe_config->pipe_bpp;
+
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
+                connector->base.base.id,
+                drm_get_connector_name(&connector->base));
+
+        /* Don't use an invalid EDID bpc value */
+        if (connector->base.display_info.bpc &&
+            connector->base.display_info.bpc * 3 < bpp) {
+                DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
+                              bpp, connector->base.display_info.bpc*3);
+                pipe_config->pipe_bpp = connector->base.display_info.bpc*3;
+        }
+
+        /* Clamp bpp to 8 on screens without EDID 1.4 */
+        if (connector->base.display_info.bpc == 0 && bpp > 24) {
+                DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n",
+                              bpp);
+                pipe_config->pipe_bpp = 24;
+        }
+}
+
 static int
-pipe_config_set_bpp(struct drm_crtc *crtc,
-                    struct drm_framebuffer *fb,
-                    struct intel_crtc_config *pipe_config)
+compute_baseline_pipe_bpp(struct intel_crtc *crtc,
+                          struct drm_framebuffer *fb,
+                          struct intel_crtc_config *pipe_config)
 {
-        struct drm_device *dev = crtc->dev;
-        struct drm_connector *connector;
+        struct drm_device *dev = crtc->base.dev;
+        struct intel_connector *connector;
         int bpp;
 
         switch (fb->pixel_format) {
@@ -7580,22 +7726,66 @@ pipe_config_set_bpp(struct drm_crtc *crtc,
 
         /* Clamp display bpp to EDID value */
         list_for_each_entry(connector, &dev->mode_config.connector_list,
-                            head) {
-                if (connector->encoder && connector->encoder->crtc != crtc)
+                            base.head) {
+                if (!connector->new_encoder ||
+                    connector->new_encoder->new_crtc != crtc)
                         continue;
 
-                /* Don't use an invalid EDID bpc value */
-                if (connector->display_info.bpc &&
-                    connector->display_info.bpc * 3 < bpp) {
-                        DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
-                                      bpp, connector->display_info.bpc*3);
-                        pipe_config->pipe_bpp = connector->display_info.bpc*3;
-                }
+                connected_sink_compute_bpp(connector, pipe_config);
         }
 
         return bpp;
 }
 
+static void intel_dump_pipe_config(struct intel_crtc *crtc,
+                                   struct intel_crtc_config *pipe_config,
+                                   const char *context)
+{
+        DRM_DEBUG_KMS("[CRTC:%d]%s config for pipe %c\n", crtc->base.base.id,
+                      context, pipe_name(crtc->pipe));
+
+        DRM_DEBUG_KMS("cpu_transcoder: %c\n", transcoder_name(pipe_config->cpu_transcoder));
+        DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n",
+                      pipe_config->pipe_bpp, pipe_config->dither);
+        DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
+                      pipe_config->has_pch_encoder,
+                      pipe_config->fdi_lanes,
+                      pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
+                      pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
+                      pipe_config->fdi_m_n.tu);
+        DRM_DEBUG_KMS("requested mode:\n");
+        drm_mode_debug_printmodeline(&pipe_config->requested_mode);
+        DRM_DEBUG_KMS("adjusted mode:\n");
+        drm_mode_debug_printmodeline(&pipe_config->adjusted_mode);
+        DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
+                      pipe_config->gmch_pfit.control,
+                      pipe_config->gmch_pfit.pgm_ratios,
+                      pipe_config->gmch_pfit.lvds_border_bits);
+        DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x\n",
+                      pipe_config->pch_pfit.pos,
+                      pipe_config->pch_pfit.size);
+        DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
+}
+
+static bool check_encoder_cloning(struct drm_crtc *crtc)
+{
+        int num_encoders = 0;
+        bool uncloneable_encoders = false;
+        struct intel_encoder *encoder;
+
+        list_for_each_entry(encoder, &crtc->dev->mode_config.encoder_list,
+                            base.head) {
+                if (&encoder->new_crtc->base != crtc)
+                        continue;
+
+                num_encoders++;
+                if (!encoder->cloneable)
+                        uncloneable_encoders = true;
+        }
+
+        return !(num_encoders > 1 && uncloneable_encoders);
+}
+
 static struct intel_crtc_config *
 intel_modeset_pipe_config(struct drm_crtc *crtc,
                           struct drm_framebuffer *fb,
@@ -7605,7 +7795,13 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
         struct drm_encoder_helper_funcs *encoder_funcs;
         struct intel_encoder *encoder;
         struct intel_crtc_config *pipe_config;
-        int plane_bpp;
+        int plane_bpp, ret = -EINVAL;
+        bool retry = true;
+
+        if (!check_encoder_cloning(crtc)) {
+                DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
+                return ERR_PTR(-EINVAL);
+        }
 
         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
         if (!pipe_config)
@@ -7613,11 +7809,23 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
 
         drm_mode_copy(&pipe_config->adjusted_mode, mode);
         drm_mode_copy(&pipe_config->requested_mode, mode);
-
-        plane_bpp = pipe_config_set_bpp(crtc, fb, pipe_config);
+        pipe_config->cpu_transcoder = to_intel_crtc(crtc)->pipe;
+        pipe_config->shared_dpll = DPLL_ID_PRIVATE;
+
+        /* Compute a starting value for pipe_config->pipe_bpp taking the source
+         * plane pixel format and any sink constraints into account. Returns the
+         * source plane bpp so that dithering can be selected on mismatches
+         * after encoders and crtc also have had their say. */
+        plane_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
+                                              fb, pipe_config);
         if (plane_bpp < 0)
                 goto fail;
 
+encoder_retry:
+        /* Ensure the port clock defaults are reset when retrying. */
+        pipe_config->port_clock = 0;
+        pipe_config->pixel_multiplier = 1;
+
         /* Pass our mode to the connectors and the CRTC to give them a chance to
          * adjust it according to limitations or connector properties, and also
          * a chance to reject the mode entirely.
@@ -7646,11 +7854,27 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
                 }
         }
 
-        if (!(intel_crtc_compute_config(crtc, pipe_config))) {
+        /* Set default port clock if not overwritten by the encoder. Needs to be
+         * done afterwards in case the encoder adjusts the mode. */
+        if (!pipe_config->port_clock)
+                pipe_config->port_clock = pipe_config->adjusted_mode.clock;
+
+        ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
+        if (ret < 0) {
                 DRM_DEBUG_KMS("CRTC fixup failed\n");
                 goto fail;
         }
-        DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
+
+        if (ret == RETRY) {
+                if (WARN(!retry, "loop in pipe configuration computation\n")) {
+                        ret = -EINVAL;
+                        goto fail;
+                }
+
+                DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
+                retry = false;
+                goto encoder_retry;
+        }
 
         pipe_config->dither = pipe_config->pipe_bpp != plane_bpp;
         DRM_DEBUG_KMS("plane bpp: %i, pipe bpp: %i, dithering: %i\n",
@@ -7659,7 +7883,7 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
         return pipe_config;
 fail:
         kfree(pipe_config);
-        return ERR_PTR(-EINVAL);
+        return ERR_PTR(ret);
 }
 
 /* Computes which crtcs are affected and sets the relevant bits in the mask. For
@@ -7755,6 +7979,9 @@ intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes,
          */
         *modeset_pipes &= 1 << intel_crtc->pipe;
         *prepare_pipes &= 1 << intel_crtc->pipe;
+
+        DRM_DEBUG_KMS("set mode pipe masks: modeset: %x, prepare: %x, disable: %x\n",
+                      *modeset_pipes, *prepare_pipes, *disable_pipes);
 }
 
 static bool intel_crtc_in_use(struct drm_crtc *crtc)
@@ -7821,31 +8048,114 @@ intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes)
         list_for_each_entry((intel_crtc), \
                             &(dev)->mode_config.crtc_list, \
                             base.head) \
-                if (mask & (1 <<(intel_crtc)->pipe)) \
+                if (mask & (1 <<(intel_crtc)->pipe))
 
 static bool
-intel_pipe_config_compare(struct intel_crtc_config *current_config,
+intel_pipe_config_compare(struct drm_device *dev,
+                          struct intel_crtc_config *current_config,
                           struct intel_crtc_config *pipe_config)
 {
-        if (current_config->has_pch_encoder != pipe_config->has_pch_encoder) {
-                DRM_ERROR("mismatch in has_pch_encoder "
-                          "(expected %i, found %i)\n",
-                          current_config->has_pch_encoder,
-                          pipe_config->has_pch_encoder);
-                return false;
-        }
+#define PIPE_CONF_CHECK_X(name) \
+        if (current_config->name != pipe_config->name) { \
+                DRM_ERROR("mismatch in " #name " " \
+                          "(expected 0x%08x, found 0x%08x)\n", \
+                          current_config->name, \
+                          pipe_config->name); \
+                return false; \
+        }
+
+#define PIPE_CONF_CHECK_I(name) \
+        if (current_config->name != pipe_config->name) { \
+                DRM_ERROR("mismatch in " #name " " \
+                          "(expected %i, found %i)\n", \
+                          current_config->name, \
+                          pipe_config->name); \
+                return false; \
+        }
+
+#define PIPE_CONF_CHECK_FLAGS(name, mask)       \
+        if ((current_config->name ^ pipe_config->name) & (mask)) { \
+                DRM_ERROR("mismatch in " #name " " \
+                          "(expected %i, found %i)\n", \
+                          current_config->name & (mask), \
+                          pipe_config->name & (mask)); \
+                return false; \
+        }
+
+#define PIPE_CONF_QUIRK(quirk)  \
+        ((current_config->quirks | pipe_config->quirks) & (quirk))
+
+        PIPE_CONF_CHECK_I(cpu_transcoder);
+
+        PIPE_CONF_CHECK_I(has_pch_encoder);
+        PIPE_CONF_CHECK_I(fdi_lanes);
+        PIPE_CONF_CHECK_I(fdi_m_n.gmch_m);
+        PIPE_CONF_CHECK_I(fdi_m_n.gmch_n);
+        PIPE_CONF_CHECK_I(fdi_m_n.link_m);
+        PIPE_CONF_CHECK_I(fdi_m_n.link_n);
+        PIPE_CONF_CHECK_I(fdi_m_n.tu);
+
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_hdisplay);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_htotal);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_hblank_start);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_hblank_end);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_hsync_start);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_hsync_end);
+
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_vdisplay);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_vtotal);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_vblank_start);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_vblank_end);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_vsync_start);
+        PIPE_CONF_CHECK_I(adjusted_mode.crtc_vsync_end);
+
+        if (!HAS_PCH_SPLIT(dev))
+                PIPE_CONF_CHECK_I(pixel_multiplier);
+
+        PIPE_CONF_CHECK_FLAGS(adjusted_mode.flags,
+                              DRM_MODE_FLAG_INTERLACE);
+
+        if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
+                PIPE_CONF_CHECK_FLAGS(adjusted_mode.flags,
+                                      DRM_MODE_FLAG_PHSYNC);
+                PIPE_CONF_CHECK_FLAGS(adjusted_mode.flags,
+                                      DRM_MODE_FLAG_NHSYNC);
+                PIPE_CONF_CHECK_FLAGS(adjusted_mode.flags,
+                                      DRM_MODE_FLAG_PVSYNC);
+                PIPE_CONF_CHECK_FLAGS(adjusted_mode.flags,
+                                      DRM_MODE_FLAG_NVSYNC);
+        }
+
+        PIPE_CONF_CHECK_I(requested_mode.hdisplay);
+        PIPE_CONF_CHECK_I(requested_mode.vdisplay);
+
+        PIPE_CONF_CHECK_I(gmch_pfit.control);
+        /* pfit ratios are autocomputed by the hw on gen4+ */
+        if (INTEL_INFO(dev)->gen < 4)
+                PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios);
+        PIPE_CONF_CHECK_I(gmch_pfit.lvds_border_bits);
+        PIPE_CONF_CHECK_I(pch_pfit.pos);
+        PIPE_CONF_CHECK_I(pch_pfit.size);
+
+        PIPE_CONF_CHECK_I(ips_enabled);
+
+        PIPE_CONF_CHECK_I(shared_dpll);
+        PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
+        PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
+        PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
+
+#undef PIPE_CONF_CHECK_X
+#undef PIPE_CONF_CHECK_I
+#undef PIPE_CONF_CHECK_FLAGS
+#undef PIPE_CONF_QUIRK
 
         return true;
 }
 
-void
-intel_modeset_check_state(struct drm_device *dev)
+static void
+check_connector_state(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct intel_crtc *crtc;
-        struct intel_encoder *encoder;
         struct intel_connector *connector;
-        struct intel_crtc_config pipe_config;
 
         list_for_each_entry(connector, &dev->mode_config.connector_list,
                             base.head) {
@@ -7856,6 +8166,13 @@ intel_modeset_check_state(struct drm_device *dev)
                 WARN(&connector->new_encoder->base != connector->base.encoder,
                      "connector's staged encoder doesn't match current encoder\n");
         }
+}
+
+static void
+check_encoder_state(struct drm_device *dev)
+{
+        struct intel_encoder *encoder;
+        struct intel_connector *connector;
 
         list_for_each_entry(encoder, &dev->mode_config.encoder_list,
                             base.head) {
@@ -7907,12 +8224,23 @@ intel_modeset_check_state(struct drm_device *dev)
                      tracked_pipe, pipe);
 
         }
+}
+
+static void
+check_crtc_state(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_crtc *crtc;
+        struct intel_encoder *encoder;
+        struct intel_crtc_config pipe_config;
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list,
                             base.head) {
                 bool enabled = false;
                 bool active = false;
 
+                memset(&pipe_config, 0, sizeof(pipe_config));
+
                 DRM_DEBUG_KMS("[CRTC:%d]\n",
                               crtc->base.base.id);
 
@@ -7927,6 +8255,7 @@ intel_modeset_check_state(struct drm_device *dev)
                         if (encoder->connectors_active)
                                 active = true;
                 }
+
                 WARN(active != crtc->active,
                      "crtc's computed active state doesn't match tracked active state "
                      "(expected %i, found %i)\n", active, crtc->active);
@@ -7934,7 +8263,6 @@ intel_modeset_check_state(struct drm_device *dev)
                      "crtc's computed enabled state doesn't match tracked enabled state "
                      "(expected %i, found %i)\n", enabled, crtc->base.enabled);
 
-                memset(&pipe_config, 0, sizeof(pipe_config));
                 active = dev_priv->display.get_pipe_config(crtc,
                                                            &pipe_config);
 
@@ -7942,16 +8270,86 @@ intel_modeset_check_state(struct drm_device *dev)
                 if (crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
                         active = crtc->active;
 
+                list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+                                    base.head) {
+                        if (encoder->base.crtc != &crtc->base)
+                                continue;
+                        if (encoder->get_config)
+                                encoder->get_config(encoder, &pipe_config);
+                }
+
                 WARN(crtc->active != active,
                      "crtc active state doesn't match with hw state "
                      "(expected %i, found %i)\n", crtc->active, active);
 
-                WARN(active &&
-                     !intel_pipe_config_compare(&crtc->config, &pipe_config),
-                     "pipe state doesn't match!\n");
+                if (active &&
+                    !intel_pipe_config_compare(dev, &crtc->config, &pipe_config)) {
+                        WARN(1, "pipe state doesn't match!\n");
+                        intel_dump_pipe_config(crtc, &pipe_config,
+                                               "[hw state]");
+                        intel_dump_pipe_config(crtc, &crtc->config,
+                                               "[sw state]");
+                }
         }
 }
 
+static void
+check_shared_dpll_state(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_crtc *crtc;
+        struct intel_dpll_hw_state dpll_hw_state;
+        int i;
+
+        for (i = 0; i < dev_priv->num_shared_dpll; i++) {
+                struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
+                int enabled_crtcs = 0, active_crtcs = 0;
+                bool active;
+
+                memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
+
+                DRM_DEBUG_KMS("%s\n", pll->name);
+
+                active = pll->get_hw_state(dev_priv, pll, &dpll_hw_state);
+
+                WARN(pll->active > pll->refcount,
+                     "more active pll users than references: %i vs %i\n",
+                     pll->active, pll->refcount);
+                WARN(pll->active && !pll->on,
+                     "pll in active use but not on in sw tracking\n");
+                WARN(pll->on != active,
+                     "pll on state mismatch (expected %i, found %i)\n",
+                     pll->on, active);
+
+                list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+                                    base.head) {
+                        if (crtc->base.enabled && intel_crtc_to_shared_dpll(crtc) == pll)
+                                enabled_crtcs++;
+                        if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll)
+                                active_crtcs++;
+                }
+                WARN(pll->active != active_crtcs,
+                     "pll active crtcs mismatch (expected %i, found %i)\n",
+                     pll->active, active_crtcs);
+                WARN(pll->refcount != enabled_crtcs,
+                     "pll enabled crtcs mismatch (expected %i, found %i)\n",
+                     pll->refcount, enabled_crtcs);
+
+                WARN(pll->on && memcmp(&pll->hw_state, &dpll_hw_state,
+                                       sizeof(dpll_hw_state)),
+                     "pll hw state mismatch\n");
+        }
+}
+
+void
+intel_modeset_check_state(struct drm_device *dev)
+{
+        check_connector_state(dev);
+        check_encoder_state(dev);
+        check_crtc_state(dev);
+        check_shared_dpll_state(dev);
+}
+
 static int __intel_set_mode(struct drm_crtc *crtc,
                             struct drm_display_mode *mode,
                             int x, int y, struct drm_framebuffer *fb)
@@ -7988,11 +8386,10 @@ static int __intel_set_mode(struct drm_crtc *crtc,
 
                         goto out;
                 }
+                intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
+                                       "[modeset]");
         }
 
-        DRM_DEBUG_KMS("set mode pipe masks: modeset: %x, prepare: %x, disable: %x\n",
-                      modeset_pipes, prepare_pipes, disable_pipes);
-
         for_each_intel_crtc_masked(dev, disable_pipes, intel_crtc)
                 intel_crtc_disable(&intel_crtc->base);
 
@@ -8005,12 +8402,10 @@ static int __intel_set_mode(struct drm_crtc *crtc,
          * to set it here already despite that we pass it down the callchain.
          */
         if (modeset_pipes) {
-                enum transcoder tmp = to_intel_crtc(crtc)->config.cpu_transcoder;
                 crtc->mode = *mode;
                 /* mode_set/enable/disable functions rely on a correct pipe
                  * config. */
                 to_intel_crtc(crtc)->config = *pipe_config;
-                to_intel_crtc(crtc)->config.cpu_transcoder = tmp;
         }
 
         /* Only after disabling all output pipelines that will be changed can we
@@ -8349,12 +8744,6 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
                 goto fail;
 
         if (config->mode_changed) {
-                if (set->mode) {
-                        DRM_DEBUG_KMS("attempting to set mode from"
-                                        " userspace\n");
-                        drm_mode_debug_printmodeline(set->mode);
-                }
-
                 ret = intel_set_mode(set->crtc, set->mode,
                                      set->x, set->y, set->fb);
         } else if (config->fb_changed) {
@@ -8365,8 +8754,8 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
         }
 
         if (ret) {
-                DRM_ERROR("failed to set mode on [CRTC:%d], err = %d\n",
-                          set->crtc->base.id, ret);
+                DRM_DEBUG_KMS("failed to set mode on [CRTC:%d], err = %d\n",
+                              set->crtc->base.id, ret);
 fail:
                 intel_set_config_restore_state(dev, config);
 
@@ -8397,23 +8786,93 @@ static void intel_cpu_pll_init(struct drm_device *dev)
                 intel_ddi_pll_init(dev);
 }
 
-static void intel_pch_pll_init(struct drm_device *dev)
+static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv,
+                                      struct intel_shared_dpll *pll,
+                                      struct intel_dpll_hw_state *hw_state)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int i;
+        uint32_t val;
 
-        if (dev_priv->num_pch_pll == 0) {
-                DRM_DEBUG_KMS("No PCH PLLs on this hardware, skipping initialisation\n");
-                return;
+        val = I915_READ(PCH_DPLL(pll->id));
+        hw_state->dpll = val;
+        hw_state->fp0 = I915_READ(PCH_FP0(pll->id));
+        hw_state->fp1 = I915_READ(PCH_FP1(pll->id));
+
+        return val & DPLL_VCO_ENABLE;
+}
+
+static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
+                                struct intel_shared_dpll *pll)
+{
+        uint32_t reg, val;
+
+        /* PCH refclock must be enabled first */
+        assert_pch_refclk_enabled(dev_priv);
+
+        reg = PCH_DPLL(pll->id);
+        val = I915_READ(reg);
+        val |= DPLL_VCO_ENABLE;
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(200);
+}
+
+static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
+                                 struct intel_shared_dpll *pll)
+{
+        struct drm_device *dev = dev_priv->dev;
+        struct intel_crtc *crtc;
+        uint32_t reg, val;
+
+        /* Make sure no transcoder isn't still depending on us. */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+                if (intel_crtc_to_shared_dpll(crtc) == pll)
+                        assert_pch_transcoder_disabled(dev_priv, crtc->pipe);
         }
 
-        for (i = 0; i < dev_priv->num_pch_pll; i++) {
-                dev_priv->pch_plls[i].pll_reg = _PCH_DPLL(i);
-                dev_priv->pch_plls[i].fp0_reg = _PCH_FP0(i);
-                dev_priv->pch_plls[i].fp1_reg = _PCH_FP1(i);
+        reg = PCH_DPLL(pll->id);
+        val = I915_READ(reg);
+        val &= ~DPLL_VCO_ENABLE;
+        I915_WRITE(reg, val);
+        POSTING_READ(reg);
+        udelay(200);
+}
+
+static char *ibx_pch_dpll_names[] = {
+        "PCH DPLL A",
+        "PCH DPLL B",
+};
+
+static void ibx_pch_dpll_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
+
+        dev_priv->num_shared_dpll = 2;
+
+        for (i = 0; i < dev_priv->num_shared_dpll; i++) {
+                dev_priv->shared_dplls[i].id = i;
+                dev_priv->shared_dplls[i].name = ibx_pch_dpll_names[i];
+                dev_priv->shared_dplls[i].enable = ibx_pch_dpll_enable;
+                dev_priv->shared_dplls[i].disable = ibx_pch_dpll_disable;
+                dev_priv->shared_dplls[i].get_hw_state =
+                        ibx_pch_dpll_get_hw_state;
         }
 }
 
+static void intel_shared_dpll_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+                ibx_pch_dpll_init(dev);
+        else
+                dev_priv->num_shared_dpll = 0;
+
+        BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
+        DRM_DEBUG_KMS("%i shared PLLs initialized\n",
+                      dev_priv->num_shared_dpll);
+}
+
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
@@ -8436,7 +8895,6 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         /* Swap pipes & planes for FBC on pre-965 */
         intel_crtc->pipe = pipe;
         intel_crtc->plane = pipe;
-        intel_crtc->config.cpu_transcoder = pipe;
         if (IS_MOBILE(dev) && IS_GEN3(dev)) {
                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
                 intel_crtc->plane = !pipe;
@@ -8519,13 +8977,8 @@ static void intel_setup_outputs(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_encoder *encoder;
         bool dpd_is_edp = false;
-        bool has_lvds;
 
-        has_lvds = intel_lvds_init(dev);
-        if (!has_lvds && !HAS_PCH_SPLIT(dev)) {
-                /* disable the panel fitter on everything but LVDS */
-                I915_WRITE(PFIT_CONTROL, 0);
-        }
+        intel_lvds_init(dev);
 
         if (!IS_ULT(dev))
                 intel_crt_init(dev);
@@ -8598,10 +9051,8 @@ static void intel_setup_outputs(struct drm_device *dev)
                                 intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
                         }
 
-                        if (!found && SUPPORTS_INTEGRATED_DP(dev)) {
-                                DRM_DEBUG_KMS("probing DP_B\n");
+                        if (!found && SUPPORTS_INTEGRATED_DP(dev))
                                 intel_dp_init(dev, DP_B, PORT_B);
-                        }
                 }
 
                 /* Before G4X SDVOC doesn't have its own detect register */
@@ -8617,17 +9068,13 @@ static void intel_setup_outputs(struct drm_device *dev)
                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
                                 intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
                         }
-                        if (SUPPORTS_INTEGRATED_DP(dev)) {
-                                DRM_DEBUG_KMS("probing DP_C\n");
+                        if (SUPPORTS_INTEGRATED_DP(dev))
                                 intel_dp_init(dev, DP_C, PORT_C);
-                        }
                 }
 
                 if (SUPPORTS_INTEGRATED_DP(dev) &&
-                    (I915_READ(DP_D) & DP_DETECTED)) {
-                        DRM_DEBUG_KMS("probing DP_D\n");
+                    (I915_READ(DP_D) & DP_DETECTED))
                         intel_dp_init(dev, DP_D, PORT_D);
-                }
         } else if (IS_GEN2(dev))
                 intel_dvo_init(dev);
 
@@ -8675,6 +9122,7 @@ int intel_framebuffer_init(struct drm_device *dev,
                            struct drm_mode_fb_cmd2 *mode_cmd,
                            struct drm_i915_gem_object *obj)
 {
+        int pitch_limit;
         int ret;
 
         if (obj->tiling_mode == I915_TILING_Y) {
@@ -8688,10 +9136,26 @@ int intel_framebuffer_init(struct drm_device *dev,
                 return -EINVAL;
         }
 
-        /* FIXME <= Gen4 stride limits are bit unclear */
-        if (mode_cmd->pitches[0] > 32768) {
-                DRM_DEBUG("pitch (%d) must be at less than 32768\n",
-                          mode_cmd->pitches[0]);
+        if (INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev)) {
+                pitch_limit = 32*1024;
+        } else if (INTEL_INFO(dev)->gen >= 4) {
+                if (obj->tiling_mode)
+                        pitch_limit = 16*1024;
+                else
+                        pitch_limit = 32*1024;
+        } else if (INTEL_INFO(dev)->gen >= 3) {
+                if (obj->tiling_mode)
+                        pitch_limit = 8*1024;
+                else
+                        pitch_limit = 16*1024;
+        } else
+                /* XXX DSPC is limited to 4k tiled */
+                pitch_limit = 8*1024;
+
+        if (mode_cmd->pitches[0] > pitch_limit) {
+                DRM_DEBUG("%s pitch (%d) must be at less than %d\n",
+                          obj->tiling_mode ? "tiled" : "linear",
+                          mode_cmd->pitches[0], pitch_limit);
                 return -EINVAL;
         }
 
@@ -8712,7 +9176,8 @@ int intel_framebuffer_init(struct drm_device *dev,
         case DRM_FORMAT_XRGB1555:
         case DRM_FORMAT_ARGB1555:
                 if (INTEL_INFO(dev)->gen > 3) {
-                        DRM_DEBUG("invalid format: 0x%08x\n", mode_cmd->pixel_format);
+                        DRM_DEBUG("unsupported pixel format: %s\n",
+                                  drm_get_format_name(mode_cmd->pixel_format));
                         return -EINVAL;
                 }
                 break;
@@ -8723,7 +9188,8 @@ int intel_framebuffer_init(struct drm_device *dev,
         case DRM_FORMAT_XBGR2101010:
         case DRM_FORMAT_ABGR2101010:
                 if (INTEL_INFO(dev)->gen < 4) {
-                        DRM_DEBUG("invalid format: 0x%08x\n", mode_cmd->pixel_format);
+                        DRM_DEBUG("unsupported pixel format: %s\n",
+                                  drm_get_format_name(mode_cmd->pixel_format));
                         return -EINVAL;
                 }
                 break;
@@ -8732,12 +9198,14 @@ int intel_framebuffer_init(struct drm_device *dev,
         case DRM_FORMAT_YVYU:
         case DRM_FORMAT_VYUY:
                 if (INTEL_INFO(dev)->gen < 5) {
-                        DRM_DEBUG("invalid format: 0x%08x\n", mode_cmd->pixel_format);
+                        DRM_DEBUG("unsupported pixel format: %s\n",
+                                  drm_get_format_name(mode_cmd->pixel_format));
                         return -EINVAL;
                 }
                 break;
         default:
-                DRM_DEBUG("unsupported pixel format 0x%08x\n", mode_cmd->pixel_format);
+                DRM_DEBUG("unsupported pixel format: %s\n",
+                          drm_get_format_name(mode_cmd->pixel_format));
                 return -EINVAL;
         }
 
@@ -8782,6 +9250,15 @@ static void intel_init_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        if (HAS_PCH_SPLIT(dev) || IS_G4X(dev))
+                dev_priv->display.find_dpll = g4x_find_best_dpll;
+        else if (IS_VALLEYVIEW(dev))
+                dev_priv->display.find_dpll = vlv_find_best_dpll;
+        else if (IS_PINEVIEW(dev))
+                dev_priv->display.find_dpll = pnv_find_best_dpll;
+        else
+                dev_priv->display.find_dpll = i9xx_find_best_dpll;
+
         if (HAS_DDI(dev)) {
                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
                 dev_priv->display.crtc_mode_set = haswell_crtc_mode_set;
@@ -8796,6 +9273,13 @@ static void intel_init_display(struct drm_device *dev)
                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
                 dev_priv->display.off = ironlake_crtc_off;
                 dev_priv->display.update_plane = ironlake_update_plane;
+        } else if (IS_VALLEYVIEW(dev)) {
+                dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
+                dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
+                dev_priv->display.crtc_enable = valleyview_crtc_enable;
+                dev_priv->display.crtc_disable = i9xx_crtc_disable;
+                dev_priv->display.off = i9xx_crtc_off;
+                dev_priv->display.update_plane = i9xx_update_plane;
         } else {
                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
                 dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
@@ -9037,6 +9521,11 @@ void intel_modeset_init_hw(struct drm_device *dev)
         mutex_unlock(&dev->struct_mutex);
 }
 
+void intel_modeset_suspend_hw(struct drm_device *dev)
+{
+        intel_suspend_hw(dev);
+}
+
 void intel_modeset_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -9082,13 +9571,13 @@ void intel_modeset_init(struct drm_device *dev)
                 for (j = 0; j < dev_priv->num_plane; j++) {
                         ret = intel_plane_init(dev, i, j);
                         if (ret)
-                                DRM_DEBUG_KMS("pipe %d plane %d init failed: %d\n",
-                                              i, j, ret);
+                                DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
+                                              pipe_name(i), sprite_name(i, j), ret);
                 }
         }
 
         intel_cpu_pll_init(dev);
-        intel_pch_pll_init(dev);
+        intel_shared_dpll_init(dev);
 
         /* Just disable it once at startup */
         i915_disable_vga(dev);
@@ -9289,57 +9778,18 @@ void i915_redisable_vga(struct drm_device *dev)
         }
 }
 
-/* Scan out the current hw modeset state, sanitizes it and maps it into the drm
- * and i915 state tracking structures. */
-void intel_modeset_setup_hw_state(struct drm_device *dev,
-                                  bool force_restore)
+static void intel_modeset_readout_hw_state(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe;
-        u32 tmp;
-        struct drm_plane *plane;
         struct intel_crtc *crtc;
         struct intel_encoder *encoder;
         struct intel_connector *connector;
+        int i;
 
-        if (HAS_DDI(dev)) {
-                tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
-
-                if (tmp & TRANS_DDI_FUNC_ENABLE) {
-                        switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
-                        case TRANS_DDI_EDP_INPUT_A_ON:
-                        case TRANS_DDI_EDP_INPUT_A_ONOFF:
-                                pipe = PIPE_A;
-                                break;
-                        case TRANS_DDI_EDP_INPUT_B_ONOFF:
-                                pipe = PIPE_B;
-                                break;
-                        case TRANS_DDI_EDP_INPUT_C_ONOFF:
-                                pipe = PIPE_C;
-                                break;
-                        default:
-                                /* A bogus value has been programmed, disable
-                                 * the transcoder */
-                                WARN(1, "Bogus eDP source %08x\n", tmp);
-                                intel_ddi_disable_transcoder_func(dev_priv,
-                                                TRANSCODER_EDP);
-                                goto setup_pipes;
-                        }
-
-                        crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
-                        crtc->config.cpu_transcoder = TRANSCODER_EDP;
-
-                        DRM_DEBUG_KMS("Pipe %c using transcoder EDP\n",
-                                      pipe_name(pipe));
-                }
-        }
-
-setup_pipes:
         list_for_each_entry(crtc, &dev->mode_config.crtc_list,
                             base.head) {
-                enum transcoder tmp = crtc->config.cpu_transcoder;
                 memset(&crtc->config, 0, sizeof(crtc->config));
-                crtc->config.cpu_transcoder = tmp;
 
                 crtc->active = dev_priv->display.get_pipe_config(crtc,
                                                                  &crtc->config);
@@ -9351,16 +9801,35 @@ setup_pipes:
                               crtc->active ? "enabled" : "disabled");
         }
 
+        /* FIXME: Smash this into the new shared dpll infrastructure. */
         if (HAS_DDI(dev))
                 intel_ddi_setup_hw_pll_state(dev);
 
+        for (i = 0; i < dev_priv->num_shared_dpll; i++) {
+                struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
+
+                pll->on = pll->get_hw_state(dev_priv, pll, &pll->hw_state);
+                pll->active = 0;
+                list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+                                    base.head) {
+                        if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll)
+                                pll->active++;
+                }
+                pll->refcount = pll->active;
+
+                DRM_DEBUG_KMS("%s hw state readout: refcount %i\n",
+                              pll->name, pll->refcount);
+        }
+
         list_for_each_entry(encoder, &dev->mode_config.encoder_list,
                             base.head) {
                 pipe = 0;
 
                 if (encoder->get_hw_state(encoder, &pipe)) {
-                        encoder->base.crtc =
-                                dev_priv->pipe_to_crtc_mapping[pipe];
+                        crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+                        encoder->base.crtc = &crtc->base;
+                        if (encoder->get_config)
+                                encoder->get_config(encoder, &crtc->config);
                 } else {
                         encoder->base.crtc = NULL;
                 }
@@ -9388,6 +9857,20 @@ setup_pipes:
                               drm_get_connector_name(&connector->base),
                               connector->base.encoder ? "enabled" : "disabled");
         }
+}
+
+/* Scan out the current hw modeset state, sanitizes it and maps it into the drm
+ * and i915 state tracking structures. */
+void intel_modeset_setup_hw_state(struct drm_device *dev,
+                                  bool force_restore)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe pipe;
+        struct drm_plane *plane;
+        struct intel_crtc *crtc;
+        struct intel_encoder *encoder;
+
+        intel_modeset_readout_hw_state(dev);
 
         /* HW state is read out, now we need to sanitize this mess. */
         list_for_each_entry(encoder, &dev->mode_config.encoder_list,
@@ -9398,6 +9881,7 @@ setup_pipes:
         for_each_pipe(pipe) {
                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
                 intel_sanitize_crtc(crtc);
+                intel_dump_pipe_config(crtc, &crtc->config, "[setup_hw_state]");
         }
 
         if (force_restore) {
@@ -9440,12 +9924,23 @@ void intel_modeset_cleanup(struct drm_device *dev)
         struct drm_crtc *crtc;
         struct intel_crtc *intel_crtc;
 
+        /*
+         * Interrupts and polling as the first thing to avoid creating havoc.
+         * Too much stuff here (turning of rps, connectors, ...) would
+         * experience fancy races otherwise.
+         */
+        drm_irq_uninstall(dev);
+        cancel_work_sync(&dev_priv->hotplug_work);
+        /*
+         * Due to the hpd irq storm handling the hotplug work can re-arm the
+         * poll handlers. Hence disable polling after hpd handling is shut down.
+         */
         drm_kms_helper_poll_fini(dev);
+
         mutex_lock(&dev->struct_mutex);
 
         intel_unregister_dsm_handler();
 
-
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 /* Skip inactive CRTCs */
                 if (!crtc->fb)
@@ -9461,17 +9956,8 @@ void intel_modeset_cleanup(struct drm_device *dev)
 
         ironlake_teardown_rc6(dev);
 
-        if (IS_VALLEYVIEW(dev))
-                vlv_init_dpio(dev);
-
         mutex_unlock(&dev->struct_mutex);
 
-        /* Disable the irq before mode object teardown, for the irq might
-         * enqueue unpin/hotplug work. */
-        drm_irq_uninstall(dev);
-        cancel_work_sync(&dev_priv->hotplug_work);
-        cancel_work_sync(&dev_priv->rps.work);
-
         /* flush any delayed tasks or pending work */
         flush_scheduled_work();
 
@@ -9520,6 +10006,9 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
 #include <linux/seq_file.h>
 
 struct intel_display_error_state {
+
+        u32 power_well_driver;
+
         struct intel_cursor_error_state {
                 u32 control;
                 u32 position;
@@ -9528,6 +10017,7 @@ struct intel_display_error_state {
         } cursor[I915_MAX_PIPES];
 
         struct intel_pipe_error_state {
+                enum transcoder cpu_transcoder;
                 u32 conf;
                 u32 source;
 
@@ -9562,8 +10052,12 @@ intel_display_capture_error_state(struct drm_device *dev)
         if (error == NULL)
                 return NULL;
 
+        if (HAS_POWER_WELL(dev))
+                error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
+
         for_each_pipe(i) {
                 cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, i);
+                error->pipe[i].cpu_transcoder = cpu_transcoder;
 
                 if (INTEL_INFO(dev)->gen <= 6 || IS_VALLEYVIEW(dev)) {
                         error->cursor[i].control = I915_READ(CURCNTR(i));
@@ -9598,46 +10092,60 @@ intel_display_capture_error_state(struct drm_device *dev)
                 error->pipe[i].vsync = I915_READ(VSYNC(cpu_transcoder));
         }
 
+        /* In the code above we read the registers without checking if the power
+         * well was on, so here we have to clear the FPGA_DBG_RM_NOCLAIM bit to
+         * prevent the next I915_WRITE from detecting it and printing an error
+         * message. */
+        if (HAS_POWER_WELL(dev))
+                I915_WRITE_NOTRACE(FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
+
         return error;
 }
 
+#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
+
 void
-intel_display_print_error_state(struct seq_file *m,
+intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                                 struct drm_device *dev,
                                 struct intel_display_error_state *error)
 {
         int i;
 
-        seq_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
+        err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
+        if (HAS_POWER_WELL(dev))
+                err_printf(m, "PWR_WELL_CTL2: %08x\n",
+                           error->power_well_driver);
         for_each_pipe(i) {
-                seq_printf(m, "Pipe [%d]:\n", i);
-                seq_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
-                seq_printf(m, "  SRC: %08x\n", error->pipe[i].source);
-                seq_printf(m, "  HTOTAL: %08x\n", error->pipe[i].htotal);
-                seq_printf(m, "  HBLANK: %08x\n", error->pipe[i].hblank);
-                seq_printf(m, "  HSYNC: %08x\n", error->pipe[i].hsync);
-                seq_printf(m, "  VTOTAL: %08x\n", error->pipe[i].vtotal);
-                seq_printf(m, "  VBLANK: %08x\n", error->pipe[i].vblank);
-                seq_printf(m, "  VSYNC: %08x\n", error->pipe[i].vsync);
-
-                seq_printf(m, "Plane [%d]:\n", i);
-                seq_printf(m, "  CNTR: %08x\n", error->plane[i].control);
-                seq_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
+                err_printf(m, "Pipe [%d]:\n", i);
+                err_printf(m, "  CPU transcoder: %c\n",
+                           transcoder_name(error->pipe[i].cpu_transcoder));
+                err_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
+                err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
+                err_printf(m, "  HTOTAL: %08x\n", error->pipe[i].htotal);
+                err_printf(m, "  HBLANK: %08x\n", error->pipe[i].hblank);
+                err_printf(m, "  HSYNC: %08x\n", error->pipe[i].hsync);
+                err_printf(m, "  VTOTAL: %08x\n", error->pipe[i].vtotal);
+                err_printf(m, "  VBLANK: %08x\n", error->pipe[i].vblank);
+                err_printf(m, "  VSYNC: %08x\n", error->pipe[i].vsync);
+
+                err_printf(m, "Plane [%d]:\n", i);
+                err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
+                err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
                 if (INTEL_INFO(dev)->gen <= 3) {
-                        seq_printf(m, "  SIZE: %08x\n", error->plane[i].size);
-                        seq_printf(m, "  POS: %08x\n", error->plane[i].pos);
+                        err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
+                        err_printf(m, "  POS: %08x\n", error->plane[i].pos);
                 }
                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
-                        seq_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
+                        err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
                 if (INTEL_INFO(dev)->gen >= 4) {
-                        seq_printf(m, "  SURF: %08x\n", error->plane[i].surface);
-                        seq_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
+                        err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
+                        err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
                 }
 
-                seq_printf(m, "Cursor [%d]:\n", i);
-                seq_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
-                seq_printf(m, "  POS: %08x\n", error->cursor[i].position);
-                seq_printf(m, "  BASE: %08x\n", error->cursor[i].base);
+                err_printf(m, "Cursor [%d]:\n", i);
+                err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
+                err_printf(m, "  POS: %08x\n", error->cursor[i].position);
+                err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
         }
 }
 #endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 70789b1b5642..b73971234013 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -52,30 +52,6 @@ static bool is_edp(struct intel_dp *intel_dp)
         return intel_dig_port->base.type == INTEL_OUTPUT_EDP;
 }
 
-/**
- * is_pch_edp - is the port on the PCH and attached to an eDP panel?
- * @intel_dp: DP struct
- *
- * Returns true if the given DP struct corresponds to a PCH DP port attached
- * to an eDP panel, false otherwise.  Helpful for determining whether we
- * may need FDI resources for a given DP output or not.
- */
-static bool is_pch_edp(struct intel_dp *intel_dp)
-{
-        return intel_dp->is_pch_edp;
-}
-
-/**
- * is_cpu_edp - is the port on the CPU and attached to an eDP panel?
- * @intel_dp: DP struct
- *
- * Returns true if the given DP struct corresponds to a CPU eDP port.
- */
-static bool is_cpu_edp(struct intel_dp *intel_dp)
-{
-        return is_edp(intel_dp) && !is_pch_edp(intel_dp);
-}
-
 static struct drm_device *intel_dp_to_dev(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
@@ -88,25 +64,6 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
         return enc_to_intel_dp(&intel_attached_encoder(connector)->base);
 }
 
-/**
- * intel_encoder_is_pch_edp - is the given encoder a PCH attached eDP?
- * @encoder: DRM encoder
- *
- * Return true if @encoder corresponds to a PCH attached eDP panel.  Needed
- * by intel_display.c.
- */
-bool intel_encoder_is_pch_edp(struct drm_encoder *encoder)
-{
-        struct intel_dp *intel_dp;
-
-        if (!encoder)
-                return false;
-
-        intel_dp = enc_to_intel_dp(encoder);
-
-        return is_pch_edp(intel_dp);
-}
-
 static void intel_dp_link_down(struct intel_dp *intel_dp);
 
 static int
@@ -344,11 +301,12 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
          * Note that PCH attached eDP panels should use a 125MHz input
          * clock divider.
          */
-        if (is_cpu_edp(intel_dp)) {
+        if (IS_VALLEYVIEW(dev)) {
+                aux_clock_divider = 100;
+        } else if (intel_dig_port->port == PORT_A) {
                 if (HAS_DDI(dev))
-                        aux_clock_divider = intel_ddi_get_cdclk_freq(dev_priv) >> 1;
-                else if (IS_VALLEYVIEW(dev))
-                        aux_clock_divider = 100;
+                        aux_clock_divider = DIV_ROUND_CLOSEST(
+                                intel_ddi_get_cdclk_freq(dev_priv), 2000);
                 else if (IS_GEN6(dev) || IS_GEN7(dev))
                         aux_clock_divider = 200; /* SNB & IVB eDP input clock at 400Mhz */
                 else
@@ -660,6 +618,49 @@ intel_dp_i2c_init(struct intel_dp *intel_dp,
         return ret;
 }
 
+static void
+intel_dp_set_clock(struct intel_encoder *encoder,
+                   struct intel_crtc_config *pipe_config, int link_bw)
+{
+        struct drm_device *dev = encoder->base.dev;
+
+        if (IS_G4X(dev)) {
+                if (link_bw == DP_LINK_BW_1_62) {
+                        pipe_config->dpll.p1 = 2;
+                        pipe_config->dpll.p2 = 10;
+                        pipe_config->dpll.n = 2;
+                        pipe_config->dpll.m1 = 23;
+                        pipe_config->dpll.m2 = 8;
+                } else {
+                        pipe_config->dpll.p1 = 1;
+                        pipe_config->dpll.p2 = 10;
+                        pipe_config->dpll.n = 1;
+                        pipe_config->dpll.m1 = 14;
+                        pipe_config->dpll.m2 = 2;
+                }
+                pipe_config->clock_set = true;
+        } else if (IS_HASWELL(dev)) {
+                /* Haswell has special-purpose DP DDI clocks. */
+        } else if (HAS_PCH_SPLIT(dev)) {
+                if (link_bw == DP_LINK_BW_1_62) {
+                        pipe_config->dpll.n = 1;
+                        pipe_config->dpll.p1 = 2;
+                        pipe_config->dpll.p2 = 10;
+                        pipe_config->dpll.m1 = 12;
+                        pipe_config->dpll.m2 = 9;
+                } else {
+                        pipe_config->dpll.n = 2;
+                        pipe_config->dpll.p1 = 1;
+                        pipe_config->dpll.p2 = 10;
+                        pipe_config->dpll.m1 = 14;
+                        pipe_config->dpll.m2 = 8;
+                }
+                pipe_config->clock_set = true;
+        } else if (IS_VALLEYVIEW(dev)) {
+                /* FIXME: Need to figure out optimized DP clocks for vlv. */
+        }
+}
+
 bool
 intel_dp_compute_config(struct intel_encoder *encoder,
                         struct intel_crtc_config *pipe_config)
@@ -667,17 +668,18 @@ intel_dp_compute_config(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_display_mode *adjusted_mode = &pipe_config->adjusted_mode;
-        struct drm_display_mode *mode = &pipe_config->requested_mode;
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        enum port port = dp_to_dig_port(intel_dp)->port;
+        struct intel_crtc *intel_crtc = encoder->new_crtc;
         struct intel_connector *intel_connector = intel_dp->attached_connector;
         int lane_count, clock;
         int max_lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
         int max_clock = intel_dp_max_link_bw(intel_dp) == DP_LINK_BW_2_7 ? 1 : 0;
         int bpp, mode_rate;
         static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 };
-        int target_clock, link_avail, link_clock;
+        int link_avail, link_clock;
 
-        if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && !is_cpu_edp(intel_dp))
+        if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A)
                 pipe_config->has_pch_encoder = true;
 
         pipe_config->has_dp_encoder = true;
@@ -685,12 +687,13 @@ intel_dp_compute_config(struct intel_encoder *encoder,
         if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
                 intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
                                        adjusted_mode);
-                intel_pch_panel_fitting(dev,
-                                        intel_connector->panel.fitting_mode,
-                                        mode, adjusted_mode);
+                if (!HAS_PCH_SPLIT(dev))
+                        intel_gmch_panel_fitting(intel_crtc, pipe_config,
+                                                 intel_connector->panel.fitting_mode);
+                else
+                        intel_pch_panel_fitting(intel_crtc, pipe_config,
+                                                intel_connector->panel.fitting_mode);
         }
-        /* We need to take the panel's fixed mode into account. */
-        target_clock = adjusted_mode->clock;
 
         if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK)
                 return false;
@@ -701,12 +704,12 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
         /* Walk through all bpp values. Luckily they're all nicely spaced with 2
          * bpc in between. */
-        bpp = min_t(int, 8*3, pipe_config->pipe_bpp);
-        if (is_edp(intel_dp) && dev_priv->edp.bpp)
-                bpp = min_t(int, bpp, dev_priv->edp.bpp);
+        bpp = pipe_config->pipe_bpp;
+        if (is_edp(intel_dp) && dev_priv->vbt.edp_bpp)
+                bpp = min_t(int, bpp, dev_priv->vbt.edp_bpp);
 
         for (; bpp >= 6*3; bpp -= 2*3) {
-                mode_rate = intel_dp_link_required(target_clock, bpp);
+                mode_rate = intel_dp_link_required(adjusted_mode->clock, bpp);
 
                 for (clock = 0; clock <= max_clock; clock++) {
                         for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) {
@@ -741,20 +744,21 @@ found:
 
         intel_dp->link_bw = bws[clock];
         intel_dp->lane_count = lane_count;
-        adjusted_mode->clock = drm_dp_bw_code_to_link_rate(intel_dp->link_bw);
         pipe_config->pipe_bpp = bpp;
-        pipe_config->pixel_target_clock = target_clock;
+        pipe_config->port_clock = drm_dp_bw_code_to_link_rate(intel_dp->link_bw);
 
         DRM_DEBUG_KMS("DP link bw %02x lane count %d clock %d bpp %d\n",
                       intel_dp->link_bw, intel_dp->lane_count,
-                      adjusted_mode->clock, bpp);
+                      pipe_config->port_clock, bpp);
         DRM_DEBUG_KMS("DP link bw required %i available %i\n",
                       mode_rate, link_avail);
 
         intel_link_compute_m_n(bpp, lane_count,
-                               target_clock, adjusted_mode->clock,
+                               adjusted_mode->clock, pipe_config->port_clock,
                                &pipe_config->dp_m_n);
 
+        intel_dp_set_clock(encoder, pipe_config, intel_dp->link_bw);
+
         return true;
 }
 
@@ -773,24 +777,28 @@ void intel_dp_init_link_config(struct intel_dp *intel_dp)
         }
 }
 
-static void ironlake_set_pll_edp(struct drm_crtc *crtc, int clock)
+static void ironlake_set_pll_cpu_edp(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = crtc->dev;
+        struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+        struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc);
+        struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 dpa_ctl;
 
-        DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", clock);
+        DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", crtc->config.port_clock);
         dpa_ctl = I915_READ(DP_A);
         dpa_ctl &= ~DP_PLL_FREQ_MASK;
 
-        if (clock < 200000) {
+        if (crtc->config.port_clock == 162000) {
                 /* For a long time we've carried around a ILK-DevA w/a for the
                  * 160MHz clock. If we're really unlucky, it's still required.
                  */
                 DRM_DEBUG_KMS("160MHz cpu eDP clock, might need ilk devA w/a\n");
                 dpa_ctl |= DP_PLL_FREQ_160MHZ;
+                intel_dp->DP |= DP_PLL_FREQ_160MHZ;
         } else {
                 dpa_ctl |= DP_PLL_FREQ_270MHZ;
+                intel_dp->DP |= DP_PLL_FREQ_270MHZ;
         }
 
         I915_WRITE(DP_A, dpa_ctl);
@@ -806,8 +814,8 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-        struct drm_crtc *crtc = encoder->crtc;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        enum port port = dp_to_dig_port(intel_dp)->port;
+        struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
 
         /*
          * There are four kinds of DP registers:
@@ -833,21 +841,11 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 
         /* Handle DP bits in common between all three register formats */
         intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
+        intel_dp->DP |= DP_PORT_WIDTH(intel_dp->lane_count);
 
-        switch (intel_dp->lane_count) {
-        case 1:
-                intel_dp->DP |= DP_PORT_WIDTH_1;
-                break;
-        case 2:
-                intel_dp->DP |= DP_PORT_WIDTH_2;
-                break;
-        case 4:
-                intel_dp->DP |= DP_PORT_WIDTH_4;
-                break;
-        }
         if (intel_dp->has_audio) {
                 DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n",
-                                 pipe_name(intel_crtc->pipe));
+                                 pipe_name(crtc->pipe));
                 intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE;
                 intel_write_eld(encoder, adjusted_mode);
         }
@@ -856,7 +854,7 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 
         /* Split out the IBX/CPU vs CPT settings */
 
-        if (is_cpu_edp(intel_dp) && IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) {
+        if (port == PORT_A && IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) {
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                         intel_dp->DP |= DP_SYNC_HS_HIGH;
                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
@@ -866,14 +864,8 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 if (intel_dp->link_configuration[1] & DP_LANE_COUNT_ENHANCED_FRAME_EN)
                         intel_dp->DP |= DP_ENHANCED_FRAMING;
 
-                intel_dp->DP |= intel_crtc->pipe << 29;
-
-                /* don't miss out required setting for eDP */
-                if (adjusted_mode->clock < 200000)
-                        intel_dp->DP |= DP_PLL_FREQ_160MHZ;
-                else
-                        intel_dp->DP |= DP_PLL_FREQ_270MHZ;
-        } else if (!HAS_PCH_CPT(dev) || is_cpu_edp(intel_dp)) {
+                intel_dp->DP |= crtc->pipe << 29;
+        } else if (!HAS_PCH_CPT(dev) || port == PORT_A) {
                 if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev))
                         intel_dp->DP |= intel_dp->color_range;
 
@@ -886,22 +878,14 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
                 if (intel_dp->link_configuration[1] & DP_LANE_COUNT_ENHANCED_FRAME_EN)
                         intel_dp->DP |= DP_ENHANCED_FRAMING;
 
-                if (intel_crtc->pipe == 1)
+                if (crtc->pipe == 1)
                         intel_dp->DP |= DP_PIPEB_SELECT;
-
-                if (is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev)) {
-                        /* don't miss out required setting for eDP */
-                        if (adjusted_mode->clock < 200000)
-                                intel_dp->DP |= DP_PLL_FREQ_160MHZ;
-                        else
-                                intel_dp->DP |= DP_PLL_FREQ_270MHZ;
-                }
         } else {
                 intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT;
         }
 
-        if (is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev))
-                ironlake_set_pll_edp(crtc, adjusted_mode->clock);
+        if (port == PORT_A && !IS_VALLEYVIEW(dev))
+                ironlake_set_pll_cpu_edp(intel_dp);
 }
 
 #define IDLE_ON_MASK            (PP_ON | 0        | PP_SEQUENCE_MASK | 0                     | PP_SEQUENCE_STATE_MASK)
@@ -1290,6 +1274,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
                                   enum pipe *pipe)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        enum port port = dp_to_dig_port(intel_dp)->port;
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 tmp = I915_READ(intel_dp->output_reg);
@@ -1297,9 +1282,9 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
         if (!(tmp & DP_PORT_EN))
                 return false;
 
-        if (is_cpu_edp(intel_dp) && IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) {
+        if (port == PORT_A && IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) {
                 *pipe = PORT_TO_PIPE_CPT(tmp);
-        } else if (!HAS_PCH_CPT(dev) || is_cpu_edp(intel_dp)) {
+        } else if (!HAS_PCH_CPT(dev) || port == PORT_A) {
                 *pipe = PORT_TO_PIPE(tmp);
         } else {
                 u32 trans_sel;
@@ -1335,9 +1320,48 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
         return true;
 }
 
+static void intel_dp_get_config(struct intel_encoder *encoder,
+                                struct intel_crtc_config *pipe_config)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        u32 tmp, flags = 0;
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum port port = dp_to_dig_port(intel_dp)->port;
+        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+
+        if ((port == PORT_A) || !HAS_PCH_CPT(dev)) {
+                tmp = I915_READ(intel_dp->output_reg);
+                if (tmp & DP_SYNC_HS_HIGH)
+                        flags |= DRM_MODE_FLAG_PHSYNC;
+                else
+                        flags |= DRM_MODE_FLAG_NHSYNC;
+
+                if (tmp & DP_SYNC_VS_HIGH)
+                        flags |= DRM_MODE_FLAG_PVSYNC;
+                else
+                        flags |= DRM_MODE_FLAG_NVSYNC;
+        } else {
+                tmp = I915_READ(TRANS_DP_CTL(crtc->pipe));
+                if (tmp & TRANS_DP_HSYNC_ACTIVE_HIGH)
+                        flags |= DRM_MODE_FLAG_PHSYNC;
+                else
+                        flags |= DRM_MODE_FLAG_NHSYNC;
+
+                if (tmp & TRANS_DP_VSYNC_ACTIVE_HIGH)
+                        flags |= DRM_MODE_FLAG_PVSYNC;
+                else
+                        flags |= DRM_MODE_FLAG_NVSYNC;
+        }
+
+        pipe_config->adjusted_mode.flags |= flags;
+}
+
 static void intel_disable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        enum port port = dp_to_dig_port(intel_dp)->port;
+        struct drm_device *dev = encoder->base.dev;
 
         /* Make sure the panel is off before trying to change the mode. But also
          * ensure that we have vdd while we switch off the panel. */
@@ -1347,16 +1371,17 @@ static void intel_disable_dp(struct intel_encoder *encoder)
         ironlake_edp_panel_off(intel_dp);
 
         /* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */
-        if (!is_cpu_edp(intel_dp))
+        if (!(port == PORT_A || IS_VALLEYVIEW(dev)))
                 intel_dp_link_down(intel_dp);
 }
 
 static void intel_post_disable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        enum port port = dp_to_dig_port(intel_dp)->port;
         struct drm_device *dev = encoder->base.dev;
 
-        if (is_cpu_edp(intel_dp)) {
+        if (port == PORT_A || IS_VALLEYVIEW(dev)) {
                 intel_dp_link_down(intel_dp);
                 if (!IS_VALLEYVIEW(dev))
                         ironlake_edp_pll_off(intel_dp);
@@ -1381,15 +1406,73 @@ static void intel_enable_dp(struct intel_encoder *encoder)
         intel_dp_complete_link_train(intel_dp);
         intel_dp_stop_link_train(intel_dp);
         ironlake_edp_backlight_on(intel_dp);
+
+        if (IS_VALLEYVIEW(dev)) {
+                struct intel_digital_port *dport =
+                        enc_to_dig_port(&encoder->base);
+                int channel = vlv_dport_to_channel(dport);
+
+                vlv_wait_port_ready(dev_priv, channel);
+        }
 }
 
 static void intel_pre_enable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
         struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev))
+        if (dport->port == PORT_A && !IS_VALLEYVIEW(dev))
                 ironlake_edp_pll_on(intel_dp);
+
+        if (IS_VALLEYVIEW(dev)) {
+                struct intel_crtc *intel_crtc =
+                        to_intel_crtc(encoder->base.crtc);
+                int port = vlv_dport_to_channel(dport);
+                int pipe = intel_crtc->pipe;
+                u32 val;
+
+                val = vlv_dpio_read(dev_priv, DPIO_DATA_LANE_A(port));
+                val = 0;
+                if (pipe)
+                        val |= (1<<21);
+                else
+                        val &= ~(1<<21);
+                val |= 0x001000c4;
+                vlv_dpio_write(dev_priv, DPIO_DATA_CHANNEL(port), val);
+
+                vlv_dpio_write(dev_priv, DPIO_PCS_CLOCKBUF0(port),
+                                 0x00760018);
+                vlv_dpio_write(dev_priv, DPIO_PCS_CLOCKBUF8(port),
+                                 0x00400888);
+        }
+}
+
+static void intel_dp_pre_pll_enable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int port = vlv_dport_to_channel(dport);
+
+        if (!IS_VALLEYVIEW(dev))
+                return;
+
+        /* Program Tx lane resets to default */
+        vlv_dpio_write(dev_priv, DPIO_PCS_TX(port),
+                         DPIO_PCS_TX_LANE2_RESET |
+                         DPIO_PCS_TX_LANE1_RESET);
+        vlv_dpio_write(dev_priv, DPIO_PCS_CLK(port),
+                         DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
+                         DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
+                         (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
+                                 DPIO_PCS_CLK_SOFT_RESET);
+
+        /* Fix up inter-pair skew failure */
+        vlv_dpio_write(dev_priv, DPIO_PCS_STAGGER1(port), 0x00750f00);
+        vlv_dpio_write(dev_priv, DPIO_TX_CTL(port), 0x00001500);
+        vlv_dpio_write(dev_priv, DPIO_TX_LANE(port), 0x40400000);
 }
 
 /*
@@ -1451,10 +1534,13 @@ static uint8_t
 intel_dp_voltage_max(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        enum port port = dp_to_dig_port(intel_dp)->port;
 
-        if (IS_GEN7(dev) && is_cpu_edp(intel_dp))
+        if (IS_VALLEYVIEW(dev))
+                return DP_TRAIN_VOLTAGE_SWING_1200;
+        else if (IS_GEN7(dev) && port == PORT_A)
                 return DP_TRAIN_VOLTAGE_SWING_800;
-        else if (HAS_PCH_CPT(dev) && !is_cpu_edp(intel_dp))
+        else if (HAS_PCH_CPT(dev) && port != PORT_A)
                 return DP_TRAIN_VOLTAGE_SWING_1200;
         else
                 return DP_TRAIN_VOLTAGE_SWING_800;
@@ -1464,6 +1550,7 @@ static uint8_t
 intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        enum port port = dp_to_dig_port(intel_dp)->port;
 
         if (HAS_DDI(dev)) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
@@ -1477,7 +1564,19 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
                 default:
                         return DP_TRAIN_PRE_EMPHASIS_0;
                 }
-        } else if (IS_GEN7(dev) && is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev)) {
+                switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
+                case DP_TRAIN_VOLTAGE_SWING_400:
+                        return DP_TRAIN_PRE_EMPHASIS_9_5;
+                case DP_TRAIN_VOLTAGE_SWING_600:
+                        return DP_TRAIN_PRE_EMPHASIS_6;
+                case DP_TRAIN_VOLTAGE_SWING_800:
+                        return DP_TRAIN_PRE_EMPHASIS_3_5;
+                case DP_TRAIN_VOLTAGE_SWING_1200:
+                default:
+                        return DP_TRAIN_PRE_EMPHASIS_0;
+                }
+        } else if (IS_GEN7(dev) && port == PORT_A) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
                 case DP_TRAIN_VOLTAGE_SWING_400:
                         return DP_TRAIN_PRE_EMPHASIS_6;
@@ -1502,6 +1601,101 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
         }
 }
 
+static uint32_t intel_vlv_signal_levels(struct intel_dp *intel_dp)
+{
+        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
+        unsigned long demph_reg_value, preemph_reg_value,
+                uniqtranscale_reg_value;
+        uint8_t train_set = intel_dp->train_set[0];
+        int port = vlv_dport_to_channel(dport);
+
+        switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
+        case DP_TRAIN_PRE_EMPHASIS_0:
+                preemph_reg_value = 0x0004000;
+                switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
+                case DP_TRAIN_VOLTAGE_SWING_400:
+                        demph_reg_value = 0x2B405555;
+                        uniqtranscale_reg_value = 0x552AB83A;
+                        break;
+                case DP_TRAIN_VOLTAGE_SWING_600:
+                        demph_reg_value = 0x2B404040;
+                        uniqtranscale_reg_value = 0x5548B83A;
+                        break;
+                case DP_TRAIN_VOLTAGE_SWING_800:
+                        demph_reg_value = 0x2B245555;
+                        uniqtranscale_reg_value = 0x5560B83A;
+                        break;
+                case DP_TRAIN_VOLTAGE_SWING_1200:
+                        demph_reg_value = 0x2B405555;
+                        uniqtranscale_reg_value = 0x5598DA3A;
+                        break;
+                default:
+                        return 0;
+                }
+                break;
+        case DP_TRAIN_PRE_EMPHASIS_3_5:
+                preemph_reg_value = 0x0002000;
+                switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
+                case DP_TRAIN_VOLTAGE_SWING_400:
+                        demph_reg_value = 0x2B404040;
+                        uniqtranscale_reg_value = 0x5552B83A;
+                        break;
+                case DP_TRAIN_VOLTAGE_SWING_600:
+                        demph_reg_value = 0x2B404848;
+                        uniqtranscale_reg_value = 0x5580B83A;
+                        break;
+                case DP_TRAIN_VOLTAGE_SWING_800:
+                        demph_reg_value = 0x2B404040;
+                        uniqtranscale_reg_value = 0x55ADDA3A;
+                        break;
+                default:
+                        return 0;
+                }
+                break;
+        case DP_TRAIN_PRE_EMPHASIS_6:
+                preemph_reg_value = 0x0000000;
+                switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
+                case DP_TRAIN_VOLTAGE_SWING_400:
+                        demph_reg_value = 0x2B305555;
+                        uniqtranscale_reg_value = 0x5570B83A;
+                        break;
+                case DP_TRAIN_VOLTAGE_SWING_600:
+                        demph_reg_value = 0x2B2B4040;
+                        uniqtranscale_reg_value = 0x55ADDA3A;
+                        break;
+                default:
+                        return 0;
+                }
+                break;
+        case DP_TRAIN_PRE_EMPHASIS_9_5:
+                preemph_reg_value = 0x0006000;
+                switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
+                case DP_TRAIN_VOLTAGE_SWING_400:
+                        demph_reg_value = 0x1B405555;
+                        uniqtranscale_reg_value = 0x55ADDA3A;
+                        break;
+                default:
+                        return 0;
+                }
+                break;
+        default:
+                return 0;
+        }
+
+        vlv_dpio_write(dev_priv, DPIO_TX_OCALINIT(port), 0x00000000);
+        vlv_dpio_write(dev_priv, DPIO_TX_SWING_CTL4(port), demph_reg_value);
+        vlv_dpio_write(dev_priv, DPIO_TX_SWING_CTL2(port),
+                         uniqtranscale_reg_value);
+        vlv_dpio_write(dev_priv, DPIO_TX_SWING_CTL3(port), 0x0C782040);
+        vlv_dpio_write(dev_priv, DPIO_PCS_STAGGER0(port), 0x00030000);
+        vlv_dpio_write(dev_priv, DPIO_PCS_CTL_OVER1(port), preemph_reg_value);
+        vlv_dpio_write(dev_priv, DPIO_TX_OCALINIT(port), 0x80000000);
+
+        return 0;
+}
+
 static void
 intel_get_adjust_train(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE])
 {
@@ -1669,6 +1863,7 @@ static void
 intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        enum port port = intel_dig_port->port;
         struct drm_device *dev = intel_dig_port->base.base.dev;
         uint32_t signal_levels, mask;
         uint8_t train_set = intel_dp->train_set[0];
@@ -1676,10 +1871,13 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP)
         if (HAS_DDI(dev)) {
                 signal_levels = intel_hsw_signal_levels(train_set);
                 mask = DDI_BUF_EMP_MASK;
-        } else if (IS_GEN7(dev) && is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev)) {
+                signal_levels = intel_vlv_signal_levels(intel_dp);
+                mask = 0;
+        } else if (IS_GEN7(dev) && port == PORT_A) {
                 signal_levels = intel_gen7_edp_signal_levels(train_set);
                 mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB;
-        } else if (IS_GEN6(dev) && is_cpu_edp(intel_dp)) {
+        } else if (IS_GEN6(dev) && port == PORT_A) {
                 signal_levels = intel_gen6_edp_signal_levels(train_set);
                 mask = EDP_LINK_TRAIN_VOL_EMP_MASK_SNB;
         } else {
@@ -1729,8 +1927,7 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
                 }
                 I915_WRITE(DP_TP_CTL(port), temp);
 
-        } else if (HAS_PCH_CPT(dev) &&
-                   (IS_GEN7(dev) || !is_cpu_edp(intel_dp))) {
+        } else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) {
                 dp_reg_value &= ~DP_LINK_TRAIN_MASK_CPT;
 
                 switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
@@ -1981,6 +2178,7 @@ static void
 intel_dp_link_down(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        enum port port = intel_dig_port->port;
         struct drm_device *dev = intel_dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc =
@@ -2010,7 +2208,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 
         DRM_DEBUG_KMS("\n");
 
-        if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp))) {
+        if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) {
                 DP &= ~DP_LINK_TRAIN_MASK_CPT;
                 I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT);
         } else {
@@ -2301,11 +2499,10 @@ intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
                         return NULL;
 
                 size = (intel_connector->edid->extensions + 1) * EDID_LENGTH;
-                edid = kmalloc(size, GFP_KERNEL);
+                edid = kmemdup(intel_connector->edid, size, GFP_KERNEL);
                 if (!edid)
                         return NULL;
 
-                memcpy(edid, intel_connector->edid, size);
                 return edid;
         }
 
@@ -2499,15 +2696,16 @@ done:
 }
 
 static void
-intel_dp_destroy(struct drm_connector *connector)
+intel_dp_connector_destroy(struct drm_connector *connector)
 {
-        struct intel_dp *intel_dp = intel_attached_dp(connector);
         struct intel_connector *intel_connector = to_intel_connector(connector);
 
         if (!IS_ERR_OR_NULL(intel_connector->edid))
                 kfree(intel_connector->edid);
 
-        if (is_edp(intel_dp))
+        /* Can't call is_edp() since the encoder may have been destroyed
+         * already. */
+        if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
                 intel_panel_fini(&intel_connector->panel);
 
         drm_sysfs_connector_remove(connector);
@@ -2541,7 +2739,7 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = {
         .detect = intel_dp_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_dp_set_property,
-        .destroy = intel_dp_destroy,
+        .destroy = intel_dp_connector_destroy,
 };
 
 static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = {
@@ -2588,11 +2786,11 @@ bool intel_dpd_is_edp(struct drm_device *dev)
         struct child_device_config *p_child;
         int i;
 
-        if (!dev_priv->child_dev_num)
+        if (!dev_priv->vbt.child_dev_num)
                 return false;
 
-        for (i = 0; i < dev_priv->child_dev_num; i++) {
-                p_child = dev_priv->child_dev + i;
+        for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
+                p_child = dev_priv->vbt.child_dev + i;
 
                 if (p_child->dvo_port == PORT_IDPD &&
                     p_child->device_type == DEVICE_TYPE_eDP)
@@ -2670,7 +2868,7 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
         DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
                       cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12);
 
-        vbt = dev_priv->edp.pps;
+        vbt = dev_priv->vbt.edp_pps;
 
         /* Upper limits from eDP 1.3 spec. Note that we use the clunky units of
          * our hw here, which are all in 100usec. */
@@ -2738,9 +2936,6 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                 pp_div_reg = PIPEA_PP_DIVISOR;
         }
 
-        if (IS_VALLEYVIEW(dev))
-                port_sel = I915_READ(pp_on_reg) & 0xc0000000;
-
         /* And finally store the new values in the power sequencer. */
         pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) |
                 (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT);
@@ -2754,8 +2949,10 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
         /* Haswell doesn't have any port selection bits for the panel
          * power sequencer any more. */
-        if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
-                if (is_cpu_edp(intel_dp))
+        if (IS_VALLEYVIEW(dev)) {
+                port_sel = I915_READ(pp_on_reg) & 0xc0000000;
+        } else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
+                if (dp_to_dig_port(intel_dp)->port == PORT_A)
                         port_sel = PANEL_POWER_PORT_DP_A;
                 else
                         port_sel = PANEL_POWER_PORT_DP_D;
@@ -2773,7 +2970,85 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                       I915_READ(pp_div_reg));
 }
 
-void
+static bool intel_edp_init_connector(struct intel_dp *intel_dp,
+                                     struct intel_connector *intel_connector)
+{
+        struct drm_connector *connector = &intel_connector->base;
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_display_mode *fixed_mode = NULL;
+        struct edp_power_seq power_seq = { 0 };
+        bool has_dpcd;
+        struct drm_display_mode *scan;
+        struct edid *edid;
+
+        if (!is_edp(intel_dp))
+                return true;
+
+        intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
+
+        /* Cache DPCD and EDID for edp. */
+        ironlake_edp_panel_vdd_on(intel_dp);
+        has_dpcd = intel_dp_get_dpcd(intel_dp);
+        ironlake_edp_panel_vdd_off(intel_dp, false);
+
+        if (has_dpcd) {
+                if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
+                        dev_priv->no_aux_handshake =
+                                intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
+                                DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
+        } else {
+                /* if this fails, presume the device is a ghost */
+                DRM_INFO("failed to retrieve link info, disabling eDP\n");
+                return false;
+        }
+
+        /* We now know it's not a ghost, init power sequence regs. */
+        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
+                                                      &power_seq);
+
+        ironlake_edp_panel_vdd_on(intel_dp);
+        edid = drm_get_edid(connector, &intel_dp->adapter);
+        if (edid) {
+                if (drm_add_edid_modes(connector, edid)) {
+                        drm_mode_connector_update_edid_property(connector,
+                                                                edid);
+                        drm_edid_to_eld(connector, edid);
+                } else {
+                        kfree(edid);
+                        edid = ERR_PTR(-EINVAL);
+                }
+        } else {
+                edid = ERR_PTR(-ENOENT);
+        }
+        intel_connector->edid = edid;
+
+        /* prefer fixed mode from EDID if available */
+        list_for_each_entry(scan, &connector->probed_modes, head) {
+                if ((scan->type & DRM_MODE_TYPE_PREFERRED)) {
+                        fixed_mode = drm_mode_duplicate(dev, scan);
+                        break;
+                }
+        }
+
+        /* fallback to VBT if available for eDP */
+        if (!fixed_mode && dev_priv->vbt.lfp_lvds_vbt_mode) {
+                fixed_mode = drm_mode_duplicate(dev,
+                                        dev_priv->vbt.lfp_lvds_vbt_mode);
+                if (fixed_mode)
+                        fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
+        }
+
+        ironlake_edp_panel_vdd_off(intel_dp, false);
+
+        intel_panel_init(&intel_connector->panel, fixed_mode);
+        intel_panel_setup_backlight(connector);
+
+        return true;
+}
+
+bool
 intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                         struct intel_connector *intel_connector)
 {
@@ -2782,38 +3057,47 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_display_mode *fixed_mode = NULL;
-        struct edp_power_seq power_seq = { 0 };
         enum port port = intel_dig_port->port;
         const char *name = NULL;
-        int type;
+        int type, error;
 
         /* Preserve the current hw state. */
         intel_dp->DP = I915_READ(intel_dp->output_reg);
         intel_dp->attached_connector = intel_connector;
 
-        if (HAS_PCH_SPLIT(dev) && port == PORT_D)
-                if (intel_dpd_is_edp(dev))
-                        intel_dp->is_pch_edp = true;
-
+        type = DRM_MODE_CONNECTOR_DisplayPort;
         /*
          * FIXME : We need to initialize built-in panels before external panels.
          * For X0, DP_C is fixed as eDP. Revisit this as part of VLV eDP cleanup
          */
-        if (IS_VALLEYVIEW(dev) && port == PORT_C) {
-                type = DRM_MODE_CONNECTOR_eDP;
-                intel_encoder->type = INTEL_OUTPUT_EDP;
-        } else if (port == PORT_A || is_pch_edp(intel_dp)) {
+        switch (port) {
+        case PORT_A:
                 type = DRM_MODE_CONNECTOR_eDP;
-                intel_encoder->type = INTEL_OUTPUT_EDP;
-        } else {
-                /* The intel_encoder->type value may be INTEL_OUTPUT_UNKNOWN for
-                 * DDI or INTEL_OUTPUT_DISPLAYPORT for the older gens, so don't
-                 * rewrite it.
-                 */
-                type = DRM_MODE_CONNECTOR_DisplayPort;
+                break;
+        case PORT_C:
+                if (IS_VALLEYVIEW(dev))
+                        type = DRM_MODE_CONNECTOR_eDP;
+                break;
+        case PORT_D:
+                if (HAS_PCH_SPLIT(dev) && intel_dpd_is_edp(dev))
+                        type = DRM_MODE_CONNECTOR_eDP;
+                break;
+        default:        /* silence GCC warning */
+                break;
         }
 
+        /*
+         * For eDP we always set the encoder type to INTEL_OUTPUT_EDP, but
+         * for DP the encoder type can be set by the caller to
+         * INTEL_OUTPUT_UNKNOWN for DDI, so don't rewrite it.
+         */
+        if (type == DRM_MODE_CONNECTOR_eDP)
+                intel_encoder->type = INTEL_OUTPUT_EDP;
+
+        DRM_DEBUG_KMS("Adding %s connector on port %c\n",
+                        type == DRM_MODE_CONNECTOR_eDP ? "eDP" : "DP",
+                        port_name(port));
+
         drm_connector_init(dev, connector, &intel_dp_connector_funcs, type);
         drm_connector_helper_add(connector, &intel_dp_connector_helper_funcs);
 
@@ -2873,74 +3157,21 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 BUG();
         }
 
-        if (is_edp(intel_dp))
-                intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
-
-        intel_dp_i2c_init(intel_dp, intel_connector, name);
-
-        /* Cache DPCD and EDID for edp. */
-        if (is_edp(intel_dp)) {
-                bool ret;
-                struct drm_display_mode *scan;
-                struct edid *edid;
-
-                ironlake_edp_panel_vdd_on(intel_dp);
-                ret = intel_dp_get_dpcd(intel_dp);
-                ironlake_edp_panel_vdd_off(intel_dp, false);
+        error = intel_dp_i2c_init(intel_dp, intel_connector, name);
+        WARN(error, "intel_dp_i2c_init failed with error %d for port %c\n",
+             error, port_name(port));
 
-                if (ret) {
-                        if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
-                                dev_priv->no_aux_handshake =
-                                        intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
-                                        DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
-                } else {
-                        /* if this fails, presume the device is a ghost */
-                        DRM_INFO("failed to retrieve link info, disabling eDP\n");
-                        intel_dp_encoder_destroy(&intel_encoder->base);
-                        intel_dp_destroy(connector);
-                        return;
-                }
-
-                /* We now know it's not a ghost, init power sequence regs. */
-                intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
-                                                              &power_seq);
-
-                ironlake_edp_panel_vdd_on(intel_dp);
-                edid = drm_get_edid(connector, &intel_dp->adapter);
-                if (edid) {
-                        if (drm_add_edid_modes(connector, edid)) {
-                                drm_mode_connector_update_edid_property(connector, edid);
-                                drm_edid_to_eld(connector, edid);
-                        } else {
-                                kfree(edid);
-                                edid = ERR_PTR(-EINVAL);
-                        }
-                } else {
-                        edid = ERR_PTR(-ENOENT);
+        if (!intel_edp_init_connector(intel_dp, intel_connector)) {
+                i2c_del_adapter(&intel_dp->adapter);
+                if (is_edp(intel_dp)) {
+                        cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
+                        mutex_lock(&dev->mode_config.mutex);
+                        ironlake_panel_vdd_off_sync(intel_dp);
+                        mutex_unlock(&dev->mode_config.mutex);
                 }
-                intel_connector->edid = edid;
-
-                /* prefer fixed mode from EDID if available */
-                list_for_each_entry(scan, &connector->probed_modes, head) {
-                        if ((scan->type & DRM_MODE_TYPE_PREFERRED)) {
-                                fixed_mode = drm_mode_duplicate(dev, scan);
-                                break;
-                        }
-                }
-
-                /* fallback to VBT if available for eDP */
-                if (!fixed_mode && dev_priv->lfp_lvds_vbt_mode) {
-                        fixed_mode = drm_mode_duplicate(dev, dev_priv->lfp_lvds_vbt_mode);
-                        if (fixed_mode)
-                                fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
-                }
-
-                ironlake_edp_panel_vdd_off(intel_dp, false);
-        }
-
-        if (is_edp(intel_dp)) {
-                intel_panel_init(&intel_connector->panel, fixed_mode);
-                intel_panel_setup_backlight(connector);
+                drm_sysfs_connector_remove(connector);
+                drm_connector_cleanup(connector);
+                return false;
         }
 
         intel_dp_add_properties(intel_dp, connector);
@@ -2953,6 +3184,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 u32 temp = I915_READ(PEG_BAND_GAP_DATA);
                 I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
         }
+
+        return true;
 }
 
 void
@@ -2986,6 +3219,9 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
         intel_encoder->disable = intel_disable_dp;
         intel_encoder->post_disable = intel_post_disable_dp;
         intel_encoder->get_hw_state = intel_dp_get_hw_state;
+        intel_encoder->get_config = intel_dp_get_config;
+        if (IS_VALLEYVIEW(dev))
+                intel_encoder->pre_pll_enable = intel_dp_pre_pll_enable;
 
         intel_dig_port->port = port;
         intel_dig_port->dp.output_reg = output_reg;
@@ -2995,5 +3231,9 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
         intel_encoder->cloneable = false;
         intel_encoder->hot_plug = intel_dp_hot_plug;
 
-        intel_dp_init_connector(intel_dig_port, intel_connector);
+        if (!intel_dp_init_connector(intel_dig_port, intel_connector)) {
+                drm_encoder_cleanup(encoder);
+                kfree(intel_dig_port);
+                kfree(intel_connector);
+        }
 }
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 624a9e6b8d71..c8c9b6f48230 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -120,7 +120,6 @@ struct intel_encoder {
         struct intel_crtc *new_crtc;
 
         int type;
-        bool needs_tv_clock;
         /*
          * Intel hw has only one MUX where encoders could be clone, hence a
          * simple flag is enough to compute the possible_clones mask.
@@ -140,6 +139,12 @@ struct intel_encoder {
          * the encoder is active. If the encoder is enabled it also set the pipe
          * it is connected to in the pipe parameter. */
         bool (*get_hw_state)(struct intel_encoder *, enum pipe *pipe);
+        /* Reconstructs the equivalent mode flags for the current hardware
+         * state. This must be called _after_ display->get_pipe_config has
+         * pre-filled the pipe config. Note that intel_encoder->base.crtc must
+         * be set correctly before calling this function. */
+        void (*get_config)(struct intel_encoder *,
+                           struct intel_crtc_config *pipe_config);
         int crtc_mask;
         enum hpd_pin hpd_pin;
 };
@@ -177,7 +182,30 @@ struct intel_connector {
         u8 polled;
 };
 
+typedef struct dpll {
+        /* given values */
+        int n;
+        int m1, m2;
+        int p1, p2;
+        /* derived values */
+        int     dot;
+        int     vco;
+        int     m;
+        int     p;
+} intel_clock_t;
+
 struct intel_crtc_config {
+        /**
+         * quirks - bitfield with hw state readout quirks
+         *
+         * For various reasons the hw state readout code might not be able to
+         * completely faithfully read out the current state. These cases are
+         * tracked with quirk flags so that fastboot and state checker can act
+         * accordingly.
+         */
+#define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */
+        unsigned long quirks;
+
         struct drm_display_mode requested_mode;
         struct drm_display_mode adjusted_mode;
         /* This flag must be set by the encoder's compute_config callback if it
@@ -201,29 +229,67 @@ struct intel_crtc_config {
         /* DP has a bunch of special case unfortunately, so mark the pipe
          * accordingly. */
         bool has_dp_encoder;
+
+        /*
+         * Enable dithering, used when the selected pipe bpp doesn't match the
+         * plane bpp.
+         */
         bool dither;
 
         /* Controls for the clock computation, to override various stages. */
         bool clock_set;
 
+        /* SDVO TV has a bunch of special case. To make multifunction encoders
+         * work correctly, we need to track this at runtime.*/
+        bool sdvo_tv_clock;
+
+        /*
+         * crtc bandwidth limit, don't increase pipe bpp or clock if not really
+         * required. This is set in the 2nd loop of calling encoder's
+         * ->compute_config if the first pick doesn't work out.
+         */
+        bool bw_constrained;
+
         /* Settings for the intel dpll used on pretty much everything but
          * haswell. */
-        struct dpll {
-                unsigned n;
-                unsigned m1, m2;
-                unsigned p1, p2;
-        } dpll;
+        struct dpll dpll;
+
+        /* Selected dpll when shared or DPLL_ID_PRIVATE. */
+        enum intel_dpll_id shared_dpll;
+
+        /* Actual register state of the dpll, for shared dpll cross-checking. */
+        struct intel_dpll_hw_state dpll_hw_state;
 
         int pipe_bpp;
         struct intel_link_m_n dp_m_n;
-        /**
-         * This is currently used by DP and HDMI encoders since those can have a
-         * target pixel clock != the port link clock (which is currently stored
-         * in adjusted_mode->clock).
+
+        /*
+         * Frequence the dpll for the port should run at. Differs from the
+         * adjusted dotclock e.g. for DP or 12bpc hdmi mode.
          */
-        int pixel_target_clock;
+        int port_clock;
+
         /* Used by SDVO (and if we ever fix it, HDMI). */
         unsigned pixel_multiplier;
+
+        /* Panel fitter controls for gen2-gen4 + VLV */
+        struct {
+                u32 control;
+                u32 pgm_ratios;
+                u32 lvds_border_bits;
+        } gmch_pfit;
+
+        /* Panel fitter placement and size for Ironlake+ */
+        struct {
+                u32 pos;
+                u32 size;
+        } pch_pfit;
+
+        /* FDI configuration, only valid if has_pch_encoder is set. */
+        int fdi_lanes;
+        struct intel_link_m_n fdi_m_n;
+
+        bool ips_enabled;
 };
 
 struct intel_crtc {
@@ -242,7 +308,6 @@ struct intel_crtc {
         bool lowfreq_avail;
         struct intel_overlay *overlay;
         struct intel_unpin_work *unpin_work;
-        int fdi_lanes;
 
         atomic_t unpin_work_count;
 
@@ -259,12 +324,14 @@ struct intel_crtc {
 
         struct intel_crtc_config config;
 
-        /* We can share PLLs across outputs if the timings match */
-        struct intel_pch_pll *pch_pll;
         uint32_t ddi_pll_sel;
 
         /* reset counter value when the last flip was submitted */
         unsigned int reset_counter;
+
+        /* Access to these should be protected by dev_priv->irq_lock. */
+        bool cpu_fifo_underrun_disabled;
+        bool pch_fifo_underrun_disabled;
 };
 
 struct intel_plane {
@@ -279,6 +346,18 @@ struct intel_plane {
         unsigned int crtc_w, crtc_h;
         uint32_t src_x, src_y;
         uint32_t src_w, src_h;
+
+        /* Since we need to change the watermarks before/after
+         * enabling/disabling the planes, we need to store the parameters here
+         * as the other pieces of the struct may not reflect the values we want
+         * for the watermark calculations. Currently only Haswell uses this.
+         */
+        struct {
+                bool enable;
+                uint8_t bytes_per_pixel;
+                uint32_t horiz_pixels;
+        } wm;
+
         void (*update_plane)(struct drm_plane *plane,
                              struct drm_framebuffer *fb,
                              struct drm_i915_gem_object *obj,
@@ -411,7 +490,6 @@ struct intel_dp {
         uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
         struct i2c_adapter adapter;
         struct i2c_algo_dp_aux_data algo;
-        bool is_pch_edp;
         uint8_t train_set[4];
         int panel_power_up_delay;
         int panel_power_down_delay;
@@ -431,6 +509,19 @@ struct intel_digital_port {
         struct intel_hdmi hdmi;
 };
 
+static inline int
+vlv_dport_to_channel(struct intel_digital_port *dport)
+{
+        switch (dport->port) {
+        case PORT_B:
+                return 0;
+        case PORT_C:
+                return 1;
+        default:
+                BUG();
+        }
+}
+
 static inline struct drm_crtc *
 intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
 {
@@ -474,6 +565,7 @@ int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
 extern void intel_attach_force_audio_property(struct drm_connector *connector);
 extern void intel_attach_broadcast_rgb_property(struct drm_connector *connector);
 
+extern bool intel_pipe_has_type(struct drm_crtc *crtc, int type);
 extern void intel_crt_init(struct drm_device *dev);
 extern void intel_hdmi_init(struct drm_device *dev,
                             int hdmi_reg, enum port port);
@@ -488,13 +580,14 @@ extern bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg,
 extern void intel_dvo_init(struct drm_device *dev);
 extern void intel_tv_init(struct drm_device *dev);
 extern void intel_mark_busy(struct drm_device *dev);
-extern void intel_mark_fb_busy(struct drm_i915_gem_object *obj);
+extern void intel_mark_fb_busy(struct drm_i915_gem_object *obj,
+                               struct intel_ring_buffer *ring);
 extern void intel_mark_idle(struct drm_device *dev);
-extern bool intel_lvds_init(struct drm_device *dev);
+extern void intel_lvds_init(struct drm_device *dev);
 extern bool intel_is_dual_link_lvds(struct drm_device *dev);
 extern void intel_dp_init(struct drm_device *dev, int output_reg,
                           enum port port);
-extern void intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
+extern bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                                     struct intel_connector *intel_connector);
 extern void intel_dp_init_link_config(struct intel_dp *intel_dp);
 extern void intel_dp_start_link_train(struct intel_dp *intel_dp);
@@ -512,7 +605,6 @@ extern void ironlake_edp_panel_on(struct intel_dp *intel_dp);
 extern void ironlake_edp_panel_off(struct intel_dp *intel_dp);
 extern void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp);
 extern void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
-extern bool intel_encoder_is_pch_edp(struct drm_encoder *encoder);
 extern int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane);
 extern void intel_flush_display_plane(struct drm_i915_private *dev_priv,
                                       enum plane plane);
@@ -524,12 +616,14 @@ extern void intel_panel_fini(struct intel_panel *panel);
 
 extern void intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
                                    struct drm_display_mode *adjusted_mode);
-extern void intel_pch_panel_fitting(struct drm_device *dev,
-                                    int fitting_mode,
-                                    const struct drm_display_mode *mode,
-                                    struct drm_display_mode *adjusted_mode);
-extern u32 intel_panel_get_max_backlight(struct drm_device *dev);
-extern void intel_panel_set_backlight(struct drm_device *dev, u32 level);
+extern void intel_pch_panel_fitting(struct intel_crtc *crtc,
+                                    struct intel_crtc_config *pipe_config,
+                                    int fitting_mode);
+extern void intel_gmch_panel_fitting(struct intel_crtc *crtc,
+                                     struct intel_crtc_config *pipe_config,
+                                     int fitting_mode);
+extern void intel_panel_set_backlight(struct drm_device *dev,
+                                      u32 level, u32 max);
 extern int intel_panel_setup_backlight(struct drm_connector *connector);
 extern void intel_panel_enable_backlight(struct drm_device *dev,
                                          enum pipe pipe);
@@ -553,11 +647,11 @@ extern void intel_crtc_load_lut(struct drm_crtc *crtc);
 extern void intel_crtc_update_dpms(struct drm_crtc *crtc);
 extern void intel_encoder_destroy(struct drm_encoder *encoder);
 extern void intel_encoder_dpms(struct intel_encoder *encoder, int mode);
-extern bool intel_encoder_check_is_cloned(struct intel_encoder *encoder);
 extern void intel_connector_dpms(struct drm_connector *, int mode);
 extern bool intel_connector_get_hw_state(struct intel_connector *connector);
 extern void intel_modeset_check_state(struct drm_device *dev);
 extern void intel_plane_restore(struct drm_plane *plane);
+extern void intel_plane_disable(struct drm_plane *plane);
 
 
 static inline struct intel_encoder *intel_attached_encoder(struct drm_connector *connector)
@@ -565,19 +659,17 @@ static inline struct intel_encoder *intel_attached_encoder(struct drm_connector
         return to_intel_connector(connector)->encoder;
 }
 
-static inline struct intel_dp *enc_to_intel_dp(struct drm_encoder *encoder)
-{
-        struct intel_digital_port *intel_dig_port =
-                container_of(encoder, struct intel_digital_port, base.base);
-        return &intel_dig_port->dp;
-}
-
 static inline struct intel_digital_port *
 enc_to_dig_port(struct drm_encoder *encoder)
 {
         return container_of(encoder, struct intel_digital_port, base.base);
 }
 
+static inline struct intel_dp *enc_to_intel_dp(struct drm_encoder *encoder)
+{
+        return &enc_to_dig_port(encoder)->dp;
+}
+
 static inline struct intel_digital_port *
 dp_to_dig_port(struct intel_dp *intel_dp)
 {
@@ -607,6 +699,7 @@ intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 extern void intel_wait_for_vblank(struct drm_device *dev, int pipe);
 extern void intel_wait_for_pipe_off(struct drm_device *dev, int pipe);
 extern int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp);
+extern void vlv_wait_port_ready(struct drm_i915_private *dev_priv, int port);
 
 struct intel_load_detect_pipe {
         struct drm_framebuffer *release_fb;
@@ -660,13 +753,9 @@ extern void assert_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
 #define assert_pipe_disabled(d, p) assert_pipe(d, p, false)
 
 extern void intel_init_clock_gating(struct drm_device *dev);
+extern void intel_suspend_hw(struct drm_device *dev);
 extern void intel_write_eld(struct drm_encoder *encoder,
                             struct drm_display_mode *mode);
-extern void intel_cpt_verify_modeset(struct drm_device *dev, int pipe);
-extern void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
-                                         struct intel_link_m_n *m_n);
-extern void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
-                                         struct intel_link_m_n *m_n);
 extern void intel_prepare_ddi(struct drm_device *dev);
 extern void hsw_fdi_link_train(struct drm_crtc *crtc);
 extern void intel_ddi_init(struct drm_device *dev, enum port port);
@@ -675,9 +764,7 @@ extern void intel_ddi_init(struct drm_device *dev, enum port port);
 extern void intel_update_watermarks(struct drm_device *dev);
 extern void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
                                            uint32_t sprite_width,
-                                           int pixel_size);
-extern void intel_update_linetime_watermarks(struct drm_device *dev, int pipe,
-                         struct drm_display_mode *mode);
+                                           int pixel_size, bool enable);
 
 extern unsigned long intel_gen4_compute_page_offset(int *x, int *y,
                                                     unsigned int tiling_mode,
@@ -689,8 +776,6 @@ extern int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 extern int intel_sprite_get_colorkey(struct drm_device *dev, void *data,
                                      struct drm_file *file_priv);
 
-extern u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg);
-
 /* Power-related functions, located in intel_pm.c */
 extern void intel_init_pm(struct drm_device *dev);
 /* FBC */
@@ -701,7 +786,12 @@ extern void intel_update_fbc(struct drm_device *dev);
 extern void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
 extern void intel_gpu_ips_teardown(void);
 
-extern bool intel_using_power_well(struct drm_device *dev);
+/* Power well */
+extern int i915_init_power_well(struct drm_device *dev);
+extern void i915_remove_power_well(struct drm_device *dev);
+
+extern bool intel_display_power_enabled(struct drm_device *dev,
+                                        enum intel_display_power_domain domain);
 extern void intel_init_power_well(struct drm_device *dev);
 extern void intel_set_power_well(struct drm_device *dev, bool enable);
 extern void intel_enable_gt_powersave(struct drm_device *dev);
@@ -719,7 +809,7 @@ extern void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
 extern void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc);
 extern void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc);
 extern void intel_ddi_setup_hw_pll_state(struct drm_device *dev);
-extern bool intel_ddi_pll_mode_set(struct drm_crtc *crtc, int clock);
+extern bool intel_ddi_pll_mode_set(struct drm_crtc *crtc);
 extern void intel_ddi_put_crtc_pll(struct drm_crtc *crtc);
 extern void intel_ddi_set_pipe_settings(struct drm_crtc *crtc);
 extern void intel_ddi_prepare_link_retrain(struct drm_encoder *encoder);
@@ -728,5 +818,11 @@ intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
 extern void intel_ddi_fdi_disable(struct drm_crtc *crtc);
 
 extern void intel_display_handle_reset(struct drm_device *dev);
+extern bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
+                                                  enum pipe pipe,
+                                                  bool enable);
+extern bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
+                                                 enum transcoder pch_transcoder,
+                                                 bool enable);
 
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index cc70b16d5d42..eb2020eb2b7e 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -54,6 +54,13 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
                 .dev_ops = &ch7xxx_ops,
         },
         {
+                .type = INTEL_DVO_CHIP_TMDS,
+                .name = "ch7xxx",
+                .dvo_reg = DVOC,
+                .slave_addr = 0x75, /* For some ch7010 */
+                .dev_ops = &ch7xxx_ops,
+        },
+        {
                 .type = INTEL_DVO_CHIP_LVDS,
                 .name = "ivch",
                 .dvo_reg = DVOA,
@@ -129,6 +136,26 @@ static bool intel_dvo_get_hw_state(struct intel_encoder *encoder,
         return true;
 }
 
+static void intel_dvo_get_config(struct intel_encoder *encoder,
+                                 struct intel_crtc_config *pipe_config)
+{
+        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct intel_dvo *intel_dvo = enc_to_intel_dvo(&encoder->base);
+        u32 tmp, flags = 0;
+
+        tmp = I915_READ(intel_dvo->dev.dvo_reg);
+        if (tmp & DVO_HSYNC_ACTIVE_HIGH)
+                flags |= DRM_MODE_FLAG_PHSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NHSYNC;
+        if (tmp & DVO_VSYNC_ACTIVE_HIGH)
+                flags |= DRM_MODE_FLAG_PVSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NVSYNC;
+
+        pipe_config->adjusted_mode.flags |= flags;
+}
+
 static void intel_disable_dvo(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
@@ -153,6 +180,7 @@ static void intel_enable_dvo(struct intel_encoder *encoder)
         intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
 }
 
+/* Special dpms function to support cloning between dvo/sdvo/crt. */
 static void intel_dvo_dpms(struct drm_connector *connector, int mode)
 {
         struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
@@ -174,6 +202,8 @@ static void intel_dvo_dpms(struct drm_connector *connector, int mode)
                 return;
         }
 
+        /* We call connector dpms manually below in case pipe dpms doesn't
+         * change due to cloning. */
         if (mode == DRM_MODE_DPMS_ON) {
                 intel_dvo->base.connectors_active = true;
 
@@ -440,6 +470,7 @@ void intel_dvo_init(struct drm_device *dev)
         intel_encoder->disable = intel_disable_dvo;
         intel_encoder->enable = intel_enable_dvo;
         intel_encoder->get_hw_state = intel_dvo_get_hw_state;
+        intel_encoder->get_config = intel_dvo_get_config;
         intel_connector->get_hw_state = intel_dvo_connector_get_hw_state;
 
         /* Now, try to find a controller */
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index 6b7c3ca2c035..dff669e2387f 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -60,8 +60,9 @@ static struct fb_ops intelfb_ops = {
 static int intelfb_create(struct drm_fb_helper *helper,
                           struct drm_fb_helper_surface_size *sizes)
 {
-        struct intel_fbdev *ifbdev = (struct intel_fbdev *)helper;
-        struct drm_device *dev = ifbdev->helper.dev;
+        struct intel_fbdev *ifbdev =
+                container_of(helper, struct intel_fbdev, helper);
+        struct drm_device *dev = helper->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct fb_info *info;
         struct drm_framebuffer *fb;
@@ -108,7 +109,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
                 goto out_unpin;
         }
 
-        info->par = ifbdev;
+        info->par = helper;
 
         ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj);
         if (ret)
@@ -217,7 +218,7 @@ static void intel_fbdev_destroy(struct drm_device *dev,
 int intel_fbdev_init(struct drm_device *dev)
 {
         struct intel_fbdev *ifbdev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL);
@@ -242,7 +243,7 @@ int intel_fbdev_init(struct drm_device *dev)
 
 void intel_fbdev_initial_config(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         /* Due to peculiar init order wrt to hpd handling this is separate. */
         drm_fb_helper_initial_config(&dev_priv->fbdev->helper, 32);
@@ -250,7 +251,7 @@ void intel_fbdev_initial_config(struct drm_device *dev)
 
 void intel_fbdev_fini(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         if (!dev_priv->fbdev)
                 return;
 
@@ -261,7 +262,7 @@ void intel_fbdev_fini(struct drm_device *dev)
 
 void intel_fbdev_set_suspend(struct drm_device *dev, int state)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_fbdev *ifbdev = dev_priv->fbdev;
         struct fb_info *info;
 
@@ -274,7 +275,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
          * been restored from swap. If the object is stolen however, it will be
          * full of whatever garbage was left in there.
          */
-        if (!state && ifbdev->ifb.obj->stolen)
+        if (state == FBINFO_STATE_RUNNING && ifbdev->ifb.obj->stolen)
                 memset_io(info->screen_base, 0, info->screen_size);
 
         fb_set_suspend(info, state);
@@ -284,16 +285,14 @@ MODULE_LICENSE("GPL and additional rights");
 
 void intel_fb_output_poll_changed(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper);
 }
 
 void intel_fb_restore_mode(struct drm_device *dev)
 {
         int ret;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_mode_config *config = &dev->mode_config;
-        struct drm_plane *plane;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (INTEL_INFO(dev)->num_pipes == 0)
                 return;
@@ -304,10 +303,5 @@ void intel_fb_restore_mode(struct drm_device *dev)
         if (ret)
                 DRM_DEBUG("failed to restore crtc mode\n");
 
-        /* Be sure to shut off any planes that may be active */
-        list_for_each_entry(plane, &config->plane_list, head)
-                if (plane->enabled)
-                        plane->funcs->disable_plane(plane);
-
         drm_modeset_unlock_all(dev);
 }
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index a9057930f2b2..98df2a0c85bd 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -602,7 +602,7 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
         u32 hdmi_val;
 
         hdmi_val = SDVO_ENCODING_HDMI;
-        if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev))
+        if (!HAS_PCH_SPLIT(dev))
                 hdmi_val |= intel_hdmi->color_range;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                 hdmi_val |= SDVO_VSYNC_ACTIVE_HIGH;
@@ -658,6 +658,28 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
         return true;
 }
 
+static void intel_hdmi_get_config(struct intel_encoder *encoder,
+                                  struct intel_crtc_config *pipe_config)
+{
+        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        u32 tmp, flags = 0;
+
+        tmp = I915_READ(intel_hdmi->hdmi_reg);
+
+        if (tmp & SDVO_HSYNC_ACTIVE_HIGH)
+                flags |= DRM_MODE_FLAG_PHSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NHSYNC;
+
+        if (tmp & SDVO_VSYNC_ACTIVE_HIGH)
+                flags |= DRM_MODE_FLAG_PVSYNC;
+        else
+                flags |= DRM_MODE_FLAG_NVSYNC;
+
+        pipe_config->adjusted_mode.flags |= flags;
+}
+
 static void intel_enable_hdmi(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
@@ -697,6 +719,14 @@ static void intel_enable_hdmi(struct intel_encoder *encoder)
                 I915_WRITE(intel_hdmi->hdmi_reg, temp);
                 POSTING_READ(intel_hdmi->hdmi_reg);
         }
+
+        if (IS_VALLEYVIEW(dev)) {
+                struct intel_digital_port *dport =
+                        enc_to_dig_port(&encoder->base);
+                int channel = vlv_dport_to_channel(dport);
+
+                vlv_wait_port_ready(dev_priv, channel);
+        }
 }
 
 static void intel_disable_hdmi(struct intel_encoder *encoder)
@@ -775,6 +805,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         struct drm_device *dev = encoder->base.dev;
         struct drm_display_mode *adjusted_mode = &pipe_config->adjusted_mode;
+        int clock_12bpc = pipe_config->requested_mode.clock * 3 / 2;
+        int desired_bpp;
 
         if (intel_hdmi->color_range_auto) {
                 /* See CEA-861-E - 5.1 Default Encoding Parameters */
@@ -794,14 +826,29 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
         /*
          * HDMI is either 12 or 8, so if the display lets 10bpc sneak
          * through, clamp it down. Note that g4x/vlv don't support 12bpc hdmi
-         * outputs.
+         * outputs. We also need to check that the higher clock still fits
+         * within limits.
          */
-        if (pipe_config->pipe_bpp > 8*3 && HAS_PCH_SPLIT(dev)) {
-                DRM_DEBUG_KMS("forcing bpc to 12 for HDMI\n");
-                pipe_config->pipe_bpp = 12*3;
+        if (pipe_config->pipe_bpp > 8*3 && clock_12bpc <= 225000
+            && HAS_PCH_SPLIT(dev)) {
+                DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n");
+                desired_bpp = 12*3;
+
+                /* Need to adjust the port link by 1.5x for 12bpc. */
+                pipe_config->port_clock = clock_12bpc;
         } else {
-                DRM_DEBUG_KMS("forcing bpc to 8 for HDMI\n");
-                pipe_config->pipe_bpp = 8*3;
+                DRM_DEBUG_KMS("picking bpc to 8 for HDMI output\n");
+                desired_bpp = 8*3;
+        }
+
+        if (!pipe_config->bw_constrained) {
+                DRM_DEBUG_KMS("forcing pipe bpc to %i for HDMI\n", desired_bpp);
+                pipe_config->pipe_bpp = desired_bpp;
+        }
+
+        if (adjusted_mode->clock > 225000) {
+                DRM_DEBUG_KMS("too high HDMI clock, rejecting mode\n");
+                return false;
         }
 
         return true;
@@ -955,6 +1002,97 @@ done:
         return 0;
 }
 
+static void intel_hdmi_pre_enable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc =
+                to_intel_crtc(encoder->base.crtc);
+        int port = vlv_dport_to_channel(dport);
+        int pipe = intel_crtc->pipe;
+        u32 val;
+
+        if (!IS_VALLEYVIEW(dev))
+                return;
+
+        /* Enable clock channels for this port */
+        val = vlv_dpio_read(dev_priv, DPIO_DATA_LANE_A(port));
+        val = 0;
+        if (pipe)
+                val |= (1<<21);
+        else
+                val &= ~(1<<21);
+        val |= 0x001000c4;
+        vlv_dpio_write(dev_priv, DPIO_DATA_CHANNEL(port), val);
+
+        /* HDMI 1.0V-2dB */
+        vlv_dpio_write(dev_priv, DPIO_TX_OCALINIT(port), 0);
+        vlv_dpio_write(dev_priv, DPIO_TX_SWING_CTL4(port),
+                         0x2b245f5f);
+        vlv_dpio_write(dev_priv, DPIO_TX_SWING_CTL2(port),
+                         0x5578b83a);
+        vlv_dpio_write(dev_priv, DPIO_TX_SWING_CTL3(port),
+                         0x0c782040);
+        vlv_dpio_write(dev_priv, DPIO_TX3_SWING_CTL4(port),
+                         0x2b247878);
+        vlv_dpio_write(dev_priv, DPIO_PCS_STAGGER0(port), 0x00030000);
+        vlv_dpio_write(dev_priv, DPIO_PCS_CTL_OVER1(port),
+                         0x00002000);
+        vlv_dpio_write(dev_priv, DPIO_TX_OCALINIT(port),
+                         DPIO_TX_OCALINIT_EN);
+
+        /* Program lane clock */
+        vlv_dpio_write(dev_priv, DPIO_PCS_CLOCKBUF0(port),
+                         0x00760018);
+        vlv_dpio_write(dev_priv, DPIO_PCS_CLOCKBUF8(port),
+                         0x00400888);
+}
+
+static void intel_hdmi_pre_pll_enable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int port = vlv_dport_to_channel(dport);
+
+        if (!IS_VALLEYVIEW(dev))
+                return;
+
+        /* Program Tx lane resets to default */
+        vlv_dpio_write(dev_priv, DPIO_PCS_TX(port),
+                         DPIO_PCS_TX_LANE2_RESET |
+                         DPIO_PCS_TX_LANE1_RESET);
+        vlv_dpio_write(dev_priv, DPIO_PCS_CLK(port),
+                         DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
+                         DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
+                         (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
+                         DPIO_PCS_CLK_SOFT_RESET);
+
+        /* Fix up inter-pair skew failure */
+        vlv_dpio_write(dev_priv, DPIO_PCS_STAGGER1(port), 0x00750f00);
+        vlv_dpio_write(dev_priv, DPIO_TX_CTL(port), 0x00001500);
+        vlv_dpio_write(dev_priv, DPIO_TX_LANE(port), 0x40400000);
+
+        vlv_dpio_write(dev_priv, DPIO_PCS_CTL_OVER1(port),
+                         0x00002000);
+        vlv_dpio_write(dev_priv, DPIO_TX_OCALINIT(port),
+                         DPIO_TX_OCALINIT_EN);
+}
+
+static void intel_hdmi_post_disable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        int port = vlv_dport_to_channel(dport);
+
+        /* Reset lanes to avoid HDMI flicker (VLV w/a) */
+        mutex_lock(&dev_priv->dpio_lock);
+        vlv_dpio_write(dev_priv, DPIO_PCS_TX(port), 0x00000000);
+        vlv_dpio_write(dev_priv, DPIO_PCS_CLK(port), 0x00e00060);
+        mutex_unlock(&dev_priv->dpio_lock);
+}
+
 static void intel_hdmi_destroy(struct drm_connector *connector)
 {
         drm_sysfs_connector_remove(connector);
@@ -1094,6 +1232,12 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port)
         intel_encoder->enable = intel_enable_hdmi;
         intel_encoder->disable = intel_disable_hdmi;
         intel_encoder->get_hw_state = intel_hdmi_get_hw_state;
+        intel_encoder->get_config = intel_hdmi_get_config;
+        if (IS_VALLEYVIEW(dev)) {
+                intel_encoder->pre_enable = intel_hdmi_pre_enable;
+                intel_encoder->pre_pll_enable = intel_hdmi_pre_pll_enable;
+                intel_encoder->post_disable = intel_hdmi_post_disable;
+        }
 
         intel_encoder->type = INTEL_OUTPUT_HDMI;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 29412cc89c7a..2abb2d3c727b 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -49,8 +49,6 @@ struct intel_lvds_connector {
 struct intel_lvds_encoder {
         struct intel_encoder base;
 
-        u32 pfit_control;
-        u32 pfit_pgm_ratios;
         bool is_dual_link;
         u32 reg;
 
@@ -88,6 +86,31 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
         return true;
 }
 
+static void intel_lvds_get_config(struct intel_encoder *encoder,
+                                  struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 lvds_reg, tmp, flags = 0;
+
+        if (HAS_PCH_SPLIT(dev))
+                lvds_reg = PCH_LVDS;
+        else
+                lvds_reg = LVDS;
+
+        tmp = I915_READ(lvds_reg);
+        if (tmp & LVDS_HSYNC_POLARITY)
+                flags |= DRM_MODE_FLAG_NHSYNC;
+        else
+                flags |= DRM_MODE_FLAG_PHSYNC;
+        if (tmp & LVDS_VSYNC_POLARITY)
+                flags |= DRM_MODE_FLAG_NVSYNC;
+        else
+                flags |= DRM_MODE_FLAG_PVSYNC;
+
+        pipe_config->adjusted_mode.flags |= flags;
+}
+
 /* The LVDS pin pair needs to be on before the DPLLs are enabled.
  * This is an exception to the general rule that mode_set doesn't turn
  * things on.
@@ -118,7 +141,8 @@ static void intel_pre_pll_enable_lvds(struct intel_encoder *encoder)
         }
 
         /* set the corresponsding LVDS_BORDER bit */
-        temp |= dev_priv->lvds_border_bits;
+        temp &= ~LVDS_BORDER_ENABLE;
+        temp |= intel_crtc->config.gmch_pfit.lvds_border_bits;
         /* Set the B0-B3 data pairs corresponding to whether we're going to
          * set the DPLLs for dual-channel mode or not.
          */
@@ -136,7 +160,10 @@ static void intel_pre_pll_enable_lvds(struct intel_encoder *encoder)
          * special lvds dither control bit on pch-split platforms, dithering is
          * only controlled through the PIPECONF reg. */
         if (INTEL_INFO(dev)->gen == 4) {
-                if (dev_priv->lvds_dither)
+                /* Bspec wording suggests that LVDS port dithering only exists
+                 * for 18bpp panels. */
+                if (intel_crtc->config.dither &&
+                    intel_crtc->config.pipe_bpp == 18)
                         temp |= LVDS_ENABLE_DITHER;
                 else
                         temp &= ~LVDS_ENABLE_DITHER;
@@ -150,29 +177,6 @@ static void intel_pre_pll_enable_lvds(struct intel_encoder *encoder)
         I915_WRITE(lvds_encoder->reg, temp);
 }
 
-static void intel_pre_enable_lvds(struct intel_encoder *encoder)
-{
-        struct drm_device *dev = encoder->base.dev;
-        struct intel_lvds_encoder *enc = to_lvds_encoder(&encoder->base);
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (HAS_PCH_SPLIT(dev) || !enc->pfit_control)
-                return;
-
-        /*
-         * Enable automatic panel scaling so that non-native modes
-         * fill the screen.  The panel fitter should only be
-         * adjusted whilst the pipe is disabled, according to
-         * register description and PRM.
-         */
-        DRM_DEBUG_KMS("applying panel-fitter: %x, %x\n",
-                      enc->pfit_control,
-                      enc->pfit_pgm_ratios);
-
-        I915_WRITE(PFIT_PGM_RATIOS, enc->pfit_pgm_ratios);
-        I915_WRITE(PFIT_CONTROL, enc->pfit_control);
-}
-
 /**
  * Sets the power state for the panel.
  */
@@ -241,62 +245,6 @@ static int intel_lvds_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static void
-centre_horizontally(struct drm_display_mode *mode,
-                    int width)
-{
-        u32 border, sync_pos, blank_width, sync_width;
-
-        /* keep the hsync and hblank widths constant */
-        sync_width = mode->crtc_hsync_end - mode->crtc_hsync_start;
-        blank_width = mode->crtc_hblank_end - mode->crtc_hblank_start;
-        sync_pos = (blank_width - sync_width + 1) / 2;
-
-        border = (mode->hdisplay - width + 1) / 2;
-        border += border & 1; /* make the border even */
-
-        mode->crtc_hdisplay = width;
-        mode->crtc_hblank_start = width + border;
-        mode->crtc_hblank_end = mode->crtc_hblank_start + blank_width;
-
-        mode->crtc_hsync_start = mode->crtc_hblank_start + sync_pos;
-        mode->crtc_hsync_end = mode->crtc_hsync_start + sync_width;
-}
-
-static void
-centre_vertically(struct drm_display_mode *mode,
-                  int height)
-{
-        u32 border, sync_pos, blank_width, sync_width;
-
-        /* keep the vsync and vblank widths constant */
-        sync_width = mode->crtc_vsync_end - mode->crtc_vsync_start;
-        blank_width = mode->crtc_vblank_end - mode->crtc_vblank_start;
-        sync_pos = (blank_width - sync_width + 1) / 2;
-
-        border = (mode->vdisplay - height + 1) / 2;
-
-        mode->crtc_vdisplay = height;
-        mode->crtc_vblank_start = height + border;
-        mode->crtc_vblank_end = mode->crtc_vblank_start + blank_width;
-
-        mode->crtc_vsync_start = mode->crtc_vblank_start + sync_pos;
-        mode->crtc_vsync_end = mode->crtc_vsync_start + sync_width;
-}
-
-static inline u32 panel_fitter_scaling(u32 source, u32 target)
-{
-        /*
-         * Floating point operation is not supported. So the FACTOR
-         * is defined, which can avoid the floating point computation
-         * when calculating the panel ratio.
-         */
-#define ACCURACY 12
-#define FACTOR (1 << ACCURACY)
-        u32 ratio = source * FACTOR / target;
-        return (FACTOR * ratio + FACTOR/2) / FACTOR;
-}
-
 static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
                                       struct intel_crtc_config *pipe_config)
 {
@@ -307,11 +255,8 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
         struct intel_connector *intel_connector =
                 &lvds_encoder->attached_connector->base;
         struct drm_display_mode *adjusted_mode = &pipe_config->adjusted_mode;
-        struct drm_display_mode *mode = &pipe_config->requested_mode;
         struct intel_crtc *intel_crtc = lvds_encoder->base.new_crtc;
-        u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
         unsigned int lvds_bpp;
-        int pipe;
 
         /* Should never happen!! */
         if (INTEL_INFO(dev)->gen < 4 && intel_crtc->pipe == 0) {
@@ -319,20 +264,18 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
                 return false;
         }
 
-        if (intel_encoder_check_is_cloned(&lvds_encoder->base))
-                return false;
-
         if ((I915_READ(lvds_encoder->reg) & LVDS_A3_POWER_MASK) ==
             LVDS_A3_POWER_UP)
                 lvds_bpp = 8*3;
         else
                 lvds_bpp = 6*3;
 
-        if (lvds_bpp != pipe_config->pipe_bpp) {
+        if (lvds_bpp != pipe_config->pipe_bpp && !pipe_config->bw_constrained) {
                 DRM_DEBUG_KMS("forcing display bpp (was %d) to LVDS (%d)\n",
                               pipe_config->pipe_bpp, lvds_bpp);
                 pipe_config->pipe_bpp = lvds_bpp;
         }
+
         /*
          * We have timings from the BIOS for the panel, put them in
          * to the adjusted mode.  The CRTC will be set up for this mode,
@@ -345,139 +288,17 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
         if (HAS_PCH_SPLIT(dev)) {
                 pipe_config->has_pch_encoder = true;
 
-                intel_pch_panel_fitting(dev,
-                                        intel_connector->panel.fitting_mode,
-                                        mode, adjusted_mode);
+                intel_pch_panel_fitting(intel_crtc, pipe_config,
+                                        intel_connector->panel.fitting_mode);
                 return true;
+        } else {
+                intel_gmch_panel_fitting(intel_crtc, pipe_config,
+                                         intel_connector->panel.fitting_mode);
         }
 
-        /* Native modes don't need fitting */
-        if (adjusted_mode->hdisplay == mode->hdisplay &&
-            adjusted_mode->vdisplay == mode->vdisplay)
-                goto out;
-
-        /* 965+ wants fuzzy fitting */
-        if (INTEL_INFO(dev)->gen >= 4)
-                pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) |
-                                 PFIT_FILTER_FUZZY);
-
-        /*
-         * Enable automatic panel scaling for non-native modes so that they fill
-         * the screen.  Should be enabled before the pipe is enabled, according
-         * to register description and PRM.
-         * Change the value here to see the borders for debugging
-         */
-        for_each_pipe(pipe)
-                I915_WRITE(BCLRPAT(pipe), 0);
-
         drm_mode_set_crtcinfo(adjusted_mode, 0);
         pipe_config->timings_set = true;
 
-        switch (intel_connector->panel.fitting_mode) {
-        case DRM_MODE_SCALE_CENTER:
-                /*
-                 * For centered modes, we have to calculate border widths &
-                 * heights and modify the values programmed into the CRTC.
-                 */
-                centre_horizontally(adjusted_mode, mode->hdisplay);
-                centre_vertically(adjusted_mode, mode->vdisplay);
-                border = LVDS_BORDER_ENABLE;
-                break;
-
-        case DRM_MODE_SCALE_ASPECT:
-                /* Scale but preserve the aspect ratio */
-                if (INTEL_INFO(dev)->gen >= 4) {
-                        u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
-                        u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
-
-                        /* 965+ is easy, it does everything in hw */
-                        if (scaled_width > scaled_height)
-                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_PILLAR;
-                        else if (scaled_width < scaled_height)
-                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER;
-                        else if (adjusted_mode->hdisplay != mode->hdisplay)
-                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
-                } else {
-                        u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
-                        u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
-                        /*
-                         * For earlier chips we have to calculate the scaling
-                         * ratio by hand and program it into the
-                         * PFIT_PGM_RATIO register
-                         */
-                        if (scaled_width > scaled_height) { /* pillar */
-                                centre_horizontally(adjusted_mode, scaled_height / mode->vdisplay);
-
-                                border = LVDS_BORDER_ENABLE;
-                                if (mode->vdisplay != adjusted_mode->vdisplay) {
-                                        u32 bits = panel_fitter_scaling(mode->vdisplay, adjusted_mode->vdisplay);
-                                        pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT |
-                                                            bits << PFIT_VERT_SCALE_SHIFT);
-                                        pfit_control |= (PFIT_ENABLE |
-                                                         VERT_INTERP_BILINEAR |
-                                                         HORIZ_INTERP_BILINEAR);
-                                }
-                        } else if (scaled_width < scaled_height) { /* letter */
-                                centre_vertically(adjusted_mode, scaled_width / mode->hdisplay);
-
-                                border = LVDS_BORDER_ENABLE;
-                                if (mode->hdisplay != adjusted_mode->hdisplay) {
-                                        u32 bits = panel_fitter_scaling(mode->hdisplay, adjusted_mode->hdisplay);
-                                        pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT |
-                                                            bits << PFIT_VERT_SCALE_SHIFT);
-                                        pfit_control |= (PFIT_ENABLE |
-                                                         VERT_INTERP_BILINEAR |
-                                                         HORIZ_INTERP_BILINEAR);
-                                }
-                        } else
-                                /* Aspects match, Let hw scale both directions */
-                                pfit_control |= (PFIT_ENABLE |
-                                                 VERT_AUTO_SCALE | HORIZ_AUTO_SCALE |
-                                                 VERT_INTERP_BILINEAR |
-                                                 HORIZ_INTERP_BILINEAR);
-                }
-                break;
-
-        case DRM_MODE_SCALE_FULLSCREEN:
-                /*
-                 * Full scaling, even if it changes the aspect ratio.
-                 * Fortunately this is all done for us in hw.
-                 */
-                if (mode->vdisplay != adjusted_mode->vdisplay ||
-                    mode->hdisplay != adjusted_mode->hdisplay) {
-                        pfit_control |= PFIT_ENABLE;
-                        if (INTEL_INFO(dev)->gen >= 4)
-                                pfit_control |= PFIT_SCALING_AUTO;
-                        else
-                                pfit_control |= (VERT_AUTO_SCALE |
-                                                 VERT_INTERP_BILINEAR |
-                                                 HORIZ_AUTO_SCALE |
-                                                 HORIZ_INTERP_BILINEAR);
-                }
-                break;
-
-        default:
-                break;
-        }
-
-out:
-        /* If not enabling scaling, be consistent and always use 0. */
-        if ((pfit_control & PFIT_ENABLE) == 0) {
-                pfit_control = 0;
-                pfit_pgm_ratios = 0;
-        }
-
-        /* Make sure pre-965 set dither correctly */
-        if (INTEL_INFO(dev)->gen < 4 && dev_priv->lvds_dither)
-                pfit_control |= PANEL_8TO6_DITHER_ENABLE;
-
-        if (pfit_control != lvds_encoder->pfit_control ||
-            pfit_pgm_ratios != lvds_encoder->pfit_pgm_ratios) {
-                lvds_encoder->pfit_control = pfit_control;
-                lvds_encoder->pfit_pgm_ratios = pfit_pgm_ratios;
-        }
-        dev_priv->lvds_border_bits = border;
-
         /*
          * XXX: It would be nice to support lower refresh rates on the
          * panels to reduce power consumption, and perhaps match the
@@ -937,11 +758,11 @@ static bool lvds_is_present_in_vbt(struct drm_device *dev,
         struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
-        if (!dev_priv->child_dev_num)
+        if (!dev_priv->vbt.child_dev_num)
                 return true;
 
-        for (i = 0; i < dev_priv->child_dev_num; i++) {
-                struct child_device_config *child = dev_priv->child_dev + i;
+        for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
+                struct child_device_config *child = dev_priv->vbt.child_dev + i;
 
                 /* If the device type is not LFP, continue.
                  * We have to check both the new identifiers as well as the
@@ -1029,7 +850,7 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
          */
         val = I915_READ(lvds_encoder->reg);
         if (!(val & ~(LVDS_PIPE_MASK | LVDS_DETECTED)))
-                val = dev_priv->bios_lvds_val;
+                val = dev_priv->vbt.bios_lvds_val;
 
         return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP;
 }
@@ -1056,7 +877,7 @@ static bool intel_lvds_supported(struct drm_device *dev)
  * Create the connector, register the LVDS DDC bus, and try to figure out what
  * modes we can display on the LVDS panel (if present).
  */
-bool intel_lvds_init(struct drm_device *dev)
+void intel_lvds_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds_encoder *lvds_encoder;
@@ -1074,43 +895,39 @@ bool intel_lvds_init(struct drm_device *dev)
         u8 pin;
 
         if (!intel_lvds_supported(dev))
-                return false;
+                return;
 
         /* Skip init on machines we know falsely report LVDS */
         if (dmi_check_system(intel_no_lvds))
-                return false;
+                return;
 
         pin = GMBUS_PORT_PANEL;
         if (!lvds_is_present_in_vbt(dev, &pin)) {
                 DRM_DEBUG_KMS("LVDS is not present in VBT\n");
-                return false;
+                return;
         }
 
         if (HAS_PCH_SPLIT(dev)) {
                 if ((I915_READ(PCH_LVDS) & LVDS_DETECTED) == 0)
-                        return false;
-                if (dev_priv->edp.support) {
+                        return;
+                if (dev_priv->vbt.edp_support) {
                         DRM_DEBUG_KMS("disable LVDS for eDP support\n");
-                        return false;
+                        return;
                 }
         }
 
         lvds_encoder = kzalloc(sizeof(struct intel_lvds_encoder), GFP_KERNEL);
         if (!lvds_encoder)
-                return false;
+                return;
 
         lvds_connector = kzalloc(sizeof(struct intel_lvds_connector), GFP_KERNEL);
         if (!lvds_connector) {
                 kfree(lvds_encoder);
-                return false;
+                return;
         }
 
         lvds_encoder->attached_connector = lvds_connector;
 
-        if (!HAS_PCH_SPLIT(dev)) {
-                lvds_encoder->pfit_control = I915_READ(PFIT_CONTROL);
-        }
-
         intel_encoder = &lvds_encoder->base;
         encoder = &intel_encoder->base;
         intel_connector = &lvds_connector->base;
@@ -1122,11 +939,11 @@ bool intel_lvds_init(struct drm_device *dev)
                          DRM_MODE_ENCODER_LVDS);
 
         intel_encoder->enable = intel_enable_lvds;
-        intel_encoder->pre_enable = intel_pre_enable_lvds;
         intel_encoder->pre_pll_enable = intel_pre_pll_enable_lvds;
         intel_encoder->compute_config = intel_lvds_compute_config;
         intel_encoder->disable = intel_disable_lvds;
         intel_encoder->get_hw_state = intel_lvds_get_hw_state;
+        intel_encoder->get_config = intel_lvds_get_config;
         intel_connector->get_hw_state = intel_connector_get_hw_state;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
@@ -1212,11 +1029,11 @@ bool intel_lvds_init(struct drm_device *dev)
         }
 
         /* Failed to get EDID, what about VBT? */
-        if (dev_priv->lfp_lvds_vbt_mode) {
+        if (dev_priv->vbt.lfp_lvds_vbt_mode) {
                 DRM_DEBUG_KMS("using mode from VBT: ");
-                drm_mode_debug_printmodeline(dev_priv->lfp_lvds_vbt_mode);
+                drm_mode_debug_printmodeline(dev_priv->vbt.lfp_lvds_vbt_mode);
 
-                fixed_mode = drm_mode_duplicate(dev, dev_priv->lfp_lvds_vbt_mode);
+                fixed_mode = drm_mode_duplicate(dev, dev_priv->vbt.lfp_lvds_vbt_mode);
                 if (fixed_mode) {
                         fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
                         goto out;
@@ -1277,7 +1094,7 @@ out:
         intel_panel_init(&intel_connector->panel, fixed_mode);
         intel_panel_setup_backlight(connector);
 
-        return true;
+        return;
 
 failed:
         DRM_DEBUG_KMS("No LVDS modes found, disabling.\n");
@@ -1287,5 +1104,5 @@ failed:
                 drm_mode_destroy(dev, fixed_mode);
         kfree(lvds_encoder);
         kfree(lvds_connector);
-        return false;
+        return;
 }
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index a8117e614009..cfb8fb68f09c 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -110,6 +110,10 @@ struct opregion_asle {
         u8 rsvd[102];
 } __attribute__((packed));
 
+/* Driver readiness indicator */
+#define ASLE_ARDY_READY         (1 << 0)
+#define ASLE_ARDY_NOT_READY     (0 << 0)
+
 /* ASLE irq request bits */
 #define ASLE_SET_ALS_ILLUM     (1 << 0)
 #define ASLE_SET_BACKLIGHT     (1 << 1)
@@ -123,6 +127,12 @@ struct opregion_asle {
 #define ASLE_PFIT_FAILED        (1<<14)
 #define ASLE_PWM_FREQ_FAILED    (1<<16)
 
+/* Technology enabled indicator */
+#define ASLE_TCHE_ALS_EN        (1 << 0)
+#define ASLE_TCHE_BLC_EN        (1 << 1)
+#define ASLE_TCHE_PFIT_EN       (1 << 2)
+#define ASLE_TCHE_PFMB_EN       (1 << 3)
+
 /* ASLE backlight brightness to set */
 #define ASLE_BCLP_VALID                (1<<31)
 #define ASLE_BCLP_MSK          (~(1<<31))
@@ -152,7 +162,6 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
-        u32 max;
 
         DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp);
 
@@ -163,8 +172,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
         if (bclp > 255)
                 return ASLE_BACKLIGHT_FAILED;
 
-        max = intel_panel_get_max_backlight(dev);
-        intel_panel_set_backlight(dev, bclp * max / 255);
+        intel_panel_set_backlight(dev, bclp, 255);
         iowrite32((bclp*0x64)/0xff | ASLE_CBLV_VALID, &asle->cblv);
 
         return 0;
@@ -174,29 +182,22 @@ static u32 asle_set_als_illum(struct drm_device *dev, u32 alsi)
 {
         /* alsi is the current ALS reading in lux. 0 indicates below sensor
            range, 0xffff indicates above sensor range. 1-0xfffe are valid */
-        return 0;
+        DRM_DEBUG_DRIVER("Illum is not supported\n");
+        return ASLE_ALS_ILLUM_FAILED;
 }
 
 static u32 asle_set_pwm_freq(struct drm_device *dev, u32 pfmb)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        if (pfmb & ASLE_PFMB_PWM_VALID) {
-                u32 blc_pwm_ctl = I915_READ(BLC_PWM_CTL);
-                u32 pwm = pfmb & ASLE_PFMB_PWM_MASK;
-                blc_pwm_ctl &= BACKLIGHT_DUTY_CYCLE_MASK;
-                pwm = pwm >> 9;
-                /* FIXME - what do we do with the PWM? */
-        }
-        return 0;
+        DRM_DEBUG_DRIVER("PWM freq is not supported\n");
+        return ASLE_PWM_FREQ_FAILED;
 }
 
 static u32 asle_set_pfit(struct drm_device *dev, u32 pfit)
 {
         /* Panel fitting is currently controlled by the X code, so this is a
            noop until modesetting support works fully */
-        if (!(pfit & ASLE_PFIT_VALID))
-                return ASLE_PFIT_FAILED;
-        return 0;
+        DRM_DEBUG_DRIVER("Pfit is not supported\n");
+        return ASLE_PFIT_FAILED;
 }
 
 void intel_opregion_asle_intr(struct drm_device *dev)
@@ -231,64 +232,6 @@ void intel_opregion_asle_intr(struct drm_device *dev)
         iowrite32(asle_stat, &asle->aslc);
 }
 
-void intel_opregion_gse_intr(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
-        u32 asle_stat = 0;
-        u32 asle_req;
-
-        if (!asle)
-                return;
-
-        asle_req = ioread32(&asle->aslc) & ASLE_REQ_MSK;
-
-        if (!asle_req) {
-                DRM_DEBUG_DRIVER("non asle set request??\n");
-                return;
-        }
-
-        if (asle_req & ASLE_SET_ALS_ILLUM) {
-                DRM_DEBUG_DRIVER("Illum is not supported\n");
-                asle_stat |= ASLE_ALS_ILLUM_FAILED;
-        }
-
-        if (asle_req & ASLE_SET_BACKLIGHT)
-                asle_stat |= asle_set_backlight(dev, ioread32(&asle->bclp));
-
-        if (asle_req & ASLE_SET_PFIT) {
-                DRM_DEBUG_DRIVER("Pfit is not supported\n");
-                asle_stat |= ASLE_PFIT_FAILED;
-        }
-
-        if (asle_req & ASLE_SET_PWM_FREQ) {
-                DRM_DEBUG_DRIVER("PWM freq is not supported\n");
-                asle_stat |= ASLE_PWM_FREQ_FAILED;
-        }
-
-        iowrite32(asle_stat, &asle->aslc);
-}
-#define ASLE_ALS_EN    (1<<0)
-#define ASLE_BLC_EN    (1<<1)
-#define ASLE_PFIT_EN   (1<<2)
-#define ASLE_PFMB_EN   (1<<3)
-
-void intel_opregion_enable_asle(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
-
-        if (asle) {
-                if (IS_MOBILE(dev))
-                        intel_enable_asle(dev);
-
-                iowrite32(ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN |
-                          ASLE_PFMB_EN,
-                          &asle->tche);
-                iowrite32(1, &asle->ardy);
-        }
-}
-
 #define ACPI_EV_DISPLAY_SWITCH (1<<0)
 #define ACPI_EV_LID            (1<<1)
 #define ACPI_EV_DOCK           (1<<2)
@@ -368,8 +311,8 @@ static void intel_didl_outputs(struct drm_device *dev)
 
         list_for_each_entry(acpi_cdev, &acpi_video_bus->children, node) {
                 if (i >= 8) {
-                        dev_printk(KERN_ERR, &dev->pdev->dev,
-                                    "More than 8 outputs detected\n");
+                        dev_dbg(&dev->pdev->dev,
+                                "More than 8 outputs detected via ACPI\n");
                         return;
                 }
                 status =
@@ -395,8 +338,8 @@ blind_set:
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 int output_type = ACPI_OTHER_OUTPUT;
                 if (i >= 8) {
-                        dev_printk(KERN_ERR, &dev->pdev->dev,
-                                    "More than 8 outputs detected\n");
+                        dev_dbg(&dev->pdev->dev,
+                                "More than 8 outputs in connector list\n");
                         return;
                 }
                 switch (connector->connector_type) {
@@ -472,8 +415,10 @@ void intel_opregion_init(struct drm_device *dev)
                 register_acpi_notifier(&intel_opregion_notifier);
         }
 
-        if (opregion->asle)
-                intel_opregion_enable_asle(dev);
+        if (opregion->asle) {
+                iowrite32(ASLE_TCHE_BLC_EN, &opregion->asle->tche);
+                iowrite32(ASLE_ARDY_READY, &opregion->asle->ardy);
+        }
 }
 
 void intel_opregion_fini(struct drm_device *dev)
@@ -484,6 +429,9 @@ void intel_opregion_fini(struct drm_device *dev)
         if (!opregion->header)
                 return;
 
+        if (opregion->asle)
+                iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy);
+
         if (opregion->acpi) {
                 iowrite32(0, &opregion->acpi->drdy);
 
@@ -546,6 +494,8 @@ int intel_opregion_setup(struct drm_device *dev)
         if (mboxes & MBOX_ASLE) {
                 DRM_DEBUG_DRIVER("ASLE supported\n");
                 opregion->asle = base + OPREGION_ASLE_OFFSET;
+
+                iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 67a2501d519d..a3698812e9c7 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -217,7 +217,7 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
         int ret;
 
         BUG_ON(overlay->last_flip_req);
-        ret = i915_add_request(ring, NULL, &overlay->last_flip_req);
+        ret = i915_add_request(ring, &overlay->last_flip_req);
         if (ret)
                 return ret;
 
@@ -286,7 +286,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
         intel_ring_emit(ring, flip_addr);
         intel_ring_advance(ring);
 
-        return i915_add_request(ring, NULL, &overlay->last_flip_req);
+        return i915_add_request(ring, &overlay->last_flip_req);
 }
 
 static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
@@ -1485,14 +1485,15 @@ err:
 }
 
 void
-intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error)
+intel_overlay_print_error_state(struct drm_i915_error_state_buf *m,
+                                struct intel_overlay_error_state *error)
 {
-        seq_printf(m, "Overlay, status: 0x%08x, interrupt: 0x%08x\n",
-                   error->dovsta, error->isr);
-        seq_printf(m, "  Register file at 0x%08lx:\n",
-                   error->base);
+        i915_error_printf(m, "Overlay, status: 0x%08x, interrupt: 0x%08x\n",
+                          error->dovsta, error->isr);
+        i915_error_printf(m, "  Register file at 0x%08lx:\n",
+                          error->base);
 
-#define P(x) seq_printf(m, "    " #x ": 0x%08x\n", error->regs.x)
+#define P(x) i915_error_printf(m, "    " #x ":  0x%08x\n", error->regs.x)
         P(OBUF_0Y);
         P(OBUF_1Y);
         P(OBUF_0U);
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index eb5e6e95f3c7..80bea1d3209f 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -54,14 +54,16 @@ intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
 
 /* adjusted_mode has been preset to be the panel's fixed mode */
 void
-intel_pch_panel_fitting(struct drm_device *dev,
-                        int fitting_mode,
-                        const struct drm_display_mode *mode,
-                        struct drm_display_mode *adjusted_mode)
+intel_pch_panel_fitting(struct intel_crtc *intel_crtc,
+                        struct intel_crtc_config *pipe_config,
+                        int fitting_mode)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_display_mode *mode, *adjusted_mode;
         int x, y, width, height;
 
+        mode = &pipe_config->requested_mode;
+        adjusted_mode = &pipe_config->adjusted_mode;
+
         x = y = width = height = 0;
 
         /* Native modes don't need fitting */
@@ -104,17 +106,209 @@ intel_pch_panel_fitting(struct drm_device *dev,
                 }
                 break;
 
-        default:
         case DRM_MODE_SCALE_FULLSCREEN:
                 x = y = 0;
                 width = adjusted_mode->hdisplay;
                 height = adjusted_mode->vdisplay;
                 break;
+
+        default:
+                WARN(1, "bad panel fit mode: %d\n", fitting_mode);
+                return;
         }
 
 done:
-        dev_priv->pch_pf_pos = (x << 16) | y;
-        dev_priv->pch_pf_size = (width << 16) | height;
+        pipe_config->pch_pfit.pos = (x << 16) | y;
+        pipe_config->pch_pfit.size = (width << 16) | height;
+}
+
+static void
+centre_horizontally(struct drm_display_mode *mode,
+                    int width)
+{
+        u32 border, sync_pos, blank_width, sync_width;
+
+        /* keep the hsync and hblank widths constant */
+        sync_width = mode->crtc_hsync_end - mode->crtc_hsync_start;
+        blank_width = mode->crtc_hblank_end - mode->crtc_hblank_start;
+        sync_pos = (blank_width - sync_width + 1) / 2;
+
+        border = (mode->hdisplay - width + 1) / 2;
+        border += border & 1; /* make the border even */
+
+        mode->crtc_hdisplay = width;
+        mode->crtc_hblank_start = width + border;
+        mode->crtc_hblank_end = mode->crtc_hblank_start + blank_width;
+
+        mode->crtc_hsync_start = mode->crtc_hblank_start + sync_pos;
+        mode->crtc_hsync_end = mode->crtc_hsync_start + sync_width;
+}
+
+static void
+centre_vertically(struct drm_display_mode *mode,
+                  int height)
+{
+        u32 border, sync_pos, blank_width, sync_width;
+
+        /* keep the vsync and vblank widths constant */
+        sync_width = mode->crtc_vsync_end - mode->crtc_vsync_start;
+        blank_width = mode->crtc_vblank_end - mode->crtc_vblank_start;
+        sync_pos = (blank_width - sync_width + 1) / 2;
+
+        border = (mode->vdisplay - height + 1) / 2;
+
+        mode->crtc_vdisplay = height;
+        mode->crtc_vblank_start = height + border;
+        mode->crtc_vblank_end = mode->crtc_vblank_start + blank_width;
+
+        mode->crtc_vsync_start = mode->crtc_vblank_start + sync_pos;
+        mode->crtc_vsync_end = mode->crtc_vsync_start + sync_width;
+}
+
+static inline u32 panel_fitter_scaling(u32 source, u32 target)
+{
+        /*
+         * Floating point operation is not supported. So the FACTOR
+         * is defined, which can avoid the floating point computation
+         * when calculating the panel ratio.
+         */
+#define ACCURACY 12
+#define FACTOR (1 << ACCURACY)
+        u32 ratio = source * FACTOR / target;
+        return (FACTOR * ratio + FACTOR/2) / FACTOR;
+}
+
+void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
+                              struct intel_crtc_config *pipe_config,
+                              int fitting_mode)
+{
+        struct drm_device *dev = intel_crtc->base.dev;
+        u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
+        struct drm_display_mode *mode, *adjusted_mode;
+
+        mode = &pipe_config->requested_mode;
+        adjusted_mode = &pipe_config->adjusted_mode;
+
+        /* Native modes don't need fitting */
+        if (adjusted_mode->hdisplay == mode->hdisplay &&
+            adjusted_mode->vdisplay == mode->vdisplay)
+                goto out;
+
+        switch (fitting_mode) {
+        case DRM_MODE_SCALE_CENTER:
+                /*
+                 * For centered modes, we have to calculate border widths &
+                 * heights and modify the values programmed into the CRTC.
+                 */
+                centre_horizontally(adjusted_mode, mode->hdisplay);
+                centre_vertically(adjusted_mode, mode->vdisplay);
+                border = LVDS_BORDER_ENABLE;
+                break;
+        case DRM_MODE_SCALE_ASPECT:
+                /* Scale but preserve the aspect ratio */
+                if (INTEL_INFO(dev)->gen >= 4) {
+                        u32 scaled_width = adjusted_mode->hdisplay *
+                                mode->vdisplay;
+                        u32 scaled_height = mode->hdisplay *
+                                adjusted_mode->vdisplay;
+
+                        /* 965+ is easy, it does everything in hw */
+                        if (scaled_width > scaled_height)
+                                pfit_control |= PFIT_ENABLE |
+                                        PFIT_SCALING_PILLAR;
+                        else if (scaled_width < scaled_height)
+                                pfit_control |= PFIT_ENABLE |
+                                        PFIT_SCALING_LETTER;
+                        else if (adjusted_mode->hdisplay != mode->hdisplay)
+                                pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
+                } else {
+                        u32 scaled_width = adjusted_mode->hdisplay *
+                                mode->vdisplay;
+                        u32 scaled_height = mode->hdisplay *
+                                adjusted_mode->vdisplay;
+                        /*
+                         * For earlier chips we have to calculate the scaling
+                         * ratio by hand and program it into the
+                         * PFIT_PGM_RATIO register
+                         */
+                        if (scaled_width > scaled_height) { /* pillar */
+                                centre_horizontally(adjusted_mode,
+                                                    scaled_height /
+                                                    mode->vdisplay);
+
+                                border = LVDS_BORDER_ENABLE;
+                                if (mode->vdisplay != adjusted_mode->vdisplay) {
+                                        u32 bits = panel_fitter_scaling(mode->vdisplay, adjusted_mode->vdisplay);
+                                        pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT |
+                                                            bits << PFIT_VERT_SCALE_SHIFT);
+                                        pfit_control |= (PFIT_ENABLE |
+                                                         VERT_INTERP_BILINEAR |
+                                                         HORIZ_INTERP_BILINEAR);
+                                }
+                        } else if (scaled_width < scaled_height) { /* letter */
+                                centre_vertically(adjusted_mode,
+                                                  scaled_width /
+                                                  mode->hdisplay);
+
+                                border = LVDS_BORDER_ENABLE;
+                                if (mode->hdisplay != adjusted_mode->hdisplay) {
+                                        u32 bits = panel_fitter_scaling(mode->hdisplay, adjusted_mode->hdisplay);
+                                        pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT |
+                                                            bits << PFIT_VERT_SCALE_SHIFT);
+                                        pfit_control |= (PFIT_ENABLE |
+                                                         VERT_INTERP_BILINEAR |
+                                                         HORIZ_INTERP_BILINEAR);
+                                }
+                        } else {
+                                /* Aspects match, Let hw scale both directions */
+                                pfit_control |= (PFIT_ENABLE |
+                                                 VERT_AUTO_SCALE | HORIZ_AUTO_SCALE |
+                                                 VERT_INTERP_BILINEAR |
+                                                 HORIZ_INTERP_BILINEAR);
+                        }
+                }
+                break;
+        case DRM_MODE_SCALE_FULLSCREEN:
+                /*
+                 * Full scaling, even if it changes the aspect ratio.
+                 * Fortunately this is all done for us in hw.
+                 */
+                if (mode->vdisplay != adjusted_mode->vdisplay ||
+                    mode->hdisplay != adjusted_mode->hdisplay) {
+                        pfit_control |= PFIT_ENABLE;
+                        if (INTEL_INFO(dev)->gen >= 4)
+                                pfit_control |= PFIT_SCALING_AUTO;
+                        else
+                                pfit_control |= (VERT_AUTO_SCALE |
+                                                 VERT_INTERP_BILINEAR |
+                                                 HORIZ_AUTO_SCALE |
+                                                 HORIZ_INTERP_BILINEAR);
+                }
+                break;
+        default:
+                WARN(1, "bad panel fit mode: %d\n", fitting_mode);
+                return;
+        }
+
+        /* 965+ wants fuzzy fitting */
+        /* FIXME: handle multiple panels by failing gracefully */
+        if (INTEL_INFO(dev)->gen >= 4)
+                pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) |
+                                 PFIT_FILTER_FUZZY);
+
+out:
+        if ((pfit_control & PFIT_ENABLE) == 0) {
+                pfit_control = 0;
+                pfit_pgm_ratios = 0;
+        }
+
+        /* Make sure pre-965 set dither correctly for 18bpp panels. */
+        if (INTEL_INFO(dev)->gen < 4 && pipe_config->pipe_bpp == 18)
+                pfit_control |= PANEL_8TO6_DITHER_ENABLE;
+
+        pipe_config->gmch_pfit.control = pfit_control;
+        pipe_config->gmch_pfit.pgm_ratios = pfit_pgm_ratios;
+        pipe_config->gmch_pfit.lvds_border_bits = border;
 }
 
 static int is_backlight_combination_mode(struct drm_device *dev)
@@ -130,11 +324,16 @@ static int is_backlight_combination_mode(struct drm_device *dev)
         return 0;
 }
 
+/* XXX: query mode clock or hardware clock and program max PWM appropriately
+ * when it's 0.
+ */
 static u32 i915_read_blc_pwm_ctl(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val;
 
+        WARN_ON_SMP(!spin_is_locked(&dev_priv->backlight.lock));
+
         /* Restore the CTL value if it lost, e.g. GPU reset */
 
         if (HAS_PCH_SPLIT(dev_priv->dev)) {
@@ -164,7 +363,7 @@ static u32 i915_read_blc_pwm_ctl(struct drm_device *dev)
         return val;
 }
 
-static u32 _intel_panel_get_max_backlight(struct drm_device *dev)
+static u32 intel_panel_get_max_backlight(struct drm_device *dev)
 {
         u32 max;
 
@@ -182,23 +381,8 @@ static u32 _intel_panel_get_max_backlight(struct drm_device *dev)
                         max *= 0xff;
         }
 
-        return max;
-}
-
-u32 intel_panel_get_max_backlight(struct drm_device *dev)
-{
-        u32 max;
-
-        max = _intel_panel_get_max_backlight(dev);
-        if (max == 0) {
-                /* XXX add code here to query mode clock or hardware clock
-                 * and program max PWM appropriately.
-                 */
-                pr_warn_once("fixme: max PWM is zero\n");
-                return 1;
-        }
-
         DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max);
+
         return max;
 }
 
@@ -217,8 +401,11 @@ static u32 intel_panel_compute_brightness(struct drm_device *dev, u32 val)
                 return val;
 
         if (i915_panel_invert_brightness > 0 ||
-            dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS)
-                return intel_panel_get_max_backlight(dev) - val;
+            dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) {
+                u32 max = intel_panel_get_max_backlight(dev);
+                if (max)
+                        return max - val;
+        }
 
         return val;
 }
@@ -227,6 +414,9 @@ static u32 intel_panel_get_backlight(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
 
         if (HAS_PCH_SPLIT(dev)) {
                 val = I915_READ(BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
@@ -244,6 +434,9 @@ static u32 intel_panel_get_backlight(struct drm_device *dev)
         }
 
         val = intel_panel_compute_brightness(dev, val);
+
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+
         DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
         return val;
 }
@@ -270,6 +463,10 @@ static void intel_panel_actually_set_backlight(struct drm_device *dev, u32 level
                 u32 max = intel_panel_get_max_backlight(dev);
                 u8 lbpc;
 
+                /* we're screwed, but keep behaviour backwards compatible */
+                if (!max)
+                        max = 1;
+
                 lbpc = level * 0xfe / max + 1;
                 level /= lbpc;
                 pci_write_config_byte(dev->pdev, PCI_LBPC, lbpc);
@@ -282,9 +479,23 @@ static void intel_panel_actually_set_backlight(struct drm_device *dev, u32 level
         I915_WRITE(BLC_PWM_CTL, tmp | level);
 }
 
-void intel_panel_set_backlight(struct drm_device *dev, u32 level)
+/* set backlight brightness to level in range [0..max] */
+void intel_panel_set_backlight(struct drm_device *dev, u32 level, u32 max)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 freq;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+
+        freq = intel_panel_get_max_backlight(dev);
+        if (!freq) {
+                /* we are screwed, bail out */
+                goto out;
+        }
+
+        /* scale to hardware */
+        level = level * freq / max;
 
         dev_priv->backlight.level = level;
         if (dev_priv->backlight.device)
@@ -292,11 +503,16 @@ void intel_panel_set_backlight(struct drm_device *dev, u32 level)
 
         if (dev_priv->backlight.enabled)
                 intel_panel_actually_set_backlight(dev, level);
+out:
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
 }
 
 void intel_panel_disable_backlight(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
 
         dev_priv->backlight.enabled = false;
         intel_panel_actually_set_backlight(dev, 0);
@@ -314,12 +530,19 @@ void intel_panel_disable_backlight(struct drm_device *dev)
                         I915_WRITE(BLC_PWM_PCH_CTL1, tmp);
                 }
         }
+
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
 }
 
 void intel_panel_enable_backlight(struct drm_device *dev,
                                   enum pipe pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        enum transcoder cpu_transcoder =
+                intel_pipe_to_cpu_transcoder(dev_priv, pipe);
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
 
         if (dev_priv->backlight.level == 0) {
                 dev_priv->backlight.level = intel_panel_get_max_backlight(dev);
@@ -347,7 +570,10 @@ void intel_panel_enable_backlight(struct drm_device *dev,
                 else
                         tmp &= ~BLM_PIPE_SELECT;
 
-                tmp |= BLM_PIPE(pipe);
+                if (cpu_transcoder == TRANSCODER_EDP)
+                        tmp |= BLM_TRANSCODER_EDP;
+                else
+                        tmp |= BLM_PIPE(cpu_transcoder);
                 tmp &= ~BLM_PWM_ENABLE;
 
                 I915_WRITE(reg, tmp);
@@ -369,6 +595,8 @@ set_level:
          */
         dev_priv->backlight.enabled = true;
         intel_panel_actually_set_backlight(dev, dev_priv->backlight.level);
+
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
 }
 
 static void intel_panel_init_backlight(struct drm_device *dev)
@@ -405,7 +633,8 @@ intel_panel_detect(struct drm_device *dev)
 static int intel_panel_update_status(struct backlight_device *bd)
 {
         struct drm_device *dev = bl_get_data(bd);
-        intel_panel_set_backlight(dev, bd->props.brightness);
+        intel_panel_set_backlight(dev, bd->props.brightness,
+                                  bd->props.max_brightness);
         return 0;
 }
 
@@ -425,6 +654,7 @@ int intel_panel_setup_backlight(struct drm_connector *connector)
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct backlight_properties props;
+        unsigned long flags;
 
         intel_panel_init_backlight(dev);
 
@@ -434,7 +664,11 @@ int intel_panel_setup_backlight(struct drm_connector *connector)
         memset(&props, 0, sizeof(props));
         props.type = BACKLIGHT_RAW;
         props.brightness = dev_priv->backlight.level;
-        props.max_brightness = _intel_panel_get_max_backlight(dev);
+
+        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+        props.max_brightness = intel_panel_get_max_backlight(dev);
+        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+
         if (props.max_brightness == 0) {
                 DRM_DEBUG_DRIVER("Failed to get maximum backlight value\n");
                 return -ENODEV;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index aa01128ff192..ccbdd83f5220 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -113,8 +113,8 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         fbc_ctl |= obj->fence_reg;
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
-        DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
-                      cfb_pitch, crtc->y, intel_crtc->plane);
+        DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
+                      cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
 }
 
 static bool i8xx_fbc_enabled(struct drm_device *dev)
@@ -148,7 +148,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         /* enable it... */
         I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
 
-        DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
+        DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
 }
 
 static void g4x_disable_fbc(struct drm_device *dev)
@@ -228,7 +228,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
                 sandybridge_blit_fbc_update(dev);
         }
 
-        DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
+        DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
 }
 
 static void ironlake_disable_fbc(struct drm_device *dev)
@@ -242,6 +242,18 @@ static void ironlake_disable_fbc(struct drm_device *dev)
                 dpfc_ctl &= ~DPFC_CTL_EN;
                 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
 
+                if (IS_IVYBRIDGE(dev))
+                        /* WaFbcDisableDpfcClockGating:ivb */
+                        I915_WRITE(ILK_DSPCLK_GATE_D,
+                                   I915_READ(ILK_DSPCLK_GATE_D) &
+                                   ~ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
+
+                if (IS_HASWELL(dev))
+                        /* WaFbcDisableDpfcClockGating:hsw */
+                        I915_WRITE(HSW_CLKGATE_DISABLE_PART_1,
+                                   I915_READ(HSW_CLKGATE_DISABLE_PART_1) &
+                                   ~HSW_DPFC_GATING_DISABLE);
+
                 DRM_DEBUG_KMS("disabled FBC\n");
         }
 }
@@ -253,6 +265,47 @@ static bool ironlake_fbc_enabled(struct drm_device *dev)
         return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
 }
 
+static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_framebuffer *fb = crtc->fb;
+        struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+        I915_WRITE(IVB_FBC_RT_BASE, obj->gtt_offset);
+
+        I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X |
+                   IVB_DPFC_CTL_FENCE_EN |
+                   intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
+
+        if (IS_IVYBRIDGE(dev)) {
+                /* WaFbcAsynchFlipDisableFbcQueue:ivb */
+                I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
+                /* WaFbcDisableDpfcClockGating:ivb */
+                I915_WRITE(ILK_DSPCLK_GATE_D,
+                           I915_READ(ILK_DSPCLK_GATE_D) |
+                           ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
+        } else {
+                /* WaFbcAsynchFlipDisableFbcQueue:hsw */
+                I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
+                           HSW_BYPASS_FBC_QUEUE);
+                /* WaFbcDisableDpfcClockGating:hsw */
+                I915_WRITE(HSW_CLKGATE_DISABLE_PART_1,
+                           I915_READ(HSW_CLKGATE_DISABLE_PART_1) |
+                           HSW_DPFC_GATING_DISABLE);
+        }
+
+        I915_WRITE(SNB_DPFC_CTL_SA,
+                   SNB_CPU_FENCE_ENABLE | obj->fence_reg);
+        I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
+
+        sandybridge_blit_fbc_update(dev);
+
+        DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
+}
+
 bool intel_fbc_enabled(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -378,7 +431,7 @@ void intel_disable_fbc(struct drm_device *dev)
  *   - no pixel mulitply/line duplication
  *   - no alpha buffer discard
  *   - no dual wide
- *   - framebuffer <= 2048 in width, 1536 in height
+ *   - framebuffer <= max_hdisplay in width, max_vdisplay in height
  *
  * We can't assume that any compression will take place (worst case),
  * so the compressed buffer has to be the same size as the uncompressed
@@ -396,6 +449,7 @@ void intel_update_fbc(struct drm_device *dev)
         struct intel_framebuffer *intel_fb;
         struct drm_i915_gem_object *obj;
         int enable_fbc;
+        unsigned int max_hdisplay, max_vdisplay;
 
         if (!i915_powersave)
                 return;
@@ -439,7 +493,7 @@ void intel_update_fbc(struct drm_device *dev)
         if (enable_fbc < 0) {
                 DRM_DEBUG_KMS("fbc set to per-chip default\n");
                 enable_fbc = 1;
-                if (INTEL_INFO(dev)->gen <= 6)
+                if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
                         enable_fbc = 0;
         }
         if (!enable_fbc) {
@@ -454,13 +508,22 @@ void intel_update_fbc(struct drm_device *dev)
                 dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
                 goto out_disable;
         }
-        if ((crtc->mode.hdisplay > 2048) ||
-            (crtc->mode.vdisplay > 1536)) {
+
+        if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
+                max_hdisplay = 4096;
+                max_vdisplay = 2048;
+        } else {
+                max_hdisplay = 2048;
+                max_vdisplay = 1536;
+        }
+        if ((crtc->mode.hdisplay > max_hdisplay) ||
+            (crtc->mode.vdisplay > max_vdisplay)) {
                 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
                 dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
                 goto out_disable;
         }
-        if ((IS_I915GM(dev) || IS_I945GM(dev)) && intel_crtc->plane != 0) {
+        if ((IS_I915GM(dev) || IS_I945GM(dev) || IS_HASWELL(dev)) &&
+            intel_crtc->plane != 0) {
                 DRM_DEBUG_KMS("plane not 0, disabling compression\n");
                 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
                 goto out_disable;
@@ -481,8 +544,6 @@ void intel_update_fbc(struct drm_device *dev)
                 goto out_disable;
 
         if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
-                DRM_INFO("not enough stolen space for compressed buffer (need %zd bytes), disabling\n", intel_fb->obj->base.size);
-                DRM_INFO("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
                 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
                 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
                 goto out_disable;
@@ -1633,6 +1694,10 @@ static bool ironlake_check_srwm(struct drm_device *dev, int level,
                 I915_WRITE(DISP_ARB_CTL,
                            I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
                 return false;
+        } else if (INTEL_INFO(dev)->gen >= 6) {
+                /* enable FBC WM (except on ILK, where it must remain off) */
+                I915_WRITE(DISP_ARB_CTL,
+                           I915_READ(DISP_ARB_CTL) & ~DISP_FBC_WM_DIS);
         }
 
         if (display_wm > display->max_wm) {
@@ -2016,31 +2081,558 @@ static void ivybridge_update_wm(struct drm_device *dev)
                    cursor_wm);
 }
 
-static void
-haswell_update_linetime_wm(struct drm_device *dev, int pipe,
-                                 struct drm_display_mode *mode)
+static uint32_t hsw_wm_get_pixel_rate(struct drm_device *dev,
+                                      struct drm_crtc *crtc)
+{
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        uint32_t pixel_rate, pfit_size;
+
+        pixel_rate = intel_crtc->config.adjusted_mode.clock;
+
+        /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
+         * adjust the pixel_rate here. */
+
+        pfit_size = intel_crtc->config.pch_pfit.size;
+        if (pfit_size) {
+                uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
+
+                pipe_w = intel_crtc->config.requested_mode.hdisplay;
+                pipe_h = intel_crtc->config.requested_mode.vdisplay;
+                pfit_w = (pfit_size >> 16) & 0xFFFF;
+                pfit_h = pfit_size & 0xFFFF;
+                if (pipe_w < pfit_w)
+                        pipe_w = pfit_w;
+                if (pipe_h < pfit_h)
+                        pipe_h = pfit_h;
+
+                pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
+                                     pfit_w * pfit_h);
+        }
+
+        return pixel_rate;
+}
+
+static uint32_t hsw_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
+                               uint32_t latency)
+{
+        uint64_t ret;
+
+        ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
+        ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
+
+        return ret;
+}
+
+static uint32_t hsw_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
+                               uint32_t horiz_pixels, uint8_t bytes_per_pixel,
+                               uint32_t latency)
+{
+        uint32_t ret;
+
+        ret = (latency * pixel_rate) / (pipe_htotal * 10000);
+        ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
+        ret = DIV_ROUND_UP(ret, 64) + 2;
+        return ret;
+}
+
+static uint32_t hsw_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
+                           uint8_t bytes_per_pixel)
+{
+        return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
+}
+
+struct hsw_pipe_wm_parameters {
+        bool active;
+        bool sprite_enabled;
+        uint8_t pri_bytes_per_pixel;
+        uint8_t spr_bytes_per_pixel;
+        uint8_t cur_bytes_per_pixel;
+        uint32_t pri_horiz_pixels;
+        uint32_t spr_horiz_pixels;
+        uint32_t cur_horiz_pixels;
+        uint32_t pipe_htotal;
+        uint32_t pixel_rate;
+};
+
+struct hsw_wm_maximums {
+        uint16_t pri;
+        uint16_t spr;
+        uint16_t cur;
+        uint16_t fbc;
+};
+
+struct hsw_lp_wm_result {
+        bool enable;
+        bool fbc_enable;
+        uint32_t pri_val;
+        uint32_t spr_val;
+        uint32_t cur_val;
+        uint32_t fbc_val;
+};
+
+struct hsw_wm_values {
+        uint32_t wm_pipe[3];
+        uint32_t wm_lp[3];
+        uint32_t wm_lp_spr[3];
+        uint32_t wm_linetime[3];
+        bool enable_fbc_wm;
+};
+
+enum hsw_data_buf_partitioning {
+        HSW_DATA_BUF_PART_1_2,
+        HSW_DATA_BUF_PART_5_6,
+};
+
+/* For both WM_PIPE and WM_LP. */
+static uint32_t hsw_compute_pri_wm(struct hsw_pipe_wm_parameters *params,
+                                   uint32_t mem_value,
+                                   bool is_lp)
+{
+        uint32_t method1, method2;
+
+        /* TODO: for now, assume the primary plane is always enabled. */
+        if (!params->active)
+                return 0;
+
+        method1 = hsw_wm_method1(params->pixel_rate,
+                                 params->pri_bytes_per_pixel,
+                                 mem_value);
+
+        if (!is_lp)
+                return method1;
+
+        method2 = hsw_wm_method2(params->pixel_rate,
+                                 params->pipe_htotal,
+                                 params->pri_horiz_pixels,
+                                 params->pri_bytes_per_pixel,
+                                 mem_value);
+
+        return min(method1, method2);
+}
+
+/* For both WM_PIPE and WM_LP. */
+static uint32_t hsw_compute_spr_wm(struct hsw_pipe_wm_parameters *params,
+                                   uint32_t mem_value)
+{
+        uint32_t method1, method2;
+
+        if (!params->active || !params->sprite_enabled)
+                return 0;
+
+        method1 = hsw_wm_method1(params->pixel_rate,
+                                 params->spr_bytes_per_pixel,
+                                 mem_value);
+        method2 = hsw_wm_method2(params->pixel_rate,
+                                 params->pipe_htotal,
+                                 params->spr_horiz_pixels,
+                                 params->spr_bytes_per_pixel,
+                                 mem_value);
+        return min(method1, method2);
+}
+
+/* For both WM_PIPE and WM_LP. */
+static uint32_t hsw_compute_cur_wm(struct hsw_pipe_wm_parameters *params,
+                                   uint32_t mem_value)
+{
+        if (!params->active)
+                return 0;
+
+        return hsw_wm_method2(params->pixel_rate,
+                              params->pipe_htotal,
+                              params->cur_horiz_pixels,
+                              params->cur_bytes_per_pixel,
+                              mem_value);
+}
+
+/* Only for WM_LP. */
+static uint32_t hsw_compute_fbc_wm(struct hsw_pipe_wm_parameters *params,
+                                   uint32_t pri_val,
+                                   uint32_t mem_value)
+{
+        if (!params->active)
+                return 0;
+
+        return hsw_wm_fbc(pri_val,
+                          params->pri_horiz_pixels,
+                          params->pri_bytes_per_pixel);
+}
+
+static bool hsw_compute_lp_wm(uint32_t mem_value, struct hsw_wm_maximums *max,
+                              struct hsw_pipe_wm_parameters *params,
+                              struct hsw_lp_wm_result *result)
+{
+        enum pipe pipe;
+        uint32_t pri_val[3], spr_val[3], cur_val[3], fbc_val[3];
+
+        for (pipe = PIPE_A; pipe <= PIPE_C; pipe++) {
+                struct hsw_pipe_wm_parameters *p = &params[pipe];
+
+                pri_val[pipe] = hsw_compute_pri_wm(p, mem_value, true);
+                spr_val[pipe] = hsw_compute_spr_wm(p, mem_value);
+                cur_val[pipe] = hsw_compute_cur_wm(p, mem_value);
+                fbc_val[pipe] = hsw_compute_fbc_wm(p, pri_val[pipe], mem_value);
+        }
+
+        result->pri_val = max3(pri_val[0], pri_val[1], pri_val[2]);
+        result->spr_val = max3(spr_val[0], spr_val[1], spr_val[2]);
+        result->cur_val = max3(cur_val[0], cur_val[1], cur_val[2]);
+        result->fbc_val = max3(fbc_val[0], fbc_val[1], fbc_val[2]);
+
+        if (result->fbc_val > max->fbc) {
+                result->fbc_enable = false;
+                result->fbc_val = 0;
+        } else {
+                result->fbc_enable = true;
+        }
+
+        result->enable = result->pri_val <= max->pri &&
+                         result->spr_val <= max->spr &&
+                         result->cur_val <= max->cur;
+        return result->enable;
+}
+
+static uint32_t hsw_compute_wm_pipe(struct drm_i915_private *dev_priv,
+                                    uint32_t mem_value, enum pipe pipe,
+                                    struct hsw_pipe_wm_parameters *params)
+{
+        uint32_t pri_val, cur_val, spr_val;
+
+        pri_val = hsw_compute_pri_wm(params, mem_value, false);
+        spr_val = hsw_compute_spr_wm(params, mem_value);
+        cur_val = hsw_compute_cur_wm(params, mem_value);
+
+        WARN(pri_val > 127,
+             "Primary WM error, mode not supported for pipe %c\n",
+             pipe_name(pipe));
+        WARN(spr_val > 127,
+             "Sprite WM error, mode not supported for pipe %c\n",
+             pipe_name(pipe));
+        WARN(cur_val > 63,
+             "Cursor WM error, mode not supported for pipe %c\n",
+             pipe_name(pipe));
+
+        return (pri_val << WM0_PIPE_PLANE_SHIFT) |
+               (spr_val << WM0_PIPE_SPRITE_SHIFT) |
+               cur_val;
+}
+
+static uint32_t
+hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 temp;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
+        u32 linetime, ips_linetime;
 
-        temp = I915_READ(PIPE_WM_LINETIME(pipe));
-        temp &= ~PIPE_WM_LINETIME_MASK;
+        if (!intel_crtc_active(crtc))
+                return 0;
 
         /* The WM are computed with base on how long it takes to fill a single
          * row at the given clock rate, multiplied by 8.
          * */
-        temp |= PIPE_WM_LINETIME_TIME(
-                ((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
+        linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8, mode->clock);
+        ips_linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8,
+                                         intel_ddi_get_cdclk_freq(dev_priv));
 
-        /* IPS watermarks are only used by pipe A, and are ignored by
-         * pipes B and C.  They are calculated similarly to the common
-         * linetime values, except that we are using CD clock frequency
-         * in MHz instead of pixel rate for the division.
-         *
-         * This is a placeholder for the IPS watermark calculation code.
-         */
+        return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
+               PIPE_WM_LINETIME_TIME(linetime);
+}
+
+static void hsw_compute_wm_parameters(struct drm_device *dev,
+                                      struct hsw_pipe_wm_parameters *params,
+                                      uint32_t *wm,
+                                      struct hsw_wm_maximums *lp_max_1_2,
+                                      struct hsw_wm_maximums *lp_max_5_6)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc;
+        struct drm_plane *plane;
+        uint64_t sskpd = I915_READ64(MCH_SSKPD);
+        enum pipe pipe;
+        int pipes_active = 0, sprites_enabled = 0;
+
+        if ((sskpd >> 56) & 0xFF)
+                wm[0] = (sskpd >> 56) & 0xFF;
+        else
+                wm[0] = sskpd & 0xF;
+        wm[1] = ((sskpd >> 4) & 0xFF) * 5;
+        wm[2] = ((sskpd >> 12) & 0xFF) * 5;
+        wm[3] = ((sskpd >> 20) & 0x1FF) * 5;
+        wm[4] = ((sskpd >> 32) & 0x1FF) * 5;
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+                struct hsw_pipe_wm_parameters *p;
+
+                pipe = intel_crtc->pipe;
+                p = &params[pipe];
+
+                p->active = intel_crtc_active(crtc);
+                if (!p->active)
+                        continue;
+
+                pipes_active++;
+
+                p->pipe_htotal = intel_crtc->config.adjusted_mode.htotal;
+                p->pixel_rate = hsw_wm_get_pixel_rate(dev, crtc);
+                p->pri_bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
+                p->cur_bytes_per_pixel = 4;
+                p->pri_horiz_pixels =
+                        intel_crtc->config.requested_mode.hdisplay;
+                p->cur_horiz_pixels = 64;
+        }
+
+        list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+                struct intel_plane *intel_plane = to_intel_plane(plane);
+                struct hsw_pipe_wm_parameters *p;
+
+                pipe = intel_plane->pipe;
+                p = &params[pipe];
+
+                p->sprite_enabled = intel_plane->wm.enable;
+                p->spr_bytes_per_pixel = intel_plane->wm.bytes_per_pixel;
+                p->spr_horiz_pixels = intel_plane->wm.horiz_pixels;
+
+                if (p->sprite_enabled)
+                        sprites_enabled++;
+        }
 
-        I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
+        if (pipes_active > 1) {
+                lp_max_1_2->pri = lp_max_5_6->pri = sprites_enabled ? 128 : 256;
+                lp_max_1_2->spr = lp_max_5_6->spr = 128;
+                lp_max_1_2->cur = lp_max_5_6->cur = 64;
+        } else {
+                lp_max_1_2->pri = sprites_enabled ? 384 : 768;
+                lp_max_5_6->pri = sprites_enabled ? 128 : 768;
+                lp_max_1_2->spr = 384;
+                lp_max_5_6->spr = 640;
+                lp_max_1_2->cur = lp_max_5_6->cur = 255;
+        }
+        lp_max_1_2->fbc = lp_max_5_6->fbc = 15;
+}
+
+static void hsw_compute_wm_results(struct drm_device *dev,
+                                   struct hsw_pipe_wm_parameters *params,
+                                   uint32_t *wm,
+                                   struct hsw_wm_maximums *lp_maximums,
+                                   struct hsw_wm_values *results)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc;
+        struct hsw_lp_wm_result lp_results[4] = {};
+        enum pipe pipe;
+        int level, max_level, wm_lp;
+
+        for (level = 1; level <= 4; level++)
+                if (!hsw_compute_lp_wm(wm[level], lp_maximums, params,
+                                       &lp_results[level - 1]))
+                        break;
+        max_level = level - 1;
+
+        /* The spec says it is preferred to disable FBC WMs instead of disabling
+         * a WM level. */
+        results->enable_fbc_wm = true;
+        for (level = 1; level <= max_level; level++) {
+                if (!lp_results[level - 1].fbc_enable) {
+                        results->enable_fbc_wm = false;
+                        break;
+                }
+        }
+
+        memset(results, 0, sizeof(*results));
+        for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
+                const struct hsw_lp_wm_result *r;
+
+                level = (max_level == 4 && wm_lp > 1) ? wm_lp + 1 : wm_lp;
+                if (level > max_level)
+                        break;
+
+                r = &lp_results[level - 1];
+                results->wm_lp[wm_lp - 1] = HSW_WM_LP_VAL(level * 2,
+                                                          r->fbc_val,
+                                                          r->pri_val,
+                                                          r->cur_val);
+                results->wm_lp_spr[wm_lp - 1] = r->spr_val;
+        }
+
+        for_each_pipe(pipe)
+                results->wm_pipe[pipe] = hsw_compute_wm_pipe(dev_priv, wm[0],
+                                                             pipe,
+                                                             &params[pipe]);
+
+        for_each_pipe(pipe) {
+                crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+                results->wm_linetime[pipe] = hsw_compute_linetime_wm(dev, crtc);
+        }
+}
+
+/* Find the result with the highest level enabled. Check for enable_fbc_wm in
+ * case both are at the same level. Prefer r1 in case they're the same. */
+struct hsw_wm_values *hsw_find_best_result(struct hsw_wm_values *r1,
+                                           struct hsw_wm_values *r2)
+{
+        int i, val_r1 = 0, val_r2 = 0;
+
+        for (i = 0; i < 3; i++) {
+                if (r1->wm_lp[i] & WM3_LP_EN)
+                        val_r1 = r1->wm_lp[i] & WM1_LP_LATENCY_MASK;
+                if (r2->wm_lp[i] & WM3_LP_EN)
+                        val_r2 = r2->wm_lp[i] & WM1_LP_LATENCY_MASK;
+        }
+
+        if (val_r1 == val_r2) {
+                if (r2->enable_fbc_wm && !r1->enable_fbc_wm)
+                        return r2;
+                else
+                        return r1;
+        } else if (val_r1 > val_r2) {
+                return r1;
+        } else {
+                return r2;
+        }
+}
+
+/*
+ * The spec says we shouldn't write when we don't need, because every write
+ * causes WMs to be re-evaluated, expending some power.
+ */
+static void hsw_write_wm_values(struct drm_i915_private *dev_priv,
+                                struct hsw_wm_values *results,
+                                enum hsw_data_buf_partitioning partitioning)
+{
+        struct hsw_wm_values previous;
+        uint32_t val;
+        enum hsw_data_buf_partitioning prev_partitioning;
+        bool prev_enable_fbc_wm;
+
+        previous.wm_pipe[0] = I915_READ(WM0_PIPEA_ILK);
+        previous.wm_pipe[1] = I915_READ(WM0_PIPEB_ILK);
+        previous.wm_pipe[2] = I915_READ(WM0_PIPEC_IVB);
+        previous.wm_lp[0] = I915_READ(WM1_LP_ILK);
+        previous.wm_lp[1] = I915_READ(WM2_LP_ILK);
+        previous.wm_lp[2] = I915_READ(WM3_LP_ILK);
+        previous.wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
+        previous.wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
+        previous.wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
+        previous.wm_linetime[0] = I915_READ(PIPE_WM_LINETIME(PIPE_A));
+        previous.wm_linetime[1] = I915_READ(PIPE_WM_LINETIME(PIPE_B));
+        previous.wm_linetime[2] = I915_READ(PIPE_WM_LINETIME(PIPE_C));
+
+        prev_partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
+                            HSW_DATA_BUF_PART_5_6 : HSW_DATA_BUF_PART_1_2;
+
+        prev_enable_fbc_wm = !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
+
+        if (memcmp(results->wm_pipe, previous.wm_pipe,
+                   sizeof(results->wm_pipe)) == 0 &&
+            memcmp(results->wm_lp, previous.wm_lp,
+                   sizeof(results->wm_lp)) == 0 &&
+            memcmp(results->wm_lp_spr, previous.wm_lp_spr,
+                   sizeof(results->wm_lp_spr)) == 0 &&
+            memcmp(results->wm_linetime, previous.wm_linetime,
+                   sizeof(results->wm_linetime)) == 0 &&
+            partitioning == prev_partitioning &&
+            results->enable_fbc_wm == prev_enable_fbc_wm)
+                return;
+
+        if (previous.wm_lp[2] != 0)
+                I915_WRITE(WM3_LP_ILK, 0);
+        if (previous.wm_lp[1] != 0)
+                I915_WRITE(WM2_LP_ILK, 0);
+        if (previous.wm_lp[0] != 0)
+                I915_WRITE(WM1_LP_ILK, 0);
+
+        if (previous.wm_pipe[0] != results->wm_pipe[0])
+                I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
+        if (previous.wm_pipe[1] != results->wm_pipe[1])
+                I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
+        if (previous.wm_pipe[2] != results->wm_pipe[2])
+                I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
+
+        if (previous.wm_linetime[0] != results->wm_linetime[0])
+                I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
+        if (previous.wm_linetime[1] != results->wm_linetime[1])
+                I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
+        if (previous.wm_linetime[2] != results->wm_linetime[2])
+                I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
+
+        if (prev_partitioning != partitioning) {
+                val = I915_READ(WM_MISC);
+                if (partitioning == HSW_DATA_BUF_PART_1_2)
+                        val &= ~WM_MISC_DATA_PARTITION_5_6;
+                else
+                        val |= WM_MISC_DATA_PARTITION_5_6;
+                I915_WRITE(WM_MISC, val);
+        }
+
+        if (prev_enable_fbc_wm != results->enable_fbc_wm) {
+                val = I915_READ(DISP_ARB_CTL);
+                if (results->enable_fbc_wm)
+                        val &= ~DISP_FBC_WM_DIS;
+                else
+                        val |= DISP_FBC_WM_DIS;
+                I915_WRITE(DISP_ARB_CTL, val);
+        }
+
+        if (previous.wm_lp_spr[0] != results->wm_lp_spr[0])
+                I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
+        if (previous.wm_lp_spr[1] != results->wm_lp_spr[1])
+                I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
+        if (previous.wm_lp_spr[2] != results->wm_lp_spr[2])
+                I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
+
+        if (results->wm_lp[0] != 0)
+                I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
+        if (results->wm_lp[1] != 0)
+                I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
+        if (results->wm_lp[2] != 0)
+                I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
+}
+
+static void haswell_update_wm(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct hsw_wm_maximums lp_max_1_2, lp_max_5_6;
+        struct hsw_pipe_wm_parameters params[3];
+        struct hsw_wm_values results_1_2, results_5_6, *best_results;
+        uint32_t wm[5];
+        enum hsw_data_buf_partitioning partitioning;
+
+        hsw_compute_wm_parameters(dev, params, wm, &lp_max_1_2, &lp_max_5_6);
+
+        hsw_compute_wm_results(dev, params, wm, &lp_max_1_2, &results_1_2);
+        if (lp_max_1_2.pri != lp_max_5_6.pri) {
+                hsw_compute_wm_results(dev, params, wm, &lp_max_5_6,
+                                       &results_5_6);
+                best_results = hsw_find_best_result(&results_1_2, &results_5_6);
+        } else {
+                best_results = &results_1_2;
+        }
+
+        partitioning = (best_results == &results_1_2) ?
+                       HSW_DATA_BUF_PART_1_2 : HSW_DATA_BUF_PART_5_6;
+
+        hsw_write_wm_values(dev_priv, best_results, partitioning);
+}
+
+static void haswell_update_sprite_wm(struct drm_device *dev, int pipe,
+                                     uint32_t sprite_width, int pixel_size,
+                                     bool enable)
+{
+        struct drm_plane *plane;
+
+        list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+                struct intel_plane *intel_plane = to_intel_plane(plane);
+
+                if (intel_plane->pipe == pipe) {
+                        intel_plane->wm.enable = enable;
+                        intel_plane->wm.horiz_pixels = sprite_width + 1;
+                        intel_plane->wm.bytes_per_pixel = pixel_size;
+                        break;
+                }
+        }
+
+        haswell_update_wm(dev);
 }
 
 static bool
@@ -2120,7 +2712,8 @@ sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
 }
 
 static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
-                                         uint32_t sprite_width, int pixel_size)
+                                         uint32_t sprite_width, int pixel_size,
+                                         bool enable)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int latency = SNB_READ_WM0_LATENCY() * 100;     /* In unit 0.1us */
@@ -2128,6 +2721,9 @@ static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
         int sprite_wm, reg;
         int ret;
 
+        if (!enable)
+                return;
+
         switch (pipe) {
         case 0:
                 reg = WM0_PIPEA_ILK;
@@ -2146,15 +2742,15 @@ static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
                                             &sandybridge_display_wm_info,
                                             latency, &sprite_wm);
         if (!ret) {
-                DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
-                              pipe);
+                DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
+                              pipe_name(pipe));
                 return;
         }
 
         val = I915_READ(reg);
         val &= ~WM0_PIPE_SPRITE_MASK;
         I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
-        DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
+        DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm);
 
 
         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
@@ -2163,8 +2759,8 @@ static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
                                               SNB_READ_WM1_LATENCY() * 500,
                                               &sprite_wm);
         if (!ret) {
-                DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
-                              pipe);
+                DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
+                              pipe_name(pipe));
                 return;
         }
         I915_WRITE(WM1S_LP_ILK, sprite_wm);
@@ -2179,8 +2775,8 @@ static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
                                               SNB_READ_WM2_LATENCY() * 500,
                                               &sprite_wm);
         if (!ret) {
-                DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
-                              pipe);
+                DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
+                              pipe_name(pipe));
                 return;
         }
         I915_WRITE(WM2S_LP_IVB, sprite_wm);
@@ -2191,8 +2787,8 @@ static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
                                               SNB_READ_WM3_LATENCY() * 500,
                                               &sprite_wm);
         if (!ret) {
-                DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
-                              pipe);
+                DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
+                              pipe_name(pipe));
                 return;
         }
         I915_WRITE(WM3S_LP_IVB, sprite_wm);
@@ -2238,23 +2834,15 @@ void intel_update_watermarks(struct drm_device *dev)
                 dev_priv->display.update_wm(dev);
 }
 
-void intel_update_linetime_watermarks(struct drm_device *dev,
-                int pipe, struct drm_display_mode *mode)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (dev_priv->display.update_linetime_wm)
-                dev_priv->display.update_linetime_wm(dev, pipe, mode);
-}
-
 void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
-                                    uint32_t sprite_width, int pixel_size)
+                                    uint32_t sprite_width, int pixel_size,
+                                    bool enable)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (dev_priv->display.update_sprite_wm)
                 dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
-                                                   pixel_size);
+                                                   pixel_size, enable);
 }
 
 static struct drm_i915_gem_object *
@@ -2481,6 +3069,67 @@ void gen6_set_rps(struct drm_device *dev, u8 val)
         trace_intel_gpu_freq_change(val * 50);
 }
 
+/*
+ * Wait until the previous freq change has completed,
+ * or the timeout elapsed, and then update our notion
+ * of the current GPU frequency.
+ */
+static void vlv_update_rps_cur_delay(struct drm_i915_private *dev_priv)
+{
+        unsigned long timeout = jiffies + msecs_to_jiffies(10);
+        u32 pval;
+
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        do {
+                pval = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
+                if (time_after(jiffies, timeout)) {
+                        DRM_DEBUG_DRIVER("timed out waiting for Punit\n");
+                        break;
+                }
+                udelay(10);
+        } while (pval & 1);
+
+        pval >>= 8;
+
+        if (pval != dev_priv->rps.cur_delay)
+                DRM_DEBUG_DRIVER("Punit overrode GPU freq: %d MHz (%u) requested, but got %d Mhz (%u)\n",
+                                 vlv_gpu_freq(dev_priv->mem_freq, dev_priv->rps.cur_delay),
+                                 dev_priv->rps.cur_delay,
+                                 vlv_gpu_freq(dev_priv->mem_freq, pval), pval);
+
+        dev_priv->rps.cur_delay = pval;
+}
+
+void valleyview_set_rps(struct drm_device *dev, u8 val)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        gen6_rps_limits(dev_priv, &val);
+
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+        WARN_ON(val > dev_priv->rps.max_delay);
+        WARN_ON(val < dev_priv->rps.min_delay);
+
+        vlv_update_rps_cur_delay(dev_priv);
+
+        DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
+                         vlv_gpu_freq(dev_priv->mem_freq,
+                                      dev_priv->rps.cur_delay),
+                         dev_priv->rps.cur_delay,
+                         vlv_gpu_freq(dev_priv->mem_freq, val), val);
+
+        if (val == dev_priv->rps.cur_delay)
+                return;
+
+        vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
+
+        dev_priv->rps.cur_delay = val;
+
+        trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv->mem_freq, val));
+}
+
+
 static void gen6_disable_rps(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2488,6 +3137,25 @@ static void gen6_disable_rps(struct drm_device *dev)
         I915_WRITE(GEN6_RC_CONTROL, 0);
         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
         I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
+        I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
+        /* Complete PM interrupt masking here doesn't race with the rps work
+         * item again unmasking PM interrupts because that is using a different
+         * register (PMIMR) to mask PM interrupts. The only risk is in leaving
+         * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
+
+        spin_lock_irq(&dev_priv->rps.lock);
+        dev_priv->rps.pm_iir = 0;
+        spin_unlock_irq(&dev_priv->rps.lock);
+
+        I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
+}
+
+static void valleyview_disable_rps(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(GEN6_RC_CONTROL, 0);
+        I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
         I915_WRITE(GEN6_PMIER, 0);
         /* Complete PM interrupt masking here doesn't race with the rps work
          * item again unmasking PM interrupts because that is using a different
@@ -2499,6 +3167,11 @@ static void gen6_disable_rps(struct drm_device *dev)
         spin_unlock_irq(&dev_priv->rps.lock);
 
         I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
+
+        if (dev_priv->vlv_pctx) {
+                drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
+                dev_priv->vlv_pctx = NULL;
+        }
 }
 
 int intel_enable_rc6(const struct drm_device *dev)
@@ -2655,12 +3328,15 @@ static void gen6_enable_rps(struct drm_device *dev)
         gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
 
         /* requires MSI enabled */
-        I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
+        I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) | GEN6_PM_RPS_EVENTS);
         spin_lock_irq(&dev_priv->rps.lock);
-        WARN_ON(dev_priv->rps.pm_iir != 0);
-        I915_WRITE(GEN6_PMIMR, 0);
+        /* FIXME: Our interrupt enabling sequence is bonghits.
+         * dev_priv->rps.pm_iir really should be 0 here. */
+        dev_priv->rps.pm_iir = 0;
+        I915_WRITE(GEN6_PMIMR, I915_READ(GEN6_PMIMR) & ~GEN6_PM_RPS_EVENTS);
+        I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
         spin_unlock_irq(&dev_priv->rps.lock);
-        /* enable all PM interrupts */
+        /* unmask all PM interrupts */
         I915_WRITE(GEN6_PMINTRMSK, 0);
 
         rc6vids = 0;
@@ -2742,6 +3418,207 @@ static void gen6_update_ring_freq(struct drm_device *dev)
         }
 }
 
+int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
+{
+        u32 val, rp0;
+
+        val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
+
+        rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
+        /* Clamp to max */
+        rp0 = min_t(u32, rp0, 0xea);
+
+        return rp0;
+}
+
+static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
+{
+        u32 val, rpe;
+
+        val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
+        rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
+        val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
+        rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
+
+        return rpe;
+}
+
+int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
+{
+        return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
+}
+
+static void vlv_rps_timer_work(struct work_struct *work)
+{
+        drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
+                                                    rps.vlv_work.work);
+
+        /*
+         * Timer fired, we must be idle.  Drop to min voltage state.
+         * Note: we use RPe here since it should match the
+         * Vmin we were shooting for.  That should give us better
+         * perf when we come back out of RC6 than if we used the
+         * min freq available.
+         */
+        mutex_lock(&dev_priv->rps.hw_lock);
+        if (dev_priv->rps.cur_delay > dev_priv->rps.rpe_delay)
+                valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
+        mutex_unlock(&dev_priv->rps.hw_lock);
+}
+
+static void valleyview_setup_pctx(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *pctx;
+        unsigned long pctx_paddr;
+        u32 pcbr;
+        int pctx_size = 24*1024;
+
+        pcbr = I915_READ(VLV_PCBR);
+        if (pcbr) {
+                /* BIOS set it up already, grab the pre-alloc'd space */
+                int pcbr_offset;
+
+                pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
+                pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
+                                                                      pcbr_offset,
+                                                                      -1,
+                                                                      pctx_size);
+                goto out;
+        }
+
+        /*
+         * From the Gunit register HAS:
+         * The Gfx driver is expected to program this register and ensure
+         * proper allocation within Gfx stolen memory.  For example, this
+         * register should be programmed such than the PCBR range does not
+         * overlap with other ranges, such as the frame buffer, protected
+         * memory, or any other relevant ranges.
+         */
+        pctx = i915_gem_object_create_stolen(dev, pctx_size);
+        if (!pctx) {
+                DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
+                return;
+        }
+
+        pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
+        I915_WRITE(VLV_PCBR, pctx_paddr);
+
+out:
+        dev_priv->vlv_pctx = pctx;
+}
+
+static void valleyview_enable_rps(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        u32 gtfifodbg, val;
+        int i;
+
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        if ((gtfifodbg = I915_READ(GTFIFODBG))) {
+                DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
+                I915_WRITE(GTFIFODBG, gtfifodbg);
+        }
+
+        valleyview_setup_pctx(dev);
+
+        gen6_gt_force_wake_get(dev_priv);
+
+        I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
+        I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
+        I915_WRITE(GEN6_RP_UP_EI, 66000);
+        I915_WRITE(GEN6_RP_DOWN_EI, 350000);
+
+        I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
+
+        I915_WRITE(GEN6_RP_CONTROL,
+                   GEN6_RP_MEDIA_TURBO |
+                   GEN6_RP_MEDIA_HW_NORMAL_MODE |
+                   GEN6_RP_MEDIA_IS_GFX |
+                   GEN6_RP_ENABLE |
+                   GEN6_RP_UP_BUSY_AVG |
+                   GEN6_RP_DOWN_IDLE_CONT);
+
+        I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
+        I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
+        I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
+
+        for_each_ring(ring, dev_priv, i)
+                I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
+
+        I915_WRITE(GEN6_RC6_THRESHOLD, 0xc350);
+
+        /* allows RC6 residency counter to work */
+        I915_WRITE(0x138104, _MASKED_BIT_ENABLE(0x3));
+        I915_WRITE(GEN6_RC_CONTROL,
+                   GEN7_RC_CTL_TO_MODE);
+
+        val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
+        switch ((val >> 6) & 3) {
+        case 0:
+        case 1:
+                dev_priv->mem_freq = 800;
+                break;
+        case 2:
+                dev_priv->mem_freq = 1066;
+                break;
+        case 3:
+                dev_priv->mem_freq = 1333;
+                break;
+        }
+        DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
+
+        DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
+        DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
+
+        dev_priv->rps.cur_delay = (val >> 8) & 0xff;
+        DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
+                         vlv_gpu_freq(dev_priv->mem_freq,
+                                      dev_priv->rps.cur_delay),
+                         dev_priv->rps.cur_delay);
+
+        dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
+        dev_priv->rps.hw_max = dev_priv->rps.max_delay;
+        DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
+                         vlv_gpu_freq(dev_priv->mem_freq,
+                                      dev_priv->rps.max_delay),
+                         dev_priv->rps.max_delay);
+
+        dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv);
+        DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
+                         vlv_gpu_freq(dev_priv->mem_freq,
+                                      dev_priv->rps.rpe_delay),
+                         dev_priv->rps.rpe_delay);
+
+        dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv);
+        DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
+                         vlv_gpu_freq(dev_priv->mem_freq,
+                                      dev_priv->rps.min_delay),
+                         dev_priv->rps.min_delay);
+
+        DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
+                         vlv_gpu_freq(dev_priv->mem_freq,
+                                      dev_priv->rps.rpe_delay),
+                         dev_priv->rps.rpe_delay);
+
+        INIT_DELAYED_WORK(&dev_priv->rps.vlv_work, vlv_rps_timer_work);
+
+        valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
+
+        /* requires MSI enabled */
+        I915_WRITE(GEN6_PMIER, GEN6_PM_RPS_EVENTS);
+        spin_lock_irq(&dev_priv->rps.lock);
+        WARN_ON(dev_priv->rps.pm_iir != 0);
+        I915_WRITE(GEN6_PMIMR, 0);
+        spin_unlock_irq(&dev_priv->rps.lock);
+        /* enable all PM interrupts */
+        I915_WRITE(GEN6_PMINTRMSK, 0);
+
+        gen6_gt_force_wake_put(dev_priv);
+}
+
 void ironlake_teardown_rc6(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3465,13 +4342,22 @@ void intel_disable_gt_powersave(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        /* Interrupts should be disabled already to avoid re-arming. */
+        WARN_ON(dev->irq_enabled);
+
         if (IS_IRONLAKE_M(dev)) {
                 ironlake_disable_drps(dev);
                 ironlake_disable_rc6(dev);
-        } else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
+        } else if (INTEL_INFO(dev)->gen >= 6) {
                 cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
+                cancel_work_sync(&dev_priv->rps.work);
+                if (IS_VALLEYVIEW(dev))
+                        cancel_delayed_work_sync(&dev_priv->rps.vlv_work);
                 mutex_lock(&dev_priv->rps.hw_lock);
-                gen6_disable_rps(dev);
+                if (IS_VALLEYVIEW(dev))
+                        valleyview_disable_rps(dev);
+                else
+                        gen6_disable_rps(dev);
                 mutex_unlock(&dev_priv->rps.hw_lock);
         }
 }
@@ -3484,8 +4370,13 @@ static void intel_gen6_powersave_work(struct work_struct *work)
         struct drm_device *dev = dev_priv->dev;
 
         mutex_lock(&dev_priv->rps.hw_lock);
-        gen6_enable_rps(dev);
-        gen6_update_ring_freq(dev);
+
+        if (IS_VALLEYVIEW(dev)) {
+                valleyview_enable_rps(dev);
+        } else {
+                gen6_enable_rps(dev);
+                gen6_update_ring_freq(dev);
+        }
         mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
@@ -3497,7 +4388,7 @@ void intel_enable_gt_powersave(struct drm_device *dev)
                 ironlake_enable_drps(dev);
                 ironlake_enable_rc6(dev);
                 intel_init_emon(dev);
-        } else if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
+        } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
                 /*
                  * PCU communication is slow and this doesn't need to be
                  * done at any specific time, so do this out of our fast path
@@ -3520,6 +4411,19 @@ static void ibx_init_clock_gating(struct drm_device *dev)
         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 }
 
+static void g4x_disable_trickle_feed(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
+
+        for_each_pipe(pipe) {
+                I915_WRITE(DSPCNTR(pipe),
+                           I915_READ(DSPCNTR(pipe)) |
+                           DISPPLANE_TRICKLE_FEED_DISABLE);
+                intel_flush_display_plane(dev_priv, pipe);
+        }
+}
+
 static void ironlake_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3579,10 +4483,12 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
                    _3D_CHICKEN2_WM_READ_PIPELINED);
 
-        /* WaDisableRenderCachePipelinedFlush */
+        /* WaDisableRenderCachePipelinedFlush:ilk */
         I915_WRITE(CACHE_MODE_0,
                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
 
+        g4x_disable_trickle_feed(dev);
+
         ibx_init_clock_gating(dev);
 }
 
@@ -3607,7 +4513,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
                 val = I915_READ(TRANS_CHICKEN2(pipe));
                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
-                if (dev_priv->fdi_rx_polarity_inverted)
+                if (dev_priv->vbt.fdi_rx_polarity_inverted)
                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
@@ -3637,7 +4543,6 @@ static void gen6_check_mch_setup(struct drm_device *dev)
 static void gen6_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe;
         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
@@ -3646,11 +4551,11 @@ static void gen6_init_clock_gating(struct drm_device *dev)
                    I915_READ(ILK_DISPLAY_CHICKEN2) |
                    ILK_ELPIN_409_SELECT);
 
-        /* WaDisableHiZPlanesWhenMSAAEnabled */
+        /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
         I915_WRITE(_3D_CHICKEN,
                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
 
-        /* WaSetupGtModeTdRowDispatch */
+        /* WaSetupGtModeTdRowDispatch:snb */
         if (IS_SNB_GT1(dev))
                 I915_WRITE(GEN6_GT_MODE,
                            _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
@@ -3677,8 +4582,8 @@ static void gen6_init_clock_gating(struct drm_device *dev)
          * According to the spec, bit 11 (RCCUNIT) must also be set,
          * but we didn't debug actual testcases to find it out.
          *
-         * Also apply WaDisableVDSUnitClockGating and
-         * WaDisableRCPBUnitClockGating.
+         * Also apply WaDisableVDSUnitClockGating:snb and
+         * WaDisableRCPBUnitClockGating:snb.
          */
         I915_WRITE(GEN6_UCGCTL2,
                    GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
@@ -3709,16 +4614,11 @@ static void gen6_init_clock_gating(struct drm_device *dev)
                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 
-        /* WaMbcDriverBootEnable */
+        /* WaMbcDriverBootEnable:snb */
         I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
                    GEN6_MBCTL_ENABLE_BOOT_FETCH);
 
-        for_each_pipe(pipe) {
-                I915_WRITE(DSPCNTR(pipe),
-                           I915_READ(DSPCNTR(pipe)) |
-                           DISPPLANE_TRICKLE_FEED_DISABLE);
-                intel_flush_display_plane(dev_priv, pipe);
-        }
+        g4x_disable_trickle_feed(dev);
 
         /* The default value should be 0x200 according to docs, but the two
          * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
@@ -3739,7 +4639,6 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
         reg |= GEN7_FF_VS_SCHED_HW;
         reg |= GEN7_FF_DS_SCHED_HW;
 
-        /* WaVSRefCountFullforceMissDisable */
         if (IS_HASWELL(dev_priv->dev))
                 reg &= ~GEN7_FF_VS_REF_CNT_FFME;
 
@@ -3758,65 +4657,72 @@ static void lpt_init_clock_gating(struct drm_device *dev)
                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
                            I915_READ(SOUTH_DSPCLK_GATE_D) |
                            PCH_LP_PARTITION_LEVEL_DISABLE);
+
+        /* WADPOClockGatingDisable:hsw */
+        I915_WRITE(_TRANSA_CHICKEN1,
+                   I915_READ(_TRANSA_CHICKEN1) |
+                   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
+}
+
+static void lpt_suspend_hw(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
+                uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
+
+                val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
+                I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
+        }
 }
 
 static void haswell_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe;
 
         I915_WRITE(WM3_LP_ILK, 0);
         I915_WRITE(WM2_LP_ILK, 0);
         I915_WRITE(WM1_LP_ILK, 0);
 
         /* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-         * This implements the WaDisableRCZUnitClockGating workaround.
+         * This implements the WaDisableRCZUnitClockGating:hsw workaround.
          */
         I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 
-        /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
+        /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 
-        /* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
+        /* WaApplyL3ControlAndL3ChickenMode:hsw */
         I915_WRITE(GEN7_L3CNTLREG1,
                         GEN7_WA_FOR_GEN7_L3_CONTROL);
         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
                         GEN7_WA_L3_CHICKEN_MODE);
 
-        /* This is required by WaCatErrorRejectionIssue */
+        /* This is required by WaCatErrorRejectionIssue:hsw */
         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-        for_each_pipe(pipe) {
-                I915_WRITE(DSPCNTR(pipe),
-                           I915_READ(DSPCNTR(pipe)) |
-                           DISPPLANE_TRICKLE_FEED_DISABLE);
-                intel_flush_display_plane(dev_priv, pipe);
-        }
+        g4x_disable_trickle_feed(dev);
 
+        /* WaVSRefCountFullforceMissDisable:hsw */
         gen7_setup_fixed_func_scheduler(dev_priv);
 
-        /* WaDisable4x2SubspanOptimization */
+        /* WaDisable4x2SubspanOptimization:hsw */
         I915_WRITE(CACHE_MODE_1,
                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 
-        /* WaMbcDriverBootEnable */
+        /* WaMbcDriverBootEnable:hsw */
         I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
                    GEN6_MBCTL_ENABLE_BOOT_FETCH);
 
-        /* WaSwitchSolVfFArbitrationPriority */
+        /* WaSwitchSolVfFArbitrationPriority:hsw */
         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
 
-        /* XXX: This is a workaround for early silicon revisions and should be
-         * removed later.
-         */
-        I915_WRITE(WM_DBG,
-                        I915_READ(WM_DBG) |
-                        WM_DBG_DISALLOW_MULTIPLE_LP |
-                        WM_DBG_DISALLOW_SPRITE |
-                        WM_DBG_DISALLOW_MAXFIFO);
+        /* WaRsPkgCStateDisplayPMReq:hsw */
+        I915_WRITE(CHICKEN_PAR1_1,
+                   I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
 
         lpt_init_clock_gating(dev);
 }
@@ -3824,7 +4730,6 @@ static void haswell_init_clock_gating(struct drm_device *dev)
 static void ivybridge_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe;
         uint32_t snpcr;
 
         I915_WRITE(WM3_LP_ILK, 0);
@@ -3833,16 +4738,16 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 
         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 
-        /* WaDisableEarlyCull */
+        /* WaDisableEarlyCull:ivb */
         I915_WRITE(_3D_CHICKEN3,
                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
 
-        /* WaDisableBackToBackFlipFix */
+        /* WaDisableBackToBackFlipFix:ivb */
         I915_WRITE(IVB_CHICKEN3,
                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
-        /* WaDisablePSDDualDispatchEnable */
+        /* WaDisablePSDDualDispatchEnable:ivb */
         if (IS_IVB_GT1(dev))
                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
@@ -3850,11 +4755,11 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
 
-        /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
+        /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 
-        /* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
+        /* WaApplyL3ControlAndL3ChickenMode:ivb */
         I915_WRITE(GEN7_L3CNTLREG1,
                         GEN7_WA_FOR_GEN7_L3_CONTROL);
         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
@@ -3867,7 +4772,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 
 
-        /* WaForceL3Serialization */
+        /* WaForceL3Serialization:ivb */
         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
 
@@ -3882,31 +4787,27 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
          * but we didn't debug actual testcases to find it out.
          *
          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-         * This implements the WaDisableRCZUnitClockGating workaround.
+         * This implements the WaDisableRCZUnitClockGating:ivb workaround.
          */
         I915_WRITE(GEN6_UCGCTL2,
                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 
-        /* This is required by WaCatErrorRejectionIssue */
+        /* This is required by WaCatErrorRejectionIssue:ivb */
         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-        for_each_pipe(pipe) {
-                I915_WRITE(DSPCNTR(pipe),
-                           I915_READ(DSPCNTR(pipe)) |
-                           DISPPLANE_TRICKLE_FEED_DISABLE);
-                intel_flush_display_plane(dev_priv, pipe);
-        }
+        g4x_disable_trickle_feed(dev);
 
-        /* WaMbcDriverBootEnable */
+        /* WaMbcDriverBootEnable:ivb */
         I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
                    GEN6_MBCTL_ENABLE_BOOT_FETCH);
 
+        /* WaVSRefCountFullforceMissDisable:ivb */
         gen7_setup_fixed_func_scheduler(dev_priv);
 
-        /* WaDisable4x2SubspanOptimization */
+        /* WaDisable4x2SubspanOptimization:ivb */
         I915_WRITE(CACHE_MODE_1,
                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 
@@ -3924,54 +4825,45 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 static void valleyview_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe;
-
-        I915_WRITE(WM3_LP_ILK, 0);
-        I915_WRITE(WM2_LP_ILK, 0);
-        I915_WRITE(WM1_LP_ILK, 0);
 
-        I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+        I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
 
-        /* WaDisableEarlyCull */
+        /* WaDisableEarlyCull:vlv */
         I915_WRITE(_3D_CHICKEN3,
                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
 
-        /* WaDisableBackToBackFlipFix */
+        /* WaDisableBackToBackFlipFix:vlv */
         I915_WRITE(IVB_CHICKEN3,
                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
-        /* WaDisablePSDDualDispatchEnable */
+        /* WaDisablePSDDualDispatchEnable:vlv */
         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
 
-        /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
+        /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 
-        /* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
+        /* WaApplyL3ControlAndL3ChickenMode:vlv */
         I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS);
         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 
-        /* WaForceL3Serialization */
+        /* WaForceL3Serialization:vlv */
         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
 
-        /* WaDisableDopClockGating */
+        /* WaDisableDopClockGating:vlv */
         I915_WRITE(GEN7_ROW_CHICKEN2,
                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 
-        /* WaForceL3Serialization */
-        I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
-                   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
-
-        /* This is required by WaCatErrorRejectionIssue */
+        /* This is required by WaCatErrorRejectionIssue:vlv */
         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-        /* WaMbcDriverBootEnable */
+        /* WaMbcDriverBootEnable:vlv */
         I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) |
                    GEN6_MBCTL_ENABLE_BOOT_FETCH);
 
@@ -3987,10 +4879,10 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
          * but we didn't debug actual testcases to find it out.
          *
          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-         * This implements the WaDisableRCZUnitClockGating workaround.
+         * This implements the WaDisableRCZUnitClockGating:vlv workaround.
          *
-         * Also apply WaDisableVDSUnitClockGating and
-         * WaDisableRCPBUnitClockGating.
+         * Also apply WaDisableVDSUnitClockGating:vlv and
+         * WaDisableRCPBUnitClockGating:vlv.
          */
         I915_WRITE(GEN6_UCGCTL2,
                    GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
@@ -4001,18 +4893,13 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
 
         I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 
-        for_each_pipe(pipe) {
-                I915_WRITE(DSPCNTR(pipe),
-                           I915_READ(DSPCNTR(pipe)) |
-                           DISPPLANE_TRICKLE_FEED_DISABLE);
-                intel_flush_display_plane(dev_priv, pipe);
-        }
+        I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 
         I915_WRITE(CACHE_MODE_1,
                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 
         /*
-         * WaDisableVLVClockGating_VBIIssue
+         * WaDisableVLVClockGating_VBIIssue:vlv
          * Disable clock gating on th GCFG unit to prevent a delay
          * in the reporting of vblank events.
          */
@@ -4048,6 +4935,8 @@ static void g4x_init_clock_gating(struct drm_device *dev)
         /* WaDisableRenderCachePipelinedFlush */
         I915_WRITE(CACHE_MODE_0,
                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
+
+        g4x_disable_trickle_feed(dev);
 }
 
 static void crestline_init_clock_gating(struct drm_device *dev)
@@ -4059,6 +4948,8 @@ static void crestline_init_clock_gating(struct drm_device *dev)
         I915_WRITE(DSPCLK_GATE_D, 0);
         I915_WRITE(RAMCLK_GATE_D, 0);
         I915_WRITE16(DEUC, 0);
+        I915_WRITE(MI_ARB_STATE,
+                   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void broadwater_init_clock_gating(struct drm_device *dev)
@@ -4071,6 +4962,8 @@ static void broadwater_init_clock_gating(struct drm_device *dev)
                    I965_ISC_CLOCK_GATE_DISABLE |
                    I965_FBC_CLOCK_GATE_DISABLE);
         I915_WRITE(RENCLK_GATE_D2, 0);
+        I915_WRITE(MI_ARB_STATE,
+                   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void gen3_init_clock_gating(struct drm_device *dev)
@@ -4110,34 +5003,50 @@ void intel_init_clock_gating(struct drm_device *dev)
         dev_priv->display.init_clock_gating(dev);
 }
 
+void intel_suspend_hw(struct drm_device *dev)
+{
+        if (HAS_PCH_LPT(dev))
+                lpt_suspend_hw(dev);
+}
+
 /**
  * We should only use the power well if we explicitly asked the hardware to
  * enable it, so check if it's enabled and also check if we've requested it to
  * be enabled.
  */
-bool intel_using_power_well(struct drm_device *dev)
+bool intel_display_power_enabled(struct drm_device *dev,
+                                 enum intel_display_power_domain domain)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (IS_HASWELL(dev))
+        if (!HAS_POWER_WELL(dev))
+                return true;
+
+        switch (domain) {
+        case POWER_DOMAIN_PIPE_A:
+        case POWER_DOMAIN_TRANSCODER_EDP:
+                return true;
+        case POWER_DOMAIN_PIPE_B:
+        case POWER_DOMAIN_PIPE_C:
+        case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
+        case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
+        case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
+        case POWER_DOMAIN_TRANSCODER_A:
+        case POWER_DOMAIN_TRANSCODER_B:
+        case POWER_DOMAIN_TRANSCODER_C:
                 return I915_READ(HSW_PWR_WELL_DRIVER) ==
                        (HSW_PWR_WELL_ENABLE | HSW_PWR_WELL_STATE);
-        else
-                return true;
+        default:
+                BUG();
+        }
 }
 
-void intel_set_power_well(struct drm_device *dev, bool enable)
+static void __intel_set_power_well(struct drm_device *dev, bool enable)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         bool is_enabled, enable_requested;
         uint32_t tmp;
 
-        if (!HAS_POWER_WELL(dev))
-                return;
-
-        if (!i915_disable_power_well && !enable)
-                return;
-
         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
         is_enabled = tmp & HSW_PWR_WELL_STATE;
         enable_requested = tmp & HSW_PWR_WELL_ENABLE;
@@ -4160,6 +5069,79 @@ void intel_set_power_well(struct drm_device *dev, bool enable)
         }
 }
 
+static struct i915_power_well *hsw_pwr;
+
+/* Display audio driver power well request */
+void i915_request_power_well(void)
+{
+        if (WARN_ON(!hsw_pwr))
+                return;
+
+        spin_lock_irq(&hsw_pwr->lock);
+        if (!hsw_pwr->count++ &&
+                        !hsw_pwr->i915_request)
+                __intel_set_power_well(hsw_pwr->device, true);
+        spin_unlock_irq(&hsw_pwr->lock);
+}
+EXPORT_SYMBOL_GPL(i915_request_power_well);
+
+/* Display audio driver power well release */
+void i915_release_power_well(void)
+{
+        if (WARN_ON(!hsw_pwr))
+                return;
+
+        spin_lock_irq(&hsw_pwr->lock);
+        WARN_ON(!hsw_pwr->count);
+        if (!--hsw_pwr->count &&
+                       !hsw_pwr->i915_request)
+                __intel_set_power_well(hsw_pwr->device, false);
+        spin_unlock_irq(&hsw_pwr->lock);
+}
+EXPORT_SYMBOL_GPL(i915_release_power_well);
+
+int i915_init_power_well(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        hsw_pwr = &dev_priv->power_well;
+
+        hsw_pwr->device = dev;
+        spin_lock_init(&hsw_pwr->lock);
+        hsw_pwr->count = 0;
+
+        return 0;
+}
+
+void i915_remove_power_well(struct drm_device *dev)
+{
+        hsw_pwr = NULL;
+}
+
+void intel_set_power_well(struct drm_device *dev, bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct i915_power_well *power_well = &dev_priv->power_well;
+
+        if (!HAS_POWER_WELL(dev))
+                return;
+
+        if (!i915_disable_power_well && !enable)
+                return;
+
+        spin_lock_irq(&power_well->lock);
+        power_well->i915_request = enable;
+
+        /* only reject "disable" power well request */
+        if (power_well->count && !enable) {
+                spin_unlock_irq(&power_well->lock);
+                return;
+        }
+
+        __intel_set_power_well(dev, enable);
+        spin_unlock_irq(&power_well->lock);
+}
+
 /*
  * Starting with Haswell, we have a "Power Down Well" that can be turned off
  * when not needed anymore. We have 4 registers that can request the power well
@@ -4190,7 +5172,12 @@ void intel_init_pm(struct drm_device *dev)
         if (I915_HAS_FBC(dev)) {
                 if (HAS_PCH_SPLIT(dev)) {
                         dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
-                        dev_priv->display.enable_fbc = ironlake_enable_fbc;
+                        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+                                dev_priv->display.enable_fbc =
+                                        gen7_enable_fbc;
+                        else
+                                dev_priv->display.enable_fbc =
+                                        ironlake_enable_fbc;
                         dev_priv->display.disable_fbc = ironlake_disable_fbc;
                 } else if (IS_GM45(dev)) {
                         dev_priv->display.fbc_enabled = g4x_fbc_enabled;
@@ -4242,10 +5229,10 @@ void intel_init_pm(struct drm_device *dev)
                         }
                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
                 } else if (IS_HASWELL(dev)) {
-                        if (SNB_READ_WM0_LATENCY()) {
-                                dev_priv->display.update_wm = sandybridge_update_wm;
-                                dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
-                                dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
+                        if (I915_READ64(MCH_SSKPD)) {
+                                dev_priv->display.update_wm = haswell_update_wm;
+                                dev_priv->display.update_sprite_wm =
+                                        haswell_update_sprite_wm;
                         } else {
                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
                                               "Disable CxSR\n");
@@ -4340,6 +5327,7 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
                             FORCEWAKE_ACK_TIMEOUT_MS))
                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 
+        /* WaRsForcewakeWaitTC0:snb */
         __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
@@ -4371,6 +5359,7 @@ static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
                             FORCEWAKE_ACK_TIMEOUT_MS))
                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 
+        /* WaRsForcewakeWaitTC0:ivb,hsw */
         __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
@@ -4474,6 +5463,7 @@ static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
                             FORCEWAKE_ACK_TIMEOUT_MS))
                 DRM_ERROR("Timed out waiting for media to ack forcewake request.\n");
 
+        /* WaRsForcewakeWaitTC0:vlv */
         __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
@@ -4568,55 +5558,58 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
         return 0;
 }
 
-static int vlv_punit_rw(struct drm_i915_private *dev_priv, u8 opcode,
-                        u8 addr, u32 *val)
+int vlv_gpu_freq(int ddr_freq, int val)
 {
-        u32 cmd, devfn, port, be, bar;
-
-        bar = 0;
-        be = 0xf;
-        port = IOSF_PORT_PUNIT;
-        devfn = PCI_DEVFN(2, 0);
+        int mult, base;
 
-        cmd = (devfn << IOSF_DEVFN_SHIFT) | (opcode << IOSF_OPCODE_SHIFT) |
-                (port << IOSF_PORT_SHIFT) | (be << IOSF_BYTE_ENABLES_SHIFT) |
-                (bar << IOSF_BAR_SHIFT);
-
-        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
-
-        if (I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) {
-                DRM_DEBUG_DRIVER("warning: pcode (%s) mailbox access failed\n",
-                                 opcode == PUNIT_OPCODE_REG_READ ?
-                                 "read" : "write");
-                return -EAGAIN;
+        switch (ddr_freq) {
+        case 800:
+                mult = 20;
+                base = 120;
+                break;
+        case 1066:
+                mult = 22;
+                base = 133;
+                break;
+        case 1333:
+                mult = 21;
+                base = 125;
+                break;
+        default:
+                return -1;
         }
 
-        I915_WRITE(VLV_IOSF_ADDR, addr);
-        if (opcode == PUNIT_OPCODE_REG_WRITE)
-                I915_WRITE(VLV_IOSF_DATA, *val);
-        I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
+        return ((val - 0xbd) * mult) + base;
+}
 
-        if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0,
-                     500)) {
-                DRM_ERROR("timeout waiting for pcode %s (%d) to finish\n",
-                          opcode == PUNIT_OPCODE_REG_READ ? "read" : "write",
-                          addr);
-                return -ETIMEDOUT;
+int vlv_freq_opcode(int ddr_freq, int val)
+{
+        int mult, base;
+
+        switch (ddr_freq) {
+        case 800:
+                mult = 20;
+                base = 120;
+                break;
+        case 1066:
+                mult = 22;
+                base = 133;
+                break;
+        case 1333:
+                mult = 21;
+                base = 125;
+                break;
+        default:
+                return -1;
         }
 
-        if (opcode == PUNIT_OPCODE_REG_READ)
-                *val = I915_READ(VLV_IOSF_DATA);
-        I915_WRITE(VLV_IOSF_DATA, 0);
+        val /= mult;
+        val -= base / mult;
+        val += 0xbd;
 
-        return 0;
-}
+        if (val > 0xea)
+                val = 0xea;
 
-int valleyview_punit_read(struct drm_i915_private *dev_priv, u8 addr, u32 *val)
-{
-        return vlv_punit_rw(dev_priv, PUNIT_OPCODE_REG_READ, addr, val);
+        return val;
 }
 
-int valleyview_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val)
-{
-        return vlv_punit_rw(dev_priv, PUNIT_OPCODE_REG_WRITE, addr, &val);
-}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 1d5d613eb6be..e51ab552046c 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -280,6 +280,27 @@ gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
         return 0;
 }
 
+static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value)
+{
+        int ret;
+
+        if (!ring->fbc_dirty)
+                return 0;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+        intel_ring_emit(ring, MI_NOOP);
+        /* WaFbcNukeOn3DBlt:ivb/hsw */
+        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+        intel_ring_emit(ring, MSG_FBC_REND_STATE);
+        intel_ring_emit(ring, value);
+        intel_ring_advance(ring);
+
+        ring->fbc_dirty = false;
+        return 0;
+}
+
 static int
 gen7_render_ring_flush(struct intel_ring_buffer *ring,
                        u32 invalidate_domains, u32 flush_domains)
@@ -336,6 +357,9 @@ gen7_render_ring_flush(struct intel_ring_buffer *ring,
         intel_ring_emit(ring, 0);
         intel_ring_advance(ring);
 
+        if (flush_domains)
+                return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
+
         return 0;
 }
 
@@ -429,6 +453,8 @@ static int init_ring_common(struct intel_ring_buffer *ring)
                 ring->last_retired_head = -1;
         }
 
+        memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
+
 out:
         if (HAS_FORCE_WAKE(dev))
                 gen6_gt_force_wake_put(dev_priv);
@@ -464,9 +490,11 @@ init_pipe_control(struct intel_ring_buffer *ring)
                 goto err_unref;
 
         pc->gtt_offset = obj->gtt_offset;
-        pc->cpu_page =  kmap(sg_page(obj->pages->sgl));
-        if (pc->cpu_page == NULL)
+        pc->cpu_page = kmap(sg_page(obj->pages->sgl));
+        if (pc->cpu_page == NULL) {
+                ret = -ENOMEM;
                 goto err_unpin;
+        }
 
         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
                          ring->name, pc->gtt_offset);
@@ -515,6 +543,8 @@ static int init_render_ring(struct intel_ring_buffer *ring)
         /* We need to disable the AsyncFlip performance optimisations in order
          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
          * programmed to '1' on all products.
+         *
+         * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
          */
         if (INTEL_INFO(dev)->gen >= 6)
                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
@@ -556,7 +586,7 @@ static int init_render_ring(struct intel_ring_buffer *ring)
                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 
         if (HAS_L3_GPU_CACHE(dev))
-                I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
+                I915_WRITE_IMR(ring, ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 
         return ret;
 }
@@ -578,9 +608,16 @@ static void
 update_mboxes(struct intel_ring_buffer *ring,
               u32 mmio_offset)
 {
+/* NB: In order to be able to do semaphore MBOX updates for varying number
+ * of rings, it's easiest if we round up each individual update to a
+ * multiple of 2 (since ring updates must always be a multiple of 2)
+ * even though the actual update only requires 3 dwords.
+ */
+#define MBOX_UPDATE_DWORDS 4
         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
         intel_ring_emit(ring, mmio_offset);
         intel_ring_emit(ring, ring->outstanding_lazy_request);
+        intel_ring_emit(ring, MI_NOOP);
 }
 
 /**
@@ -595,19 +632,24 @@ update_mboxes(struct intel_ring_buffer *ring,
 static int
 gen6_add_request(struct intel_ring_buffer *ring)
 {
-        u32 mbox1_reg;
-        u32 mbox2_reg;
-        int ret;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *useless;
+        int i, ret;
 
-        ret = intel_ring_begin(ring, 10);
+        ret = intel_ring_begin(ring, ((I915_NUM_RINGS-1) *
+                                      MBOX_UPDATE_DWORDS) +
+                                      4);
         if (ret)
                 return ret;
+#undef MBOX_UPDATE_DWORDS
 
-        mbox1_reg = ring->signal_mbox[0];
-        mbox2_reg = ring->signal_mbox[1];
+        for_each_ring(useless, dev_priv, i) {
+                u32 mbox_reg = ring->signal_mbox[i];
+                if (mbox_reg != GEN6_NOSYNC)
+                        update_mboxes(ring, mbox_reg);
+        }
 
-        update_mboxes(ring, mbox1_reg);
-        update_mboxes(ring, mbox2_reg);
         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
         intel_ring_emit(ring, ring->outstanding_lazy_request);
@@ -779,7 +821,7 @@ gen5_ring_get_irq(struct intel_ring_buffer *ring)
                 return false;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (ring->irq_refcount++ == 0) {
+        if (ring->irq_refcount.gt++ == 0) {
                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
                 POSTING_READ(GTIMR);
@@ -797,7 +839,7 @@ gen5_ring_put_irq(struct intel_ring_buffer *ring)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--ring->irq_refcount == 0) {
+        if (--ring->irq_refcount.gt == 0) {
                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
                 POSTING_READ(GTIMR);
@@ -816,7 +858,7 @@ i9xx_ring_get_irq(struct intel_ring_buffer *ring)
                 return false;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (ring->irq_refcount++ == 0) {
+        if (ring->irq_refcount.gt++ == 0) {
                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
                 I915_WRITE(IMR, dev_priv->irq_mask);
                 POSTING_READ(IMR);
@@ -834,7 +876,7 @@ i9xx_ring_put_irq(struct intel_ring_buffer *ring)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--ring->irq_refcount == 0) {
+        if (--ring->irq_refcount.gt == 0) {
                 dev_priv->irq_mask |= ring->irq_enable_mask;
                 I915_WRITE(IMR, dev_priv->irq_mask);
                 POSTING_READ(IMR);
@@ -853,7 +895,7 @@ i8xx_ring_get_irq(struct intel_ring_buffer *ring)
                 return false;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (ring->irq_refcount++ == 0) {
+        if (ring->irq_refcount.gt++ == 0) {
                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
                 I915_WRITE16(IMR, dev_priv->irq_mask);
                 POSTING_READ16(IMR);
@@ -871,7 +913,7 @@ i8xx_ring_put_irq(struct intel_ring_buffer *ring)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--ring->irq_refcount == 0) {
+        if (--ring->irq_refcount.gt == 0) {
                 dev_priv->irq_mask |= ring->irq_enable_mask;
                 I915_WRITE16(IMR, dev_priv->irq_mask);
                 POSTING_READ16(IMR);
@@ -899,6 +941,9 @@ void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
                 case VCS:
                         mmio = BSD_HWS_PGA_GEN7;
                         break;
+                case VECS:
+                        mmio = VEBOX_HWS_PGA_GEN7;
+                        break;
                 }
         } else if (IS_GEN6(ring->dev)) {
                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
@@ -961,10 +1006,11 @@ gen6_ring_get_irq(struct intel_ring_buffer *ring)
         gen6_gt_force_wake_get(dev_priv);
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (ring->irq_refcount++ == 0) {
+        if (ring->irq_refcount.gt++ == 0) {
                 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
-                        I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
-                                                GEN6_RENDER_L3_PARITY_ERROR));
+                        I915_WRITE_IMR(ring,
+                                       ~(ring->irq_enable_mask |
+                                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
                 else
                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
@@ -984,9 +1030,10 @@ gen6_ring_put_irq(struct intel_ring_buffer *ring)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--ring->irq_refcount == 0) {
+        if (--ring->irq_refcount.gt == 0) {
                 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
-                        I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
+                        I915_WRITE_IMR(ring,
+                                       ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
                 else
                         I915_WRITE_IMR(ring, ~0);
                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
@@ -998,6 +1045,48 @@ gen6_ring_put_irq(struct intel_ring_buffer *ring)
         gen6_gt_force_wake_put(dev_priv);
 }
 
+static bool
+hsw_vebox_get_irq(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        if (!dev->irq_enabled)
+                return false;
+
+        spin_lock_irqsave(&dev_priv->rps.lock, flags);
+        if (ring->irq_refcount.pm++ == 0) {
+                u32 pm_imr = I915_READ(GEN6_PMIMR);
+                I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
+                I915_WRITE(GEN6_PMIMR, pm_imr & ~ring->irq_enable_mask);
+                POSTING_READ(GEN6_PMIMR);
+        }
+        spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
+
+        return true;
+}
+
+static void
+hsw_vebox_put_irq(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        if (!dev->irq_enabled)
+                return;
+
+        spin_lock_irqsave(&dev_priv->rps.lock, flags);
+        if (--ring->irq_refcount.pm == 0) {
+                u32 pm_imr = I915_READ(GEN6_PMIMR);
+                I915_WRITE_IMR(ring, ~0);
+                I915_WRITE(GEN6_PMIMR, pm_imr | ring->irq_enable_mask);
+                POSTING_READ(GEN6_PMIMR);
+        }
+        spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
+}
+
 static int
 i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
                          u32 offset, u32 length,
@@ -1423,7 +1512,7 @@ int intel_ring_idle(struct intel_ring_buffer *ring)
 
         /* We need to add any requests required to flush the objects and ring */
         if (ring->outstanding_lazy_request) {
-                ret = i915_add_request(ring, NULL, NULL);
+                ret = i915_add_request(ring, NULL);
                 if (ret)
                         return ret;
         }
@@ -1500,6 +1589,7 @@ void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
         }
 
         ring->set_seqno(ring, seqno);
+        ring->hangcheck.seqno = seqno;
 }
 
 void intel_ring_advance(struct intel_ring_buffer *ring)
@@ -1546,8 +1636,8 @@ static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
 }
 
-static int gen6_ring_flush(struct intel_ring_buffer *ring,
-                           u32 invalidate, u32 flush)
+static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
+                               u32 invalidate, u32 flush)
 {
         uint32_t cmd;
         int ret;
@@ -1618,9 +1708,10 @@ gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 
 /* Blitter support (SandyBridge+) */
 
-static int blt_ring_flush(struct intel_ring_buffer *ring,
-                          u32 invalidate, u32 flush)
+static int gen6_ring_flush(struct intel_ring_buffer *ring,
+                           u32 invalidate, u32 flush)
 {
+        struct drm_device *dev = ring->dev;
         uint32_t cmd;
         int ret;
 
@@ -1643,6 +1734,10 @@ static int blt_ring_flush(struct intel_ring_buffer *ring,
         intel_ring_emit(ring, 0);
         intel_ring_emit(ring, MI_NOOP);
         intel_ring_advance(ring);
+
+        if (IS_GEN7(dev) && flush)
+                return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
+
         return 0;
 }
 
@@ -1662,15 +1757,18 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
                         ring->flush = gen6_render_ring_flush;
                 ring->irq_get = gen6_ring_get_irq;
                 ring->irq_put = gen6_ring_put_irq;
-                ring->irq_enable_mask = GT_USER_INTERRUPT;
+                ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
                 ring->get_seqno = gen6_ring_get_seqno;
                 ring->set_seqno = ring_set_seqno;
                 ring->sync_to = gen6_ring_sync;
-                ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
-                ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
-                ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
-                ring->signal_mbox[0] = GEN6_VRSYNC;
-                ring->signal_mbox[1] = GEN6_BRSYNC;
+                ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
+                ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
+                ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
+                ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE;
+                ring->signal_mbox[RCS] = GEN6_NOSYNC;
+                ring->signal_mbox[VCS] = GEN6_VRSYNC;
+                ring->signal_mbox[BCS] = GEN6_BRSYNC;
+                ring->signal_mbox[VECS] = GEN6_VERSYNC;
         } else if (IS_GEN5(dev)) {
                 ring->add_request = pc_render_add_request;
                 ring->flush = gen4_render_ring_flush;
@@ -1678,7 +1776,8 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
                 ring->set_seqno = pc_render_set_seqno;
                 ring->irq_get = gen5_ring_get_irq;
                 ring->irq_put = gen5_ring_put_irq;
-                ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
+                ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
+                                        GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
         } else {
                 ring->add_request = i9xx_add_request;
                 if (INTEL_INFO(dev)->gen < 4)
@@ -1816,20 +1915,23 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
                 /* gen6 bsd needs a special wa for tail updates */
                 if (IS_GEN6(dev))
                         ring->write_tail = gen6_bsd_ring_write_tail;
-                ring->flush = gen6_ring_flush;
+                ring->flush = gen6_bsd_ring_flush;
                 ring->add_request = gen6_add_request;
                 ring->get_seqno = gen6_ring_get_seqno;
                 ring->set_seqno = ring_set_seqno;
-                ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
+                ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
                 ring->irq_get = gen6_ring_get_irq;
                 ring->irq_put = gen6_ring_put_irq;
                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
                 ring->sync_to = gen6_ring_sync;
-                ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
-                ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
-                ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
-                ring->signal_mbox[0] = GEN6_RVSYNC;
-                ring->signal_mbox[1] = GEN6_BVSYNC;
+                ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
+                ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
+                ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
+                ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE;
+                ring->signal_mbox[RCS] = GEN6_RVSYNC;
+                ring->signal_mbox[VCS] = GEN6_NOSYNC;
+                ring->signal_mbox[BCS] = GEN6_BVSYNC;
+                ring->signal_mbox[VECS] = GEN6_VEVSYNC;
         } else {
                 ring->mmio_base = BSD_RING_BASE;
                 ring->flush = bsd_ring_flush;
@@ -1837,7 +1939,7 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
                 ring->get_seqno = ring_get_seqno;
                 ring->set_seqno = ring_set_seqno;
                 if (IS_GEN5(dev)) {
-                        ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
+                        ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
                         ring->irq_get = gen5_ring_get_irq;
                         ring->irq_put = gen5_ring_put_irq;
                 } else {
@@ -1862,20 +1964,56 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
 
         ring->mmio_base = BLT_RING_BASE;
         ring->write_tail = ring_write_tail;
-        ring->flush = blt_ring_flush;
+        ring->flush = gen6_ring_flush;
         ring->add_request = gen6_add_request;
         ring->get_seqno = gen6_ring_get_seqno;
         ring->set_seqno = ring_set_seqno;
-        ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
+        ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
         ring->irq_get = gen6_ring_get_irq;
         ring->irq_put = gen6_ring_put_irq;
         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
         ring->sync_to = gen6_ring_sync;
-        ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
-        ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
-        ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
-        ring->signal_mbox[0] = GEN6_RBSYNC;
-        ring->signal_mbox[1] = GEN6_VBSYNC;
+        ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
+        ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
+        ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
+        ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE;
+        ring->signal_mbox[RCS] = GEN6_RBSYNC;
+        ring->signal_mbox[VCS] = GEN6_VBSYNC;
+        ring->signal_mbox[BCS] = GEN6_NOSYNC;
+        ring->signal_mbox[VECS] = GEN6_VEBSYNC;
+        ring->init = init_ring_common;
+
+        return intel_init_ring_buffer(dev, ring);
+}
+
+int intel_init_vebox_ring_buffer(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
+
+        ring->name = "video enhancement ring";
+        ring->id = VECS;
+
+        ring->mmio_base = VEBOX_RING_BASE;
+        ring->write_tail = ring_write_tail;
+        ring->flush = gen6_ring_flush;
+        ring->add_request = gen6_add_request;
+        ring->get_seqno = gen6_ring_get_seqno;
+        ring->set_seqno = ring_set_seqno;
+        ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT |
+                PM_VEBOX_CS_ERROR_INTERRUPT;
+        ring->irq_get = hsw_vebox_get_irq;
+        ring->irq_put = hsw_vebox_put_irq;
+        ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+        ring->sync_to = gen6_ring_sync;
+        ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
+        ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
+        ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
+        ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID;
+        ring->signal_mbox[RCS] = GEN6_RVESYNC;
+        ring->signal_mbox[VCS] = GEN6_VVESYNC;
+        ring->signal_mbox[BCS] = GEN6_BVESYNC;
+        ring->signal_mbox[VECS] = GEN6_NOSYNC;
         ring->init = init_ring_common;
 
         return intel_init_ring_buffer(dev, ring);
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index d66208c2c48b..799f04c9da45 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -37,14 +37,25 @@ struct  intel_hw_status_page {
 #define I915_READ_SYNC_0(ring) I915_READ(RING_SYNC_0((ring)->mmio_base))
 #define I915_READ_SYNC_1(ring) I915_READ(RING_SYNC_1((ring)->mmio_base))
 
+enum intel_ring_hangcheck_action { wait, active, kick, hung };
+
+struct intel_ring_hangcheck {
+        bool deadlock;
+        u32 seqno;
+        u32 acthd;
+        int score;
+        enum intel_ring_hangcheck_action action;
+};
+
 struct  intel_ring_buffer {
         const char      *name;
         enum intel_ring_id {
                 RCS = 0x0,
                 VCS,
                 BCS,
+                VECS,
         } id;
-#define I915_NUM_RINGS 3
+#define I915_NUM_RINGS 4
         u32             mmio_base;
         void            __iomem *virtual_start;
         struct          drm_device *dev;
@@ -67,7 +78,10 @@ struct  intel_ring_buffer {
          */
         u32             last_retired_head;
 
-        u32             irq_refcount;           /* protected by dev_priv->irq_lock */
+        struct {
+                u32     gt; /*  protected by dev_priv->irq_lock */
+                u32     pm; /*  protected by dev_priv->rps.lock (sucks) */
+        } irq_refcount;
         u32             irq_enable_mask;        /* bitmask to enable ring interrupt */
         u32             trace_irq_seqno;
         u32             sync_seqno[I915_NUM_RINGS-1];
@@ -102,8 +116,11 @@ struct  intel_ring_buffer {
                                    struct intel_ring_buffer *to,
                                    u32 seqno);
 
-        u32             semaphore_register[3]; /*our mbox written by others */
-        u32             signal_mbox[2]; /* mboxes this ring signals to */
+        /* our mbox written by others */
+        u32             semaphore_register[I915_NUM_RINGS];
+        /* mboxes this ring signals to */
+        u32             signal_mbox[I915_NUM_RINGS];
+
         /**
          * List of objects currently involved in rendering from the
          * ringbuffer.
@@ -127,6 +144,7 @@ struct  intel_ring_buffer {
          */
         u32 outstanding_lazy_request;
         bool gpu_caches_dirty;
+        bool fbc_dirty;
 
         wait_queue_head_t irq_queue;
 
@@ -135,7 +153,9 @@ struct  intel_ring_buffer {
          */
         bool itlb_before_ctx_switch;
         struct i915_hw_context *default_context;
-        struct drm_i915_gem_object *last_context_obj;
+        struct i915_hw_context *last_context;
+
+        struct intel_ring_hangcheck hangcheck;
 
         void *private;
 };
@@ -224,6 +244,7 @@ int intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring);
 int intel_init_render_ring_buffer(struct drm_device *dev);
 int intel_init_bsd_ring_buffer(struct drm_device *dev);
 int intel_init_blt_ring_buffer(struct drm_device *dev);
+int intel_init_vebox_ring_buffer(struct drm_device *dev);
 
 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring);
 void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index d4ea6c265ce1..2628d5622449 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -80,7 +80,7 @@ struct intel_sdvo {
 
         /*
          * Capabilities of the SDVO device returned by
-         * i830_sdvo_get_capabilities()
+         * intel_sdvo_get_capabilities()
          */
         struct intel_sdvo_caps caps;
 
@@ -712,6 +712,13 @@ static bool intel_sdvo_set_timing(struct intel_sdvo *intel_sdvo, u8 cmd,
                 intel_sdvo_set_value(intel_sdvo, cmd + 1, &dtd->part2, sizeof(dtd->part2));
 }
 
+static bool intel_sdvo_get_timing(struct intel_sdvo *intel_sdvo, u8 cmd,
+                                  struct intel_sdvo_dtd *dtd)
+{
+        return intel_sdvo_get_value(intel_sdvo, cmd, &dtd->part1, sizeof(dtd->part1)) &&
+                intel_sdvo_get_value(intel_sdvo, cmd + 1, &dtd->part2, sizeof(dtd->part2));
+}
+
 static bool intel_sdvo_set_input_timing(struct intel_sdvo *intel_sdvo,
                                          struct intel_sdvo_dtd *dtd)
 {
@@ -726,6 +733,13 @@ static bool intel_sdvo_set_output_timing(struct intel_sdvo *intel_sdvo,
                                      SDVO_CMD_SET_OUTPUT_TIMINGS_PART1, dtd);
 }
 
+static bool intel_sdvo_get_input_timing(struct intel_sdvo *intel_sdvo,
+                                        struct intel_sdvo_dtd *dtd)
+{
+        return intel_sdvo_get_timing(intel_sdvo,
+                                     SDVO_CMD_GET_INPUT_TIMINGS_PART1, dtd);
+}
+
 static bool
 intel_sdvo_create_preferred_input_timing(struct intel_sdvo *intel_sdvo,
                                          uint16_t clock,
@@ -1041,6 +1055,32 @@ intel_sdvo_get_preferred_input_mode(struct intel_sdvo *intel_sdvo,
         return true;
 }
 
+static void i9xx_adjust_sdvo_tv_clock(struct intel_crtc_config *pipe_config)
+{
+        unsigned dotclock = pipe_config->adjusted_mode.clock;
+        struct dpll *clock = &pipe_config->dpll;
+
+        /* SDVO TV has fixed PLL values depend on its clock range,
+           this mirrors vbios setting. */
+        if (dotclock >= 100000 && dotclock < 140500) {
+                clock->p1 = 2;
+                clock->p2 = 10;
+                clock->n = 3;
+                clock->m1 = 16;
+                clock->m2 = 8;
+        } else if (dotclock >= 140500 && dotclock <= 200000) {
+                clock->p1 = 1;
+                clock->p2 = 10;
+                clock->n = 6;
+                clock->m1 = 12;
+                clock->m2 = 8;
+        } else {
+                WARN(1, "SDVO TV clock out of range: %i\n", dotclock);
+        }
+
+        pipe_config->clock_set = true;
+}
+
 static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
                                       struct intel_crtc_config *pipe_config)
 {
@@ -1066,6 +1106,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
                 (void) intel_sdvo_get_preferred_input_mode(intel_sdvo,
                                                            mode,
                                                            adjusted_mode);
+                pipe_config->sdvo_tv_clock = true;
         } else if (intel_sdvo->is_lvds) {
                 if (!intel_sdvo_set_output_timings_from_mode(intel_sdvo,
                                                              intel_sdvo->sdvo_lvds_fixed_mode))
@@ -1097,6 +1138,10 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
         if (intel_sdvo->color_range)
                 pipe_config->limited_color_range = true;
 
+        /* Clock computation needs to happen after pixel multiplier. */
+        if (intel_sdvo->is_tv)
+                i9xx_adjust_sdvo_tv_clock(pipe_config);
+
         return true;
 }
 
@@ -1174,6 +1219,7 @@ static void intel_sdvo_mode_set(struct intel_encoder *intel_encoder)
 
         switch (intel_crtc->config.pixel_multiplier) {
         default:
+                WARN(1, "unknown pixel mutlipler specified\n");
         case 1: rate = SDVO_CLOCK_RATE_MULT_1X; break;
         case 2: rate = SDVO_CLOCK_RATE_MULT_2X; break;
         case 4: rate = SDVO_CLOCK_RATE_MULT_4X; break;
@@ -1231,7 +1277,7 @@ static bool intel_sdvo_connector_get_hw_state(struct intel_connector *connector)
         struct intel_sdvo_connector *intel_sdvo_connector =
                 to_intel_sdvo_connector(&connector->base);
         struct intel_sdvo *intel_sdvo = intel_attached_sdvo(&connector->base);
-        u16 active_outputs;
+        u16 active_outputs = 0;
 
         intel_sdvo_get_active_outputs(intel_sdvo, &active_outputs);
 
@@ -1247,7 +1293,7 @@ static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
-        u16 active_outputs;
+        u16 active_outputs = 0;
         u32 tmp;
 
         tmp = I915_READ(intel_sdvo->sdvo_reg);
@@ -1264,6 +1310,74 @@ static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder,
         return true;
 }
 
+static void intel_sdvo_get_config(struct intel_encoder *encoder,
+                                  struct intel_crtc_config *pipe_config)
+{
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
+        struct intel_sdvo_dtd dtd;
+        int encoder_pixel_multiplier = 0;
+        u32 flags = 0, sdvox;
+        u8 val;
+        bool ret;
+
+        ret = intel_sdvo_get_input_timing(intel_sdvo, &dtd);
+        if (!ret) {
+                /* Some sdvo encoders are not spec compliant and don't
+                 * implement the mandatory get_timings function. */
+                DRM_DEBUG_DRIVER("failed to retrieve SDVO DTD\n");
+                pipe_config->quirks |= PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS;
+        } else {
+                if (dtd.part2.dtd_flags & DTD_FLAG_HSYNC_POSITIVE)
+                        flags |= DRM_MODE_FLAG_PHSYNC;
+                else
+                        flags |= DRM_MODE_FLAG_NHSYNC;
+
+                if (dtd.part2.dtd_flags & DTD_FLAG_VSYNC_POSITIVE)
+                        flags |= DRM_MODE_FLAG_PVSYNC;
+                else
+                        flags |= DRM_MODE_FLAG_NVSYNC;
+        }
+
+        pipe_config->adjusted_mode.flags |= flags;
+
+        /*
+         * pixel multiplier readout is tricky: Only on i915g/gm it is stored in
+         * the sdvo port register, on all other platforms it is part of the dpll
+         * state. Since the general pipe state readout happens before the
+         * encoder->get_config we so already have a valid pixel multplier on all
+         * other platfroms.
+         */
+        if (IS_I915G(dev) || IS_I915GM(dev)) {
+                sdvox = I915_READ(intel_sdvo->sdvo_reg);
+                pipe_config->pixel_multiplier =
+                        ((sdvox & SDVO_PORT_MULTIPLY_MASK)
+                         >> SDVO_PORT_MULTIPLY_SHIFT) + 1;
+        }
+
+        /* Cross check the port pixel multiplier with the sdvo encoder state. */
+        intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_CLOCK_RATE_MULT, &val, 1);
+        switch (val) {
+        case SDVO_CLOCK_RATE_MULT_1X:
+                encoder_pixel_multiplier = 1;
+                break;
+        case SDVO_CLOCK_RATE_MULT_2X:
+                encoder_pixel_multiplier = 2;
+                break;
+        case SDVO_CLOCK_RATE_MULT_4X:
+                encoder_pixel_multiplier = 4;
+                break;
+        }
+
+        if(HAS_PCH_SPLIT(dev))
+                return; /* no pixel multiplier readout support yet */
+
+        WARN(encoder_pixel_multiplier != pipe_config->pixel_multiplier,
+             "SDVO pixel multiplier mismatch, port: %i, encoder: %i\n",
+             pipe_config->pixel_multiplier, encoder_pixel_multiplier);
+}
+
 static void intel_disable_sdvo(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
@@ -1344,6 +1458,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder)
         intel_sdvo_set_active_outputs(intel_sdvo, intel_sdvo->attached_output);
 }
 
+/* Special dpms function to support cloning between dvo/sdvo/crt. */
 static void intel_sdvo_dpms(struct drm_connector *connector, int mode)
 {
         struct drm_crtc *crtc;
@@ -1365,6 +1480,8 @@ static void intel_sdvo_dpms(struct drm_connector *connector, int mode)
                 return;
         }
 
+        /* We set active outputs manually below in case pipe dpms doesn't change
+         * due to cloning. */
         if (mode != DRM_MODE_DPMS_ON) {
                 intel_sdvo_set_active_outputs(intel_sdvo, 0);
                 if (0)
@@ -1495,7 +1612,7 @@ intel_sdvo_get_analog_edid(struct drm_connector *connector)
 
         return drm_get_edid(connector,
                             intel_gmbus_get_adapter(dev_priv,
-                                                    dev_priv->crt_ddc_pin));
+                                                    dev_priv->vbt.crt_ddc_pin));
 }
 
 static enum drm_connector_status
@@ -1625,12 +1742,9 @@ intel_sdvo_detect(struct drm_connector *connector, bool force)
         if (ret == connector_status_connected) {
                 intel_sdvo->is_tv = false;
                 intel_sdvo->is_lvds = false;
-                intel_sdvo->base.needs_tv_clock = false;
 
-                if (response & SDVO_TV_MASK) {
+                if (response & SDVO_TV_MASK)
                         intel_sdvo->is_tv = true;
-                        intel_sdvo->base.needs_tv_clock = true;
-                }
                 if (response & SDVO_LVDS_MASK)
                         intel_sdvo->is_lvds = intel_sdvo->sdvo_lvds_fixed_mode != NULL;
         }
@@ -1772,21 +1886,12 @@ static void intel_sdvo_get_lvds_modes(struct drm_connector *connector)
         struct drm_display_mode *newmode;
 
         /*
-         * Attempt to get the mode list from DDC.
-         * Assume that the preferred modes are
-         * arranged in priority order.
-         */
-        intel_ddc_get_modes(connector, &intel_sdvo->ddc);
-
-        /*
          * Fetch modes from VBT. For SDVO prefer the VBT mode since some
-         * SDVO->LVDS transcoders can't cope with the EDID mode. Since
-         * drm_mode_probed_add adds the mode at the head of the list we add it
-         * last.
+         * SDVO->LVDS transcoders can't cope with the EDID mode.
          */
-        if (dev_priv->sdvo_lvds_vbt_mode != NULL) {
+        if (dev_priv->vbt.sdvo_lvds_vbt_mode != NULL) {
                 newmode = drm_mode_duplicate(connector->dev,
-                                             dev_priv->sdvo_lvds_vbt_mode);
+                                             dev_priv->vbt.sdvo_lvds_vbt_mode);
                 if (newmode != NULL) {
                         /* Guarantee the mode is preferred */
                         newmode->type = (DRM_MODE_TYPE_PREFERRED |
@@ -1795,6 +1900,13 @@ static void intel_sdvo_get_lvds_modes(struct drm_connector *connector)
                 }
         }
 
+        /*
+         * Attempt to get the mode list from DDC.
+         * Assume that the preferred modes are
+         * arranged in priority order.
+         */
+        intel_ddc_get_modes(connector, &intel_sdvo->ddc);
+
         list_for_each_entry(newmode, &connector->probed_modes, head) {
                 if (newmode->type & DRM_MODE_TYPE_PREFERRED) {
                         intel_sdvo->sdvo_lvds_fixed_mode =
@@ -2329,7 +2441,6 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type)
         intel_sdvo_connector->output_flag = type;
 
         intel_sdvo->is_tv = true;
-        intel_sdvo->base.needs_tv_clock = true;
 
         intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
 
@@ -2417,7 +2528,6 @@ static bool
 intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, uint16_t flags)
 {
         intel_sdvo->is_tv = false;
-        intel_sdvo->base.needs_tv_clock = false;
         intel_sdvo->is_lvds = false;
 
         /* SDVO requires XXX1 function may not exist unless it has XXX0 function.*/
@@ -2751,7 +2861,6 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_encoder *intel_encoder;
         struct intel_sdvo *intel_sdvo;
-        u32 hotplug_mask;
         int i;
         intel_sdvo = kzalloc(sizeof(struct intel_sdvo), GFP_KERNEL);
         if (!intel_sdvo)
@@ -2780,23 +2889,12 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
                 }
         }
 
-        hotplug_mask = 0;
-        if (IS_G4X(dev)) {
-                hotplug_mask = intel_sdvo->is_sdvob ?
-                        SDVOB_HOTPLUG_INT_STATUS_G4X : SDVOC_HOTPLUG_INT_STATUS_G4X;
-        } else if (IS_GEN4(dev)) {
-                hotplug_mask = intel_sdvo->is_sdvob ?
-                        SDVOB_HOTPLUG_INT_STATUS_I965 : SDVOC_HOTPLUG_INT_STATUS_I965;
-        } else {
-                hotplug_mask = intel_sdvo->is_sdvob ?
-                        SDVOB_HOTPLUG_INT_STATUS_I915 : SDVOC_HOTPLUG_INT_STATUS_I915;
-        }
-
         intel_encoder->compute_config = intel_sdvo_compute_config;
         intel_encoder->disable = intel_disable_sdvo;
         intel_encoder->mode_set = intel_sdvo_mode_set;
         intel_encoder->enable = intel_enable_sdvo;
         intel_encoder->get_hw_state = intel_sdvo_get_hw_state;
+        intel_encoder->get_config = intel_sdvo_get_config;
 
         /* In default case sdvo lvds is false */
         if (!intel_sdvo_get_capabilities(intel_sdvo, &intel_sdvo->caps))
diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c
new file mode 100644
index 000000000000..9a0e6c5ea540
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_sideband.c
@@ -0,0 +1,177 @@
+/*
+ * Copyright © 2013 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "i915_drv.h"
+#include "intel_drv.h"
+
+/* IOSF sideband */
+static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
+                           u32 port, u32 opcode, u32 addr, u32 *val)
+{
+        u32 cmd, be = 0xf, bar = 0;
+        bool is_read = (opcode == PUNIT_OPCODE_REG_READ ||
+                        opcode == DPIO_OPCODE_REG_READ);
+
+        cmd = (devfn << IOSF_DEVFN_SHIFT) | (opcode << IOSF_OPCODE_SHIFT) |
+                (port << IOSF_PORT_SHIFT) | (be << IOSF_BYTE_ENABLES_SHIFT) |
+                (bar << IOSF_BAR_SHIFT);
+
+        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
+
+        if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0, 5)) {
+                DRM_DEBUG_DRIVER("IOSF sideband idle wait (%s) timed out\n",
+                                 is_read ? "read" : "write");
+                return -EAGAIN;
+        }
+
+        I915_WRITE(VLV_IOSF_ADDR, addr);
+        if (!is_read)
+                I915_WRITE(VLV_IOSF_DATA, *val);
+        I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
+
+        if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0, 5)) {
+                DRM_DEBUG_DRIVER("IOSF sideband finish wait (%s) timed out\n",
+                                 is_read ? "read" : "write");
+                return -ETIMEDOUT;
+        }
+
+        if (is_read)
+                *val = I915_READ(VLV_IOSF_DATA);
+        I915_WRITE(VLV_IOSF_DATA, 0);
+
+        return 0;
+}
+
+u32 vlv_punit_read(struct drm_i915_private *dev_priv, u8 addr)
+{
+        u32 val = 0;
+
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        mutex_lock(&dev_priv->dpio_lock);
+        vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_PUNIT,
+                        PUNIT_OPCODE_REG_READ, addr, &val);
+        mutex_unlock(&dev_priv->dpio_lock);
+
+        return val;
+}
+
+void vlv_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val)
+{
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        mutex_lock(&dev_priv->dpio_lock);
+        vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_PUNIT,
+                        PUNIT_OPCODE_REG_WRITE, addr, &val);
+        mutex_unlock(&dev_priv->dpio_lock);
+}
+
+u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr)
+{
+        u32 val = 0;
+
+        WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
+
+        mutex_lock(&dev_priv->dpio_lock);
+        vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_NC,
+                        PUNIT_OPCODE_REG_READ, addr, &val);
+        mutex_unlock(&dev_priv->dpio_lock);
+
+        return val;
+}
+
+u32 vlv_dpio_read(struct drm_i915_private *dev_priv, int reg)
+{
+        u32 val = 0;
+
+        vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_DPIO,
+                        DPIO_OPCODE_REG_READ, reg, &val);
+
+        return val;
+}
+
+void vlv_dpio_write(struct drm_i915_private *dev_priv, int reg, u32 val)
+{
+        vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_DPIO,
+                        DPIO_OPCODE_REG_WRITE, reg, &val);
+}
+
+/* SBI access */
+u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
+                   enum intel_sbi_destination destination)
+{
+        u32 value = 0;
+        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
+
+        if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0,
+                                100)) {
+                DRM_ERROR("timeout waiting for SBI to become ready\n");
+                return 0;
+        }
+
+        I915_WRITE(SBI_ADDR, (reg << 16));
+
+        if (destination == SBI_ICLK)
+                value = SBI_CTL_DEST_ICLK | SBI_CTL_OP_CRRD;
+        else
+                value = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD;
+        I915_WRITE(SBI_CTL_STAT, value | SBI_BUSY);
+
+        if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0,
+                                100)) {
+                DRM_ERROR("timeout waiting for SBI to complete read transaction\n");
+                return 0;
+        }
+
+        return I915_READ(SBI_DATA);
+}
+
+void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
+                     enum intel_sbi_destination destination)
+{
+        u32 tmp;
+
+        WARN_ON(!mutex_is_locked(&dev_priv->dpio_lock));
+
+        if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0,
+                                100)) {
+                DRM_ERROR("timeout waiting for SBI to become ready\n");
+                return;
+        }
+
+        I915_WRITE(SBI_ADDR, (reg << 16));
+        I915_WRITE(SBI_DATA, value);
+
+        if (destination == SBI_ICLK)
+                tmp = SBI_CTL_DEST_ICLK | SBI_CTL_OP_CRWR;
+        else
+                tmp = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IOWR;
+        I915_WRITE(SBI_CTL_STAT, SBI_BUSY | tmp);
+
+        if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0,
+                                100)) {
+                DRM_ERROR("timeout waiting for SBI to complete write transaction\n");
+                return;
+        }
+}
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index c7d25c5dd4e6..1fa5612a4572 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -32,6 +32,7 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_rect.h>
 #include "intel_drv.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
@@ -113,7 +114,7 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_framebuffer *fb,
         crtc_w--;
         crtc_h--;
 
-        intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
+        intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size, true);
 
         I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
         I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x);
@@ -267,7 +268,7 @@ ivb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
         crtc_w--;
         crtc_h--;
 
-        intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
+        intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size, true);
 
         /*
          * IVB workaround: must disable low power watermarks for at least
@@ -334,6 +335,8 @@ ivb_disable_plane(struct drm_plane *plane)
 
         dev_priv->sprite_scaling_enabled &= ~(1 << pipe);
 
+        intel_update_sprite_watermarks(dev, pipe, 0, 0, false);
+
         /* potentially re-enable LP watermarks */
         if (scaling_was_enabled && !dev_priv->sprite_scaling_enabled)
                 intel_update_watermarks(dev);
@@ -452,7 +455,7 @@ ilk_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
         crtc_w--;
         crtc_h--;
 
-        intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
+        intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size, true);
 
         dvsscale = 0;
         if (IS_GEN5(dev) || crtc_w != src_w || crtc_h != src_h)
@@ -583,6 +586,20 @@ ilk_get_colorkey(struct drm_plane *plane, struct drm_intel_sprite_colorkey *key)
                 key->flags = I915_SET_COLORKEY_NONE;
 }
 
+static bool
+format_is_yuv(uint32_t format)
+{
+        switch (format) {
+        case DRM_FORMAT_YUYV:
+        case DRM_FORMAT_UYVY:
+        case DRM_FORMAT_VYUY:
+        case DRM_FORMAT_YVYU:
+                return true;
+        default:
+                return false;
+        }
+}
+
 static int
 intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                    struct drm_framebuffer *fb, int crtc_x, int crtc_y,
@@ -600,9 +617,29 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
                                                                       pipe);
         int ret = 0;
-        int x = src_x >> 16, y = src_y >> 16;
-        int primary_w = crtc->mode.hdisplay, primary_h = crtc->mode.vdisplay;
         bool disable_primary = false;
+        bool visible;
+        int hscale, vscale;
+        int max_scale, min_scale;
+        int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
+        struct drm_rect src = {
+                /* sample coordinates in 16.16 fixed point */
+                .x1 = src_x,
+                .x2 = src_x + src_w,
+                .y1 = src_y,
+                .y2 = src_y + src_h,
+        };
+        struct drm_rect dst = {
+                /* integer pixels */
+                .x1 = crtc_x,
+                .x2 = crtc_x + crtc_w,
+                .y1 = crtc_y,
+                .y2 = crtc_y + crtc_h,
+        };
+        const struct drm_rect clip = {
+                .x2 = crtc->mode.hdisplay,
+                .y2 = crtc->mode.vdisplay,
+        };
 
         intel_fb = to_intel_framebuffer(fb);
         obj = intel_fb->obj;
@@ -618,19 +655,23 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         intel_plane->src_w = src_w;
         intel_plane->src_h = src_h;
 
-        src_w = src_w >> 16;
-        src_h = src_h >> 16;
-
         /* Pipe must be running... */
-        if (!(I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_ENABLE))
+        if (!(I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_ENABLE)) {
+                DRM_DEBUG_KMS("Pipe disabled\n");
                 return -EINVAL;
+        }
 
-        if (crtc_x >= primary_w || crtc_y >= primary_h)
+        /* Don't modify another pipe's plane */
+        if (intel_plane->pipe != intel_crtc->pipe) {
+                DRM_DEBUG_KMS("Wrong plane <-> crtc mapping\n");
                 return -EINVAL;
+        }
 
-        /* Don't modify another pipe's plane */
-        if (intel_plane->pipe != intel_crtc->pipe)
+        /* FIXME check all gen limits */
+        if (fb->width < 3 || fb->height < 3 || fb->pitches[0] > 16384) {
+                DRM_DEBUG_KMS("Unsuitable framebuffer for plane\n");
                 return -EINVAL;
+        }
 
         /* Sprite planes can be linear or x-tiled surfaces */
         switch (obj->tiling_mode) {
@@ -638,55 +679,123 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                 case I915_TILING_X:
                         break;
                 default:
+                        DRM_DEBUG_KMS("Unsupported tiling mode\n");
                         return -EINVAL;
         }
 
         /*
-         * Clamp the width & height into the visible area.  Note we don't
-         * try to scale the source if part of the visible region is offscreen.
-         * The caller must handle that by adjusting source offset and size.
+         * FIXME the following code does a bunch of fuzzy adjustments to the
+         * coordinates and sizes. We probably need some way to decide whether
+         * more strict checking should be done instead.
          */
-        if ((crtc_x < 0) && ((crtc_x + crtc_w) > 0)) {
-                crtc_w += crtc_x;
-                crtc_x = 0;
-        }
-        if ((crtc_x + crtc_w) <= 0) /* Nothing to display */
-                goto out;
-        if ((crtc_x + crtc_w) > primary_w)
-                crtc_w = primary_w - crtc_x;
+        max_scale = intel_plane->max_downscale << 16;
+        min_scale = intel_plane->can_scale ? 1 : (1 << 16);
+
+        hscale = drm_rect_calc_hscale_relaxed(&src, &dst, min_scale, max_scale);
+        BUG_ON(hscale < 0);
+
+        vscale = drm_rect_calc_vscale_relaxed(&src, &dst, min_scale, max_scale);
+        BUG_ON(vscale < 0);
+
+        visible = drm_rect_clip_scaled(&src, &dst, &clip, hscale, vscale);
+
+        crtc_x = dst.x1;
+        crtc_y = dst.y1;
+        crtc_w = drm_rect_width(&dst);
+        crtc_h = drm_rect_height(&dst);
+
+        if (visible) {
+                /* check again in case clipping clamped the results */
+                hscale = drm_rect_calc_hscale(&src, &dst, min_scale, max_scale);
+                if (hscale < 0) {
+                        DRM_DEBUG_KMS("Horizontal scaling factor out of limits\n");
+                        drm_rect_debug_print(&src, true);
+                        drm_rect_debug_print(&dst, false);
+
+                        return hscale;
+                }
 
-        if ((crtc_y < 0) && ((crtc_y + crtc_h) > 0)) {
-                crtc_h += crtc_y;
-                crtc_y = 0;
+                vscale = drm_rect_calc_vscale(&src, &dst, min_scale, max_scale);
+                if (vscale < 0) {
+                        DRM_DEBUG_KMS("Vertical scaling factor out of limits\n");
+                        drm_rect_debug_print(&src, true);
+                        drm_rect_debug_print(&dst, false);
+
+                        return vscale;
+                }
+
+                /* Make the source viewport size an exact multiple of the scaling factors. */
+                drm_rect_adjust_size(&src,
+                                     drm_rect_width(&dst) * hscale - drm_rect_width(&src),
+                                     drm_rect_height(&dst) * vscale - drm_rect_height(&src));
+
+                /* sanity check to make sure the src viewport wasn't enlarged */
+                WARN_ON(src.x1 < (int) src_x ||
+                        src.y1 < (int) src_y ||
+                        src.x2 > (int) (src_x + src_w) ||
+                        src.y2 > (int) (src_y + src_h));
+
+                /*
+                 * Hardware doesn't handle subpixel coordinates.
+                 * Adjust to (macro)pixel boundary, but be careful not to
+                 * increase the source viewport size, because that could
+                 * push the downscaling factor out of bounds.
+                 */
+                src_x = src.x1 >> 16;
+                src_w = drm_rect_width(&src) >> 16;
+                src_y = src.y1 >> 16;
+                src_h = drm_rect_height(&src) >> 16;
+
+                if (format_is_yuv(fb->pixel_format)) {
+                        src_x &= ~1;
+                        src_w &= ~1;
+
+                        /*
+                         * Must keep src and dst the
+                         * same if we can't scale.
+                         */
+                        if (!intel_plane->can_scale)
+                                crtc_w &= ~1;
+
+                        if (crtc_w == 0)
+                                visible = false;
+                }
         }
-        if ((crtc_y + crtc_h) <= 0) /* Nothing to display */
-                goto out;
-        if (crtc_y + crtc_h > primary_h)
-                crtc_h = primary_h - crtc_y;
 
-        if (!crtc_w || !crtc_h) /* Again, nothing to display */
-                goto out;
+        /* Check size restrictions when scaling */
+        if (visible && (src_w != crtc_w || src_h != crtc_h)) {
+                unsigned int width_bytes;
 
-        /*
-         * We may not have a scaler, eg. HSW does not have it any more
-         */
-        if (!intel_plane->can_scale && (crtc_w != src_w || crtc_h != src_h))
-                return -EINVAL;
+                WARN_ON(!intel_plane->can_scale);
 
-        /*
-         * We can take a larger source and scale it down, but
-         * only so much...  16x is the max on SNB.
-         */
-        if (((src_w * src_h) / (crtc_w * crtc_h)) > intel_plane->max_downscale)
-                return -EINVAL;
+                /* FIXME interlacing min height is 6 */
+
+                if (crtc_w < 3 || crtc_h < 3)
+                        visible = false;
+
+                if (src_w < 3 || src_h < 3)
+                        visible = false;
+
+                width_bytes = ((src_x * pixel_size) & 63) + src_w * pixel_size;
+
+                if (src_w > 2048 || src_h > 2048 ||
+                    width_bytes > 4096 || fb->pitches[0] > 4096) {
+                        DRM_DEBUG_KMS("Source dimensions exceed hardware limits\n");
+                        return -EINVAL;
+                }
+        }
+
+        dst.x1 = crtc_x;
+        dst.x2 = crtc_x + crtc_w;
+        dst.y1 = crtc_y;
+        dst.y2 = crtc_y + crtc_h;
 
         /*
          * If the sprite is completely covering the primary plane,
          * we can disable the primary and save power.
          */
-        if ((crtc_x == 0) && (crtc_y == 0) &&
-            (crtc_w == primary_w) && (crtc_h == primary_h))
-                disable_primary = true;
+        disable_primary = drm_rect_equals(&dst, &clip);
+        WARN_ON(disable_primary && !visible);
 
         mutex_lock(&dev->struct_mutex);
 
@@ -708,8 +817,12 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         if (!disable_primary)
                 intel_enable_primary(crtc);
 
-        intel_plane->update_plane(plane, fb, obj, crtc_x, crtc_y,
-                                  crtc_w, crtc_h, x, y, src_w, src_h);
+        if (visible)
+                intel_plane->update_plane(plane, fb, obj,
+                                          crtc_x, crtc_y, crtc_w, crtc_h,
+                                          src_x, src_y, src_w, src_h);
+        else
+                intel_plane->disable_plane(plane);
 
         if (disable_primary)
                 intel_disable_primary(crtc);
@@ -732,7 +845,6 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
-out:
         return ret;
 }
 
@@ -845,6 +957,14 @@ void intel_plane_restore(struct drm_plane *plane)
                            intel_plane->src_w, intel_plane->src_h);
 }
 
+void intel_plane_disable(struct drm_plane *plane)
+{
+        if (!plane->crtc || !plane->fb)
+                return;
+
+        intel_disable_plane(plane);
+}
+
 static const struct drm_plane_funcs intel_plane_funcs = {
         .update_plane = intel_update_plane,
         .disable_plane = intel_disable_plane,
@@ -918,13 +1038,15 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
                 break;
 
         case 7:
-                if (IS_HASWELL(dev) || IS_VALLEYVIEW(dev))
-                        intel_plane->can_scale = false;
-                else
+                if (IS_IVYBRIDGE(dev)) {
                         intel_plane->can_scale = true;
+                        intel_plane->max_downscale = 2;
+                } else {
+                        intel_plane->can_scale = false;
+                        intel_plane->max_downscale = 1;
+                }
 
                 if (IS_VALLEYVIEW(dev)) {
-                        intel_plane->max_downscale = 1;
                         intel_plane->update_plane = vlv_update_plane;
                         intel_plane->disable_plane = vlv_disable_plane;
                         intel_plane->update_colorkey = vlv_update_colorkey;
@@ -933,7 +1055,6 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
                         plane_formats = vlv_plane_formats;
                         num_plane_formats = ARRAY_SIZE(vlv_plane_formats);
                 } else {
-                        intel_plane->max_downscale = 2;
                         intel_plane->update_plane = ivb_update_plane;
                         intel_plane->disable_plane = ivb_disable_plane;
                         intel_plane->update_colorkey = ivb_update_colorkey;
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index b945bc54207a..39debd80d190 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -914,9 +914,6 @@ intel_tv_compute_config(struct intel_encoder *encoder,
         if (!tv_mode)
                 return false;
 
-        if (intel_encoder_check_is_cloned(&intel_tv->base))
-                return false;
-
         pipe_config->adjusted_mode.clock = tv_mode->clock;
         DRM_DEBUG_KMS("forcing bpc to 8 for TV\n");
         pipe_config->pipe_bpp = 8*3;
@@ -1521,12 +1518,12 @@ static int tv_is_present_in_vbt(struct drm_device *dev)
         struct child_device_config *p_child;
         int i, ret;
 
-        if (!dev_priv->child_dev_num)
+        if (!dev_priv->vbt.child_dev_num)
                 return 1;
 
         ret = 0;
-        for (i = 0; i < dev_priv->child_dev_num; i++) {
-                p_child = dev_priv->child_dev + i;
+        for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
+                p_child = dev_priv->vbt.child_dev + i;
                 /*
                  * If the device type is not TV, continue.
                  */
@@ -1564,7 +1561,7 @@ intel_tv_init(struct drm_device *dev)
                 return;
         }
         /* Even if we have an encoder we may not have a connector */
-        if (!dev_priv->int_tv_support)
+        if (!dev_priv->vbt.int_tv_support)
                 return;
 
         /*
diff --git a/drivers/gpu/drm/mgag200/Makefile b/drivers/gpu/drm/mgag200/Makefile
index 7db592eedbf1..a9a0300f09fc 100644
--- a/drivers/gpu/drm/mgag200/Makefile
+++ b/drivers/gpu/drm/mgag200/Makefile
@@ -1,5 +1,5 @@
 ccflags-y := -Iinclude/drm
-mgag200-y   := mgag200_main.o mgag200_mode.o \
+mgag200-y   := mgag200_main.o mgag200_mode.o mgag200_cursor.o \
         mgag200_drv.o mgag200_fb.o mgag200_i2c.o mgag200_ttm.o
 
 obj-$(CONFIG_DRM_MGAG200) += mgag200.o
diff --git a/drivers/gpu/drm/mgag200/mgag200_cursor.c b/drivers/gpu/drm/mgag200/mgag200_cursor.c
new file mode 100644
index 000000000000..801731aeab61
--- /dev/null
+++ b/drivers/gpu/drm/mgag200/mgag200_cursor.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright 2013 Matrox Graphics
+ *
+ * This file is subject to the terms and conditions of the GNU General
+ * Public License version 2. See the file COPYING in the main
+ * directory of this archive for more details.
+ *
+ * Author: Christopher Harvey <charvey@matrox.com>
+ */
+
+#include <drm/drmP.h>
+#include "mgag200_drv.h"
+
+static bool warn_transparent = true;
+static bool warn_palette = true;
+
+/*
+  Hide the cursor off screen. We can't disable the cursor hardware because it
+  takes too long to re-activate and causes momentary corruption
+*/
+static void mga_hide_cursor(struct mga_device *mdev)
+{
+        WREG8(MGA_CURPOSXL, 0);
+        WREG8(MGA_CURPOSXH, 0);
+        mgag200_bo_unpin(mdev->cursor.pixels_1);
+        mgag200_bo_unpin(mdev->cursor.pixels_2);
+}
+
+int mga_crtc_cursor_set(struct drm_crtc *crtc,
+                        struct drm_file *file_priv,
+                        uint32_t handle,
+                        uint32_t width,
+                        uint32_t height)
+{
+        struct drm_device *dev = (struct drm_device *)file_priv->minor->dev;
+        struct mga_device *mdev = (struct mga_device *)dev->dev_private;
+        struct mgag200_bo *pixels_1 = mdev->cursor.pixels_1;
+        struct mgag200_bo *pixels_2 = mdev->cursor.pixels_2;
+        struct mgag200_bo *pixels_current = mdev->cursor.pixels_current;
+        struct mgag200_bo *pixels_prev = mdev->cursor.pixels_prev;
+        struct drm_gem_object *obj;
+        struct mgag200_bo *bo = NULL;
+        int ret = 0;
+        unsigned int i, row, col;
+        uint32_t colour_set[16];
+        uint32_t *next_space = &colour_set[0];
+        uint32_t *palette_iter;
+        uint32_t this_colour;
+        bool found = false;
+        int colour_count = 0;
+        u64 gpu_addr;
+        u8 reg_index;
+        u8 this_row[48];
+
+        if (!pixels_1 || !pixels_2) {
+                WREG8(MGA_CURPOSXL, 0);
+                WREG8(MGA_CURPOSXH, 0);
+                return -ENOTSUPP; /* Didn't allocate space for cursors */
+        }
+
+        if ((width != 64 || height != 64) && handle) {
+                WREG8(MGA_CURPOSXL, 0);
+                WREG8(MGA_CURPOSXH, 0);
+                return -EINVAL;
+        }
+
+        BUG_ON(pixels_1 != pixels_current && pixels_1 != pixels_prev);
+        BUG_ON(pixels_2 != pixels_current && pixels_2 != pixels_prev);
+        BUG_ON(pixels_current == pixels_prev);
+
+        ret = mgag200_bo_reserve(pixels_1, true);
+        if (ret) {
+                WREG8(MGA_CURPOSXL, 0);
+                WREG8(MGA_CURPOSXH, 0);
+                return ret;
+        }
+        ret = mgag200_bo_reserve(pixels_2, true);
+        if (ret) {
+                WREG8(MGA_CURPOSXL, 0);
+                WREG8(MGA_CURPOSXH, 0);
+                mgag200_bo_unreserve(pixels_1);
+                return ret;
+        }
+
+        if (!handle) {
+                mga_hide_cursor(mdev);
+                ret = 0;
+                goto out1;
+        }
+
+        /* Move cursor buffers into VRAM if they aren't already */
+        if (!pixels_1->pin_count) {
+                ret = mgag200_bo_pin(pixels_1, TTM_PL_FLAG_VRAM,
+                                     &mdev->cursor.pixels_1_gpu_addr);
+                if (ret)
+                        goto out1;
+        }
+        if (!pixels_2->pin_count) {
+                ret = mgag200_bo_pin(pixels_2, TTM_PL_FLAG_VRAM,
+                                     &mdev->cursor.pixels_2_gpu_addr);
+                if (ret) {
+                        mgag200_bo_unpin(pixels_1);
+                        goto out1;
+                }
+        }
+
+        mutex_lock(&dev->struct_mutex);
+        obj = drm_gem_object_lookup(dev, file_priv, handle);
+        if (!obj) {
+                mutex_unlock(&dev->struct_mutex);
+                ret = -ENOENT;
+                goto out1;
+        }
+        drm_gem_object_unreference(obj);
+        mutex_unlock(&dev->struct_mutex);
+
+        bo = gem_to_mga_bo(obj);
+        ret = mgag200_bo_reserve(bo, true);
+        if (ret) {
+                dev_err(&dev->pdev->dev, "failed to reserve user bo\n");
+                goto out1;
+        }
+        if (!bo->kmap.virtual) {
+                ret = ttm_bo_kmap(&bo->bo, 0, bo->bo.num_pages, &bo->kmap);
+                if (ret) {
+                        dev_err(&dev->pdev->dev, "failed to kmap user buffer updates\n");
+                        goto out2;
+                }
+        }
+
+        memset(&colour_set[0], 0, sizeof(uint32_t)*16);
+        /* width*height*4 = 16384 */
+        for (i = 0; i < 16384; i += 4) {
+                this_colour = ioread32(bo->kmap.virtual + i);
+                /* No transparency */
+                if (this_colour>>24 != 0xff &&
+                        this_colour>>24 != 0x0) {
+                        if (warn_transparent) {
+                                dev_info(&dev->pdev->dev, "Video card doesn't support cursors with partial transparency.\n");
+                                dev_info(&dev->pdev->dev, "Not enabling hardware cursor.\n");
+                                warn_transparent = false; /* Only tell the user once. */
+                        }
+                        ret = -EINVAL;
+                        goto out3;
+                }
+                /* Don't need to store transparent pixels as colours */
+                if (this_colour>>24 == 0x0)
+                        continue;
+                found = false;
+                for (palette_iter = &colour_set[0]; palette_iter != next_space; palette_iter++) {
+                        if (*palette_iter == this_colour) {
+                                found = true;
+                                break;
+                        }
+                }
+                if (found)
+                        continue;
+                /* We only support 4bit paletted cursors */
+                if (colour_count >= 16) {
+                        if (warn_palette) {
+                                dev_info(&dev->pdev->dev, "Video card only supports cursors with up to 16 colours.\n");
+                                dev_info(&dev->pdev->dev, "Not enabling hardware cursor.\n");
+                                warn_palette = false; /* Only tell the user once. */
+                        }
+                        ret = -EINVAL;
+                        goto out3;
+                }
+                *next_space = this_colour;
+                next_space++;
+                colour_count++;
+        }
+
+        /* Program colours from cursor icon into palette */
+        for (i = 0; i < colour_count; i++) {
+                if (i <= 2)
+                        reg_index = 0x8 + i*0x4;
+                else
+                        reg_index = 0x60 + i*0x3;
+                WREG_DAC(reg_index, colour_set[i] & 0xff);
+                WREG_DAC(reg_index+1, colour_set[i]>>8 & 0xff);
+                WREG_DAC(reg_index+2, colour_set[i]>>16 & 0xff);
+                BUG_ON((colour_set[i]>>24 & 0xff) != 0xff);
+        }
+
+        /* Map up-coming buffer to write colour indices */
+        if (!pixels_prev->kmap.virtual) {
+                ret = ttm_bo_kmap(&pixels_prev->bo, 0,
+                                  pixels_prev->bo.num_pages,
+                                  &pixels_prev->kmap);
+                if (ret) {
+                        dev_err(&dev->pdev->dev, "failed to kmap cursor updates\n");
+                        goto out3;
+                }
+        }
+
+        /* now write colour indices into hardware cursor buffer */
+        for (row = 0; row < 64; row++) {
+                memset(&this_row[0], 0, 48);
+                for (col = 0; col < 64; col++) {
+                        this_colour = ioread32(bo->kmap.virtual + 4*(col + 64*row));
+                        /* write transparent pixels */
+                        if (this_colour>>24 == 0x0) {
+                                this_row[47 - col/8] |= 0x80>>(col%8);
+                                continue;
+                        }
+
+                        /* write colour index here */
+                        for (i = 0; i < colour_count; i++) {
+                                if (colour_set[i] == this_colour) {
+                                        if (col % 2)
+                                                this_row[col/2] |= i<<4;
+                                        else
+                                                this_row[col/2] |= i;
+                                        break;
+                                }
+                        }
+                }
+                memcpy_toio(pixels_prev->kmap.virtual + row*48, &this_row[0], 48);
+        }
+
+        /* Program gpu address of cursor buffer */
+        if (pixels_prev == pixels_1)
+                gpu_addr = mdev->cursor.pixels_1_gpu_addr;
+        else
+                gpu_addr = mdev->cursor.pixels_2_gpu_addr;
+        WREG_DAC(MGA1064_CURSOR_BASE_ADR_LOW, (u8)((gpu_addr>>10) & 0xff));
+        WREG_DAC(MGA1064_CURSOR_BASE_ADR_HI, (u8)((gpu_addr>>18) & 0x3f));
+
+        /* Adjust cursor control register to turn on the cursor */
+        WREG_DAC(MGA1064_CURSOR_CTL, 4); /* 16-colour palletized cursor mode */
+
+        /* Now swap internal buffer pointers */
+        if (mdev->cursor.pixels_1 == mdev->cursor.pixels_prev) {
+                mdev->cursor.pixels_prev = mdev->cursor.pixels_2;
+                mdev->cursor.pixels_current = mdev->cursor.pixels_1;
+        } else if (mdev->cursor.pixels_1 == mdev->cursor.pixels_current) {
+                mdev->cursor.pixels_prev = mdev->cursor.pixels_1;
+                mdev->cursor.pixels_current = mdev->cursor.pixels_2;
+        } else {
+                BUG();
+        }
+        ret = 0;
+
+        ttm_bo_kunmap(&pixels_prev->kmap);
+ out3:
+        ttm_bo_kunmap(&bo->kmap);
+ out2:
+        mgag200_bo_unreserve(bo);
+ out1:
+        if (ret)
+                mga_hide_cursor(mdev);
+        mgag200_bo_unreserve(pixels_1);
+        mgag200_bo_unreserve(pixels_2);
+        return ret;
+}
+
+int mga_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
+{
+        struct mga_device *mdev = (struct mga_device *)crtc->dev->dev_private;
+        /* Our origin is at (64,64) */
+        x += 64;
+        y += 64;
+
+        BUG_ON(x <= 0);
+        BUG_ON(y <= 0);
+        BUG_ON(x & ~0xffff);
+        BUG_ON(y & ~0xffff);
+
+        WREG8(MGA_CURPOSXL, x & 0xff);
+        WREG8(MGA_CURPOSXH, (x>>8) & 0xff);
+
+        WREG8(MGA_CURPOSYL, y & 0xff);
+        WREG8(MGA_CURPOSYH, (y>>8) & 0xff);
+        return 0;
+}
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index bf29b2f4d68d..12e2499d9352 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -149,6 +149,21 @@ struct mga_connector {
         struct mga_i2c_chan *i2c;
 };
 
+struct mga_cursor {
+        /*
+           We have to have 2 buffers for the cursor to avoid occasional
+           corruption while switching cursor icons.
+           If either of these is NULL, then don't do hardware cursors, and
+           fall back to software.
+        */
+        struct mgag200_bo *pixels_1;
+        struct mgag200_bo *pixels_2;
+        u64 pixels_1_gpu_addr, pixels_2_gpu_addr;
+        /* The currently displayed icon, this points to one of pixels_1, or pixels_2 */
+        struct mgag200_bo *pixels_current;
+        /* The previously displayed icon */
+        struct mgag200_bo *pixels_prev;
+};
 
 struct mga_mc {
         resource_size_t                 vram_size;
@@ -181,6 +196,7 @@ struct mga_device {
         struct mga_mode_info            mode_info;
 
         struct mga_fbdev *mfbdev;
+        struct mga_cursor cursor;
 
         bool                            suspended;
         int                             num_crtc;
@@ -198,7 +214,8 @@ struct mga_device {
                 struct ttm_bo_device bdev;
         } ttm;
 
-        u32 reg_1e24; /* SE model number */
+        /* SE model number stored in reg 0x1e24 */
+        u32 unique_rev_id;
 };
 
 
@@ -263,8 +280,24 @@ void mgag200_i2c_destroy(struct mga_i2c_chan *i2c);
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 void mgag200_ttm_placement(struct mgag200_bo *bo, int domain);
 
-int mgag200_bo_reserve(struct mgag200_bo *bo, bool no_wait);
-void mgag200_bo_unreserve(struct mgag200_bo *bo);
+static inline int mgag200_bo_reserve(struct mgag200_bo *bo, bool no_wait)
+{
+        int ret;
+
+        ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
+        if (ret) {
+                if (ret != -ERESTARTSYS && ret != -EBUSY)
+                        DRM_ERROR("reserve failed %p\n", bo);
+                return ret;
+        }
+        return 0;
+}
+
+static inline void mgag200_bo_unreserve(struct mgag200_bo *bo)
+{
+        ttm_bo_unreserve(&bo->bo);
+}
+
 int mgag200_bo_create(struct drm_device *dev, int size, int align,
                       uint32_t flags, struct mgag200_bo **pastbo);
 int mgag200_mm_init(struct mga_device *mdev);
@@ -273,4 +306,9 @@ int mgag200_mmap(struct file *filp, struct vm_area_struct *vma);
 int mgag200_bo_pin(struct mgag200_bo *bo, u32 pl_flag, u64 *gpu_addr);
 int mgag200_bo_unpin(struct mgag200_bo *bo);
 int mgag200_bo_push_sysram(struct mgag200_bo *bo);
+                           /* mgag200_cursor.c */
+int mga_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
+                                                uint32_t handle, uint32_t width, uint32_t height);
+int mga_crtc_cursor_move(struct drm_crtc *crtc, int x, int y);
+
 #endif                          /* __MGAG200_DRV_H__ */
diff --git a/drivers/gpu/drm/mgag200/mgag200_fb.c b/drivers/gpu/drm/mgag200/mgag200_fb.c
index 5da824ce9ba1..964f58cee5ea 100644
--- a/drivers/gpu/drm/mgag200/mgag200_fb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_fb.c
@@ -27,7 +27,7 @@ static void mga_dirty_update(struct mga_fbdev *mfbdev,
         struct mgag200_bo *bo;
         int src_offset, dst_offset;
         int bpp = (mfbdev->mfb.base.bits_per_pixel + 7)/8;
-        int ret;
+        int ret = -EBUSY;
         bool unmap = false;
         bool store_for_later = false;
         int x2, y2;
@@ -41,7 +41,8 @@ static void mga_dirty_update(struct mga_fbdev *mfbdev,
          * then the BO is being moved and we should
          * store up the damage until later.
          */
-        ret = mgag200_bo_reserve(bo, true);
+        if (!in_interrupt())
+                ret = mgag200_bo_reserve(bo, true);
         if (ret) {
                 if (ret != -EBUSY)
                         return;
diff --git a/drivers/gpu/drm/mgag200/mgag200_main.c b/drivers/gpu/drm/mgag200/mgag200_main.c
index 99059237da38..9fa5685baee0 100644
--- a/drivers/gpu/drm/mgag200/mgag200_main.c
+++ b/drivers/gpu/drm/mgag200/mgag200_main.c
@@ -176,7 +176,7 @@ static int mgag200_device_init(struct drm_device *dev,
 
         /* stash G200 SE model number for later use */
         if (IS_G200_SE(mdev))
-                mdev->reg_1e24 = RREG32(0x1e24);
+                mdev->unique_rev_id = RREG32(0x1e24);
 
         ret = mga_vram_init(mdev);
         if (ret)
@@ -209,7 +209,7 @@ int mgag200_driver_load(struct drm_device *dev, unsigned long flags)
         r = mgag200_device_init(dev, flags);
         if (r) {
                 dev_err(&dev->pdev->dev, "Fatal error during GPU init: %d\n", r);
-                goto out;
+                return r;
         }
         r = mgag200_mm_init(mdev);
         if (r)
@@ -221,8 +221,27 @@ int mgag200_driver_load(struct drm_device *dev, unsigned long flags)
         dev->mode_config.prefer_shadow = 1;
 
         r = mgag200_modeset_init(mdev);
-        if (r)
+        if (r) {
                 dev_err(&dev->pdev->dev, "Fatal error during modeset init: %d\n", r);
+                goto out;
+        }
+
+        /* Make small buffers to store a hardware cursor (double buffered icon updates) */
+        mgag200_bo_create(dev, roundup(48*64, PAGE_SIZE), 0, 0,
+                                          &mdev->cursor.pixels_1);
+        mgag200_bo_create(dev, roundup(48*64, PAGE_SIZE), 0, 0,
+                                          &mdev->cursor.pixels_2);
+        if (!mdev->cursor.pixels_2 || !mdev->cursor.pixels_1)
+                goto cursor_nospace;
+        mdev->cursor.pixels_current = mdev->cursor.pixels_1;
+        mdev->cursor.pixels_prev = mdev->cursor.pixels_2;
+        goto cursor_done;
+ cursor_nospace:
+        mdev->cursor.pixels_1 = NULL;
+        mdev->cursor.pixels_2 = NULL;
+        dev_warn(&dev->pdev->dev, "Could not allocate space for cursors. Not doing hardware cursors.\n");
+ cursor_done:
+
 out:
         if (r)
                 mgag200_driver_unload(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index ee66badc8bb6..251784aa2225 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -1008,7 +1008,7 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
 
 
         if (IS_G200_SE(mdev)) {
-                if (mdev->reg_1e24 >= 0x02) {
+                if (mdev->unique_rev_id >= 0x02) {
                         u8 hi_pri_lvl;
                         u32 bpp;
                         u32 mb;
@@ -1038,7 +1038,7 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
                         WREG8(MGAREG_CRTCEXT_DATA, hi_pri_lvl);
                 } else {
                         WREG8(MGAREG_CRTCEXT_INDEX, 0x06);
-                        if (mdev->reg_1e24 >= 0x01)
+                        if (mdev->unique_rev_id >= 0x01)
                                 WREG8(MGAREG_CRTCEXT_DATA, 0x03);
                         else
                                 WREG8(MGAREG_CRTCEXT_DATA, 0x04);
@@ -1253,6 +1253,8 @@ static void mga_crtc_destroy(struct drm_crtc *crtc)
 
 /* These provide the minimum set of functions required to handle a CRTC */
 static const struct drm_crtc_funcs mga_crtc_funcs = {
+        .cursor_set = mga_crtc_cursor_set,
+        .cursor_move = mga_crtc_cursor_move,
         .gamma_set = mga_crtc_gamma_set,
         .set_config = drm_crtc_helper_set_config,
         .destroy = mga_crtc_destroy,
@@ -1410,6 +1412,32 @@ static int mga_vga_get_modes(struct drm_connector *connector)
         return ret;
 }
 
+static uint32_t mga_vga_calculate_mode_bandwidth(struct drm_display_mode *mode,
+                                                        int bits_per_pixel)
+{
+        uint32_t total_area, divisor;
+        int64_t active_area, pixels_per_second, bandwidth;
+        uint64_t bytes_per_pixel = (bits_per_pixel + 7) / 8;
+
+        divisor = 1024;
+
+        if (!mode->htotal || !mode->vtotal || !mode->clock)
+                return 0;
+
+        active_area = mode->hdisplay * mode->vdisplay;
+        total_area = mode->htotal * mode->vtotal;
+
+        pixels_per_second = active_area * mode->clock * 1000;
+        do_div(pixels_per_second, total_area);
+
+        bandwidth = pixels_per_second * bytes_per_pixel * 100;
+        do_div(bandwidth, divisor);
+
+        return (uint32_t)(bandwidth);
+}
+
+#define MODE_BANDWIDTH  MODE_BAD
+
 static int mga_vga_mode_valid(struct drm_connector *connector,
                                  struct drm_display_mode *mode)
 {
@@ -1421,7 +1449,45 @@ static int mga_vga_mode_valid(struct drm_connector *connector,
         int bpp = 32;
         int i = 0;
 
-        /* FIXME: Add bandwidth and g200se limitations */
+        if (IS_G200_SE(mdev)) {
+                if (mdev->unique_rev_id == 0x01) {
+                        if (mode->hdisplay > 1600)
+                                return MODE_VIRTUAL_X;
+                        if (mode->vdisplay > 1200)
+                                return MODE_VIRTUAL_Y;
+                        if (mga_vga_calculate_mode_bandwidth(mode, bpp)
+                                > (24400 * 1024))
+                                return MODE_BANDWIDTH;
+                } else if (mdev->unique_rev_id >= 0x02) {
+                        if (mode->hdisplay > 1920)
+                                return MODE_VIRTUAL_X;
+                        if (mode->vdisplay > 1200)
+                                return MODE_VIRTUAL_Y;
+                        if (mga_vga_calculate_mode_bandwidth(mode, bpp)
+                                > (30100 * 1024))
+                                return MODE_BANDWIDTH;
+                }
+        } else if (mdev->type == G200_WB) {
+                if (mode->hdisplay > 1280)
+                        return MODE_VIRTUAL_X;
+                if (mode->vdisplay > 1024)
+                        return MODE_VIRTUAL_Y;
+                if (mga_vga_calculate_mode_bandwidth(mode,
+                        bpp > (31877 * 1024)))
+                        return MODE_BANDWIDTH;
+        } else if (mdev->type == G200_EV &&
+                (mga_vga_calculate_mode_bandwidth(mode, bpp)
+                        > (32700 * 1024))) {
+                return MODE_BANDWIDTH;
+        } else if (mode->type == G200_EH &&
+                (mga_vga_calculate_mode_bandwidth(mode, bpp)
+                        > (37500 * 1024))) {
+                return MODE_BANDWIDTH;
+        } else if (mode->type == G200_ER &&
+                (mga_vga_calculate_mode_bandwidth(mode,
+                        bpp) > (55000 * 1024))) {
+                return MODE_BANDWIDTH;
+        }
 
         if (mode->crtc_hdisplay > 2048 || mode->crtc_hsync_start > 4096 ||
             mode->crtc_hsync_end > 4096 || mode->crtc_htotal > 4096 ||
diff --git a/drivers/gpu/drm/mgag200/mgag200_reg.h b/drivers/gpu/drm/mgag200/mgag200_reg.h
index fb24d8655feb..3ae442a64bd6 100644
--- a/drivers/gpu/drm/mgag200/mgag200_reg.h
+++ b/drivers/gpu/drm/mgag200/mgag200_reg.h
@@ -235,7 +235,11 @@
 #define MGAREG_CRTCEXT_INDEX    0x1fde
 #define MGAREG_CRTCEXT_DATA     0x1fdf
 
-
+/* Cursor X and Y position */
+#define MGA_CURPOSXL 0x3c0c
+#define MGA_CURPOSXH 0x3c0d
+#define MGA_CURPOSYL 0x3c0e
+#define MGA_CURPOSYH 0x3c0f
 
 /* MGA bits for registers PCI_OPTION_REG */
 #define MGA1064_OPT_SYS_CLK_PCI                 ( 0x00 << 0 )
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 401c9891d3a8..3acb2b044c7b 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -270,26 +270,20 @@ int mgag200_mm_init(struct mga_device *mdev)
                 return ret;
         }
 
-        mdev->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 0),
-                                    pci_resource_len(dev->pdev, 0),
-                                    DRM_MTRR_WC);
+        mdev->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0),
+                                         pci_resource_len(dev->pdev, 0));
 
         return 0;
 }
 
 void mgag200_mm_fini(struct mga_device *mdev)
 {
-        struct drm_device *dev = mdev->dev;
         ttm_bo_device_release(&mdev->ttm.bdev);
 
         mgag200_ttm_global_release(mdev);
 
-        if (mdev->fb_mtrr >= 0) {
-                drm_mtrr_del(mdev->fb_mtrr,
-                             pci_resource_start(dev->pdev, 0),
-                             pci_resource_len(dev->pdev, 0), DRM_MTRR_WC);
-                mdev->fb_mtrr = -1;
-        }
+        arch_phys_wc_del(mdev->fb_mtrr);
+        mdev->fb_mtrr = 0;
 }
 
 void mgag200_ttm_placement(struct mgag200_bo *bo, int domain)
@@ -309,24 +303,6 @@ void mgag200_ttm_placement(struct mgag200_bo *bo, int domain)
         bo->placement.num_busy_placement = c;
 }
 
-int mgag200_bo_reserve(struct mgag200_bo *bo, bool no_wait)
-{
-        int ret;
-
-        ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
-        if (ret) {
-                if (ret != -ERESTARTSYS && ret != -EBUSY)
-                        DRM_ERROR("reserve failed %p %d\n", bo, ret);
-                return ret;
-        }
-        return 0;
-}
-
-void mgag200_bo_unreserve(struct mgag200_bo *bo)
-{
-        ttm_bo_unreserve(&bo->bo);
-}
-
 int mgag200_bo_create(struct drm_device *dev, int size, int align,
                   uint32_t flags, struct mgag200_bo **pmgabo)
 {
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index a7ff6d5a34b9..ff80f12480ea 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -15,6 +15,13 @@ config DRM_NOUVEAU
         select ACPI_WMI if ACPI && X86
         select MXM_WMI if ACPI && X86
         select POWER_SUPPLY
+        # Similar to i915, we need to select ACPI_VIDEO and it's dependencies
+        select BACKLIGHT_LCD_SUPPORT if ACPI && X86
+        select BACKLIGHT_CLASS_DEVICE if ACPI && X86
+        select VIDEO_OUTPUT_CONTROL if ACPI && X86
+        select INPUT if ACPI && X86
+        select THERMAL if ACPI && X86
+        select ACPI_VIDEO if ACPI && X86
         help
           Choose this option for open-source nVidia support.
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 1c4c6c9161ac..8f467e7bfd19 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -129,6 +129,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
 
         if (chan->ntfy) {
                 nouveau_bo_vma_del(chan->ntfy, &chan->ntfy_vma);
+                nouveau_bo_unpin(chan->ntfy);
                 drm_gem_object_unreference_unlocked(chan->ntfy->gem);
         }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 0b6c296e6ef3..708b2d1c0037 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -26,6 +26,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/ttm/ttm_execbuf_util.h>
 
 #include "nouveau_fbcon.h"
 #include "dispnv04/hw.h"
@@ -462,51 +463,6 @@ nouveau_display_resume(struct drm_device *dev)
 }
 
 static int
-nouveau_page_flip_reserve(struct nouveau_bo *old_bo,
-                          struct nouveau_bo *new_bo)
-{
-        int ret;
-
-        ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM);
-        if (ret)
-                return ret;
-
-        ret = ttm_bo_reserve(&new_bo->bo, false, false, false, 0);
-        if (ret)
-                goto fail;
-
-        if (likely(old_bo != new_bo)) {
-                ret = ttm_bo_reserve(&old_bo->bo, false, false, false, 0);
-                if (ret)
-                        goto fail_unreserve;
-        }
-
-        return 0;
-
-fail_unreserve:
-        ttm_bo_unreserve(&new_bo->bo);
-fail:
-        nouveau_bo_unpin(new_bo);
-        return ret;
-}
-
-static void
-nouveau_page_flip_unreserve(struct nouveau_bo *old_bo,
-                            struct nouveau_bo *new_bo,
-                            struct nouveau_fence *fence)
-{
-        nouveau_bo_fence(new_bo, fence);
-        ttm_bo_unreserve(&new_bo->bo);
-
-        if (likely(old_bo != new_bo)) {
-                nouveau_bo_fence(old_bo, fence);
-                ttm_bo_unreserve(&old_bo->bo);
-        }
-
-        nouveau_bo_unpin(old_bo);
-}
-
-static int
 nouveau_page_flip_emit(struct nouveau_channel *chan,
                        struct nouveau_bo *old_bo,
                        struct nouveau_bo *new_bo,
@@ -568,6 +524,9 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         struct nouveau_page_flip_state *s;
         struct nouveau_channel *chan = NULL;
         struct nouveau_fence *fence;
+        struct list_head res;
+        struct ttm_validate_buffer res_val[2];
+        struct ww_acquire_ctx ticket;
         int ret;
 
         if (!drm->channel)
@@ -577,25 +536,43 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         if (!s)
                 return -ENOMEM;
 
-        /* Don't let the buffers go away while we flip */
-        ret = nouveau_page_flip_reserve(old_bo, new_bo);
-        if (ret)
-                goto fail_free;
-
-        /* Initialize a page flip struct */
-        *s = (struct nouveau_page_flip_state)
-                { { }, event, nouveau_crtc(crtc)->index,
-                  fb->bits_per_pixel, fb->pitches[0], crtc->x, crtc->y,
-                  new_bo->bo.offset };
-
         /* Choose the channel the flip will be handled in */
+        spin_lock(&old_bo->bo.bdev->fence_lock);
         fence = new_bo->bo.sync_obj;
         if (fence)
                 chan = fence->channel;
         if (!chan)
                 chan = drm->channel;
+        spin_unlock(&old_bo->bo.bdev->fence_lock);
+
         mutex_lock(&chan->cli->mutex);
 
+        if (new_bo != old_bo) {
+                ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM);
+                if (likely(!ret)) {
+                        res_val[0].bo = &old_bo->bo;
+                        res_val[1].bo = &new_bo->bo;
+                        INIT_LIST_HEAD(&res);
+                        list_add_tail(&res_val[0].head, &res);
+                        list_add_tail(&res_val[1].head, &res);
+                        ret = ttm_eu_reserve_buffers(&ticket, &res);
+                        if (ret)
+                                nouveau_bo_unpin(new_bo);
+                }
+        } else
+                ret = ttm_bo_reserve(&new_bo->bo, false, false, false, 0);
+
+        if (ret) {
+                mutex_unlock(&chan->cli->mutex);
+                goto fail_free;
+        }
+
+        /* Initialize a page flip struct */
+        *s = (struct nouveau_page_flip_state)
+                { { }, event, nouveau_crtc(crtc)->index,
+                  fb->bits_per_pixel, fb->pitches[0], crtc->x, crtc->y,
+                  new_bo->bo.offset };
+
         /* Emit a page flip */
         if (nv_device(drm->device)->card_type >= NV_50) {
                 ret = nv50_display_flip_next(crtc, fb, chan, 0);
@@ -613,12 +590,22 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         /* Update the crtc struct and cleanup */
         crtc->fb = fb;
 
-        nouveau_page_flip_unreserve(old_bo, new_bo, fence);
+        if (old_bo != new_bo) {
+                ttm_eu_fence_buffer_objects(&ticket, &res, fence);
+                nouveau_bo_unpin(old_bo);
+        } else {
+                nouveau_bo_fence(new_bo, fence);
+                ttm_bo_unreserve(&new_bo->bo);
+        }
         nouveau_fence_unref(&fence);
         return 0;
 
 fail_unreserve:
-        nouveau_page_flip_unreserve(old_bo, new_bo, NULL);
+        if (old_bo != new_bo) {
+                ttm_eu_backoff_reservation(&ticket, &res);
+                nouveau_bo_unpin(new_bo);
+        } else
+                ttm_bo_unreserve(&new_bo->bo);
 fail_free:
         kfree(s);
         return ret;
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 383f4e6ea9d1..218a4b522fe5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -702,6 +702,7 @@ driver = {
         .gem_prime_export = drm_gem_prime_export,
         .gem_prime_import = drm_gem_prime_import,
         .gem_prime_pin = nouveau_gem_prime_pin,
+        .gem_prime_unpin = nouveau_gem_prime_unpin,
         .gem_prime_get_sg_table = nouveau_gem_prime_get_sg_table,
         .gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table,
         .gem_prime_vmap = nouveau_gem_prime_vmap,
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 51fe6406edab..9352010030e9 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -289,16 +289,13 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
         ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM);
         if (ret) {
                 NV_ERROR(drm, "failed to pin fb: %d\n", ret);
-                nouveau_bo_ref(NULL, &nvbo);
-                goto out;
+                goto out_unref;
         }
 
         ret = nouveau_bo_map(nvbo);
         if (ret) {
                 NV_ERROR(drm, "failed to map fb: %d\n", ret);
-                nouveau_bo_unpin(nvbo);
-                nouveau_bo_ref(NULL, &nvbo);
-                goto out;
+                goto out_unpin;
         }
 
         chan = nouveau_nofbaccel ? NULL : drm->channel;
@@ -316,13 +313,14 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
         info = framebuffer_alloc(0, &pdev->dev);
         if (!info) {
                 ret = -ENOMEM;
-                goto out_unref;
+                goto out_unlock;
         }
 
         ret = fb_alloc_cmap(&info->cmap, 256, 0);
         if (ret) {
                 ret = -ENOMEM;
-                goto out_unref;
+                framebuffer_release(info);
+                goto out_unlock;
         }
 
         info->par = fbcon;
@@ -337,7 +335,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
         fbcon->helper.fbdev = info;
 
         strcpy(info->fix.id, "nouveaufb");
-        if (nouveau_nofbaccel)
+        if (!chan)
                 info->flags = FBINFO_DEFAULT | FBINFO_HWACCEL_DISABLED;
         else
                 info->flags = FBINFO_DEFAULT | FBINFO_HWACCEL_COPYAREA |
@@ -383,8 +381,14 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
         vga_switcheroo_client_fb_set(dev->pdev, info);
         return 0;
 
-out_unref:
+out_unlock:
         mutex_unlock(&dev->struct_mutex);
+        if (chan)
+                nouveau_bo_vma_del(nvbo, &fbcon->nouveau_fb.vma);
+out_unpin:
+        nouveau_bo_unpin(nvbo);
+out_unref:
+        nouveau_bo_ref(NULL, &nvbo);
 out:
         return ret;
 }
@@ -413,6 +417,7 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon)
         if (nouveau_fb->nvbo) {
                 nouveau_bo_unmap(nouveau_fb->nvbo);
                 nouveau_bo_vma_del(nouveau_fb->nvbo, &nouveau_fb->vma);
+                nouveau_bo_unpin(nouveau_fb->nvbo);
                 drm_gem_object_unreference_unlocked(nouveau_fb->nvbo->gem);
                 nouveau_fb->nvbo = NULL;
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index c0e324b557c1..e72d09c068a8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -50,7 +50,8 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
                 return;
         nvbo->gem = NULL;
 
-        if (unlikely(nvbo->pin_refcnt)) {
+        /* Lockdep hates you for doing reserve with gem object lock held */
+        if (WARN_ON_ONCE(nvbo->pin_refcnt)) {
                 nvbo->pin_refcnt = 1;
                 nouveau_bo_unpin(nvbo);
         }
@@ -309,10 +310,12 @@ struct validate_op {
         struct list_head vram_list;
         struct list_head gart_list;
         struct list_head both_list;
+        struct ww_acquire_ctx ticket;
 };
 
 static void
-validate_fini_list(struct list_head *list, struct nouveau_fence *fence)
+validate_fini_list(struct list_head *list, struct nouveau_fence *fence,
+                   struct ww_acquire_ctx *ticket)
 {
         struct list_head *entry, *tmp;
         struct nouveau_bo *nvbo;
@@ -329,17 +332,24 @@ validate_fini_list(struct list_head *list, struct nouveau_fence *fence)
 
                 list_del(&nvbo->entry);
                 nvbo->reserved_by = NULL;
-                ttm_bo_unreserve(&nvbo->bo);
+                ttm_bo_unreserve_ticket(&nvbo->bo, ticket);
                 drm_gem_object_unreference_unlocked(nvbo->gem);
         }
 }
 
 static void
-validate_fini(struct validate_op *op, struct nouveau_fence* fence)
+validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence)
 {
-        validate_fini_list(&op->vram_list, fence);
-        validate_fini_list(&op->gart_list, fence);
-        validate_fini_list(&op->both_list, fence);
+        validate_fini_list(&op->vram_list, fence, &op->ticket);
+        validate_fini_list(&op->gart_list, fence, &op->ticket);
+        validate_fini_list(&op->both_list, fence, &op->ticket);
+}
+
+static void
+validate_fini(struct validate_op *op, struct nouveau_fence *fence)
+{
+        validate_fini_no_ticket(op, fence);
+        ww_acquire_fini(&op->ticket);
 }
 
 static int
@@ -349,13 +359,11 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 {
         struct nouveau_cli *cli = nouveau_cli(file_priv);
         struct drm_device *dev = chan->drm->dev;
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        uint32_t sequence;
         int trycnt = 0;
         int ret, i;
         struct nouveau_bo *res_bo = NULL;
 
-        sequence = atomic_add_return(1, &drm->ttm.validate_sequence);
+        ww_acquire_init(&op->ticket, &reservation_ww_class);
 retry:
         if (++trycnt > 100000) {
                 NV_ERROR(cli, "%s failed and gave up.\n", __func__);
@@ -370,6 +378,7 @@ retry:
                 gem = drm_gem_object_lookup(dev, file_priv, b->handle);
                 if (!gem) {
                         NV_ERROR(cli, "Unknown handle 0x%08x\n", b->handle);
+                        ww_acquire_done(&op->ticket);
                         validate_fini(op, NULL);
                         return -ENOENT;
                 }
@@ -384,21 +393,23 @@ retry:
                         NV_ERROR(cli, "multiple instances of buffer %d on "
                                       "validation list\n", b->handle);
                         drm_gem_object_unreference_unlocked(gem);
+                        ww_acquire_done(&op->ticket);
                         validate_fini(op, NULL);
                         return -EINVAL;
                 }
 
-                ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence);
+                ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket);
                 if (ret) {
-                        validate_fini(op, NULL);
-                        if (unlikely(ret == -EAGAIN)) {
-                                sequence = atomic_add_return(1, &drm->ttm.validate_sequence);
+                        validate_fini_no_ticket(op, NULL);
+                        if (unlikely(ret == -EDEADLK)) {
                                 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
-                                                              sequence);
+                                                              &op->ticket);
                                 if (!ret)
                                         res_bo = nvbo;
                         }
                         if (unlikely(ret)) {
+                                ww_acquire_done(&op->ticket);
+                                ww_acquire_fini(&op->ticket);
                                 drm_gem_object_unreference_unlocked(gem);
                                 if (ret != -ERESTARTSYS)
                                         NV_ERROR(cli, "fail reserve\n");
@@ -422,6 +433,7 @@ retry:
                         NV_ERROR(cli, "invalid valid domains: 0x%08x\n",
                                  b->valid_domains);
                         list_add_tail(&nvbo->entry, &op->both_list);
+                        ww_acquire_done(&op->ticket);
                         validate_fini(op, NULL);
                         return -EINVAL;
                 }
@@ -429,6 +441,7 @@ retry:
                         goto retry;
         }
 
+        ww_acquire_done(&op->ticket);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.h b/drivers/gpu/drm/nouveau/nouveau_gem.h
index 8d7a3f0aeb86..502e4290aa8f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.h
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.h
@@ -36,6 +36,7 @@ extern int nouveau_gem_ioctl_info(struct drm_device *, void *,
                                   struct drm_file *);
 
 extern int nouveau_gem_prime_pin(struct drm_gem_object *);
+extern void nouveau_gem_prime_unpin(struct drm_gem_object *);
 extern struct sg_table *nouveau_gem_prime_get_sg_table(struct drm_gem_object *);
 extern struct drm_gem_object *nouveau_gem_prime_import_sg_table(
         struct drm_device *, size_t size, struct sg_table *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_prime.c b/drivers/gpu/drm/nouveau/nouveau_prime.c
index f53e10874cae..e90468d5e5c0 100644
--- a/drivers/gpu/drm/nouveau/nouveau_prime.c
+++ b/drivers/gpu/drm/nouveau/nouveau_prime.c
@@ -84,7 +84,7 @@ struct drm_gem_object *nouveau_gem_prime_import_sg_table(struct drm_device *dev,
 int nouveau_gem_prime_pin(struct drm_gem_object *obj)
 {
         struct nouveau_bo *nvbo = nouveau_gem_object(obj);
-        int ret = 0;
+        int ret;
 
         /* pin buffer into GTT */
         ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_TT);
@@ -93,3 +93,10 @@ int nouveau_gem_prime_pin(struct drm_gem_object *obj)
 
         return 0;
 }
+
+void nouveau_gem_prime_unpin(struct drm_gem_object *obj)
+{
+        struct nouveau_bo *nvbo = nouveau_gem_object(obj);
+
+        nouveau_bo_unpin(nvbo);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index d0382f7e86c8..19e3757291fb 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -393,9 +393,8 @@ nouveau_ttm_init(struct nouveau_drm *drm)
                 return ret;
         }
 
-        drm->ttm.mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 1),
-                                     pci_resource_len(dev->pdev, 1),
-                                     DRM_MTRR_WC);
+        drm->ttm.mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 1),
+                                         pci_resource_len(dev->pdev, 1));
 
         /* GART init */
         if (drm->agp.stat != ENABLED) {
@@ -428,10 +427,6 @@ nouveau_ttm_fini(struct nouveau_drm *drm)
 
         nouveau_ttm_global_release(drm);
 
-        if (drm->ttm.mtrr >= 0) {
-                drm_mtrr_del(drm->ttm.mtrr,
-                             pci_resource_start(drm->dev->pdev, 1),
-                             pci_resource_len(drm->dev->pdev, 1), DRM_MTRR_WC);
-                drm->ttm.mtrr = -1;
-        }
+        arch_phys_wc_del(drm->ttm.mtrr);
+        drm->ttm.mtrr = 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/Kconfig b/drivers/gpu/drm/omapdrm/Kconfig
index 09f65dc3d2c8..20c41e73d448 100644
--- a/drivers/gpu/drm/omapdrm/Kconfig
+++ b/drivers/gpu/drm/omapdrm/Kconfig
@@ -1,7 +1,7 @@
 
 config DRM_OMAP
         tristate "OMAP DRM"
-        depends on DRM && !CONFIG_FB_OMAP2
+        depends on DRM
         depends on ARCH_OMAP2PLUS || ARCH_MULTIPLATFORM
         depends on OMAP2_DSS
         select DRM_KMS_HELPER
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index 79b200aee18a..ef161ea982e6 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -253,10 +253,6 @@ static int omap_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
                         NULL, NULL);
 }
 
-static void omap_crtc_load_lut(struct drm_crtc *crtc)
-{
-}
-
 static void vblank_cb(void *arg)
 {
         struct drm_crtc *crtc = arg;
@@ -366,7 +362,6 @@ static const struct drm_crtc_helper_funcs omap_crtc_helper_funcs = {
         .prepare = omap_crtc_prepare,
         .commit = omap_crtc_commit,
         .mode_set_base = omap_crtc_mode_set_base,
-        .load_lut = omap_crtc_load_lut,
 };
 
 const struct omap_video_timings *omap_crtc_timings(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index b11ce609fcc2..002988d09021 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -281,21 +281,7 @@ fail:
         return ret;
 }
 
-static void omap_crtc_fb_gamma_set(struct drm_crtc *crtc,
-                u16 red, u16 green, u16 blue, int regno)
-{
-        DBG("fbdev: set gamma");
-}
-
-static void omap_crtc_fb_gamma_get(struct drm_crtc *crtc,
-                u16 *red, u16 *green, u16 *blue, int regno)
-{
-        DBG("fbdev: get gamma");
-}
-
 static struct drm_fb_helper_funcs omap_fb_helper_funcs = {
-        .gamma_set = omap_crtc_fb_gamma_set,
-        .gamma_get = omap_crtc_fb_gamma_get,
         .fb_probe = omap_fbdev_create,
 };
 
diff --git a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
index be7cd97a0db0..4fcca8d42796 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
@@ -136,44 +136,21 @@ static void omap_gem_dmabuf_kunmap(struct dma_buf *buffer,
         kunmap(pages[page_num]);
 }
 
-/*
- * TODO maybe we can split up drm_gem_mmap to avoid duplicating
- * some here.. or at least have a drm_dmabuf_mmap helper.
- */
 static int omap_gem_dmabuf_mmap(struct dma_buf *buffer,
                 struct vm_area_struct *vma)
 {
         struct drm_gem_object *obj = buffer->priv;
+        struct drm_device *dev = obj->dev;
         int ret = 0;
 
         if (WARN_ON(!obj->filp))
                 return -EINVAL;
 
-        /* Check for valid size. */
-        if (omap_gem_mmap_size(obj) < vma->vm_end - vma->vm_start) {
-                ret = -EINVAL;
-                goto out_unlock;
-        }
-
-        if (!obj->dev->driver->gem_vm_ops) {
-                ret = -EINVAL;
-                goto out_unlock;
-        }
-
-        vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
-        vma->vm_ops = obj->dev->driver->gem_vm_ops;
-        vma->vm_private_data = obj;
-        vma->vm_page_prot =  pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
-
-        /* Take a ref for this mapping of the object, so that the fault
-         * handler can dereference the mmap offset's pointer to the object.
-         * This reference is cleaned up by the corresponding vm_close
-         * (which should happen whether the vma was created by this call, or
-         * by a vm_open due to mremap or partial unmap or whatever).
-         */
-        vma->vm_ops->open(vma);
-
-out_unlock:
+        mutex_lock(&dev->struct_mutex);
+        ret = drm_gem_mmap_obj(obj, omap_gem_mmap_size(obj), vma);
+        mutex_unlock(&dev->struct_mutex);
+        if (ret < 0)
+                return ret;
 
         return omap_gem_mmap_obj(obj, vma);
 }
diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c
index f86771481317..93c2f2cceb51 100644
--- a/drivers/gpu/drm/qxl/qxl_cmd.c
+++ b/drivers/gpu/drm/qxl/qxl_cmd.c
@@ -49,6 +49,11 @@ void qxl_ring_free(struct qxl_ring *ring)
         kfree(ring);
 }
 
+void qxl_ring_init_hdr(struct qxl_ring *ring)
+{
+        ring->ring->header.notify_on_prod = ring->n_elements;
+}
+
 struct qxl_ring *
 qxl_ring_create(struct qxl_ring_header *header,
                 int element_size,
@@ -69,7 +74,7 @@ qxl_ring_create(struct qxl_ring_header *header,
         ring->prod_notify = prod_notify;
         ring->push_event = push_event;
         if (set_prod_notify)
-                header->notify_on_prod = ring->n_elements;
+                qxl_ring_init_hdr(ring);
         spin_lock_init(&ring->lock);
         return ring;
 }
@@ -87,7 +92,7 @@ static int qxl_check_header(struct qxl_ring *ring)
         return ret;
 }
 
-static int qxl_check_idle(struct qxl_ring *ring)
+int qxl_check_idle(struct qxl_ring *ring)
 {
         int ret;
         struct qxl_ring_header *header = &(ring->ring->header);
@@ -375,8 +380,8 @@ void qxl_io_destroy_primary(struct qxl_device *qdev)
         wait_for_io_cmd(qdev, 0, QXL_IO_DESTROY_PRIMARY_ASYNC);
 }
 
-void qxl_io_create_primary(struct qxl_device *qdev, unsigned width,
-                           unsigned height, unsigned offset, struct qxl_bo *bo)
+void qxl_io_create_primary(struct qxl_device *qdev,
+                           unsigned offset, struct qxl_bo *bo)
 {
         struct qxl_surface_create *create;
 
@@ -384,8 +389,8 @@ void qxl_io_create_primary(struct qxl_device *qdev, unsigned width,
                  qdev->ram_header);
         create = &qdev->ram_header->create_surface;
         create->format = bo->surf.format;
-        create->width = width;
-        create->height = height;
+        create->width = bo->surf.width;
+        create->height = bo->surf.height;
         create->stride = bo->surf.stride;
         create->mem = qxl_bo_physical_address(qdev, bo, offset);
 
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 823d29e926ec..f76f5dd7bfc4 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -30,53 +30,9 @@
 #include "qxl_object.h"
 #include "drm_crtc_helper.h"
 
-static void qxl_crtc_set_to_mode(struct qxl_device *qdev,
-                                 struct drm_connector *connector,
-                                 struct qxl_head *head)
+static bool qxl_head_enabled(struct qxl_head *head)
 {
-        struct drm_device *dev = connector->dev;
-        struct drm_display_mode *mode, *t;
-        int width = head->width;
-        int height = head->height;
-
-        if (width < 320 || height < 240) {
-                qxl_io_log(qdev, "%s: bad head: %dx%d", width, height);
-                width = 1024;
-                height = 768;
-        }
-        if (width * height * 4 > 16*1024*1024) {
-                width = 1024;
-                height = 768;
-        }
-        /* TODO: go over regular modes and removed preferred? */
-        list_for_each_entry_safe(mode, t, &connector->probed_modes, head)
-                drm_mode_remove(connector, mode);
-        mode = drm_cvt_mode(dev, width, height, 60, false, false, false);
-        mode->type |= DRM_MODE_TYPE_PREFERRED;
-        mode->status = MODE_OK;
-        drm_mode_probed_add(connector, mode);
-        qxl_io_log(qdev, "%s: %d x %d\n", __func__, width, height);
-}
-
-void qxl_crtc_set_from_monitors_config(struct qxl_device *qdev)
-{
-        struct drm_connector *connector;
-        int i;
-        struct drm_device *dev = qdev->ddev;
-
-        i = 0;
-        qxl_io_log(qdev, "%s: %d, %d\n", __func__,
-                   dev->mode_config.num_connector,
-                   qdev->monitors_config->count);
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                if (i > qdev->monitors_config->count) {
-                        /* crtc will be reported as disabled */
-                        continue;
-                }
-                qxl_crtc_set_to_mode(qdev, connector,
-                                     &qdev->monitors_config->heads[i]);
-                ++i;
-        }
+        return head->width && head->height;
 }
 
 void qxl_alloc_client_monitors_config(struct qxl_device *qdev, unsigned count)
@@ -106,7 +62,6 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
         int num_monitors;
         uint32_t crc;
 
-        BUG_ON(!qdev->monitors_config);
         num_monitors = qdev->rom->client_monitors_config.count;
         crc = crc32(0, (const uint8_t *)&qdev->rom->client_monitors_config,
                   sizeof(qdev->rom->client_monitors_config));
@@ -117,8 +72,8 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
                 return 1;
         }
         if (num_monitors > qdev->monitors_config->max_allowed) {
-                DRM_INFO("client monitors list will be truncated: %d < %d\n",
-                         qdev->monitors_config->max_allowed, num_monitors);
+                DRM_DEBUG_KMS("client monitors list will be truncated: %d < %d\n",
+                              qdev->monitors_config->max_allowed, num_monitors);
                 num_monitors = qdev->monitors_config->max_allowed;
         } else {
                 num_monitors = qdev->rom->client_monitors_config.count;
@@ -132,18 +87,15 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
                         &qdev->rom->client_monitors_config.heads[i];
                 struct qxl_head *client_head =
                         &qdev->client_monitors_config->heads[i];
-                struct qxl_head *head = &qdev->monitors_config->heads[i];
-                client_head->x = head->x = c_rect->left;
-                client_head->y = head->y = c_rect->top;
-                client_head->width = head->width =
-                                                c_rect->right - c_rect->left;
-                client_head->height = head->height =
-                                                c_rect->bottom - c_rect->top;
-                client_head->surface_id = head->surface_id = 0;
-                client_head->id = head->id = i;
-                client_head->flags = head->flags = 0;
-                QXL_DEBUG(qdev, "read %dx%d+%d+%d\n", head->width, head->height,
-                          head->x, head->y);
+                client_head->x = c_rect->left;
+                client_head->y = c_rect->top;
+                client_head->width = c_rect->right - c_rect->left;
+                client_head->height = c_rect->bottom - c_rect->top;
+                client_head->surface_id = 0;
+                client_head->id = i;
+                client_head->flags = 0;
+                DRM_DEBUG_KMS("read %dx%d+%d+%d\n", client_head->width, client_head->height,
+                          client_head->x, client_head->y);
         }
         return 0;
 }
@@ -155,10 +107,7 @@ void qxl_display_read_client_monitors_config(struct qxl_device *qdev)
                 qxl_io_log(qdev, "failed crc check for client_monitors_config,"
                                  " retrying\n");
         }
-        qxl_crtc_set_from_monitors_config(qdev);
-        /* fire off a uevent and let userspace tell us what to do */
-        qxl_io_log(qdev, "calling drm_sysfs_hotplug_event\n");
-        drm_sysfs_hotplug_event(qdev->ddev);
+        drm_helper_hpd_irq_event(qdev->ddev);
 }
 
 static int qxl_add_monitors_config_modes(struct drm_connector *connector)
@@ -170,9 +119,9 @@ static int qxl_add_monitors_config_modes(struct drm_connector *connector)
         struct drm_display_mode *mode = NULL;
         struct qxl_head *head;
 
-        if (!qdev->monitors_config)
+        if (!qdev->client_monitors_config)
                 return 0;
-        head = &qdev->monitors_config->heads[h];
+        head = &qdev->client_monitors_config->heads[h];
 
         mode = drm_cvt_mode(dev, head->width, head->height, 60, false, false,
                             false);
@@ -222,12 +171,6 @@ static int qxl_add_common_modes(struct drm_connector *connector)
         return i - 1;
 }
 
-static void qxl_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                               u16 *blue, uint32_t start, uint32_t size)
-{
-        /* TODO */
-}
-
 static void qxl_crtc_destroy(struct drm_crtc *crtc)
 {
         struct qxl_crtc *qxl_crtc = to_qxl_crtc(crtc);
@@ -255,11 +198,11 @@ qxl_hide_cursor(struct qxl_device *qdev)
         qxl_release_unreserve(qdev, release);
 }
 
-static int qxl_crtc_cursor_set(struct drm_crtc *crtc,
-                               struct drm_file *file_priv,
-                               uint32_t handle,
-                               uint32_t width,
-                               uint32_t height)
+static int qxl_crtc_cursor_set2(struct drm_crtc *crtc,
+                                struct drm_file *file_priv,
+                                uint32_t handle,
+                                uint32_t width,
+                                uint32_t height, int32_t hot_x, int32_t hot_y)
 {
         struct drm_device *dev = crtc->dev;
         struct qxl_device *qdev = dev->dev_private;
@@ -315,8 +258,8 @@ static int qxl_crtc_cursor_set(struct drm_crtc *crtc,
         cursor->header.type = SPICE_CURSOR_TYPE_ALPHA;
         cursor->header.width = 64;
         cursor->header.height = 64;
-        cursor->header.hot_spot_x = 0;
-        cursor->header.hot_spot_y = 0;
+        cursor->header.hot_spot_x = hot_x;
+        cursor->header.hot_spot_y = hot_y;
         cursor->data_size = size;
         cursor->chunk.next_chunk = 0;
         cursor->chunk.prev_chunk = 0;
@@ -397,9 +340,8 @@ static int qxl_crtc_cursor_move(struct drm_crtc *crtc,
 
 
 static const struct drm_crtc_funcs qxl_crtc_funcs = {
-        .cursor_set = qxl_crtc_cursor_set,
+        .cursor_set2 = qxl_crtc_cursor_set2,
         .cursor_move = qxl_crtc_cursor_move,
-        .gamma_set = qxl_crtc_gamma_set,
         .set_config = drm_crtc_helper_set_config,
         .destroy = qxl_crtc_destroy,
 };
@@ -506,7 +448,7 @@ qxl_send_monitors_config(struct qxl_device *qdev)
         for (i = 0 ; i < qdev->monitors_config->count ; ++i) {
                 struct qxl_head *head = &qdev->monitors_config->heads[i];
 
-                if (head->y > 8192 || head->y < head->x ||
+                if (head->y > 8192 || head->x > 8192 ||
                     head->width > 8192 || head->height > 8192) {
                         DRM_ERROR("head %d wrong: %dx%d+%d+%d\n",
                                   i, head->width, head->height,
@@ -517,16 +459,19 @@ qxl_send_monitors_config(struct qxl_device *qdev)
         qxl_io_monitors_config(qdev);
 }
 
-static void qxl_monitors_config_set_single(struct qxl_device *qdev,
-                                           unsigned x, unsigned y,
-                                           unsigned width, unsigned height)
+static void qxl_monitors_config_set(struct qxl_device *qdev,
+                                    int index,
+                                    unsigned x, unsigned y,
+                                    unsigned width, unsigned height,
+                                    unsigned surf_id)
 {
-        DRM_DEBUG("%dx%d+%d+%d\n", width, height, x, y);
-        qdev->monitors_config->count = 1;
-        qdev->monitors_config->heads[0].x = x;
-        qdev->monitors_config->heads[0].y = y;
-        qdev->monitors_config->heads[0].width = width;
-        qdev->monitors_config->heads[0].height = height;
+        DRM_DEBUG_KMS("%d:%dx%d+%d+%d\n", index, width, height, x, y);
+        qdev->monitors_config->heads[index].x = x;
+        qdev->monitors_config->heads[index].y = y;
+        qdev->monitors_config->heads[index].width = width;
+        qdev->monitors_config->heads[index].height = height;
+        qdev->monitors_config->heads[index].surface_id = surf_id;
+
 }
 
 static int qxl_crtc_mode_set(struct drm_crtc *crtc,
@@ -540,10 +485,11 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
         struct qxl_mode *m = (void *)mode->private;
         struct qxl_framebuffer *qfb;
         struct qxl_bo *bo, *old_bo = NULL;
+        struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
         uint32_t width, height, base_offset;
         bool recreate_primary = false;
         int ret;
-
+        int surf_id;
         if (!crtc->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
@@ -567,7 +513,8 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
                   adjusted_mode->hdisplay,
                   adjusted_mode->vdisplay);
 
-        recreate_primary = true;
+        if (qcrtc->index == 0)
+                recreate_primary = true;
 
         width = mode->hdisplay;
         height = mode->vdisplay;
@@ -588,8 +535,11 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
                            "recreate primary: %dx%d (was %dx%d,%d,%d)\n",
                            width, height, bo->surf.width,
                            bo->surf.height, bo->surf.stride, bo->surf.format);
-                qxl_io_create_primary(qdev, width, height, base_offset, bo);
+                qxl_io_create_primary(qdev, base_offset, bo);
                 bo->is_primary = true;
+                surf_id = 0;
+        } else {
+                surf_id = bo->surface_id;
         }
 
         if (old_bo && old_bo != bo) {
@@ -599,11 +549,9 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
                 qxl_bo_unreserve(old_bo);
         }
 
-        if (qdev->monitors_config->count == 0) {
-                qxl_monitors_config_set_single(qdev, x, y,
-                                               mode->hdisplay,
-                                               mode->vdisplay);
-        }
+        qxl_monitors_config_set(qdev, qcrtc->index, x, y,
+                                mode->hdisplay,
+                                mode->vdisplay, surf_id);
         return 0;
 }
 
@@ -619,21 +567,36 @@ static void qxl_crtc_commit(struct drm_crtc *crtc)
         DRM_DEBUG("\n");
 }
 
-static void qxl_crtc_load_lut(struct drm_crtc *crtc)
+static void qxl_crtc_disable(struct drm_crtc *crtc)
 {
-        DRM_DEBUG("\n");
+        struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
+        struct drm_device *dev = crtc->dev;
+        struct qxl_device *qdev = dev->dev_private;
+        if (crtc->fb) {
+                struct qxl_framebuffer *qfb = to_qxl_framebuffer(crtc->fb);
+                struct qxl_bo *bo = gem_to_qxl_bo(qfb->obj);
+                int ret;
+                ret = qxl_bo_reserve(bo, false);
+                qxl_bo_unpin(bo);
+                qxl_bo_unreserve(bo);
+                crtc->fb = NULL;
+        }
+
+        qxl_monitors_config_set(qdev, qcrtc->index, 0, 0, 0, 0, 0);
+
+        qxl_send_monitors_config(qdev);
 }
 
 static const struct drm_crtc_helper_funcs qxl_crtc_helper_funcs = {
         .dpms = qxl_crtc_dpms,
+        .disable = qxl_crtc_disable,
         .mode_fixup = qxl_crtc_mode_fixup,
         .mode_set = qxl_crtc_mode_set,
         .prepare = qxl_crtc_prepare,
         .commit = qxl_crtc_commit,
-        .load_lut = qxl_crtc_load_lut,
 };
 
-static int qdev_crtc_init(struct drm_device *dev, int num_crtc)
+static int qdev_crtc_init(struct drm_device *dev, int crtc_id)
 {
         struct qxl_crtc *qxl_crtc;
 
@@ -642,7 +605,7 @@ static int qdev_crtc_init(struct drm_device *dev, int num_crtc)
                 return -ENOMEM;
 
         drm_crtc_init(dev, &qxl_crtc->base, &qxl_crtc_funcs);
-
+        qxl_crtc->index = crtc_id;
         drm_mode_crtc_set_gamma_size(&qxl_crtc->base, 256);
         drm_crtc_helper_add(&qxl_crtc->base, &qxl_crtc_helper_funcs);
         return 0;
@@ -670,18 +633,13 @@ static void qxl_write_monitors_config_for_encoder(struct qxl_device *qdev,
                 struct drm_encoder *encoder)
 {
         int i;
+        struct qxl_output *output = drm_encoder_to_qxl_output(encoder);
         struct qxl_head *head;
         struct drm_display_mode *mode;
 
         BUG_ON(!encoder);
         /* TODO: ugly, do better */
-        for (i = 0 ; (encoder->possible_crtcs != (1 << i)) && i < 32; ++i)
-                ;
-        if (encoder->possible_crtcs != (1 << i)) {
-                DRM_ERROR("encoder has wrong possible_crtcs: %x\n",
-                          encoder->possible_crtcs);
-                return;
-        }
+        i = output->index;
         if (!qdev->monitors_config ||
             qdev->monitors_config->max_allowed <= i) {
                 DRM_ERROR(
@@ -699,7 +657,6 @@ static void qxl_write_monitors_config_for_encoder(struct qxl_device *qdev,
                 DRM_DEBUG("missing for multiple monitors: no head holes\n");
         head = &qdev->monitors_config->heads[i];
         head->id = i;
-        head->surface_id = 0;
         if (encoder->crtc->enabled) {
                 mode = &encoder->crtc->mode;
                 head->width = mode->hdisplay;
@@ -714,8 +671,8 @@ static void qxl_write_monitors_config_for_encoder(struct qxl_device *qdev,
                 head->x = 0;
                 head->y = 0;
         }
-        DRM_DEBUG("setting head %d to +%d+%d %dx%d\n",
-                  i, head->x, head->y, head->width, head->height);
+        DRM_DEBUG_KMS("setting head %d to +%d+%d %dx%d out of %d\n",
+                      i, head->x, head->y, head->width, head->height, qdev->monitors_config->count);
         head->flags = 0;
         /* TODO - somewhere else to call this for multiple monitors
          * (config_commit?) */
@@ -810,8 +767,9 @@ static enum drm_connector_status qxl_conn_detect(
 
         /* The first monitor is always connected */
         connected = (output->index == 0) ||
-                    (qdev->monitors_config &&
-                     qdev->monitors_config->count > output->index);
+                    (qdev->client_monitors_config &&
+                     qdev->client_monitors_config->count > output->index &&
+                     qxl_head_enabled(&qdev->client_monitors_config->heads[output->index]));
 
         DRM_DEBUG("\n");
         return connected ? connector_status_connected
@@ -875,6 +833,8 @@ static int qdev_output_init(struct drm_device *dev, int num_output)
         drm_encoder_init(dev, &qxl_output->enc, &qxl_enc_funcs,
                          DRM_MODE_ENCODER_VIRTUAL);
 
+        /* we get HPD via client monitors config */
+        connector->polled = DRM_CONNECTOR_POLL_HPD;
         encoder->possible_crtcs = 1 << num_output;
         drm_mode_connector_attach_encoder(&qxl_output->base,
                                           &qxl_output->enc);
@@ -914,16 +874,14 @@ static const struct drm_mode_config_funcs qxl_mode_funcs = {
         .fb_create = qxl_user_framebuffer_create,
 };
 
-int qxl_modeset_init(struct qxl_device *qdev)
+int qxl_create_monitors_object(struct qxl_device *qdev)
 {
-        int i;
         int ret;
         struct drm_gem_object *gobj;
-        int max_allowed = QXL_NUM_OUTPUTS;
+        int max_allowed = qxl_num_crtc;
         int monitors_config_size = sizeof(struct qxl_monitors_config) +
-                                   max_allowed * sizeof(struct qxl_head);
+                max_allowed * sizeof(struct qxl_head);
 
-        drm_mode_config_init(qdev->ddev);
         ret = qxl_gem_object_create(qdev, monitors_config_size, 0,
                                     QXL_GEM_DOMAIN_VRAM,
                                     false, false, NULL, &gobj);
@@ -932,13 +890,59 @@ int qxl_modeset_init(struct qxl_device *qdev)
                 return -ENOMEM;
         }
         qdev->monitors_config_bo = gem_to_qxl_bo(gobj);
+
+        ret = qxl_bo_reserve(qdev->monitors_config_bo, false);
+        if (ret)
+                return ret;
+
+        ret = qxl_bo_pin(qdev->monitors_config_bo, QXL_GEM_DOMAIN_VRAM, NULL);
+        if (ret) {
+                qxl_bo_unreserve(qdev->monitors_config_bo);
+                return ret;
+        }
+
+        qxl_bo_unreserve(qdev->monitors_config_bo);
+
         qxl_bo_kmap(qdev->monitors_config_bo, NULL);
+
         qdev->monitors_config = qdev->monitors_config_bo->kptr;
         qdev->ram_header->monitors_config =
                 qxl_bo_physical_address(qdev, qdev->monitors_config_bo, 0);
 
         memset(qdev->monitors_config, 0, monitors_config_size);
         qdev->monitors_config->max_allowed = max_allowed;
+        return 0;
+}
+
+int qxl_destroy_monitors_object(struct qxl_device *qdev)
+{
+        int ret;
+
+        qdev->monitors_config = NULL;
+        qdev->ram_header->monitors_config = 0;
+
+        qxl_bo_kunmap(qdev->monitors_config_bo);
+        ret = qxl_bo_reserve(qdev->monitors_config_bo, false);
+        if (ret)
+                return ret;
+
+        qxl_bo_unpin(qdev->monitors_config_bo);
+        qxl_bo_unreserve(qdev->monitors_config_bo);
+
+        qxl_bo_unref(&qdev->monitors_config_bo);
+        return 0;
+}
+
+int qxl_modeset_init(struct qxl_device *qdev)
+{
+        int i;
+        int ret;
+
+        drm_mode_config_init(qdev->ddev);
+
+        ret = qxl_create_monitors_object(qdev);
+        if (ret)
+                return ret;
 
         qdev->ddev->mode_config.funcs = (void *)&qxl_mode_funcs;
 
@@ -949,7 +953,7 @@ int qxl_modeset_init(struct qxl_device *qdev)
         qdev->ddev->mode_config.max_height = 8192;
 
         qdev->ddev->mode_config.fb_base = qdev->vram_base;
-        for (i = 0 ; i < QXL_NUM_OUTPUTS; ++i) {
+        for (i = 0 ; i < qxl_num_crtc; ++i) {
                 qdev_crtc_init(qdev->ddev, i);
                 qdev_output_init(qdev->ddev, i);
         }
@@ -966,6 +970,8 @@ int qxl_modeset_init(struct qxl_device *qdev)
 void qxl_modeset_fini(struct qxl_device *qdev)
 {
         qxl_fbdev_fini(qdev);
+
+        qxl_destroy_monitors_object(qdev);
         if (qdev->mode_info.mode_config_initialized) {
                 drm_mode_config_cleanup(qdev->ddev);
                 qdev->mode_info.mode_config_initialized = false;
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index aa291d8a98a2..df0b577a6608 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -33,8 +33,9 @@
 
 #include "drmP.h"
 #include "drm/drm.h"
-
+#include "drm_crtc_helper.h"
 #include "qxl_drv.h"
+#include "qxl_object.h"
 
 extern int qxl_max_ioctls;
 static DEFINE_PCI_DEVICE_TABLE(pciidlist) = {
@@ -47,10 +48,14 @@ static DEFINE_PCI_DEVICE_TABLE(pciidlist) = {
 MODULE_DEVICE_TABLE(pci, pciidlist);
 
 static int qxl_modeset = -1;
+int qxl_num_crtc = 4;
 
 MODULE_PARM_DESC(modeset, "Disable/Enable modesetting");
 module_param_named(modeset, qxl_modeset, int, 0400);
 
+MODULE_PARM_DESC(num_heads, "Number of virtual crtcs to expose (default 4)");
+module_param_named(num_heads, qxl_num_crtc, int, 0400);
+
 static struct drm_driver qxl_driver;
 static struct pci_driver qxl_pci_driver;
 
@@ -73,13 +78,6 @@ qxl_pci_remove(struct pci_dev *pdev)
         drm_put_dev(dev);
 }
 
-static struct pci_driver qxl_pci_driver = {
-         .name = DRIVER_NAME,
-         .id_table = pciidlist,
-         .probe = qxl_pci_probe,
-         .remove = qxl_pci_remove,
-};
-
 static const struct file_operations qxl_fops = {
         .owner = THIS_MODULE,
         .open = drm_open,
@@ -90,6 +88,130 @@ static const struct file_operations qxl_fops = {
         .mmap = qxl_mmap,
 };
 
+static int qxl_drm_freeze(struct drm_device *dev)
+{
+        struct pci_dev *pdev = dev->pdev;
+        struct qxl_device *qdev = dev->dev_private;
+        struct drm_crtc *crtc;
+
+        drm_kms_helper_poll_disable(dev);
+
+        console_lock();
+        qxl_fbdev_set_suspend(qdev, 1);
+        console_unlock();
+
+        /* unpin the front buffers */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
+                if (crtc->enabled)
+                        (*crtc_funcs->disable)(crtc);
+        }
+
+        qxl_destroy_monitors_object(qdev);
+        qxl_surf_evict(qdev);
+        qxl_vram_evict(qdev);
+
+        while (!qxl_check_idle(qdev->command_ring));
+        while (!qxl_check_idle(qdev->release_ring))
+                qxl_queue_garbage_collect(qdev, 1);
+
+        pci_save_state(pdev);
+
+        return 0;
+}
+
+static int qxl_drm_resume(struct drm_device *dev, bool thaw)
+{
+        struct qxl_device *qdev = dev->dev_private;
+
+        qdev->ram_header->int_mask = QXL_INTERRUPT_MASK;
+        if (!thaw) {
+                qxl_reinit_memslots(qdev);
+                qxl_ring_init_hdr(qdev->release_ring);
+        }
+
+        qxl_create_monitors_object(qdev);
+        drm_helper_resume_force_mode(dev);
+
+        console_lock();
+        qxl_fbdev_set_suspend(qdev, 0);
+        console_unlock();
+
+        drm_kms_helper_poll_enable(dev);
+        return 0;
+}
+
+static int qxl_pm_suspend(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        int error;
+
+        error = qxl_drm_freeze(drm_dev);
+        if (error)
+                return error;
+
+        pci_disable_device(pdev);
+        pci_set_power_state(pdev, PCI_D3hot);
+        return 0;
+}
+
+static int qxl_pm_resume(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+        pci_set_power_state(pdev, PCI_D0);
+        pci_restore_state(pdev);
+        if (pci_enable_device(pdev)) {
+                return -EIO;
+        }
+
+        return qxl_drm_resume(drm_dev, false);
+}
+
+static int qxl_pm_thaw(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+        return qxl_drm_resume(drm_dev, true);
+}
+
+static int qxl_pm_freeze(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+        return qxl_drm_freeze(drm_dev);
+}
+
+static int qxl_pm_restore(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        struct qxl_device *qdev = drm_dev->dev_private;
+
+        qxl_io_reset(qdev);
+        return qxl_drm_resume(drm_dev, false);
+}
+
+static const struct dev_pm_ops qxl_pm_ops = {
+        .suspend = qxl_pm_suspend,
+        .resume = qxl_pm_resume,
+        .freeze = qxl_pm_freeze,
+        .thaw = qxl_pm_thaw,
+        .poweroff = qxl_pm_freeze,
+        .restore = qxl_pm_restore,
+};
+static struct pci_driver qxl_pci_driver = {
+         .name = DRIVER_NAME,
+         .id_table = pciidlist,
+         .probe = qxl_pci_probe,
+         .remove = qxl_pci_remove,
+         .driver.pm = &qxl_pm_ops,
+};
+
 static struct drm_driver qxl_driver = {
         .driver_features = DRIVER_GEM | DRIVER_MODESET |
                            DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED,
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 43d06ab28a21..aacb791464a3 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -55,11 +55,10 @@
 #define DRIVER_MINOR 1
 #define DRIVER_PATCHLEVEL 0
 
-#define QXL_NUM_OUTPUTS 1
-
 #define QXL_DEBUGFS_MAX_COMPONENTS              32
 
 extern int qxl_log_level;
+extern int qxl_num_crtc;
 
 enum {
         QXL_INFO_LEVEL = 1,
@@ -139,6 +138,7 @@ struct qxl_reloc_list {
 
 struct qxl_crtc {
         struct drm_crtc base;
+        int index;
         int cur_x;
         int cur_y;
 };
@@ -156,7 +156,7 @@ struct qxl_framebuffer {
 
 #define to_qxl_crtc(x) container_of(x, struct qxl_crtc, base)
 #define drm_connector_to_qxl_output(x) container_of(x, struct qxl_output, base)
-#define drm_encoder_to_qxl_output(x) container_of(x, struct qxl_output, base)
+#define drm_encoder_to_qxl_output(x) container_of(x, struct qxl_output, enc)
 #define to_qxl_framebuffer(x) container_of(x, struct qxl_framebuffer, base)
 
 struct qxl_mman {
@@ -331,6 +331,10 @@ void qxl_modeset_fini(struct qxl_device *qdev);
 int qxl_bo_init(struct qxl_device *qdev);
 void qxl_bo_fini(struct qxl_device *qdev);
 
+void qxl_reinit_memslots(struct qxl_device *qdev);
+int qxl_surf_evict(struct qxl_device *qdev);
+int qxl_vram_evict(struct qxl_device *qdev);
+
 struct qxl_ring *qxl_ring_create(struct qxl_ring_header *header,
                                  int element_size,
                                  int n_elements,
@@ -338,6 +342,8 @@ struct qxl_ring *qxl_ring_create(struct qxl_ring_header *header,
                                  bool set_prod_notify,
                                  wait_queue_head_t *push_event);
 void qxl_ring_free(struct qxl_ring *ring);
+void qxl_ring_init_hdr(struct qxl_ring *ring);
+int qxl_check_idle(struct qxl_ring *ring);
 
 static inline void *
 qxl_fb_virtual_address(struct qxl_device *qdev, unsigned long physical)
@@ -365,6 +371,7 @@ void qxl_fbdev_fini(struct qxl_device *qdev);
 int qxl_get_handle_for_primary_fb(struct qxl_device *qdev,
                                   struct drm_file *file_priv,
                                   uint32_t *handle);
+void qxl_fbdev_set_suspend(struct qxl_device *qdev, int state);
 
 /* qxl_display.c */
 int
@@ -374,6 +381,8 @@ qxl_framebuffer_init(struct drm_device *dev,
                      struct drm_gem_object *obj);
 void qxl_display_read_client_monitors_config(struct qxl_device *qdev);
 void qxl_send_monitors_config(struct qxl_device *qdev);
+int qxl_create_monitors_object(struct qxl_device *qdev);
+int qxl_destroy_monitors_object(struct qxl_device *qdev);
 
 /* used by qxl_debugfs only */
 void qxl_crtc_set_from_monitors_config(struct qxl_device *qdev);
@@ -435,7 +444,7 @@ void qxl_update_screen(struct qxl_device *qxl);
 /* qxl io operations (qxl_cmd.c) */
 
 void qxl_io_create_primary(struct qxl_device *qdev,
-                           unsigned width, unsigned height, unsigned offset,
+                           unsigned offset,
                            struct qxl_bo *bo);
 void qxl_io_destroy_primary(struct qxl_device *qdev);
 void qxl_io_memslot_add(struct qxl_device *qdev, uint8_t id);
@@ -528,6 +537,7 @@ irqreturn_t qxl_irq_handler(DRM_IRQ_ARGS);
 
 /* qxl_fb.c */
 int qxl_fb_init(struct qxl_device *qdev);
+bool qxl_fbdev_qobj_is_fb(struct qxl_device *qdev, struct qxl_bo *qobj);
 
 int qxl_debugfs_add_files(struct qxl_device *qdev,
                           struct drm_info_list *files,
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index b3c51275df5c..76f39d88d684 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -520,10 +520,6 @@ static int qxl_fbdev_destroy(struct drm_device *dev, struct qxl_fbdev *qfbdev)
 }
 
 static struct drm_fb_helper_funcs qxl_fb_helper_funcs = {
-        /* TODO
-        .gamma_set = qxl_crtc_fb_gamma_set,
-        .gamma_get = qxl_crtc_fb_gamma_get,
-        */
         .fb_probe = qxl_fb_find_or_create_single,
 };
 
@@ -542,7 +538,7 @@ int qxl_fbdev_init(struct qxl_device *qdev)
         qfbdev->helper.funcs = &qxl_fb_helper_funcs;
 
         ret = drm_fb_helper_init(qdev->ddev, &qfbdev->helper,
-                                 1 /* num_crtc - QXL supports just 1 */,
+                                 qxl_num_crtc /* num_crtc - QXL supports just 1 */,
                                  QXLFB_CONN_LIMIT);
         if (ret) {
                 kfree(qfbdev);
@@ -564,4 +560,14 @@ void qxl_fbdev_fini(struct qxl_device *qdev)
         qdev->mode_info.qfbdev = NULL;
 }
 
+void qxl_fbdev_set_suspend(struct qxl_device *qdev, int state)
+{
+        fb_set_suspend(qdev->mode_info.qfbdev->helper.fbdev, state);
+}
 
+bool qxl_fbdev_qobj_is_fb(struct qxl_device *qdev, struct qxl_bo *qobj)
+{
+        if (qobj == gem_to_qxl_bo(qdev->mode_info.qfbdev->qfb.obj))
+                return true;
+        return false;
+}
diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c
index a4b71b25fa53..6ba49d9922f2 100644
--- a/drivers/gpu/drm/qxl/qxl_ioctl.c
+++ b/drivers/gpu/drm/qxl/qxl_ioctl.c
@@ -183,6 +183,12 @@ static int qxl_execbuffer_ioctl(struct drm_device *dev, void *data,
                 /* TODO copy slow path code from i915 */
                 fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_SIZE));
                 unwritten = __copy_from_user_inatomic_nocache(fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_SIZE), (void *)(unsigned long)user_cmd.command, user_cmd.command_size);
+
+                {
+                        struct qxl_drawable *draw = fb_cmd;
+
+                        draw->mm_time = qdev->rom->mm_clock;
+                }
                 qxl_bo_kunmap_atomic_page(qdev, cmd_bo, fb_cmd);
                 if (unwritten) {
                         DRM_ERROR("got unwritten %d\n", unwritten);
diff --git a/drivers/gpu/drm/qxl/qxl_kms.c b/drivers/gpu/drm/qxl/qxl_kms.c
index e27ce2a907cf..9e8da9ee9731 100644
--- a/drivers/gpu/drm/qxl/qxl_kms.c
+++ b/drivers/gpu/drm/qxl/qxl_kms.c
@@ -26,6 +26,7 @@
 #include "qxl_drv.h"
 #include "qxl_object.h"
 
+#include <drm/drm_crtc_helper.h>
 #include <linux/io-mapping.h>
 
 int qxl_log_level;
@@ -72,21 +73,28 @@ static bool qxl_check_device(struct qxl_device *qdev)
         return true;
 }
 
+static void setup_hw_slot(struct qxl_device *qdev, int slot_index,
+                          struct qxl_memslot *slot)
+{
+        qdev->ram_header->mem_slot.mem_start = slot->start_phys_addr;
+        qdev->ram_header->mem_slot.mem_end = slot->end_phys_addr;
+        qxl_io_memslot_add(qdev, slot_index);
+}
+
 static uint8_t setup_slot(struct qxl_device *qdev, uint8_t slot_index_offset,
         unsigned long start_phys_addr, unsigned long end_phys_addr)
 {
         uint64_t high_bits;
         struct qxl_memslot *slot;
         uint8_t slot_index;
-        struct qxl_ram_header *ram_header = qdev->ram_header;
 
         slot_index = qdev->rom->slots_start + slot_index_offset;
         slot = &qdev->mem_slots[slot_index];
         slot->start_phys_addr = start_phys_addr;
         slot->end_phys_addr = end_phys_addr;
-        ram_header->mem_slot.mem_start = slot->start_phys_addr;
-        ram_header->mem_slot.mem_end = slot->end_phys_addr;
-        qxl_io_memslot_add(qdev, slot_index);
+
+        setup_hw_slot(qdev, slot_index, slot);
+
         slot->generation = qdev->rom->slot_generation;
         high_bits = slot_index << qdev->slot_gen_bits;
         high_bits |= slot->generation;
@@ -95,6 +103,12 @@ static uint8_t setup_slot(struct qxl_device *qdev, uint8_t slot_index_offset,
         return slot_index;
 }
 
+void qxl_reinit_memslots(struct qxl_device *qdev)
+{
+        setup_hw_slot(qdev, qdev->main_mem_slot, &qdev->mem_slots[qdev->main_mem_slot]);
+        setup_hw_slot(qdev, qdev->surfaces_mem_slot, &qdev->mem_slots[qdev->surfaces_mem_slot]);
+}
+
 static void qxl_gc_work(struct work_struct *work)
 {
         struct qxl_device *qdev = container_of(work, struct qxl_device, gc_work);
@@ -294,6 +308,8 @@ int qxl_driver_load(struct drm_device *dev, unsigned long flags)
                 goto out;
         }
 
+        drm_kms_helper_poll_init(qdev->ddev);
+
         return 0;
 out:
         kfree(qdev);
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index d9b12e7bc6e1..1191fe7788c9 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -363,3 +363,13 @@ int qxl_bo_list_add(struct qxl_reloc_list *reloc_list, struct qxl_bo *bo)
                 return ret;
         return 0;
 }
+
+int qxl_surf_evict(struct qxl_device *qdev)
+{
+        return ttm_bo_evict_mm(&qdev->mman.bdev, TTM_PL_PRIV0);
+}
+
+int qxl_vram_evict(struct qxl_device *qdev)
+{
+        return ttm_bo_evict_mm(&qdev->mman.bdev, TTM_PL_VRAM);
+}
diff --git a/drivers/gpu/drm/qxl/qxl_object.h b/drivers/gpu/drm/qxl/qxl_object.h
index b4fd89fbd8b7..ee7ad79ce781 100644
--- a/drivers/gpu/drm/qxl/qxl_object.h
+++ b/drivers/gpu/drm/qxl/qxl_object.h
@@ -57,11 +57,6 @@ static inline unsigned long qxl_bo_size(struct qxl_bo *bo)
         return bo->tbo.num_pages << PAGE_SHIFT;
 }
 
-static inline bool qxl_bo_is_reserved(struct qxl_bo *bo)
-{
-        return !!atomic_read(&bo->tbo.reserved);
-}
-
 static inline u64 qxl_bo_mmap_offset(struct qxl_bo *bo)
 {
         return bo->tbo.addr_space_offset;
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index 86c5e3611892..c3df52c1a60c 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -76,7 +76,10 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
         evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \
         evergreen_hdmi.o radeon_trace_points.o ni.o cayman_blit_shaders.o \
         atombios_encoders.o radeon_semaphore.o radeon_sa.o atombios_i2c.o si.o \
-        si_blit_shaders.o radeon_prime.o radeon_uvd.o
+        si_blit_shaders.o radeon_prime.o radeon_uvd.o cik.o cik_blit_shaders.o \
+        r600_dpm.o rs780_dpm.o rv6xx_dpm.o rv770_dpm.o rv730_dpm.o rv740_dpm.o \
+        rv770_smc.o cypress_dpm.o btc_dpm.o sumo_dpm.o sumo_smc.o trinity_dpm.o \
+        trinity_smc.o ni_dpm.o si_smc.o si_dpm.o
 
 radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
 radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
diff --git a/drivers/gpu/drm/radeon/ObjectID.h b/drivers/gpu/drm/radeon/ObjectID.h
index ca4b038050d2..06192698bd96 100644
--- a/drivers/gpu/drm/radeon/ObjectID.h
+++ b/drivers/gpu/drm/radeon/ObjectID.h
@@ -69,6 +69,8 @@
 #define ENCODER_OBJECT_ID_ALMOND                  0x22
 #define ENCODER_OBJECT_ID_TRAVIS                  0x23
 #define ENCODER_OBJECT_ID_NUTMEG                  0x22
+#define ENCODER_OBJECT_ID_HDMI_ANX9805            0x26
+
 /* Kaleidoscope (KLDSCP) Class Display Hardware (internal) */
 #define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1   0x13
 #define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1    0x14
@@ -86,6 +88,8 @@
 #define ENCODER_OBJECT_ID_INTERNAL_UNIPHY1        0x20
 #define ENCODER_OBJECT_ID_INTERNAL_UNIPHY2        0x21
 #define ENCODER_OBJECT_ID_INTERNAL_VCE            0x24
+#define ENCODER_OBJECT_ID_INTERNAL_UNIPHY3        0x25
+#define ENCODER_OBJECT_ID_INTERNAL_AMCLK          0x27
 
 #define ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO    0xFF
 
@@ -364,6 +368,14 @@
                                                  GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
                                                  ENCODER_OBJECT_ID_INTERNAL_UNIPHY2 << OBJECT_ID_SHIFT)
 
+#define ENCODER_INTERNAL_UNIPHY3_ENUM_ID1         ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                 ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 << OBJECT_ID_SHIFT)
+
+#define ENCODER_INTERNAL_UNIPHY3_ENUM_ID2         ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
+                                                 ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 << OBJECT_ID_SHIFT)
+
 #define ENCODER_GENERAL_EXTERNAL_DVO_ENUM_ID1    ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
                                                   GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                   ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO << OBJECT_ID_SHIFT)
@@ -392,6 +404,10 @@
                                                   GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                   ENCODER_OBJECT_ID_INTERNAL_VCE << OBJECT_ID_SHIFT)
 
+#define ENCODER_HDMI_ANX9805_ENUM_ID1            ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_HDMI_ANX9805 << OBJECT_ID_SHIFT)
+
 /****************************************************/
 /* Connector Object ID definition - Shared with BIOS */
 /****************************************************/
@@ -461,6 +477,14 @@
                                                  GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
 
+#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID5   ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
+
+#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID6   ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
+
 #define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1     ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT)
@@ -473,6 +497,10 @@
                                                  GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT)
 
+#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID4     ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT)
+
 #define CONNECTOR_VGA_ENUM_ID1                 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_VGA << OBJECT_ID_SHIFT)
@@ -541,6 +569,18 @@
                                                  GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT)
 
+#define CONNECTOR_HDMI_TYPE_A_ENUM_ID4         ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT)
+
+#define CONNECTOR_HDMI_TYPE_A_ENUM_ID5         ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT)
+
+#define CONNECTOR_HDMI_TYPE_A_ENUM_ID6         ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
+                                                 GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\
+                                                 CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT)
+
 #define CONNECTOR_HDMI_TYPE_B_ENUM_ID1         ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
                                                  GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                  CONNECTOR_OBJECT_ID_HDMI_TYPE_B << OBJECT_ID_SHIFT)
diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h
index 0ee573743de9..16b120c3f144 100644
--- a/drivers/gpu/drm/radeon/atombios.h
+++ b/drivers/gpu/drm/radeon/atombios.h
@@ -74,6 +74,8 @@
 #define ATOM_PPLL2            1
 #define ATOM_DCPLL            2
 #define ATOM_PPLL0            2
+#define ATOM_PPLL3            3
+
 #define ATOM_EXT_PLL1         8
 #define ATOM_EXT_PLL2         9
 #define ATOM_EXT_CLOCK        10
@@ -259,7 +261,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
   USHORT AdjustDisplayPll;                                                                                       //Atomic Table,  used by various SW componentes. 
   USHORT AdjustMemoryController;                 //Atomic Table,  indirectly used by various SW components,called from SetMemoryClock                
   USHORT EnableASIC_StaticPwrMgt;                //Atomic Table,  only used by Bios
-  USHORT ASIC_StaticPwrMgtStatusChange;          //Obsolete ,     only used by Bios   
+  USHORT SetUniphyInstance;                      //Atomic Table,  only used by Bios   
   USHORT DAC_LoadDetection;                      //Atomic Table,  directly used by various SW components,latest version 1.2  
   USHORT LVTMAEncoderControl;                    //Atomic Table,directly used by various SW components,latest version 1.3
   USHORT HW_Misc_Operation;                      //Atomic Table,  directly used by various SW components,latest version 1.1 
@@ -271,7 +273,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
   USHORT TVEncoderControl;                       //Function Table,directly used by various SW components,latest version 1.1
   USHORT PatchMCSetting;                         //only used by BIOS
   USHORT MC_SEQ_Control;                         //only used by BIOS
-  USHORT TV1OutputControl;                       //Atomic Table,  Obsolete from Ry6xx, use DAC2 Output instead
+  USHORT Gfx_Harvesting;                         //Atomic Table,  Obsolete from Ry6xx, Now only used by BIOS for GFX harvesting
   USHORT EnableScaler;                           //Atomic Table,  used only by Bios
   USHORT BlankCRTC;                              //Atomic Table,  directly used by various SW components,latest version 1.1 
   USHORT EnableCRTC;                             //Atomic Table,  directly used by various SW components,latest version 1.1 
@@ -328,7 +330,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
 #define UNIPHYTransmitterControl                             DIG1TransmitterControl
 #define LVTMATransmitterControl                              DIG2TransmitterControl
 #define SetCRTC_DPM_State                        GetConditionalGoldenSetting
-#define SetUniphyInstance                        ASIC_StaticPwrMgtStatusChange
+#define ASIC_StaticPwrMgtStatusChange            SetUniphyInstance 
 #define HPDInterruptService                      ReadHWAssistedI2CStatus
 #define EnableVGA_Access                         GetSCLKOverMCLKRatio
 #define EnableYUV                                GetDispObjectInfo                         
@@ -338,7 +340,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
 #define TMDSAEncoderControl                      PatchMCSetting
 #define LVDSEncoderControl                       MC_SEQ_Control
 #define LCD1OutputControl                        HW_Misc_Operation
-
+#define TV1OutputControl                         Gfx_Harvesting
 
 typedef struct _ATOM_MASTER_COMMAND_TABLE
 {
@@ -478,11 +480,11 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3
 typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4
 {
 #if ATOM_BIG_ENDIAN
-  ULONG  ucPostDiv;          //return parameter: post divider which is used to program to register directly
+  ULONG  ucPostDiv:8;        //return parameter: post divider which is used to program to register directly
   ULONG  ulClock:24;         //Input= target clock, output = actual clock 
 #else
   ULONG  ulClock:24;         //Input= target clock, output = actual clock 
-  ULONG  ucPostDiv;          //return parameter: post divider which is used to program to register directly
+  ULONG  ucPostDiv:8;        //return parameter: post divider which is used to program to register directly
 #endif
 }COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4;
 
@@ -504,6 +506,32 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5
   UCHAR   ucReserved;                       
 }COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5;
 
+
+typedef struct _COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6
+{
+  ATOM_COMPUTE_CLOCK_FREQ  ulClock;         //Input Parameter
+  ULONG   ulReserved[2];
+}COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6;
+
+//ATOM_COMPUTE_CLOCK_FREQ.ulComputeClockFlag
+#define COMPUTE_GPUCLK_INPUT_FLAG_CLK_TYPE_MASK            0x0f
+#define COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK           0x00
+#define COMPUTE_GPUCLK_INPUT_FLAG_SCLK                     0x01
+
+typedef struct _COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6
+{
+  COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4  ulClock;         //Output Parameter: ucPostDiv=DFS divider
+  ATOM_S_MPLL_FB_DIVIDER   ulFbDiv;         //Output Parameter: PLL FB divider
+  UCHAR   ucPllRefDiv;                      //Output Parameter: PLL ref divider      
+  UCHAR   ucPllPostDiv;                     //Output Parameter: PLL post divider      
+  UCHAR   ucPllCntlFlag;                    //Output Flags: control flag
+  UCHAR   ucReserved;                       
+}COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6;
+
+//ucPllCntlFlag
+#define SPLL_CNTL_FLAG_VCO_MODE_MASK            0x03 
+
+
 // ucInputFlag
 #define ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN  1   // 1-StrobeMode, 0-PerformanceMode
 
@@ -1686,6 +1714,7 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V6
 #define PIXEL_CLOCK_V6_MISC_HDMI_30BPP              0x08
 #define PIXEL_CLOCK_V6_MISC_HDMI_48BPP              0x0c
 #define PIXEL_CLOCK_V6_MISC_REF_DIV_SRC             0x10
+#define PIXEL_CLOCK_V6_MISC_GEN_DPREFCLK            0x40
 
 typedef struct _GET_DISP_PLL_STATUS_INPUT_PARAMETERS_V2
 {
@@ -2102,6 +2131,17 @@ typedef struct _DVO_ENCODER_CONTROL_PARAMETERS_V3
 }DVO_ENCODER_CONTROL_PARAMETERS_V3;
 #define DVO_ENCODER_CONTROL_PS_ALLOCATION_V3    DVO_ENCODER_CONTROL_PARAMETERS_V3
 
+typedef struct _DVO_ENCODER_CONTROL_PARAMETERS_V1_4
+{
+  USHORT usPixelClock; 
+  UCHAR  ucDVOConfig;
+  UCHAR  ucAction;                                                                                                              //ATOM_ENABLE/ATOM_DISABLE/ATOM_HPD_INIT
+  UCHAR  ucBitPerColor;                       //please refer to definition of PANEL_xBIT_PER_COLOR
+  UCHAR  ucReseved[3];
+}DVO_ENCODER_CONTROL_PARAMETERS_V1_4;
+#define DVO_ENCODER_CONTROL_PS_ALLOCATION_V1_4  DVO_ENCODER_CONTROL_PARAMETERS_V1_4
+
+
 //ucTableFormatRevision=1
 //ucTableContentRevision=3 structure is not changed but usMisc add bit 1 as another input for 
 // bit1=0: non-coherent mode
@@ -2165,7 +2205,7 @@ typedef struct _DVO_ENCODER_CONTROL_PARAMETERS_V3
 #define SET_ASIC_VOLTAGE_MODE_SOURCE_B         0x4
 
 #define SET_ASIC_VOLTAGE_MODE_SET_VOLTAGE      0x0
-#define SET_ASIC_VOLTAGE_MODE_GET_GPIOVAL      0x1      
+#define SET_ASIC_VOLTAGE_MODE_GET_GPIOVAL      0x1
 #define SET_ASIC_VOLTAGE_MODE_GET_GPIOMASK     0x2
 
 typedef struct  _SET_VOLTAGE_PARAMETERS
@@ -2200,15 +2240,20 @@ typedef struct	_SET_VOLTAGE_PARAMETERS_V1_3
 //SET_VOLTAGE_PARAMETERS_V3.ucVoltageMode
 #define ATOM_SET_VOLTAGE                     0        //Set voltage Level
 #define ATOM_INIT_VOLTAGE_REGULATOR          3        //Init Regulator
-#define ATOM_SET_VOLTAGE_PHASE               4        //Set Vregulator Phase
-#define ATOM_GET_MAX_VOLTAGE                 6        //Get Max Voltage, not used in SetVoltageTable v1.3
-#define ATOM_GET_VOLTAGE_LEVEL               6        //Get Voltage level from vitual voltage ID
+#define ATOM_SET_VOLTAGE_PHASE               4        //Set Vregulator Phase, only for SVID/PVID regulator
+#define ATOM_GET_MAX_VOLTAGE                 6        //Get Max Voltage, not used from SetVoltageTable v1.3
+#define ATOM_GET_VOLTAGE_LEVEL               6        //Get Voltage level from vitual voltage ID, not used for SetVoltage v1.4
+#define ATOM_GET_LEAKAGE_ID                  8        //Get Leakage Voltage Id ( starting from SMU7x IP ), SetVoltage v1.4 
 
 // define vitual voltage id in usVoltageLevel
 #define ATOM_VIRTUAL_VOLTAGE_ID0             0xff01
 #define ATOM_VIRTUAL_VOLTAGE_ID1             0xff02
 #define ATOM_VIRTUAL_VOLTAGE_ID2             0xff03
 #define ATOM_VIRTUAL_VOLTAGE_ID3             0xff04
+#define ATOM_VIRTUAL_VOLTAGE_ID4             0xff05
+#define ATOM_VIRTUAL_VOLTAGE_ID5             0xff06
+#define ATOM_VIRTUAL_VOLTAGE_ID6             0xff07
+#define ATOM_VIRTUAL_VOLTAGE_ID7             0xff08
 
 typedef struct _SET_VOLTAGE_PS_ALLOCATION
 {
@@ -2628,7 +2673,8 @@ typedef struct _ATOM_FIRMWARE_INFO_V2_2
   ULONG                           ulFirmwareRevision;
   ULONG                           ulDefaultEngineClock;       //In 10Khz unit
   ULONG                           ulDefaultMemoryClock;       //In 10Khz unit
-  ULONG                           ulReserved[2];
+  ULONG                           ulSPLL_OutputFreq;          //In 10Khz unit  
+  ULONG                           ulGPUPLL_OutputFreq;        //In 10Khz unit
   ULONG                           ulReserved1;                //Was ulMaxEngineClockPLL_Output; //In 10Khz unit*
   ULONG                           ulReserved2;                //Was ulMaxMemoryClockPLL_Output; //In 10Khz unit*
   ULONG                           ulMaxPixelClockPLL_Output;  //In 10Khz unit
@@ -3813,6 +3859,12 @@ typedef struct _ATOM_GPIO_PIN_ASSIGNMENT
   UCHAR                    ucGPIO_ID;
 }ATOM_GPIO_PIN_ASSIGNMENT;
 
+//ucGPIO_ID pre-define id for multiple usage
+//from SMU7.x, if ucGPIO_ID=PP_AC_DC_SWITCH_GPIO_PINID in GPIO_LUTTable, AC/DC swithing feature is enable
+#define PP_AC_DC_SWITCH_GPIO_PINID          60
+//from SMU7.x, if ucGPIO_ID=VDDC_REGULATOR_VRHOT_GPIO_PINID in GPIO_LUTable, VRHot feature is enable
+#define VDDC_VRHOT_GPIO_PINID               61
+
 typedef struct _ATOM_GPIO_PIN_LUT
 {
   ATOM_COMMON_TABLE_HEADER  sHeader;
@@ -4074,17 +4126,19 @@ typedef struct _EXT_DISPLAY_PATH
 
 //usCaps
 #define  EXT_DISPLAY_PATH_CAPS__HBR2_DISABLE          0x01
+#define  EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN        0x02
 
 typedef  struct _ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO
 {
   ATOM_COMMON_TABLE_HEADER sHeader;
   UCHAR                    ucGuid [NUMBER_OF_UCHAR_FOR_GUID];     // a GUID is a 16 byte long string
   EXT_DISPLAY_PATH         sPath[MAX_NUMBER_OF_EXT_DISPLAY_PATH]; // total of fixed 7 entries.
-  UCHAR                    ucChecksum;                            // a  simple Checksum of the sum of whole structure equal to 0x0. 
+  UCHAR                    ucChecksum;                            // a simple Checksum of the sum of whole structure equal to 0x0. 
   UCHAR                    uc3DStereoPinId;                       // use for eDP panel
   UCHAR                    ucRemoteDisplayConfig;
   UCHAR                    uceDPToLVDSRxId;
-  UCHAR                    Reserved[4];                           // for potential expansion
+  UCHAR                    ucFixDPVoltageSwing;                   // usCaps[1]=1, this indicate DP_LANE_SET value
+  UCHAR                    Reserved[3];                           // for potential expansion
 }ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO;
 
 //Related definitions, all records are different but they have a commond header
@@ -4416,6 +4470,13 @@ typedef struct _ATOM_VOLTAGE_CONTROL
 #define VOLTAGE_CONTROL_ID_CHL822x                                              0x08                                                                    
 #define VOLTAGE_CONTROL_ID_VT1586M                                              0x09
 #define VOLTAGE_CONTROL_ID_UP1637                                               0x0A
+#define VOLTAGE_CONTROL_ID_CHL8214            0x0B
+#define VOLTAGE_CONTROL_ID_UP1801             0x0C
+#define VOLTAGE_CONTROL_ID_ST6788A            0x0D
+#define VOLTAGE_CONTROL_ID_CHLIR3564SVI2      0x0E
+#define VOLTAGE_CONTROL_ID_AD527x             0x0F
+#define VOLTAGE_CONTROL_ID_NCP81022           0x10
+#define VOLTAGE_CONTROL_ID_LTC2635                        0x11
 
 typedef struct  _ATOM_VOLTAGE_OBJECT
 {
@@ -4458,6 +4519,15 @@ typedef struct _ATOM_VOLTAGE_OBJECT_HEADER_V3{
          USHORT         usSize;                                                                                                 //Size of Object        
 }ATOM_VOLTAGE_OBJECT_HEADER_V3;
 
+// ATOM_VOLTAGE_OBJECT_HEADER_V3.ucVoltageMode
+#define VOLTAGE_OBJ_GPIO_LUT                 0        //VOLTAGE and GPIO Lookup table ->ATOM_GPIO_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_VR_I2C_INIT_SEQ          3        //VOLTAGE REGULATOR INIT sequece through I2C -> ATOM_I2C_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_PHASE_LUT                4        //Set Vregulator Phase lookup table ->ATOM_GPIO_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_SVID2                    7        //Indicate voltage control by SVID2 ->ATOM_SVID2_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_PWRBOOST_LEAKAGE_LUT     0x10     //Powerboost Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_HIGH_STATE_LEAKAGE_LUT   0x11     //High voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_HIGH1_STATE_LEAKAGE_LUT  0x12     //High1 voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
+
 typedef struct  _VOLTAGE_LUT_ENTRY_V2
 {
          ULONG          ulVoltageId;                                                                      // The Voltage ID which is used to program GPIO register
@@ -4473,7 +4543,7 @@ typedef struct  _LEAKAGE_VOLTAGE_LUT_ENTRY_V2
 
 typedef struct  _ATOM_I2C_VOLTAGE_OBJECT_V3
 {
-   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;
+   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;    // voltage mode = VOLTAGE_OBJ_VR_I2C_INIT_SEQ
    UCHAR        ucVoltageRegulatorId;                                     //Indicate Voltage Regulator Id
    UCHAR    ucVoltageControlI2cLine;
    UCHAR    ucVoltageControlAddress;
@@ -4484,7 +4554,7 @@ typedef struct  _ATOM_I2C_VOLTAGE_OBJECT_V3
 
 typedef struct  _ATOM_GPIO_VOLTAGE_OBJECT_V3
 {
-   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;   
+   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;   // voltage mode = VOLTAGE_OBJ_GPIO_LUT or VOLTAGE_OBJ_PHASE_LUT
    UCHAR    ucVoltageGpioCntlId;         // default is 0 which indicate control through CG VID mode 
    UCHAR    ucGpioEntryNum;              // indiate the entry numbers of Votlage/Gpio value Look up table
    UCHAR    ucPhaseDelay;                // phase delay in unit of micro second
@@ -4495,7 +4565,7 @@ typedef struct  _ATOM_GPIO_VOLTAGE_OBJECT_V3
 
 typedef struct  _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
 {
-   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;
+   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;    // voltage mode = 0x10/0x11/0x12
    UCHAR    ucLeakageCntlId;             // default is 0
    UCHAR    ucLeakageEntryNum;           // indicate the entry number of LeakageId/Voltage Lut table
    UCHAR    ucReserved[2];               
@@ -4503,10 +4573,26 @@ typedef struct  _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
    LEAKAGE_VOLTAGE_LUT_ENTRY_V2 asLeakageIdLut[1];   
 }ATOM_LEAKAGE_VOLTAGE_OBJECT_V3;
 
+
+typedef struct  _ATOM_SVID2_VOLTAGE_OBJECT_V3
+{
+   ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;    // voltage mode = VOLTAGE_OBJ_SVID2
+// 14:7 – PSI0_VID
+// 6 – PSI0_EN
+// 5 – PSI1
+// 4:2 – load line slope trim. 
+// 1:0 – offset trim, 
+   USHORT   usLoadLine_PSI;    
+// GPU GPIO pin Id to SVID2 regulator VRHot pin. possible value 0~31. 0 means GPIO0, 31 means GPIO31
+   UCHAR    ucReserved[2];
+   ULONG    ulReserved;
+}ATOM_SVID2_VOLTAGE_OBJECT_V3;
+
 typedef union _ATOM_VOLTAGE_OBJECT_V3{
   ATOM_GPIO_VOLTAGE_OBJECT_V3 asGpioVoltageObj;
   ATOM_I2C_VOLTAGE_OBJECT_V3 asI2cVoltageObj;
   ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 asLeakageObj;
+  ATOM_SVID2_VOLTAGE_OBJECT_V3 asSVID2Obj;
 }ATOM_VOLTAGE_OBJECT_V3;
 
 typedef struct  _ATOM_VOLTAGE_OBJECT_INFO_V3_1
@@ -4536,6 +4622,21 @@ typedef struct  _ATOM_ASIC_PROFILING_INFO
         ATOM_ASIC_PROFILE_VOLTAGE                       asVoltage;
 }ATOM_ASIC_PROFILING_INFO;
 
+typedef struct  _ATOM_ASIC_PROFILING_INFO_V2_1
+{
+  ATOM_COMMON_TABLE_HEADER                      asHeader; 
+  UCHAR  ucLeakageBinNum;                // indicate the entry number of LeakageId/Voltage Lut table
+  USHORT usLeakageBinArrayOffset;        // offset of USHORT Leakage Bin list array ( from lower LeakageId to higher) 
+
+  UCHAR  ucElbVDDC_Num;               
+  USHORT usElbVDDC_IdArrayOffset;        // offset of USHORT virtual VDDC voltage id ( 0xff01~0xff08 )
+  USHORT usElbVDDC_LevelArrayOffset;     // offset of 2 dimension voltage level USHORT array
+
+  UCHAR  ucElbVDDCI_Num;
+  USHORT usElbVDDCI_IdArrayOffset;       // offset of USHORT virtual VDDCI voltage id ( 0xff01~0xff08 )
+  USHORT usElbVDDCI_LevelArrayOffset;    // offset of 2 dimension voltage level USHORT array
+}ATOM_ASIC_PROFILING_INFO_V2_1;
+
 typedef struct _ATOM_POWER_SOURCE_OBJECT
 {
         UCHAR   ucPwrSrcId;                                                                                                     // Power source
@@ -4652,6 +4753,8 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V6
 #define SYS_INFO_LVDSMISC__888_BPC                                                   0x04
 #define SYS_INFO_LVDSMISC__OVERRIDE_EN                                               0x08
 #define SYS_INFO_LVDSMISC__BLON_ACTIVE_LOW                                           0x10
+// new since Trinity
+#define SYS_INFO_LVDSMISC__TRAVIS_LVDS_VOL_OVERRIDE_EN                               0x20
 
 // not used any more
 #define SYS_INFO_LVDSMISC__VSYNC_ACTIVE_LOW                                          0x04
@@ -4752,6 +4855,29 @@ typedef struct _ATOM_FUSION_SYSTEM_INFO_V1
   ATOM_INTEGRATED_SYSTEM_INFO_V6    sIntegratedSysInfo;   
   ULONG  ulPowerplayTable[128];  
 }ATOM_FUSION_SYSTEM_INFO_V1; 
+
+
+typedef struct _ATOM_TDP_CONFIG_BITS
+{
+#if ATOM_BIG_ENDIAN
+  ULONG   uReserved:2;
+  ULONG   uTDP_Value:14;  // Original TDP value in tens of milli watts
+  ULONG   uCTDP_Value:14; // Override value in tens of milli watts
+  ULONG   uCTDP_Enable:2; // = (uCTDP_Value > uTDP_Value? 2: (uCTDP_Value < uTDP_Value))
+#else
+  ULONG   uCTDP_Enable:2; // = (uCTDP_Value > uTDP_Value? 2: (uCTDP_Value < uTDP_Value))
+  ULONG   uCTDP_Value:14; // Override value in tens of milli watts
+  ULONG   uTDP_Value:14;  // Original TDP value in tens of milli watts
+  ULONG   uReserved:2;
+#endif
+}ATOM_TDP_CONFIG_BITS;
+
+typedef union _ATOM_TDP_CONFIG
+{
+  ATOM_TDP_CONFIG_BITS TDP_config;
+  ULONG            TDP_config_all;
+}ATOM_TDP_CONFIG;
+
 /**********************************************************************************************************************
   ATOM_FUSION_SYSTEM_INFO_V1 Description
 sIntegratedSysInfo:               refer to ATOM_INTEGRATED_SYSTEM_INFO_V6 definition.
@@ -4784,7 +4910,8 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7
   UCHAR  ucMemoryType;  
   UCHAR  ucUMAChannelNumber;
   UCHAR  strVBIOSMsg[40];
-  ULONG  ulReserved[20];
+  ATOM_TDP_CONFIG  asTdpConfig;
+  ULONG  ulReserved[19];
   ATOM_AVAILABLE_SCLK_LIST   sAvail_SCLK[5];
   ULONG  ulGMCRestoreResetTime;
   ULONG  ulMinimumNClk;
@@ -4809,7 +4936,7 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7
   USHORT GnbTdpLimit;
   USHORT usMaxLVDSPclkFreqInSingleLink;
   UCHAR  ucLvdsMisc;
-  UCHAR  ucLVDSReserved;
+  UCHAR  ucTravisLVDSVolAdjust;
   UCHAR  ucLVDSPwrOnSeqDIGONtoDE_in4Ms;
   UCHAR  ucLVDSPwrOnSeqDEtoVARY_BL_in4Ms;
   UCHAR  ucLVDSPwrOffSeqVARY_BLtoDE_in4Ms;
@@ -4817,7 +4944,7 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7
   UCHAR  ucLVDSOffToOnDelay_in4Ms;
   UCHAR  ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms;
   UCHAR  ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms;
-  UCHAR  ucLVDSReserved1;
+  UCHAR  ucMinAllowedBL_Level;
   ULONG  ulLCDBitDepthControlVal;
   ULONG  ulNbpStateMemclkFreq[4];
   USHORT usNBP2Voltage;               
@@ -4846,6 +4973,7 @@ typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7
 #define SYS_INFO_GPUCAPS__TMDSHDMI_COHERENT_SINGLEPLL_MODE                0x01
 #define SYS_INFO_GPUCAPS__DP_SINGLEPLL_MODE                               0x02
 #define SYS_INFO_GPUCAPS__DISABLE_AUX_MODE_DETECT                         0x08
+#define SYS_INFO_GPUCAPS__ENABEL_DFS_BYPASS                               0x10
 
 /**********************************************************************************************************************
   ATOM_INTEGRATED_SYSTEM_INFO_V1_7 Description
@@ -4945,6 +5073,9 @@ ucLVDSMisc:                       [bit0] LVDS 888bit panel mode =0: LVDS 888 pan
                                   [bit2] LVDS 888bit per color mode  =0: 666 bit per color =1:888 bit per color
                                   [bit3] LVDS parameter override enable  =0: ucLvdsMisc parameter are not used =1: ucLvdsMisc parameter should be used
                                   [bit4] Polarity of signal sent to digital BLON output pin. =0: not inverted(active high) =1: inverted ( active low )
+                                  [bit5] Travid LVDS output voltage override enable, when =1, use ucTravisLVDSVolAdjust value to overwrite Traivs register LVDS_CTRL_4
+ucTravisLVDSVolAdjust             When ucLVDSMisc[5]=1,it means platform SBIOS want to overwrite TravisLVDSVoltage. Then VBIOS will use ucTravisLVDSVolAdjust 
+                                  value to program Travis register LVDS_CTRL_4
 ucLVDSPwrOnSeqDIGONtoDE_in4Ms:    LVDS power up sequence time in unit of 4ms, time delay from DIGON signal active to data enable signal active( DE ).
                                   =0 mean use VBIOS default which is 8 ( 32ms ). The LVDS power up sequence is as following: DIGON->DE->VARY_BL->BLON. 
                                   This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
@@ -4964,18 +5095,241 @@ ucLVDSOffToOnDelay_in4Ms:         LVDS power down sequence time in unit of 4ms.
                                   =0 means to use VBIOS default delay which is 125 ( 500ms ).
                                   This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
 
-ucLVDSPwrOnVARY_BLtoBLON_in4Ms:   LVDS power up sequence time in unit of 4ms. Time delay from VARY_BL signal on to DLON signal active. 
+ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms:
+                                  LVDS power up sequence time in unit of 4ms. Time delay from VARY_BL signal on to DLON signal active. 
                                   =0 means to use VBIOS default delay which is 0 ( 0ms ).
                                   This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
 
-ucLVDSPwrOffBLONtoVARY_BL_in4Ms:  LVDS power down sequence time in unit of 4ms. Time delay from BLON signal off to VARY_BL signal off. 
+ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms:  
+                                  LVDS power down sequence time in unit of 4ms. Time delay from BLON signal off to VARY_BL signal off. 
                                   =0 means to use VBIOS default delay which is 0 ( 0ms ).
                                   This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
 
+ucMinAllowedBL_Level:             Lowest LCD backlight PWM level. This is customer platform specific parameters. By default it is 0. 
+
 ulNbpStateMemclkFreq[4]:          system memory clock frequncey in unit of 10Khz in different NB pstate. 
 
 **********************************************************************************************************************/
 
+// this IntegrateSystemInfoTable is used for Kaveri & Kabini APU
+typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8
+{
+  ATOM_COMMON_TABLE_HEADER   sHeader;
+  ULONG  ulBootUpEngineClock;
+  ULONG  ulDentistVCOFreq;
+  ULONG  ulBootUpUMAClock;
+  ATOM_CLK_VOLT_CAPABILITY   sDISPCLK_Voltage[4];
+  ULONG  ulBootUpReqDisplayVector;
+  ULONG  ulVBIOSMisc;
+  ULONG  ulGPUCapInfo;
+  ULONG  ulDISP_CLK2Freq;
+  USHORT usRequestedPWMFreqInHz;
+  UCHAR  ucHtcTmpLmt;
+  UCHAR  ucHtcHystLmt;
+  ULONG  ulReserved2;
+  ULONG  ulSystemConfig;            
+  ULONG  ulCPUCapInfo;
+  ULONG  ulReserved3;
+  USHORT usGPUReservedSysMemSize;
+  USHORT usExtDispConnInfoOffset;
+  USHORT usPanelRefreshRateRange;     
+  UCHAR  ucMemoryType;  
+  UCHAR  ucUMAChannelNumber;
+  UCHAR  strVBIOSMsg[40];
+  ATOM_TDP_CONFIG  asTdpConfig;
+  ULONG  ulReserved[19];
+  ATOM_AVAILABLE_SCLK_LIST   sAvail_SCLK[5];
+  ULONG  ulGMCRestoreResetTime;
+  ULONG  ulReserved4;
+  ULONG  ulIdleNClk;
+  ULONG  ulDDR_DLL_PowerUpTime;
+  ULONG  ulDDR_PLL_PowerUpTime;
+  USHORT usPCIEClkSSPercentage;
+  USHORT usPCIEClkSSType;
+  USHORT usLvdsSSPercentage;
+  USHORT usLvdsSSpreadRateIn10Hz;
+  USHORT usHDMISSPercentage;
+  USHORT usHDMISSpreadRateIn10Hz;
+  USHORT usDVISSPercentage;
+  USHORT usDVISSpreadRateIn10Hz;
+  ULONG  ulGPUReservedSysMemBaseAddrLo;
+  ULONG  ulGPUReservedSysMemBaseAddrHi;
+  ULONG  ulReserved5[3];
+  USHORT usMaxLVDSPclkFreqInSingleLink;
+  UCHAR  ucLvdsMisc;
+  UCHAR  ucTravisLVDSVolAdjust;
+  UCHAR  ucLVDSPwrOnSeqDIGONtoDE_in4Ms;
+  UCHAR  ucLVDSPwrOnSeqDEtoVARY_BL_in4Ms;
+  UCHAR  ucLVDSPwrOffSeqVARY_BLtoDE_in4Ms;
+  UCHAR  ucLVDSPwrOffSeqDEtoDIGON_in4Ms;
+  UCHAR  ucLVDSOffToOnDelay_in4Ms;
+  UCHAR  ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms;
+  UCHAR  ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms;
+  UCHAR  ucMinAllowedBL_Level;
+  ULONG  ulLCDBitDepthControlVal;
+  ULONG  ulNbpStateMemclkFreq[4];
+  ULONG  ulReserved6;               
+  ULONG  ulNbpStateNClkFreq[4];
+  USHORT usNBPStateVoltage[4];            
+  USHORT usBootUpNBVoltage;   
+  USHORT usReserved2; 
+  ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO sExtDispConnInfo;
+}ATOM_INTEGRATED_SYSTEM_INFO_V1_8;
+
+/**********************************************************************************************************************
+  ATOM_INTEGRATED_SYSTEM_INFO_V1_8 Description
+ulBootUpEngineClock:              VBIOS bootup Engine clock frequency, in 10kHz unit. if it is equal 0, then VBIOS use pre-defined bootup engine clock
+ulDentistVCOFreq:                 Dentist VCO clock in 10kHz unit. 
+ulBootUpUMAClock:                 System memory boot up clock frequency in 10Khz unit. 
+sDISPCLK_Voltage:                 Report Display clock frequency requirement on GNB voltage(up to 4 voltage levels).
+ 
+ulBootUpReqDisplayVector:         VBIOS boot up display IDs, following are supported devices in Trinity projects:
+                                  ATOM_DEVICE_CRT1_SUPPORT                  0x0001
+                                  ATOM_DEVICE_DFP1_SUPPORT                  0x0008
+                                  ATOM_DEVICE_DFP6_SUPPORT                  0x0040
+                                  ATOM_DEVICE_DFP2_SUPPORT                  0x0080
+                                  ATOM_DEVICE_DFP3_SUPPORT                  0x0200
+                                  ATOM_DEVICE_DFP4_SUPPORT                  0x0400
+                                  ATOM_DEVICE_DFP5_SUPPORT                  0x0800
+                                  ATOM_DEVICE_LCD1_SUPPORT                  0x0002
+
+ulVBIOSMisc:                          Miscellenous flags for VBIOS requirement and interface 
+                                  bit[0]=0: INT15 callback function Get LCD EDID ( ax=4e08, bl=1b ) is not supported by SBIOS. 
+                                        =1: INT15 callback function Get LCD EDID ( ax=4e08, bl=1b ) is supported by SBIOS.
+                                  bit[1]=0: INT15 callback function Get boot display( ax=4e08, bl=01h) is not supported by SBIOS
+                                        =1: INT15 callback function Get boot display( ax=4e08, bl=01h) is supported by SBIOS
+                                  bit[2]=0: INT15 callback function Get panel Expansion ( ax=4e08, bl=02h) is not supported by SBIOS
+                                        =1: INT15 callback function Get panel Expansion ( ax=4e08, bl=02h) is supported by SBIOS
+                                  bit[3]=0: VBIOS fast boot is disable
+                                        =1: VBIOS fast boot is enable. ( VBIOS skip display device detection in every set mode if LCD panel is connect and LID is open)
+
+ulGPUCapInfo:                     bit[0~2]= Reserved
+                                  bit[3]=0: Enable AUX HW mode detection logic
+                                        =1: Disable AUX HW mode detection logic
+                                  bit[4]=0: Disable DFS bypass feature
+                                        =1: Enable DFS bypass feature
+
+usRequestedPWMFreqInHz:           When it's set to 0x0 by SBIOS: the LCD BackLight is not controlled by GPU(SW). 
+                                  Any attempt to change BL using VBIOS function or enable VariBri from PP table is not effective since ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==0;
+                                  
+                                  When it's set to a non-zero frequency, the BackLight is controlled by GPU (SW) in one of two ways below:
+                                  1. SW uses the GPU BL PWM output to control the BL, in chis case, this non-zero frequency determines what freq GPU should use;
+                                  VBIOS will set up proper PWM frequency and ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1,as the result,
+                                  Changing BL using VBIOS function is functional in both driver and non-driver present environment; 
+                                  and enabling VariBri under the driver environment from PP table is optional.
+
+                                  2. SW uses other means to control BL (like DPCD),this non-zero frequency serves as a flag only indicating
+                                  that BL control from GPU is expected.
+                                  VBIOS will NOT set up PWM frequency but make ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1
+                                  Changing BL using VBIOS function could be functional in both driver and non-driver present environment,but
+                                  it's per platform 
+                                  and enabling VariBri under the driver environment from PP table is optional.
+
+ucHtcTmpLmt:                      Refer to D18F3x64 bit[22:16], HtcTmpLmt. Threshold on value to enter HTC_active state.
+ucHtcHystLmt:                     Refer to D18F3x64 bit[27:24], HtcHystLmt. 
+                                  To calculate threshold off value to exit HTC_active state, which is Threshold on vlaue minus ucHtcHystLmt.
+
+ulSystemConfig:                   Bit[0]=0: PCIE Power Gating Disabled 
+                                        =1: PCIE Power Gating Enabled
+                                  Bit[1]=0: DDR-DLL shut-down feature disabled.
+                                         1: DDR-DLL shut-down feature enabled.
+                                  Bit[2]=0: DDR-PLL Power down feature disabled.
+                                         1: DDR-PLL Power down feature enabled. 
+                                  Bit[3]=0: GNB DPM is disabled
+                                        =1: GNB DPM is enabled                                
+ulCPUCapInfo:                     TBD
+
+usExtDispConnInfoOffset:          Offset to sExtDispConnInfo inside the structure
+usPanelRefreshRateRange:          Bit vector for LCD supported refresh rate range. If DRR is requestd by the platform, at least two bits need to be set
+                                  to indicate a range.
+                                  SUPPORTED_LCD_REFRESHRATE_30Hz          0x0004
+                                  SUPPORTED_LCD_REFRESHRATE_40Hz          0x0008
+                                  SUPPORTED_LCD_REFRESHRATE_50Hz          0x0010
+                                  SUPPORTED_LCD_REFRESHRATE_60Hz          0x0020
+
+ucMemoryType:                     [3:0]=1:DDR1;=2:DDR2;=3:DDR3;=5:GDDR5; [7:4] is reserved.
+ucUMAChannelNumber:                     System memory channel numbers. 
+
+strVBIOSMsg[40]:                  VBIOS boot up customized message string 
+
+sAvail_SCLK[5]:                   Arrays to provide availabe list of SLCK and corresponding voltage, order from low to high  
+
+ulGMCRestoreResetTime:            GMC power restore and GMC reset time to calculate data reconnection latency. Unit in ns. 
+ulIdleNClk:                       NCLK speed while memory runs in self-refresh state, used to calculate self-refresh latency. Unit in 10kHz.
+ulDDR_DLL_PowerUpTime:            DDR PHY DLL power up time. Unit in ns.
+ulDDR_PLL_PowerUpTime:            DDR PHY PLL power up time. Unit in ns.
+
+usPCIEClkSSPercentage:            PCIE Clock Spread Spectrum Percentage in unit 0.01%; 100 mean 1%.
+usPCIEClkSSType:                  PCIE Clock Spread Spectrum Type. 0 for Down spread(default); 1 for Center spread.
+usLvdsSSPercentage:               LVDS panel ( not include eDP ) Spread Spectrum Percentage in unit of 0.01%, =0, use VBIOS default setting. 
+usLvdsSSpreadRateIn10Hz:          LVDS panel ( not include eDP ) Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting. 
+usHDMISSPercentage:               HDMI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%,  =0, use VBIOS default setting. 
+usHDMISSpreadRateIn10Hz:          HDMI Spread Spectrum frequency in unit of 10Hz,  =0, use VBIOS default setting. 
+usDVISSPercentage:                DVI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%,  =0, use VBIOS default setting. 
+usDVISSpreadRateIn10Hz:           DVI Spread Spectrum frequency in unit of 10Hz,  =0, use VBIOS default setting. 
+
+usGPUReservedSysMemSize:          Reserved system memory size for ACP engine in APU GNB, units in MB. 0/2/4MB based on CMOS options, current default could be 0MB. KV only, not on KB.
+ulGPUReservedSysMemBaseAddrLo:    Low 32 bits base address to the reserved system memory. 
+ulGPUReservedSysMemBaseAddrHi:    High 32 bits base address to the reserved system memory. 
+
+usMaxLVDSPclkFreqInSingleLink:    Max pixel clock LVDS panel single link, if=0 means VBIOS use default threhold, right now it is 85Mhz
+ucLVDSMisc:                       [bit0] LVDS 888bit panel mode =0: LVDS 888 panel in LDI mode, =1: LVDS 888 panel in FPDI mode
+                                  [bit1] LVDS panel lower and upper link mapping =0: lower link and upper link not swap, =1: lower link and upper link are swapped
+                                  [bit2] LVDS 888bit per color mode  =0: 666 bit per color =1:888 bit per color
+                                  [bit3] LVDS parameter override enable  =0: ucLvdsMisc parameter are not used =1: ucLvdsMisc parameter should be used
+                                  [bit4] Polarity of signal sent to digital BLON output pin. =0: not inverted(active high) =1: inverted ( active low )
+                                  [bit5] Travid LVDS output voltage override enable, when =1, use ucTravisLVDSVolAdjust value to overwrite Traivs register LVDS_CTRL_4
+ucTravisLVDSVolAdjust             When ucLVDSMisc[5]=1,it means platform SBIOS want to overwrite TravisLVDSVoltage. Then VBIOS will use ucTravisLVDSVolAdjust 
+                                  value to program Travis register LVDS_CTRL_4
+ucLVDSPwrOnSeqDIGONtoDE_in4Ms:    
+                                  LVDS power up sequence time in unit of 4ms, time delay from DIGON signal active to data enable signal active( DE ).
+                                  =0 mean use VBIOS default which is 8 ( 32ms ). The LVDS power up sequence is as following: DIGON->DE->VARY_BL->BLON. 
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+ucLVDSPwrOnDEtoVARY_BL_in4Ms:     
+                                  LVDS power up sequence time in unit of 4ms., time delay from DE( data enable ) active to Vary Brightness enable signal active( VARY_BL ).  
+                                  =0 mean use VBIOS default which is 90 ( 360ms ). The LVDS power up sequence is as following: DIGON->DE->VARY_BL->BLON. 
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+ucLVDSPwrOffVARY_BLtoDE_in4Ms:    
+                                  LVDS power down sequence time in unit of 4ms, time delay from data enable ( DE ) signal off to LCDVCC (DIGON) off. 
+                                  =0 mean use VBIOS default delay which is 8 ( 32ms ). The LVDS power down sequence is as following: BLON->VARY_BL->DE->DIGON
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+ucLVDSPwrOffDEtoDIGON_in4Ms:      
+                                   LVDS power down sequence time in unit of 4ms, time delay from vary brightness enable signal( VARY_BL) off to data enable ( DE ) signal off. 
+                                  =0 mean use VBIOS default which is 90 ( 360ms ). The LVDS power down sequence is as following: BLON->VARY_BL->DE->DIGON
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+ucLVDSOffToOnDelay_in4Ms:         
+                                  LVDS power down sequence time in unit of 4ms. Time delay from DIGON signal off to DIGON signal active. 
+                                  =0 means to use VBIOS default delay which is 125 ( 500ms ).
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+ucLVDSPwrOnSeqVARY_BLtoBLON_in4Ms:
+                                  LVDS power up sequence time in unit of 4ms. Time delay from VARY_BL signal on to DLON signal active. 
+                                  =0 means to use VBIOS default delay which is 0 ( 0ms ).
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+
+ucLVDSPwrOffSeqBLONtoVARY_BL_in4Ms:  
+                                  LVDS power down sequence time in unit of 4ms. Time delay from BLON signal off to VARY_BL signal off. 
+                                  =0 means to use VBIOS default delay which is 0 ( 0ms ).
+                                  This parameter is used by VBIOS only. VBIOS will patch LVDS_InfoTable.
+ucMinAllowedBL_Level:             Lowest LCD backlight PWM level. This is customer platform specific parameters. By default it is 0. 
+
+ulLCDBitDepthControlVal:          GPU display control encoder bit dither control setting, used to program register mmFMT_BIT_DEPTH_CONTROL
+
+ulNbpStateMemclkFreq[4]:          system memory clock frequncey in unit of 10Khz in different NB P-State(P0, P1, P2 & P3).
+ulNbpStateNClkFreq[4]:            NB P-State NClk frequency in different NB P-State
+usNBPStateVoltage[4]:             NB P-State (P0/P1 & P2/P3) voltage; NBP3 refers to lowes voltage
+usBootUpNBVoltage:                NB P-State voltage during boot up before driver loaded 
+sExtDispConnInfo:                 Display connector information table provided to VBIOS
+
+**********************************************************************************************************************/
+
+// this Table is used for Kaveri/Kabini APU
+typedef struct _ATOM_FUSION_SYSTEM_INFO_V2
+{
+  ATOM_INTEGRATED_SYSTEM_INFO_V1_8    sIntegratedSysInfo;       // refer to ATOM_INTEGRATED_SYSTEM_INFO_V1_8 definition
+  ULONG                               ulPowerplayTable[128];    // Update comments here to link new powerplay table definition structure
+}ATOM_FUSION_SYSTEM_INFO_V2; 
+
+
 /**************************************************************************/
 // This portion is only used when ext thermal chip or engine/memory clock SS chip is populated on a design
 //Memory SS Info Table
@@ -5026,22 +5380,24 @@ typedef struct _ATOM_ASIC_SS_ASSIGNMENT
 
 //Define ucClockIndication, SW uses the IDs below to search if the SS is required/enabled on a clock branch/signal type.
 //SS is not required or enabled if a match is not found.
-#define ASIC_INTERNAL_MEMORY_SS                 1
-#define ASIC_INTERNAL_ENGINE_SS                 2
-#define ASIC_INTERNAL_UVD_SS        3
-#define ASIC_INTERNAL_SS_ON_TMDS    4
-#define ASIC_INTERNAL_SS_ON_HDMI    5
-#define ASIC_INTERNAL_SS_ON_LVDS    6
-#define ASIC_INTERNAL_SS_ON_DP      7
-#define ASIC_INTERNAL_SS_ON_DCPLL   8
-#define ASIC_EXTERNAL_SS_ON_DP_CLOCK 9
-#define ASIC_INTERNAL_VCE_SS        10
+#define ASIC_INTERNAL_MEMORY_SS          1
+#define ASIC_INTERNAL_ENGINE_SS          2
+#define ASIC_INTERNAL_UVD_SS             3
+#define ASIC_INTERNAL_SS_ON_TMDS         4
+#define ASIC_INTERNAL_SS_ON_HDMI         5
+#define ASIC_INTERNAL_SS_ON_LVDS         6
+#define ASIC_INTERNAL_SS_ON_DP           7
+#define ASIC_INTERNAL_SS_ON_DCPLL        8
+#define ASIC_EXTERNAL_SS_ON_DP_CLOCK     9
+#define ASIC_INTERNAL_VCE_SS             10
+#define ASIC_INTERNAL_GPUPLL_SS          11
+
 
 typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V2
 {
         ULONG                                                           ulTargetClockRange;                                             //For mem/engine/uvd, Clock Out frequence (VCO ), in unit of 10Khz
                                                     //For TMDS/HDMI/LVDS, it is pixel clock , for DP, it is link clock ( 27000 or 16200 )
-  USHORT              usSpreadSpectrumPercentage;               //in unit of 0.01%
+  USHORT              usSpreadSpectrumPercentage;               //in unit of 0.01% or 0.001%, decided by ucSpreadSpectrumMode bit4
         USHORT                                                  usSpreadRateIn10Hz;                                             //in unit of 10Hz, modulation freq
   UCHAR               ucClockIndication;                                          //Indicate which clock source needs SS
         UCHAR                                                           ucSpreadSpectrumMode;                                   //Bit0=0 Down Spread,=1 Center Spread, bit1=0: internal SS bit1=1: external SS
@@ -5079,6 +5435,11 @@ typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V3
         UCHAR                                                           ucReserved[2];
 }ATOM_ASIC_SS_ASSIGNMENT_V3;
 
+//ATOM_ASIC_SS_ASSIGNMENT_V3.ucSpreadSpectrumMode
+#define SS_MODE_V3_CENTRE_SPREAD_MASK             0x01
+#define SS_MODE_V3_EXTERNAL_SS_MASK               0x02
+#define SS_MODE_V3_PERCENTAGE_DIV_BY_1000_MASK    0x10
+
 typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 {
   ATOM_COMMON_TABLE_HEADER            sHeader; 
@@ -5719,6 +6080,7 @@ typedef struct _INDIRECT_IO_ACCESS
 #define INDIRECT_IO_PCIE           3
 #define INDIRECT_IO_PCIEP          4
 #define INDIRECT_IO_NBMISC         5
+#define INDIRECT_IO_SMU            5
 
 #define INDIRECT_IO_PLL_READ       INDIRECT_IO_PLL   | INDIRECT_READ
 #define INDIRECT_IO_PLL_WRITE      INDIRECT_IO_PLL   | INDIRECT_WRITE
@@ -5730,6 +6092,8 @@ typedef struct _INDIRECT_IO_ACCESS
 #define INDIRECT_IO_PCIEP_WRITE    INDIRECT_IO_PCIEP | INDIRECT_WRITE
 #define INDIRECT_IO_NBMISC_READ    INDIRECT_IO_NBMISC | INDIRECT_READ
 #define INDIRECT_IO_NBMISC_WRITE   INDIRECT_IO_NBMISC | INDIRECT_WRITE
+#define INDIRECT_IO_SMU_READ       INDIRECT_IO_SMU | INDIRECT_READ
+#define INDIRECT_IO_SMU_WRITE      INDIRECT_IO_SMU | INDIRECT_WRITE
 
 typedef struct _ATOM_OEM_INFO
 { 
@@ -5875,6 +6239,7 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
 #define _64Mx32             0x43
 #define _128Mx8             0x51
 #define _128Mx16            0x52
+#define _128Mx32            0x53
 #define _256Mx8             0x61
 #define _256Mx16            0x62
 
@@ -5893,6 +6258,8 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
 #define PROMOS              MOSEL
 #define KRETON              INFINEON
 #define ELIXIR              NANYA
+#define MEZZA               ELPIDA
+
 
 /////////////Support for GDDR5 MC uCode to reside in upper 64K of ROM/////////////
 
@@ -6625,6 +6992,10 @@ typedef struct _ATOM_DISP_OUT_INFO_V3
         ASIC_TRANSMITTER_INFO_V2  asTransmitterInfo[1];     // for alligment only
 }ATOM_DISP_OUT_INFO_V3;
 
+//ucDispCaps
+#define DISPLAY_CAPS__DP_PCLK_FROM_PPLL        0x01
+#define DISPLAY_CAPS__FORCE_DISPDEV_CONNECTED  0x02
+
 typedef enum CORE_REF_CLK_SOURCE{
   CLOCK_SRC_XTALIN=0,
   CLOCK_SRC_XO_IN=1,
@@ -6829,6 +7200,17 @@ typedef struct _DIG_TRANSMITTER_INFO_HEADER_V3_1{
   USHORT usPhyPllSettingOffset;          // offset of CLOCK_CONDITION_SETTING_ENTRY* with Phy Pll Settings
 }DIG_TRANSMITTER_INFO_HEADER_V3_1;
 
+typedef struct _DIG_TRANSMITTER_INFO_HEADER_V3_2{  
+  ATOM_COMMON_TABLE_HEADER sHeader;  
+  USHORT usDPVsPreEmphSettingOffset;     // offset of PHY_ANALOG_SETTING_INFO * with DP Voltage Swing and Pre-Emphasis for each Link clock 
+  USHORT usPhyAnalogRegListOffset;       // offset of CLOCK_CONDITION_REGESTER_INFO* with None-DP mode Analog Setting's register Info 
+  USHORT usPhyAnalogSettingOffset;       // offset of CLOCK_CONDITION_SETTING_ENTRY* with None-DP mode Analog Setting for each link clock range
+  USHORT usPhyPllRegListOffset;          // offset of CLOCK_CONDITION_REGESTER_INFO* with Phy Pll register Info 
+  USHORT usPhyPllSettingOffset;          // offset of CLOCK_CONDITION_SETTING_ENTRY* with Phy Pll Settings
+  USHORT usDPSSRegListOffset;            // offset of CLOCK_CONDITION_REGESTER_INFO* with Phy SS Pll register Info 
+  USHORT usDPSSSettingOffset;            // offset of CLOCK_CONDITION_SETTING_ENTRY* with Phy SS Pll Settings
+}DIG_TRANSMITTER_INFO_HEADER_V3_2;
+
 typedef struct _CLOCK_CONDITION_REGESTER_INFO{
   USHORT usRegisterIndex;
   UCHAR  ucStartBit;
@@ -6852,12 +7234,24 @@ typedef struct _PHY_CONDITION_REG_VAL{
   ULONG  ulRegVal;
 }PHY_CONDITION_REG_VAL;
 
+typedef struct _PHY_CONDITION_REG_VAL_V2{
+  ULONG  ulCondition;
+  UCHAR  ucCondition2;
+  ULONG  ulRegVal;
+}PHY_CONDITION_REG_VAL_V2;
+
 typedef struct _PHY_CONDITION_REG_INFO{
   USHORT usRegIndex;
   USHORT usSize;
   PHY_CONDITION_REG_VAL asRegVal[1];
 }PHY_CONDITION_REG_INFO;
 
+typedef struct _PHY_CONDITION_REG_INFO_V2{
+  USHORT usRegIndex;
+  USHORT usSize;
+  PHY_CONDITION_REG_VAL_V2 asRegVal[1];
+}PHY_CONDITION_REG_INFO_V2;
+
 typedef struct _PHY_ANALOG_SETTING_INFO{
   UCHAR  ucEncodeMode;
   UCHAR  ucPhySel;
@@ -6865,6 +7259,25 @@ typedef struct _PHY_ANALOG_SETTING_INFO{
   PHY_CONDITION_REG_INFO  asAnalogSetting[1];
 }PHY_ANALOG_SETTING_INFO;
 
+typedef struct _PHY_ANALOG_SETTING_INFO_V2{
+  UCHAR  ucEncodeMode;
+  UCHAR  ucPhySel;
+  USHORT usSize;
+  PHY_CONDITION_REG_INFO_V2  asAnalogSetting[1];
+}PHY_ANALOG_SETTING_INFO_V2;
+
+typedef struct _GFX_HAVESTING_PARAMETERS {
+  UCHAR ucGfxBlkId;                        //GFX blk id to be harvested, like CU, RB or PRIM
+  UCHAR ucReserved;                        //reserved 
+  UCHAR ucActiveUnitNumPerSH;              //requested active CU/RB/PRIM number per shader array
+  UCHAR ucMaxUnitNumPerSH;                 //max CU/RB/PRIM number per shader array   
+} GFX_HAVESTING_PARAMETERS;
+
+//ucGfxBlkId
+#define GFX_HARVESTING_CU_ID               0
+#define GFX_HARVESTING_RB_ID               1
+#define GFX_HARVESTING_PRIM_ID             2
+
 /****************************************************************************/  
 //Portion VI: Definitinos for vbios MC scratch registers that driver used
 /****************************************************************************/
@@ -6875,8 +7288,17 @@ typedef struct _PHY_ANALOG_SETTING_INFO{
 #define MC_MISC0__MEMORY_TYPE__GDDR3  0x30000000
 #define MC_MISC0__MEMORY_TYPE__GDDR4  0x40000000
 #define MC_MISC0__MEMORY_TYPE__GDDR5  0x50000000
+#define MC_MISC0__MEMORY_TYPE__HBM    0x60000000
 #define MC_MISC0__MEMORY_TYPE__DDR3   0xB0000000
 
+#define ATOM_MEM_TYPE_DDR_STRING      "DDR"
+#define ATOM_MEM_TYPE_DDR2_STRING     "DDR2"
+#define ATOM_MEM_TYPE_GDDR3_STRING    "GDDR3"
+#define ATOM_MEM_TYPE_GDDR4_STRING    "GDDR4"
+#define ATOM_MEM_TYPE_GDDR5_STRING    "GDDR5"
+#define ATOM_MEM_TYPE_HBM_STRING      "HBM"
+#define ATOM_MEM_TYPE_DDR3_STRING     "DDR3"
+
 /****************************************************************************/  
 //Portion VI: Definitinos being oboselete
 /****************************************************************************/
@@ -7274,6 +7696,7 @@ typedef struct _ATOM_PPLIB_THERMALCONTROLLER
 #define ATOM_PP_THERMALCONTROLLER_NISLANDS  15
 #define ATOM_PP_THERMALCONTROLLER_SISLANDS  16
 #define ATOM_PP_THERMALCONTROLLER_LM96163   17
+#define ATOM_PP_THERMALCONTROLLER_CISLANDS  18
 
 // Thermal controller 'combo type' to use an external controller for Fan control and an internal controller for thermal.
 // We probably should reserve the bit 0x80 for this use.
@@ -7316,6 +7739,8 @@ typedef struct _ATOM_PPLIB_EXTENDEDHEADER
     // Add extra system parameters here, always adjust size to include all fields.
     USHORT  usVCETableOffset; //points to ATOM_PPLIB_VCE_Table
     USHORT  usUVDTableOffset;   //points to ATOM_PPLIB_UVD_Table
+    USHORT  usSAMUTableOffset;  //points to ATOM_PPLIB_SAMU_Table
+    USHORT  usPPMTableOffset;   //points to ATOM_PPLIB_PPM_Table
 } ATOM_PPLIB_EXTENDEDHEADER;
 
 //// ATOM_PPLIB_POWERPLAYTABLE::ulPlatformCaps
@@ -7337,7 +7762,10 @@ typedef struct _ATOM_PPLIB_EXTENDEDHEADER
 #define ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL 0x8000                   // Does the driver control VDDCI independently from VDDC.
 #define ATOM_PP_PLATFORM_CAP_REGULATOR_HOT 0x00010000               // Enable the 'regulator hot' feature.
 #define ATOM_PP_PLATFORM_CAP_BACO          0x00020000               // Does the driver supports BACO state.
-
+#define ATOM_PP_PLATFORM_CAP_NEW_CAC_VOLTAGE   0x00040000           // Does the driver supports new CAC voltage table.
+#define ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY   0x00080000     // Does the driver supports revert GPIO5 polarity.
+#define ATOM_PP_PLATFORM_CAP_OUTPUT_THERMAL2GPIO17   0x00100000     // Does the driver supports thermal2GPIO17.
+#define ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE   0x00200000   // Does the driver supports VR HOT GPIO Configurable.
 
 typedef struct _ATOM_PPLIB_POWERPLAYTABLE
 {
@@ -7398,7 +7826,7 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE4
     USHORT                     usVddcDependencyOnMCLKOffset;
     USHORT                     usMaxClockVoltageOnDCOffset;
     USHORT                     usVddcPhaseShedLimitsTableOffset;    // Points to ATOM_PPLIB_PhaseSheddingLimits_Table
-    USHORT                     usReserved;  
+    USHORT                     usMvddDependencyOnMCLKOffset;  
 } ATOM_PPLIB_POWERPLAYTABLE4, *LPATOM_PPLIB_POWERPLAYTABLE4;
 
 typedef struct _ATOM_PPLIB_POWERPLAYTABLE5
@@ -7563,6 +7991,17 @@ typedef struct _ATOM_PPLIB_SI_CLOCK_INFO
 
 } ATOM_PPLIB_SI_CLOCK_INFO;
 
+typedef struct _ATOM_PPLIB_CI_CLOCK_INFO
+{
+      USHORT usEngineClockLow;
+      UCHAR  ucEngineClockHigh;
+
+      USHORT usMemoryClockLow;
+      UCHAR  ucMemoryClockHigh;
+      
+      UCHAR  ucPCIEGen;
+      USHORT usPCIELane;
+} ATOM_PPLIB_CI_CLOCK_INFO;
 
 typedef struct _ATOM_PPLIB_RS780_CLOCK_INFO
 
@@ -7680,8 +8119,8 @@ typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Table
 
 typedef struct _ATOM_PPLIB_CAC_Leakage_Record
 {
-    USHORT usVddc;  // We use this field for the "fake" standardized VDDC for power calculations                                                  
-    ULONG  ulLeakageValue;
+    USHORT usVddc;  // We use this field for the "fake" standardized VDDC for power calculations; For CI and newer, we use this as the real VDDC value.
+    ULONG  ulLeakageValue;  // For CI and newer we use this as the "fake" standar VDDC value.
 }ATOM_PPLIB_CAC_Leakage_Record;
 
 typedef struct _ATOM_PPLIB_CAC_Leakage_Table
@@ -7796,6 +8235,42 @@ typedef struct _ATOM_PPLIB_UVD_Table
 //    ATOM_PPLIB_UVD_State_Table states;
 }ATOM_PPLIB_UVD_Table;
 
+
+typedef struct _ATOM_PPLIB_SAMClk_Voltage_Limit_Record
+{
+      USHORT usVoltage;
+      USHORT usSAMClockLow;
+      UCHAR  ucSAMClockHigh;
+}ATOM_PPLIB_SAMClk_Voltage_Limit_Record;
+
+typedef struct _ATOM_PPLIB_SAMClk_Voltage_Limit_Table{
+    UCHAR numEntries;
+    ATOM_PPLIB_SAMClk_Voltage_Limit_Record entries[1];
+}ATOM_PPLIB_SAMClk_Voltage_Limit_Table;
+
+typedef struct _ATOM_PPLIB_SAMU_Table
+{
+      UCHAR revid;
+      ATOM_PPLIB_SAMClk_Voltage_Limit_Table limits;
+}ATOM_PPLIB_SAMU_Table;
+
+#define ATOM_PPM_A_A    1
+#define ATOM_PPM_A_I    2
+typedef struct _ATOM_PPLIB_PPM_Table
+{
+      UCHAR  ucRevId;
+      UCHAR  ucPpmDesign;          //A+I or A+A
+      USHORT usCpuCoreNumber;
+      ULONG  ulPlatformTDP;
+      ULONG  ulSmallACPlatformTDP;
+      ULONG  ulPlatformTDC;
+      ULONG  ulSmallACPlatformTDC;
+      ULONG  ulApuTDP;
+      ULONG  ulDGpuTDP;  
+      ULONG  ulDGpuUlvPower;
+      ULONG  ulTjmax;
+} ATOM_PPLIB_PPM_Table;
+
 /**************************************************************************/
 
 
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index d5df8fd10217..c7ad4b930850 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -555,7 +555,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                 if (rdev->family < CHIP_RV770)
                         radeon_crtc->pll_flags |= RADEON_PLL_PREFER_MINM_OVER_MAXP;
                 /* use frac fb div on APUs */
-                if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev))
+                if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev) || ASIC_IS_DCE8(rdev))
                         radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV;
                 /* use frac fb div on RS780/RS880 */
                 if ((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880))
@@ -743,7 +743,7 @@ static void atombios_crtc_set_disp_eng_pll(struct radeon_device *rdev,
                          * SetPixelClock provides the dividers
                          */
                         args.v6.ulDispEngClkFreq = cpu_to_le32(dispclk);
-                        if (ASIC_IS_DCE61(rdev))
+                        if (ASIC_IS_DCE61(rdev) || ASIC_IS_DCE8(rdev))
                                 args.v6.ucPpll = ATOM_EXT_PLL1;
                         else if (ASIC_IS_DCE6(rdev))
                                 args.v6.ucPpll = ATOM_PPLL0;
@@ -1143,7 +1143,9 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
         }
 
         if (tiling_flags & RADEON_TILING_MACRO) {
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        tmp = rdev->config.cik.tile_config;
+                else if (rdev->family >= CHIP_TAHITI)
                         tmp = rdev->config.si.tile_config;
                 else if (rdev->family >= CHIP_CAYMAN)
                         tmp = rdev->config.cayman.tile_config;
@@ -1170,11 +1172,29 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
                 fb_format |= EVERGREEN_GRPH_BANK_WIDTH(bankw);
                 fb_format |= EVERGREEN_GRPH_BANK_HEIGHT(bankh);
                 fb_format |= EVERGREEN_GRPH_MACRO_TILE_ASPECT(mtaspect);
+                if (rdev->family >= CHIP_BONAIRE) {
+                        /* XXX need to know more about the surface tiling mode */
+                        fb_format |= CIK_GRPH_MICRO_TILE_MODE(CIK_DISPLAY_MICRO_TILING);
+                }
         } else if (tiling_flags & RADEON_TILING_MICRO)
                 fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_1D_TILED_THIN1);
 
-        if ((rdev->family == CHIP_TAHITI) ||
-            (rdev->family == CHIP_PITCAIRN))
+        if (rdev->family >= CHIP_BONAIRE) {
+                u32 num_pipe_configs = rdev->config.cik.max_tile_pipes;
+                u32 num_rb = rdev->config.cik.max_backends_per_se;
+                if (num_pipe_configs > 8)
+                        num_pipe_configs = 8;
+                if (num_pipe_configs == 8)
+                        fb_format |= CIK_GRPH_PIPE_CONFIG(CIK_ADDR_SURF_P8_32x32_16x16);
+                else if (num_pipe_configs == 4) {
+                        if (num_rb == 4)
+                                fb_format |= CIK_GRPH_PIPE_CONFIG(CIK_ADDR_SURF_P4_16x16);
+                        else if (num_rb < 4)
+                                fb_format |= CIK_GRPH_PIPE_CONFIG(CIK_ADDR_SURF_P4_8x16);
+                } else if (num_pipe_configs == 2)
+                        fb_format |= CIK_GRPH_PIPE_CONFIG(CIK_ADDR_SURF_P2);
+        } else if ((rdev->family == CHIP_TAHITI) ||
+                   (rdev->family == CHIP_PITCAIRN))
                 fb_format |= SI_GRPH_PIPE_CONFIG(SI_ADDR_SURF_P8_32x32_8x16);
         else if (rdev->family == CHIP_VERDE)
                 fb_format |= SI_GRPH_PIPE_CONFIG(SI_ADDR_SURF_P4_8x16);
@@ -1224,8 +1244,12 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
         WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset, fb_pitch_pixels);
         WREG32(EVERGREEN_GRPH_ENABLE + radeon_crtc->crtc_offset, 1);
 
-        WREG32(EVERGREEN_DESKTOP_HEIGHT + radeon_crtc->crtc_offset,
-               target_fb->height);
+        if (rdev->family >= CHIP_BONAIRE)
+                WREG32(CIK_LB_DESKTOP_HEIGHT + radeon_crtc->crtc_offset,
+                       target_fb->height);
+        else
+                WREG32(EVERGREEN_DESKTOP_HEIGHT + radeon_crtc->crtc_offset,
+                       target_fb->height);
         x &= ~3;
         y &= ~1;
         WREG32(EVERGREEN_VIEWPORT_START + radeon_crtc->crtc_offset,
@@ -1597,6 +1621,12 @@ static int radeon_get_shared_nondp_ppll(struct drm_crtc *crtc)
  *
  * Asic specific PLL information
  *
+ * DCE 8.x
+ * KB/KV
+ * - PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP)
+ * CI
+ * - PPLL0, PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP) and DAC
+ *
  * DCE 6.1
  * - PPLL2 is only available to UNIPHYA (both DP and non-DP)
  * - PPLL0, PPLL1 are available for UNIPHYB/C/D/E/F (both DP and non-DP)
@@ -1623,7 +1653,47 @@ static int radeon_atom_pick_pll(struct drm_crtc *crtc)
         u32 pll_in_use;
         int pll;
 
-        if (ASIC_IS_DCE61(rdev)) {
+        if (ASIC_IS_DCE8(rdev)) {
+                if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(radeon_crtc->encoder))) {
+                        if (rdev->clock.dp_extclk)
+                                /* skip PPLL programming if using ext clock */
+                                return ATOM_PPLL_INVALID;
+                        else {
+                                /* use the same PPLL for all DP monitors */
+                                pll = radeon_get_shared_dp_ppll(crtc);
+                                if (pll != ATOM_PPLL_INVALID)
+                                        return pll;
+                        }
+                } else {
+                        /* use the same PPLL for all monitors with the same clock */
+                        pll = radeon_get_shared_nondp_ppll(crtc);
+                        if (pll != ATOM_PPLL_INVALID)
+                                return pll;
+                }
+                /* otherwise, pick one of the plls */
+                if ((rdev->family == CHIP_KAVERI) ||
+                    (rdev->family == CHIP_KABINI)) {
+                        /* KB/KV has PPLL1 and PPLL2 */
+                        pll_in_use = radeon_get_pll_use_mask(crtc);
+                        if (!(pll_in_use & (1 << ATOM_PPLL2)))
+                                return ATOM_PPLL2;
+                        if (!(pll_in_use & (1 << ATOM_PPLL1)))
+                                return ATOM_PPLL1;
+                        DRM_ERROR("unable to allocate a PPLL\n");
+                        return ATOM_PPLL_INVALID;
+                } else {
+                        /* CI has PPLL0, PPLL1, and PPLL2 */
+                        pll_in_use = radeon_get_pll_use_mask(crtc);
+                        if (!(pll_in_use & (1 << ATOM_PPLL2)))
+                                return ATOM_PPLL2;
+                        if (!(pll_in_use & (1 << ATOM_PPLL1)))
+                                return ATOM_PPLL1;
+                        if (!(pll_in_use & (1 << ATOM_PPLL0)))
+                                return ATOM_PPLL0;
+                        DRM_ERROR("unable to allocate a PPLL\n");
+                        return ATOM_PPLL_INVALID;
+                }
+        } else if (ASIC_IS_DCE61(rdev)) {
                 struct radeon_encoder_atom_dig *dig =
                         radeon_encoder->enc_priv;
 
@@ -1861,7 +1931,7 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
                 break;
         case ATOM_PPLL0:
                 /* disable the ppll */
-                if (ASIC_IS_DCE61(rdev))
+                if ((rdev->family == CHIP_ARUBA) || (rdev->family == CHIP_BONAIRE))
                         atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id,
                                                   0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss);
                 break;
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 8406c8251fbf..092275d53d4a 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -186,6 +186,13 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder,
         u8 backlight_level;
         char bl_name[16];
 
+        /* Mac laptops with multiple GPUs use the gmux driver for backlight
+         * so don't register a backlight device
+         */
+        if ((rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) &&
+            (rdev->pdev->device == 0x6741))
+                return;
+
         if (!radeon_encoder->enc_priv)
                 return;
 
@@ -296,6 +303,7 @@ static inline bool radeon_encoder_is_digital(struct drm_encoder *encoder)
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                 return true;
         default:
                 return false;
@@ -479,11 +487,11 @@ static u8 radeon_atom_get_bpc(struct drm_encoder *encoder)
         }
 }
 
-
 union dvo_encoder_control {
         ENABLE_EXTERNAL_TMDS_ENCODER_PS_ALLOCATION ext_tmds;
         DVO_ENCODER_CONTROL_PS_ALLOCATION dvo;
         DVO_ENCODER_CONTROL_PS_ALLOCATION_V3 dvo_v3;
+        DVO_ENCODER_CONTROL_PS_ALLOCATION_V1_4 dvo_v4;
 };
 
 void
@@ -533,6 +541,13 @@ atombios_dvo_setup(struct drm_encoder *encoder, int action)
                         args.dvo_v3.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
                         args.dvo_v3.ucDVOConfig = 0; /* XXX */
                         break;
+                case 4:
+                        /* DCE8 */
+                        args.dvo_v4.ucAction = action;
+                        args.dvo_v4.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
+                        args.dvo_v4.ucDVOConfig = 0; /* XXX */
+                        args.dvo_v4.ucBitPerColor = radeon_atom_get_bpc(encoder);
+                        break;
                 default:
                         DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
                         break;
@@ -915,10 +930,14 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo
                                 args.v4.ucLaneNum = 4;
 
                         if (ENCODER_MODE_IS_DP(args.v4.ucEncoderMode)) {
-                                if (dp_clock == 270000)
-                                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ;
-                                else if (dp_clock == 540000)
+                                if (dp_clock == 540000)
                                         args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ;
+                                else if (dp_clock == 324000)
+                                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_3_24GHZ;
+                                else if (dp_clock == 270000)
+                                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ;
+                                else
+                                        args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_1_62GHZ;
                         }
                         args.v4.acConfig.ucDigSel = dig->dig_encoder;
                         args.v4.ucBitPerColor = radeon_atom_get_bpc(encoder);
@@ -1012,6 +1031,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                 index = GetIndexIntoMasterTable(COMMAND, UNIPHYTransmitterControl);
                 break;
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
@@ -1271,6 +1291,9 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
                                 else
                                         args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYE;
                                 break;
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
+                                args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYG;
+                                break;
                         }
                         if (is_dp)
                                 args.v5.ucLaneNum = dp_lane_count;
@@ -1735,6 +1758,7 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                 radeon_atom_encoder_dpms_dig(encoder, mode);
                 break;
@@ -1872,6 +1896,7 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder)
                         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
                         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
                         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                                 dig = radeon_encoder->enc_priv;
                                 switch (dig->dig_encoder) {
@@ -1893,6 +1918,9 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder)
                                 case 5:
                                         args.v2.ucEncoderID = ASIC_INT_DIG6_ENCODER_ID;
                                         break;
+                                case 6:
+                                        args.v2.ucEncoderID = ASIC_INT_DIG7_ENCODER_ID;
+                                        break;
                                 }
                                 break;
                         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
@@ -1955,7 +1983,13 @@ atombios_apply_encoder_quirks(struct drm_encoder *encoder,
         /* set scaler clears this on some chips */
         if (ASIC_IS_AVIVO(rdev) &&
             (!(radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)))) {
-                if (ASIC_IS_DCE4(rdev)) {
+                if (ASIC_IS_DCE8(rdev)) {
+                        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                                WREG32(CIK_LB_DATA_FORMAT + radeon_crtc->crtc_offset,
+                                       CIK_INTERLEAVE_EN);
+                        else
+                                WREG32(CIK_LB_DATA_FORMAT + radeon_crtc->crtc_offset, 0);
+                } else if (ASIC_IS_DCE4(rdev)) {
                         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
                                 WREG32(EVERGREEN_DATA_FORMAT + radeon_crtc->crtc_offset,
                                        EVERGREEN_INTERLEAVE_EN);
@@ -2002,6 +2036,9 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder)
                         else
                                 return 4;
                         break;
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
+                        return 6;
+                        break;
                 }
         } else if (ASIC_IS_DCE4(rdev)) {
                 /* DCE4/5 */
@@ -2086,6 +2123,7 @@ radeon_atom_encoder_init(struct radeon_device *rdev)
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_INIT, 0, 0);
                         break;
@@ -2130,6 +2168,7 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                 /* handled in dpms */
                 break;
@@ -2395,6 +2434,7 @@ static void radeon_atom_encoder_disable(struct drm_encoder *encoder)
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                 /* handled in dpms */
                 break;
@@ -2626,6 +2666,7 @@ radeon_add_atom_encoder(struct drm_device *dev,
         case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+        case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                 if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
                         radeon_encoder->rmx_type = RMX_FULL;
                         drm_encoder_init(dev, encoder, &radeon_atom_enc_funcs, DRM_MODE_ENCODER_LVDS);
diff --git a/drivers/gpu/drm/radeon/btc_dpm.c b/drivers/gpu/drm/radeon/btc_dpm.c
new file mode 100644
index 000000000000..f072660c7665
--- /dev/null
+++ b/drivers/gpu/drm/radeon/btc_dpm.c
@@ -0,0 +1,2737 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "btcd.h"
+#include "r600_dpm.h"
+#include "cypress_dpm.h"
+#include "btc_dpm.h"
+#include "atom.h"
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define MC_CG_SEQ_DRAMCONF_S0       0x05
+#define MC_CG_SEQ_DRAMCONF_S1       0x06
+#define MC_CG_SEQ_YCLK_SUSPEND      0x04
+#define MC_CG_SEQ_YCLK_RESUME       0x0a
+
+#define SMC_RAM_END 0x8000
+
+#ifndef BTC_MGCG_SEQUENCE
+#define BTC_MGCG_SEQUENCE  300
+
+struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps);
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
+struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
+
+
+//********* BARTS **************//
+static const u32 barts_cgcg_cgls_default[] =
+{
+        /* Register,   Value,     Mask bits */
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define BARTS_CGCG_CGLS_DEFAULT_LENGTH sizeof(barts_cgcg_cgls_default) / (3 * sizeof(u32))
+
+static const u32 barts_cgcg_cgls_disable[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00000644, 0x000f7912, 0x001f4180,
+        0x00000644, 0x000f3812, 0x001f4180
+};
+#define BARTS_CGCG_CGLS_DISABLE_LENGTH sizeof(barts_cgcg_cgls_disable) / (3 * sizeof(u32))
+
+static const u32 barts_cgcg_cgls_enable[] =
+{
+        /* 0x0000c124, 0x84180000, 0x00180000, */
+        0x00000644, 0x000f7892, 0x001f4080,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff
+};
+#define BARTS_CGCG_CGLS_ENABLE_LENGTH sizeof(barts_cgcg_cgls_enable) / (3 * sizeof(u32))
+
+static const u32 barts_mgcg_default[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00005448, 0x00000100, 0xffffffff,
+        0x000055e4, 0x00600100, 0xffffffff,
+        0x0000160c, 0x00000100, 0xffffffff,
+        0x0000c164, 0x00000100, 0xffffffff,
+        0x00008a18, 0x00000100, 0xffffffff,
+        0x0000897c, 0x06000100, 0xffffffff,
+        0x00008b28, 0x00000100, 0xffffffff,
+        0x00009144, 0x00000100, 0xffffffff,
+        0x00009a60, 0x00000100, 0xffffffff,
+        0x00009868, 0x00000100, 0xffffffff,
+        0x00008d58, 0x00000100, 0xffffffff,
+        0x00009510, 0x00000100, 0xffffffff,
+        0x0000949c, 0x00000100, 0xffffffff,
+        0x00009654, 0x00000100, 0xffffffff,
+        0x00009030, 0x00000100, 0xffffffff,
+        0x00009034, 0x00000100, 0xffffffff,
+        0x00009038, 0x00000100, 0xffffffff,
+        0x0000903c, 0x00000100, 0xffffffff,
+        0x00009040, 0x00000100, 0xffffffff,
+        0x0000a200, 0x00000100, 0xffffffff,
+        0x0000a204, 0x00000100, 0xffffffff,
+        0x0000a208, 0x00000100, 0xffffffff,
+        0x0000a20c, 0x00000100, 0xffffffff,
+        0x0000977c, 0x00000100, 0xffffffff,
+        0x00003f80, 0x00000100, 0xffffffff,
+        0x0000a210, 0x00000100, 0xffffffff,
+        0x0000a214, 0x00000100, 0xffffffff,
+        0x000004d8, 0x00000100, 0xffffffff,
+        0x00009784, 0x00000100, 0xffffffff,
+        0x00009698, 0x00000100, 0xffffffff,
+        0x000004d4, 0x00000200, 0xffffffff,
+        0x000004d0, 0x00000000, 0xffffffff,
+        0x000030cc, 0x00000100, 0xffffffff,
+        0x0000d0c0, 0xff000100, 0xffffffff,
+        0x0000802c, 0x40000000, 0xffffffff,
+        0x0000915c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091b0, 0x00070000, 0xffffffff,
+        0x000091b4, 0x00030002, 0xffffffff,
+        0x000091b8, 0x00050004, 0xffffffff,
+        0x000091c4, 0x00010006, 0xffffffff,
+        0x000091c8, 0x00090008, 0xffffffff,
+        0x000091cc, 0x00070000, 0xffffffff,
+        0x000091d0, 0x00030002, 0xffffffff,
+        0x000091d4, 0x00050004, 0xffffffff,
+        0x000091e0, 0x00010006, 0xffffffff,
+        0x000091e4, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x000091ec, 0x00070000, 0xffffffff,
+        0x000091f0, 0x00030002, 0xffffffff,
+        0x000091f4, 0x00050004, 0xffffffff,
+        0x00009200, 0x00010006, 0xffffffff,
+        0x00009204, 0x00090008, 0xffffffff,
+        0x00009208, 0x00070000, 0xffffffff,
+        0x0000920c, 0x00030002, 0xffffffff,
+        0x00009210, 0x00050004, 0xffffffff,
+        0x0000921c, 0x00010006, 0xffffffff,
+        0x00009220, 0x00090008, 0xffffffff,
+        0x00009224, 0x00070000, 0xffffffff,
+        0x00009228, 0x00030002, 0xffffffff,
+        0x0000922c, 0x00050004, 0xffffffff,
+        0x00009238, 0x00010006, 0xffffffff,
+        0x0000923c, 0x00090008, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff,
+        0x0000802c, 0x40010000, 0xffffffff,
+        0x0000915c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091b0, 0x00070000, 0xffffffff,
+        0x000091b4, 0x00030002, 0xffffffff,
+        0x000091b8, 0x00050004, 0xffffffff,
+        0x000091c4, 0x00010006, 0xffffffff,
+        0x000091c8, 0x00090008, 0xffffffff,
+        0x000091cc, 0x00070000, 0xffffffff,
+        0x000091d0, 0x00030002, 0xffffffff,
+        0x000091d4, 0x00050004, 0xffffffff,
+        0x000091e0, 0x00010006, 0xffffffff,
+        0x000091e4, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x000091ec, 0x00070000, 0xffffffff,
+        0x000091f0, 0x00030002, 0xffffffff,
+        0x000091f4, 0x00050004, 0xffffffff,
+        0x00009200, 0x00010006, 0xffffffff,
+        0x00009204, 0x00090008, 0xffffffff,
+        0x00009208, 0x00070000, 0xffffffff,
+        0x0000920c, 0x00030002, 0xffffffff,
+        0x00009210, 0x00050004, 0xffffffff,
+        0x0000921c, 0x00010006, 0xffffffff,
+        0x00009220, 0x00090008, 0xffffffff,
+        0x00009224, 0x00070000, 0xffffffff,
+        0x00009228, 0x00030002, 0xffffffff,
+        0x0000922c, 0x00050004, 0xffffffff,
+        0x00009238, 0x00010006, 0xffffffff,
+        0x0000923c, 0x00090008, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff,
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define BARTS_MGCG_DEFAULT_LENGTH sizeof(barts_mgcg_default) / (3 * sizeof(u32))
+
+static const u32 barts_mgcg_disable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x00009150, 0x00600000, 0xffffffff
+};
+#define BARTS_MGCG_DISABLE_LENGTH sizeof(barts_mgcg_disable) / (3 * sizeof(u32))
+
+static const u32 barts_mgcg_enable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00009150, 0x81944000, 0xffffffff
+};
+#define BARTS_MGCG_ENABLE_LENGTH sizeof(barts_mgcg_enable) / (3 * sizeof(u32))
+
+//********* CAICOS **************//
+static const u32 caicos_cgcg_cgls_default[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define CAICOS_CGCG_CGLS_DEFAULT_LENGTH sizeof(caicos_cgcg_cgls_default) / (3 * sizeof(u32))
+
+static const u32 caicos_cgcg_cgls_disable[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00000644, 0x000f7912, 0x001f4180,
+        0x00000644, 0x000f3812, 0x001f4180
+};
+#define CAICOS_CGCG_CGLS_DISABLE_LENGTH sizeof(caicos_cgcg_cgls_disable) / (3 * sizeof(u32))
+
+static const u32 caicos_cgcg_cgls_enable[] =
+{
+        /* 0x0000c124, 0x84180000, 0x00180000, */
+        0x00000644, 0x000f7892, 0x001f4080,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff
+};
+#define CAICOS_CGCG_CGLS_ENABLE_LENGTH sizeof(caicos_cgcg_cgls_enable) / (3 * sizeof(u32))
+
+static const u32 caicos_mgcg_default[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00005448, 0x00000100, 0xffffffff,
+        0x000055e4, 0x00600100, 0xffffffff,
+        0x0000160c, 0x00000100, 0xffffffff,
+        0x0000c164, 0x00000100, 0xffffffff,
+        0x00008a18, 0x00000100, 0xffffffff,
+        0x0000897c, 0x06000100, 0xffffffff,
+        0x00008b28, 0x00000100, 0xffffffff,
+        0x00009144, 0x00000100, 0xffffffff,
+        0x00009a60, 0x00000100, 0xffffffff,
+        0x00009868, 0x00000100, 0xffffffff,
+        0x00008d58, 0x00000100, 0xffffffff,
+        0x00009510, 0x00000100, 0xffffffff,
+        0x0000949c, 0x00000100, 0xffffffff,
+        0x00009654, 0x00000100, 0xffffffff,
+        0x00009030, 0x00000100, 0xffffffff,
+        0x00009034, 0x00000100, 0xffffffff,
+        0x00009038, 0x00000100, 0xffffffff,
+        0x0000903c, 0x00000100, 0xffffffff,
+        0x00009040, 0x00000100, 0xffffffff,
+        0x0000a200, 0x00000100, 0xffffffff,
+        0x0000a204, 0x00000100, 0xffffffff,
+        0x0000a208, 0x00000100, 0xffffffff,
+        0x0000a20c, 0x00000100, 0xffffffff,
+        0x0000977c, 0x00000100, 0xffffffff,
+        0x00003f80, 0x00000100, 0xffffffff,
+        0x0000a210, 0x00000100, 0xffffffff,
+        0x0000a214, 0x00000100, 0xffffffff,
+        0x000004d8, 0x00000100, 0xffffffff,
+        0x00009784, 0x00000100, 0xffffffff,
+        0x00009698, 0x00000100, 0xffffffff,
+        0x000004d4, 0x00000200, 0xffffffff,
+        0x000004d0, 0x00000000, 0xffffffff,
+        0x000030cc, 0x00000100, 0xffffffff,
+        0x0000d0c0, 0xff000100, 0xffffffff,
+        0x0000915c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define CAICOS_MGCG_DEFAULT_LENGTH sizeof(caicos_mgcg_default) / (3 * sizeof(u32))
+
+static const u32 caicos_mgcg_disable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x00009150, 0x00600000, 0xffffffff
+};
+#define CAICOS_MGCG_DISABLE_LENGTH sizeof(caicos_mgcg_disable) / (3 * sizeof(u32))
+
+static const u32 caicos_mgcg_enable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00009150, 0x46944040, 0xffffffff
+};
+#define CAICOS_MGCG_ENABLE_LENGTH sizeof(caicos_mgcg_enable) / (3 * sizeof(u32))
+
+//********* TURKS **************//
+static const u32 turks_cgcg_cgls_default[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define TURKS_CGCG_CGLS_DEFAULT_LENGTH  sizeof(turks_cgcg_cgls_default) / (3 * sizeof(u32))
+
+static const u32 turks_cgcg_cgls_disable[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00000644, 0x000f7912, 0x001f4180,
+        0x00000644, 0x000f3812, 0x001f4180
+};
+#define TURKS_CGCG_CGLS_DISABLE_LENGTH sizeof(turks_cgcg_cgls_disable) / (3 * sizeof(u32))
+
+static const u32 turks_cgcg_cgls_enable[] =
+{
+        /* 0x0000c124, 0x84180000, 0x00180000, */
+        0x00000644, 0x000f7892, 0x001f4080,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff
+};
+#define TURKS_CGCG_CGLS_ENABLE_LENGTH sizeof(turks_cgcg_cgls_enable) / (3 * sizeof(u32))
+
+// These are the sequences for turks_mgcg_shls
+static const u32 turks_mgcg_default[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00005448, 0x00000100, 0xffffffff,
+        0x000055e4, 0x00600100, 0xffffffff,
+        0x0000160c, 0x00000100, 0xffffffff,
+        0x0000c164, 0x00000100, 0xffffffff,
+        0x00008a18, 0x00000100, 0xffffffff,
+        0x0000897c, 0x06000100, 0xffffffff,
+        0x00008b28, 0x00000100, 0xffffffff,
+        0x00009144, 0x00000100, 0xffffffff,
+        0x00009a60, 0x00000100, 0xffffffff,
+        0x00009868, 0x00000100, 0xffffffff,
+        0x00008d58, 0x00000100, 0xffffffff,
+        0x00009510, 0x00000100, 0xffffffff,
+        0x0000949c, 0x00000100, 0xffffffff,
+        0x00009654, 0x00000100, 0xffffffff,
+        0x00009030, 0x00000100, 0xffffffff,
+        0x00009034, 0x00000100, 0xffffffff,
+        0x00009038, 0x00000100, 0xffffffff,
+        0x0000903c, 0x00000100, 0xffffffff,
+        0x00009040, 0x00000100, 0xffffffff,
+        0x0000a200, 0x00000100, 0xffffffff,
+        0x0000a204, 0x00000100, 0xffffffff,
+        0x0000a208, 0x00000100, 0xffffffff,
+        0x0000a20c, 0x00000100, 0xffffffff,
+        0x0000977c, 0x00000100, 0xffffffff,
+        0x00003f80, 0x00000100, 0xffffffff,
+        0x0000a210, 0x00000100, 0xffffffff,
+        0x0000a214, 0x00000100, 0xffffffff,
+        0x000004d8, 0x00000100, 0xffffffff,
+        0x00009784, 0x00000100, 0xffffffff,
+        0x00009698, 0x00000100, 0xffffffff,
+        0x000004d4, 0x00000200, 0xffffffff,
+        0x000004d0, 0x00000000, 0xffffffff,
+        0x000030cc, 0x00000100, 0xffffffff,
+        0x0000d0c0, 0x00000100, 0xffffffff,
+        0x0000915c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091b0, 0x00070000, 0xffffffff,
+        0x000091b4, 0x00030002, 0xffffffff,
+        0x000091b8, 0x00050004, 0xffffffff,
+        0x000091c4, 0x00010006, 0xffffffff,
+        0x000091c8, 0x00090008, 0xffffffff,
+        0x000091cc, 0x00070000, 0xffffffff,
+        0x000091d0, 0x00030002, 0xffffffff,
+        0x000091d4, 0x00050004, 0xffffffff,
+        0x000091e0, 0x00010006, 0xffffffff,
+        0x000091e4, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x000091ec, 0x00070000, 0xffffffff,
+        0x000091f0, 0x00030002, 0xffffffff,
+        0x000091f4, 0x00050004, 0xffffffff,
+        0x00009200, 0x00010006, 0xffffffff,
+        0x00009204, 0x00090008, 0xffffffff,
+        0x00009208, 0x00070000, 0xffffffff,
+        0x0000920c, 0x00030002, 0xffffffff,
+        0x00009210, 0x00050004, 0xffffffff,
+        0x0000921c, 0x00010006, 0xffffffff,
+        0x00009220, 0x00090008, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define TURKS_MGCG_DEFAULT_LENGTH sizeof(turks_mgcg_default) / (3 * sizeof(u32))
+
+static const u32 turks_mgcg_disable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x00009150, 0x00600000, 0xffffffff
+};
+#define TURKS_MGCG_DISABLE_LENGTH sizeof(turks_mgcg_disable) / (3 * sizeof(u32))
+
+static const u32 turks_mgcg_enable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00009150, 0x6e944000, 0xffffffff
+};
+#define TURKS_MGCG_ENABLE_LENGTH sizeof(turks_mgcg_enable) / (3 * sizeof(u32))
+
+#endif
+
+#ifndef BTC_SYSLS_SEQUENCE
+#define BTC_SYSLS_SEQUENCE  100
+
+
+//********* BARTS **************//
+static const u32 barts_sysls_default[] =
+{
+        /* Register,   Value,     Mask bits */
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+#define BARTS_SYSLS_DEFAULT_LENGTH sizeof(barts_sysls_default) / (3 * sizeof(u32))
+
+static const u32 barts_sysls_disable[] =
+{
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x00041401, 0xffffffff,
+        0x0000264c, 0x00040400, 0xffffffff,
+        0x00002648, 0x00040400, 0xffffffff,
+        0x00002650, 0x00040400, 0xffffffff,
+        0x000020b8, 0x00040400, 0xffffffff,
+        0x000020bc, 0x00040400, 0xffffffff,
+        0x000020c0, 0x00040c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680000, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00007ffd, 0xffffffff,
+        0x00000c7c, 0x0000ff00, 0xffffffff,
+        0x00006dfc, 0x0000007f, 0xffffffff
+};
+#define BARTS_SYSLS_DISABLE_LENGTH sizeof(barts_sysls_disable) / (3 * sizeof(u32))
+
+static const u32 barts_sysls_enable[] =
+{
+        0x000055e8, 0x00000001, 0xffffffff,
+        0x0000d0bc, 0x00000100, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x000004c8, 0x00000000, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+#define BARTS_SYSLS_ENABLE_LENGTH sizeof(barts_sysls_enable) / (3 * sizeof(u32))
+
+//********* CAICOS **************//
+static const u32 caicos_sysls_default[] =
+{
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+#define CAICOS_SYSLS_DEFAULT_LENGTH sizeof(caicos_sysls_default) / (3 * sizeof(u32))
+
+static const u32 caicos_sysls_disable[] =
+{
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x00041401, 0xffffffff,
+        0x0000264c, 0x00040400, 0xffffffff,
+        0x00002648, 0x00040400, 0xffffffff,
+        0x00002650, 0x00040400, 0xffffffff,
+        0x000020b8, 0x00040400, 0xffffffff,
+        0x000020bc, 0x00040400, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680000, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00007ffd, 0xffffffff,
+        0x00000c7c, 0x0000ff00, 0xffffffff,
+        0x00006dfc, 0x0000007f, 0xffffffff
+};
+#define CAICOS_SYSLS_DISABLE_LENGTH sizeof(caicos_sysls_disable) / (3 * sizeof(u32))
+
+static const u32 caicos_sysls_enable[] =
+{
+        0x000055e8, 0x00000001, 0xffffffff,
+        0x0000d0bc, 0x00000100, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff,
+        0x000004c8, 0x00000000, 0xffffffff
+};
+#define CAICOS_SYSLS_ENABLE_LENGTH sizeof(caicos_sysls_enable) / (3 * sizeof(u32))
+
+//********* TURKS **************//
+static const u32 turks_sysls_default[] =
+{
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+#define TURKS_SYSLS_DEFAULT_LENGTH sizeof(turks_sysls_default) / (3 * sizeof(u32))
+
+static const u32 turks_sysls_disable[] =
+{
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x00041401, 0xffffffff,
+        0x0000264c, 0x00040400, 0xffffffff,
+        0x00002648, 0x00040400, 0xffffffff,
+        0x00002650, 0x00040400, 0xffffffff,
+        0x000020b8, 0x00040400, 0xffffffff,
+        0x000020bc, 0x00040400, 0xffffffff,
+        0x000020c0, 0x00040c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680000, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00007ffd, 0xffffffff,
+        0x00000c7c, 0x0000ff00, 0xffffffff,
+        0x00006dfc, 0x0000007f, 0xffffffff
+};
+#define TURKS_SYSLS_DISABLE_LENGTH sizeof(turks_sysls_disable) / (3 * sizeof(u32))
+
+static const u32 turks_sysls_enable[] =
+{
+        0x000055e8, 0x00000001, 0xffffffff,
+        0x0000d0bc, 0x00000100, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x000004c8, 0x00000000, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+#define TURKS_SYSLS_ENABLE_LENGTH sizeof(turks_sysls_enable) / (3 * sizeof(u32))
+
+#endif
+
+u32 btc_valid_sclk[40] =
+{
+        5000,   10000,  15000,  20000,  25000,  30000,  35000,  40000,  45000,  50000,
+        55000,  60000,  65000,  70000,  75000,  80000,  85000,  90000,  95000,  100000,
+        105000, 110000, 11500,  120000, 125000, 130000, 135000, 140000, 145000, 150000,
+        155000, 160000, 165000, 170000, 175000, 180000, 185000, 190000, 195000, 200000
+};
+
+static const struct radeon_blacklist_clocks btc_blacklist_clocks[] =
+{
+        { 10000, 30000, RADEON_SCLK_UP },
+        { 15000, 30000, RADEON_SCLK_UP },
+        { 20000, 30000, RADEON_SCLK_UP },
+        { 25000, 30000, RADEON_SCLK_UP }
+};
+
+void btc_apply_voltage_dependency_rules(struct radeon_clock_voltage_dependency_table *table,
+                                        u32 clock, u16 max_voltage, u16 *voltage)
+{
+        u32 i;
+
+        if ((table == NULL) || (table->count == 0))
+                return;
+
+        for (i= 0; i < table->count; i++) {
+                if (clock <= table->entries[i].clk) {
+                        if (*voltage < table->entries[i].v)
+                                *voltage = (u16)((table->entries[i].v < max_voltage) ?
+                                                  table->entries[i].v : max_voltage);
+                        return;
+                }
+        }
+
+        *voltage = (*voltage > max_voltage) ? *voltage : max_voltage;
+}
+
+static u32 btc_find_valid_clock(struct radeon_clock_array *clocks,
+                                u32 max_clock, u32 requested_clock)
+{
+        unsigned int i;
+
+        if ((clocks == NULL) || (clocks->count == 0))
+                return (requested_clock < max_clock) ? requested_clock : max_clock;
+
+        for (i = 0; i < clocks->count; i++) {
+                if (clocks->values[i] >= requested_clock)
+                        return (clocks->values[i] < max_clock) ? clocks->values[i] : max_clock;
+        }
+
+        return (clocks->values[clocks->count - 1] < max_clock) ?
+                clocks->values[clocks->count - 1] : max_clock;
+}
+
+static u32 btc_get_valid_mclk(struct radeon_device *rdev,
+                              u32 max_mclk, u32 requested_mclk)
+{
+        return btc_find_valid_clock(&rdev->pm.dpm.dyn_state.valid_mclk_values,
+                                    max_mclk, requested_mclk);
+}
+
+static u32 btc_get_valid_sclk(struct radeon_device *rdev,
+                              u32 max_sclk, u32 requested_sclk)
+{
+        return btc_find_valid_clock(&rdev->pm.dpm.dyn_state.valid_sclk_values,
+                                    max_sclk, requested_sclk);
+}
+
+void btc_skip_blacklist_clocks(struct radeon_device *rdev,
+                               const u32 max_sclk, const u32 max_mclk,
+                               u32 *sclk, u32 *mclk)
+{
+        int i, num_blacklist_clocks;
+
+        if ((sclk == NULL) || (mclk == NULL))
+                return;
+
+        num_blacklist_clocks = ARRAY_SIZE(btc_blacklist_clocks);
+
+        for (i = 0; i < num_blacklist_clocks; i++) {
+                if ((btc_blacklist_clocks[i].sclk == *sclk) &&
+                    (btc_blacklist_clocks[i].mclk == *mclk))
+                        break;
+        }
+
+        if (i < num_blacklist_clocks) {
+                if (btc_blacklist_clocks[i].action == RADEON_SCLK_UP) {
+                        *sclk = btc_get_valid_sclk(rdev, max_sclk, *sclk + 1);
+
+                        if (*sclk < max_sclk)
+                                btc_skip_blacklist_clocks(rdev, max_sclk, max_mclk, sclk, mclk);
+                }
+        }
+}
+
+void btc_adjust_clock_combinations(struct radeon_device *rdev,
+                                   const struct radeon_clock_and_voltage_limits *max_limits,
+                                   struct rv7xx_pl *pl)
+{
+
+        if ((pl->mclk == 0) || (pl->sclk == 0))
+                return;
+
+        if (pl->mclk == pl->sclk)
+                return;
+
+        if (pl->mclk > pl->sclk) {
+                if (((pl->mclk + (pl->sclk - 1)) / pl->sclk) > rdev->pm.dpm.dyn_state.mclk_sclk_ratio)
+                        pl->sclk = btc_get_valid_sclk(rdev,
+                                                      max_limits->sclk,
+                                                      (pl->mclk +
+                                                       (rdev->pm.dpm.dyn_state.mclk_sclk_ratio - 1)) /
+                                                      rdev->pm.dpm.dyn_state.mclk_sclk_ratio);
+        } else {
+                if ((pl->sclk - pl->mclk) > rdev->pm.dpm.dyn_state.sclk_mclk_delta)
+                        pl->mclk = btc_get_valid_mclk(rdev,
+                                                      max_limits->mclk,
+                                                      pl->sclk -
+                                                      rdev->pm.dpm.dyn_state.sclk_mclk_delta);
+        }
+}
+
+static u16 btc_find_voltage(struct atom_voltage_table *table, u16 voltage)
+{
+        unsigned int i;
+
+        for (i = 0; i < table->count; i++) {
+                if (voltage <= table->entries[i].value)
+                        return table->entries[i].value;
+        }
+
+        return table->entries[table->count - 1].value;
+}
+
+void btc_apply_voltage_delta_rules(struct radeon_device *rdev,
+                                   u16 max_vddc, u16 max_vddci,
+                                   u16 *vddc, u16 *vddci)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u16 new_voltage;
+
+        if ((0 == *vddc) || (0 == *vddci))
+                return;
+
+        if (*vddc > *vddci) {
+                if ((*vddc - *vddci) > rdev->pm.dpm.dyn_state.vddc_vddci_delta) {
+                        new_voltage = btc_find_voltage(&eg_pi->vddci_voltage_table,
+                                                       (*vddc - rdev->pm.dpm.dyn_state.vddc_vddci_delta));
+                        *vddci = (new_voltage < max_vddci) ? new_voltage : max_vddci;
+                }
+        } else {
+                if ((*vddci - *vddc) > rdev->pm.dpm.dyn_state.vddc_vddci_delta) {
+                        new_voltage = btc_find_voltage(&eg_pi->vddc_voltage_table,
+                                                       (*vddci - rdev->pm.dpm.dyn_state.vddc_vddci_delta));
+                        *vddc = (new_voltage < max_vddc) ? new_voltage : max_vddc;
+                }
+        }
+}
+
+static void btc_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                             bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp, bif;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        if (enable) {
+                if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+                    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+                        if (!pi->boot_in_gen2) {
+                                bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
+                                bif |= CG_CLIENT_REQ(0xd);
+                                WREG32(CG_BIF_REQ_AND_RSP, bif);
+
+                                tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                                tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
+                                tmp |= LC_GEN2_EN_STRAP;
+
+                                tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                                udelay(10);
+                                tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                        }
+                }
+        } else {
+                if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
+                    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+                        if (!pi->boot_in_gen2) {
+                                bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
+                                bif |= CG_CLIENT_REQ(0xd);
+                                WREG32(CG_BIF_REQ_AND_RSP, bif);
+
+                                tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                                tmp &= ~LC_GEN2_EN_STRAP;
+                        }
+                        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                }
+        }
+}
+
+static void btc_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                         bool enable)
+{
+        btc_enable_bif_dynamic_pcie_gen2(rdev, enable);
+
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
+}
+
+static int btc_disable_ulv(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (eg_pi->ulv.supported) {
+                if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) != PPSMC_Result_OK)
+                        return -EINVAL;
+        }
+        return 0;
+}
+
+static int btc_populate_ulv_state(struct radeon_device *rdev,
+                                  RV770_SMC_STATETABLE *table)
+{
+        int ret = -EINVAL;
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_pl *ulv_pl = eg_pi->ulv.pl;
+
+        if (ulv_pl->vddc) {
+                ret = cypress_convert_power_level_to_smc(rdev,
+                                                         ulv_pl,
+                                                         &table->ULVState.levels[0],
+                                                         PPSMC_DISPLAY_WATERMARK_LOW);
+                if (ret == 0) {
+                        table->ULVState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
+                        table->ULVState.levels[0].ACIndex = 1;
+
+                        table->ULVState.levels[1] = table->ULVState.levels[0];
+                        table->ULVState.levels[2] = table->ULVState.levels[0];
+
+                        table->ULVState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+                        WREG32(CG_ULV_CONTROL, BTC_CGULVCONTROL_DFLT);
+                        WREG32(CG_ULV_PARAMETER, BTC_CGULVPARAMETER_DFLT);
+                }
+        }
+
+        return ret;
+}
+
+static int btc_populate_smc_acpi_state(struct radeon_device *rdev,
+                                       RV770_SMC_STATETABLE *table)
+{
+        int ret = cypress_populate_smc_acpi_state(rdev, table);
+
+        if (ret == 0) {
+                table->ACPIState.levels[0].ACIndex = 0;
+                table->ACPIState.levels[1].ACIndex = 0;
+                table->ACPIState.levels[2].ACIndex = 0;
+        }
+
+        return ret;
+}
+
+void btc_program_mgcg_hw_sequence(struct radeon_device *rdev,
+                                  const u32 *sequence, u32 count)
+{
+        u32 i, length = count * 3;
+        u32 tmp;
+
+        for (i = 0; i < length; i+=3) {
+                tmp = RREG32(sequence[i]);
+                tmp &= ~sequence[i+2];
+                tmp |= sequence[i+1] & sequence[i+2];
+                WREG32(sequence[i], tmp);
+        }
+}
+
+static void btc_cg_clock_gating_default(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *p = NULL;
+
+        if (rdev->family == CHIP_BARTS) {
+                p = (const u32 *)&barts_cgcg_cgls_default;
+                count = BARTS_CGCG_CGLS_DEFAULT_LENGTH;
+        } else if (rdev->family == CHIP_TURKS) {
+                p = (const u32 *)&turks_cgcg_cgls_default;
+                count = TURKS_CGCG_CGLS_DEFAULT_LENGTH;
+        } else if (rdev->family == CHIP_CAICOS) {
+                p = (const u32 *)&caicos_cgcg_cgls_default;
+                count = CAICOS_CGCG_CGLS_DEFAULT_LENGTH;
+        } else
+                return;
+
+        btc_program_mgcg_hw_sequence(rdev, p, count);
+}
+
+static void btc_cg_clock_gating_enable(struct radeon_device *rdev,
+                                       bool enable)
+{
+        u32 count;
+        const u32 *p = NULL;
+
+        if (enable) {
+                if (rdev->family == CHIP_BARTS) {
+                        p = (const u32 *)&barts_cgcg_cgls_enable;
+                        count = BARTS_CGCG_CGLS_ENABLE_LENGTH;
+                } else if (rdev->family == CHIP_TURKS) {
+                        p = (const u32 *)&turks_cgcg_cgls_enable;
+                        count = TURKS_CGCG_CGLS_ENABLE_LENGTH;
+                } else if (rdev->family == CHIP_CAICOS) {
+                        p = (const u32 *)&caicos_cgcg_cgls_enable;
+                        count = CAICOS_CGCG_CGLS_ENABLE_LENGTH;
+                } else
+                        return;
+        } else {
+                if (rdev->family == CHIP_BARTS) {
+                        p = (const u32 *)&barts_cgcg_cgls_disable;
+                        count = BARTS_CGCG_CGLS_DISABLE_LENGTH;
+                } else if (rdev->family == CHIP_TURKS) {
+                        p = (const u32 *)&turks_cgcg_cgls_disable;
+                        count = TURKS_CGCG_CGLS_DISABLE_LENGTH;
+                } else if (rdev->family == CHIP_CAICOS) {
+                        p = (const u32 *)&caicos_cgcg_cgls_disable;
+                        count = CAICOS_CGCG_CGLS_DISABLE_LENGTH;
+                } else
+                        return;
+        }
+
+        btc_program_mgcg_hw_sequence(rdev, p, count);
+}
+
+static void btc_mg_clock_gating_default(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *p = NULL;
+
+        if (rdev->family == CHIP_BARTS) {
+                p = (const u32 *)&barts_mgcg_default;
+                count = BARTS_MGCG_DEFAULT_LENGTH;
+        } else if (rdev->family == CHIP_TURKS) {
+                p = (const u32 *)&turks_mgcg_default;
+                count = TURKS_MGCG_DEFAULT_LENGTH;
+        } else if (rdev->family == CHIP_CAICOS) {
+                p = (const u32 *)&caicos_mgcg_default;
+                count = CAICOS_MGCG_DEFAULT_LENGTH;
+        } else
+                return;
+
+        btc_program_mgcg_hw_sequence(rdev, p, count);
+}
+
+static void btc_mg_clock_gating_enable(struct radeon_device *rdev,
+                                       bool enable)
+{
+        u32 count;
+        const u32 *p = NULL;
+
+        if (enable) {
+                if (rdev->family == CHIP_BARTS) {
+                        p = (const u32 *)&barts_mgcg_enable;
+                        count = BARTS_MGCG_ENABLE_LENGTH;
+                } else if (rdev->family == CHIP_TURKS) {
+                        p = (const u32 *)&turks_mgcg_enable;
+                        count = TURKS_MGCG_ENABLE_LENGTH;
+                } else if (rdev->family == CHIP_CAICOS) {
+                        p = (const u32 *)&caicos_mgcg_enable;
+                        count = CAICOS_MGCG_ENABLE_LENGTH;
+                } else
+                        return;
+        } else {
+                if (rdev->family == CHIP_BARTS) {
+                        p = (const u32 *)&barts_mgcg_disable[0];
+                        count = BARTS_MGCG_DISABLE_LENGTH;
+                } else if (rdev->family == CHIP_TURKS) {
+                        p = (const u32 *)&turks_mgcg_disable[0];
+                        count = TURKS_MGCG_DISABLE_LENGTH;
+                } else if (rdev->family == CHIP_CAICOS) {
+                        p = (const u32 *)&caicos_mgcg_disable[0];
+                        count = CAICOS_MGCG_DISABLE_LENGTH;
+                } else
+                        return;
+        }
+
+        btc_program_mgcg_hw_sequence(rdev, p, count);
+}
+
+static void btc_ls_clock_gating_default(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *p = NULL;
+
+        if (rdev->family == CHIP_BARTS) {
+                p = (const u32 *)&barts_sysls_default;
+                count = BARTS_SYSLS_DEFAULT_LENGTH;
+        } else if (rdev->family == CHIP_TURKS) {
+                p = (const u32 *)&turks_sysls_default;
+                count = TURKS_SYSLS_DEFAULT_LENGTH;
+        } else if (rdev->family == CHIP_CAICOS) {
+                p = (const u32 *)&caicos_sysls_default;
+                count = CAICOS_SYSLS_DEFAULT_LENGTH;
+        } else
+                return;
+
+        btc_program_mgcg_hw_sequence(rdev, p, count);
+}
+
+static void btc_ls_clock_gating_enable(struct radeon_device *rdev,
+                                       bool enable)
+{
+        u32 count;
+        const u32 *p = NULL;
+
+        if (enable) {
+                if (rdev->family == CHIP_BARTS) {
+                        p = (const u32 *)&barts_sysls_enable;
+                        count = BARTS_SYSLS_ENABLE_LENGTH;
+                } else if (rdev->family == CHIP_TURKS) {
+                        p = (const u32 *)&turks_sysls_enable;
+                        count = TURKS_SYSLS_ENABLE_LENGTH;
+                } else if (rdev->family == CHIP_CAICOS) {
+                        p = (const u32 *)&caicos_sysls_enable;
+                        count = CAICOS_SYSLS_ENABLE_LENGTH;
+                } else
+                        return;
+        } else {
+                if (rdev->family == CHIP_BARTS) {
+                        p = (const u32 *)&barts_sysls_disable;
+                        count = BARTS_SYSLS_DISABLE_LENGTH;
+                } else if (rdev->family == CHIP_TURKS) {
+                        p = (const u32 *)&turks_sysls_disable;
+                        count = TURKS_SYSLS_DISABLE_LENGTH;
+                } else if (rdev->family == CHIP_CAICOS) {
+                        p = (const u32 *)&caicos_sysls_disable;
+                        count = CAICOS_SYSLS_DISABLE_LENGTH;
+                } else
+                        return;
+        }
+
+        btc_program_mgcg_hw_sequence(rdev, p, count);
+}
+
+bool btc_dpm_enabled(struct radeon_device *rdev)
+{
+        if (rv770_is_smc_running(rdev))
+                return true;
+        else
+                return false;
+}
+
+static int btc_init_smc_table(struct radeon_device *rdev,
+                              struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        RV770_SMC_STATETABLE *table = &pi->smc_statetable;
+        int ret;
+
+        memset(table, 0, sizeof(RV770_SMC_STATETABLE));
+
+        cypress_populate_smc_voltage_tables(rdev, table);
+
+        switch (rdev->pm.int_thermal_type) {
+        case THERMAL_TYPE_EVERGREEN:
+        case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
+                break;
+        case THERMAL_TYPE_NONE:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
+                break;
+        default:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
+                break;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+        if (pi->mem_gddr5)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+        ret = cypress_populate_smc_initial_state(rdev, radeon_boot_state, table);
+        if (ret)
+                return ret;
+
+        if (eg_pi->sclk_deep_sleep)
+                WREG32_P(SCLK_PSKIP_CNTL, PSKIP_ON_ALLOW_STOP_HI(32),
+                         ~PSKIP_ON_ALLOW_STOP_HI_MASK);
+
+        ret = btc_populate_smc_acpi_state(rdev, table);
+        if (ret)
+                return ret;
+
+        if (eg_pi->ulv.supported) {
+                ret = btc_populate_ulv_state(rdev, table);
+                if (ret)
+                        eg_pi->ulv.supported = false;
+        }
+
+        table->driverState = table->initialState;
+
+        return rv770_copy_bytes_to_smc(rdev,
+                                       pi->state_table_start,
+                                       (u8 *)table,
+                                       sizeof(RV770_SMC_STATETABLE),
+                                       pi->sram_end);
+}
+
+static void btc_set_at_for_uvd(struct radeon_device *rdev,
+                               struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        int idx = 0;
+
+        if (r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2))
+                idx = 1;
+
+        if ((idx == 1) && !eg_pi->smu_uvd_hs) {
+                pi->rlp = 10;
+                pi->rmp = 100;
+                pi->lhp = 100;
+                pi->lmp = 10;
+        } else {
+                pi->rlp = eg_pi->ats[idx].rlp;
+                pi->rmp = eg_pi->ats[idx].rmp;
+                pi->lhp = eg_pi->ats[idx].lhp;
+                pi->lmp = eg_pi->ats[idx].lmp;
+        }
+
+}
+
+void btc_notify_uvd_to_smc(struct radeon_device *rdev,
+                           struct radeon_ps *radeon_new_state)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
+                rv770_write_smc_soft_register(rdev,
+                                              RV770_SMC_SOFT_REGISTER_uvd_enabled, 1);
+                eg_pi->uvd_enabled = true;
+        } else {
+                rv770_write_smc_soft_register(rdev,
+                                              RV770_SMC_SOFT_REGISTER_uvd_enabled, 0);
+                eg_pi->uvd_enabled = false;
+        }
+}
+
+int btc_reset_to_default(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return 0;
+}
+
+static void btc_stop_smc(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (((RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK) >> LB_SYNC_RESET_SEL_SHIFT) != 1)
+                        break;
+                udelay(1);
+        }
+        udelay(100);
+
+        r7xx_stop_smc(rdev);
+}
+
+void btc_read_arb_registers(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct evergreen_arb_registers *arb_registers =
+                &eg_pi->bootup_arb_registers;
+
+        arb_registers->mc_arb_dram_timing = RREG32(MC_ARB_DRAM_TIMING);
+        arb_registers->mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+        arb_registers->mc_arb_rfsh_rate = RREG32(MC_ARB_RFSH_RATE);
+        arb_registers->mc_arb_burst_time = RREG32(MC_ARB_BURST_TIME);
+}
+
+
+static void btc_set_arb0_registers(struct radeon_device *rdev,
+                                   struct evergreen_arb_registers *arb_registers)
+{
+        u32 val;
+
+        WREG32(MC_ARB_DRAM_TIMING,  arb_registers->mc_arb_dram_timing);
+        WREG32(MC_ARB_DRAM_TIMING2, arb_registers->mc_arb_dram_timing2);
+
+        val = (arb_registers->mc_arb_rfsh_rate & POWERMODE0_MASK) >>
+                POWERMODE0_SHIFT;
+        WREG32_P(MC_ARB_RFSH_RATE, POWERMODE0(val), ~POWERMODE0_MASK);
+
+        val = (arb_registers->mc_arb_burst_time & STATE0_MASK) >>
+                STATE0_SHIFT;
+        WREG32_P(MC_ARB_BURST_TIME, STATE0(val), ~STATE0_MASK);
+}
+
+static void btc_set_boot_state_timing(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (eg_pi->ulv.supported)
+                btc_set_arb0_registers(rdev, &eg_pi->bootup_arb_registers);
+}
+
+static bool btc_is_state_ulv_compatible(struct radeon_device *rdev,
+                                        struct radeon_ps *radeon_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_pl *ulv_pl = eg_pi->ulv.pl;
+
+        if (state->low.mclk != ulv_pl->mclk)
+                return false;
+
+        if (state->low.vddci != ulv_pl->vddci)
+                return false;
+
+        /* XXX check minclocks, etc. */
+
+        return true;
+}
+
+
+static int btc_set_ulv_dram_timing(struct radeon_device *rdev)
+{
+        u32 val;
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_pl *ulv_pl = eg_pi->ulv.pl;
+
+        radeon_atom_set_engine_dram_timings(rdev,
+                                            ulv_pl->sclk,
+                                            ulv_pl->mclk);
+
+        val = rv770_calculate_memory_refresh_rate(rdev, ulv_pl->sclk);
+        WREG32_P(MC_ARB_RFSH_RATE, POWERMODE0(val), ~POWERMODE0_MASK);
+
+        val = cypress_calculate_burst_time(rdev, ulv_pl->sclk, ulv_pl->mclk);
+        WREG32_P(MC_ARB_BURST_TIME, STATE0(val), ~STATE0_MASK);
+
+        return 0;
+}
+
+static int btc_enable_ulv(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return 0;
+}
+
+static int btc_set_power_state_conditionally_enable_ulv(struct radeon_device *rdev,
+                                                        struct radeon_ps *radeon_new_state)
+{
+        int ret = 0;
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (eg_pi->ulv.supported) {
+                if (btc_is_state_ulv_compatible(rdev, radeon_new_state)) {
+                        // Set ARB[0] to reflect the DRAM timing needed for ULV.
+                        ret = btc_set_ulv_dram_timing(rdev);
+                        if (ret == 0)
+                                ret = btc_enable_ulv(rdev);
+                }
+        }
+
+        return ret;
+}
+
+static bool btc_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
+{
+        bool result = true;
+
+        switch (in_reg) {
+        case MC_SEQ_RAS_TIMING >> 2:
+                *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_CAS_TIMING >> 2:
+                *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_MISC_TIMING >> 2:
+                *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_MISC_TIMING2 >> 2:
+                *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
+                break;
+        case MC_SEQ_RD_CTL_D0 >> 2:
+                *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
+                break;
+        case MC_SEQ_RD_CTL_D1 >> 2:
+                *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_D0 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_D1 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
+                break;
+        case MC_PMG_CMD_EMRS >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS1 >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+                break;
+        default:
+                result = false;
+                break;
+        }
+
+        return result;
+}
+
+static void btc_set_valid_flag(struct evergreen_mc_reg_table *table)
+{
+        u8 i, j;
+
+        for (i = 0; i < table->last; i++) {
+                for (j = 1; j < table->num_entries; j++) {
+                        if (table->mc_reg_table_entry[j-1].mc_data[i] !=
+                            table->mc_reg_table_entry[j].mc_data[i]) {
+                                table->valid_flag |= (1 << i);
+                                break;
+                        }
+                }
+        }
+}
+
+static int btc_set_mc_special_registers(struct radeon_device *rdev,
+                                        struct evergreen_mc_reg_table *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 i, j, k;
+        u32 tmp;
+
+        for (i = 0, j = table->last; i < table->last; i++) {
+                switch (table->mc_reg_address[i].s1) {
+                case MC_SEQ_MISC1 >> 2:
+                        tmp = RREG32(MC_PMG_CMD_EMRS);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++) {
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        ((tmp & 0xffff0000)) |
+                                        ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+                        }
+                        j++;
+
+                        if (j > SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+
+                        tmp = RREG32(MC_PMG_CMD_MRS);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++) {
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        (tmp & 0xffff0000) |
+                                        (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+                                if (!pi->mem_gddr5)
+                                        table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+                        }
+                        j++;
+
+                        if (j > SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+                        break;
+                case MC_SEQ_RESERVE_M >> 2:
+                        tmp = RREG32(MC_PMG_CMD_MRS1);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++) {
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        (tmp & 0xffff0000) |
+                                        (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+                        }
+                        j++;
+
+                        if (j > SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        table->last = j;
+
+        return 0;
+}
+
+static void btc_set_s0_mc_reg_index(struct evergreen_mc_reg_table *table)
+{
+        u32 i;
+        u16 address;
+
+        for (i = 0; i < table->last; i++) {
+                table->mc_reg_address[i].s0 =
+                        btc_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
+                        address : table->mc_reg_address[i].s1;
+        }
+}
+
+static int btc_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
+                                       struct evergreen_mc_reg_table *eg_table)
+{
+        u8 i, j;
+
+        if (table->last > SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE)
+                return -EINVAL;
+
+        if (table->num_entries > MAX_AC_TIMING_ENTRIES)
+                return -EINVAL;
+
+        for (i = 0; i < table->last; i++)
+                eg_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+        eg_table->last = table->last;
+
+        for (i = 0; i < table->num_entries; i++) {
+                eg_table->mc_reg_table_entry[i].mclk_max =
+                        table->mc_reg_table_entry[i].mclk_max;
+                for(j = 0; j < table->last; j++)
+                        eg_table->mc_reg_table_entry[i].mc_data[j] =
+                                table->mc_reg_table_entry[i].mc_data[j];
+        }
+        eg_table->num_entries = table->num_entries;
+
+        return 0;
+}
+
+static int btc_initialize_mc_reg_table(struct radeon_device *rdev)
+{
+        int ret;
+        struct atom_mc_reg_table *table;
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct evergreen_mc_reg_table *eg_table = &eg_pi->mc_reg_table;
+        u8 module_index = rv770_get_memory_module_index(rdev);
+
+        table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
+        if (!table)
+                return -ENOMEM;
+
+        /* Program additional LP registers that are no longer programmed by VBIOS */
+        WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
+        WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
+        WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
+        WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
+        WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
+        WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
+        WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
+        WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
+        WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
+        WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
+        WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
+
+        ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
+
+        if (ret)
+                goto init_mc_done;
+
+        ret = btc_copy_vbios_mc_reg_table(table, eg_table);
+
+        if (ret)
+                goto init_mc_done;
+
+        btc_set_s0_mc_reg_index(eg_table);
+        ret = btc_set_mc_special_registers(rdev, eg_table);
+
+        if (ret)
+                goto init_mc_done;
+
+        btc_set_valid_flag(eg_table);
+
+init_mc_done:
+        kfree(table);
+
+        return ret;
+}
+
+static void btc_init_stutter_mode(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp;
+
+        if (pi->mclk_stutter_mode_threshold) {
+                if (pi->mem_gddr5) {
+                        tmp = RREG32(MC_PMG_AUTO_CFG);
+                        if ((0x200 & tmp) == 0) {
+                                tmp = (tmp & 0xfffffc0b) | 0x204;
+                                WREG32(MC_PMG_AUTO_CFG, tmp);
+                        }
+                }
+        }
+}
+
+static void btc_apply_state_adjust_rules(struct radeon_device *rdev,
+                                         struct radeon_ps *rps)
+{
+        struct rv7xx_ps *ps = rv770_get_ps(rps);
+        struct radeon_clock_and_voltage_limits *max_limits;
+        bool disable_mclk_switching;
+        u32 mclk, sclk;
+        u16 vddc, vddci;
+
+        if (rdev->pm.dpm.new_active_crtc_count > 1)
+                disable_mclk_switching = true;
+        else
+                disable_mclk_switching = false;
+
+        if (rdev->pm.dpm.ac_power)
+                max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
+        else
+                max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
+
+        if (rdev->pm.dpm.ac_power == false) {
+                if (ps->high.mclk > max_limits->mclk)
+                        ps->high.mclk = max_limits->mclk;
+                if (ps->high.sclk > max_limits->sclk)
+                        ps->high.sclk = max_limits->sclk;
+                if (ps->high.vddc > max_limits->vddc)
+                        ps->high.vddc = max_limits->vddc;
+                if (ps->high.vddci > max_limits->vddci)
+                        ps->high.vddci = max_limits->vddci;
+
+                if (ps->medium.mclk > max_limits->mclk)
+                        ps->medium.mclk = max_limits->mclk;
+                if (ps->medium.sclk > max_limits->sclk)
+                        ps->medium.sclk = max_limits->sclk;
+                if (ps->medium.vddc > max_limits->vddc)
+                        ps->medium.vddc = max_limits->vddc;
+                if (ps->medium.vddci > max_limits->vddci)
+                        ps->medium.vddci = max_limits->vddci;
+
+                if (ps->low.mclk > max_limits->mclk)
+                        ps->low.mclk = max_limits->mclk;
+                if (ps->low.sclk > max_limits->sclk)
+                        ps->low.sclk = max_limits->sclk;
+                if (ps->low.vddc > max_limits->vddc)
+                        ps->low.vddc = max_limits->vddc;
+                if (ps->low.vddci > max_limits->vddci)
+                        ps->low.vddci = max_limits->vddci;
+        }
+
+        /* XXX validate the min clocks required for display */
+
+        if (disable_mclk_switching) {
+                sclk = ps->low.sclk;
+                mclk = ps->high.mclk;
+                vddc = ps->low.vddc;
+                vddci = ps->high.vddci;
+        } else {
+                sclk = ps->low.sclk;
+                mclk = ps->low.mclk;
+                vddc = ps->low.vddc;
+                vddci = ps->low.vddci;
+        }
+
+        /* adjusted low state */
+        ps->low.sclk = sclk;
+        ps->low.mclk = mclk;
+        ps->low.vddc = vddc;
+        ps->low.vddci = vddci;
+
+        btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
+                                  &ps->low.sclk, &ps->low.mclk);
+
+        /* adjusted medium, high states */
+        if (ps->medium.sclk < ps->low.sclk)
+                ps->medium.sclk = ps->low.sclk;
+        if (ps->medium.vddc < ps->low.vddc)
+                ps->medium.vddc = ps->low.vddc;
+        if (ps->high.sclk < ps->medium.sclk)
+                ps->high.sclk = ps->medium.sclk;
+        if (ps->high.vddc < ps->medium.vddc)
+                ps->high.vddc = ps->medium.vddc;
+
+        if (disable_mclk_switching) {
+                mclk = ps->low.mclk;
+                if (mclk < ps->medium.mclk)
+                        mclk = ps->medium.mclk;
+                if (mclk < ps->high.mclk)
+                        mclk = ps->high.mclk;
+                ps->low.mclk = mclk;
+                ps->low.vddci = vddci;
+                ps->medium.mclk = mclk;
+                ps->medium.vddci = vddci;
+                ps->high.mclk = mclk;
+                ps->high.vddci = vddci;
+        } else {
+                if (ps->medium.mclk < ps->low.mclk)
+                        ps->medium.mclk = ps->low.mclk;
+                if (ps->medium.vddci < ps->low.vddci)
+                        ps->medium.vddci = ps->low.vddci;
+                if (ps->high.mclk < ps->medium.mclk)
+                        ps->high.mclk = ps->medium.mclk;
+                if (ps->high.vddci < ps->medium.vddci)
+                        ps->high.vddci = ps->medium.vddci;
+        }
+
+        btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
+                                  &ps->medium.sclk, &ps->medium.mclk);
+        btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
+                                  &ps->high.sclk, &ps->high.mclk);
+
+        btc_adjust_clock_combinations(rdev, max_limits, &ps->low);
+        btc_adjust_clock_combinations(rdev, max_limits, &ps->medium);
+        btc_adjust_clock_combinations(rdev, max_limits, &ps->high);
+
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+                                           ps->low.sclk, max_limits->vddc, &ps->low.vddc);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+                                           ps->low.mclk, max_limits->vddci, &ps->low.vddci);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+                                           ps->low.mclk, max_limits->vddc, &ps->low.vddc);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
+                                           rdev->clock.current_dispclk, max_limits->vddc, &ps->low.vddc);
+
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+                                           ps->medium.sclk, max_limits->vddc, &ps->medium.vddc);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+                                           ps->medium.mclk, max_limits->vddci, &ps->medium.vddci);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+                                           ps->medium.mclk, max_limits->vddc, &ps->medium.vddc);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
+                                           rdev->clock.current_dispclk, max_limits->vddc, &ps->medium.vddc);
+
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+                                           ps->high.sclk, max_limits->vddc, &ps->high.vddc);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+                                           ps->high.mclk, max_limits->vddci, &ps->high.vddci);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+                                           ps->high.mclk, max_limits->vddc, &ps->high.vddc);
+        btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
+                                           rdev->clock.current_dispclk, max_limits->vddc, &ps->high.vddc);
+
+        btc_apply_voltage_delta_rules(rdev, max_limits->vddc, max_limits->vddci,
+                                      &ps->low.vddc, &ps->low.vddci);
+        btc_apply_voltage_delta_rules(rdev, max_limits->vddc, max_limits->vddci,
+                                      &ps->medium.vddc, &ps->medium.vddci);
+        btc_apply_voltage_delta_rules(rdev, max_limits->vddc, max_limits->vddci,
+                                      &ps->high.vddc, &ps->high.vddci);
+
+        if ((ps->high.vddc <= rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc) &&
+            (ps->medium.vddc <= rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc) &&
+            (ps->low.vddc <= rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc))
+                ps->dc_compatible = true;
+        else
+                ps->dc_compatible = false;
+
+        if (ps->low.vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
+                ps->low.flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
+        if (ps->medium.vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
+                ps->medium.flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
+        if (ps->high.vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
+                ps->high.flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
+}
+
+static void btc_update_current_ps(struct radeon_device *rdev,
+                                  struct radeon_ps *rps)
+{
+        struct rv7xx_ps *new_ps = rv770_get_ps(rps);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        eg_pi->current_rps = *rps;
+        eg_pi->current_ps = *new_ps;
+        eg_pi->current_rps.ps_priv = &eg_pi->current_ps;
+}
+
+static void btc_update_requested_ps(struct radeon_device *rdev,
+                                    struct radeon_ps *rps)
+{
+        struct rv7xx_ps *new_ps = rv770_get_ps(rps);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        eg_pi->requested_rps = *rps;
+        eg_pi->requested_ps = *new_ps;
+        eg_pi->requested_rps.ps_priv = &eg_pi->requested_ps;
+}
+
+void btc_dpm_reset_asic(struct radeon_device *rdev)
+{
+        rv770_restrict_performance_levels_before_switch(rdev);
+        btc_disable_ulv(rdev);
+        btc_set_boot_state_timing(rdev);
+        rv770_set_boot_state(rdev);
+}
+
+int btc_dpm_pre_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
+        struct radeon_ps *new_ps = &requested_ps;
+
+        btc_update_requested_ps(rdev, new_ps);
+
+        btc_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
+
+        return 0;
+}
+
+int btc_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = &eg_pi->requested_rps;
+        struct radeon_ps *old_ps = &eg_pi->current_rps;
+        int ret;
+
+        ret = btc_disable_ulv(rdev);
+        btc_set_boot_state_timing(rdev);
+        ret = rv770_restrict_performance_levels_before_switch(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n");
+                return ret;
+        }
+        if (eg_pi->pcie_performance_request)
+                cypress_notify_link_speed_change_before_state_change(rdev, new_ps, old_ps);
+
+        rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+        ret = rv770_halt_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_halt_smc failed\n");
+                return ret;
+        }
+        btc_set_at_for_uvd(rdev, new_ps);
+        if (eg_pi->smu_uvd_hs)
+                btc_notify_uvd_to_smc(rdev, new_ps);
+        ret = cypress_upload_sw_state(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("cypress_upload_sw_state failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = cypress_upload_mc_reg_table(rdev, new_ps);
+                if (ret) {
+                        DRM_ERROR("cypress_upload_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+
+        cypress_program_memory_timing_parameters(rdev, new_ps);
+
+        ret = rv770_resume_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_resume_smc failed\n");
+                return ret;
+        }
+        ret = rv770_set_sw_state(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_set_sw_state failed\n");
+                return ret;
+        }
+        rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+
+        if (eg_pi->pcie_performance_request)
+                cypress_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps);
+
+        ret = btc_set_power_state_conditionally_enable_ulv(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("btc_set_power_state_conditionally_enable_ulv failed\n");
+                return ret;
+        }
+
+        ret = rv770_unrestrict_performance_levels_after_switch(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_unrestrict_performance_levels_after_switch failed\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+void btc_dpm_post_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = &eg_pi->requested_rps;
+
+        btc_update_current_ps(rdev, new_ps);
+}
+
+int btc_dpm_enable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        if (pi->gfx_clock_gating)
+                btc_cg_clock_gating_default(rdev);
+
+        if (btc_dpm_enabled(rdev))
+                return -EINVAL;
+
+        if (pi->mg_clock_gating)
+                btc_mg_clock_gating_default(rdev);
+
+        if (eg_pi->ls_clock_gating)
+                btc_ls_clock_gating_default(rdev);
+
+        if (pi->voltage_control) {
+                rv770_enable_voltage_control(rdev, true);
+                ret = cypress_construct_voltage_tables(rdev);
+                if (ret) {
+                        DRM_ERROR("cypress_construct_voltage_tables failed\n");
+                        return ret;
+                }
+        }
+
+        if (pi->mvdd_control) {
+                ret = cypress_get_mvdd_configuration(rdev);
+                if (ret) {
+                        DRM_ERROR("cypress_get_mvdd_configuration failed\n");
+                        return ret;
+                }
+        }
+
+        if (eg_pi->dynamic_ac_timing) {
+                ret = btc_initialize_mc_reg_table(rdev);
+                if (ret)
+                        eg_pi->dynamic_ac_timing = false;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv770_enable_backbias(rdev, true);
+
+        if (pi->dynamic_ss)
+                cypress_enable_spread_spectrum(rdev, true);
+
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, true);
+
+        rv770_setup_bsp(rdev);
+        rv770_program_git(rdev);
+        rv770_program_tp(rdev);
+        rv770_program_tpp(rdev);
+        rv770_program_sstp(rdev);
+        rv770_program_engine_speed_parameters(rdev);
+        cypress_enable_display_gap(rdev);
+        rv770_program_vc(rdev);
+
+        if (pi->dynamic_pcie_gen2)
+                btc_enable_dynamic_pcie_gen2(rdev, true);
+
+        ret = rv770_upload_firmware(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_upload_firmware failed\n");
+                return ret;
+        }
+        ret = cypress_get_table_locations(rdev);
+        if (ret) {
+                DRM_ERROR("cypress_get_table_locations failed\n");
+                return ret;
+        }
+        ret = btc_init_smc_table(rdev, boot_ps);
+        if (ret)
+                return ret;
+
+        if (eg_pi->dynamic_ac_timing) {
+                ret = cypress_populate_mc_reg_table(rdev, boot_ps);
+                if (ret) {
+                        DRM_ERROR("cypress_populate_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+
+        cypress_program_response_times(rdev);
+        r7xx_start_smc(rdev);
+        ret = cypress_notify_smc_display_change(rdev, false);
+        if (ret) {
+                DRM_ERROR("cypress_notify_smc_display_change failed\n");
+                return ret;
+        }
+        cypress_enable_sclk_control(rdev, true);
+
+        if (eg_pi->memory_transition)
+                cypress_enable_mclk_control(rdev, true);
+
+        cypress_start_dpm(rdev);
+
+        if (pi->gfx_clock_gating)
+                btc_cg_clock_gating_enable(rdev, true);
+
+        if (pi->mg_clock_gating)
+                btc_mg_clock_gating_enable(rdev, true);
+
+        if (eg_pi->ls_clock_gating)
+                btc_ls_clock_gating_enable(rdev, true);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                PPSMC_Result result;
+
+                ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+                result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
+
+                if (result != PPSMC_Result_OK)
+                        DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
+        }
+
+        rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+
+        btc_init_stutter_mode(rdev);
+
+        btc_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
+
+        return 0;
+};
+
+void btc_dpm_disable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (!btc_dpm_enabled(rdev))
+                return;
+
+        rv770_clear_vc(rdev);
+
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, false);
+
+        if (pi->dynamic_pcie_gen2)
+                btc_enable_dynamic_pcie_gen2(rdev, false);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        if (pi->gfx_clock_gating)
+                btc_cg_clock_gating_enable(rdev, false);
+
+        if (pi->mg_clock_gating)
+                btc_mg_clock_gating_enable(rdev, false);
+
+        if (eg_pi->ls_clock_gating)
+                btc_ls_clock_gating_enable(rdev, false);
+
+        rv770_stop_dpm(rdev);
+        btc_reset_to_default(rdev);
+        btc_stop_smc(rdev);
+        cypress_enable_spread_spectrum(rdev, false);
+
+        btc_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
+}
+
+void btc_dpm_setup_asic(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        rv770_get_memory_type(rdev);
+        rv740_read_clock_registers(rdev);
+        btc_read_arb_registers(rdev);
+        rv770_read_voltage_smio_registers(rdev);
+
+        if (eg_pi->pcie_performance_request)
+                cypress_advertise_gen2_capability(rdev);
+
+        rv770_get_pcie_gen2_status(rdev);
+        rv770_enable_acpi_pm(rdev);
+}
+
+int btc_dpm_init(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi;
+        struct evergreen_power_info *eg_pi;
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        u16 data_offset, size;
+        u8 frev, crev;
+        struct atom_clock_dividers dividers;
+        int ret;
+
+        eg_pi = kzalloc(sizeof(struct evergreen_power_info), GFP_KERNEL);
+        if (eg_pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = eg_pi;
+        pi = &eg_pi->rv7xx;
+
+        rv770_get_max_vddc(rdev);
+
+        eg_pi->ulv.supported = false;
+        pi->acpi_vddc = 0;
+        eg_pi->acpi_vddci = 0;
+        pi->min_vddc_in_table = 0;
+        pi->max_vddc_in_table = 0;
+
+        ret = rv7xx_parse_power_table(rdev);
+        if (ret)
+                return ret;
+        ret = r600_parse_extended_power_table(rdev);
+        if (ret)
+                return ret;
+
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
+                kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
+        if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
+                r600_free_extended_power_table(rdev);
+                return -ENOMEM;
+        }
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 800;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 800;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 800;
+
+        if (rdev->pm.dpm.voltage_response_time == 0)
+                rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+        if (rdev->pm.dpm.backbias_response_time == 0)
+                rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->ref_div = dividers.ref_div + 1;
+        else
+                pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        pi->mclk_strobe_mode_threshold = 40000;
+        pi->mclk_edc_enable_threshold = 40000;
+        eg_pi->mclk_edc_wr_enable_threshold = 40000;
+
+        pi->rlp = RV770_RLP_DFLT;
+        pi->rmp = RV770_RMP_DFLT;
+        pi->lhp = RV770_LHP_DFLT;
+        pi->lmp = RV770_LMP_DFLT;
+
+        eg_pi->ats[0].rlp = RV770_RLP_DFLT;
+        eg_pi->ats[0].rmp = RV770_RMP_DFLT;
+        eg_pi->ats[0].lhp = RV770_LHP_DFLT;
+        eg_pi->ats[0].lmp = RV770_LMP_DFLT;
+
+        eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
+        eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
+        eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
+        eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
+
+        eg_pi->smu_uvd_hs = true;
+
+        pi->voltage_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
+
+        pi->mvdd_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
+
+        eg_pi->vddci_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                pi->sclk_ss = true;
+                pi->mclk_ss = true;
+                pi->dynamic_ss = true;
+        } else {
+                pi->sclk_ss = false;
+                pi->mclk_ss = false;
+                pi->dynamic_ss = true;
+        }
+
+        pi->asi = RV770_ASI_DFLT;
+        pi->pasi = CYPRESS_HASI_DFLT;
+        pi->vrc = CYPRESS_VRC_DFLT;
+
+        pi->power_gating = false;
+
+        pi->gfx_clock_gating = true;
+
+        pi->mg_clock_gating = true;
+        pi->mgcgtssm = true;
+        eg_pi->ls_clock_gating = false;
+        eg_pi->sclk_deep_sleep = false;
+
+        pi->dynamic_pcie_gen2 = true;
+
+        if (pi->gfx_clock_gating &&
+            (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
+                pi->thermal_protection = true;
+        else
+                pi->thermal_protection = false;
+
+        pi->display_gap = true;
+
+        if (rdev->flags & RADEON_IS_MOBILITY)
+                pi->dcodt = true;
+        else
+                pi->dcodt = false;
+
+        pi->ulps = true;
+
+        eg_pi->dynamic_ac_timing = true;
+        eg_pi->abm = true;
+        eg_pi->mcls = true;
+        eg_pi->light_sleep = true;
+        eg_pi->memory_transition = true;
+#if defined(CONFIG_ACPI)
+        eg_pi->pcie_performance_request =
+                radeon_acpi_is_pcie_performance_request_supported(rdev);
+#else
+        eg_pi->pcie_performance_request = false;
+#endif
+
+        if (rdev->family == CHIP_BARTS)
+                eg_pi->dll_default_on = true;
+        else
+                eg_pi->dll_default_on = false;
+
+        eg_pi->sclk_deep_sleep = false;
+        if (ASIC_IS_LOMBOK(rdev))
+                pi->mclk_stutter_mode_threshold = 30000;
+        else
+                pi->mclk_stutter_mode_threshold = 0;
+
+        pi->sram_end = SMC_RAM_END;
+
+        rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4;
+        rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
+        rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
+        rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
+        rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
+        rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
+        rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
+
+        if (rdev->family == CHIP_TURKS)
+                rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000;
+        else
+                rdev->pm.dpm.dyn_state.sclk_mclk_delta = 10000;
+
+        return 0;
+}
+
+void btc_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+        kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
+        r600_free_extended_power_table(rdev);
+}
+
+u32 btc_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_ps *requested_state = rv770_get_ps(&eg_pi->requested_rps);
+
+        if (low)
+                return requested_state->low.sclk;
+        else
+                return requested_state->high.sclk;
+}
+
+u32 btc_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_ps *requested_state = rv770_get_ps(&eg_pi->requested_rps);
+
+        if (low)
+                return requested_state->low.mclk;
+        else
+                return requested_state->high.mclk;
+}
diff --git a/drivers/gpu/drm/radeon/btc_dpm.h b/drivers/gpu/drm/radeon/btc_dpm.h
new file mode 100644
index 000000000000..1a15e0e41950
--- /dev/null
+++ b/drivers/gpu/drm/radeon/btc_dpm.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __BTC_DPM_H__
+#define __BTC_DPM_H__
+
+#define BTC_RLP_UVD_DFLT                              20
+#define BTC_RMP_UVD_DFLT                              50
+#define BTC_LHP_UVD_DFLT                              50
+#define BTC_LMP_UVD_DFLT                              20
+#define BARTS_MGCGCGTSSMCTRL_DFLT                     0x81944000
+#define TURKS_MGCGCGTSSMCTRL_DFLT                     0x6e944000
+#define CAICOS_MGCGCGTSSMCTRL_DFLT                    0x46944040
+#define BTC_CGULVPARAMETER_DFLT                       0x00040035
+#define BTC_CGULVCONTROL_DFLT                         0x00001450
+
+extern u32 btc_valid_sclk[40];
+
+void btc_read_arb_registers(struct radeon_device *rdev);
+void btc_program_mgcg_hw_sequence(struct radeon_device *rdev,
+                                  const u32 *sequence, u32 count);
+void btc_skip_blacklist_clocks(struct radeon_device *rdev,
+                               const u32 max_sclk, const u32 max_mclk,
+                               u32 *sclk, u32 *mclk);
+void btc_adjust_clock_combinations(struct radeon_device *rdev,
+                                   const struct radeon_clock_and_voltage_limits *max_limits,
+                                   struct rv7xx_pl *pl);
+void btc_apply_voltage_dependency_rules(struct radeon_clock_voltage_dependency_table *table,
+                                        u32 clock, u16 max_voltage, u16 *voltage);
+void btc_apply_voltage_delta_rules(struct radeon_device *rdev,
+                                   u16 max_vddc, u16 max_vddci,
+                                   u16 *vddc, u16 *vddci);
+bool btc_dpm_enabled(struct radeon_device *rdev);
+int btc_reset_to_default(struct radeon_device *rdev);
+void btc_notify_uvd_to_smc(struct radeon_device *rdev,
+                           struct radeon_ps *radeon_new_state);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/btcd.h b/drivers/gpu/drm/radeon/btcd.h
new file mode 100644
index 000000000000..29e32de7e025
--- /dev/null
+++ b/drivers/gpu/drm/radeon/btcd.h
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef _BTCD_H_
+#define _BTCD_H_
+
+/* pm registers */
+
+#define GENERAL_PWRMGT                                  0x63c
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define ENABLE_GEN2PCIE                          (1 << 4)
+#       define ENABLE_GEN2XSP                           (1 << 5)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (3 << 6)
+#       define SW_SMIO_INDEX_SHIFT                      6
+#       define LOW_VOLT_D2_ACPI                         (1 << 8)
+#       define LOW_VOLT_D3_ACPI                         (1 << 9)
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define BACKBIAS_PAD_EN                          (1 << 18)
+#       define BACKBIAS_VALUE                           (1 << 19)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#       define AC_DC_SW                                 (1 << 24)
+
+#define CG_BIF_REQ_AND_RSP                              0x7f4
+#define         CG_CLIENT_REQ(x)                        ((x) << 0)
+#define         CG_CLIENT_REQ_MASK                      (0xff << 0)
+#define         CG_CLIENT_REQ_SHIFT                     0
+#define         CG_CLIENT_RESP(x)                       ((x) << 8)
+#define         CG_CLIENT_RESP_MASK                     (0xff << 8)
+#define         CG_CLIENT_RESP_SHIFT                    8
+#define         CLIENT_CG_REQ(x)                        ((x) << 16)
+#define         CLIENT_CG_REQ_MASK                      (0xff << 16)
+#define         CLIENT_CG_REQ_SHIFT                     16
+#define         CLIENT_CG_RESP(x)                       ((x) << 24)
+#define         CLIENT_CG_RESP_MASK                     (0xff << 24)
+#define         CLIENT_CG_RESP_SHIFT                    24
+
+#define SCLK_PSKIP_CNTL                                 0x8c0
+#define         PSKIP_ON_ALLOW_STOP_HI(x)               ((x) << 16)
+#define         PSKIP_ON_ALLOW_STOP_HI_MASK             (0xff << 16)
+#define         PSKIP_ON_ALLOW_STOP_HI_SHIFT            16
+
+#define CG_ULV_CONTROL                                  0x8c8
+#define CG_ULV_PARAMETER                                0x8cc
+
+#define MC_ARB_DRAM_TIMING                              0x2774
+#define MC_ARB_DRAM_TIMING2                             0x2778
+
+#define MC_ARB_RFSH_RATE                                0x27b0
+#define         POWERMODE0(x)                           ((x) << 0)
+#define         POWERMODE0_MASK                         (0xff << 0)
+#define         POWERMODE0_SHIFT                        0
+#define         POWERMODE1(x)                           ((x) << 8)
+#define         POWERMODE1_MASK                         (0xff << 8)
+#define         POWERMODE1_SHIFT                        8
+#define         POWERMODE2(x)                           ((x) << 16)
+#define         POWERMODE2_MASK                         (0xff << 16)
+#define         POWERMODE2_SHIFT                        16
+#define         POWERMODE3(x)                           ((x) << 24)
+#define         POWERMODE3_MASK                         (0xff << 24)
+#define         POWERMODE3_SHIFT                        24
+
+#define MC_ARB_BURST_TIME                               0x2808
+#define         STATE0(x)                               ((x) << 0)
+#define         STATE0_MASK                             (0x1f << 0)
+#define         STATE0_SHIFT                            0
+#define         STATE1(x)                               ((x) << 5)
+#define         STATE1_MASK                             (0x1f << 5)
+#define         STATE1_SHIFT                            5
+#define         STATE2(x)                               ((x) << 10)
+#define         STATE2_MASK                             (0x1f << 10)
+#define         STATE2_SHIFT                            10
+#define         STATE3(x)                               ((x) << 15)
+#define         STATE3_MASK                             (0x1f << 15)
+#define         STATE3_SHIFT                            15
+
+#define MC_SEQ_RAS_TIMING                               0x28a0
+#define MC_SEQ_CAS_TIMING                               0x28a4
+#define MC_SEQ_MISC_TIMING                              0x28a8
+#define MC_SEQ_MISC_TIMING2                             0x28ac
+
+#define MC_SEQ_RD_CTL_D0                                0x28b4
+#define MC_SEQ_RD_CTL_D1                                0x28b8
+#define MC_SEQ_WR_CTL_D0                                0x28bc
+#define MC_SEQ_WR_CTL_D1                                0x28c0
+
+#define MC_PMG_AUTO_CFG                                 0x28d4
+
+#define MC_SEQ_STATUS_M                                 0x29f4
+#       define PMG_PWRSTATE                             (1 << 16)
+
+#define MC_SEQ_MISC0                                    0x2a00
+#define         MC_SEQ_MISC0_GDDR5_SHIFT                28
+#define         MC_SEQ_MISC0_GDDR5_MASK                 0xf0000000
+#define         MC_SEQ_MISC0_GDDR5_VALUE                5
+#define MC_SEQ_MISC1                                    0x2a04
+#define MC_SEQ_RESERVE_M                                0x2a08
+#define MC_PMG_CMD_EMRS                                 0x2a0c
+
+#define MC_SEQ_MISC3                                    0x2a2c
+
+#define MC_SEQ_MISC5                                    0x2a54
+#define MC_SEQ_MISC6                                    0x2a58
+
+#define MC_SEQ_MISC7                                    0x2a64
+
+#define MC_SEQ_CG                                       0x2a68
+#define         CG_SEQ_REQ(x)                           ((x) << 0)
+#define         CG_SEQ_REQ_MASK                         (0xff << 0)
+#define         CG_SEQ_REQ_SHIFT                        0
+#define         CG_SEQ_RESP(x)                          ((x) << 8)
+#define         CG_SEQ_RESP_MASK                        (0xff << 8)
+#define         CG_SEQ_RESP_SHIFT                       8
+#define         SEQ_CG_REQ(x)                           ((x) << 16)
+#define         SEQ_CG_REQ_MASK                         (0xff << 16)
+#define         SEQ_CG_REQ_SHIFT                        16
+#define         SEQ_CG_RESP(x)                          ((x) << 24)
+#define         SEQ_CG_RESP_MASK                        (0xff << 24)
+#define         SEQ_CG_RESP_SHIFT                       24
+#define MC_SEQ_RAS_TIMING_LP                            0x2a6c
+#define MC_SEQ_CAS_TIMING_LP                            0x2a70
+#define MC_SEQ_MISC_TIMING_LP                           0x2a74
+#define MC_SEQ_MISC_TIMING2_LP                          0x2a78
+#define MC_SEQ_WR_CTL_D0_LP                             0x2a7c
+#define MC_SEQ_WR_CTL_D1_LP                             0x2a80
+#define MC_SEQ_PMG_CMD_EMRS_LP                          0x2a84
+#define MC_SEQ_PMG_CMD_MRS_LP                           0x2a88
+
+#define MC_PMG_CMD_MRS                                  0x2aac
+
+#define MC_SEQ_RD_CTL_D0_LP                             0x2b1c
+#define MC_SEQ_RD_CTL_D1_LP                             0x2b20
+
+#define MC_PMG_CMD_MRS1                                 0x2b44
+#define MC_SEQ_PMG_CMD_MRS1_LP                          0x2b48
+
+#define LB_SYNC_RESET_SEL                               0x6b28
+#define         LB_SYNC_RESET_SEL_MASK                  (3 << 0)
+#define         LB_SYNC_RESET_SEL_SHIFT                 0
+
+/* PCIE link stuff */
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_HW_VOLTAGE_IF_CONTROL(x)                ((x) << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_MASK              (3 << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_SHIFT             12
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
new file mode 100644
index 000000000000..ed1d91025928
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -0,0 +1,6987 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#include <linux/firmware.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include "drmP.h"
+#include "radeon.h"
+#include "radeon_asic.h"
+#include "cikd.h"
+#include "atom.h"
+#include "cik_blit_shaders.h"
+
+/* GFX */
+#define CIK_PFP_UCODE_SIZE 2144
+#define CIK_ME_UCODE_SIZE 2144
+#define CIK_CE_UCODE_SIZE 2144
+/* compute */
+#define CIK_MEC_UCODE_SIZE 4192
+/* interrupts */
+#define BONAIRE_RLC_UCODE_SIZE 2048
+#define KB_RLC_UCODE_SIZE 2560
+#define KV_RLC_UCODE_SIZE 2560
+/* gddr controller */
+#define CIK_MC_UCODE_SIZE 7866
+/* sdma */
+#define CIK_SDMA_UCODE_SIZE 1050
+#define CIK_SDMA_UCODE_VERSION 64
+
+MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
+MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
+MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
+MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
+MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
+MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
+MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
+MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
+MODULE_FIRMWARE("radeon/KAVERI_me.bin");
+MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
+MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
+MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
+MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
+MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
+MODULE_FIRMWARE("radeon/KABINI_me.bin");
+MODULE_FIRMWARE("radeon/KABINI_ce.bin");
+MODULE_FIRMWARE("radeon/KABINI_mec.bin");
+MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
+MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
+
+extern int r600_ih_ring_alloc(struct radeon_device *rdev);
+extern void r600_ih_ring_fini(struct radeon_device *rdev);
+extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
+extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
+extern bool evergreen_is_display_hung(struct radeon_device *rdev);
+extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
+extern void si_rlc_fini(struct radeon_device *rdev);
+extern int si_rlc_init(struct radeon_device *rdev);
+static void cik_rlc_stop(struct radeon_device *rdev);
+
+/*
+ * Indirect registers accessor
+ */
+u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(PCIE_INDEX, reg);
+        (void)RREG32(PCIE_INDEX);
+        r = RREG32(PCIE_DATA);
+        return r;
+}
+
+void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(PCIE_INDEX, reg);
+        (void)RREG32(PCIE_INDEX);
+        WREG32(PCIE_DATA, v);
+        (void)RREG32(PCIE_DATA);
+}
+
+static const u32 bonaire_golden_spm_registers[] =
+{
+        0x30800, 0xe0ffffff, 0xe0000000
+};
+
+static const u32 bonaire_golden_common_registers[] =
+{
+        0xc770, 0xffffffff, 0x00000800,
+        0xc774, 0xffffffff, 0x00000800,
+        0xc798, 0xffffffff, 0x00007fbf,
+        0xc79c, 0xffffffff, 0x00007faf
+};
+
+static const u32 bonaire_golden_registers[] =
+{
+        0x3354, 0x00000333, 0x00000333,
+        0x3350, 0x000c0fc0, 0x00040200,
+        0x9a10, 0x00010000, 0x00058208,
+        0x3c000, 0xffff1fff, 0x00140000,
+        0x3c200, 0xfdfc0fff, 0x00000100,
+        0x3c234, 0x40000000, 0x40000200,
+        0x9830, 0xffffffff, 0x00000000,
+        0x9834, 0xf00fffff, 0x00000400,
+        0x9838, 0x0002021c, 0x00020200,
+        0xc78, 0x00000080, 0x00000000,
+        0x5bb0, 0x000000f0, 0x00000070,
+        0x5bc0, 0xf0311fff, 0x80300000,
+        0x98f8, 0x73773777, 0x12010001,
+        0x350c, 0x00810000, 0x408af000,
+        0x7030, 0x31000111, 0x00000011,
+        0x2f48, 0x73773777, 0x12010001,
+        0x220c, 0x00007fb6, 0x0021a1b1,
+        0x2210, 0x00007fb6, 0x002021b1,
+        0x2180, 0x00007fb6, 0x00002191,
+        0x2218, 0x00007fb6, 0x002121b1,
+        0x221c, 0x00007fb6, 0x002021b1,
+        0x21dc, 0x00007fb6, 0x00002191,
+        0x21e0, 0x00007fb6, 0x00002191,
+        0x3628, 0x0000003f, 0x0000000a,
+        0x362c, 0x0000003f, 0x0000000a,
+        0x2ae4, 0x00073ffe, 0x000022a2,
+        0x240c, 0x000007ff, 0x00000000,
+        0x8a14, 0xf000003f, 0x00000007,
+        0x8bf0, 0x00002001, 0x00000001,
+        0x8b24, 0xffffffff, 0x00ffffff,
+        0x30a04, 0x0000ff0f, 0x00000000,
+        0x28a4c, 0x07ffffff, 0x06000000,
+        0x4d8, 0x00000fff, 0x00000100,
+        0x3e78, 0x00000001, 0x00000002,
+        0x9100, 0x03000000, 0x0362c688,
+        0x8c00, 0x000000ff, 0x00000001,
+        0xe40, 0x00001fff, 0x00001fff,
+        0x9060, 0x0000007f, 0x00000020,
+        0x9508, 0x00010000, 0x00010000,
+        0xac14, 0x000003ff, 0x000000f3,
+        0xac0c, 0xffffffff, 0x00001032
+};
+
+static const u32 bonaire_mgcg_cgcg_init[] =
+{
+        0xc420, 0xffffffff, 0xfffffffc,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c2a0, 0xffffffff, 0x00000100,
+        0x3c208, 0xffffffff, 0x00000100,
+        0x3c2c0, 0xffffffff, 0xc0000100,
+        0x3c2c8, 0xffffffff, 0xc0000100,
+        0x3c2c4, 0xffffffff, 0xc0000100,
+        0x55e4, 0xffffffff, 0x00600100,
+        0x3c280, 0xffffffff, 0x00000100,
+        0x3c214, 0xffffffff, 0x06000100,
+        0x3c220, 0xffffffff, 0x00000100,
+        0x3c218, 0xffffffff, 0x06000100,
+        0x3c204, 0xffffffff, 0x00000100,
+        0x3c2e0, 0xffffffff, 0x00000100,
+        0x3c224, 0xffffffff, 0x00000100,
+        0x3c200, 0xffffffff, 0x00000100,
+        0x3c230, 0xffffffff, 0x00000100,
+        0x3c234, 0xffffffff, 0x00000100,
+        0x3c250, 0xffffffff, 0x00000100,
+        0x3c254, 0xffffffff, 0x00000100,
+        0x3c258, 0xffffffff, 0x00000100,
+        0x3c25c, 0xffffffff, 0x00000100,
+        0x3c260, 0xffffffff, 0x00000100,
+        0x3c27c, 0xffffffff, 0x00000100,
+        0x3c278, 0xffffffff, 0x00000100,
+        0x3c210, 0xffffffff, 0x06000100,
+        0x3c290, 0xffffffff, 0x00000100,
+        0x3c274, 0xffffffff, 0x00000100,
+        0x3c2b4, 0xffffffff, 0x00000100,
+        0x3c2b0, 0xffffffff, 0x00000100,
+        0x3c270, 0xffffffff, 0x00000100,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c020, 0xffffffff, 0x00010000,
+        0x3c024, 0xffffffff, 0x00030002,
+        0x3c028, 0xffffffff, 0x00040007,
+        0x3c02c, 0xffffffff, 0x00060005,
+        0x3c030, 0xffffffff, 0x00090008,
+        0x3c034, 0xffffffff, 0x00010000,
+        0x3c038, 0xffffffff, 0x00030002,
+        0x3c03c, 0xffffffff, 0x00040007,
+        0x3c040, 0xffffffff, 0x00060005,
+        0x3c044, 0xffffffff, 0x00090008,
+        0x3c048, 0xffffffff, 0x00010000,
+        0x3c04c, 0xffffffff, 0x00030002,
+        0x3c050, 0xffffffff, 0x00040007,
+        0x3c054, 0xffffffff, 0x00060005,
+        0x3c058, 0xffffffff, 0x00090008,
+        0x3c05c, 0xffffffff, 0x00010000,
+        0x3c060, 0xffffffff, 0x00030002,
+        0x3c064, 0xffffffff, 0x00040007,
+        0x3c068, 0xffffffff, 0x00060005,
+        0x3c06c, 0xffffffff, 0x00090008,
+        0x3c070, 0xffffffff, 0x00010000,
+        0x3c074, 0xffffffff, 0x00030002,
+        0x3c078, 0xffffffff, 0x00040007,
+        0x3c07c, 0xffffffff, 0x00060005,
+        0x3c080, 0xffffffff, 0x00090008,
+        0x3c084, 0xffffffff, 0x00010000,
+        0x3c088, 0xffffffff, 0x00030002,
+        0x3c08c, 0xffffffff, 0x00040007,
+        0x3c090, 0xffffffff, 0x00060005,
+        0x3c094, 0xffffffff, 0x00090008,
+        0x3c098, 0xffffffff, 0x00010000,
+        0x3c09c, 0xffffffff, 0x00030002,
+        0x3c0a0, 0xffffffff, 0x00040007,
+        0x3c0a4, 0xffffffff, 0x00060005,
+        0x3c0a8, 0xffffffff, 0x00090008,
+        0x3c000, 0xffffffff, 0x96e00200,
+        0x8708, 0xffffffff, 0x00900100,
+        0xc424, 0xffffffff, 0x0020003f,
+        0x38, 0xffffffff, 0x0140001c,
+        0x3c, 0x000f0000, 0x000f0000,
+        0x220, 0xffffffff, 0xC060000C,
+        0x224, 0xc0000fff, 0x00000100,
+        0xf90, 0xffffffff, 0x00000100,
+        0xf98, 0x00000101, 0x00000000,
+        0x20a8, 0xffffffff, 0x00000104,
+        0x55e4, 0xff000fff, 0x00000100,
+        0x30cc, 0xc0000fff, 0x00000104,
+        0xc1e4, 0x00000001, 0x00000001,
+        0xd00c, 0xff000ff0, 0x00000100,
+        0xd80c, 0xff000ff0, 0x00000100
+};
+
+static const u32 spectre_golden_spm_registers[] =
+{
+        0x30800, 0xe0ffffff, 0xe0000000
+};
+
+static const u32 spectre_golden_common_registers[] =
+{
+        0xc770, 0xffffffff, 0x00000800,
+        0xc774, 0xffffffff, 0x00000800,
+        0xc798, 0xffffffff, 0x00007fbf,
+        0xc79c, 0xffffffff, 0x00007faf
+};
+
+static const u32 spectre_golden_registers[] =
+{
+        0x3c000, 0xffff1fff, 0x96940200,
+        0x3c00c, 0xffff0001, 0xff000000,
+        0x3c200, 0xfffc0fff, 0x00000100,
+        0x6ed8, 0x00010101, 0x00010000,
+        0x9834, 0xf00fffff, 0x00000400,
+        0x9838, 0xfffffffc, 0x00020200,
+        0x5bb0, 0x000000f0, 0x00000070,
+        0x5bc0, 0xf0311fff, 0x80300000,
+        0x98f8, 0x73773777, 0x12010001,
+        0x9b7c, 0x00ff0000, 0x00fc0000,
+        0x2f48, 0x73773777, 0x12010001,
+        0x8a14, 0xf000003f, 0x00000007,
+        0x8b24, 0xffffffff, 0x00ffffff,
+        0x28350, 0x3f3f3fff, 0x00000082,
+        0x28355, 0x0000003f, 0x00000000,
+        0x3e78, 0x00000001, 0x00000002,
+        0x913c, 0xffff03df, 0x00000004,
+        0xc768, 0x00000008, 0x00000008,
+        0x8c00, 0x000008ff, 0x00000800,
+        0x9508, 0x00010000, 0x00010000,
+        0xac0c, 0xffffffff, 0x54763210,
+        0x214f8, 0x01ff01ff, 0x00000002,
+        0x21498, 0x007ff800, 0x00200000,
+        0x2015c, 0xffffffff, 0x00000f40,
+        0x30934, 0xffffffff, 0x00000001
+};
+
+static const u32 spectre_mgcg_cgcg_init[] =
+{
+        0xc420, 0xffffffff, 0xfffffffc,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c2a0, 0xffffffff, 0x00000100,
+        0x3c208, 0xffffffff, 0x00000100,
+        0x3c2c0, 0xffffffff, 0x00000100,
+        0x3c2c8, 0xffffffff, 0x00000100,
+        0x3c2c4, 0xffffffff, 0x00000100,
+        0x55e4, 0xffffffff, 0x00600100,
+        0x3c280, 0xffffffff, 0x00000100,
+        0x3c214, 0xffffffff, 0x06000100,
+        0x3c220, 0xffffffff, 0x00000100,
+        0x3c218, 0xffffffff, 0x06000100,
+        0x3c204, 0xffffffff, 0x00000100,
+        0x3c2e0, 0xffffffff, 0x00000100,
+        0x3c224, 0xffffffff, 0x00000100,
+        0x3c200, 0xffffffff, 0x00000100,
+        0x3c230, 0xffffffff, 0x00000100,
+        0x3c234, 0xffffffff, 0x00000100,
+        0x3c250, 0xffffffff, 0x00000100,
+        0x3c254, 0xffffffff, 0x00000100,
+        0x3c258, 0xffffffff, 0x00000100,
+        0x3c25c, 0xffffffff, 0x00000100,
+        0x3c260, 0xffffffff, 0x00000100,
+        0x3c27c, 0xffffffff, 0x00000100,
+        0x3c278, 0xffffffff, 0x00000100,
+        0x3c210, 0xffffffff, 0x06000100,
+        0x3c290, 0xffffffff, 0x00000100,
+        0x3c274, 0xffffffff, 0x00000100,
+        0x3c2b4, 0xffffffff, 0x00000100,
+        0x3c2b0, 0xffffffff, 0x00000100,
+        0x3c270, 0xffffffff, 0x00000100,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c020, 0xffffffff, 0x00010000,
+        0x3c024, 0xffffffff, 0x00030002,
+        0x3c028, 0xffffffff, 0x00040007,
+        0x3c02c, 0xffffffff, 0x00060005,
+        0x3c030, 0xffffffff, 0x00090008,
+        0x3c034, 0xffffffff, 0x00010000,
+        0x3c038, 0xffffffff, 0x00030002,
+        0x3c03c, 0xffffffff, 0x00040007,
+        0x3c040, 0xffffffff, 0x00060005,
+        0x3c044, 0xffffffff, 0x00090008,
+        0x3c048, 0xffffffff, 0x00010000,
+        0x3c04c, 0xffffffff, 0x00030002,
+        0x3c050, 0xffffffff, 0x00040007,
+        0x3c054, 0xffffffff, 0x00060005,
+        0x3c058, 0xffffffff, 0x00090008,
+        0x3c05c, 0xffffffff, 0x00010000,
+        0x3c060, 0xffffffff, 0x00030002,
+        0x3c064, 0xffffffff, 0x00040007,
+        0x3c068, 0xffffffff, 0x00060005,
+        0x3c06c, 0xffffffff, 0x00090008,
+        0x3c070, 0xffffffff, 0x00010000,
+        0x3c074, 0xffffffff, 0x00030002,
+        0x3c078, 0xffffffff, 0x00040007,
+        0x3c07c, 0xffffffff, 0x00060005,
+        0x3c080, 0xffffffff, 0x00090008,
+        0x3c084, 0xffffffff, 0x00010000,
+        0x3c088, 0xffffffff, 0x00030002,
+        0x3c08c, 0xffffffff, 0x00040007,
+        0x3c090, 0xffffffff, 0x00060005,
+        0x3c094, 0xffffffff, 0x00090008,
+        0x3c098, 0xffffffff, 0x00010000,
+        0x3c09c, 0xffffffff, 0x00030002,
+        0x3c0a0, 0xffffffff, 0x00040007,
+        0x3c0a4, 0xffffffff, 0x00060005,
+        0x3c0a8, 0xffffffff, 0x00090008,
+        0x3c0ac, 0xffffffff, 0x00010000,
+        0x3c0b0, 0xffffffff, 0x00030002,
+        0x3c0b4, 0xffffffff, 0x00040007,
+        0x3c0b8, 0xffffffff, 0x00060005,
+        0x3c0bc, 0xffffffff, 0x00090008,
+        0x3c000, 0xffffffff, 0x96e00200,
+        0x8708, 0xffffffff, 0x00900100,
+        0xc424, 0xffffffff, 0x0020003f,
+        0x38, 0xffffffff, 0x0140001c,
+        0x3c, 0x000f0000, 0x000f0000,
+        0x220, 0xffffffff, 0xC060000C,
+        0x224, 0xc0000fff, 0x00000100,
+        0xf90, 0xffffffff, 0x00000100,
+        0xf98, 0x00000101, 0x00000000,
+        0x20a8, 0xffffffff, 0x00000104,
+        0x55e4, 0xff000fff, 0x00000100,
+        0x30cc, 0xc0000fff, 0x00000104,
+        0xc1e4, 0x00000001, 0x00000001,
+        0xd00c, 0xff000ff0, 0x00000100,
+        0xd80c, 0xff000ff0, 0x00000100
+};
+
+static const u32 kalindi_golden_spm_registers[] =
+{
+        0x30800, 0xe0ffffff, 0xe0000000
+};
+
+static const u32 kalindi_golden_common_registers[] =
+{
+        0xc770, 0xffffffff, 0x00000800,
+        0xc774, 0xffffffff, 0x00000800,
+        0xc798, 0xffffffff, 0x00007fbf,
+        0xc79c, 0xffffffff, 0x00007faf
+};
+
+static const u32 kalindi_golden_registers[] =
+{
+        0x3c000, 0xffffdfff, 0x6e944040,
+        0x55e4, 0xff607fff, 0xfc000100,
+        0x3c220, 0xff000fff, 0x00000100,
+        0x3c224, 0xff000fff, 0x00000100,
+        0x3c200, 0xfffc0fff, 0x00000100,
+        0x6ed8, 0x00010101, 0x00010000,
+        0x9830, 0xffffffff, 0x00000000,
+        0x9834, 0xf00fffff, 0x00000400,
+        0x5bb0, 0x000000f0, 0x00000070,
+        0x5bc0, 0xf0311fff, 0x80300000,
+        0x98f8, 0x73773777, 0x12010001,
+        0x98fc, 0xffffffff, 0x00000010,
+        0x9b7c, 0x00ff0000, 0x00fc0000,
+        0x8030, 0x00001f0f, 0x0000100a,
+        0x2f48, 0x73773777, 0x12010001,
+        0x2408, 0x000fffff, 0x000c007f,
+        0x8a14, 0xf000003f, 0x00000007,
+        0x8b24, 0x3fff3fff, 0x00ffcfff,
+        0x30a04, 0x0000ff0f, 0x00000000,
+        0x28a4c, 0x07ffffff, 0x06000000,
+        0x4d8, 0x00000fff, 0x00000100,
+        0x3e78, 0x00000001, 0x00000002,
+        0xc768, 0x00000008, 0x00000008,
+        0x8c00, 0x000000ff, 0x00000003,
+        0x214f8, 0x01ff01ff, 0x00000002,
+        0x21498, 0x007ff800, 0x00200000,
+        0x2015c, 0xffffffff, 0x00000f40,
+        0x88c4, 0x001f3ae3, 0x00000082,
+        0x88d4, 0x0000001f, 0x00000010,
+        0x30934, 0xffffffff, 0x00000000
+};
+
+static const u32 kalindi_mgcg_cgcg_init[] =
+{
+        0xc420, 0xffffffff, 0xfffffffc,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c2a0, 0xffffffff, 0x00000100,
+        0x3c208, 0xffffffff, 0x00000100,
+        0x3c2c0, 0xffffffff, 0x00000100,
+        0x3c2c8, 0xffffffff, 0x00000100,
+        0x3c2c4, 0xffffffff, 0x00000100,
+        0x55e4, 0xffffffff, 0x00600100,
+        0x3c280, 0xffffffff, 0x00000100,
+        0x3c214, 0xffffffff, 0x06000100,
+        0x3c220, 0xffffffff, 0x00000100,
+        0x3c218, 0xffffffff, 0x06000100,
+        0x3c204, 0xffffffff, 0x00000100,
+        0x3c2e0, 0xffffffff, 0x00000100,
+        0x3c224, 0xffffffff, 0x00000100,
+        0x3c200, 0xffffffff, 0x00000100,
+        0x3c230, 0xffffffff, 0x00000100,
+        0x3c234, 0xffffffff, 0x00000100,
+        0x3c250, 0xffffffff, 0x00000100,
+        0x3c254, 0xffffffff, 0x00000100,
+        0x3c258, 0xffffffff, 0x00000100,
+        0x3c25c, 0xffffffff, 0x00000100,
+        0x3c260, 0xffffffff, 0x00000100,
+        0x3c27c, 0xffffffff, 0x00000100,
+        0x3c278, 0xffffffff, 0x00000100,
+        0x3c210, 0xffffffff, 0x06000100,
+        0x3c290, 0xffffffff, 0x00000100,
+        0x3c274, 0xffffffff, 0x00000100,
+        0x3c2b4, 0xffffffff, 0x00000100,
+        0x3c2b0, 0xffffffff, 0x00000100,
+        0x3c270, 0xffffffff, 0x00000100,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c020, 0xffffffff, 0x00010000,
+        0x3c024, 0xffffffff, 0x00030002,
+        0x3c028, 0xffffffff, 0x00040007,
+        0x3c02c, 0xffffffff, 0x00060005,
+        0x3c030, 0xffffffff, 0x00090008,
+        0x3c034, 0xffffffff, 0x00010000,
+        0x3c038, 0xffffffff, 0x00030002,
+        0x3c03c, 0xffffffff, 0x00040007,
+        0x3c040, 0xffffffff, 0x00060005,
+        0x3c044, 0xffffffff, 0x00090008,
+        0x3c000, 0xffffffff, 0x96e00200,
+        0x8708, 0xffffffff, 0x00900100,
+        0xc424, 0xffffffff, 0x0020003f,
+        0x38, 0xffffffff, 0x0140001c,
+        0x3c, 0x000f0000, 0x000f0000,
+        0x220, 0xffffffff, 0xC060000C,
+        0x224, 0xc0000fff, 0x00000100,
+        0x20a8, 0xffffffff, 0x00000104,
+        0x55e4, 0xff000fff, 0x00000100,
+        0x30cc, 0xc0000fff, 0x00000104,
+        0xc1e4, 0x00000001, 0x00000001,
+        0xd00c, 0xff000ff0, 0x00000100,
+        0xd80c, 0xff000ff0, 0x00000100
+};
+
+static void cik_init_golden_registers(struct radeon_device *rdev)
+{
+        switch (rdev->family) {
+        case CHIP_BONAIRE:
+                radeon_program_register_sequence(rdev,
+                                                 bonaire_mgcg_cgcg_init,
+                                                 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
+                radeon_program_register_sequence(rdev,
+                                                 bonaire_golden_registers,
+                                                 (const u32)ARRAY_SIZE(bonaire_golden_registers));
+                radeon_program_register_sequence(rdev,
+                                                 bonaire_golden_common_registers,
+                                                 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
+                radeon_program_register_sequence(rdev,
+                                                 bonaire_golden_spm_registers,
+                                                 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
+                break;
+        case CHIP_KABINI:
+                radeon_program_register_sequence(rdev,
+                                                 kalindi_mgcg_cgcg_init,
+                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
+                radeon_program_register_sequence(rdev,
+                                                 kalindi_golden_registers,
+                                                 (const u32)ARRAY_SIZE(kalindi_golden_registers));
+                radeon_program_register_sequence(rdev,
+                                                 kalindi_golden_common_registers,
+                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
+                radeon_program_register_sequence(rdev,
+                                                 kalindi_golden_spm_registers,
+                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
+                break;
+        case CHIP_KAVERI:
+                radeon_program_register_sequence(rdev,
+                                                 spectre_mgcg_cgcg_init,
+                                                 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
+                radeon_program_register_sequence(rdev,
+                                                 spectre_golden_registers,
+                                                 (const u32)ARRAY_SIZE(spectre_golden_registers));
+                radeon_program_register_sequence(rdev,
+                                                 spectre_golden_common_registers,
+                                                 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
+                radeon_program_register_sequence(rdev,
+                                                 spectre_golden_spm_registers,
+                                                 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
+                break;
+        default:
+                break;
+        }
+}
+
+/**
+ * cik_get_xclk - get the xclk
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Returns the reference clock used by the gfx engine
+ * (CIK).
+ */
+u32 cik_get_xclk(struct radeon_device *rdev)
+{
+        u32 reference_clock = rdev->clock.spll.reference_freq;
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
+                        return reference_clock / 2;
+        } else {
+                if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
+                        return reference_clock / 4;
+        }
+        return reference_clock;
+}
+
+/**
+ * cik_mm_rdoorbell - read a doorbell dword
+ *
+ * @rdev: radeon_device pointer
+ * @offset: byte offset into the aperture
+ *
+ * Returns the value in the doorbell aperture at the
+ * requested offset (CIK).
+ */
+u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
+{
+        if (offset < rdev->doorbell.size) {
+                return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
+        } else {
+                DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
+                return 0;
+        }
+}
+
+/**
+ * cik_mm_wdoorbell - write a doorbell dword
+ *
+ * @rdev: radeon_device pointer
+ * @offset: byte offset into the aperture
+ * @v: value to write
+ *
+ * Writes @v to the doorbell aperture at the
+ * requested offset (CIK).
+ */
+void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
+{
+        if (offset < rdev->doorbell.size) {
+                writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
+        } else {
+                DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
+        }
+}
+
+#define BONAIRE_IO_MC_REGS_SIZE 36
+
+static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
+{
+        {0x00000070, 0x04400000},
+        {0x00000071, 0x80c01803},
+        {0x00000072, 0x00004004},
+        {0x00000073, 0x00000100},
+        {0x00000074, 0x00ff0000},
+        {0x00000075, 0x34000000},
+        {0x00000076, 0x08000014},
+        {0x00000077, 0x00cc08ec},
+        {0x00000078, 0x00000400},
+        {0x00000079, 0x00000000},
+        {0x0000007a, 0x04090000},
+        {0x0000007c, 0x00000000},
+        {0x0000007e, 0x4408a8e8},
+        {0x0000007f, 0x00000304},
+        {0x00000080, 0x00000000},
+        {0x00000082, 0x00000001},
+        {0x00000083, 0x00000002},
+        {0x00000084, 0xf3e4f400},
+        {0x00000085, 0x052024e3},
+        {0x00000087, 0x00000000},
+        {0x00000088, 0x01000000},
+        {0x0000008a, 0x1c0a0000},
+        {0x0000008b, 0xff010000},
+        {0x0000008d, 0xffffefff},
+        {0x0000008e, 0xfff3efff},
+        {0x0000008f, 0xfff3efbf},
+        {0x00000092, 0xf7ffffff},
+        {0x00000093, 0xffffff7f},
+        {0x00000095, 0x00101101},
+        {0x00000096, 0x00000fff},
+        {0x00000097, 0x00116fff},
+        {0x00000098, 0x60010000},
+        {0x00000099, 0x10010000},
+        {0x0000009a, 0x00006000},
+        {0x0000009b, 0x00001000},
+        {0x0000009f, 0x00b48000}
+};
+
+/**
+ * cik_srbm_select - select specific register instances
+ *
+ * @rdev: radeon_device pointer
+ * @me: selected ME (micro engine)
+ * @pipe: pipe
+ * @queue: queue
+ * @vmid: VMID
+ *
+ * Switches the currently active registers instances.  Some
+ * registers are instanced per VMID, others are instanced per
+ * me/pipe/queue combination.
+ */
+static void cik_srbm_select(struct radeon_device *rdev,
+                            u32 me, u32 pipe, u32 queue, u32 vmid)
+{
+        u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
+                             MEID(me & 0x3) |
+                             VMID(vmid & 0xf) |
+                             QUEUEID(queue & 0x7));
+        WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
+}
+
+/* ucode loading */
+/**
+ * ci_mc_load_microcode - load MC ucode into the hw
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Load the GDDR MC ucode into the hw (CIK).
+ * Returns 0 on success, error on failure.
+ */
+static int ci_mc_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        u32 running, blackout = 0;
+        u32 *io_mc_regs;
+        int i, ucode_size, regs_size;
+
+        if (!rdev->mc_fw)
+                return -EINVAL;
+
+        switch (rdev->family) {
+        case CHIP_BONAIRE:
+        default:
+                io_mc_regs = (u32 *)&bonaire_io_mc_regs;
+                ucode_size = CIK_MC_UCODE_SIZE;
+                regs_size = BONAIRE_IO_MC_REGS_SIZE;
+                break;
+        }
+
+        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
+
+        if (running == 0) {
+                if (running) {
+                        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
+                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
+                }
+
+                /* reset the engine and set to writable */
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
+
+                /* load mc io regs */
+                for (i = 0; i < regs_size; i++) {
+                        WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
+                        WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
+                }
+                /* load the MC ucode */
+                fw_data = (const __be32 *)rdev->mc_fw->data;
+                for (i = 0; i < ucode_size; i++)
+                        WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
+
+                /* put the engine back into the active state */
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
+                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
+
+                /* wait for training to complete */
+                for (i = 0; i < rdev->usec_timeout; i++) {
+                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
+                                break;
+                        udelay(1);
+                }
+                for (i = 0; i < rdev->usec_timeout; i++) {
+                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
+                                break;
+                        udelay(1);
+                }
+
+                if (running)
+                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
+        }
+
+        return 0;
+}
+
+/**
+ * cik_init_microcode - load ucode images from disk
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Use the firmware interface to load the ucode images into
+ * the driver (not loaded into hw).
+ * Returns 0 on success, error on failure.
+ */
+static int cik_init_microcode(struct radeon_device *rdev)
+{
+        struct platform_device *pdev;
+        const char *chip_name;
+        size_t pfp_req_size, me_req_size, ce_req_size,
+                mec_req_size, rlc_req_size, mc_req_size,
+                sdma_req_size;
+        char fw_name[30];
+        int err;
+
+        DRM_DEBUG("\n");
+
+        pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
+        err = IS_ERR(pdev);
+        if (err) {
+                printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
+                return -EINVAL;
+        }
+
+        switch (rdev->family) {
+        case CHIP_BONAIRE:
+                chip_name = "BONAIRE";
+                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
+                me_req_size = CIK_ME_UCODE_SIZE * 4;
+                ce_req_size = CIK_CE_UCODE_SIZE * 4;
+                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
+                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
+                mc_req_size = CIK_MC_UCODE_SIZE * 4;
+                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
+                break;
+        case CHIP_KAVERI:
+                chip_name = "KAVERI";
+                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
+                me_req_size = CIK_ME_UCODE_SIZE * 4;
+                ce_req_size = CIK_CE_UCODE_SIZE * 4;
+                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
+                rlc_req_size = KV_RLC_UCODE_SIZE * 4;
+                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
+                break;
+        case CHIP_KABINI:
+                chip_name = "KABINI";
+                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
+                me_req_size = CIK_ME_UCODE_SIZE * 4;
+                ce_req_size = CIK_CE_UCODE_SIZE * 4;
+                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
+                rlc_req_size = KB_RLC_UCODE_SIZE * 4;
+                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
+                break;
+        default: BUG();
+        }
+
+        DRM_INFO("Loading %s Microcode\n", chip_name);
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
+        err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->pfp_fw->size != pfp_req_size) {
+                printk(KERN_ERR
+                       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->pfp_fw->size, fw_name);
+                err = -EINVAL;
+                goto out;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
+        err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->me_fw->size != me_req_size) {
+                printk(KERN_ERR
+                       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->me_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
+        err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->ce_fw->size != ce_req_size) {
+                printk(KERN_ERR
+                       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->ce_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
+        err = request_firmware(&rdev->mec_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->mec_fw->size != mec_req_size) {
+                printk(KERN_ERR
+                       "cik_cp: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->mec_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
+        err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->rlc_fw->size != rlc_req_size) {
+                printk(KERN_ERR
+                       "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->rlc_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
+        err = request_firmware(&rdev->sdma_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->sdma_fw->size != sdma_req_size) {
+                printk(KERN_ERR
+                       "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->sdma_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
+        /* No MC ucode on APUs */
+        if (!(rdev->flags & RADEON_IS_IGP)) {
+                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
+                err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
+                if (err)
+                        goto out;
+                if (rdev->mc_fw->size != mc_req_size) {
+                        printk(KERN_ERR
+                               "cik_mc: Bogus length %zu in firmware \"%s\"\n",
+                               rdev->mc_fw->size, fw_name);
+                        err = -EINVAL;
+                }
+        }
+
+out:
+        platform_device_unregister(pdev);
+
+        if (err) {
+                if (err != -EINVAL)
+                        printk(KERN_ERR
+                               "cik_cp: Failed to load firmware \"%s\"\n",
+                               fw_name);
+                release_firmware(rdev->pfp_fw);
+                rdev->pfp_fw = NULL;
+                release_firmware(rdev->me_fw);
+                rdev->me_fw = NULL;
+                release_firmware(rdev->ce_fw);
+                rdev->ce_fw = NULL;
+                release_firmware(rdev->rlc_fw);
+                rdev->rlc_fw = NULL;
+                release_firmware(rdev->mc_fw);
+                rdev->mc_fw = NULL;
+        }
+        return err;
+}
+
+/*
+ * Core functions
+ */
+/**
+ * cik_tiling_mode_table_init - init the hw tiling table
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Starting with SI, the tiling setup is done globally in a
+ * set of 32 tiling modes.  Rather than selecting each set of
+ * parameters per surface as on older asics, we just select
+ * which index in the tiling table we want to use, and the
+ * surface uses those parameters (CIK).
+ */
+static void cik_tiling_mode_table_init(struct radeon_device *rdev)
+{
+        const u32 num_tile_mode_states = 32;
+        const u32 num_secondary_tile_mode_states = 16;
+        u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
+        u32 num_pipe_configs;
+        u32 num_rbs = rdev->config.cik.max_backends_per_se *
+                rdev->config.cik.max_shader_engines;
+
+        switch (rdev->config.cik.mem_row_size_in_kb) {
+        case 1:
+                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
+                break;
+        case 2:
+        default:
+                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
+                break;
+        case 4:
+                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
+                break;
+        }
+
+        num_pipe_configs = rdev->config.cik.max_tile_pipes;
+        if (num_pipe_configs > 8)
+                num_pipe_configs = 8; /* ??? */
+
+        if (num_pipe_configs == 8) {
+                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
+                                break;
+                        case 1:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
+                                break;
+                        case 2:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                break;
+                        case 3:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
+                                break;
+                        case 4:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(split_equal_to_row_size));
+                                break;
+                        case 5:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+                                break;
+                        case 6:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                break;
+                        case 7:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 TILE_SPLIT(split_equal_to_row_size));
+                                break;
+                        case 8:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
+                                break;
+                        case 9:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
+                                break;
+                        case 10:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 11:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 12:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 13:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
+                                break;
+                        case 14:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 16:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 17:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 27:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
+                                break;
+                        case 28:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 29:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 30:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
+                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 1:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 2:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 3:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 4:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 5:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_4_BANK));
+                                break;
+                        case 6:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_2_BANK));
+                                break;
+                        case 8:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 9:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 10:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 11:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 12:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 13:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_4_BANK));
+                                break;
+                        case 14:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_2_BANK));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+        } else if (num_pipe_configs == 4) {
+                if (num_rbs == 4) {
+                        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+                                switch (reg_offset) {
+                                case 0:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
+                                        break;
+                                case 1:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
+                                        break;
+                                case 2:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                        break;
+                                case 3:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
+                                        break;
+                                case 4:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(split_equal_to_row_size));
+                                        break;
+                                case 5:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+                                        break;
+                                case 6:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                        break;
+                                case 7:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         TILE_SPLIT(split_equal_to_row_size));
+                                        break;
+                                case 8:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16));
+                                        break;
+                                case 9:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
+                                        break;
+                                case 10:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 11:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 12:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 13:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
+                                        break;
+                                case 14:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 16:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 17:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 27:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
+                                        break;
+                                case 28:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 29:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 30:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                default:
+                                        gb_tile_moden = 0;
+                                        break;
+                                }
+                                rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
+                                WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                        }
+                } else if (num_rbs < 4) {
+                        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+                                switch (reg_offset) {
+                                case 0:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
+                                        break;
+                                case 1:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
+                                        break;
+                                case 2:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                        break;
+                                case 3:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
+                                        break;
+                                case 4:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(split_equal_to_row_size));
+                                        break;
+                                case 5:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+                                        break;
+                                case 6:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                        break;
+                                case 7:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         TILE_SPLIT(split_equal_to_row_size));
+                                        break;
+                                case 8:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+                                                 PIPE_CONFIG(ADDR_SURF_P4_8x16));
+                                        break;
+                                case 9:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
+                                        break;
+                                case 10:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 11:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 12:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 13:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
+                                        break;
+                                case 14:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 16:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 17:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 27:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
+                                        break;
+                                case 28:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 29:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                case 30:
+                                        gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                         MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+                                                         SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                        break;
+                                default:
+                                        gb_tile_moden = 0;
+                                        break;
+                                }
+                                rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
+                                WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                        }
+                }
+                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 1:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 2:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 3:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 4:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 5:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 6:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_4_BANK));
+                                break;
+                        case 8:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 9:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 10:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 11:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 12:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 13:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 14:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_4_BANK));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+        } else if (num_pipe_configs == 2) {
+                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
+                                break;
+                        case 1:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
+                                break;
+                        case 2:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                break;
+                        case 3:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
+                                break;
+                        case 4:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(split_equal_to_row_size));
+                                break;
+                        case 5:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+                                break;
+                        case 6:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                break;
+                        case 7:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 TILE_SPLIT(split_equal_to_row_size));
+                                break;
+                        case 8:
+                                gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
+                                break;
+                        case 9:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
+                                break;
+                        case 10:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 11:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 12:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 13:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
+                                break;
+                        case 14:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 16:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 17:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 27:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
+                                break;
+                        case 28:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 29:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 30:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
+                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 1:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 2:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 3:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 4:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 5:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 6:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 8:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 9:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 10:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 11:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 12:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 13:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 14:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+        } else
+                DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
+}
+
+/**
+ * cik_select_se_sh - select which SE, SH to address
+ *
+ * @rdev: radeon_device pointer
+ * @se_num: shader engine to address
+ * @sh_num: sh block to address
+ *
+ * Select which SE, SH combinations to address. Certain
+ * registers are instanced per SE or SH.  0xffffffff means
+ * broadcast to all SEs or SHs (CIK).
+ */
+static void cik_select_se_sh(struct radeon_device *rdev,
+                             u32 se_num, u32 sh_num)
+{
+        u32 data = INSTANCE_BROADCAST_WRITES;
+
+        if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
+                data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
+        else if (se_num == 0xffffffff)
+                data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
+        else if (sh_num == 0xffffffff)
+                data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
+        else
+                data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
+        WREG32(GRBM_GFX_INDEX, data);
+}
+
+/**
+ * cik_create_bitmask - create a bitmask
+ *
+ * @bit_width: length of the mask
+ *
+ * create a variable length bit mask (CIK).
+ * Returns the bitmask.
+ */
+static u32 cik_create_bitmask(u32 bit_width)
+{
+        u32 i, mask = 0;
+
+        for (i = 0; i < bit_width; i++) {
+                mask <<= 1;
+                mask |= 1;
+        }
+        return mask;
+}
+
+/**
+ * cik_select_se_sh - select which SE, SH to address
+ *
+ * @rdev: radeon_device pointer
+ * @max_rb_num: max RBs (render backends) for the asic
+ * @se_num: number of SEs (shader engines) for the asic
+ * @sh_per_se: number of SH blocks per SE for the asic
+ *
+ * Calculates the bitmask of disabled RBs (CIK).
+ * Returns the disabled RB bitmask.
+ */
+static u32 cik_get_rb_disabled(struct radeon_device *rdev,
+                              u32 max_rb_num, u32 se_num,
+                              u32 sh_per_se)
+{
+        u32 data, mask;
+
+        data = RREG32(CC_RB_BACKEND_DISABLE);
+        if (data & 1)
+                data &= BACKEND_DISABLE_MASK;
+        else
+                data = 0;
+        data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
+
+        data >>= BACKEND_DISABLE_SHIFT;
+
+        mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
+
+        return data & mask;
+}
+
+/**
+ * cik_setup_rb - setup the RBs on the asic
+ *
+ * @rdev: radeon_device pointer
+ * @se_num: number of SEs (shader engines) for the asic
+ * @sh_per_se: number of SH blocks per SE for the asic
+ * @max_rb_num: max RBs (render backends) for the asic
+ *
+ * Configures per-SE/SH RB registers (CIK).
+ */
+static void cik_setup_rb(struct radeon_device *rdev,
+                         u32 se_num, u32 sh_per_se,
+                         u32 max_rb_num)
+{
+        int i, j;
+        u32 data, mask;
+        u32 disabled_rbs = 0;
+        u32 enabled_rbs = 0;
+
+        for (i = 0; i < se_num; i++) {
+                for (j = 0; j < sh_per_se; j++) {
+                        cik_select_se_sh(rdev, i, j);
+                        data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
+                        disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
+                }
+        }
+        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
+
+        mask = 1;
+        for (i = 0; i < max_rb_num; i++) {
+                if (!(disabled_rbs & mask))
+                        enabled_rbs |= mask;
+                mask <<= 1;
+        }
+
+        for (i = 0; i < se_num; i++) {
+                cik_select_se_sh(rdev, i, 0xffffffff);
+                data = 0;
+                for (j = 0; j < sh_per_se; j++) {
+                        switch (enabled_rbs & 3) {
+                        case 1:
+                                data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
+                                break;
+                        case 2:
+                                data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
+                                break;
+                        case 3:
+                        default:
+                                data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
+                                break;
+                        }
+                        enabled_rbs >>= 2;
+                }
+                WREG32(PA_SC_RASTER_CONFIG, data);
+        }
+        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
+}
+
+/**
+ * cik_gpu_init - setup the 3D engine
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Configures the 3D engine and tiling configuration
+ * registers so that the 3D engine is usable.
+ */
+static void cik_gpu_init(struct radeon_device *rdev)
+{
+        u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
+        u32 mc_shared_chmap, mc_arb_ramcfg;
+        u32 hdp_host_path_cntl;
+        u32 tmp;
+        int i, j;
+
+        switch (rdev->family) {
+        case CHIP_BONAIRE:
+                rdev->config.cik.max_shader_engines = 2;
+                rdev->config.cik.max_tile_pipes = 4;
+                rdev->config.cik.max_cu_per_sh = 7;
+                rdev->config.cik.max_sh_per_se = 1;
+                rdev->config.cik.max_backends_per_se = 2;
+                rdev->config.cik.max_texture_channel_caches = 4;
+                rdev->config.cik.max_gprs = 256;
+                rdev->config.cik.max_gs_threads = 32;
+                rdev->config.cik.max_hw_contexts = 8;
+
+                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
+                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
+                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
+                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
+                break;
+        case CHIP_KAVERI:
+                /* TODO */
+                break;
+        case CHIP_KABINI:
+        default:
+                rdev->config.cik.max_shader_engines = 1;
+                rdev->config.cik.max_tile_pipes = 2;
+                rdev->config.cik.max_cu_per_sh = 2;
+                rdev->config.cik.max_sh_per_se = 1;
+                rdev->config.cik.max_backends_per_se = 1;
+                rdev->config.cik.max_texture_channel_caches = 2;
+                rdev->config.cik.max_gprs = 256;
+                rdev->config.cik.max_gs_threads = 16;
+                rdev->config.cik.max_hw_contexts = 8;
+
+                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
+                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
+                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
+                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
+                break;
+        }
+
+        /* Initialize HDP */
+        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
+                WREG32((0x2c14 + j), 0x00000000);
+                WREG32((0x2c18 + j), 0x00000000);
+                WREG32((0x2c1c + j), 0x00000000);
+                WREG32((0x2c20 + j), 0x00000000);
+                WREG32((0x2c24 + j), 0x00000000);
+        }
+
+        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
+
+        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
+
+        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
+        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
+
+        rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
+        rdev->config.cik.mem_max_burst_length_bytes = 256;
+        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
+        rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
+        if (rdev->config.cik.mem_row_size_in_kb > 4)
+                rdev->config.cik.mem_row_size_in_kb = 4;
+        /* XXX use MC settings? */
+        rdev->config.cik.shader_engine_tile_size = 32;
+        rdev->config.cik.num_gpus = 1;
+        rdev->config.cik.multi_gpu_tile_size = 64;
+
+        /* fix up row size */
+        gb_addr_config &= ~ROW_SIZE_MASK;
+        switch (rdev->config.cik.mem_row_size_in_kb) {
+        case 1:
+        default:
+                gb_addr_config |= ROW_SIZE(0);
+                break;
+        case 2:
+                gb_addr_config |= ROW_SIZE(1);
+                break;
+        case 4:
+                gb_addr_config |= ROW_SIZE(2);
+                break;
+        }
+
+        /* setup tiling info dword.  gb_addr_config is not adequate since it does
+         * not have bank info, so create a custom tiling dword.
+         * bits 3:0   num_pipes
+         * bits 7:4   num_banks
+         * bits 11:8  group_size
+         * bits 15:12 row_size
+         */
+        rdev->config.cik.tile_config = 0;
+        switch (rdev->config.cik.num_tile_pipes) {
+        case 1:
+                rdev->config.cik.tile_config |= (0 << 0);
+                break;
+        case 2:
+                rdev->config.cik.tile_config |= (1 << 0);
+                break;
+        case 4:
+                rdev->config.cik.tile_config |= (2 << 0);
+                break;
+        case 8:
+        default:
+                /* XXX what about 12? */
+                rdev->config.cik.tile_config |= (3 << 0);
+                break;
+        }
+        if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
+                rdev->config.cik.tile_config |= 1 << 4;
+        else
+                rdev->config.cik.tile_config |= 0 << 4;
+        rdev->config.cik.tile_config |=
+                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
+        rdev->config.cik.tile_config |=
+                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
+
+        WREG32(GB_ADDR_CONFIG, gb_addr_config);
+        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
+        WREG32(DMIF_ADDR_CALC, gb_addr_config);
+        WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
+        WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
+        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
+        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
+        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
+
+        cik_tiling_mode_table_init(rdev);
+
+        cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
+                     rdev->config.cik.max_sh_per_se,
+                     rdev->config.cik.max_backends_per_se);
+
+        /* set HW defaults for 3D engine */
+        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
+
+        WREG32(SX_DEBUG_1, 0x20);
+
+        WREG32(TA_CNTL_AUX, 0x00010000);
+
+        tmp = RREG32(SPI_CONFIG_CNTL);
+        tmp |= 0x03000000;
+        WREG32(SPI_CONFIG_CNTL, tmp);
+
+        WREG32(SQ_CONFIG, 1);
+
+        WREG32(DB_DEBUG, 0);
+
+        tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
+        tmp |= 0x00000400;
+        WREG32(DB_DEBUG2, tmp);
+
+        tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
+        tmp |= 0x00020200;
+        WREG32(DB_DEBUG3, tmp);
+
+        tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
+        tmp |= 0x00018208;
+        WREG32(CB_HW_CONTROL, tmp);
+
+        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
+
+        WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
+                                 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
+                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
+                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
+
+        WREG32(VGT_NUM_INSTANCES, 1);
+
+        WREG32(CP_PERFMON_CNTL, 0);
+
+        WREG32(SQ_CONFIG, 0);
+
+        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
+                                          FORCE_EOV_MAX_REZ_CNT(255)));
+
+        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
+               AUTO_INVLD_EN(ES_AND_GS_AUTO));
+
+        WREG32(VGT_GS_VERTEX_REUSE, 16);
+        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
+
+        tmp = RREG32(HDP_MISC_CNTL);
+        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
+        WREG32(HDP_MISC_CNTL, tmp);
+
+        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
+        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
+
+        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
+        WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
+
+        udelay(50);
+}
+
+/*
+ * GPU scratch registers helpers function.
+ */
+/**
+ * cik_scratch_init - setup driver info for CP scratch regs
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Set up the number and offset of the CP scratch registers.
+ * NOTE: use of CP scratch registers is a legacy inferface and
+ * is not used by default on newer asics (r6xx+).  On newer asics,
+ * memory buffers are used for fences rather than scratch regs.
+ */
+static void cik_scratch_init(struct radeon_device *rdev)
+{
+        int i;
+
+        rdev->scratch.num_reg = 7;
+        rdev->scratch.reg_base = SCRATCH_REG0;
+        for (i = 0; i < rdev->scratch.num_reg; i++) {
+                rdev->scratch.free[i] = true;
+                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
+        }
+}
+
+/**
+ * cik_ring_test - basic gfx ring test
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ *
+ * Allocate a scratch register and write to it using the gfx ring (CIK).
+ * Provides a basic gfx ring test to verify that the ring is working.
+ * Used by cik_cp_gfx_resume();
+ * Returns 0 on success, error on failure.
+ */
+int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        uint32_t scratch;
+        uint32_t tmp = 0;
+        unsigned i;
+        int r;
+
+        r = radeon_scratch_get(rdev, &scratch);
+        if (r) {
+                DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
+                return r;
+        }
+        WREG32(scratch, 0xCAFEDEAD);
+        r = radeon_ring_lock(rdev, ring, 3);
+        if (r) {
+                DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
+                radeon_scratch_free(rdev, scratch);
+                return r;
+        }
+        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+        radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
+        radeon_ring_write(ring, 0xDEADBEEF);
+        radeon_ring_unlock_commit(rdev, ring);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = RREG32(scratch);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i < rdev->usec_timeout) {
+                DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+        } else {
+                DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
+                          ring->idx, scratch, tmp);
+                r = -EINVAL;
+        }
+        radeon_scratch_free(rdev, scratch);
+        return r;
+}
+
+/**
+ * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
+ *
+ * @rdev: radeon_device pointer
+ * @fence: radeon fence object
+ *
+ * Emits a fence sequnce number on the gfx ring and flushes
+ * GPU caches.
+ */
+void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
+                             struct radeon_fence *fence)
+{
+        struct radeon_ring *ring = &rdev->ring[fence->ring];
+        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
+
+        /* EVENT_WRITE_EOP - flush caches, send int */
+        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
+        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
+                                 EOP_TC_ACTION_EN |
+                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
+                                 EVENT_INDEX(5)));
+        radeon_ring_write(ring, addr & 0xfffffffc);
+        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
+        radeon_ring_write(ring, fence->seq);
+        radeon_ring_write(ring, 0);
+        /* HDP flush */
+        /* We should be using the new WAIT_REG_MEM special op packet here
+         * but it causes the CP to hang
+         */
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0);
+}
+
+/**
+ * cik_fence_compute_ring_emit - emit a fence on the compute ring
+ *
+ * @rdev: radeon_device pointer
+ * @fence: radeon fence object
+ *
+ * Emits a fence sequnce number on the compute ring and flushes
+ * GPU caches.
+ */
+void cik_fence_compute_ring_emit(struct radeon_device *rdev,
+                                 struct radeon_fence *fence)
+{
+        struct radeon_ring *ring = &rdev->ring[fence->ring];
+        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
+
+        /* RELEASE_MEM - flush caches, send int */
+        radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
+        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
+                                 EOP_TC_ACTION_EN |
+                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
+                                 EVENT_INDEX(5)));
+        radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
+        radeon_ring_write(ring, addr & 0xfffffffc);
+        radeon_ring_write(ring, upper_32_bits(addr));
+        radeon_ring_write(ring, fence->seq);
+        radeon_ring_write(ring, 0);
+        /* HDP flush */
+        /* We should be using the new WAIT_REG_MEM special op packet here
+         * but it causes the CP to hang
+         */
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0);
+}
+
+void cik_semaphore_ring_emit(struct radeon_device *rdev,
+                             struct radeon_ring *ring,
+                             struct radeon_semaphore *semaphore,
+                             bool emit_wait)
+{
+        uint64_t addr = semaphore->gpu_addr;
+        unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
+
+        radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
+        radeon_ring_write(ring, addr & 0xffffffff);
+        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
+}
+
+/*
+ * IB stuff
+ */
+/**
+ * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
+ *
+ * @rdev: radeon_device pointer
+ * @ib: radeon indirect buffer object
+ *
+ * Emits an DE (drawing engine) or CE (constant engine) IB
+ * on the gfx ring.  IBs are usually generated by userspace
+ * acceleration drivers and submitted to the kernel for
+ * sheduling on the ring.  This function schedules the IB
+ * on the gfx ring for execution by the GPU.
+ */
+void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        struct radeon_ring *ring = &rdev->ring[ib->ring];
+        u32 header, control = INDIRECT_BUFFER_VALID;
+
+        if (ib->is_const_ib) {
+                /* set switch buffer packet before const IB */
+                radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+                radeon_ring_write(ring, 0);
+
+                header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
+        } else {
+                u32 next_rptr;
+                if (ring->rptr_save_reg) {
+                        next_rptr = ring->wptr + 3 + 4;
+                        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+                        radeon_ring_write(ring, ((ring->rptr_save_reg -
+                                                  PACKET3_SET_UCONFIG_REG_START) >> 2));
+                        radeon_ring_write(ring, next_rptr);
+                } else if (rdev->wb.enabled) {
+                        next_rptr = ring->wptr + 5 + 4;
+                        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+                        radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
+                        radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
+                        radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
+                        radeon_ring_write(ring, next_rptr);
+                }
+
+                header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
+        }
+
+        control |= ib->length_dw |
+                (ib->vm ? (ib->vm->id << 24) : 0);
+
+        radeon_ring_write(ring, header);
+        radeon_ring_write(ring,
+#ifdef __BIG_ENDIAN
+                          (2 << 0) |
+#endif
+                          (ib->gpu_addr & 0xFFFFFFFC));
+        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
+        radeon_ring_write(ring, control);
+}
+
+/**
+ * cik_ib_test - basic gfx ring IB test
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ *
+ * Allocate an IB and execute it on the gfx ring (CIK).
+ * Provides a basic gfx ring test to verify that IBs are working.
+ * Returns 0 on success, error on failure.
+ */
+int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        struct radeon_ib ib;
+        uint32_t scratch;
+        uint32_t tmp = 0;
+        unsigned i;
+        int r;
+
+        r = radeon_scratch_get(rdev, &scratch);
+        if (r) {
+                DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
+                return r;
+        }
+        WREG32(scratch, 0xCAFEDEAD);
+        r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
+        if (r) {
+                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
+                return r;
+        }
+        ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
+        ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
+        ib.ptr[2] = 0xDEADBEEF;
+        ib.length_dw = 3;
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r) {
+                radeon_scratch_free(rdev, scratch);
+                radeon_ib_free(rdev, &ib);
+                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
+                return r;
+        }
+        r = radeon_fence_wait(ib.fence, false);
+        if (r) {
+                DRM_ERROR("radeon: fence wait failed (%d).\n", r);
+                return r;
+        }
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = RREG32(scratch);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i < rdev->usec_timeout) {
+                DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
+        } else {
+                DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
+                          scratch, tmp);
+                r = -EINVAL;
+        }
+        radeon_scratch_free(rdev, scratch);
+        radeon_ib_free(rdev, &ib);
+        return r;
+}
+
+/*
+ * CP.
+ * On CIK, gfx and compute now have independant command processors.
+ *
+ * GFX
+ * Gfx consists of a single ring and can process both gfx jobs and
+ * compute jobs.  The gfx CP consists of three microengines (ME):
+ * PFP - Pre-Fetch Parser
+ * ME - Micro Engine
+ * CE - Constant Engine
+ * The PFP and ME make up what is considered the Drawing Engine (DE).
+ * The CE is an asynchronous engine used for updating buffer desciptors
+ * used by the DE so that they can be loaded into cache in parallel
+ * while the DE is processing state update packets.
+ *
+ * Compute
+ * The compute CP consists of two microengines (ME):
+ * MEC1 - Compute MicroEngine 1
+ * MEC2 - Compute MicroEngine 2
+ * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
+ * The queues are exposed to userspace and are programmed directly
+ * by the compute runtime.
+ */
+/**
+ * cik_cp_gfx_enable - enable/disable the gfx CP MEs
+ *
+ * @rdev: radeon_device pointer
+ * @enable: enable or disable the MEs
+ *
+ * Halts or unhalts the gfx MEs.
+ */
+static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32(CP_ME_CNTL, 0);
+        else {
+                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
+                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
+        }
+        udelay(50);
+}
+
+/**
+ * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Loads the gfx PFP, ME, and CE ucode.
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        int i;
+
+        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
+                return -EINVAL;
+
+        cik_cp_gfx_enable(rdev, false);
+
+        /* PFP */
+        fw_data = (const __be32 *)rdev->pfp_fw->data;
+        WREG32(CP_PFP_UCODE_ADDR, 0);
+        for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
+                WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
+        WREG32(CP_PFP_UCODE_ADDR, 0);
+
+        /* CE */
+        fw_data = (const __be32 *)rdev->ce_fw->data;
+        WREG32(CP_CE_UCODE_ADDR, 0);
+        for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
+                WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
+        WREG32(CP_CE_UCODE_ADDR, 0);
+
+        /* ME */
+        fw_data = (const __be32 *)rdev->me_fw->data;
+        WREG32(CP_ME_RAM_WADDR, 0);
+        for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
+                WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
+        WREG32(CP_ME_RAM_WADDR, 0);
+
+        WREG32(CP_PFP_UCODE_ADDR, 0);
+        WREG32(CP_CE_UCODE_ADDR, 0);
+        WREG32(CP_ME_RAM_WADDR, 0);
+        WREG32(CP_ME_RAM_RADDR, 0);
+        return 0;
+}
+
+/**
+ * cik_cp_gfx_start - start the gfx ring
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Enables the ring and loads the clear state context and other
+ * packets required to init the ring.
+ * Returns 0 for success, error for failure.
+ */
+static int cik_cp_gfx_start(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
+        int r, i;
+
+        /* init the CP */
+        WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
+        WREG32(CP_ENDIAN_SWAP, 0);
+        WREG32(CP_DEVICE_ID, 1);
+
+        cik_cp_gfx_enable(rdev, true);
+
+        r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
+        if (r) {
+                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
+                return r;
+        }
+
+        /* init the CE partitions.  CE only used for gfx on CIK */
+        radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
+        radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
+        radeon_ring_write(ring, 0xc000);
+        radeon_ring_write(ring, 0xc000);
+
+        /* setup clear context state */
+        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+        radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+        radeon_ring_write(ring, 0x80000000);
+        radeon_ring_write(ring, 0x80000000);
+
+        for (i = 0; i < cik_default_size; i++)
+                radeon_ring_write(ring, cik_default_state[i]);
+
+        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+        /* set clear context state */
+        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+        radeon_ring_write(ring, 0);
+
+        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
+        radeon_ring_write(ring, 0x00000316);
+        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
+
+        radeon_ring_unlock_commit(rdev, ring);
+
+        return 0;
+}
+
+/**
+ * cik_cp_gfx_fini - stop the gfx ring
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Stop the gfx ring and tear down the driver ring
+ * info.
+ */
+static void cik_cp_gfx_fini(struct radeon_device *rdev)
+{
+        cik_cp_gfx_enable(rdev, false);
+        radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
+}
+
+/**
+ * cik_cp_gfx_resume - setup the gfx ring buffer registers
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Program the location and size of the gfx ring buffer
+ * and test it to make sure it's working.
+ * Returns 0 for success, error for failure.
+ */
+static int cik_cp_gfx_resume(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring;
+        u32 tmp;
+        u32 rb_bufsz;
+        u64 rb_addr;
+        int r;
+
+        WREG32(CP_SEM_WAIT_TIMER, 0x0);
+        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
+
+        /* Set the write pointer delay */
+        WREG32(CP_RB_WPTR_DELAY, 0);
+
+        /* set the RB to use vmid 0 */
+        WREG32(CP_RB_VMID, 0);
+
+        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
+
+        /* ring 0 - compute and gfx */
+        /* Set ring buffer size */
+        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
+        rb_bufsz = drm_order(ring->ring_size / 8);
+        tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+#ifdef __BIG_ENDIAN
+        tmp |= BUF_SWAP_32BIT;
+#endif
+        WREG32(CP_RB0_CNTL, tmp);
+
+        /* Initialize the ring buffer's read and write pointers */
+        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
+        ring->wptr = 0;
+        WREG32(CP_RB0_WPTR, ring->wptr);
+
+        /* set the wb address wether it's enabled or not */
+        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
+
+        /* scratch register shadowing is no longer supported */
+        WREG32(SCRATCH_UMSK, 0);
+
+        if (!rdev->wb.enabled)
+                tmp |= RB_NO_UPDATE;
+
+        mdelay(1);
+        WREG32(CP_RB0_CNTL, tmp);
+
+        rb_addr = ring->gpu_addr >> 8;
+        WREG32(CP_RB0_BASE, rb_addr);
+        WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
+
+        ring->rptr = RREG32(CP_RB0_RPTR);
+
+        /* start the ring */
+        cik_cp_gfx_start(rdev);
+        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
+        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
+        if (r) {
+                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
+                return r;
+        }
+        return 0;
+}
+
+u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
+                              struct radeon_ring *ring)
+{
+        u32 rptr;
+
+
+
+        if (rdev->wb.enabled) {
+                rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
+        } else {
+                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
+                rptr = RREG32(CP_HQD_PQ_RPTR);
+                cik_srbm_select(rdev, 0, 0, 0, 0);
+        }
+        rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
+
+        return rptr;
+}
+
+u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
+                              struct radeon_ring *ring)
+{
+        u32 wptr;
+
+        if (rdev->wb.enabled) {
+                wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
+        } else {
+                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
+                wptr = RREG32(CP_HQD_PQ_WPTR);
+                cik_srbm_select(rdev, 0, 0, 0, 0);
+        }
+        wptr = (wptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
+
+        return wptr;
+}
+
+void cik_compute_ring_set_wptr(struct radeon_device *rdev,
+                               struct radeon_ring *ring)
+{
+        u32 wptr = (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask;
+
+        rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(wptr);
+        WDOORBELL32(ring->doorbell_offset, wptr);
+}
+
+/**
+ * cik_cp_compute_enable - enable/disable the compute CP MEs
+ *
+ * @rdev: radeon_device pointer
+ * @enable: enable or disable the MEs
+ *
+ * Halts or unhalts the compute MEs.
+ */
+static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32(CP_MEC_CNTL, 0);
+        else
+                WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
+        udelay(50);
+}
+
+/**
+ * cik_cp_compute_load_microcode - load the compute CP ME ucode
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Loads the compute MEC1&2 ucode.
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        int i;
+
+        if (!rdev->mec_fw)
+                return -EINVAL;
+
+        cik_cp_compute_enable(rdev, false);
+
+        /* MEC1 */
+        fw_data = (const __be32 *)rdev->mec_fw->data;
+        WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
+        for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
+                WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
+        WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
+
+        if (rdev->family == CHIP_KAVERI) {
+                /* MEC2 */
+                fw_data = (const __be32 *)rdev->mec_fw->data;
+                WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
+                for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
+                        WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
+                WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
+        }
+
+        return 0;
+}
+
+/**
+ * cik_cp_compute_start - start the compute queues
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Enable the compute queues.
+ * Returns 0 for success, error for failure.
+ */
+static int cik_cp_compute_start(struct radeon_device *rdev)
+{
+        cik_cp_compute_enable(rdev, true);
+
+        return 0;
+}
+
+/**
+ * cik_cp_compute_fini - stop the compute queues
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Stop the compute queues and tear down the driver queue
+ * info.
+ */
+static void cik_cp_compute_fini(struct radeon_device *rdev)
+{
+        int i, idx, r;
+
+        cik_cp_compute_enable(rdev, false);
+
+        for (i = 0; i < 2; i++) {
+                if (i == 0)
+                        idx = CAYMAN_RING_TYPE_CP1_INDEX;
+                else
+                        idx = CAYMAN_RING_TYPE_CP2_INDEX;
+
+                if (rdev->ring[idx].mqd_obj) {
+                        r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
+                        if (unlikely(r != 0))
+                                dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
+
+                        radeon_bo_unpin(rdev->ring[idx].mqd_obj);
+                        radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
+
+                        radeon_bo_unref(&rdev->ring[idx].mqd_obj);
+                        rdev->ring[idx].mqd_obj = NULL;
+                }
+        }
+}
+
+static void cik_mec_fini(struct radeon_device *rdev)
+{
+        int r;
+
+        if (rdev->mec.hpd_eop_obj) {
+                r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
+                if (unlikely(r != 0))
+                        dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
+                radeon_bo_unpin(rdev->mec.hpd_eop_obj);
+                radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
+
+                radeon_bo_unref(&rdev->mec.hpd_eop_obj);
+                rdev->mec.hpd_eop_obj = NULL;
+        }
+}
+
+#define MEC_HPD_SIZE 2048
+
+static int cik_mec_init(struct radeon_device *rdev)
+{
+        int r;
+        u32 *hpd;
+
+        /*
+         * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
+         * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
+         */
+        if (rdev->family == CHIP_KAVERI)
+                rdev->mec.num_mec = 2;
+        else
+                rdev->mec.num_mec = 1;
+        rdev->mec.num_pipe = 4;
+        rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
+
+        if (rdev->mec.hpd_eop_obj == NULL) {
+                r = radeon_bo_create(rdev,
+                                     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
+                                     PAGE_SIZE, true,
+                                     RADEON_GEM_DOMAIN_GTT, NULL,
+                                     &rdev->mec.hpd_eop_obj);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
+                        return r;
+                }
+        }
+
+        r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
+        if (unlikely(r != 0)) {
+                cik_mec_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
+                          &rdev->mec.hpd_eop_gpu_addr);
+        if (r) {
+                dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
+                cik_mec_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
+        if (r) {
+                dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
+                cik_mec_fini(rdev);
+                return r;
+        }
+
+        /* clear memory.  Not sure if this is required or not */
+        memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
+
+        radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
+        radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
+
+        return 0;
+}
+
+struct hqd_registers
+{
+        u32 cp_mqd_base_addr;
+        u32 cp_mqd_base_addr_hi;
+        u32 cp_hqd_active;
+        u32 cp_hqd_vmid;
+        u32 cp_hqd_persistent_state;
+        u32 cp_hqd_pipe_priority;
+        u32 cp_hqd_queue_priority;
+        u32 cp_hqd_quantum;
+        u32 cp_hqd_pq_base;
+        u32 cp_hqd_pq_base_hi;
+        u32 cp_hqd_pq_rptr;
+        u32 cp_hqd_pq_rptr_report_addr;
+        u32 cp_hqd_pq_rptr_report_addr_hi;
+        u32 cp_hqd_pq_wptr_poll_addr;
+        u32 cp_hqd_pq_wptr_poll_addr_hi;
+        u32 cp_hqd_pq_doorbell_control;
+        u32 cp_hqd_pq_wptr;
+        u32 cp_hqd_pq_control;
+        u32 cp_hqd_ib_base_addr;
+        u32 cp_hqd_ib_base_addr_hi;
+        u32 cp_hqd_ib_rptr;
+        u32 cp_hqd_ib_control;
+        u32 cp_hqd_iq_timer;
+        u32 cp_hqd_iq_rptr;
+        u32 cp_hqd_dequeue_request;
+        u32 cp_hqd_dma_offload;
+        u32 cp_hqd_sema_cmd;
+        u32 cp_hqd_msg_type;
+        u32 cp_hqd_atomic0_preop_lo;
+        u32 cp_hqd_atomic0_preop_hi;
+        u32 cp_hqd_atomic1_preop_lo;
+        u32 cp_hqd_atomic1_preop_hi;
+        u32 cp_hqd_hq_scheduler0;
+        u32 cp_hqd_hq_scheduler1;
+        u32 cp_mqd_control;
+};
+
+struct bonaire_mqd
+{
+        u32 header;
+        u32 dispatch_initiator;
+        u32 dimensions[3];
+        u32 start_idx[3];
+        u32 num_threads[3];
+        u32 pipeline_stat_enable;
+        u32 perf_counter_enable;
+        u32 pgm[2];
+        u32 tba[2];
+        u32 tma[2];
+        u32 pgm_rsrc[2];
+        u32 vmid;
+        u32 resource_limits;
+        u32 static_thread_mgmt01[2];
+        u32 tmp_ring_size;
+        u32 static_thread_mgmt23[2];
+        u32 restart[3];
+        u32 thread_trace_enable;
+        u32 reserved1;
+        u32 user_data[16];
+        u32 vgtcs_invoke_count[2];
+        struct hqd_registers queue_state;
+        u32 dequeue_cntr;
+        u32 interrupt_queue[64];
+};
+
+/**
+ * cik_cp_compute_resume - setup the compute queue registers
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Program the compute queues and test them to make sure they
+ * are working.
+ * Returns 0 for success, error for failure.
+ */
+static int cik_cp_compute_resume(struct radeon_device *rdev)
+{
+        int r, i, idx;
+        u32 tmp;
+        bool use_doorbell = true;
+        u64 hqd_gpu_addr;
+        u64 mqd_gpu_addr;
+        u64 eop_gpu_addr;
+        u64 wb_gpu_addr;
+        u32 *buf;
+        struct bonaire_mqd *mqd;
+
+        r = cik_cp_compute_start(rdev);
+        if (r)
+                return r;
+
+        /* fix up chicken bits */
+        tmp = RREG32(CP_CPF_DEBUG);
+        tmp |= (1 << 23);
+        WREG32(CP_CPF_DEBUG, tmp);
+
+        /* init the pipes */
+        for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
+                int me = (i < 4) ? 1 : 2;
+                int pipe = (i < 4) ? i : (i - 4);
+
+                eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
+
+                cik_srbm_select(rdev, me, pipe, 0, 0);
+
+                /* write the EOP addr */
+                WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
+                WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
+
+                /* set the VMID assigned */
+                WREG32(CP_HPD_EOP_VMID, 0);
+
+                /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+                tmp = RREG32(CP_HPD_EOP_CONTROL);
+                tmp &= ~EOP_SIZE_MASK;
+                tmp |= drm_order(MEC_HPD_SIZE / 8);
+                WREG32(CP_HPD_EOP_CONTROL, tmp);
+        }
+        cik_srbm_select(rdev, 0, 0, 0, 0);
+
+        /* init the queues.  Just two for now. */
+        for (i = 0; i < 2; i++) {
+                if (i == 0)
+                        idx = CAYMAN_RING_TYPE_CP1_INDEX;
+                else
+                        idx = CAYMAN_RING_TYPE_CP2_INDEX;
+
+                if (rdev->ring[idx].mqd_obj == NULL) {
+                        r = radeon_bo_create(rdev,
+                                             sizeof(struct bonaire_mqd),
+                                             PAGE_SIZE, true,
+                                             RADEON_GEM_DOMAIN_GTT, NULL,
+                                             &rdev->ring[idx].mqd_obj);
+                        if (r) {
+                                dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
+                                return r;
+                        }
+                }
+
+                r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
+                if (unlikely(r != 0)) {
+                        cik_cp_compute_fini(rdev);
+                        return r;
+                }
+                r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
+                                  &mqd_gpu_addr);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
+                        cik_cp_compute_fini(rdev);
+                        return r;
+                }
+                r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
+                        cik_cp_compute_fini(rdev);
+                        return r;
+                }
+
+                /* doorbell offset */
+                rdev->ring[idx].doorbell_offset =
+                        (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
+
+                /* init the mqd struct */
+                memset(buf, 0, sizeof(struct bonaire_mqd));
+
+                mqd = (struct bonaire_mqd *)buf;
+                mqd->header = 0xC0310800;
+                mqd->static_thread_mgmt01[0] = 0xffffffff;
+                mqd->static_thread_mgmt01[1] = 0xffffffff;
+                mqd->static_thread_mgmt23[0] = 0xffffffff;
+                mqd->static_thread_mgmt23[1] = 0xffffffff;
+
+                cik_srbm_select(rdev, rdev->ring[idx].me,
+                                rdev->ring[idx].pipe,
+                                rdev->ring[idx].queue, 0);
+
+                /* disable wptr polling */
+                tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
+                tmp &= ~WPTR_POLL_EN;
+                WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
+
+                /* enable doorbell? */
+                mqd->queue_state.cp_hqd_pq_doorbell_control =
+                        RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
+                if (use_doorbell)
+                        mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
+                else
+                        mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
+                WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
+                       mqd->queue_state.cp_hqd_pq_doorbell_control);
+
+                /* disable the queue if it's active */
+                mqd->queue_state.cp_hqd_dequeue_request = 0;
+                mqd->queue_state.cp_hqd_pq_rptr = 0;
+                mqd->queue_state.cp_hqd_pq_wptr= 0;
+                if (RREG32(CP_HQD_ACTIVE) & 1) {
+                        WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
+                        for (i = 0; i < rdev->usec_timeout; i++) {
+                                if (!(RREG32(CP_HQD_ACTIVE) & 1))
+                                        break;
+                                udelay(1);
+                        }
+                        WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
+                        WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
+                        WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
+                }
+
+                /* set the pointer to the MQD */
+                mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
+                mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
+                WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
+                WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
+                /* set MQD vmid to 0 */
+                mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
+                mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
+                WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
+
+                /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+                hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
+                mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
+                mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
+                WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
+                WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
+
+                /* set up the HQD, this is similar to CP_RB0_CNTL */
+                mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
+                mqd->queue_state.cp_hqd_pq_control &=
+                        ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
+
+                mqd->queue_state.cp_hqd_pq_control |=
+                        drm_order(rdev->ring[idx].ring_size / 8);
+                mqd->queue_state.cp_hqd_pq_control |=
+                        (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8);
+#ifdef __BIG_ENDIAN
+                mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
+#endif
+                mqd->queue_state.cp_hqd_pq_control &=
+                        ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
+                mqd->queue_state.cp_hqd_pq_control |=
+                        PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
+                WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
+
+                /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
+                if (i == 0)
+                        wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
+                else
+                        wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
+                mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
+                mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
+                WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
+                WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
+                       mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
+
+                /* set the wb address wether it's enabled or not */
+                if (i == 0)
+                        wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
+                else
+                        wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
+                mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
+                mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
+                        upper_32_bits(wb_gpu_addr) & 0xffff;
+                WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
+                       mqd->queue_state.cp_hqd_pq_rptr_report_addr);
+                WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+                       mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
+
+                /* enable the doorbell if requested */
+                if (use_doorbell) {
+                        mqd->queue_state.cp_hqd_pq_doorbell_control =
+                                RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
+                        mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
+                        mqd->queue_state.cp_hqd_pq_doorbell_control |=
+                                DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
+                        mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
+                        mqd->queue_state.cp_hqd_pq_doorbell_control &=
+                                ~(DOORBELL_SOURCE | DOORBELL_HIT);
+
+                } else {
+                        mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
+                }
+                WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
+                       mqd->queue_state.cp_hqd_pq_doorbell_control);
+
+                /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+                rdev->ring[idx].wptr = 0;
+                mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
+                WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
+                rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
+                mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
+
+                /* set the vmid for the queue */
+                mqd->queue_state.cp_hqd_vmid = 0;
+                WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
+
+                /* activate the queue */
+                mqd->queue_state.cp_hqd_active = 1;
+                WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
+
+                cik_srbm_select(rdev, 0, 0, 0, 0);
+
+                radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
+                radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
+
+                rdev->ring[idx].ready = true;
+                r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
+                if (r)
+                        rdev->ring[idx].ready = false;
+        }
+
+        return 0;
+}
+
+static void cik_cp_enable(struct radeon_device *rdev, bool enable)
+{
+        cik_cp_gfx_enable(rdev, enable);
+        cik_cp_compute_enable(rdev, enable);
+}
+
+static int cik_cp_load_microcode(struct radeon_device *rdev)
+{
+        int r;
+
+        r = cik_cp_gfx_load_microcode(rdev);
+        if (r)
+                return r;
+        r = cik_cp_compute_load_microcode(rdev);
+        if (r)
+                return r;
+
+        return 0;
+}
+
+static void cik_cp_fini(struct radeon_device *rdev)
+{
+        cik_cp_gfx_fini(rdev);
+        cik_cp_compute_fini(rdev);
+}
+
+static int cik_cp_resume(struct radeon_device *rdev)
+{
+        int r;
+
+        /* Reset all cp blocks */
+        WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
+        RREG32(GRBM_SOFT_RESET);
+        mdelay(15);
+        WREG32(GRBM_SOFT_RESET, 0);
+        RREG32(GRBM_SOFT_RESET);
+
+        r = cik_cp_load_microcode(rdev);
+        if (r)
+                return r;
+
+        r = cik_cp_gfx_resume(rdev);
+        if (r)
+                return r;
+        r = cik_cp_compute_resume(rdev);
+        if (r)
+                return r;
+
+        return 0;
+}
+
+/*
+ * sDMA - System DMA
+ * Starting with CIK, the GPU has new asynchronous
+ * DMA engines.  These engines are used for compute
+ * and gfx.  There are two DMA engines (SDMA0, SDMA1)
+ * and each one supports 1 ring buffer used for gfx
+ * and 2 queues used for compute.
+ *
+ * The programming model is very similar to the CP
+ * (ring buffer, IBs, etc.), but sDMA has it's own
+ * packet format that is different from the PM4 format
+ * used by the CP. sDMA supports copying data, writing
+ * embedded data, solid fills, and a number of other
+ * things.  It also has support for tiling/detiling of
+ * buffers.
+ */
+/**
+ * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
+ *
+ * @rdev: radeon_device pointer
+ * @ib: IB object to schedule
+ *
+ * Schedule an IB in the DMA ring (CIK).
+ */
+void cik_sdma_ring_ib_execute(struct radeon_device *rdev,
+                              struct radeon_ib *ib)
+{
+        struct radeon_ring *ring = &rdev->ring[ib->ring];
+        u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf;
+
+        if (rdev->wb.enabled) {
+                u32 next_rptr = ring->wptr + 5;
+                while ((next_rptr & 7) != 4)
+                        next_rptr++;
+                next_rptr += 4;
+                radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
+                radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
+                radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
+                radeon_ring_write(ring, 1); /* number of DWs to follow */
+                radeon_ring_write(ring, next_rptr);
+        }
+
+        /* IB packet must end on a 8 DW boundary */
+        while ((ring->wptr & 7) != 4)
+                radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
+        radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
+        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
+        radeon_ring_write(ring, ib->length_dw);
+
+}
+
+/**
+ * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
+ *
+ * @rdev: radeon_device pointer
+ * @fence: radeon fence object
+ *
+ * Add a DMA fence packet to the ring to write
+ * the fence seq number and DMA trap packet to generate
+ * an interrupt if needed (CIK).
+ */
+void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
+                              struct radeon_fence *fence)
+{
+        struct radeon_ring *ring = &rdev->ring[fence->ring];
+        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
+        u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
+                          SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
+        u32 ref_and_mask;
+
+        if (fence->ring == R600_RING_TYPE_DMA_INDEX)
+                ref_and_mask = SDMA0;
+        else
+                ref_and_mask = SDMA1;
+
+        /* write the fence */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
+        radeon_ring_write(ring, addr & 0xffffffff);
+        radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+        radeon_ring_write(ring, fence->seq);
+        /* generate an interrupt */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
+        /* flush HDP */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
+        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
+        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
+        radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
+        radeon_ring_write(ring, ref_and_mask); /* MASK */
+        radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
+}
+
+/**
+ * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ * @semaphore: radeon semaphore object
+ * @emit_wait: wait or signal semaphore
+ *
+ * Add a DMA semaphore packet to the ring wait on or signal
+ * other rings (CIK).
+ */
+void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
+                                  struct radeon_ring *ring,
+                                  struct radeon_semaphore *semaphore,
+                                  bool emit_wait)
+{
+        u64 addr = semaphore->gpu_addr;
+        u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));
+        radeon_ring_write(ring, addr & 0xfffffff8);
+        radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+}
+
+/**
+ * cik_sdma_gfx_stop - stop the gfx async dma engines
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Stop the gfx async dma ring buffers (CIK).
+ */
+static void cik_sdma_gfx_stop(struct radeon_device *rdev)
+{
+        u32 rb_cntl, reg_offset;
+        int i;
+
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
+
+        for (i = 0; i < 2; i++) {
+                if (i == 0)
+                        reg_offset = SDMA0_REGISTER_OFFSET;
+                else
+                        reg_offset = SDMA1_REGISTER_OFFSET;
+                rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset);
+                rb_cntl &= ~SDMA_RB_ENABLE;
+                WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
+                WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);
+        }
+}
+
+/**
+ * cik_sdma_rlc_stop - stop the compute async dma engines
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Stop the compute async dma queues (CIK).
+ */
+static void cik_sdma_rlc_stop(struct radeon_device *rdev)
+{
+        /* XXX todo */
+}
+
+/**
+ * cik_sdma_enable - stop the async dma engines
+ *
+ * @rdev: radeon_device pointer
+ * @enable: enable/disable the DMA MEs.
+ *
+ * Halt or unhalt the async dma engines (CIK).
+ */
+static void cik_sdma_enable(struct radeon_device *rdev, bool enable)
+{
+        u32 me_cntl, reg_offset;
+        int i;
+
+        for (i = 0; i < 2; i++) {
+                if (i == 0)
+                        reg_offset = SDMA0_REGISTER_OFFSET;
+                else
+                        reg_offset = SDMA1_REGISTER_OFFSET;
+                me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset);
+                if (enable)
+                        me_cntl &= ~SDMA_HALT;
+                else
+                        me_cntl |= SDMA_HALT;
+                WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl);
+        }
+}
+
+/**
+ * cik_sdma_gfx_resume - setup and start the async dma engines
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Set up the gfx DMA ring buffers and enable them (CIK).
+ * Returns 0 for success, error for failure.
+ */
+static int cik_sdma_gfx_resume(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring;
+        u32 rb_cntl, ib_cntl;
+        u32 rb_bufsz;
+        u32 reg_offset, wb_offset;
+        int i, r;
+
+        for (i = 0; i < 2; i++) {
+                if (i == 0) {
+                        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
+                        reg_offset = SDMA0_REGISTER_OFFSET;
+                        wb_offset = R600_WB_DMA_RPTR_OFFSET;
+                } else {
+                        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
+                        reg_offset = SDMA1_REGISTER_OFFSET;
+                        wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
+                }
+
+                WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
+                WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
+
+                /* Set ring buffer size in dwords */
+                rb_bufsz = drm_order(ring->ring_size / 4);
+                rb_cntl = rb_bufsz << 1;
+#ifdef __BIG_ENDIAN
+                rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE;
+#endif
+                WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
+
+                /* Initialize the ring buffer's read and write pointers */
+                WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);
+                WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);
+
+                /* set the wb address whether it's enabled or not */
+                WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset,
+                       upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
+                WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset,
+                       ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
+
+                if (rdev->wb.enabled)
+                        rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE;
+
+                WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);
+                WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);
+
+                ring->wptr = 0;
+                WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
+
+                ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
+
+                /* enable DMA RB */
+                WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
+
+                ib_cntl = SDMA_IB_ENABLE;
+#ifdef __BIG_ENDIAN
+                ib_cntl |= SDMA_IB_SWAP_ENABLE;
+#endif
+                /* enable DMA IBs */
+                WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl);
+
+                ring->ready = true;
+
+                r = radeon_ring_test(rdev, ring->idx, ring);
+                if (r) {
+                        ring->ready = false;
+                        return r;
+                }
+        }
+
+        radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
+
+        return 0;
+}
+
+/**
+ * cik_sdma_rlc_resume - setup and start the async dma engines
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Set up the compute DMA queues and enable them (CIK).
+ * Returns 0 for success, error for failure.
+ */
+static int cik_sdma_rlc_resume(struct radeon_device *rdev)
+{
+        /* XXX todo */
+        return 0;
+}
+
+/**
+ * cik_sdma_load_microcode - load the sDMA ME ucode
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Loads the sDMA0/1 ucode.
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int cik_sdma_load_microcode(struct radeon_device *rdev)
+{
+        const __be32 *fw_data;
+        int i;
+
+        if (!rdev->sdma_fw)
+                return -EINVAL;
+
+        /* stop the gfx rings and rlc compute queues */
+        cik_sdma_gfx_stop(rdev);
+        cik_sdma_rlc_stop(rdev);
+
+        /* halt the MEs */
+        cik_sdma_enable(rdev, false);
+
+        /* sdma0 */
+        fw_data = (const __be32 *)rdev->sdma_fw->data;
+        WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
+        for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
+                WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++));
+        WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
+
+        /* sdma1 */
+        fw_data = (const __be32 *)rdev->sdma_fw->data;
+        WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
+        for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
+                WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++));
+        WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
+
+        WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
+        WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
+        return 0;
+}
+
+/**
+ * cik_sdma_resume - setup and start the async dma engines
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Set up the DMA engines and enable them (CIK).
+ * Returns 0 for success, error for failure.
+ */
+static int cik_sdma_resume(struct radeon_device *rdev)
+{
+        int r;
+
+        /* Reset dma */
+        WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1);
+        RREG32(SRBM_SOFT_RESET);
+        udelay(50);
+        WREG32(SRBM_SOFT_RESET, 0);
+        RREG32(SRBM_SOFT_RESET);
+
+        r = cik_sdma_load_microcode(rdev);
+        if (r)
+                return r;
+
+        /* unhalt the MEs */
+        cik_sdma_enable(rdev, true);
+
+        /* start the gfx rings and rlc compute queues */
+        r = cik_sdma_gfx_resume(rdev);
+        if (r)
+                return r;
+        r = cik_sdma_rlc_resume(rdev);
+        if (r)
+                return r;
+
+        return 0;
+}
+
+/**
+ * cik_sdma_fini - tear down the async dma engines
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Stop the async dma engines and free the rings (CIK).
+ */
+static void cik_sdma_fini(struct radeon_device *rdev)
+{
+        /* stop the gfx rings and rlc compute queues */
+        cik_sdma_gfx_stop(rdev);
+        cik_sdma_rlc_stop(rdev);
+        /* halt the MEs */
+        cik_sdma_enable(rdev, false);
+        radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
+        radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
+        /* XXX - compute dma queue tear down */
+}
+
+/**
+ * cik_copy_dma - copy pages using the DMA engine
+ *
+ * @rdev: radeon_device pointer
+ * @src_offset: src GPU address
+ * @dst_offset: dst GPU address
+ * @num_gpu_pages: number of GPU pages to xfer
+ * @fence: radeon fence object
+ *
+ * Copy GPU paging using the DMA engine (CIK).
+ * Used by the radeon ttm implementation to move pages if
+ * registered as the asic copy callback.
+ */
+int cik_copy_dma(struct radeon_device *rdev,
+                 uint64_t src_offset, uint64_t dst_offset,
+                 unsigned num_gpu_pages,
+                 struct radeon_fence **fence)
+{
+        struct radeon_semaphore *sem = NULL;
+        int ring_index = rdev->asic->copy.dma_ring_index;
+        struct radeon_ring *ring = &rdev->ring[ring_index];
+        u32 size_in_bytes, cur_size_in_bytes;
+        int i, num_loops;
+        int r = 0;
+
+        r = radeon_semaphore_create(rdev, &sem);
+        if (r) {
+                DRM_ERROR("radeon: moving bo (%d).\n", r);
+                return r;
+        }
+
+        size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
+        num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
+        r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
+        if (r) {
+                DRM_ERROR("radeon: moving bo (%d).\n", r);
+                radeon_semaphore_free(rdev, &sem, NULL);
+                return r;
+        }
+
+        if (radeon_fence_need_sync(*fence, ring->idx)) {
+                radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
+                                            ring->idx);
+                radeon_fence_note_sync(*fence, ring->idx);
+        } else {
+                radeon_semaphore_free(rdev, &sem, NULL);
+        }
+
+        for (i = 0; i < num_loops; i++) {
+                cur_size_in_bytes = size_in_bytes;
+                if (cur_size_in_bytes > 0x1fffff)
+                        cur_size_in_bytes = 0x1fffff;
+                size_in_bytes -= cur_size_in_bytes;
+                radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
+                radeon_ring_write(ring, cur_size_in_bytes);
+                radeon_ring_write(ring, 0); /* src/dst endian swap */
+                radeon_ring_write(ring, src_offset & 0xffffffff);
+                radeon_ring_write(ring, upper_32_bits(src_offset) & 0xffffffff);
+                radeon_ring_write(ring, dst_offset & 0xfffffffc);
+                radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xffffffff);
+                src_offset += cur_size_in_bytes;
+                dst_offset += cur_size_in_bytes;
+        }
+
+        r = radeon_fence_emit(rdev, fence, ring->idx);
+        if (r) {
+                radeon_ring_unlock_undo(rdev, ring);
+                return r;
+        }
+
+        radeon_ring_unlock_commit(rdev, ring);
+        radeon_semaphore_free(rdev, &sem, *fence);
+
+        return r;
+}
+
+/**
+ * cik_sdma_ring_test - simple async dma engine test
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ *
+ * Test the DMA engine by writing using it to write an
+ * value to memory. (CIK).
+ * Returns 0 for success, error for failure.
+ */
+int cik_sdma_ring_test(struct radeon_device *rdev,
+                       struct radeon_ring *ring)
+{
+        unsigned i;
+        int r;
+        void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
+        u32 tmp;
+
+        if (!ptr) {
+                DRM_ERROR("invalid vram scratch pointer\n");
+                return -EINVAL;
+        }
+
+        tmp = 0xCAFEDEAD;
+        writel(tmp, ptr);
+
+        r = radeon_ring_lock(rdev, ring, 4);
+        if (r) {
+                DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
+                return r;
+        }
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
+        radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
+        radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff);
+        radeon_ring_write(ring, 1); /* number of DWs to follow */
+        radeon_ring_write(ring, 0xDEADBEEF);
+        radeon_ring_unlock_commit(rdev, ring);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = readl(ptr);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+
+        if (i < rdev->usec_timeout) {
+                DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+        } else {
+                DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
+                          ring->idx, tmp);
+                r = -EINVAL;
+        }
+        return r;
+}
+
+/**
+ * cik_sdma_ib_test - test an IB on the DMA engine
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ *
+ * Test a simple IB in the DMA ring (CIK).
+ * Returns 0 on success, error on failure.
+ */
+int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        struct radeon_ib ib;
+        unsigned i;
+        int r;
+        void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
+        u32 tmp = 0;
+
+        if (!ptr) {
+                DRM_ERROR("invalid vram scratch pointer\n");
+                return -EINVAL;
+        }
+
+        tmp = 0xCAFEDEAD;
+        writel(tmp, ptr);
+
+        r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
+        if (r) {
+                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
+                return r;
+        }
+
+        ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
+        ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
+        ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff;
+        ib.ptr[3] = 1;
+        ib.ptr[4] = 0xDEADBEEF;
+        ib.length_dw = 5;
+
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r) {
+                radeon_ib_free(rdev, &ib);
+                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
+                return r;
+        }
+        r = radeon_fence_wait(ib.fence, false);
+        if (r) {
+                DRM_ERROR("radeon: fence wait failed (%d).\n", r);
+                return r;
+        }
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = readl(ptr);
+                if (tmp == 0xDEADBEEF)
+                        break;
+                DRM_UDELAY(1);
+        }
+        if (i < rdev->usec_timeout) {
+                DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
+        } else {
+                DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
+                r = -EINVAL;
+        }
+        radeon_ib_free(rdev, &ib);
+        return r;
+}
+
+
+static void cik_print_gpu_status_regs(struct radeon_device *rdev)
+{
+        dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
+                RREG32(GRBM_STATUS));
+        dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
+                RREG32(GRBM_STATUS2));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
+                RREG32(GRBM_STATUS_SE0));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
+                RREG32(GRBM_STATUS_SE1));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
+                RREG32(GRBM_STATUS_SE2));
+        dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
+                RREG32(GRBM_STATUS_SE3));
+        dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
+                RREG32(SRBM_STATUS));
+        dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
+                RREG32(SRBM_STATUS2));
+        dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
+                RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
+        dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
+                 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
+        dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
+        dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
+                 RREG32(CP_STALLED_STAT1));
+        dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
+                 RREG32(CP_STALLED_STAT2));
+        dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
+                 RREG32(CP_STALLED_STAT3));
+        dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
+                 RREG32(CP_CPF_BUSY_STAT));
+        dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
+                 RREG32(CP_CPF_STALLED_STAT1));
+        dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
+        dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
+        dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
+                 RREG32(CP_CPC_STALLED_STAT1));
+        dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
+}
+
+/**
+ * cik_gpu_check_soft_reset - check which blocks are busy
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Check which blocks are busy and return the relevant reset
+ * mask to be used by cik_gpu_soft_reset().
+ * Returns a mask of the blocks to be reset.
+ */
+static u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
+{
+        u32 reset_mask = 0;
+        u32 tmp;
+
+        /* GRBM_STATUS */
+        tmp = RREG32(GRBM_STATUS);
+        if (tmp & (PA_BUSY | SC_BUSY |
+                   BCI_BUSY | SX_BUSY |
+                   TA_BUSY | VGT_BUSY |
+                   DB_BUSY | CB_BUSY |
+                   GDS_BUSY | SPI_BUSY |
+                   IA_BUSY | IA_BUSY_NO_DMA))
+                reset_mask |= RADEON_RESET_GFX;
+
+        if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
+                reset_mask |= RADEON_RESET_CP;
+
+        /* GRBM_STATUS2 */
+        tmp = RREG32(GRBM_STATUS2);
+        if (tmp & RLC_BUSY)
+                reset_mask |= RADEON_RESET_RLC;
+
+        /* SDMA0_STATUS_REG */
+        tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
+        if (!(tmp & SDMA_IDLE))
+                reset_mask |= RADEON_RESET_DMA;
+
+        /* SDMA1_STATUS_REG */
+        tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
+        if (!(tmp & SDMA_IDLE))
+                reset_mask |= RADEON_RESET_DMA1;
+
+        /* SRBM_STATUS2 */
+        tmp = RREG32(SRBM_STATUS2);
+        if (tmp & SDMA_BUSY)
+                reset_mask |= RADEON_RESET_DMA;
+
+        if (tmp & SDMA1_BUSY)
+                reset_mask |= RADEON_RESET_DMA1;
+
+        /* SRBM_STATUS */
+        tmp = RREG32(SRBM_STATUS);
+
+        if (tmp & IH_BUSY)
+                reset_mask |= RADEON_RESET_IH;
+
+        if (tmp & SEM_BUSY)
+                reset_mask |= RADEON_RESET_SEM;
+
+        if (tmp & GRBM_RQ_PENDING)
+                reset_mask |= RADEON_RESET_GRBM;
+
+        if (tmp & VMC_BUSY)
+                reset_mask |= RADEON_RESET_VMC;
+
+        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
+                   MCC_BUSY | MCD_BUSY))
+                reset_mask |= RADEON_RESET_MC;
+
+        if (evergreen_is_display_hung(rdev))
+                reset_mask |= RADEON_RESET_DISPLAY;
+
+        /* Skip MC reset as it's mostly likely not hung, just busy */
+        if (reset_mask & RADEON_RESET_MC) {
+                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
+                reset_mask &= ~RADEON_RESET_MC;
+        }
+
+        return reset_mask;
+}
+
+/**
+ * cik_gpu_soft_reset - soft reset GPU
+ *
+ * @rdev: radeon_device pointer
+ * @reset_mask: mask of which blocks to reset
+ *
+ * Soft reset the blocks specified in @reset_mask.
+ */
+static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
+{
+        struct evergreen_mc_save save;
+        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
+        u32 tmp;
+
+        if (reset_mask == 0)
+                return;
+
+        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
+
+        cik_print_gpu_status_regs(rdev);
+        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
+        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
+
+        /* stop the rlc */
+        cik_rlc_stop(rdev);
+
+        /* Disable GFX parsing/prefetching */
+        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
+
+        /* Disable MEC parsing/prefetching */
+        WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
+
+        if (reset_mask & RADEON_RESET_DMA) {
+                /* sdma0 */
+                tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
+                tmp |= SDMA_HALT;
+                WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
+        }
+        if (reset_mask & RADEON_RESET_DMA1) {
+                /* sdma1 */
+                tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
+                tmp |= SDMA_HALT;
+                WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
+        }
+
+        evergreen_mc_stop(rdev, &save);
+        if (evergreen_mc_wait_for_idle(rdev)) {
+                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
+        }
+
+        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
+                grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
+
+        if (reset_mask & RADEON_RESET_CP) {
+                grbm_soft_reset |= SOFT_RESET_CP;
+
+                srbm_soft_reset |= SOFT_RESET_GRBM;
+        }
+
+        if (reset_mask & RADEON_RESET_DMA)
+                srbm_soft_reset |= SOFT_RESET_SDMA;
+
+        if (reset_mask & RADEON_RESET_DMA1)
+                srbm_soft_reset |= SOFT_RESET_SDMA1;
+
+        if (reset_mask & RADEON_RESET_DISPLAY)
+                srbm_soft_reset |= SOFT_RESET_DC;
+
+        if (reset_mask & RADEON_RESET_RLC)
+                grbm_soft_reset |= SOFT_RESET_RLC;
+
+        if (reset_mask & RADEON_RESET_SEM)
+                srbm_soft_reset |= SOFT_RESET_SEM;
+
+        if (reset_mask & RADEON_RESET_IH)
+                srbm_soft_reset |= SOFT_RESET_IH;
+
+        if (reset_mask & RADEON_RESET_GRBM)
+                srbm_soft_reset |= SOFT_RESET_GRBM;
+
+        if (reset_mask & RADEON_RESET_VMC)
+                srbm_soft_reset |= SOFT_RESET_VMC;
+
+        if (!(rdev->flags & RADEON_IS_IGP)) {
+                if (reset_mask & RADEON_RESET_MC)
+                        srbm_soft_reset |= SOFT_RESET_MC;
+        }
+
+        if (grbm_soft_reset) {
+                tmp = RREG32(GRBM_SOFT_RESET);
+                tmp |= grbm_soft_reset;
+                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
+                WREG32(GRBM_SOFT_RESET, tmp);
+                tmp = RREG32(GRBM_SOFT_RESET);
+
+                udelay(50);
+
+                tmp &= ~grbm_soft_reset;
+                WREG32(GRBM_SOFT_RESET, tmp);
+                tmp = RREG32(GRBM_SOFT_RESET);
+        }
+
+        if (srbm_soft_reset) {
+                tmp = RREG32(SRBM_SOFT_RESET);
+                tmp |= srbm_soft_reset;
+                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
+                WREG32(SRBM_SOFT_RESET, tmp);
+                tmp = RREG32(SRBM_SOFT_RESET);
+
+                udelay(50);
+
+                tmp &= ~srbm_soft_reset;
+                WREG32(SRBM_SOFT_RESET, tmp);
+                tmp = RREG32(SRBM_SOFT_RESET);
+        }
+
+        /* Wait a little for things to settle down */
+        udelay(50);
+
+        evergreen_mc_resume(rdev, &save);
+        udelay(50);
+
+        cik_print_gpu_status_regs(rdev);
+}
+
+/**
+ * cik_asic_reset - soft reset GPU
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Look up which blocks are hung and attempt
+ * to reset them.
+ * Returns 0 for success.
+ */
+int cik_asic_reset(struct radeon_device *rdev)
+{
+        u32 reset_mask;
+
+        reset_mask = cik_gpu_check_soft_reset(rdev);
+
+        if (reset_mask)
+                r600_set_bios_scratch_engine_hung(rdev, true);
+
+        cik_gpu_soft_reset(rdev, reset_mask);
+
+        reset_mask = cik_gpu_check_soft_reset(rdev);
+
+        if (!reset_mask)
+                r600_set_bios_scratch_engine_hung(rdev, false);
+
+        return 0;
+}
+
+/**
+ * cik_gfx_is_lockup - check if the 3D engine is locked up
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ *
+ * Check if the 3D engine is locked up (CIK).
+ * Returns true if the engine is locked, false if not.
+ */
+bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        u32 reset_mask = cik_gpu_check_soft_reset(rdev);
+
+        if (!(reset_mask & (RADEON_RESET_GFX |
+                            RADEON_RESET_COMPUTE |
+                            RADEON_RESET_CP))) {
+                radeon_ring_lockup_update(ring);
+                return false;
+        }
+        /* force CP activities */
+        radeon_ring_force_activity(rdev, ring);
+        return radeon_ring_test_lockup(rdev, ring);
+}
+
+/**
+ * cik_sdma_is_lockup - Check if the DMA engine is locked up
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring structure holding ring information
+ *
+ * Check if the async DMA engine is locked up (CIK).
+ * Returns true if the engine appears to be locked up, false if not.
+ */
+bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        u32 reset_mask = cik_gpu_check_soft_reset(rdev);
+        u32 mask;
+
+        if (ring->idx == R600_RING_TYPE_DMA_INDEX)
+                mask = RADEON_RESET_DMA;
+        else
+                mask = RADEON_RESET_DMA1;
+
+        if (!(reset_mask & mask)) {
+                radeon_ring_lockup_update(ring);
+                return false;
+        }
+        /* force ring activities */
+        radeon_ring_force_activity(rdev, ring);
+        return radeon_ring_test_lockup(rdev, ring);
+}
+
+/* MC */
+/**
+ * cik_mc_program - program the GPU memory controller
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Set the location of vram, gart, and AGP in the GPU's
+ * physical address space (CIK).
+ */
+static void cik_mc_program(struct radeon_device *rdev)
+{
+        struct evergreen_mc_save save;
+        u32 tmp;
+        int i, j;
+
+        /* Initialize HDP */
+        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
+                WREG32((0x2c14 + j), 0x00000000);
+                WREG32((0x2c18 + j), 0x00000000);
+                WREG32((0x2c1c + j), 0x00000000);
+                WREG32((0x2c20 + j), 0x00000000);
+                WREG32((0x2c24 + j), 0x00000000);
+        }
+        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
+
+        evergreen_mc_stop(rdev, &save);
+        if (radeon_mc_wait_for_idle(rdev)) {
+                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
+        }
+        /* Lockout access through VGA aperture*/
+        WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
+        /* Update configuration */
+        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
+               rdev->mc.vram_start >> 12);
+        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+               rdev->mc.vram_end >> 12);
+        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
+               rdev->vram_scratch.gpu_addr >> 12);
+        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
+        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
+        WREG32(MC_VM_FB_LOCATION, tmp);
+        /* XXX double check these! */
+        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
+        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
+        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
+        WREG32(MC_VM_AGP_BASE, 0);
+        WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
+        WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
+        if (radeon_mc_wait_for_idle(rdev)) {
+                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
+        }
+        evergreen_mc_resume(rdev, &save);
+        /* we need to own VRAM, so turn off the VGA renderer here
+         * to stop it overwriting our objects */
+        rv515_vga_render_disable(rdev);
+}
+
+/**
+ * cik_mc_init - initialize the memory controller driver params
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Look up the amount of vram, vram width, and decide how to place
+ * vram and gart within the GPU's physical address space (CIK).
+ * Returns 0 for success.
+ */
+static int cik_mc_init(struct radeon_device *rdev)
+{
+        u32 tmp;
+        int chansize, numchan;
+
+        /* Get VRAM informations */
+        rdev->mc.vram_is_ddr = true;
+        tmp = RREG32(MC_ARB_RAMCFG);
+        if (tmp & CHANSIZE_MASK) {
+                chansize = 64;
+        } else {
+                chansize = 32;
+        }
+        tmp = RREG32(MC_SHARED_CHMAP);
+        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+        case 0:
+        default:
+                numchan = 1;
+                break;
+        case 1:
+                numchan = 2;
+                break;
+        case 2:
+                numchan = 4;
+                break;
+        case 3:
+                numchan = 8;
+                break;
+        case 4:
+                numchan = 3;
+                break;
+        case 5:
+                numchan = 6;
+                break;
+        case 6:
+                numchan = 10;
+                break;
+        case 7:
+                numchan = 12;
+                break;
+        case 8:
+                numchan = 16;
+                break;
+        }
+        rdev->mc.vram_width = numchan * chansize;
+        /* Could aper size report 0 ? */
+        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
+        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
+        /* size in MB on si */
+        rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
+        rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
+        rdev->mc.visible_vram_size = rdev->mc.aper_size;
+        si_vram_gtt_location(rdev, &rdev->mc);
+        radeon_update_bandwidth_info(rdev);
+
+        return 0;
+}
+
+/*
+ * GART
+ * VMID 0 is the physical GPU addresses as used by the kernel.
+ * VMIDs 1-15 are used for userspace clients and are handled
+ * by the radeon vm/hsa code.
+ */
+/**
+ * cik_pcie_gart_tlb_flush - gart tlb flush callback
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Flush the TLB for the VMID 0 page table (CIK).
+ */
+void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
+{
+        /* flush hdp cache */
+        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+
+        /* bits 0-15 are the VM contexts0-15 */
+        WREG32(VM_INVALIDATE_REQUEST, 0x1);
+}
+
+/**
+ * cik_pcie_gart_enable - gart enable
+ *
+ * @rdev: radeon_device pointer
+ *
+ * This sets up the TLBs, programs the page tables for VMID0,
+ * sets up the hw for VMIDs 1-15 which are allocated on
+ * demand, and sets up the global locations for the LDS, GDS,
+ * and GPUVM for FSA64 clients (CIK).
+ * Returns 0 for success, errors for failure.
+ */
+static int cik_pcie_gart_enable(struct radeon_device *rdev)
+{
+        int r, i;
+
+        if (rdev->gart.robj == NULL) {
+                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
+                return -EINVAL;
+        }
+        r = radeon_gart_table_vram_pin(rdev);
+        if (r)
+                return r;
+        radeon_gart_restore(rdev);
+        /* Setup TLB control */
+        WREG32(MC_VM_MX_L1_TLB_CNTL,
+               (0xA << 7) |
+               ENABLE_L1_TLB |
+               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+               ENABLE_ADVANCED_DRIVER_MODEL |
+               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+        /* Setup L2 cache */
+        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
+               ENABLE_L2_FRAGMENT_PROCESSING |
+               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
+               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
+               EFFECTIVE_L2_QUEUE_SIZE(7) |
+               CONTEXT1_IDENTITY_ACCESS_MODE(1));
+        WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
+        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
+               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
+        /* setup context0 */
+        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
+        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
+        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
+                        (u32)(rdev->dummy_page.addr >> 12));
+        WREG32(VM_CONTEXT0_CNTL2, 0);
+        WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+                                  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
+
+        WREG32(0x15D4, 0);
+        WREG32(0x15D8, 0);
+        WREG32(0x15DC, 0);
+
+        /* empty context1-15 */
+        /* FIXME start with 4G, once using 2 level pt switch to full
+         * vm size space
+         */
+        /* set vm size, must be a multiple of 4 */
+        WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
+        WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
+        for (i = 1; i < 16; i++) {
+                if (i < 8)
+                        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
+                               rdev->gart.table_addr >> 12);
+                else
+                        WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
+                               rdev->gart.table_addr >> 12);
+        }
+
+        /* enable context1-15 */
+        WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
+               (u32)(rdev->dummy_page.addr >> 12));
+        WREG32(VM_CONTEXT1_CNTL2, 4);
+        WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
+                                RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
+                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
+                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
+                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
+                                PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
+                                PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
+                                VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
+                                VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
+                                READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
+                                READ_PROTECTION_FAULT_ENABLE_DEFAULT |
+                                WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
+                                WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
+
+        /* TC cache setup ??? */
+        WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
+        WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
+        WREG32(TC_CFG_L1_STORE_POLICY, 0);
+
+        WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
+        WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
+        WREG32(TC_CFG_L2_STORE_POLICY0, 0);
+        WREG32(TC_CFG_L2_STORE_POLICY1, 0);
+        WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
+
+        WREG32(TC_CFG_L1_VOLATILE, 0);
+        WREG32(TC_CFG_L2_VOLATILE, 0);
+
+        if (rdev->family == CHIP_KAVERI) {
+                u32 tmp = RREG32(CHUB_CONTROL);
+                tmp &= ~BYPASS_VM;
+                WREG32(CHUB_CONTROL, tmp);
+        }
+
+        /* XXX SH_MEM regs */
+        /* where to put LDS, scratch, GPUVM in FSA64 space */
+        for (i = 0; i < 16; i++) {
+                cik_srbm_select(rdev, 0, 0, 0, i);
+                /* CP and shaders */
+                WREG32(SH_MEM_CONFIG, 0);
+                WREG32(SH_MEM_APE1_BASE, 1);
+                WREG32(SH_MEM_APE1_LIMIT, 0);
+                WREG32(SH_MEM_BASES, 0);
+                /* SDMA GFX */
+                WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
+                WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
+                WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
+                WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
+                /* XXX SDMA RLC - todo */
+        }
+        cik_srbm_select(rdev, 0, 0, 0, 0);
+
+        cik_pcie_gart_tlb_flush(rdev);
+        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
+                 (unsigned)(rdev->mc.gtt_size >> 20),
+                 (unsigned long long)rdev->gart.table_addr);
+        rdev->gart.ready = true;
+        return 0;
+}
+
+/**
+ * cik_pcie_gart_disable - gart disable
+ *
+ * @rdev: radeon_device pointer
+ *
+ * This disables all VM page table (CIK).
+ */
+static void cik_pcie_gart_disable(struct radeon_device *rdev)
+{
+        /* Disable all tables */
+        WREG32(VM_CONTEXT0_CNTL, 0);
+        WREG32(VM_CONTEXT1_CNTL, 0);
+        /* Setup TLB control */
+        WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+        /* Setup L2 cache */
+        WREG32(VM_L2_CNTL,
+               ENABLE_L2_FRAGMENT_PROCESSING |
+               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
+               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
+               EFFECTIVE_L2_QUEUE_SIZE(7) |
+               CONTEXT1_IDENTITY_ACCESS_MODE(1));
+        WREG32(VM_L2_CNTL2, 0);
+        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
+               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
+        radeon_gart_table_vram_unpin(rdev);
+}
+
+/**
+ * cik_pcie_gart_fini - vm fini callback
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Tears down the driver GART/VM setup (CIK).
+ */
+static void cik_pcie_gart_fini(struct radeon_device *rdev)
+{
+        cik_pcie_gart_disable(rdev);
+        radeon_gart_table_vram_free(rdev);
+        radeon_gart_fini(rdev);
+}
+
+/* vm parser */
+/**
+ * cik_ib_parse - vm ib_parse callback
+ *
+ * @rdev: radeon_device pointer
+ * @ib: indirect buffer pointer
+ *
+ * CIK uses hw IB checking so this is a nop (CIK).
+ */
+int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        return 0;
+}
+
+/*
+ * vm
+ * VMID 0 is the physical GPU addresses as used by the kernel.
+ * VMIDs 1-15 are used for userspace clients and are handled
+ * by the radeon vm/hsa code.
+ */
+/**
+ * cik_vm_init - cik vm init callback
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Inits cik specific vm parameters (number of VMs, base of vram for
+ * VMIDs 1-15) (CIK).
+ * Returns 0 for success.
+ */
+int cik_vm_init(struct radeon_device *rdev)
+{
+        /* number of VMs */
+        rdev->vm_manager.nvm = 16;
+        /* base offset of vram pages */
+        if (rdev->flags & RADEON_IS_IGP) {
+                u64 tmp = RREG32(MC_VM_FB_OFFSET);
+                tmp <<= 22;
+                rdev->vm_manager.vram_base_offset = tmp;
+        } else
+                rdev->vm_manager.vram_base_offset = 0;
+
+        return 0;
+}
+
+/**
+ * cik_vm_fini - cik vm fini callback
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Tear down any asic specific VM setup (CIK).
+ */
+void cik_vm_fini(struct radeon_device *rdev)
+{
+}
+
+/**
+ * cik_vm_flush - cik vm flush using the CP
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Update the page table base and flush the VM TLB
+ * using the CP (CIK).
+ */
+void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
+{
+        struct radeon_ring *ring = &rdev->ring[ridx];
+
+        if (vm == NULL)
+                return;
+
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        if (vm->id < 8) {
+                radeon_ring_write(ring,
+                                  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
+        } else {
+                radeon_ring_write(ring,
+                                  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
+        }
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
+
+        /* update SH_MEM_* regs */
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, VMID(vm->id));
+
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, SH_MEM_BASES >> 2);
+        radeon_ring_write(ring, 0);
+
+        radeon_ring_write(ring, 0); /* SH_MEM_BASES */
+        radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
+        radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
+        radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
+
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, VMID(0));
+
+        /* HDP flush */
+        /* We should be using the WAIT_REG_MEM packet here like in
+         * cik_fence_ring_emit(), but it causes the CP to hang in this
+         * context...
+         */
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 0);
+
+        /* bits 0-15 are the VM contexts0-15 */
+        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+                                 WRITE_DATA_DST_SEL(0)));
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 0);
+        radeon_ring_write(ring, 1 << vm->id);
+
+        /* compute doesn't have PFP */
+        if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
+                /* sync PFP to ME, otherwise we might get invalid PFP reads */
+                radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
+                radeon_ring_write(ring, 0x0);
+        }
+}
+
+/**
+ * cik_vm_set_page - update the page tables using sDMA
+ *
+ * @rdev: radeon_device pointer
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using CP or sDMA (CIK).
+ */
+void cik_vm_set_page(struct radeon_device *rdev,
+                     struct radeon_ib *ib,
+                     uint64_t pe,
+                     uint64_t addr, unsigned count,
+                     uint32_t incr, uint32_t flags)
+{
+        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
+        uint64_t value;
+        unsigned ndw;
+
+        if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
+                /* CP */
+                while (count) {
+                        ndw = 2 + count * 2;
+                        if (ndw > 0x3FFE)
+                                ndw = 0x3FFE;
+
+                        ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
+                        ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
+                                                    WRITE_DATA_DST_SEL(1));
+                        ib->ptr[ib->length_dw++] = pe;
+                        ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+                        for (; ndw > 2; ndw -= 2, --count, pe += 8) {
+                                if (flags & RADEON_VM_PAGE_SYSTEM) {
+                                        value = radeon_vm_map_gart(rdev, addr);
+                                        value &= 0xFFFFFFFFFFFFF000ULL;
+                                } else if (flags & RADEON_VM_PAGE_VALID) {
+                                        value = addr;
+                                } else {
+                                        value = 0;
+                                }
+                                addr += incr;
+                                value |= r600_flags;
+                                ib->ptr[ib->length_dw++] = value;
+                                ib->ptr[ib->length_dw++] = upper_32_bits(value);
+                        }
+                }
+        } else {
+                /* DMA */
+                if (flags & RADEON_VM_PAGE_SYSTEM) {
+                        while (count) {
+                                ndw = count * 2;
+                                if (ndw > 0xFFFFE)
+                                        ndw = 0xFFFFE;
+
+                                /* for non-physically contiguous pages (system) */
+                                ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
+                                ib->ptr[ib->length_dw++] = pe;
+                                ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+                                ib->ptr[ib->length_dw++] = ndw;
+                                for (; ndw > 0; ndw -= 2, --count, pe += 8) {
+                                        if (flags & RADEON_VM_PAGE_SYSTEM) {
+                                                value = radeon_vm_map_gart(rdev, addr);
+                                                value &= 0xFFFFFFFFFFFFF000ULL;
+                                        } else if (flags & RADEON_VM_PAGE_VALID) {
+                                                value = addr;
+                                        } else {
+                                                value = 0;
+                                        }
+                                        addr += incr;
+                                        value |= r600_flags;
+                                        ib->ptr[ib->length_dw++] = value;
+                                        ib->ptr[ib->length_dw++] = upper_32_bits(value);
+                                }
+                        }
+                } else {
+                        while (count) {
+                                ndw = count;
+                                if (ndw > 0x7FFFF)
+                                        ndw = 0x7FFFF;
+
+                                if (flags & RADEON_VM_PAGE_VALID)
+                                        value = addr;
+                                else
+                                        value = 0;
+                                /* for physically contiguous pages (vram) */
+                                ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
+                                ib->ptr[ib->length_dw++] = pe; /* dst addr */
+                                ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+                                ib->ptr[ib->length_dw++] = r600_flags; /* mask */
+                                ib->ptr[ib->length_dw++] = 0;
+                                ib->ptr[ib->length_dw++] = value; /* value */
+                                ib->ptr[ib->length_dw++] = upper_32_bits(value);
+                                ib->ptr[ib->length_dw++] = incr; /* increment size */
+                                ib->ptr[ib->length_dw++] = 0;
+                                ib->ptr[ib->length_dw++] = ndw; /* number of entries */
+                                pe += ndw * 8;
+                                addr += ndw * incr;
+                                count -= ndw;
+                        }
+                }
+                while (ib->length_dw & 0x7)
+                        ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
+        }
+}
+
+/**
+ * cik_dma_vm_flush - cik vm flush using sDMA
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Update the page table base and flush the VM TLB
+ * using sDMA (CIK).
+ */
+void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
+{
+        struct radeon_ring *ring = &rdev->ring[ridx];
+        u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
+                          SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
+        u32 ref_and_mask;
+
+        if (vm == NULL)
+                return;
+
+        if (ridx == R600_RING_TYPE_DMA_INDEX)
+                ref_and_mask = SDMA0;
+        else
+                ref_and_mask = SDMA1;
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        if (vm->id < 8) {
+                radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
+        } else {
+                radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
+        }
+        radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
+
+        /* update SH_MEM_* regs */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
+        radeon_ring_write(ring, VMID(vm->id));
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, SH_MEM_BASES >> 2);
+        radeon_ring_write(ring, 0);
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, SH_MEM_CONFIG >> 2);
+        radeon_ring_write(ring, 0);
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);
+        radeon_ring_write(ring, 1);
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);
+        radeon_ring_write(ring, 0);
+
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
+        radeon_ring_write(ring, VMID(0));
+
+        /* flush HDP */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
+        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
+        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
+        radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
+        radeon_ring_write(ring, ref_and_mask); /* MASK */
+        radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
+
+        /* flush TLB */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+        radeon_ring_write(ring, 1 << vm->id);
+}
+
+/*
+ * RLC
+ * The RLC is a multi-purpose microengine that handles a
+ * variety of functions, the most important of which is
+ * the interrupt controller.
+ */
+/**
+ * cik_rlc_stop - stop the RLC ME
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Halt the RLC ME (MicroEngine) (CIK).
+ */
+static void cik_rlc_stop(struct radeon_device *rdev)
+{
+        int i, j, k;
+        u32 mask, tmp;
+
+        tmp = RREG32(CP_INT_CNTL_RING0);
+        tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+        WREG32(CP_INT_CNTL_RING0, tmp);
+
+        RREG32(CB_CGTT_SCLK_CTRL);
+        RREG32(CB_CGTT_SCLK_CTRL);
+        RREG32(CB_CGTT_SCLK_CTRL);
+        RREG32(CB_CGTT_SCLK_CTRL);
+
+        tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
+        WREG32(RLC_CGCG_CGLS_CTRL, tmp);
+
+        WREG32(RLC_CNTL, 0);
+
+        for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
+                for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
+                        cik_select_se_sh(rdev, i, j);
+                        for (k = 0; k < rdev->usec_timeout; k++) {
+                                if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
+                                        break;
+                                udelay(1);
+                        }
+                }
+        }
+        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
+
+        mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
+        for (k = 0; k < rdev->usec_timeout; k++) {
+                if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
+                        break;
+                udelay(1);
+        }
+}
+
+/**
+ * cik_rlc_start - start the RLC ME
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Unhalt the RLC ME (MicroEngine) (CIK).
+ */
+static void cik_rlc_start(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        WREG32(RLC_CNTL, RLC_ENABLE);
+
+        tmp = RREG32(CP_INT_CNTL_RING0);
+        tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+        WREG32(CP_INT_CNTL_RING0, tmp);
+
+        udelay(50);
+}
+
+/**
+ * cik_rlc_resume - setup the RLC hw
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Initialize the RLC registers, load the ucode,
+ * and start the RLC (CIK).
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int cik_rlc_resume(struct radeon_device *rdev)
+{
+        u32 i, size;
+        u32 clear_state_info[3];
+        const __be32 *fw_data;
+
+        if (!rdev->rlc_fw)
+                return -EINVAL;
+
+        switch (rdev->family) {
+        case CHIP_BONAIRE:
+        default:
+                size = BONAIRE_RLC_UCODE_SIZE;
+                break;
+        case CHIP_KAVERI:
+                size = KV_RLC_UCODE_SIZE;
+                break;
+        case CHIP_KABINI:
+                size = KB_RLC_UCODE_SIZE;
+                break;
+        }
+
+        cik_rlc_stop(rdev);
+
+        WREG32(GRBM_SOFT_RESET, SOFT_RESET_RLC);
+        RREG32(GRBM_SOFT_RESET);
+        udelay(50);
+        WREG32(GRBM_SOFT_RESET, 0);
+        RREG32(GRBM_SOFT_RESET);
+        udelay(50);
+
+        WREG32(RLC_LB_CNTR_INIT, 0);
+        WREG32(RLC_LB_CNTR_MAX, 0x00008000);
+
+        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
+        WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
+        WREG32(RLC_LB_PARAMS, 0x00600408);
+        WREG32(RLC_LB_CNTL, 0x80000004);
+
+        WREG32(RLC_MC_CNTL, 0);
+        WREG32(RLC_UCODE_CNTL, 0);
+
+        fw_data = (const __be32 *)rdev->rlc_fw->data;
+                WREG32(RLC_GPM_UCODE_ADDR, 0);
+        for (i = 0; i < size; i++)
+                WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
+        WREG32(RLC_GPM_UCODE_ADDR, 0);
+
+        /* XXX */
+        clear_state_info[0] = 0;//upper_32_bits(rdev->rlc.save_restore_gpu_addr);
+        clear_state_info[1] = 0;//rdev->rlc.save_restore_gpu_addr;
+        clear_state_info[2] = 0;//cik_default_size;
+        WREG32(RLC_GPM_SCRATCH_ADDR, 0x3d);
+        for (i = 0; i < 3; i++)
+                WREG32(RLC_GPM_SCRATCH_DATA, clear_state_info[i]);
+        WREG32(RLC_DRIVER_DMA_STATUS, 0);
+
+        cik_rlc_start(rdev);
+
+        return 0;
+}
+
+/*
+ * Interrupts
+ * Starting with r6xx, interrupts are handled via a ring buffer.
+ * Ring buffers are areas of GPU accessible memory that the GPU
+ * writes interrupt vectors into and the host reads vectors out of.
+ * There is a rptr (read pointer) that determines where the
+ * host is currently reading, and a wptr (write pointer)
+ * which determines where the GPU has written.  When the
+ * pointers are equal, the ring is idle.  When the GPU
+ * writes vectors to the ring buffer, it increments the
+ * wptr.  When there is an interrupt, the host then starts
+ * fetching commands and processing them until the pointers are
+ * equal again at which point it updates the rptr.
+ */
+
+/**
+ * cik_enable_interrupts - Enable the interrupt ring buffer
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Enable the interrupt ring buffer (CIK).
+ */
+static void cik_enable_interrupts(struct radeon_device *rdev)
+{
+        u32 ih_cntl = RREG32(IH_CNTL);
+        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
+
+        ih_cntl |= ENABLE_INTR;
+        ih_rb_cntl |= IH_RB_ENABLE;
+        WREG32(IH_CNTL, ih_cntl);
+        WREG32(IH_RB_CNTL, ih_rb_cntl);
+        rdev->ih.enabled = true;
+}
+
+/**
+ * cik_disable_interrupts - Disable the interrupt ring buffer
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Disable the interrupt ring buffer (CIK).
+ */
+static void cik_disable_interrupts(struct radeon_device *rdev)
+{
+        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
+        u32 ih_cntl = RREG32(IH_CNTL);
+
+        ih_rb_cntl &= ~IH_RB_ENABLE;
+        ih_cntl &= ~ENABLE_INTR;
+        WREG32(IH_RB_CNTL, ih_rb_cntl);
+        WREG32(IH_CNTL, ih_cntl);
+        /* set rptr, wptr to 0 */
+        WREG32(IH_RB_RPTR, 0);
+        WREG32(IH_RB_WPTR, 0);
+        rdev->ih.enabled = false;
+        rdev->ih.rptr = 0;
+}
+
+/**
+ * cik_disable_interrupt_state - Disable all interrupt sources
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Clear all interrupt enable bits used by the driver (CIK).
+ */
+static void cik_disable_interrupt_state(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        /* gfx ring */
+        WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+        /* sdma */
+        tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
+        WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
+        tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
+        WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
+        /* compute queues */
+        WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
+        WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
+        WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
+        WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
+        WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
+        WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
+        WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
+        WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
+        /* grbm */
+        WREG32(GRBM_INT_CNTL, 0);
+        /* vline/vblank, etc. */
+        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
+        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
+        if (rdev->num_crtc >= 4) {
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+        }
+
+        /* dac hotplug */
+        WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
+
+        /* digital hotplug */
+        tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
+        WREG32(DC_HPD1_INT_CONTROL, tmp);
+        tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
+        WREG32(DC_HPD2_INT_CONTROL, tmp);
+        tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
+        WREG32(DC_HPD3_INT_CONTROL, tmp);
+        tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
+        WREG32(DC_HPD4_INT_CONTROL, tmp);
+        tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
+        WREG32(DC_HPD5_INT_CONTROL, tmp);
+        tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
+        WREG32(DC_HPD6_INT_CONTROL, tmp);
+
+}
+
+/**
+ * cik_irq_init - init and enable the interrupt ring
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Allocate a ring buffer for the interrupt controller,
+ * enable the RLC, disable interrupts, enable the IH
+ * ring buffer and enable it (CIK).
+ * Called at device load and reume.
+ * Returns 0 for success, errors for failure.
+ */
+static int cik_irq_init(struct radeon_device *rdev)
+{
+        int ret = 0;
+        int rb_bufsz;
+        u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
+
+        /* allocate ring */
+        ret = r600_ih_ring_alloc(rdev);
+        if (ret)
+                return ret;
+
+        /* disable irqs */
+        cik_disable_interrupts(rdev);
+
+        /* init rlc */
+        ret = cik_rlc_resume(rdev);
+        if (ret) {
+                r600_ih_ring_fini(rdev);
+                return ret;
+        }
+
+        /* setup interrupt control */
+        /* XXX this should actually be a bus address, not an MC address. same on older asics */
+        WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
+        interrupt_cntl = RREG32(INTERRUPT_CNTL);
+        /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
+         * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
+         */
+        interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
+        /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
+        interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
+        WREG32(INTERRUPT_CNTL, interrupt_cntl);
+
+        WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
+        rb_bufsz = drm_order(rdev->ih.ring_size / 4);
+
+        ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
+                      IH_WPTR_OVERFLOW_CLEAR |
+                      (rb_bufsz << 1));
+
+        if (rdev->wb.enabled)
+                ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
+
+        /* set the writeback address whether it's enabled or not */
+        WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
+        WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
+
+        WREG32(IH_RB_CNTL, ih_rb_cntl);
+
+        /* set rptr, wptr to 0 */
+        WREG32(IH_RB_RPTR, 0);
+        WREG32(IH_RB_WPTR, 0);
+
+        /* Default settings for IH_CNTL (disabled at first) */
+        ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
+        /* RPTR_REARM only works if msi's are enabled */
+        if (rdev->msi_enabled)
+                ih_cntl |= RPTR_REARM;
+        WREG32(IH_CNTL, ih_cntl);
+
+        /* force the active interrupt state to all disabled */
+        cik_disable_interrupt_state(rdev);
+
+        pci_set_master(rdev->pdev);
+
+        /* enable irqs */
+        cik_enable_interrupts(rdev);
+
+        return ret;
+}
+
+/**
+ * cik_irq_set - enable/disable interrupt sources
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Enable interrupt sources on the GPU (vblanks, hpd,
+ * etc.) (CIK).
+ * Returns 0 for success, errors for failure.
+ */
+int cik_irq_set(struct radeon_device *rdev)
+{
+        u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE |
+                PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
+        u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
+        u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
+        u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
+        u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
+        u32 grbm_int_cntl = 0;
+        u32 dma_cntl, dma_cntl1;
+
+        if (!rdev->irq.installed) {
+                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
+                return -EINVAL;
+        }
+        /* don't enable anything if the ih is disabled */
+        if (!rdev->ih.enabled) {
+                cik_disable_interrupts(rdev);
+                /* force the active interrupt state to all disabled */
+                cik_disable_interrupt_state(rdev);
+                return 0;
+        }
+
+        hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
+        hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
+        hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
+        hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
+        hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
+        hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
+
+        dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
+        dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
+
+        cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+        cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+
+        /* enable CP interrupts on all rings */
+        if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
+                DRM_DEBUG("cik_irq_set: sw int gfx\n");
+                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
+        }
+        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
+                struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
+                DRM_DEBUG("si_irq_set: sw int cp1\n");
+                if (ring->me == 1) {
+                        switch (ring->pipe) {
+                        case 0:
+                                cp_m1p0 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 1:
+                                cp_m1p1 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 2:
+                                cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 3:
+                                cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        default:
+                                DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
+                                break;
+                        }
+                } else if (ring->me == 2) {
+                        switch (ring->pipe) {
+                        case 0:
+                                cp_m2p0 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 1:
+                                cp_m2p1 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 2:
+                                cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 3:
+                                cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        default:
+                                DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
+                                break;
+                        }
+                } else {
+                        DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
+                }
+        }
+        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
+                struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
+                DRM_DEBUG("si_irq_set: sw int cp2\n");
+                if (ring->me == 1) {
+                        switch (ring->pipe) {
+                        case 0:
+                                cp_m1p0 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 1:
+                                cp_m1p1 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 2:
+                                cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 3:
+                                cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        default:
+                                DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
+                                break;
+                        }
+                } else if (ring->me == 2) {
+                        switch (ring->pipe) {
+                        case 0:
+                                cp_m2p0 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 1:
+                                cp_m2p1 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 2:
+                                cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        case 3:
+                                cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+                                break;
+                        default:
+                                DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
+                                break;
+                        }
+                } else {
+                        DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
+                }
+        }
+
+        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
+                DRM_DEBUG("cik_irq_set: sw int dma\n");
+                dma_cntl |= TRAP_ENABLE;
+        }
+
+        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
+                DRM_DEBUG("cik_irq_set: sw int dma1\n");
+                dma_cntl1 |= TRAP_ENABLE;
+        }
+
+        if (rdev->irq.crtc_vblank_int[0] ||
+            atomic_read(&rdev->irq.pflip[0])) {
+                DRM_DEBUG("cik_irq_set: vblank 0\n");
+                crtc1 |= VBLANK_INTERRUPT_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[1] ||
+            atomic_read(&rdev->irq.pflip[1])) {
+                DRM_DEBUG("cik_irq_set: vblank 1\n");
+                crtc2 |= VBLANK_INTERRUPT_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[2] ||
+            atomic_read(&rdev->irq.pflip[2])) {
+                DRM_DEBUG("cik_irq_set: vblank 2\n");
+                crtc3 |= VBLANK_INTERRUPT_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[3] ||
+            atomic_read(&rdev->irq.pflip[3])) {
+                DRM_DEBUG("cik_irq_set: vblank 3\n");
+                crtc4 |= VBLANK_INTERRUPT_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[4] ||
+            atomic_read(&rdev->irq.pflip[4])) {
+                DRM_DEBUG("cik_irq_set: vblank 4\n");
+                crtc5 |= VBLANK_INTERRUPT_MASK;
+        }
+        if (rdev->irq.crtc_vblank_int[5] ||
+            atomic_read(&rdev->irq.pflip[5])) {
+                DRM_DEBUG("cik_irq_set: vblank 5\n");
+                crtc6 |= VBLANK_INTERRUPT_MASK;
+        }
+        if (rdev->irq.hpd[0]) {
+                DRM_DEBUG("cik_irq_set: hpd 1\n");
+                hpd1 |= DC_HPDx_INT_EN;
+        }
+        if (rdev->irq.hpd[1]) {
+                DRM_DEBUG("cik_irq_set: hpd 2\n");
+                hpd2 |= DC_HPDx_INT_EN;
+        }
+        if (rdev->irq.hpd[2]) {
+                DRM_DEBUG("cik_irq_set: hpd 3\n");
+                hpd3 |= DC_HPDx_INT_EN;
+        }
+        if (rdev->irq.hpd[3]) {
+                DRM_DEBUG("cik_irq_set: hpd 4\n");
+                hpd4 |= DC_HPDx_INT_EN;
+        }
+        if (rdev->irq.hpd[4]) {
+                DRM_DEBUG("cik_irq_set: hpd 5\n");
+                hpd5 |= DC_HPDx_INT_EN;
+        }
+        if (rdev->irq.hpd[5]) {
+                DRM_DEBUG("cik_irq_set: hpd 6\n");
+                hpd6 |= DC_HPDx_INT_EN;
+        }
+
+        WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+
+        WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
+        WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
+
+        WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
+        WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
+        WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
+        WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
+        WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
+        WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
+        WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
+        WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
+
+        WREG32(GRBM_INT_CNTL, grbm_int_cntl);
+
+        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
+        WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
+        if (rdev->num_crtc >= 4) {
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
+        }
+        if (rdev->num_crtc >= 6) {
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
+                WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
+        }
+
+        WREG32(DC_HPD1_INT_CONTROL, hpd1);
+        WREG32(DC_HPD2_INT_CONTROL, hpd2);
+        WREG32(DC_HPD3_INT_CONTROL, hpd3);
+        WREG32(DC_HPD4_INT_CONTROL, hpd4);
+        WREG32(DC_HPD5_INT_CONTROL, hpd5);
+        WREG32(DC_HPD6_INT_CONTROL, hpd6);
+
+        return 0;
+}
+
+/**
+ * cik_irq_ack - ack interrupt sources
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Ack interrupt sources on the GPU (vblanks, hpd,
+ * etc.) (CIK).  Certain interrupts sources are sw
+ * generated and do not require an explicit ack.
+ */
+static inline void cik_irq_ack(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
+        rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
+        rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
+        rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
+        rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
+        rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
+        rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
+
+        if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
+                WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
+        if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
+                WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
+        if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
+                WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
+        if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
+                WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
+
+        if (rdev->num_crtc >= 4) {
+                if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
+                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
+                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
+                if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
+                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
+                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
+        }
+
+        if (rdev->num_crtc >= 6) {
+                if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
+                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
+                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
+                if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
+                        WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
+                if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
+                        WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
+        }
+
+        if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
+                tmp = RREG32(DC_HPD1_INT_CONTROL);
+                tmp |= DC_HPDx_INT_ACK;
+                WREG32(DC_HPD1_INT_CONTROL, tmp);
+        }
+        if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
+                tmp = RREG32(DC_HPD2_INT_CONTROL);
+                tmp |= DC_HPDx_INT_ACK;
+                WREG32(DC_HPD2_INT_CONTROL, tmp);
+        }
+        if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
+                tmp = RREG32(DC_HPD3_INT_CONTROL);
+                tmp |= DC_HPDx_INT_ACK;
+                WREG32(DC_HPD3_INT_CONTROL, tmp);
+        }
+        if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
+                tmp = RREG32(DC_HPD4_INT_CONTROL);
+                tmp |= DC_HPDx_INT_ACK;
+                WREG32(DC_HPD4_INT_CONTROL, tmp);
+        }
+        if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
+                tmp = RREG32(DC_HPD5_INT_CONTROL);
+                tmp |= DC_HPDx_INT_ACK;
+                WREG32(DC_HPD5_INT_CONTROL, tmp);
+        }
+        if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
+                tmp = RREG32(DC_HPD5_INT_CONTROL);
+                tmp |= DC_HPDx_INT_ACK;
+                WREG32(DC_HPD6_INT_CONTROL, tmp);
+        }
+}
+
+/**
+ * cik_irq_disable - disable interrupts
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Disable interrupts on the hw (CIK).
+ */
+static void cik_irq_disable(struct radeon_device *rdev)
+{
+        cik_disable_interrupts(rdev);
+        /* Wait and acknowledge irq */
+        mdelay(1);
+        cik_irq_ack(rdev);
+        cik_disable_interrupt_state(rdev);
+}
+
+/**
+ * cik_irq_disable - disable interrupts for suspend
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Disable interrupts and stop the RLC (CIK).
+ * Used for suspend.
+ */
+static void cik_irq_suspend(struct radeon_device *rdev)
+{
+        cik_irq_disable(rdev);
+        cik_rlc_stop(rdev);
+}
+
+/**
+ * cik_irq_fini - tear down interrupt support
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Disable interrupts on the hw and free the IH ring
+ * buffer (CIK).
+ * Used for driver unload.
+ */
+static void cik_irq_fini(struct radeon_device *rdev)
+{
+        cik_irq_suspend(rdev);
+        r600_ih_ring_fini(rdev);
+}
+
+/**
+ * cik_get_ih_wptr - get the IH ring buffer wptr
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Get the IH ring buffer wptr from either the register
+ * or the writeback memory buffer (CIK).  Also check for
+ * ring buffer overflow and deal with it.
+ * Used by cik_irq_process().
+ * Returns the value of the wptr.
+ */
+static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
+{
+        u32 wptr, tmp;
+
+        if (rdev->wb.enabled)
+                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
+        else
+                wptr = RREG32(IH_RB_WPTR);
+
+        if (wptr & RB_OVERFLOW) {
+                /* When a ring buffer overflow happen start parsing interrupt
+                 * from the last not overwritten vector (wptr + 16). Hopefully
+                 * this should allow us to catchup.
+                 */
+                dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
+                        wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
+                rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
+                tmp = RREG32(IH_RB_CNTL);
+                tmp |= IH_WPTR_OVERFLOW_CLEAR;
+                WREG32(IH_RB_CNTL, tmp);
+        }
+        return (wptr & rdev->ih.ptr_mask);
+}
+
+/*        CIK IV Ring
+ * Each IV ring entry is 128 bits:
+ * [7:0]    - interrupt source id
+ * [31:8]   - reserved
+ * [59:32]  - interrupt source data
+ * [63:60]  - reserved
+ * [71:64]  - RINGID
+ *            CP:
+ *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
+ *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
+ *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
+ *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
+ *            PIPE_ID - ME0 0=3D
+ *                    - ME1&2 compute dispatcher (4 pipes each)
+ *            SDMA:
+ *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
+ *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
+ *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
+ * [79:72]  - VMID
+ * [95:80]  - PASID
+ * [127:96] - reserved
+ */
+/**
+ * cik_irq_process - interrupt handler
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Interrupt hander (CIK).  Walk the IH ring,
+ * ack interrupts and schedule work to handle
+ * interrupt events.
+ * Returns irq process return code.
+ */
+int cik_irq_process(struct radeon_device *rdev)
+{
+        struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
+        struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
+        u32 wptr;
+        u32 rptr;
+        u32 src_id, src_data, ring_id;
+        u8 me_id, pipe_id, queue_id;
+        u32 ring_index;
+        bool queue_hotplug = false;
+        bool queue_reset = false;
+
+        if (!rdev->ih.enabled || rdev->shutdown)
+                return IRQ_NONE;
+
+        wptr = cik_get_ih_wptr(rdev);
+
+restart_ih:
+        /* is somebody else already processing irqs? */
+        if (atomic_xchg(&rdev->ih.lock, 1))
+                return IRQ_NONE;
+
+        rptr = rdev->ih.rptr;
+        DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
+
+        /* Order reading of wptr vs. reading of IH ring data */
+        rmb();
+
+        /* display interrupts */
+        cik_irq_ack(rdev);
+
+        while (rptr != wptr) {
+                /* wptr/rptr are in bytes! */
+                ring_index = rptr / 4;
+                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
+                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
+                ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
+
+                switch (src_id) {
+                case 1: /* D1 vblank/vline */
+                        switch (src_data) {
+                        case 0: /* D1 vblank */
+                                if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[0]) {
+                                                drm_handle_vblank(rdev->ddev, 0);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (atomic_read(&rdev->irq.pflip[0]))
+                                                radeon_crtc_handle_flip(rdev, 0);
+                                        rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
+                                        DRM_DEBUG("IH: D1 vblank\n");
+                                }
+                                break;
+                        case 1: /* D1 vline */
+                                if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
+                                        DRM_DEBUG("IH: D1 vline\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 2: /* D2 vblank/vline */
+                        switch (src_data) {
+                        case 0: /* D2 vblank */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[1]) {
+                                                drm_handle_vblank(rdev->ddev, 1);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (atomic_read(&rdev->irq.pflip[1]))
+                                                radeon_crtc_handle_flip(rdev, 1);
+                                        rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
+                                        DRM_DEBUG("IH: D2 vblank\n");
+                                }
+                                break;
+                        case 1: /* D2 vline */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
+                                        DRM_DEBUG("IH: D2 vline\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 3: /* D3 vblank/vline */
+                        switch (src_data) {
+                        case 0: /* D3 vblank */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[2]) {
+                                                drm_handle_vblank(rdev->ddev, 2);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (atomic_read(&rdev->irq.pflip[2]))
+                                                radeon_crtc_handle_flip(rdev, 2);
+                                        rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
+                                        DRM_DEBUG("IH: D3 vblank\n");
+                                }
+                                break;
+                        case 1: /* D3 vline */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
+                                        DRM_DEBUG("IH: D3 vline\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 4: /* D4 vblank/vline */
+                        switch (src_data) {
+                        case 0: /* D4 vblank */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[3]) {
+                                                drm_handle_vblank(rdev->ddev, 3);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (atomic_read(&rdev->irq.pflip[3]))
+                                                radeon_crtc_handle_flip(rdev, 3);
+                                        rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
+                                        DRM_DEBUG("IH: D4 vblank\n");
+                                }
+                                break;
+                        case 1: /* D4 vline */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
+                                        DRM_DEBUG("IH: D4 vline\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 5: /* D5 vblank/vline */
+                        switch (src_data) {
+                        case 0: /* D5 vblank */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[4]) {
+                                                drm_handle_vblank(rdev->ddev, 4);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (atomic_read(&rdev->irq.pflip[4]))
+                                                radeon_crtc_handle_flip(rdev, 4);
+                                        rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
+                                        DRM_DEBUG("IH: D5 vblank\n");
+                                }
+                                break;
+                        case 1: /* D5 vline */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
+                                        DRM_DEBUG("IH: D5 vline\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 6: /* D6 vblank/vline */
+                        switch (src_data) {
+                        case 0: /* D6 vblank */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
+                                        if (rdev->irq.crtc_vblank_int[5]) {
+                                                drm_handle_vblank(rdev->ddev, 5);
+                                                rdev->pm.vblank_sync = true;
+                                                wake_up(&rdev->irq.vblank_queue);
+                                        }
+                                        if (atomic_read(&rdev->irq.pflip[5]))
+                                                radeon_crtc_handle_flip(rdev, 5);
+                                        rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
+                                        DRM_DEBUG("IH: D6 vblank\n");
+                                }
+                                break;
+                        case 1: /* D6 vline */
+                                if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
+                                        DRM_DEBUG("IH: D6 vline\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 42: /* HPD hotplug */
+                        switch (src_data) {
+                        case 0:
+                                if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
+                                        queue_hotplug = true;
+                                        DRM_DEBUG("IH: HPD1\n");
+                                }
+                                break;
+                        case 1:
+                                if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
+                                        queue_hotplug = true;
+                                        DRM_DEBUG("IH: HPD2\n");
+                                }
+                                break;
+                        case 2:
+                                if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
+                                        queue_hotplug = true;
+                                        DRM_DEBUG("IH: HPD3\n");
+                                }
+                                break;
+                        case 3:
+                                if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
+                                        queue_hotplug = true;
+                                        DRM_DEBUG("IH: HPD4\n");
+                                }
+                                break;
+                        case 4:
+                                if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
+                                        queue_hotplug = true;
+                                        DRM_DEBUG("IH: HPD5\n");
+                                }
+                                break;
+                        case 5:
+                                if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
+                                        rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
+                                        queue_hotplug = true;
+                                        DRM_DEBUG("IH: HPD6\n");
+                                }
+                                break;
+                        default:
+                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
+                case 146:
+                case 147:
+                        dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
+                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+                                RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
+                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+                                RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
+                        /* reset addr and status */
+                        WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
+                        break;
+                case 176: /* GFX RB CP_INT */
+                case 177: /* GFX IB CP_INT */
+                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
+                        break;
+                case 181: /* CP EOP event */
+                        DRM_DEBUG("IH: CP EOP\n");
+                        /* XXX check the bitfield order! */
+                        me_id = (ring_id & 0x60) >> 5;
+                        pipe_id = (ring_id & 0x18) >> 3;
+                        queue_id = (ring_id & 0x7) >> 0;
+                        switch (me_id) {
+                        case 0:
+                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
+                                break;
+                        case 1:
+                        case 2:
+                                if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
+                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
+                                if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
+                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
+                                break;
+                        }
+                        break;
+                case 184: /* CP Privileged reg access */
+                        DRM_ERROR("Illegal register access in command stream\n");
+                        /* XXX check the bitfield order! */
+                        me_id = (ring_id & 0x60) >> 5;
+                        pipe_id = (ring_id & 0x18) >> 3;
+                        queue_id = (ring_id & 0x7) >> 0;
+                        switch (me_id) {
+                        case 0:
+                                /* This results in a full GPU reset, but all we need to do is soft
+                                 * reset the CP for gfx
+                                 */
+                                queue_reset = true;
+                                break;
+                        case 1:
+                                /* XXX compute */
+                                queue_reset = true;
+                                break;
+                        case 2:
+                                /* XXX compute */
+                                queue_reset = true;
+                                break;
+                        }
+                        break;
+                case 185: /* CP Privileged inst */
+                        DRM_ERROR("Illegal instruction in command stream\n");
+                        /* XXX check the bitfield order! */
+                        me_id = (ring_id & 0x60) >> 5;
+                        pipe_id = (ring_id & 0x18) >> 3;
+                        queue_id = (ring_id & 0x7) >> 0;
+                        switch (me_id) {
+                        case 0:
+                                /* This results in a full GPU reset, but all we need to do is soft
+                                 * reset the CP for gfx
+                                 */
+                                queue_reset = true;
+                                break;
+                        case 1:
+                                /* XXX compute */
+                                queue_reset = true;
+                                break;
+                        case 2:
+                                /* XXX compute */
+                                queue_reset = true;
+                                break;
+                        }
+                        break;
+                case 224: /* SDMA trap event */
+                        /* XXX check the bitfield order! */
+                        me_id = (ring_id & 0x3) >> 0;
+                        queue_id = (ring_id & 0xc) >> 2;
+                        DRM_DEBUG("IH: SDMA trap\n");
+                        switch (me_id) {
+                        case 0:
+                                switch (queue_id) {
+                                case 0:
+                                        radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
+                                        break;
+                                case 1:
+                                        /* XXX compute */
+                                        break;
+                                case 2:
+                                        /* XXX compute */
+                                        break;
+                                }
+                                break;
+                        case 1:
+                                switch (queue_id) {
+                                case 0:
+                                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
+                                        break;
+                                case 1:
+                                        /* XXX compute */
+                                        break;
+                                case 2:
+                                        /* XXX compute */
+                                        break;
+                                }
+                                break;
+                        }
+                        break;
+                case 241: /* SDMA Privileged inst */
+                case 247: /* SDMA Privileged inst */
+                        DRM_ERROR("Illegal instruction in SDMA command stream\n");
+                        /* XXX check the bitfield order! */
+                        me_id = (ring_id & 0x3) >> 0;
+                        queue_id = (ring_id & 0xc) >> 2;
+                        switch (me_id) {
+                        case 0:
+                                switch (queue_id) {
+                                case 0:
+                                        queue_reset = true;
+                                        break;
+                                case 1:
+                                        /* XXX compute */
+                                        queue_reset = true;
+                                        break;
+                                case 2:
+                                        /* XXX compute */
+                                        queue_reset = true;
+                                        break;
+                                }
+                                break;
+                        case 1:
+                                switch (queue_id) {
+                                case 0:
+                                        queue_reset = true;
+                                        break;
+                                case 1:
+                                        /* XXX compute */
+                                        queue_reset = true;
+                                        break;
+                                case 2:
+                                        /* XXX compute */
+                                        queue_reset = true;
+                                        break;
+                                }
+                                break;
+                        }
+                        break;
+                case 233: /* GUI IDLE */
+                        DRM_DEBUG("IH: GUI idle\n");
+                        break;
+                default:
+                        DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+                        break;
+                }
+
+                /* wptr/rptr are in bytes! */
+                rptr += 16;
+                rptr &= rdev->ih.ptr_mask;
+        }
+        if (queue_hotplug)
+                schedule_work(&rdev->hotplug_work);
+        if (queue_reset)
+                schedule_work(&rdev->reset_work);
+        rdev->ih.rptr = rptr;
+        WREG32(IH_RB_RPTR, rdev->ih.rptr);
+        atomic_set(&rdev->ih.lock, 0);
+
+        /* make sure wptr hasn't changed while processing */
+        wptr = cik_get_ih_wptr(rdev);
+        if (wptr != rptr)
+                goto restart_ih;
+
+        return IRQ_HANDLED;
+}
+
+/*
+ * startup/shutdown callbacks
+ */
+/**
+ * cik_startup - program the asic to a functional state
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Programs the asic to a functional state (CIK).
+ * Called by cik_init() and cik_resume().
+ * Returns 0 for success, error for failure.
+ */
+static int cik_startup(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring;
+        int r;
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
+                    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
+                        r = cik_init_microcode(rdev);
+                        if (r) {
+                                DRM_ERROR("Failed to load firmware!\n");
+                                return r;
+                        }
+                }
+        } else {
+                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
+                    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
+                    !rdev->mc_fw) {
+                        r = cik_init_microcode(rdev);
+                        if (r) {
+                                DRM_ERROR("Failed to load firmware!\n");
+                                return r;
+                        }
+                }
+
+                r = ci_mc_load_microcode(rdev);
+                if (r) {
+                        DRM_ERROR("Failed to load MC firmware!\n");
+                        return r;
+                }
+        }
+
+        r = r600_vram_scratch_init(rdev);
+        if (r)
+                return r;
+
+        cik_mc_program(rdev);
+        r = cik_pcie_gart_enable(rdev);
+        if (r)
+                return r;
+        cik_gpu_init(rdev);
+
+        /* allocate rlc buffers */
+        r = si_rlc_init(rdev);
+        if (r) {
+                DRM_ERROR("Failed to init rlc BOs!\n");
+                return r;
+        }
+
+        /* allocate wb buffer */
+        r = radeon_wb_init(rdev);
+        if (r)
+                return r;
+
+        /* allocate mec buffers */
+        r = cik_mec_init(rdev);
+        if (r) {
+                DRM_ERROR("Failed to init MEC BOs!\n");
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
+        if (r) {
+                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
+                return r;
+        }
+
+        r = cik_uvd_resume(rdev);
+        if (!r) {
+                r = radeon_fence_driver_start_ring(rdev,
+                                                   R600_RING_TYPE_UVD_INDEX);
+                if (r)
+                        dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
+        }
+        if (r)
+                rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
+
+        /* Enable IRQ */
+        if (!rdev->irq.installed) {
+                r = radeon_irq_kms_init(rdev);
+                if (r)
+                        return r;
+        }
+
+        r = cik_irq_init(rdev);
+        if (r) {
+                DRM_ERROR("radeon: IH init failed (%d).\n", r);
+                radeon_irq_kms_fini(rdev);
+                return r;
+        }
+        cik_irq_set(rdev);
+
+        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
+        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
+                             CP_RB0_RPTR, CP_RB0_WPTR,
+                             0, 0xfffff, RADEON_CP_PACKET2);
+        if (r)
+                return r;
+
+        /* set up the compute queues */
+        /* type-2 packets are deprecated on MEC, use type-3 instead */
+        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
+        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
+                             CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
+                             0, 0xfffff, PACKET3(PACKET3_NOP, 0x3FFF));
+        if (r)
+                return r;
+        ring->me = 1; /* first MEC */
+        ring->pipe = 0; /* first pipe */
+        ring->queue = 0; /* first queue */
+        ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
+
+        /* type-2 packets are deprecated on MEC, use type-3 instead */
+        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
+        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
+                             CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
+                             0, 0xffffffff, PACKET3(PACKET3_NOP, 0x3FFF));
+        if (r)
+                return r;
+        /* dGPU only have 1 MEC */
+        ring->me = 1; /* first MEC */
+        ring->pipe = 0; /* first pipe */
+        ring->queue = 1; /* second queue */
+        ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
+
+        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
+        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
+                             SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
+                             SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
+                             2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
+        if (r)
+                return r;
+
+        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
+        r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
+                             SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
+                             SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
+                             2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
+        if (r)
+                return r;
+
+        r = cik_cp_resume(rdev);
+        if (r)
+                return r;
+
+        r = cik_sdma_resume(rdev);
+        if (r)
+                return r;
+
+        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
+        if (ring->ring_size) {
+                r = radeon_ring_init(rdev, ring, ring->ring_size,
+                                     R600_WB_UVD_RPTR_OFFSET,
+                                     UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
+                                     0, 0xfffff, RADEON_CP_PACKET2);
+                if (!r)
+                        r = r600_uvd_init(rdev);
+                if (r)
+                        DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
+        }
+
+        r = radeon_ib_pool_init(rdev);
+        if (r) {
+                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
+                return r;
+        }
+
+        r = radeon_vm_manager_init(rdev);
+        if (r) {
+                dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
+                return r;
+        }
+
+        return 0;
+}
+
+/**
+ * cik_resume - resume the asic to a functional state
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Programs the asic to a functional state (CIK).
+ * Called at resume.
+ * Returns 0 for success, error for failure.
+ */
+int cik_resume(struct radeon_device *rdev)
+{
+        int r;
+
+        /* post card */
+        atom_asic_init(rdev->mode_info.atom_context);
+
+        /* init golden registers */
+        cik_init_golden_registers(rdev);
+
+        rdev->accel_working = true;
+        r = cik_startup(rdev);
+        if (r) {
+                DRM_ERROR("cik startup failed on resume\n");
+                rdev->accel_working = false;
+                return r;
+        }
+
+        return r;
+
+}
+
+/**
+ * cik_suspend - suspend the asic
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Bring the chip into a state suitable for suspend (CIK).
+ * Called at suspend.
+ * Returns 0 for success.
+ */
+int cik_suspend(struct radeon_device *rdev)
+{
+        radeon_vm_manager_fini(rdev);
+        cik_cp_enable(rdev, false);
+        cik_sdma_enable(rdev, false);
+        r600_uvd_rbc_stop(rdev);
+        radeon_uvd_suspend(rdev);
+        cik_irq_suspend(rdev);
+        radeon_wb_disable(rdev);
+        cik_pcie_gart_disable(rdev);
+        return 0;
+}
+
+/* Plan is to move initialization in that function and use
+ * helper function so that radeon_device_init pretty much
+ * do nothing more than calling asic specific function. This
+ * should also allow to remove a bunch of callback function
+ * like vram_info.
+ */
+/**
+ * cik_init - asic specific driver and hw init
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Setup asic specific driver variables and program the hw
+ * to a functional state (CIK).
+ * Called at driver startup.
+ * Returns 0 for success, errors for failure.
+ */
+int cik_init(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring;
+        int r;
+
+        /* Read BIOS */
+        if (!radeon_get_bios(rdev)) {
+                if (ASIC_IS_AVIVO(rdev))
+                        return -EINVAL;
+        }
+        /* Must be an ATOMBIOS */
+        if (!rdev->is_atom_bios) {
+                dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
+                return -EINVAL;
+        }
+        r = radeon_atombios_init(rdev);
+        if (r)
+                return r;
+
+        /* Post card if necessary */
+        if (!radeon_card_posted(rdev)) {
+                if (!rdev->bios) {
+                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
+                        return -EINVAL;
+                }
+                DRM_INFO("GPU not posted. posting now...\n");
+                atom_asic_init(rdev->mode_info.atom_context);
+        }
+        /* init golden registers */
+        cik_init_golden_registers(rdev);
+        /* Initialize scratch registers */
+        cik_scratch_init(rdev);
+        /* Initialize surface registers */
+        radeon_surface_init(rdev);
+        /* Initialize clocks */
+        radeon_get_clock_info(rdev->ddev);
+
+        /* Fence driver */
+        r = radeon_fence_driver_init(rdev);
+        if (r)
+                return r;
+
+        /* initialize memory controller */
+        r = cik_mc_init(rdev);
+        if (r)
+                return r;
+        /* Memory manager */
+        r = radeon_bo_init(rdev);
+        if (r)
+                return r;
+
+        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
+        ring->ring_obj = NULL;
+        r600_ring_init(rdev, ring, 1024 * 1024);
+
+        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
+        ring->ring_obj = NULL;
+        r600_ring_init(rdev, ring, 1024 * 1024);
+        r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
+        if (r)
+                return r;
+
+        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
+        ring->ring_obj = NULL;
+        r600_ring_init(rdev, ring, 1024 * 1024);
+        r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
+        if (r)
+                return r;
+
+        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
+        ring->ring_obj = NULL;
+        r600_ring_init(rdev, ring, 256 * 1024);
+
+        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
+        ring->ring_obj = NULL;
+        r600_ring_init(rdev, ring, 256 * 1024);
+
+        r = radeon_uvd_init(rdev);
+        if (!r) {
+                ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
+                ring->ring_obj = NULL;
+                r600_ring_init(rdev, ring, 4096);
+        }
+
+        rdev->ih.ring_obj = NULL;
+        r600_ih_ring_init(rdev, 64 * 1024);
+
+        r = r600_pcie_gart_init(rdev);
+        if (r)
+                return r;
+
+        rdev->accel_working = true;
+        r = cik_startup(rdev);
+        if (r) {
+                dev_err(rdev->dev, "disabling GPU acceleration\n");
+                cik_cp_fini(rdev);
+                cik_sdma_fini(rdev);
+                cik_irq_fini(rdev);
+                si_rlc_fini(rdev);
+                cik_mec_fini(rdev);
+                radeon_wb_fini(rdev);
+                radeon_ib_pool_fini(rdev);
+                radeon_vm_manager_fini(rdev);
+                radeon_irq_kms_fini(rdev);
+                cik_pcie_gart_fini(rdev);
+                rdev->accel_working = false;
+        }
+
+        /* Don't start up if the MC ucode is missing.
+         * The default clocks and voltages before the MC ucode
+         * is loaded are not suffient for advanced operations.
+         */
+        if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
+                DRM_ERROR("radeon: MC ucode required for NI+.\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+/**
+ * cik_fini - asic specific driver and hw fini
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Tear down the asic specific driver variables and program the hw
+ * to an idle state (CIK).
+ * Called at driver unload.
+ */
+void cik_fini(struct radeon_device *rdev)
+{
+        cik_cp_fini(rdev);
+        cik_sdma_fini(rdev);
+        cik_irq_fini(rdev);
+        si_rlc_fini(rdev);
+        cik_mec_fini(rdev);
+        radeon_wb_fini(rdev);
+        radeon_vm_manager_fini(rdev);
+        radeon_ib_pool_fini(rdev);
+        radeon_irq_kms_fini(rdev);
+        radeon_uvd_fini(rdev);
+        cik_pcie_gart_fini(rdev);
+        r600_vram_scratch_fini(rdev);
+        radeon_gem_fini(rdev);
+        radeon_fence_driver_fini(rdev);
+        radeon_bo_fini(rdev);
+        radeon_atombios_fini(rdev);
+        kfree(rdev->bios);
+        rdev->bios = NULL;
+}
+
+/* display watermark setup */
+/**
+ * dce8_line_buffer_adjust - Set up the line buffer
+ *
+ * @rdev: radeon_device pointer
+ * @radeon_crtc: the selected display controller
+ * @mode: the current display mode on the selected display
+ * controller
+ *
+ * Setup up the line buffer allocation for
+ * the selected display controller (CIK).
+ * Returns the line buffer size in pixels.
+ */
+static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
+                                   struct radeon_crtc *radeon_crtc,
+                                   struct drm_display_mode *mode)
+{
+        u32 tmp;
+
+        /*
+         * Line Buffer Setup
+         * There are 6 line buffers, one for each display controllers.
+         * There are 3 partitions per LB. Select the number of partitions
+         * to enable based on the display width.  For display widths larger
+         * than 4096, you need use to use 2 display controllers and combine
+         * them using the stereo blender.
+         */
+        if (radeon_crtc->base.enabled && mode) {
+                if (mode->crtc_hdisplay < 1920)
+                        tmp = 1;
+                else if (mode->crtc_hdisplay < 2560)
+                        tmp = 2;
+                else if (mode->crtc_hdisplay < 4096)
+                        tmp = 0;
+                else {
+                        DRM_DEBUG_KMS("Mode too big for LB!\n");
+                        tmp = 0;
+                }
+        } else
+                tmp = 1;
+
+        WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
+               LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
+
+        if (radeon_crtc->base.enabled && mode) {
+                switch (tmp) {
+                case 0:
+                default:
+                        return 4096 * 2;
+                case 1:
+                        return 1920 * 2;
+                case 2:
+                        return 2560 * 2;
+                }
+        }
+
+        /* controller not enabled, so no lb used */
+        return 0;
+}
+
+/**
+ * cik_get_number_of_dram_channels - get the number of dram channels
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Look up the number of video ram channels (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the number of dram channels
+ */
+static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(MC_SHARED_CHMAP);
+
+        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+        case 0:
+        default:
+                return 1;
+        case 1:
+                return 2;
+        case 2:
+                return 4;
+        case 3:
+                return 8;
+        case 4:
+                return 3;
+        case 5:
+                return 6;
+        case 6:
+                return 10;
+        case 7:
+                return 12;
+        case 8:
+                return 16;
+        }
+}
+
+struct dce8_wm_params {
+        u32 dram_channels; /* number of dram channels */
+        u32 yclk;          /* bandwidth per dram data pin in kHz */
+        u32 sclk;          /* engine clock in kHz */
+        u32 disp_clk;      /* display clock in kHz */
+        u32 src_width;     /* viewport width */
+        u32 active_time;   /* active display time in ns */
+        u32 blank_time;    /* blank time in ns */
+        bool interlaced;    /* mode is interlaced */
+        fixed20_12 vsc;    /* vertical scale ratio */
+        u32 num_heads;     /* number of active crtcs */
+        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
+        u32 lb_size;       /* line buffer allocated to pipe */
+        u32 vtaps;         /* vertical scaler taps */
+};
+
+/**
+ * dce8_dram_bandwidth - get the dram bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the raw dram bandwidth (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the dram bandwidth in MBytes/s
+ */
+static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
+{
+        /* Calculate raw DRAM Bandwidth */
+        fixed20_12 dram_efficiency; /* 0.7 */
+        fixed20_12 yclk, dram_channels, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        yclk.full = dfixed_const(wm->yclk);
+        yclk.full = dfixed_div(yclk, a);
+        dram_channels.full = dfixed_const(wm->dram_channels * 4);
+        a.full = dfixed_const(10);
+        dram_efficiency.full = dfixed_const(7);
+        dram_efficiency.full = dfixed_div(dram_efficiency, a);
+        bandwidth.full = dfixed_mul(dram_channels, yclk);
+        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
+
+        return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the dram bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the dram bandwidth for display in MBytes/s
+ */
+static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
+{
+        /* Calculate DRAM Bandwidth and the part allocated to display. */
+        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
+        fixed20_12 yclk, dram_channels, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        yclk.full = dfixed_const(wm->yclk);
+        yclk.full = dfixed_div(yclk, a);
+        dram_channels.full = dfixed_const(wm->dram_channels * 4);
+        a.full = dfixed_const(10);
+        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
+        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
+        bandwidth.full = dfixed_mul(dram_channels, yclk);
+        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
+
+        return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce8_data_return_bandwidth - get the data return bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the data return bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the data return bandwidth in MBytes/s
+ */
+static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
+{
+        /* Calculate the display Data return Bandwidth */
+        fixed20_12 return_efficiency; /* 0.8 */
+        fixed20_12 sclk, bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        sclk.full = dfixed_const(wm->sclk);
+        sclk.full = dfixed_div(sclk, a);
+        a.full = dfixed_const(10);
+        return_efficiency.full = dfixed_const(8);
+        return_efficiency.full = dfixed_div(return_efficiency, a);
+        a.full = dfixed_const(32);
+        bandwidth.full = dfixed_mul(a, sclk);
+        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
+
+        return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce8_dmif_request_bandwidth - get the dmif bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the dmif bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the dmif bandwidth in MBytes/s
+ */
+static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
+{
+        /* Calculate the DMIF Request Bandwidth */
+        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
+        fixed20_12 disp_clk, bandwidth;
+        fixed20_12 a, b;
+
+        a.full = dfixed_const(1000);
+        disp_clk.full = dfixed_const(wm->disp_clk);
+        disp_clk.full = dfixed_div(disp_clk, a);
+        a.full = dfixed_const(32);
+        b.full = dfixed_mul(a, disp_clk);
+
+        a.full = dfixed_const(10);
+        disp_clk_request_efficiency.full = dfixed_const(8);
+        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
+
+        bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
+
+        return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce8_available_bandwidth - get the min available bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the min available bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the min available bandwidth in MBytes/s
+ */
+static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
+{
+        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
+        u32 dram_bandwidth = dce8_dram_bandwidth(wm);
+        u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
+        u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
+
+        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
+}
+
+/**
+ * dce8_average_bandwidth - get the average available bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the average available bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the average available bandwidth in MBytes/s
+ */
+static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
+{
+        /* Calculate the display mode Average Bandwidth
+         * DisplayMode should contain the source and destination dimensions,
+         * timing, etc.
+         */
+        fixed20_12 bpp;
+        fixed20_12 line_time;
+        fixed20_12 src_width;
+        fixed20_12 bandwidth;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1000);
+        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
+        line_time.full = dfixed_div(line_time, a);
+        bpp.full = dfixed_const(wm->bytes_per_pixel);
+        src_width.full = dfixed_const(wm->src_width);
+        bandwidth.full = dfixed_mul(src_width, bpp);
+        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
+        bandwidth.full = dfixed_div(bandwidth, line_time);
+
+        return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce8_latency_watermark - get the latency watermark
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the latency watermark (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the latency watermark in ns
+ */
+static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
+{
+        /* First calculate the latency in ns */
+        u32 mc_latency = 2000; /* 2000 ns. */
+        u32 available_bandwidth = dce8_available_bandwidth(wm);
+        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
+        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
+        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
+        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
+                (wm->num_heads * cursor_line_pair_return_time);
+        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
+        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
+        u32 tmp, dmif_size = 12288;
+        fixed20_12 a, b, c;
+
+        if (wm->num_heads == 0)
+                return 0;
+
+        a.full = dfixed_const(2);
+        b.full = dfixed_const(1);
+        if ((wm->vsc.full > a.full) ||
+            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
+            (wm->vtaps >= 5) ||
+            ((wm->vsc.full >= a.full) && wm->interlaced))
+                max_src_lines_per_dst_line = 4;
+        else
+                max_src_lines_per_dst_line = 2;
+
+        a.full = dfixed_const(available_bandwidth);
+        b.full = dfixed_const(wm->num_heads);
+        a.full = dfixed_div(a, b);
+
+        b.full = dfixed_const(mc_latency + 512);
+        c.full = dfixed_const(wm->disp_clk);
+        b.full = dfixed_div(b, c);
+
+        c.full = dfixed_const(dmif_size);
+        b.full = dfixed_div(c, b);
+
+        tmp = min(dfixed_trunc(a), dfixed_trunc(b));
+
+        b.full = dfixed_const(1000);
+        c.full = dfixed_const(wm->disp_clk);
+        b.full = dfixed_div(c, b);
+        c.full = dfixed_const(wm->bytes_per_pixel);
+        b.full = dfixed_mul(b, c);
+
+        lb_fill_bw = min(tmp, dfixed_trunc(b));
+
+        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
+        b.full = dfixed_const(1000);
+        c.full = dfixed_const(lb_fill_bw);
+        b.full = dfixed_div(c, b);
+        a.full = dfixed_div(a, b);
+        line_fill_time = dfixed_trunc(a);
+
+        if (line_fill_time < wm->active_time)
+                return latency;
+        else
+                return latency + (line_fill_time - wm->active_time);
+
+}
+
+/**
+ * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
+ * average and available dram bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Check if the display average bandwidth fits in the display
+ * dram bandwidth (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns true if the display fits, false if not.
+ */
+static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
+{
+        if (dce8_average_bandwidth(wm) <=
+            (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
+                return true;
+        else
+                return false;
+}
+
+/**
+ * dce8_average_bandwidth_vs_available_bandwidth - check
+ * average and available bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Check if the display average bandwidth fits in the display
+ * available bandwidth (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns true if the display fits, false if not.
+ */
+static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
+{
+        if (dce8_average_bandwidth(wm) <=
+            (dce8_available_bandwidth(wm) / wm->num_heads))
+                return true;
+        else
+                return false;
+}
+
+/**
+ * dce8_check_latency_hiding - check latency hiding
+ *
+ * @wm: watermark calculation data
+ *
+ * Check latency hiding (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns true if the display fits, false if not.
+ */
+static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
+{
+        u32 lb_partitions = wm->lb_size / wm->src_width;
+        u32 line_time = wm->active_time + wm->blank_time;
+        u32 latency_tolerant_lines;
+        u32 latency_hiding;
+        fixed20_12 a;
+
+        a.full = dfixed_const(1);
+        if (wm->vsc.full > a.full)
+                latency_tolerant_lines = 1;
+        else {
+                if (lb_partitions <= (wm->vtaps + 1))
+                        latency_tolerant_lines = 1;
+                else
+                        latency_tolerant_lines = 2;
+        }
+
+        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
+
+        if (dce8_latency_watermark(wm) <= latency_hiding)
+                return true;
+        else
+                return false;
+}
+
+/**
+ * dce8_program_watermarks - program display watermarks
+ *
+ * @rdev: radeon_device pointer
+ * @radeon_crtc: the selected display controller
+ * @lb_size: line buffer size
+ * @num_heads: number of display controllers in use
+ *
+ * Calculate and program the display watermarks for the
+ * selected display controller (CIK).
+ */
+static void dce8_program_watermarks(struct radeon_device *rdev,
+                                    struct radeon_crtc *radeon_crtc,
+                                    u32 lb_size, u32 num_heads)
+{
+        struct drm_display_mode *mode = &radeon_crtc->base.mode;
+        struct dce8_wm_params wm;
+        u32 pixel_period;
+        u32 line_time = 0;
+        u32 latency_watermark_a = 0, latency_watermark_b = 0;
+        u32 tmp, wm_mask;
+
+        if (radeon_crtc->base.enabled && num_heads && mode) {
+                pixel_period = 1000000 / (u32)mode->clock;
+                line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
+
+                wm.yclk = rdev->pm.current_mclk * 10;
+                wm.sclk = rdev->pm.current_sclk * 10;
+                wm.disp_clk = mode->clock;
+                wm.src_width = mode->crtc_hdisplay;
+                wm.active_time = mode->crtc_hdisplay * pixel_period;
+                wm.blank_time = line_time - wm.active_time;
+                wm.interlaced = false;
+                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                        wm.interlaced = true;
+                wm.vsc = radeon_crtc->vsc;
+                wm.vtaps = 1;
+                if (radeon_crtc->rmx_type != RMX_OFF)
+                        wm.vtaps = 2;
+                wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
+                wm.lb_size = lb_size;
+                wm.dram_channels = cik_get_number_of_dram_channels(rdev);
+                wm.num_heads = num_heads;
+
+                /* set for high clocks */
+                latency_watermark_a = min(dce8_latency_watermark(&wm), (u32)65535);
+                /* set for low clocks */
+                /* wm.yclk = low clk; wm.sclk = low clk */
+                latency_watermark_b = min(dce8_latency_watermark(&wm), (u32)65535);
+
+                /* possibly force display priority to high */
+                /* should really do this at mode validation time... */
+                if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
+                    !dce8_average_bandwidth_vs_available_bandwidth(&wm) ||
+                    !dce8_check_latency_hiding(&wm) ||
+                    (rdev->disp_priority == 2)) {
+                        DRM_DEBUG_KMS("force priority to high\n");
+                }
+        }
+
+        /* select wm A */
+        wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
+        tmp = wm_mask;
+        tmp &= ~LATENCY_WATERMARK_MASK(3);
+        tmp |= LATENCY_WATERMARK_MASK(1);
+        WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
+        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
+               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
+                LATENCY_HIGH_WATERMARK(line_time)));
+        /* select wm B */
+        tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
+        tmp &= ~LATENCY_WATERMARK_MASK(3);
+        tmp |= LATENCY_WATERMARK_MASK(2);
+        WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
+        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
+               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
+                LATENCY_HIGH_WATERMARK(line_time)));
+        /* restore original selection */
+        WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
+}
+
+/**
+ * dce8_bandwidth_update - program display watermarks
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Calculate and program the display watermarks and line
+ * buffer allocation (CIK).
+ */
+void dce8_bandwidth_update(struct radeon_device *rdev)
+{
+        struct drm_display_mode *mode = NULL;
+        u32 num_heads = 0, lb_size;
+        int i;
+
+        radeon_update_display_priority(rdev);
+
+        for (i = 0; i < rdev->num_crtc; i++) {
+                if (rdev->mode_info.crtcs[i]->base.enabled)
+                        num_heads++;
+        }
+        for (i = 0; i < rdev->num_crtc; i++) {
+                mode = &rdev->mode_info.crtcs[i]->base.mode;
+                lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
+                dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
+        }
+}
+
+/**
+ * cik_get_gpu_clock_counter - return GPU clock counter snapshot
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Fetches a GPU clock counter snapshot (SI).
+ * Returns the 64 bit clock counter snapshot.
+ */
+uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
+{
+        uint64_t clock;
+
+        mutex_lock(&rdev->gpu_clock_mutex);
+        WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
+        clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
+                ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
+        mutex_unlock(&rdev->gpu_clock_mutex);
+        return clock;
+}
+
+static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
+                              u32 cntl_reg, u32 status_reg)
+{
+        int r, i;
+        struct atom_clock_dividers dividers;
+        uint32_t tmp;
+
+        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
+                                           clock, false, &dividers);
+        if (r)
+                return r;
+
+        tmp = RREG32_SMC(cntl_reg);
+        tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
+        tmp |= dividers.post_divider;
+        WREG32_SMC(cntl_reg, tmp);
+
+        for (i = 0; i < 100; i++) {
+                if (RREG32_SMC(status_reg) & DCLK_STATUS)
+                        break;
+                mdelay(10);
+        }
+        if (i == 100)
+                return -ETIMEDOUT;
+
+        return 0;
+}
+
+int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
+{
+        int r = 0;
+
+        r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
+        if (r)
+                return r;
+
+        r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
+        return r;
+}
+
+int cik_uvd_resume(struct radeon_device *rdev)
+{
+        uint64_t addr;
+        uint32_t size;
+        int r;
+
+        r = radeon_uvd_resume(rdev);
+        if (r)
+                return r;
+
+        /* programm the VCPU memory controller bits 0-27 */
+        addr = rdev->uvd.gpu_addr >> 3;
+        size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 4) >> 3;
+        WREG32(UVD_VCPU_CACHE_OFFSET0, addr);
+        WREG32(UVD_VCPU_CACHE_SIZE0, size);
+
+        addr += size;
+        size = RADEON_UVD_STACK_SIZE >> 3;
+        WREG32(UVD_VCPU_CACHE_OFFSET1, addr);
+        WREG32(UVD_VCPU_CACHE_SIZE1, size);
+
+        addr += size;
+        size = RADEON_UVD_HEAP_SIZE >> 3;
+        WREG32(UVD_VCPU_CACHE_OFFSET2, addr);
+        WREG32(UVD_VCPU_CACHE_SIZE2, size);
+
+        /* bits 28-31 */
+        addr = (rdev->uvd.gpu_addr >> 28) & 0xF;
+        WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0));
+
+        /* bits 32-39 */
+        addr = (rdev->uvd.gpu_addr >> 32) & 0xFF;
+        WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31));
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/radeon/cik_blit_shaders.c b/drivers/gpu/drm/radeon/cik_blit_shaders.c
new file mode 100644
index 000000000000..ff1311806e91
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cik_blit_shaders.c
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *     Alex Deucher <alexander.deucher@amd.com>
+ */
+
+#include <linux/types.h>
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+const u32 cik_default_state[] =
+{
+        0xc0066900,
+        0x00000000,
+        0x00000060, /* DB_RENDER_CONTROL */
+        0x00000000, /* DB_COUNT_CONTROL */
+        0x00000000, /* DB_DEPTH_VIEW */
+        0x0000002a, /* DB_RENDER_OVERRIDE */
+        0x00000000, /* DB_RENDER_OVERRIDE2 */
+        0x00000000, /* DB_HTILE_DATA_BASE */
+
+        0xc0046900,
+        0x00000008,
+        0x00000000, /* DB_DEPTH_BOUNDS_MIN */
+        0x00000000, /* DB_DEPTH_BOUNDS_MAX */
+        0x00000000, /* DB_STENCIL_CLEAR */
+        0x00000000, /* DB_DEPTH_CLEAR */
+
+        0xc0036900,
+        0x0000000f,
+        0x00000000, /* DB_DEPTH_INFO */
+        0x00000000, /* DB_Z_INFO */
+        0x00000000, /* DB_STENCIL_INFO */
+
+        0xc0016900,
+        0x00000080,
+        0x00000000, /* PA_SC_WINDOW_OFFSET */
+
+        0xc00d6900,
+        0x00000083,
+        0x0000ffff, /* PA_SC_CLIPRECT_RULE */
+        0x00000000, /* PA_SC_CLIPRECT_0_TL */
+        0x20002000, /* PA_SC_CLIPRECT_0_BR */
+        0x00000000,
+        0x20002000,
+        0x00000000,
+        0x20002000,
+        0x00000000,
+        0x20002000,
+        0xaaaaaaaa, /* PA_SC_EDGERULE */
+        0x00000000, /* PA_SU_HARDWARE_SCREEN_OFFSET */
+        0x0000000f, /* CB_TARGET_MASK */
+        0x0000000f, /* CB_SHADER_MASK */
+
+        0xc0226900,
+        0x00000094,
+        0x80000000, /* PA_SC_VPORT_SCISSOR_0_TL */
+        0x20002000, /* PA_SC_VPORT_SCISSOR_0_BR */
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x80000000,
+        0x20002000,
+        0x00000000, /* PA_SC_VPORT_ZMIN_0 */
+        0x3f800000, /* PA_SC_VPORT_ZMAX_0 */
+
+        0xc0046900,
+        0x00000100,
+        0xffffffff, /* VGT_MAX_VTX_INDX */
+        0x00000000, /* VGT_MIN_VTX_INDX */
+        0x00000000, /* VGT_INDX_OFFSET */
+        0x00000000, /* VGT_MULTI_PRIM_IB_RESET_INDX */
+
+        0xc0046900,
+        0x00000105,
+        0x00000000, /* CB_BLEND_RED */
+        0x00000000, /* CB_BLEND_GREEN */
+        0x00000000, /* CB_BLEND_BLUE */
+        0x00000000, /* CB_BLEND_ALPHA */
+
+        0xc0016900,
+        0x000001e0,
+        0x00000000, /* CB_BLEND0_CONTROL */
+
+        0xc00c6900,
+        0x00000200,
+        0x00000000, /* DB_DEPTH_CONTROL */
+        0x00000000, /* DB_EQAA */
+        0x00cc0010, /* CB_COLOR_CONTROL */
+        0x00000210, /* DB_SHADER_CONTROL */
+        0x00010000, /* PA_CL_CLIP_CNTL */
+        0x00000004, /* PA_SU_SC_MODE_CNTL */
+        0x00000100, /* PA_CL_VTE_CNTL */
+        0x00000000, /* PA_CL_VS_OUT_CNTL */
+        0x00000000, /* PA_CL_NANINF_CNTL */
+        0x00000000, /* PA_SU_LINE_STIPPLE_CNTL */
+        0x00000000, /* PA_SU_LINE_STIPPLE_SCALE */
+        0x00000000, /* PA_SU_PRIM_FILTER_CNTL */
+
+        0xc0116900,
+        0x00000280,
+        0x00000000, /* PA_SU_POINT_SIZE */
+        0x00000000, /* PA_SU_POINT_MINMAX */
+        0x00000008, /* PA_SU_LINE_CNTL */
+        0x00000000, /* PA_SC_LINE_STIPPLE */
+        0x00000000, /* VGT_OUTPUT_PATH_CNTL */
+        0x00000000, /* VGT_HOS_CNTL */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000, /* VGT_GS_MODE */
+
+        0xc0026900,
+        0x00000292,
+        0x00000000, /* PA_SC_MODE_CNTL_0 */
+        0x00000000, /* PA_SC_MODE_CNTL_1 */
+
+        0xc0016900,
+        0x000002a1,
+        0x00000000, /* VGT_PRIMITIVEID_EN */
+
+        0xc0016900,
+        0x000002a5,
+        0x00000000, /* VGT_MULTI_PRIM_IB_RESET_EN */
+
+        0xc0026900,
+        0x000002a8,
+        0x00000000, /* VGT_INSTANCE_STEP_RATE_0 */
+        0x00000000,
+
+        0xc0026900,
+        0x000002ad,
+        0x00000000, /* VGT_REUSE_OFF */
+        0x00000000,
+
+        0xc0016900,
+        0x000002d5,
+        0x00000000, /* VGT_SHADER_STAGES_EN */
+
+        0xc0016900,
+        0x000002dc,
+        0x0000aa00, /* DB_ALPHA_TO_MASK */
+
+        0xc0066900,
+        0x000002de,
+        0x00000000, /* PA_SU_POLY_OFFSET_DB_FMT_CNTL */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+
+        0xc0026900,
+        0x000002e5,
+        0x00000000, /* VGT_STRMOUT_CONFIG */
+        0x00000000,
+
+        0xc01b6900,
+        0x000002f5,
+        0x76543210, /* PA_SC_CENTROID_PRIORITY_0 */
+        0xfedcba98, /* PA_SC_CENTROID_PRIORITY_1 */
+        0x00000000, /* PA_SC_LINE_CNTL */
+        0x00000000, /* PA_SC_AA_CONFIG */
+        0x00000005, /* PA_SU_VTX_CNTL */
+        0x3f800000, /* PA_CL_GB_VERT_CLIP_ADJ */
+        0x3f800000, /* PA_CL_GB_VERT_DISC_ADJ */
+        0x3f800000, /* PA_CL_GB_HORZ_CLIP_ADJ */
+        0x3f800000, /* PA_CL_GB_HORZ_DISC_ADJ */
+        0x00000000, /* PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0xffffffff, /* PA_SC_AA_MASK_X0Y0_X1Y0 */
+        0xffffffff,
+
+        0xc0026900,
+        0x00000316,
+        0x0000000e, /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+        0x00000010, /*  */
+};
+
+const u32 cik_default_size = ARRAY_SIZE(cik_default_state);
diff --git a/drivers/gpu/drm/radeon/cik_blit_shaders.h b/drivers/gpu/drm/radeon/cik_blit_shaders.h
new file mode 100644
index 000000000000..dfe7314f9ff4
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cik_blit_shaders.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef CIK_BLIT_SHADERS_H
+#define CIK_BLIT_SHADERS_H
+
+extern const u32 cik_default_state[];
+
+extern const u32 cik_default_size;
+
+#endif
diff --git a/drivers/gpu/drm/radeon/cik_reg.h b/drivers/gpu/drm/radeon/cik_reg.h
new file mode 100644
index 000000000000..d71e46d571f5
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cik_reg.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef __CIK_REG_H__
+#define __CIK_REG_H__
+
+#define CIK_DC_GPIO_HPD_MASK                      0x65b0
+#define CIK_DC_GPIO_HPD_A                         0x65b4
+#define CIK_DC_GPIO_HPD_EN                        0x65b8
+#define CIK_DC_GPIO_HPD_Y                         0x65bc
+
+#define CIK_GRPH_CONTROL                          0x6804
+#       define CIK_GRPH_DEPTH(x)                  (((x) & 0x3) << 0)
+#       define CIK_GRPH_DEPTH_8BPP                0
+#       define CIK_GRPH_DEPTH_16BPP               1
+#       define CIK_GRPH_DEPTH_32BPP               2
+#       define CIK_GRPH_NUM_BANKS(x)              (((x) & 0x3) << 2)
+#       define CIK_ADDR_SURF_2_BANK               0
+#       define CIK_ADDR_SURF_4_BANK               1
+#       define CIK_ADDR_SURF_8_BANK               2
+#       define CIK_ADDR_SURF_16_BANK              3
+#       define CIK_GRPH_Z(x)                      (((x) & 0x3) << 4)
+#       define CIK_GRPH_BANK_WIDTH(x)             (((x) & 0x3) << 6)
+#       define CIK_ADDR_SURF_BANK_WIDTH_1         0
+#       define CIK_ADDR_SURF_BANK_WIDTH_2         1
+#       define CIK_ADDR_SURF_BANK_WIDTH_4         2
+#       define CIK_ADDR_SURF_BANK_WIDTH_8         3
+#       define CIK_GRPH_FORMAT(x)                 (((x) & 0x7) << 8)
+/* 8 BPP */
+#       define CIK_GRPH_FORMAT_INDEXED            0
+/* 16 BPP */
+#       define CIK_GRPH_FORMAT_ARGB1555           0
+#       define CIK_GRPH_FORMAT_ARGB565            1
+#       define CIK_GRPH_FORMAT_ARGB4444           2
+#       define CIK_GRPH_FORMAT_AI88               3
+#       define CIK_GRPH_FORMAT_MONO16             4
+#       define CIK_GRPH_FORMAT_BGRA5551           5
+/* 32 BPP */
+#       define CIK_GRPH_FORMAT_ARGB8888           0
+#       define CIK_GRPH_FORMAT_ARGB2101010        1
+#       define CIK_GRPH_FORMAT_32BPP_DIG          2
+#       define CIK_GRPH_FORMAT_8B_ARGB2101010     3
+#       define CIK_GRPH_FORMAT_BGRA1010102        4
+#       define CIK_GRPH_FORMAT_8B_BGRA1010102     5
+#       define CIK_GRPH_FORMAT_RGB111110          6
+#       define CIK_GRPH_FORMAT_BGR101111          7
+#       define CIK_GRPH_BANK_HEIGHT(x)            (((x) & 0x3) << 11)
+#       define CIK_ADDR_SURF_BANK_HEIGHT_1        0
+#       define CIK_ADDR_SURF_BANK_HEIGHT_2        1
+#       define CIK_ADDR_SURF_BANK_HEIGHT_4        2
+#       define CIK_ADDR_SURF_BANK_HEIGHT_8        3
+#       define CIK_GRPH_TILE_SPLIT(x)             (((x) & 0x7) << 13)
+#       define CIK_ADDR_SURF_TILE_SPLIT_64B       0
+#       define CIK_ADDR_SURF_TILE_SPLIT_128B      1
+#       define CIK_ADDR_SURF_TILE_SPLIT_256B      2
+#       define CIK_ADDR_SURF_TILE_SPLIT_512B      3
+#       define CIK_ADDR_SURF_TILE_SPLIT_1KB       4
+#       define CIK_ADDR_SURF_TILE_SPLIT_2KB       5
+#       define CIK_ADDR_SURF_TILE_SPLIT_4KB       6
+#       define CIK_GRPH_MACRO_TILE_ASPECT(x)      (((x) & 0x3) << 18)
+#       define CIK_ADDR_SURF_MACRO_TILE_ASPECT_1  0
+#       define CIK_ADDR_SURF_MACRO_TILE_ASPECT_2  1
+#       define CIK_ADDR_SURF_MACRO_TILE_ASPECT_4  2
+#       define CIK_ADDR_SURF_MACRO_TILE_ASPECT_8  3
+#       define CIK_GRPH_ARRAY_MODE(x)             (((x) & 0x7) << 20)
+#       define CIK_GRPH_ARRAY_LINEAR_GENERAL      0
+#       define CIK_GRPH_ARRAY_LINEAR_ALIGNED      1
+#       define CIK_GRPH_ARRAY_1D_TILED_THIN1      2
+#       define CIK_GRPH_ARRAY_2D_TILED_THIN1      4
+#       define CIK_GRPH_PIPE_CONFIG(x)           (((x) & 0x1f) << 24)
+#       define CIK_ADDR_SURF_P2                  0
+#       define CIK_ADDR_SURF_P4_8x16             4
+#       define CIK_ADDR_SURF_P4_16x16            5
+#       define CIK_ADDR_SURF_P4_16x32            6
+#       define CIK_ADDR_SURF_P4_32x32            7
+#       define CIK_ADDR_SURF_P8_16x16_8x16       8
+#       define CIK_ADDR_SURF_P8_16x32_8x16       9
+#       define CIK_ADDR_SURF_P8_32x32_8x16       10
+#       define CIK_ADDR_SURF_P8_16x32_16x16      11
+#       define CIK_ADDR_SURF_P8_32x32_16x16      12
+#       define CIK_ADDR_SURF_P8_32x32_16x32      13
+#       define CIK_ADDR_SURF_P8_32x64_32x32      14
+#       define CIK_GRPH_MICRO_TILE_MODE(x)       (((x) & 0x7) << 29)
+#       define CIK_DISPLAY_MICRO_TILING          0
+#       define CIK_THIN_MICRO_TILING             1
+#       define CIK_DEPTH_MICRO_TILING            2
+#       define CIK_ROTATED_MICRO_TILING          4
+
+/* CUR blocks at 0x6998, 0x7598, 0x10198, 0x10d98, 0x11998, 0x12598 */
+#define CIK_CUR_CONTROL                           0x6998
+#       define CIK_CURSOR_EN                      (1 << 0)
+#       define CIK_CURSOR_MODE(x)                 (((x) & 0x3) << 8)
+#       define CIK_CURSOR_MONO                    0
+#       define CIK_CURSOR_24_1                    1
+#       define CIK_CURSOR_24_8_PRE_MULT           2
+#       define CIK_CURSOR_24_8_UNPRE_MULT         3
+#       define CIK_CURSOR_2X_MAGNIFY              (1 << 16)
+#       define CIK_CURSOR_FORCE_MC_ON             (1 << 20)
+#       define CIK_CURSOR_URGENT_CONTROL(x)       (((x) & 0x7) << 24)
+#       define CIK_CURSOR_URGENT_ALWAYS           0
+#       define CIK_CURSOR_URGENT_1_8              1
+#       define CIK_CURSOR_URGENT_1_4              2
+#       define CIK_CURSOR_URGENT_3_8              3
+#       define CIK_CURSOR_URGENT_1_2              4
+#define CIK_CUR_SURFACE_ADDRESS                   0x699c
+#       define CIK_CUR_SURFACE_ADDRESS_MASK       0xfffff000
+#define CIK_CUR_SIZE                              0x69a0
+#define CIK_CUR_SURFACE_ADDRESS_HIGH              0x69a4
+#define CIK_CUR_POSITION                          0x69a8
+#define CIK_CUR_HOT_SPOT                          0x69ac
+#define CIK_CUR_COLOR1                            0x69b0
+#define CIK_CUR_COLOR2                            0x69b4
+#define CIK_CUR_UPDATE                            0x69b8
+#       define CIK_CURSOR_UPDATE_PENDING          (1 << 0)
+#       define CIK_CURSOR_UPDATE_TAKEN            (1 << 1)
+#       define CIK_CURSOR_UPDATE_LOCK             (1 << 16)
+#       define CIK_CURSOR_DISABLE_MULTIPLE_UPDATE (1 << 24)
+
+#define CIK_ALPHA_CONTROL                         0x6af0
+#       define CIK_CURSOR_ALPHA_BLND_ENA          (1 << 1)
+
+#define CIK_LB_DATA_FORMAT                        0x6b00
+#       define CIK_INTERLEAVE_EN                  (1 << 3)
+
+#define CIK_LB_DESKTOP_HEIGHT                     0x6b0c
+
+#endif
diff --git a/drivers/gpu/drm/radeon/cikd.h b/drivers/gpu/drm/radeon/cikd.h
new file mode 100644
index 000000000000..63514b95889a
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cikd.h
@@ -0,0 +1,1297 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef CIK_H
+#define CIK_H
+
+#define BONAIRE_GB_ADDR_CONFIG_GOLDEN        0x12010001
+
+#define CIK_RB_BITMAP_WIDTH_PER_SH  2
+
+/* SMC IND registers */
+#define GENERAL_PWRMGT                                    0xC0200000
+#       define GPU_COUNTER_CLK                            (1 << 15)
+
+#define CG_CLKPIN_CNTL                                    0xC05001A0
+#       define XTALIN_DIVIDE                              (1 << 1)
+
+#define PCIE_INDEX                                      0x38
+#define PCIE_DATA                                       0x3C
+
+#define VGA_HDP_CONTROL                                 0x328
+#define         VGA_MEMORY_DISABLE                              (1 << 4)
+
+#define DMIF_ADDR_CALC                                  0xC00
+
+#define SRBM_GFX_CNTL                                   0xE44
+#define         PIPEID(x)                                       ((x) << 0)
+#define         MEID(x)                                         ((x) << 2)
+#define         VMID(x)                                         ((x) << 4)
+#define         QUEUEID(x)                                      ((x) << 8)
+
+#define SRBM_STATUS2                                    0xE4C
+#define         SDMA_BUSY                               (1 << 5)
+#define         SDMA1_BUSY                              (1 << 6)
+#define SRBM_STATUS                                     0xE50
+#define         UVD_RQ_PENDING                          (1 << 1)
+#define         GRBM_RQ_PENDING                         (1 << 5)
+#define         VMC_BUSY                                (1 << 8)
+#define         MCB_BUSY                                (1 << 9)
+#define         MCB_NON_DISPLAY_BUSY                    (1 << 10)
+#define         MCC_BUSY                                (1 << 11)
+#define         MCD_BUSY                                (1 << 12)
+#define         SEM_BUSY                                (1 << 14)
+#define         IH_BUSY                                 (1 << 17)
+#define         UVD_BUSY                                (1 << 19)
+
+#define SRBM_SOFT_RESET                                 0xE60
+#define         SOFT_RESET_BIF                          (1 << 1)
+#define         SOFT_RESET_R0PLL                        (1 << 4)
+#define         SOFT_RESET_DC                           (1 << 5)
+#define         SOFT_RESET_SDMA1                        (1 << 6)
+#define         SOFT_RESET_GRBM                         (1 << 8)
+#define         SOFT_RESET_HDP                          (1 << 9)
+#define         SOFT_RESET_IH                           (1 << 10)
+#define         SOFT_RESET_MC                           (1 << 11)
+#define         SOFT_RESET_ROM                          (1 << 14)
+#define         SOFT_RESET_SEM                          (1 << 15)
+#define         SOFT_RESET_VMC                          (1 << 17)
+#define         SOFT_RESET_SDMA                         (1 << 20)
+#define         SOFT_RESET_TST                          (1 << 21)
+#define         SOFT_RESET_REGBB                        (1 << 22)
+#define         SOFT_RESET_ORB                          (1 << 23)
+#define         SOFT_RESET_VCE                          (1 << 24)
+
+#define VM_L2_CNTL                                      0x1400
+#define         ENABLE_L2_CACHE                                 (1 << 0)
+#define         ENABLE_L2_FRAGMENT_PROCESSING                   (1 << 1)
+#define         L2_CACHE_PTE_ENDIAN_SWAP_MODE(x)                ((x) << 2)
+#define         L2_CACHE_PDE_ENDIAN_SWAP_MODE(x)                ((x) << 4)
+#define         ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE         (1 << 9)
+#define         ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE        (1 << 10)
+#define         EFFECTIVE_L2_QUEUE_SIZE(x)                      (((x) & 7) << 15)
+#define         CONTEXT1_IDENTITY_ACCESS_MODE(x)                (((x) & 3) << 19)
+#define VM_L2_CNTL2                                     0x1404
+#define         INVALIDATE_ALL_L1_TLBS                          (1 << 0)
+#define         INVALIDATE_L2_CACHE                             (1 << 1)
+#define         INVALIDATE_CACHE_MODE(x)                        ((x) << 26)
+#define                 INVALIDATE_PTE_AND_PDE_CACHES           0
+#define                 INVALIDATE_ONLY_PTE_CACHES              1
+#define                 INVALIDATE_ONLY_PDE_CACHES              2
+#define VM_L2_CNTL3                                     0x1408
+#define         BANK_SELECT(x)                                  ((x) << 0)
+#define         L2_CACHE_UPDATE_MODE(x)                         ((x) << 6)
+#define         L2_CACHE_BIGK_FRAGMENT_SIZE(x)                  ((x) << 15)
+#define         L2_CACHE_BIGK_ASSOCIATIVITY                     (1 << 20)
+#define VM_L2_STATUS                                    0x140C
+#define         L2_BUSY                                         (1 << 0)
+#define VM_CONTEXT0_CNTL                                0x1410
+#define         ENABLE_CONTEXT                                  (1 << 0)
+#define         PAGE_TABLE_DEPTH(x)                             (((x) & 3) << 1)
+#define         RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT         (1 << 3)
+#define         RANGE_PROTECTION_FAULT_ENABLE_DEFAULT           (1 << 4)
+#define         DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT    (1 << 6)
+#define         DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT      (1 << 7)
+#define         PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT          (1 << 9)
+#define         PDE0_PROTECTION_FAULT_ENABLE_DEFAULT            (1 << 10)
+#define         VALID_PROTECTION_FAULT_ENABLE_INTERRUPT         (1 << 12)
+#define         VALID_PROTECTION_FAULT_ENABLE_DEFAULT           (1 << 13)
+#define         READ_PROTECTION_FAULT_ENABLE_INTERRUPT          (1 << 15)
+#define         READ_PROTECTION_FAULT_ENABLE_DEFAULT            (1 << 16)
+#define         WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT         (1 << 18)
+#define         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT           (1 << 19)
+#define VM_CONTEXT1_CNTL                                0x1414
+#define VM_CONTEXT0_CNTL2                               0x1430
+#define VM_CONTEXT1_CNTL2                               0x1434
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR                0x1438
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR                0x143c
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR               0x1440
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR               0x1444
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR               0x1448
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR               0x144c
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR               0x1450
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR               0x1454
+
+#define VM_INVALIDATE_REQUEST                           0x1478
+#define VM_INVALIDATE_RESPONSE                          0x147c
+
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS             0x14DC
+
+#define VM_CONTEXT1_PROTECTION_FAULT_ADDR               0x14FC
+
+#define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR       0x1518
+#define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR       0x151c
+
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR                0x153c
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR                0x1540
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR                0x1544
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR                0x1548
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR                0x154c
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR                0x1550
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR                0x1554
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR                0x1558
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR               0x155c
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR               0x1560
+
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR                 0x157C
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR                 0x1580
+
+#define MC_SHARED_CHMAP                                         0x2004
+#define         NOOFCHAN_SHIFT                                  12
+#define         NOOFCHAN_MASK                                   0x0000f000
+#define MC_SHARED_CHREMAP                                       0x2008
+
+#define CHUB_CONTROL                                    0x1864
+#define         BYPASS_VM                                       (1 << 0)
+
+#define MC_VM_FB_LOCATION                               0x2024
+#define MC_VM_AGP_TOP                                   0x2028
+#define MC_VM_AGP_BOT                                   0x202C
+#define MC_VM_AGP_BASE                                  0x2030
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR                  0x2034
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR                 0x2038
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR              0x203C
+
+#define MC_VM_MX_L1_TLB_CNTL                            0x2064
+#define         ENABLE_L1_TLB                                   (1 << 0)
+#define         ENABLE_L1_FRAGMENT_PROCESSING                   (1 << 1)
+#define         SYSTEM_ACCESS_MODE_PA_ONLY                      (0 << 3)
+#define         SYSTEM_ACCESS_MODE_USE_SYS_MAP                  (1 << 3)
+#define         SYSTEM_ACCESS_MODE_IN_SYS                       (2 << 3)
+#define         SYSTEM_ACCESS_MODE_NOT_IN_SYS                   (3 << 3)
+#define         SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU       (0 << 5)
+#define         ENABLE_ADVANCED_DRIVER_MODEL                    (1 << 6)
+#define MC_VM_FB_OFFSET                                 0x2068
+
+#define MC_SHARED_BLACKOUT_CNTL                         0x20ac
+
+#define MC_ARB_RAMCFG                                   0x2760
+#define         NOOFBANK_SHIFT                                  0
+#define         NOOFBANK_MASK                                   0x00000003
+#define         NOOFRANK_SHIFT                                  2
+#define         NOOFRANK_MASK                                   0x00000004
+#define         NOOFROWS_SHIFT                                  3
+#define         NOOFROWS_MASK                                   0x00000038
+#define         NOOFCOLS_SHIFT                                  6
+#define         NOOFCOLS_MASK                                   0x000000C0
+#define         CHANSIZE_SHIFT                                  8
+#define         CHANSIZE_MASK                                   0x00000100
+#define         NOOFGROUPS_SHIFT                                12
+#define         NOOFGROUPS_MASK                                 0x00001000
+
+#define MC_SEQ_SUP_CNTL                                 0x28c8
+#define         RUN_MASK                                (1 << 0)
+#define MC_SEQ_SUP_PGM                                  0x28cc
+
+#define MC_SEQ_TRAIN_WAKEUP_CNTL                        0x28e8
+#define         TRAIN_DONE_D0                           (1 << 30)
+#define         TRAIN_DONE_D1                           (1 << 31)
+
+#define MC_IO_PAD_CNTL_D0                               0x29d0
+#define         MEM_FALL_OUT_CMD                        (1 << 8)
+
+#define MC_SEQ_IO_DEBUG_INDEX                           0x2a44
+#define MC_SEQ_IO_DEBUG_DATA                            0x2a48
+
+#define HDP_HOST_PATH_CNTL                              0x2C00
+#define HDP_NONSURFACE_BASE                             0x2C04
+#define HDP_NONSURFACE_INFO                             0x2C08
+#define HDP_NONSURFACE_SIZE                             0x2C0C
+
+#define HDP_ADDR_CONFIG                                 0x2F48
+#define HDP_MISC_CNTL                                   0x2F4C
+#define         HDP_FLUSH_INVALIDATE_CACHE                      (1 << 0)
+
+#define IH_RB_CNTL                                        0x3e00
+#       define IH_RB_ENABLE                               (1 << 0)
+#       define IH_RB_SIZE(x)                              ((x) << 1) /* log2 */
+#       define IH_RB_FULL_DRAIN_ENABLE                    (1 << 6)
+#       define IH_WPTR_WRITEBACK_ENABLE                   (1 << 8)
+#       define IH_WPTR_WRITEBACK_TIMER(x)                 ((x) << 9) /* log2 */
+#       define IH_WPTR_OVERFLOW_ENABLE                    (1 << 16)
+#       define IH_WPTR_OVERFLOW_CLEAR                     (1 << 31)
+#define IH_RB_BASE                                        0x3e04
+#define IH_RB_RPTR                                        0x3e08
+#define IH_RB_WPTR                                        0x3e0c
+#       define RB_OVERFLOW                                (1 << 0)
+#       define WPTR_OFFSET_MASK                           0x3fffc
+#define IH_RB_WPTR_ADDR_HI                                0x3e10
+#define IH_RB_WPTR_ADDR_LO                                0x3e14
+#define IH_CNTL                                           0x3e18
+#       define ENABLE_INTR                                (1 << 0)
+#       define IH_MC_SWAP(x)                              ((x) << 1)
+#       define IH_MC_SWAP_NONE                            0
+#       define IH_MC_SWAP_16BIT                           1
+#       define IH_MC_SWAP_32BIT                           2
+#       define IH_MC_SWAP_64BIT                           3
+#       define RPTR_REARM                                 (1 << 4)
+#       define MC_WRREQ_CREDIT(x)                         ((x) << 15)
+#       define MC_WR_CLEAN_CNT(x)                         ((x) << 20)
+#       define MC_VMID(x)                                 ((x) << 25)
+
+#define CONFIG_MEMSIZE                                  0x5428
+
+#define INTERRUPT_CNTL                                    0x5468
+#       define IH_DUMMY_RD_OVERRIDE                       (1 << 0)
+#       define IH_DUMMY_RD_EN                             (1 << 1)
+#       define IH_REQ_NONSNOOP_EN                         (1 << 3)
+#       define GEN_IH_INT_EN                              (1 << 8)
+#define INTERRUPT_CNTL2                                   0x546c
+
+#define HDP_MEM_COHERENCY_FLUSH_CNTL                    0x5480
+
+#define BIF_FB_EN                                               0x5490
+#define         FB_READ_EN                                      (1 << 0)
+#define         FB_WRITE_EN                                     (1 << 1)
+
+#define HDP_REG_COHERENCY_FLUSH_CNTL                    0x54A0
+
+#define GPU_HDP_FLUSH_REQ                               0x54DC
+#define GPU_HDP_FLUSH_DONE                              0x54E0
+#define         CP0                                     (1 << 0)
+#define         CP1                                     (1 << 1)
+#define         CP2                                     (1 << 2)
+#define         CP3                                     (1 << 3)
+#define         CP4                                     (1 << 4)
+#define         CP5                                     (1 << 5)
+#define         CP6                                     (1 << 6)
+#define         CP7                                     (1 << 7)
+#define         CP8                                     (1 << 8)
+#define         CP9                                     (1 << 9)
+#define         SDMA0                                   (1 << 10)
+#define         SDMA1                                   (1 << 11)
+
+/* 0x6b04, 0x7704, 0x10304, 0x10f04, 0x11b04, 0x12704 */
+#define LB_MEMORY_CTRL                                  0x6b04
+#define         LB_MEMORY_SIZE(x)                       ((x) << 0)
+#define         LB_MEMORY_CONFIG(x)                     ((x) << 20)
+
+#define DPG_WATERMARK_MASK_CONTROL                      0x6cc8
+#       define LATENCY_WATERMARK_MASK(x)                ((x) << 8)
+#define DPG_PIPE_LATENCY_CONTROL                        0x6ccc
+#       define LATENCY_LOW_WATERMARK(x)                 ((x) << 0)
+#       define LATENCY_HIGH_WATERMARK(x)                ((x) << 16)
+
+/* 0x6b24, 0x7724, 0x10324, 0x10f24, 0x11b24, 0x12724 */
+#define LB_VLINE_STATUS                                 0x6b24
+#       define VLINE_OCCURRED                           (1 << 0)
+#       define VLINE_ACK                                (1 << 4)
+#       define VLINE_STAT                               (1 << 12)
+#       define VLINE_INTERRUPT                          (1 << 16)
+#       define VLINE_INTERRUPT_TYPE                     (1 << 17)
+/* 0x6b2c, 0x772c, 0x1032c, 0x10f2c, 0x11b2c, 0x1272c */
+#define LB_VBLANK_STATUS                                0x6b2c
+#       define VBLANK_OCCURRED                          (1 << 0)
+#       define VBLANK_ACK                               (1 << 4)
+#       define VBLANK_STAT                              (1 << 12)
+#       define VBLANK_INTERRUPT                         (1 << 16)
+#       define VBLANK_INTERRUPT_TYPE                    (1 << 17)
+
+/* 0x6b20, 0x7720, 0x10320, 0x10f20, 0x11b20, 0x12720 */
+#define LB_INTERRUPT_MASK                               0x6b20
+#       define VBLANK_INTERRUPT_MASK                    (1 << 0)
+#       define VLINE_INTERRUPT_MASK                     (1 << 4)
+#       define VLINE2_INTERRUPT_MASK                    (1 << 8)
+
+#define DISP_INTERRUPT_STATUS                           0x60f4
+#       define LB_D1_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D1_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD1_INTERRUPT                        (1 << 17)
+#       define DC_HPD1_RX_INTERRUPT                     (1 << 18)
+#       define DACA_AUTODETECT_INTERRUPT                (1 << 22)
+#       define DACB_AUTODETECT_INTERRUPT                (1 << 23)
+#       define DC_I2C_SW_DONE_INTERRUPT                 (1 << 24)
+#       define DC_I2C_HW_DONE_INTERRUPT                 (1 << 25)
+#define DISP_INTERRUPT_STATUS_CONTINUE                  0x60f8
+#       define LB_D2_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D2_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD2_INTERRUPT                        (1 << 17)
+#       define DC_HPD2_RX_INTERRUPT                     (1 << 18)
+#       define DISP_TIMER_INTERRUPT                     (1 << 24)
+#define DISP_INTERRUPT_STATUS_CONTINUE2                 0x60fc
+#       define LB_D3_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D3_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD3_INTERRUPT                        (1 << 17)
+#       define DC_HPD3_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE3                 0x6100
+#       define LB_D4_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D4_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD4_INTERRUPT                        (1 << 17)
+#       define DC_HPD4_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE4                 0x614c
+#       define LB_D5_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D5_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD5_INTERRUPT                        (1 << 17)
+#       define DC_HPD5_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE5                 0x6150
+#       define LB_D6_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D6_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD6_INTERRUPT                        (1 << 17)
+#       define DC_HPD6_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE6                 0x6780
+
+#define DAC_AUTODETECT_INT_CONTROL                      0x67c8
+
+#define DC_HPD1_INT_STATUS                              0x601c
+#define DC_HPD2_INT_STATUS                              0x6028
+#define DC_HPD3_INT_STATUS                              0x6034
+#define DC_HPD4_INT_STATUS                              0x6040
+#define DC_HPD5_INT_STATUS                              0x604c
+#define DC_HPD6_INT_STATUS                              0x6058
+#       define DC_HPDx_INT_STATUS                       (1 << 0)
+#       define DC_HPDx_SENSE                            (1 << 1)
+#       define DC_HPDx_SENSE_DELAYED                    (1 << 4)
+#       define DC_HPDx_RX_INT_STATUS                    (1 << 8)
+
+#define DC_HPD1_INT_CONTROL                             0x6020
+#define DC_HPD2_INT_CONTROL                             0x602c
+#define DC_HPD3_INT_CONTROL                             0x6038
+#define DC_HPD4_INT_CONTROL                             0x6044
+#define DC_HPD5_INT_CONTROL                             0x6050
+#define DC_HPD6_INT_CONTROL                             0x605c
+#       define DC_HPDx_INT_ACK                          (1 << 0)
+#       define DC_HPDx_INT_POLARITY                     (1 << 8)
+#       define DC_HPDx_INT_EN                           (1 << 16)
+#       define DC_HPDx_RX_INT_ACK                       (1 << 20)
+#       define DC_HPDx_RX_INT_EN                        (1 << 24)
+
+#define DC_HPD1_CONTROL                                   0x6024
+#define DC_HPD2_CONTROL                                   0x6030
+#define DC_HPD3_CONTROL                                   0x603c
+#define DC_HPD4_CONTROL                                   0x6048
+#define DC_HPD5_CONTROL                                   0x6054
+#define DC_HPD6_CONTROL                                   0x6060
+#       define DC_HPDx_CONNECTION_TIMER(x)                ((x) << 0)
+#       define DC_HPDx_RX_INT_TIMER(x)                    ((x) << 16)
+#       define DC_HPDx_EN                                 (1 << 28)
+
+#define GRBM_CNTL                                       0x8000
+#define         GRBM_READ_TIMEOUT(x)                            ((x) << 0)
+
+#define GRBM_STATUS2                                    0x8008
+#define         ME0PIPE1_CMDFIFO_AVAIL_MASK                     0x0000000F
+#define         ME0PIPE1_CF_RQ_PENDING                          (1 << 4)
+#define         ME0PIPE1_PF_RQ_PENDING                          (1 << 5)
+#define         ME1PIPE0_RQ_PENDING                             (1 << 6)
+#define         ME1PIPE1_RQ_PENDING                             (1 << 7)
+#define         ME1PIPE2_RQ_PENDING                             (1 << 8)
+#define         ME1PIPE3_RQ_PENDING                             (1 << 9)
+#define         ME2PIPE0_RQ_PENDING                             (1 << 10)
+#define         ME2PIPE1_RQ_PENDING                             (1 << 11)
+#define         ME2PIPE2_RQ_PENDING                             (1 << 12)
+#define         ME2PIPE3_RQ_PENDING                             (1 << 13)
+#define         RLC_RQ_PENDING                                  (1 << 14)
+#define         RLC_BUSY                                        (1 << 24)
+#define         TC_BUSY                                         (1 << 25)
+#define         CPF_BUSY                                        (1 << 28)
+#define         CPC_BUSY                                        (1 << 29)
+#define         CPG_BUSY                                        (1 << 30)
+
+#define GRBM_STATUS                                     0x8010
+#define         ME0PIPE0_CMDFIFO_AVAIL_MASK                     0x0000000F
+#define         SRBM_RQ_PENDING                                 (1 << 5)
+#define         ME0PIPE0_CF_RQ_PENDING                          (1 << 7)
+#define         ME0PIPE0_PF_RQ_PENDING                          (1 << 8)
+#define         GDS_DMA_RQ_PENDING                              (1 << 9)
+#define         DB_CLEAN                                        (1 << 12)
+#define         CB_CLEAN                                        (1 << 13)
+#define         TA_BUSY                                         (1 << 14)
+#define         GDS_BUSY                                        (1 << 15)
+#define         WD_BUSY_NO_DMA                                  (1 << 16)
+#define         VGT_BUSY                                        (1 << 17)
+#define         IA_BUSY_NO_DMA                                  (1 << 18)
+#define         IA_BUSY                                         (1 << 19)
+#define         SX_BUSY                                         (1 << 20)
+#define         WD_BUSY                                         (1 << 21)
+#define         SPI_BUSY                                        (1 << 22)
+#define         BCI_BUSY                                        (1 << 23)
+#define         SC_BUSY                                         (1 << 24)
+#define         PA_BUSY                                         (1 << 25)
+#define         DB_BUSY                                         (1 << 26)
+#define         CP_COHERENCY_BUSY                               (1 << 28)
+#define         CP_BUSY                                         (1 << 29)
+#define         CB_BUSY                                         (1 << 30)
+#define         GUI_ACTIVE                                      (1 << 31)
+#define GRBM_STATUS_SE0                                 0x8014
+#define GRBM_STATUS_SE1                                 0x8018
+#define GRBM_STATUS_SE2                                 0x8038
+#define GRBM_STATUS_SE3                                 0x803C
+#define         SE_DB_CLEAN                                     (1 << 1)
+#define         SE_CB_CLEAN                                     (1 << 2)
+#define         SE_BCI_BUSY                                     (1 << 22)
+#define         SE_VGT_BUSY                                     (1 << 23)
+#define         SE_PA_BUSY                                      (1 << 24)
+#define         SE_TA_BUSY                                      (1 << 25)
+#define         SE_SX_BUSY                                      (1 << 26)
+#define         SE_SPI_BUSY                                     (1 << 27)
+#define         SE_SC_BUSY                                      (1 << 29)
+#define         SE_DB_BUSY                                      (1 << 30)
+#define         SE_CB_BUSY                                      (1 << 31)
+
+#define GRBM_SOFT_RESET                                 0x8020
+#define         SOFT_RESET_CP                                   (1 << 0)  /* All CP blocks */
+#define         SOFT_RESET_RLC                                  (1 << 2)  /* RLC */
+#define         SOFT_RESET_GFX                                  (1 << 16) /* GFX */
+#define         SOFT_RESET_CPF                                  (1 << 17) /* CP fetcher shared by gfx and compute */
+#define         SOFT_RESET_CPC                                  (1 << 18) /* CP Compute (MEC1/2) */
+#define         SOFT_RESET_CPG                                  (1 << 19) /* CP GFX (PFP, ME, CE) */
+
+#define GRBM_INT_CNTL                                   0x8060
+#       define RDERR_INT_ENABLE                         (1 << 0)
+#       define GUI_IDLE_INT_ENABLE                      (1 << 19)
+
+#define CP_CPC_STATUS                                   0x8210
+#define CP_CPC_BUSY_STAT                                0x8214
+#define CP_CPC_STALLED_STAT1                            0x8218
+#define CP_CPF_STATUS                                   0x821c
+#define CP_CPF_BUSY_STAT                                0x8220
+#define CP_CPF_STALLED_STAT1                            0x8224
+
+#define CP_MEC_CNTL                                     0x8234
+#define         MEC_ME2_HALT                                    (1 << 28)
+#define         MEC_ME1_HALT                                    (1 << 30)
+
+#define CP_MEC_CNTL                                     0x8234
+#define         MEC_ME2_HALT                                    (1 << 28)
+#define         MEC_ME1_HALT                                    (1 << 30)
+
+#define CP_STALLED_STAT3                                0x8670
+#define CP_STALLED_STAT1                                0x8674
+#define CP_STALLED_STAT2                                0x8678
+
+#define CP_STAT                                         0x8680
+
+#define CP_ME_CNTL                                      0x86D8
+#define         CP_CE_HALT                                      (1 << 24)
+#define         CP_PFP_HALT                                     (1 << 26)
+#define         CP_ME_HALT                                      (1 << 28)
+
+#define CP_RB0_RPTR                                     0x8700
+#define CP_RB_WPTR_DELAY                                0x8704
+
+#define CP_MEQ_THRESHOLDS                               0x8764
+#define         MEQ1_START(x)                           ((x) << 0)
+#define         MEQ2_START(x)                           ((x) << 8)
+
+#define VGT_VTX_VECT_EJECT_REG                          0x88B0
+
+#define VGT_CACHE_INVALIDATION                          0x88C4
+#define         CACHE_INVALIDATION(x)                           ((x) << 0)
+#define                 VC_ONLY                                         0
+#define                 TC_ONLY                                         1
+#define                 VC_AND_TC                                       2
+#define         AUTO_INVLD_EN(x)                                ((x) << 6)
+#define                 NO_AUTO                                         0
+#define                 ES_AUTO                                         1
+#define                 GS_AUTO                                         2
+#define                 ES_AND_GS_AUTO                                  3
+
+#define VGT_GS_VERTEX_REUSE                             0x88D4
+
+#define CC_GC_SHADER_ARRAY_CONFIG                       0x89bc
+#define         INACTIVE_CUS_MASK                       0xFFFF0000
+#define         INACTIVE_CUS_SHIFT                      16
+#define GC_USER_SHADER_ARRAY_CONFIG                     0x89c0
+
+#define PA_CL_ENHANCE                                   0x8A14
+#define         CLIP_VTX_REORDER_ENA                            (1 << 0)
+#define         NUM_CLIP_SEQ(x)                                 ((x) << 1)
+
+#define PA_SC_FORCE_EOV_MAX_CNTS                        0x8B24
+#define         FORCE_EOV_MAX_CLK_CNT(x)                        ((x) << 0)
+#define         FORCE_EOV_MAX_REZ_CNT(x)                        ((x) << 16)
+
+#define PA_SC_FIFO_SIZE                                 0x8BCC
+#define         SC_FRONTEND_PRIM_FIFO_SIZE(x)                   ((x) << 0)
+#define         SC_BACKEND_PRIM_FIFO_SIZE(x)                    ((x) << 6)
+#define         SC_HIZ_TILE_FIFO_SIZE(x)                        ((x) << 15)
+#define         SC_EARLYZ_TILE_FIFO_SIZE(x)                     ((x) << 23)
+
+#define PA_SC_ENHANCE                                   0x8BF0
+#define         ENABLE_PA_SC_OUT_OF_ORDER                       (1 << 0)
+#define         DISABLE_PA_SC_GUIDANCE                          (1 << 13)
+
+#define SQ_CONFIG                                       0x8C00
+
+#define SH_MEM_BASES                                    0x8C28
+/* if PTR32, these are the bases for scratch and lds */
+#define         PRIVATE_BASE(x)                                 ((x) << 0) /* scratch */
+#define         SHARED_BASE(x)                                  ((x) << 16) /* LDS */
+#define SH_MEM_APE1_BASE                                0x8C2C
+/* if PTR32, this is the base location of GPUVM */
+#define SH_MEM_APE1_LIMIT                               0x8C30
+/* if PTR32, this is the upper limit of GPUVM */
+#define SH_MEM_CONFIG                                   0x8C34
+#define         PTR32                                           (1 << 0)
+#define         ALIGNMENT_MODE(x)                               ((x) << 2)
+#define                 SH_MEM_ALIGNMENT_MODE_DWORD                     0
+#define                 SH_MEM_ALIGNMENT_MODE_DWORD_STRICT              1
+#define                 SH_MEM_ALIGNMENT_MODE_STRICT                    2
+#define                 SH_MEM_ALIGNMENT_MODE_UNALIGNED                 3
+#define         DEFAULT_MTYPE(x)                                ((x) << 4)
+#define         APE1_MTYPE(x)                                   ((x) << 7)
+
+#define SX_DEBUG_1                                      0x9060
+
+#define SPI_CONFIG_CNTL                                 0x9100
+
+#define SPI_CONFIG_CNTL_1                               0x913C
+#define         VTX_DONE_DELAY(x)                               ((x) << 0)
+#define         INTERP_ONE_PRIM_PER_ROW                         (1 << 4)
+
+#define TA_CNTL_AUX                                     0x9508
+
+#define DB_DEBUG                                        0x9830
+#define DB_DEBUG2                                       0x9834
+#define DB_DEBUG3                                       0x9838
+
+#define CC_RB_BACKEND_DISABLE                           0x98F4
+#define         BACKEND_DISABLE(x)                      ((x) << 16)
+#define GB_ADDR_CONFIG                                  0x98F8
+#define         NUM_PIPES(x)                            ((x) << 0)
+#define         NUM_PIPES_MASK                          0x00000007
+#define         NUM_PIPES_SHIFT                         0
+#define         PIPE_INTERLEAVE_SIZE(x)                 ((x) << 4)
+#define         PIPE_INTERLEAVE_SIZE_MASK               0x00000070
+#define         PIPE_INTERLEAVE_SIZE_SHIFT              4
+#define         NUM_SHADER_ENGINES(x)                   ((x) << 12)
+#define         NUM_SHADER_ENGINES_MASK                 0x00003000
+#define         NUM_SHADER_ENGINES_SHIFT                12
+#define         SHADER_ENGINE_TILE_SIZE(x)              ((x) << 16)
+#define         SHADER_ENGINE_TILE_SIZE_MASK            0x00070000
+#define         SHADER_ENGINE_TILE_SIZE_SHIFT           16
+#define         ROW_SIZE(x)                             ((x) << 28)
+#define         ROW_SIZE_MASK                           0x30000000
+#define         ROW_SIZE_SHIFT                          28
+
+#define GB_TILE_MODE0                                   0x9910
+#       define ARRAY_MODE(x)                                    ((x) << 2)
+#              define   ARRAY_LINEAR_GENERAL                    0
+#              define   ARRAY_LINEAR_ALIGNED                    1
+#              define   ARRAY_1D_TILED_THIN1                    2
+#              define   ARRAY_2D_TILED_THIN1                    4
+#              define   ARRAY_PRT_TILED_THIN1                   5
+#              define   ARRAY_PRT_2D_TILED_THIN1                6
+#       define PIPE_CONFIG(x)                                   ((x) << 6)
+#              define   ADDR_SURF_P2                            0
+#              define   ADDR_SURF_P4_8x16                       4
+#              define   ADDR_SURF_P4_16x16                      5
+#              define   ADDR_SURF_P4_16x32                      6
+#              define   ADDR_SURF_P4_32x32                      7
+#              define   ADDR_SURF_P8_16x16_8x16                 8
+#              define   ADDR_SURF_P8_16x32_8x16                 9
+#              define   ADDR_SURF_P8_32x32_8x16                 10
+#              define   ADDR_SURF_P8_16x32_16x16                11
+#              define   ADDR_SURF_P8_32x32_16x16                12
+#              define   ADDR_SURF_P8_32x32_16x32                13
+#              define   ADDR_SURF_P8_32x64_32x32                14
+#       define TILE_SPLIT(x)                                    ((x) << 11)
+#              define   ADDR_SURF_TILE_SPLIT_64B                0
+#              define   ADDR_SURF_TILE_SPLIT_128B               1
+#              define   ADDR_SURF_TILE_SPLIT_256B               2
+#              define   ADDR_SURF_TILE_SPLIT_512B               3
+#              define   ADDR_SURF_TILE_SPLIT_1KB                4
+#              define   ADDR_SURF_TILE_SPLIT_2KB                5
+#              define   ADDR_SURF_TILE_SPLIT_4KB                6
+#       define MICRO_TILE_MODE_NEW(x)                           ((x) << 22)
+#              define   ADDR_SURF_DISPLAY_MICRO_TILING          0
+#              define   ADDR_SURF_THIN_MICRO_TILING             1
+#              define   ADDR_SURF_DEPTH_MICRO_TILING            2
+#              define   ADDR_SURF_ROTATED_MICRO_TILING          3
+#       define SAMPLE_SPLIT(x)                                  ((x) << 25)
+#              define   ADDR_SURF_SAMPLE_SPLIT_1                0
+#              define   ADDR_SURF_SAMPLE_SPLIT_2                1
+#              define   ADDR_SURF_SAMPLE_SPLIT_4                2
+#              define   ADDR_SURF_SAMPLE_SPLIT_8                3
+
+#define GB_MACROTILE_MODE0                                      0x9990
+#       define BANK_WIDTH(x)                                    ((x) << 0)
+#              define   ADDR_SURF_BANK_WIDTH_1                  0
+#              define   ADDR_SURF_BANK_WIDTH_2                  1
+#              define   ADDR_SURF_BANK_WIDTH_4                  2
+#              define   ADDR_SURF_BANK_WIDTH_8                  3
+#       define BANK_HEIGHT(x)                                   ((x) << 2)
+#              define   ADDR_SURF_BANK_HEIGHT_1                 0
+#              define   ADDR_SURF_BANK_HEIGHT_2                 1
+#              define   ADDR_SURF_BANK_HEIGHT_4                 2
+#              define   ADDR_SURF_BANK_HEIGHT_8                 3
+#       define MACRO_TILE_ASPECT(x)                             ((x) << 4)
+#              define   ADDR_SURF_MACRO_ASPECT_1                0
+#              define   ADDR_SURF_MACRO_ASPECT_2                1
+#              define   ADDR_SURF_MACRO_ASPECT_4                2
+#              define   ADDR_SURF_MACRO_ASPECT_8                3
+#       define NUM_BANKS(x)                                     ((x) << 6)
+#              define   ADDR_SURF_2_BANK                        0
+#              define   ADDR_SURF_4_BANK                        1
+#              define   ADDR_SURF_8_BANK                        2
+#              define   ADDR_SURF_16_BANK                       3
+
+#define CB_HW_CONTROL                                   0x9A10
+
+#define GC_USER_RB_BACKEND_DISABLE                      0x9B7C
+#define         BACKEND_DISABLE_MASK                    0x00FF0000
+#define         BACKEND_DISABLE_SHIFT                   16
+
+#define TCP_CHAN_STEER_LO                               0xac0c
+#define TCP_CHAN_STEER_HI                               0xac10
+
+#define TC_CFG_L1_LOAD_POLICY0                          0xAC68
+#define TC_CFG_L1_LOAD_POLICY1                          0xAC6C
+#define TC_CFG_L1_STORE_POLICY                          0xAC70
+#define TC_CFG_L2_LOAD_POLICY0                          0xAC74
+#define TC_CFG_L2_LOAD_POLICY1                          0xAC78
+#define TC_CFG_L2_STORE_POLICY0                         0xAC7C
+#define TC_CFG_L2_STORE_POLICY1                         0xAC80
+#define TC_CFG_L2_ATOMIC_POLICY                         0xAC84
+#define TC_CFG_L1_VOLATILE                              0xAC88
+#define TC_CFG_L2_VOLATILE                              0xAC8C
+
+#define CP_RB0_BASE                                     0xC100
+#define CP_RB0_CNTL                                     0xC104
+#define         RB_BUFSZ(x)                                     ((x) << 0)
+#define         RB_BLKSZ(x)                                     ((x) << 8)
+#define         BUF_SWAP_32BIT                                  (2 << 16)
+#define         RB_NO_UPDATE                                    (1 << 27)
+#define         RB_RPTR_WR_ENA                                  (1 << 31)
+
+#define CP_RB0_RPTR_ADDR                                0xC10C
+#define         RB_RPTR_SWAP_32BIT                              (2 << 0)
+#define CP_RB0_RPTR_ADDR_HI                             0xC110
+#define CP_RB0_WPTR                                     0xC114
+
+#define CP_DEVICE_ID                                    0xC12C
+#define CP_ENDIAN_SWAP                                  0xC140
+#define CP_RB_VMID                                      0xC144
+
+#define CP_PFP_UCODE_ADDR                               0xC150
+#define CP_PFP_UCODE_DATA                               0xC154
+#define CP_ME_RAM_RADDR                                 0xC158
+#define CP_ME_RAM_WADDR                                 0xC15C
+#define CP_ME_RAM_DATA                                  0xC160
+
+#define CP_CE_UCODE_ADDR                                0xC168
+#define CP_CE_UCODE_DATA                                0xC16C
+#define CP_MEC_ME1_UCODE_ADDR                           0xC170
+#define CP_MEC_ME1_UCODE_DATA                           0xC174
+#define CP_MEC_ME2_UCODE_ADDR                           0xC178
+#define CP_MEC_ME2_UCODE_DATA                           0xC17C
+
+#define CP_INT_CNTL_RING0                               0xC1A8
+#       define CNTX_BUSY_INT_ENABLE                     (1 << 19)
+#       define CNTX_EMPTY_INT_ENABLE                    (1 << 20)
+#       define PRIV_INSTR_INT_ENABLE                    (1 << 22)
+#       define PRIV_REG_INT_ENABLE                      (1 << 23)
+#       define TIME_STAMP_INT_ENABLE                    (1 << 26)
+#       define CP_RINGID2_INT_ENABLE                    (1 << 29)
+#       define CP_RINGID1_INT_ENABLE                    (1 << 30)
+#       define CP_RINGID0_INT_ENABLE                    (1 << 31)
+
+#define CP_INT_STATUS_RING0                             0xC1B4
+#       define PRIV_INSTR_INT_STAT                      (1 << 22)
+#       define PRIV_REG_INT_STAT                        (1 << 23)
+#       define TIME_STAMP_INT_STAT                      (1 << 26)
+#       define CP_RINGID2_INT_STAT                      (1 << 29)
+#       define CP_RINGID1_INT_STAT                      (1 << 30)
+#       define CP_RINGID0_INT_STAT                      (1 << 31)
+
+#define CP_CPF_DEBUG                                    0xC200
+
+#define CP_PQ_WPTR_POLL_CNTL                            0xC20C
+#define         WPTR_POLL_EN                            (1 << 31)
+
+#define CP_ME1_PIPE0_INT_CNTL                           0xC214
+#define CP_ME1_PIPE1_INT_CNTL                           0xC218
+#define CP_ME1_PIPE2_INT_CNTL                           0xC21C
+#define CP_ME1_PIPE3_INT_CNTL                           0xC220
+#define CP_ME2_PIPE0_INT_CNTL                           0xC224
+#define CP_ME2_PIPE1_INT_CNTL                           0xC228
+#define CP_ME2_PIPE2_INT_CNTL                           0xC22C
+#define CP_ME2_PIPE3_INT_CNTL                           0xC230
+#       define DEQUEUE_REQUEST_INT_ENABLE               (1 << 13)
+#       define WRM_POLL_TIMEOUT_INT_ENABLE              (1 << 17)
+#       define PRIV_REG_INT_ENABLE                      (1 << 23)
+#       define TIME_STAMP_INT_ENABLE                    (1 << 26)
+#       define GENERIC2_INT_ENABLE                      (1 << 29)
+#       define GENERIC1_INT_ENABLE                      (1 << 30)
+#       define GENERIC0_INT_ENABLE                      (1 << 31)
+#define CP_ME1_PIPE0_INT_STATUS                         0xC214
+#define CP_ME1_PIPE1_INT_STATUS                         0xC218
+#define CP_ME1_PIPE2_INT_STATUS                         0xC21C
+#define CP_ME1_PIPE3_INT_STATUS                         0xC220
+#define CP_ME2_PIPE0_INT_STATUS                         0xC224
+#define CP_ME2_PIPE1_INT_STATUS                         0xC228
+#define CP_ME2_PIPE2_INT_STATUS                         0xC22C
+#define CP_ME2_PIPE3_INT_STATUS                         0xC230
+#       define DEQUEUE_REQUEST_INT_STATUS               (1 << 13)
+#       define WRM_POLL_TIMEOUT_INT_STATUS              (1 << 17)
+#       define PRIV_REG_INT_STATUS                      (1 << 23)
+#       define TIME_STAMP_INT_STATUS                    (1 << 26)
+#       define GENERIC2_INT_STATUS                      (1 << 29)
+#       define GENERIC1_INT_STATUS                      (1 << 30)
+#       define GENERIC0_INT_STATUS                      (1 << 31)
+
+#define CP_MAX_CONTEXT                                  0xC2B8
+
+#define CP_RB0_BASE_HI                                  0xC2C4
+
+#define RLC_CNTL                                          0xC300
+#       define RLC_ENABLE                                 (1 << 0)
+
+#define RLC_MC_CNTL                                       0xC30C
+
+#define RLC_LB_CNTR_MAX                                   0xC348
+
+#define RLC_LB_CNTL                                       0xC364
+
+#define RLC_LB_CNTR_INIT                                  0xC36C
+
+#define RLC_SAVE_AND_RESTORE_BASE                         0xC374
+#define RLC_DRIVER_DMA_STATUS                             0xC378
+
+#define RLC_GPM_UCODE_ADDR                                0xC388
+#define RLC_GPM_UCODE_DATA                                0xC38C
+#define RLC_GPU_CLOCK_COUNT_LSB                           0xC390
+#define RLC_GPU_CLOCK_COUNT_MSB                           0xC394
+#define RLC_CAPTURE_GPU_CLOCK_COUNT                       0xC398
+#define RLC_UCODE_CNTL                                    0xC39C
+
+#define RLC_CGCG_CGLS_CTRL                                0xC424
+
+#define RLC_LB_INIT_CU_MASK                               0xC43C
+
+#define RLC_LB_PARAMS                                     0xC444
+
+#define RLC_SERDES_CU_MASTER_BUSY                         0xC484
+#define RLC_SERDES_NONCU_MASTER_BUSY                      0xC488
+#       define SE_MASTER_BUSY_MASK                        0x0000ffff
+#       define GC_MASTER_BUSY                             (1 << 16)
+#       define TC0_MASTER_BUSY                            (1 << 17)
+#       define TC1_MASTER_BUSY                            (1 << 18)
+
+#define RLC_GPM_SCRATCH_ADDR                              0xC4B0
+#define RLC_GPM_SCRATCH_DATA                              0xC4B4
+
+#define CP_HPD_EOP_BASE_ADDR                              0xC904
+#define CP_HPD_EOP_BASE_ADDR_HI                           0xC908
+#define CP_HPD_EOP_VMID                                   0xC90C
+#define CP_HPD_EOP_CONTROL                                0xC910
+#define         EOP_SIZE(x)                             ((x) << 0)
+#define         EOP_SIZE_MASK                           (0x3f << 0)
+#define CP_MQD_BASE_ADDR                                  0xC914
+#define CP_MQD_BASE_ADDR_HI                               0xC918
+#define CP_HQD_ACTIVE                                     0xC91C
+#define CP_HQD_VMID                                       0xC920
+
+#define CP_HQD_PQ_BASE                                    0xC934
+#define CP_HQD_PQ_BASE_HI                                 0xC938
+#define CP_HQD_PQ_RPTR                                    0xC93C
+#define CP_HQD_PQ_RPTR_REPORT_ADDR                        0xC940
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI                     0xC944
+#define CP_HQD_PQ_WPTR_POLL_ADDR                          0xC948
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI                       0xC94C
+#define CP_HQD_PQ_DOORBELL_CONTROL                        0xC950
+#define         DOORBELL_OFFSET(x)                      ((x) << 2)
+#define         DOORBELL_OFFSET_MASK                    (0x1fffff << 2)
+#define         DOORBELL_SOURCE                         (1 << 28)
+#define         DOORBELL_SCHD_HIT                       (1 << 29)
+#define         DOORBELL_EN                             (1 << 30)
+#define         DOORBELL_HIT                            (1 << 31)
+#define CP_HQD_PQ_WPTR                                    0xC954
+#define CP_HQD_PQ_CONTROL                                 0xC958
+#define         QUEUE_SIZE(x)                           ((x) << 0)
+#define         QUEUE_SIZE_MASK                         (0x3f << 0)
+#define         RPTR_BLOCK_SIZE(x)                      ((x) << 8)
+#define         RPTR_BLOCK_SIZE_MASK                    (0x3f << 8)
+#define         PQ_VOLATILE                             (1 << 26)
+#define         NO_UPDATE_RPTR                          (1 << 27)
+#define         UNORD_DISPATCH                          (1 << 28)
+#define         ROQ_PQ_IB_FLIP                          (1 << 29)
+#define         PRIV_STATE                              (1 << 30)
+#define         KMD_QUEUE                               (1 << 31)
+
+#define CP_HQD_DEQUEUE_REQUEST                          0xC974
+
+#define CP_MQD_CONTROL                                  0xC99C
+#define         MQD_VMID(x)                             ((x) << 0)
+#define         MQD_VMID_MASK                           (0xf << 0)
+
+#define PA_SC_RASTER_CONFIG                             0x28350
+#       define RASTER_CONFIG_RB_MAP_0                   0
+#       define RASTER_CONFIG_RB_MAP_1                   1
+#       define RASTER_CONFIG_RB_MAP_2                   2
+#       define RASTER_CONFIG_RB_MAP_3                   3
+
+#define VGT_EVENT_INITIATOR                             0x28a90
+#       define SAMPLE_STREAMOUTSTATS1                   (1 << 0)
+#       define SAMPLE_STREAMOUTSTATS2                   (2 << 0)
+#       define SAMPLE_STREAMOUTSTATS3                   (3 << 0)
+#       define CACHE_FLUSH_TS                           (4 << 0)
+#       define CACHE_FLUSH                              (6 << 0)
+#       define CS_PARTIAL_FLUSH                         (7 << 0)
+#       define VGT_STREAMOUT_RESET                      (10 << 0)
+#       define END_OF_PIPE_INCR_DE                      (11 << 0)
+#       define END_OF_PIPE_IB_END                       (12 << 0)
+#       define RST_PIX_CNT                              (13 << 0)
+#       define VS_PARTIAL_FLUSH                         (15 << 0)
+#       define PS_PARTIAL_FLUSH                         (16 << 0)
+#       define CACHE_FLUSH_AND_INV_TS_EVENT             (20 << 0)
+#       define ZPASS_DONE                               (21 << 0)
+#       define CACHE_FLUSH_AND_INV_EVENT                (22 << 0)
+#       define PERFCOUNTER_START                        (23 << 0)
+#       define PERFCOUNTER_STOP                         (24 << 0)
+#       define PIPELINESTAT_START                       (25 << 0)
+#       define PIPELINESTAT_STOP                        (26 << 0)
+#       define PERFCOUNTER_SAMPLE                       (27 << 0)
+#       define SAMPLE_PIPELINESTAT                      (30 << 0)
+#       define SO_VGT_STREAMOUT_FLUSH                   (31 << 0)
+#       define SAMPLE_STREAMOUTSTATS                    (32 << 0)
+#       define RESET_VTX_CNT                            (33 << 0)
+#       define VGT_FLUSH                                (36 << 0)
+#       define BOTTOM_OF_PIPE_TS                        (40 << 0)
+#       define DB_CACHE_FLUSH_AND_INV                   (42 << 0)
+#       define FLUSH_AND_INV_DB_DATA_TS                 (43 << 0)
+#       define FLUSH_AND_INV_DB_META                    (44 << 0)
+#       define FLUSH_AND_INV_CB_DATA_TS                 (45 << 0)
+#       define FLUSH_AND_INV_CB_META                    (46 << 0)
+#       define CS_DONE                                  (47 << 0)
+#       define PS_DONE                                  (48 << 0)
+#       define FLUSH_AND_INV_CB_PIXEL_DATA              (49 << 0)
+#       define THREAD_TRACE_START                       (51 << 0)
+#       define THREAD_TRACE_STOP                        (52 << 0)
+#       define THREAD_TRACE_FLUSH                       (54 << 0)
+#       define THREAD_TRACE_FINISH                      (55 << 0)
+#       define PIXEL_PIPE_STAT_CONTROL                  (56 << 0)
+#       define PIXEL_PIPE_STAT_DUMP                     (57 << 0)
+#       define PIXEL_PIPE_STAT_RESET                    (58 << 0)
+
+#define SCRATCH_REG0                                    0x30100
+#define SCRATCH_REG1                                    0x30104
+#define SCRATCH_REG2                                    0x30108
+#define SCRATCH_REG3                                    0x3010C
+#define SCRATCH_REG4                                    0x30110
+#define SCRATCH_REG5                                    0x30114
+#define SCRATCH_REG6                                    0x30118
+#define SCRATCH_REG7                                    0x3011C
+
+#define SCRATCH_UMSK                                    0x30140
+#define SCRATCH_ADDR                                    0x30144
+
+#define CP_SEM_WAIT_TIMER                               0x301BC
+
+#define CP_SEM_INCOMPLETE_TIMER_CNTL                    0x301C8
+
+#define CP_WAIT_REG_MEM_TIMEOUT                         0x301D0
+
+#define GRBM_GFX_INDEX                                  0x30800
+#define         INSTANCE_INDEX(x)                       ((x) << 0)
+#define         SH_INDEX(x)                             ((x) << 8)
+#define         SE_INDEX(x)                             ((x) << 16)
+#define         SH_BROADCAST_WRITES                     (1 << 29)
+#define         INSTANCE_BROADCAST_WRITES               (1 << 30)
+#define         SE_BROADCAST_WRITES                     (1 << 31)
+
+#define VGT_ESGS_RING_SIZE                              0x30900
+#define VGT_GSVS_RING_SIZE                              0x30904
+#define VGT_PRIMITIVE_TYPE                              0x30908
+#define VGT_INDEX_TYPE                                  0x3090C
+
+#define VGT_NUM_INDICES                                 0x30930
+#define VGT_NUM_INSTANCES                               0x30934
+#define VGT_TF_RING_SIZE                                0x30938
+#define VGT_HS_OFFCHIP_PARAM                            0x3093C
+#define VGT_TF_MEMORY_BASE                              0x30940
+
+#define PA_SU_LINE_STIPPLE_VALUE                        0x30a00
+#define PA_SC_LINE_STIPPLE_STATE                        0x30a04
+
+#define SQC_CACHES                                      0x30d20
+
+#define CP_PERFMON_CNTL                                 0x36020
+
+#define CGTS_TCC_DISABLE                                0x3c00c
+#define CGTS_USER_TCC_DISABLE                           0x3c010
+#define         TCC_DISABLE_MASK                                0xFFFF0000
+#define         TCC_DISABLE_SHIFT                               16
+
+#define CB_CGTT_SCLK_CTRL                               0x3c2a0
+
+/*
+ * PM4
+ */
+#define PACKET_TYPE0    0
+#define PACKET_TYPE1    1
+#define PACKET_TYPE2    2
+#define PACKET_TYPE3    3
+
+#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3)
+#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF)
+#define CP_PACKET0_GET_REG(h) (((h) & 0xFFFF) << 2)
+#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF)
+#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) |                         \
+                         (((reg) >> 2) & 0xFFFF) |                      \
+                         ((n) & 0x3FFF) << 16)
+#define CP_PACKET2                      0x80000000
+#define         PACKET2_PAD_SHIFT               0
+#define         PACKET2_PAD_MASK                (0x3fffffff << 0)
+
+#define PACKET2(v)      (CP_PACKET2 | REG_SET(PACKET2_PAD, (v)))
+
+#define PACKET3(op, n)  ((PACKET_TYPE3 << 30) |                         \
+                         (((op) & 0xFF) << 8) |                         \
+                         ((n) & 0x3FFF) << 16)
+
+#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1)
+
+/* Packet 3 types */
+#define PACKET3_NOP                                     0x10
+#define PACKET3_SET_BASE                                0x11
+#define         PACKET3_BASE_INDEX(x)                  ((x) << 0)
+#define                 CE_PARTITION_BASE               3
+#define PACKET3_CLEAR_STATE                             0x12
+#define PACKET3_INDEX_BUFFER_SIZE                       0x13
+#define PACKET3_DISPATCH_DIRECT                         0x15
+#define PACKET3_DISPATCH_INDIRECT                       0x16
+#define PACKET3_ATOMIC_GDS                              0x1D
+#define PACKET3_ATOMIC_MEM                              0x1E
+#define PACKET3_OCCLUSION_QUERY                         0x1F
+#define PACKET3_SET_PREDICATION                         0x20
+#define PACKET3_REG_RMW                                 0x21
+#define PACKET3_COND_EXEC                               0x22
+#define PACKET3_PRED_EXEC                               0x23
+#define PACKET3_DRAW_INDIRECT                           0x24
+#define PACKET3_DRAW_INDEX_INDIRECT                     0x25
+#define PACKET3_INDEX_BASE                              0x26
+#define PACKET3_DRAW_INDEX_2                            0x27
+#define PACKET3_CONTEXT_CONTROL                         0x28
+#define PACKET3_INDEX_TYPE                              0x2A
+#define PACKET3_DRAW_INDIRECT_MULTI                     0x2C
+#define PACKET3_DRAW_INDEX_AUTO                         0x2D
+#define PACKET3_NUM_INSTANCES                           0x2F
+#define PACKET3_DRAW_INDEX_MULTI_AUTO                   0x30
+#define PACKET3_INDIRECT_BUFFER_CONST                   0x33
+#define PACKET3_STRMOUT_BUFFER_UPDATE                   0x34
+#define PACKET3_DRAW_INDEX_OFFSET_2                     0x35
+#define PACKET3_DRAW_PREAMBLE                           0x36
+#define PACKET3_WRITE_DATA                              0x37
+#define         WRITE_DATA_DST_SEL(x)                   ((x) << 8)
+                /* 0 - register
+                 * 1 - memory (sync - via GRBM)
+                 * 2 - gl2
+                 * 3 - gds
+                 * 4 - reserved
+                 * 5 - memory (async - direct)
+                 */
+#define         WR_ONE_ADDR                             (1 << 16)
+#define         WR_CONFIRM                              (1 << 20)
+#define         WRITE_DATA_CACHE_POLICY(x)              ((x) << 25)
+                /* 0 - LRU
+                 * 1 - Stream
+                 */
+#define         WRITE_DATA_ENGINE_SEL(x)                ((x) << 30)
+                /* 0 - me
+                 * 1 - pfp
+                 * 2 - ce
+                 */
+#define PACKET3_DRAW_INDEX_INDIRECT_MULTI               0x38
+#define PACKET3_MEM_SEMAPHORE                           0x39
+#              define PACKET3_SEM_USE_MAILBOX       (0x1 << 16)
+#              define PACKET3_SEM_SEL_SIGNAL_TYPE   (0x1 << 20) /* 0 = increment, 1 = write 1 */
+#              define PACKET3_SEM_CLIENT_CODE       ((x) << 24) /* 0 = CP, 1 = CB, 2 = DB */
+#              define PACKET3_SEM_SEL_SIGNAL        (0x6 << 29)
+#              define PACKET3_SEM_SEL_WAIT          (0x7 << 29)
+#define PACKET3_COPY_DW                                 0x3B
+#define PACKET3_WAIT_REG_MEM                            0x3C
+#define         WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+                /* 0 - always
+                 * 1 - <
+                 * 2 - <=
+                 * 3 - ==
+                 * 4 - !=
+                 * 5 - >=
+                 * 6 - >
+                 */
+#define         WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+                /* 0 - reg
+                 * 1 - mem
+                 */
+#define         WAIT_REG_MEM_OPERATION(x)               ((x) << 6)
+                /* 0 - wait_reg_mem
+                 * 1 - wr_wait_wr_reg
+                 */
+#define         WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+                /* 0 - me
+                 * 1 - pfp
+                 */
+#define PACKET3_INDIRECT_BUFFER                         0x3F
+#define         INDIRECT_BUFFER_TCL2_VOLATILE           (1 << 22)
+#define         INDIRECT_BUFFER_VALID                   (1 << 23)
+#define         INDIRECT_BUFFER_CACHE_POLICY(x)         ((x) << 28)
+                /* 0 - LRU
+                 * 1 - Stream
+                 * 2 - Bypass
+                 */
+#define PACKET3_COPY_DATA                               0x40
+#define PACKET3_PFP_SYNC_ME                             0x42
+#define PACKET3_SURFACE_SYNC                            0x43
+#              define PACKET3_DEST_BASE_0_ENA      (1 << 0)
+#              define PACKET3_DEST_BASE_1_ENA      (1 << 1)
+#              define PACKET3_CB0_DEST_BASE_ENA    (1 << 6)
+#              define PACKET3_CB1_DEST_BASE_ENA    (1 << 7)
+#              define PACKET3_CB2_DEST_BASE_ENA    (1 << 8)
+#              define PACKET3_CB3_DEST_BASE_ENA    (1 << 9)
+#              define PACKET3_CB4_DEST_BASE_ENA    (1 << 10)
+#              define PACKET3_CB5_DEST_BASE_ENA    (1 << 11)
+#              define PACKET3_CB6_DEST_BASE_ENA    (1 << 12)
+#              define PACKET3_CB7_DEST_BASE_ENA    (1 << 13)
+#              define PACKET3_DB_DEST_BASE_ENA     (1 << 14)
+#              define PACKET3_TCL1_VOL_ACTION_ENA  (1 << 15)
+#              define PACKET3_TC_VOL_ACTION_ENA    (1 << 16) /* L2 */
+#              define PACKET3_TC_WB_ACTION_ENA     (1 << 18) /* L2 */
+#              define PACKET3_DEST_BASE_2_ENA      (1 << 19)
+#              define PACKET3_DEST_BASE_3_ENA      (1 << 21)
+#              define PACKET3_TCL1_ACTION_ENA      (1 << 22)
+#              define PACKET3_TC_ACTION_ENA        (1 << 23) /* L2 */
+#              define PACKET3_CB_ACTION_ENA        (1 << 25)
+#              define PACKET3_DB_ACTION_ENA        (1 << 26)
+#              define PACKET3_SH_KCACHE_ACTION_ENA (1 << 27)
+#              define PACKET3_SH_KCACHE_VOL_ACTION_ENA (1 << 28)
+#              define PACKET3_SH_ICACHE_ACTION_ENA (1 << 29)
+#define PACKET3_COND_WRITE                              0x45
+#define PACKET3_EVENT_WRITE                             0x46
+#define         EVENT_TYPE(x)                           ((x) << 0)
+#define         EVENT_INDEX(x)                          ((x) << 8)
+                /* 0 - any non-TS event
+                 * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_*
+                 * 2 - SAMPLE_PIPELINESTAT
+                 * 3 - SAMPLE_STREAMOUTSTAT*
+                 * 4 - *S_PARTIAL_FLUSH
+                 * 5 - EOP events
+                 * 6 - EOS events
+                 */
+#define PACKET3_EVENT_WRITE_EOP                         0x47
+#define         EOP_TCL1_VOL_ACTION_EN                  (1 << 12)
+#define         EOP_TC_VOL_ACTION_EN                    (1 << 13) /* L2 */
+#define         EOP_TC_WB_ACTION_EN                     (1 << 15) /* L2 */
+#define         EOP_TCL1_ACTION_EN                      (1 << 16)
+#define         EOP_TC_ACTION_EN                        (1 << 17) /* L2 */
+#define         EOP_CACHE_POLICY(x)                     ((x) << 25)
+                /* 0 - LRU
+                 * 1 - Stream
+                 * 2 - Bypass
+                 */
+#define         EOP_TCL2_VOLATILE                       (1 << 27)
+#define         DATA_SEL(x)                             ((x) << 29)
+                /* 0 - discard
+                 * 1 - send low 32bit data
+                 * 2 - send 64bit data
+                 * 3 - send 64bit GPU counter value
+                 * 4 - send 64bit sys counter value
+                 */
+#define         INT_SEL(x)                              ((x) << 24)
+                /* 0 - none
+                 * 1 - interrupt only (DATA_SEL = 0)
+                 * 2 - interrupt when data write is confirmed
+                 */
+#define         DST_SEL(x)                              ((x) << 16)
+                /* 0 - MC
+                 * 1 - TC/L2
+                 */
+#define PACKET3_EVENT_WRITE_EOS                         0x48
+#define PACKET3_RELEASE_MEM                             0x49
+#define PACKET3_PREAMBLE_CNTL                           0x4A
+#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
+#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
+#define PACKET3_DMA_DATA                                0x50
+#define PACKET3_AQUIRE_MEM                              0x58
+#define PACKET3_REWIND                                  0x59
+#define PACKET3_LOAD_UCONFIG_REG                        0x5E
+#define PACKET3_LOAD_SH_REG                             0x5F
+#define PACKET3_LOAD_CONFIG_REG                         0x60
+#define PACKET3_LOAD_CONTEXT_REG                        0x61
+#define PACKET3_SET_CONFIG_REG                          0x68
+#define         PACKET3_SET_CONFIG_REG_START                    0x00008000
+#define         PACKET3_SET_CONFIG_REG_END                      0x0000b000
+#define PACKET3_SET_CONTEXT_REG                         0x69
+#define         PACKET3_SET_CONTEXT_REG_START                   0x00028000
+#define         PACKET3_SET_CONTEXT_REG_END                     0x00029000
+#define PACKET3_SET_CONTEXT_REG_INDIRECT                0x73
+#define PACKET3_SET_SH_REG                              0x76
+#define         PACKET3_SET_SH_REG_START                        0x0000b000
+#define         PACKET3_SET_SH_REG_END                          0x0000c000
+#define PACKET3_SET_SH_REG_OFFSET                       0x77
+#define PACKET3_SET_QUEUE_REG                           0x78
+#define PACKET3_SET_UCONFIG_REG                         0x79
+#define         PACKET3_SET_UCONFIG_REG_START                   0x00030000
+#define         PACKET3_SET_UCONFIG_REG_END                     0x00031000
+#define PACKET3_SCRATCH_RAM_WRITE                       0x7D
+#define PACKET3_SCRATCH_RAM_READ                        0x7E
+#define PACKET3_LOAD_CONST_RAM                          0x80
+#define PACKET3_WRITE_CONST_RAM                         0x81
+#define PACKET3_DUMP_CONST_RAM                          0x83
+#define PACKET3_INCREMENT_CE_COUNTER                    0x84
+#define PACKET3_INCREMENT_DE_COUNTER                    0x85
+#define PACKET3_WAIT_ON_CE_COUNTER                      0x86
+#define PACKET3_WAIT_ON_DE_COUNTER_DIFF                 0x88
+#define PACKET3_SWITCH_BUFFER                           0x8B
+
+/* SDMA - first instance at 0xd000, second at 0xd800 */
+#define SDMA0_REGISTER_OFFSET                             0x0 /* not a register */
+#define SDMA1_REGISTER_OFFSET                             0x800 /* not a register */
+
+#define SDMA0_UCODE_ADDR                                  0xD000
+#define SDMA0_UCODE_DATA                                  0xD004
+
+#define SDMA0_CNTL                                        0xD010
+#       define TRAP_ENABLE                                (1 << 0)
+#       define SEM_INCOMPLETE_INT_ENABLE                  (1 << 1)
+#       define SEM_WAIT_INT_ENABLE                        (1 << 2)
+#       define DATA_SWAP_ENABLE                           (1 << 3)
+#       define FENCE_SWAP_ENABLE                          (1 << 4)
+#       define AUTO_CTXSW_ENABLE                          (1 << 18)
+#       define CTXEMPTY_INT_ENABLE                        (1 << 28)
+
+#define SDMA0_TILING_CONFIG                               0xD018
+
+#define SDMA0_SEM_INCOMPLETE_TIMER_CNTL                   0xD020
+#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL                    0xD024
+
+#define SDMA0_STATUS_REG                                  0xd034
+#       define SDMA_IDLE                                  (1 << 0)
+
+#define SDMA0_ME_CNTL                                     0xD048
+#       define SDMA_HALT                                  (1 << 0)
+
+#define SDMA0_GFX_RB_CNTL                                 0xD200
+#       define SDMA_RB_ENABLE                             (1 << 0)
+#       define SDMA_RB_SIZE(x)                            ((x) << 1) /* log2 */
+#       define SDMA_RB_SWAP_ENABLE                        (1 << 9) /* 8IN32 */
+#       define SDMA_RPTR_WRITEBACK_ENABLE                 (1 << 12)
+#       define SDMA_RPTR_WRITEBACK_SWAP_ENABLE            (1 << 13)  /* 8IN32 */
+#       define SDMA_RPTR_WRITEBACK_TIMER(x)               ((x) << 16) /* log2 */
+#define SDMA0_GFX_RB_BASE                                 0xD204
+#define SDMA0_GFX_RB_BASE_HI                              0xD208
+#define SDMA0_GFX_RB_RPTR                                 0xD20C
+#define SDMA0_GFX_RB_WPTR                                 0xD210
+
+#define SDMA0_GFX_RB_RPTR_ADDR_HI                         0xD220
+#define SDMA0_GFX_RB_RPTR_ADDR_LO                         0xD224
+#define SDMA0_GFX_IB_CNTL                                 0xD228
+#       define SDMA_IB_ENABLE                             (1 << 0)
+#       define SDMA_IB_SWAP_ENABLE                        (1 << 4)
+#       define SDMA_SWITCH_INSIDE_IB                      (1 << 8)
+#       define SDMA_CMD_VMID(x)                           ((x) << 16)
+
+#define SDMA0_GFX_VIRTUAL_ADDR                            0xD29C
+#define SDMA0_GFX_APE1_CNTL                               0xD2A0
+
+#define SDMA_PACKET(op, sub_op, e)      ((((e) & 0xFFFF) << 16) |       \
+                                         (((sub_op) & 0xFF) << 8) |     \
+                                         (((op) & 0xFF) << 0))
+/* sDMA opcodes */
+#define SDMA_OPCODE_NOP                                   0
+#define SDMA_OPCODE_COPY                                  1
+#       define SDMA_COPY_SUB_OPCODE_LINEAR                0
+#       define SDMA_COPY_SUB_OPCODE_TILED                 1
+#       define SDMA_COPY_SUB_OPCODE_SOA                   3
+#       define SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW     4
+#       define SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW      5
+#       define SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW        6
+#define SDMA_OPCODE_WRITE                                 2
+#       define SDMA_WRITE_SUB_OPCODE_LINEAR               0
+#       define SDMA_WRTIE_SUB_OPCODE_TILED                1
+#define SDMA_OPCODE_INDIRECT_BUFFER                       4
+#define SDMA_OPCODE_FENCE                                 5
+#define SDMA_OPCODE_TRAP                                  6
+#define SDMA_OPCODE_SEMAPHORE                             7
+#       define SDMA_SEMAPHORE_EXTRA_O                     (1 << 13)
+                /* 0 - increment
+                 * 1 - write 1
+                 */
+#       define SDMA_SEMAPHORE_EXTRA_S                     (1 << 14)
+                /* 0 - wait
+                 * 1 - signal
+                 */
+#       define SDMA_SEMAPHORE_EXTRA_M                     (1 << 15)
+                /* mailbox */
+#define SDMA_OPCODE_POLL_REG_MEM                          8
+#       define SDMA_POLL_REG_MEM_EXTRA_OP(x)              ((x) << 10)
+                /* 0 - wait_reg_mem
+                 * 1 - wr_wait_wr_reg
+                 */
+#       define SDMA_POLL_REG_MEM_EXTRA_FUNC(x)            ((x) << 12)
+                /* 0 - always
+                 * 1 - <
+                 * 2 - <=
+                 * 3 - ==
+                 * 4 - !=
+                 * 5 - >=
+                 * 6 - >
+                 */
+#       define SDMA_POLL_REG_MEM_EXTRA_M                  (1 << 15)
+                /* 0 = register
+                 * 1 = memory
+                 */
+#define SDMA_OPCODE_COND_EXEC                             9
+#define SDMA_OPCODE_CONSTANT_FILL                         11
+#       define SDMA_CONSTANT_FILL_EXTRA_SIZE(x)           ((x) << 14)
+                /* 0 = byte fill
+                 * 2 = DW fill
+                 */
+#define SDMA_OPCODE_GENERATE_PTE_PDE                      12
+#define SDMA_OPCODE_TIMESTAMP                             13
+#       define SDMA_TIMESTAMP_SUB_OPCODE_SET_LOCAL        0
+#       define SDMA_TIMESTAMP_SUB_OPCODE_GET_LOCAL        1
+#       define SDMA_TIMESTAMP_SUB_OPCODE_GET_GLOBAL       2
+#define SDMA_OPCODE_SRBM_WRITE                            14
+#       define SDMA_SRBM_WRITE_EXTRA_BYTE_ENABLE(x)       ((x) << 12)
+                /* byte mask */
+
+/* UVD */
+
+#define UVD_UDEC_ADDR_CONFIG            0xef4c
+#define UVD_UDEC_DB_ADDR_CONFIG         0xef50
+#define UVD_UDEC_DBW_ADDR_CONFIG        0xef54
+
+#define UVD_LMI_EXT40_ADDR              0xf498
+#define UVD_LMI_ADDR_EXT                0xf594
+#define UVD_VCPU_CACHE_OFFSET0          0xf608
+#define UVD_VCPU_CACHE_SIZE0            0xf60c
+#define UVD_VCPU_CACHE_OFFSET1          0xf610
+#define UVD_VCPU_CACHE_SIZE1            0xf614
+#define UVD_VCPU_CACHE_OFFSET2          0xf618
+#define UVD_VCPU_CACHE_SIZE2            0xf61c
+
+#define UVD_RBC_RB_RPTR                 0xf690
+#define UVD_RBC_RB_WPTR                 0xf694
+
+/* UVD clocks */
+
+#define CG_DCLK_CNTL                    0xC050009C
+#       define DCLK_DIVIDER_MASK        0x7f
+#       define DCLK_DIR_CNTL_EN         (1 << 8)
+#define CG_DCLK_STATUS                  0xC05000A0
+#       define DCLK_STATUS              (1 << 0)
+#define CG_VCLK_CNTL                    0xC05000A4
+#define CG_VCLK_STATUS                  0xC05000A8
+
+#endif
diff --git a/drivers/gpu/drm/radeon/clearstate_cayman.h b/drivers/gpu/drm/radeon/clearstate_cayman.h
new file mode 100644
index 000000000000..c00339440c5e
--- /dev/null
+++ b/drivers/gpu/drm/radeon/clearstate_cayman.h
@@ -0,0 +1,1081 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+static const u32 SECT_CONTEXT_def_1[] =
+{
+    0x00000000, // DB_RENDER_CONTROL
+    0x00000000, // DB_COUNT_CONTROL
+    0x00000000, // DB_DEPTH_VIEW
+    0x00000000, // DB_RENDER_OVERRIDE
+    0x00000000, // DB_RENDER_OVERRIDE2
+    0x00000000, // DB_HTILE_DATA_BASE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_STENCIL_CLEAR
+    0x00000000, // DB_DEPTH_CLEAR
+    0x00000000, // PA_SC_SCREEN_SCISSOR_TL
+    0x40004000, // PA_SC_SCREEN_SCISSOR_BR
+    0, // HOLE
+    0x00000000, // DB_DEPTH_INFO
+    0x00000000, // DB_Z_INFO
+    0x00000000, // DB_STENCIL_INFO
+    0x00000000, // DB_Z_READ_BASE
+    0x00000000, // DB_STENCIL_READ_BASE
+    0x00000000, // DB_Z_WRITE_BASE
+    0x00000000, // DB_STENCIL_WRITE_BASE
+    0x00000000, // DB_DEPTH_SIZE
+    0x00000000, // DB_DEPTH_SLICE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_15
+    0x00000000, // PA_SC_WINDOW_OFFSET
+    0x80000000, // PA_SC_WINDOW_SCISSOR_TL
+    0x40004000, // PA_SC_WINDOW_SCISSOR_BR
+    0x0000ffff, // PA_SC_CLIPRECT_RULE
+    0x00000000, // PA_SC_CLIPRECT_0_TL
+    0x40004000, // PA_SC_CLIPRECT_0_BR
+    0x00000000, // PA_SC_CLIPRECT_1_TL
+    0x40004000, // PA_SC_CLIPRECT_1_BR
+    0x00000000, // PA_SC_CLIPRECT_2_TL
+    0x40004000, // PA_SC_CLIPRECT_2_BR
+    0x00000000, // PA_SC_CLIPRECT_3_TL
+    0x40004000, // PA_SC_CLIPRECT_3_BR
+    0xaa99aaaa, // PA_SC_EDGERULE
+    0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET
+    0xffffffff, // CB_TARGET_MASK
+    0xffffffff, // CB_SHADER_MASK
+    0x80000000, // PA_SC_GENERIC_SCISSOR_TL
+    0x40004000, // PA_SC_GENERIC_SCISSOR_BR
+    0x00000000, // COHER_DEST_BASE_0
+    0x00000000, // COHER_DEST_BASE_1
+    0x80000000, // PA_SC_VPORT_SCISSOR_0_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_0_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_1_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_1_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_2_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_2_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_3_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_3_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_4_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_4_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_5_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_5_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_6_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_6_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_7_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_7_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_8_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_8_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_9_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_9_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_10_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_10_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_11_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_11_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_12_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_12_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_13_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_13_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_14_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_14_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_15_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_15_BR
+    0x00000000, // PA_SC_VPORT_ZMIN_0
+    0x3f800000, // PA_SC_VPORT_ZMAX_0
+    0x00000000, // PA_SC_VPORT_ZMIN_1
+    0x3f800000, // PA_SC_VPORT_ZMAX_1
+    0x00000000, // PA_SC_VPORT_ZMIN_2
+    0x3f800000, // PA_SC_VPORT_ZMAX_2
+    0x00000000, // PA_SC_VPORT_ZMIN_3
+    0x3f800000, // PA_SC_VPORT_ZMAX_3
+    0x00000000, // PA_SC_VPORT_ZMIN_4
+    0x3f800000, // PA_SC_VPORT_ZMAX_4
+    0x00000000, // PA_SC_VPORT_ZMIN_5
+    0x3f800000, // PA_SC_VPORT_ZMAX_5
+    0x00000000, // PA_SC_VPORT_ZMIN_6
+    0x3f800000, // PA_SC_VPORT_ZMAX_6
+    0x00000000, // PA_SC_VPORT_ZMIN_7
+    0x3f800000, // PA_SC_VPORT_ZMAX_7
+    0x00000000, // PA_SC_VPORT_ZMIN_8
+    0x3f800000, // PA_SC_VPORT_ZMAX_8
+    0x00000000, // PA_SC_VPORT_ZMIN_9
+    0x3f800000, // PA_SC_VPORT_ZMAX_9
+    0x00000000, // PA_SC_VPORT_ZMIN_10
+    0x3f800000, // PA_SC_VPORT_ZMAX_10
+    0x00000000, // PA_SC_VPORT_ZMIN_11
+    0x3f800000, // PA_SC_VPORT_ZMAX_11
+    0x00000000, // PA_SC_VPORT_ZMIN_12
+    0x3f800000, // PA_SC_VPORT_ZMAX_12
+    0x00000000, // PA_SC_VPORT_ZMIN_13
+    0x3f800000, // PA_SC_VPORT_ZMAX_13
+    0x00000000, // PA_SC_VPORT_ZMIN_14
+    0x3f800000, // PA_SC_VPORT_ZMAX_14
+    0x00000000, // PA_SC_VPORT_ZMIN_15
+    0x3f800000, // PA_SC_VPORT_ZMAX_15
+    0x00000000, // SX_MISC
+    0x00000000, // SX_SURFACE_SYNC
+    0x00000000, // SX_SCATTER_EXPORT_BASE
+    0x00000000, // SX_SCATTER_EXPORT_SIZE
+    0x00000000, // CP_PERFMON_CNTX_CNTL
+    0x00000000, // CP_RINGID
+    0x00000000, // CP_VMID
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_VTX_SEMANTIC_0
+    0x00000000, // SQ_VTX_SEMANTIC_1
+    0x00000000, // SQ_VTX_SEMANTIC_2
+    0x00000000, // SQ_VTX_SEMANTIC_3
+    0x00000000, // SQ_VTX_SEMANTIC_4
+    0x00000000, // SQ_VTX_SEMANTIC_5
+    0x00000000, // SQ_VTX_SEMANTIC_6
+    0x00000000, // SQ_VTX_SEMANTIC_7
+    0x00000000, // SQ_VTX_SEMANTIC_8
+    0x00000000, // SQ_VTX_SEMANTIC_9
+    0x00000000, // SQ_VTX_SEMANTIC_10
+    0x00000000, // SQ_VTX_SEMANTIC_11
+    0x00000000, // SQ_VTX_SEMANTIC_12
+    0x00000000, // SQ_VTX_SEMANTIC_13
+    0x00000000, // SQ_VTX_SEMANTIC_14
+    0x00000000, // SQ_VTX_SEMANTIC_15
+    0x00000000, // SQ_VTX_SEMANTIC_16
+    0x00000000, // SQ_VTX_SEMANTIC_17
+    0x00000000, // SQ_VTX_SEMANTIC_18
+    0x00000000, // SQ_VTX_SEMANTIC_19
+    0x00000000, // SQ_VTX_SEMANTIC_20
+    0x00000000, // SQ_VTX_SEMANTIC_21
+    0x00000000, // SQ_VTX_SEMANTIC_22
+    0x00000000, // SQ_VTX_SEMANTIC_23
+    0x00000000, // SQ_VTX_SEMANTIC_24
+    0x00000000, // SQ_VTX_SEMANTIC_25
+    0x00000000, // SQ_VTX_SEMANTIC_26
+    0x00000000, // SQ_VTX_SEMANTIC_27
+    0x00000000, // SQ_VTX_SEMANTIC_28
+    0x00000000, // SQ_VTX_SEMANTIC_29
+    0x00000000, // SQ_VTX_SEMANTIC_30
+    0x00000000, // SQ_VTX_SEMANTIC_31
+    0xffffffff, // VGT_MAX_VTX_INDX
+    0x00000000, // VGT_MIN_VTX_INDX
+    0x00000000, // VGT_INDX_OFFSET
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX
+    0x00000000, // SX_ALPHA_TEST_CONTROL
+    0x00000000, // CB_BLEND_RED
+    0x00000000, // CB_BLEND_GREEN
+    0x00000000, // CB_BLEND_BLUE
+    0x00000000, // CB_BLEND_ALPHA
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_STENCILREFMASK
+    0x00000000, // DB_STENCILREFMASK_BF
+    0x00000000, // SX_ALPHA_REF
+    0x00000000, // PA_CL_VPORT_XSCALE
+    0x00000000, // PA_CL_VPORT_XOFFSET
+    0x00000000, // PA_CL_VPORT_YSCALE
+    0x00000000, // PA_CL_VPORT_YOFFSET
+    0x00000000, // PA_CL_VPORT_ZSCALE
+    0x00000000, // PA_CL_VPORT_ZOFFSET
+    0x00000000, // PA_CL_VPORT_XSCALE_1
+    0x00000000, // PA_CL_VPORT_XOFFSET_1
+    0x00000000, // PA_CL_VPORT_YSCALE_1
+    0x00000000, // PA_CL_VPORT_YOFFSET_1
+    0x00000000, // PA_CL_VPORT_ZSCALE_1
+    0x00000000, // PA_CL_VPORT_ZOFFSET_1
+    0x00000000, // PA_CL_VPORT_XSCALE_2
+    0x00000000, // PA_CL_VPORT_XOFFSET_2
+    0x00000000, // PA_CL_VPORT_YSCALE_2
+    0x00000000, // PA_CL_VPORT_YOFFSET_2
+    0x00000000, // PA_CL_VPORT_ZSCALE_2
+    0x00000000, // PA_CL_VPORT_ZOFFSET_2
+    0x00000000, // PA_CL_VPORT_XSCALE_3
+    0x00000000, // PA_CL_VPORT_XOFFSET_3
+    0x00000000, // PA_CL_VPORT_YSCALE_3
+    0x00000000, // PA_CL_VPORT_YOFFSET_3
+    0x00000000, // PA_CL_VPORT_ZSCALE_3
+    0x00000000, // PA_CL_VPORT_ZOFFSET_3
+    0x00000000, // PA_CL_VPORT_XSCALE_4
+    0x00000000, // PA_CL_VPORT_XOFFSET_4
+    0x00000000, // PA_CL_VPORT_YSCALE_4
+    0x00000000, // PA_CL_VPORT_YOFFSET_4
+    0x00000000, // PA_CL_VPORT_ZSCALE_4
+    0x00000000, // PA_CL_VPORT_ZOFFSET_4
+    0x00000000, // PA_CL_VPORT_XSCALE_5
+    0x00000000, // PA_CL_VPORT_XOFFSET_5
+    0x00000000, // PA_CL_VPORT_YSCALE_5
+    0x00000000, // PA_CL_VPORT_YOFFSET_5
+    0x00000000, // PA_CL_VPORT_ZSCALE_5
+    0x00000000, // PA_CL_VPORT_ZOFFSET_5
+    0x00000000, // PA_CL_VPORT_XSCALE_6
+    0x00000000, // PA_CL_VPORT_XOFFSET_6
+    0x00000000, // PA_CL_VPORT_YSCALE_6
+    0x00000000, // PA_CL_VPORT_YOFFSET_6
+    0x00000000, // PA_CL_VPORT_ZSCALE_6
+    0x00000000, // PA_CL_VPORT_ZOFFSET_6
+    0x00000000, // PA_CL_VPORT_XSCALE_7
+    0x00000000, // PA_CL_VPORT_XOFFSET_7
+    0x00000000, // PA_CL_VPORT_YSCALE_7
+    0x00000000, // PA_CL_VPORT_YOFFSET_7
+    0x00000000, // PA_CL_VPORT_ZSCALE_7
+    0x00000000, // PA_CL_VPORT_ZOFFSET_7
+    0x00000000, // PA_CL_VPORT_XSCALE_8
+    0x00000000, // PA_CL_VPORT_XOFFSET_8
+    0x00000000, // PA_CL_VPORT_YSCALE_8
+    0x00000000, // PA_CL_VPORT_YOFFSET_8
+    0x00000000, // PA_CL_VPORT_ZSCALE_8
+    0x00000000, // PA_CL_VPORT_ZOFFSET_8
+    0x00000000, // PA_CL_VPORT_XSCALE_9
+    0x00000000, // PA_CL_VPORT_XOFFSET_9
+    0x00000000, // PA_CL_VPORT_YSCALE_9
+    0x00000000, // PA_CL_VPORT_YOFFSET_9
+    0x00000000, // PA_CL_VPORT_ZSCALE_9
+    0x00000000, // PA_CL_VPORT_ZOFFSET_9
+    0x00000000, // PA_CL_VPORT_XSCALE_10
+    0x00000000, // PA_CL_VPORT_XOFFSET_10
+    0x00000000, // PA_CL_VPORT_YSCALE_10
+    0x00000000, // PA_CL_VPORT_YOFFSET_10
+    0x00000000, // PA_CL_VPORT_ZSCALE_10
+    0x00000000, // PA_CL_VPORT_ZOFFSET_10
+    0x00000000, // PA_CL_VPORT_XSCALE_11
+    0x00000000, // PA_CL_VPORT_XOFFSET_11
+    0x00000000, // PA_CL_VPORT_YSCALE_11
+    0x00000000, // PA_CL_VPORT_YOFFSET_11
+    0x00000000, // PA_CL_VPORT_ZSCALE_11
+    0x00000000, // PA_CL_VPORT_ZOFFSET_11
+    0x00000000, // PA_CL_VPORT_XSCALE_12
+    0x00000000, // PA_CL_VPORT_XOFFSET_12
+    0x00000000, // PA_CL_VPORT_YSCALE_12
+    0x00000000, // PA_CL_VPORT_YOFFSET_12
+    0x00000000, // PA_CL_VPORT_ZSCALE_12
+    0x00000000, // PA_CL_VPORT_ZOFFSET_12
+    0x00000000, // PA_CL_VPORT_XSCALE_13
+    0x00000000, // PA_CL_VPORT_XOFFSET_13
+    0x00000000, // PA_CL_VPORT_YSCALE_13
+    0x00000000, // PA_CL_VPORT_YOFFSET_13
+    0x00000000, // PA_CL_VPORT_ZSCALE_13
+    0x00000000, // PA_CL_VPORT_ZOFFSET_13
+    0x00000000, // PA_CL_VPORT_XSCALE_14
+    0x00000000, // PA_CL_VPORT_XOFFSET_14
+    0x00000000, // PA_CL_VPORT_YSCALE_14
+    0x00000000, // PA_CL_VPORT_YOFFSET_14
+    0x00000000, // PA_CL_VPORT_ZSCALE_14
+    0x00000000, // PA_CL_VPORT_ZOFFSET_14
+    0x00000000, // PA_CL_VPORT_XSCALE_15
+    0x00000000, // PA_CL_VPORT_XOFFSET_15
+    0x00000000, // PA_CL_VPORT_YSCALE_15
+    0x00000000, // PA_CL_VPORT_YOFFSET_15
+    0x00000000, // PA_CL_VPORT_ZSCALE_15
+    0x00000000, // PA_CL_VPORT_ZOFFSET_15
+    0x00000000, // PA_CL_UCP_0_X
+    0x00000000, // PA_CL_UCP_0_Y
+    0x00000000, // PA_CL_UCP_0_Z
+    0x00000000, // PA_CL_UCP_0_W
+    0x00000000, // PA_CL_UCP_1_X
+    0x00000000, // PA_CL_UCP_1_Y
+    0x00000000, // PA_CL_UCP_1_Z
+    0x00000000, // PA_CL_UCP_1_W
+    0x00000000, // PA_CL_UCP_2_X
+    0x00000000, // PA_CL_UCP_2_Y
+    0x00000000, // PA_CL_UCP_2_Z
+    0x00000000, // PA_CL_UCP_2_W
+    0x00000000, // PA_CL_UCP_3_X
+    0x00000000, // PA_CL_UCP_3_Y
+    0x00000000, // PA_CL_UCP_3_Z
+    0x00000000, // PA_CL_UCP_3_W
+    0x00000000, // PA_CL_UCP_4_X
+    0x00000000, // PA_CL_UCP_4_Y
+    0x00000000, // PA_CL_UCP_4_Z
+    0x00000000, // PA_CL_UCP_4_W
+    0x00000000, // PA_CL_UCP_5_X
+    0x00000000, // PA_CL_UCP_5_Y
+    0x00000000, // PA_CL_UCP_5_Z
+    0x00000000, // PA_CL_UCP_5_W
+    0x00000000, // SPI_VS_OUT_ID_0
+    0x00000000, // SPI_VS_OUT_ID_1
+    0x00000000, // SPI_VS_OUT_ID_2
+    0x00000000, // SPI_VS_OUT_ID_3
+    0x00000000, // SPI_VS_OUT_ID_4
+    0x00000000, // SPI_VS_OUT_ID_5
+    0x00000000, // SPI_VS_OUT_ID_6
+    0x00000000, // SPI_VS_OUT_ID_7
+    0x00000000, // SPI_VS_OUT_ID_8
+    0x00000000, // SPI_VS_OUT_ID_9
+    0x00000000, // SPI_PS_INPUT_CNTL_0
+    0x00000000, // SPI_PS_INPUT_CNTL_1
+    0x00000000, // SPI_PS_INPUT_CNTL_2
+    0x00000000, // SPI_PS_INPUT_CNTL_3
+    0x00000000, // SPI_PS_INPUT_CNTL_4
+    0x00000000, // SPI_PS_INPUT_CNTL_5
+    0x00000000, // SPI_PS_INPUT_CNTL_6
+    0x00000000, // SPI_PS_INPUT_CNTL_7
+    0x00000000, // SPI_PS_INPUT_CNTL_8
+    0x00000000, // SPI_PS_INPUT_CNTL_9
+    0x00000000, // SPI_PS_INPUT_CNTL_10
+    0x00000000, // SPI_PS_INPUT_CNTL_11
+    0x00000000, // SPI_PS_INPUT_CNTL_12
+    0x00000000, // SPI_PS_INPUT_CNTL_13
+    0x00000000, // SPI_PS_INPUT_CNTL_14
+    0x00000000, // SPI_PS_INPUT_CNTL_15
+    0x00000000, // SPI_PS_INPUT_CNTL_16
+    0x00000000, // SPI_PS_INPUT_CNTL_17
+    0x00000000, // SPI_PS_INPUT_CNTL_18
+    0x00000000, // SPI_PS_INPUT_CNTL_19
+    0x00000000, // SPI_PS_INPUT_CNTL_20
+    0x00000000, // SPI_PS_INPUT_CNTL_21
+    0x00000000, // SPI_PS_INPUT_CNTL_22
+    0x00000000, // SPI_PS_INPUT_CNTL_23
+    0x00000000, // SPI_PS_INPUT_CNTL_24
+    0x00000000, // SPI_PS_INPUT_CNTL_25
+    0x00000000, // SPI_PS_INPUT_CNTL_26
+    0x00000000, // SPI_PS_INPUT_CNTL_27
+    0x00000000, // SPI_PS_INPUT_CNTL_28
+    0x00000000, // SPI_PS_INPUT_CNTL_29
+    0x00000000, // SPI_PS_INPUT_CNTL_30
+    0x00000000, // SPI_PS_INPUT_CNTL_31
+    0x00000000, // SPI_VS_OUT_CONFIG
+    0x00000001, // SPI_THREAD_GROUPING
+    0x00000002, // SPI_PS_IN_CONTROL_0
+    0x00000000, // SPI_PS_IN_CONTROL_1
+    0x00000000, // SPI_INTERP_CONTROL_0
+    0x00000000, // SPI_INPUT_Z
+    0x00000000, // SPI_FOG_CNTL
+    0x00000000, // SPI_BARYC_CNTL
+    0x00000000, // SPI_PS_IN_CONTROL_2
+    0x00000000, // SPI_COMPUTE_INPUT_CNTL
+    0x00000000, // SPI_COMPUTE_NUM_THREAD_X
+    0x00000000, // SPI_COMPUTE_NUM_THREAD_Y
+    0x00000000, // SPI_COMPUTE_NUM_THREAD_Z
+    0x00000000, // SPI_GPR_MGMT
+    0x00000000, // SPI_LDS_MGMT
+    0x00000000, // SPI_STACK_MGMT
+    0x00000000, // SPI_WAVE_MGMT_1
+    0x00000000, // SPI_WAVE_MGMT_2
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // GDS_ADDR_BASE
+    0x00003fff, // GDS_ADDR_SIZE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // GDS_ORDERED_COUNT
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // GDS_APPEND_CONSUME_UAV0
+    0x00000000, // GDS_APPEND_CONSUME_UAV1
+    0x00000000, // GDS_APPEND_CONSUME_UAV2
+    0x00000000, // GDS_APPEND_CONSUME_UAV3
+    0x00000000, // GDS_APPEND_CONSUME_UAV4
+    0x00000000, // GDS_APPEND_CONSUME_UAV5
+    0x00000000, // GDS_APPEND_CONSUME_UAV6
+    0x00000000, // GDS_APPEND_CONSUME_UAV7
+    0x00000000, // GDS_APPEND_CONSUME_UAV8
+    0x00000000, // GDS_APPEND_CONSUME_UAV9
+    0x00000000, // GDS_APPEND_CONSUME_UAV10
+    0x00000000, // GDS_APPEND_CONSUME_UAV11
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_BLEND0_CONTROL
+    0x00000000, // CB_BLEND1_CONTROL
+    0x00000000, // CB_BLEND2_CONTROL
+    0x00000000, // CB_BLEND3_CONTROL
+    0x00000000, // CB_BLEND4_CONTROL
+    0x00000000, // CB_BLEND5_CONTROL
+    0x00000000, // CB_BLEND6_CONTROL
+    0x00000000, // CB_BLEND7_CONTROL
+};
+static const u32 SECT_CONTEXT_def_2[] =
+{
+    0x00000000, // PA_CL_POINT_X_RAD
+    0x00000000, // PA_CL_POINT_Y_RAD
+    0x00000000, // PA_CL_POINT_SIZE
+    0x00000000, // PA_CL_POINT_CULL_RAD
+    0x00000000, // VGT_DMA_BASE_HI
+    0x00000000, // VGT_DMA_BASE
+};
+static const u32 SECT_CONTEXT_def_3[] =
+{
+    0x00000000, // DB_DEPTH_CONTROL
+    0x00000000, // DB_EQAA
+    0x00000000, // CB_COLOR_CONTROL
+    0x00000200, // DB_SHADER_CONTROL
+    0x00000000, // PA_CL_CLIP_CNTL
+    0x00000000, // PA_SU_SC_MODE_CNTL
+    0x00000000, // PA_CL_VTE_CNTL
+    0x00000000, // PA_CL_VS_OUT_CNTL
+    0x00000000, // PA_CL_NANINF_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_SCALE
+    0x00000000, // PA_SU_PRIM_FILTER_CNTL
+    0x00000000, // SQ_LSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_HSTMP_RING_ITEMSIZE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_PS
+    0x00000000, // SQ_PGM_RESOURCES_PS
+    0x00000000, // SQ_PGM_RESOURCES_2_PS
+    0x00000000, // SQ_PGM_EXPORTS_PS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_VS
+    0x00000000, // SQ_PGM_RESOURCES_VS
+    0x00000000, // SQ_PGM_RESOURCES_2_VS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_GS
+    0x00000000, // SQ_PGM_RESOURCES_GS
+    0x00000000, // SQ_PGM_RESOURCES_2_GS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_ES
+    0x00000000, // SQ_PGM_RESOURCES_ES
+    0x00000000, // SQ_PGM_RESOURCES_2_ES
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_FS
+    0x00000000, // SQ_PGM_RESOURCES_FS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_HS
+    0x00000000, // SQ_PGM_RESOURCES_HS
+    0x00000000, // SQ_PGM_RESOURCES_2_HS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_LS
+    0x00000000, // SQ_PGM_RESOURCES_LS
+    0x00000000, // SQ_PGM_RESOURCES_2_LS
+};
+static const u32 SECT_CONTEXT_def_4[] =
+{
+    0x00000000, // SQ_LDS_ALLOC
+    0x00000000, // SQ_LDS_ALLOC_PS
+    0x00000000, // SQ_VTX_SEMANTIC_CLEAR
+    0, // HOLE
+    0x00000000, // SQ_THREAD_TRACE_CTRL
+    0, // HOLE
+    0x00000000, // SQ_ESGS_RING_ITEMSIZE
+    0x00000000, // SQ_GSVS_RING_ITEMSIZE
+    0x00000000, // SQ_ESTMP_RING_ITEMSIZE
+    0x00000000, // SQ_GSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_VSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_PSTMP_RING_ITEMSIZE
+    0, // HOLE
+    0x00000000, // SQ_GS_VERT_ITEMSIZE
+    0x00000000, // SQ_GS_VERT_ITEMSIZE_1
+    0x00000000, // SQ_GS_VERT_ITEMSIZE_2
+    0x00000000, // SQ_GS_VERT_ITEMSIZE_3
+    0x00000000, // SQ_GSVS_RING_OFFSET_1
+    0x00000000, // SQ_GSVS_RING_OFFSET_2
+    0x00000000, // SQ_GSVS_RING_OFFSET_3
+    0x00000000, // SQ_GWS_RING_OFFSET
+    0, // HOLE
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_15
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_15
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_15
+    0x00000000, // PA_SU_POINT_SIZE
+    0x00000000, // PA_SU_POINT_MINMAX
+    0x00000000, // PA_SU_LINE_CNTL
+    0x00000000, // PA_SC_LINE_STIPPLE
+    0x00000000, // VGT_OUTPUT_PATH_CNTL
+    0x00000000, // VGT_HOS_CNTL
+    0x00000000, // VGT_HOS_MAX_TESS_LEVEL
+    0x00000000, // VGT_HOS_MIN_TESS_LEVEL
+    0x00000000, // VGT_HOS_REUSE_DEPTH
+    0x00000000, // VGT_GROUP_PRIM_TYPE
+    0x00000000, // VGT_GROUP_FIRST_DECR
+    0x00000000, // VGT_GROUP_DECR
+    0x00000000, // VGT_GROUP_VECT_0_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_CNTL
+    0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL
+    0x00000000, // VGT_GS_MODE
+    0, // HOLE
+    0x00000000, // PA_SC_MODE_CNTL_0
+    0x00000000, // PA_SC_MODE_CNTL_1
+    0x00000000, // VGT_ENHANCE
+    0x00000100, // VGT_GS_PER_ES
+    0x00000080, // VGT_ES_PER_GS
+    0x00000002, // VGT_GS_PER_VS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_GS_OUT_PRIM_TYPE
+    0x00000000, // IA_ENHANCE
+};
+static const u32 SECT_CONTEXT_def_5[] =
+{
+    0x00000000, // VGT_DMA_MAX_SIZE
+    0x00000000, // VGT_DMA_INDEX_TYPE
+    0, // HOLE
+    0x00000000, // VGT_PRIMITIVEID_EN
+    0x00000000, // VGT_DMA_NUM_INSTANCES
+};
+static const u32 SECT_CONTEXT_def_6[] =
+{
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_INSTANCE_STEP_RATE_0
+    0x00000000, // VGT_INSTANCE_STEP_RATE_1
+    0x000000ff, // IA_MULTI_VGT_PARAM
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_REUSE_OFF
+    0x00000000, // VGT_VTX_CNT_EN
+    0x00000000, // DB_HTILE_SURFACE
+    0x00000000, // DB_SRESULTS_COMPARE_STATE0
+    0x00000000, // DB_SRESULTS_COMPARE_STATE1
+    0x00000000, // DB_PRELOAD_CONTROL
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_0
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_0
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_1
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_1
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_2
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_2
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_3
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_3
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_0
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_1
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_2
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_3
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+    0, // HOLE
+    0x00000000, // VGT_GS_MAX_VERT_OUT
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_0
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_1
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_2
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_3
+    0x00000000, // VGT_SHADER_STAGES_EN
+    0x00000000, // VGT_LS_HS_CONFIG
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_TF_PARAM
+    0x00000000, // DB_ALPHA_TO_MASK
+};
+static const u32 SECT_CONTEXT_def_7[] =
+{
+    0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL
+    0x00000000, // PA_SU_POLY_OFFSET_CLAMP
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET
+    0x00000000, // VGT_GS_INSTANCE_CNT
+    0x00000000, // VGT_STRMOUT_CONFIG
+    0x00000000, // VGT_STRMOUT_BUFFER_CONFIG
+    0x00000000, // CB_IMMED0_BASE
+    0x00000000, // CB_IMMED1_BASE
+    0x00000000, // CB_IMMED2_BASE
+    0x00000000, // CB_IMMED3_BASE
+    0x00000000, // CB_IMMED4_BASE
+    0x00000000, // CB_IMMED5_BASE
+    0x00000000, // CB_IMMED6_BASE
+    0x00000000, // CB_IMMED7_BASE
+    0x00000000, // CB_IMMED8_BASE
+    0x00000000, // CB_IMMED9_BASE
+    0x00000000, // CB_IMMED10_BASE
+    0x00000000, // CB_IMMED11_BASE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // PA_SC_CENTROID_PRIORITY_0
+    0x00000000, // PA_SC_CENTROID_PRIORITY_1
+    0x00001000, // PA_SC_LINE_CNTL
+    0x00000000, // PA_SC_AA_CONFIG
+    0x00000005, // PA_SU_VTX_CNTL
+    0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_VERT_DISC_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+    0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0
+    0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1
+    0x00000000, // CB_CLRCMP_CONTROL
+    0x00000000, // CB_CLRCMP_SRC
+    0x00000000, // CB_CLRCMP_DST
+    0x00000000, // CB_CLRCMP_MSK
+    0, // HOLE
+    0, // HOLE
+    0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL
+    0x00000010, // VGT_OUT_DEALLOC_CNTL
+    0x00000000, // CB_COLOR0_BASE
+    0x00000000, // CB_COLOR0_PITCH
+    0x00000000, // CB_COLOR0_SLICE
+    0x00000000, // CB_COLOR0_VIEW
+    0x00000000, // CB_COLOR0_INFO
+    0x00000000, // CB_COLOR0_ATTRIB
+    0x00000000, // CB_COLOR0_DIM
+    0x00000000, // CB_COLOR0_CMASK
+    0x00000000, // CB_COLOR0_CMASK_SLICE
+    0x00000000, // CB_COLOR0_FMASK
+    0x00000000, // CB_COLOR0_FMASK_SLICE
+    0x00000000, // CB_COLOR0_CLEAR_WORD0
+    0x00000000, // CB_COLOR0_CLEAR_WORD1
+    0x00000000, // CB_COLOR0_CLEAR_WORD2
+    0x00000000, // CB_COLOR0_CLEAR_WORD3
+    0x00000000, // CB_COLOR1_BASE
+    0x00000000, // CB_COLOR1_PITCH
+    0x00000000, // CB_COLOR1_SLICE
+    0x00000000, // CB_COLOR1_VIEW
+    0x00000000, // CB_COLOR1_INFO
+    0x00000000, // CB_COLOR1_ATTRIB
+    0x00000000, // CB_COLOR1_DIM
+    0x00000000, // CB_COLOR1_CMASK
+    0x00000000, // CB_COLOR1_CMASK_SLICE
+    0x00000000, // CB_COLOR1_FMASK
+    0x00000000, // CB_COLOR1_FMASK_SLICE
+    0x00000000, // CB_COLOR1_CLEAR_WORD0
+    0x00000000, // CB_COLOR1_CLEAR_WORD1
+    0x00000000, // CB_COLOR1_CLEAR_WORD2
+    0x00000000, // CB_COLOR1_CLEAR_WORD3
+    0x00000000, // CB_COLOR2_BASE
+    0x00000000, // CB_COLOR2_PITCH
+    0x00000000, // CB_COLOR2_SLICE
+    0x00000000, // CB_COLOR2_VIEW
+    0x00000000, // CB_COLOR2_INFO
+    0x00000000, // CB_COLOR2_ATTRIB
+    0x00000000, // CB_COLOR2_DIM
+    0x00000000, // CB_COLOR2_CMASK
+    0x00000000, // CB_COLOR2_CMASK_SLICE
+    0x00000000, // CB_COLOR2_FMASK
+    0x00000000, // CB_COLOR2_FMASK_SLICE
+    0x00000000, // CB_COLOR2_CLEAR_WORD0
+    0x00000000, // CB_COLOR2_CLEAR_WORD1
+    0x00000000, // CB_COLOR2_CLEAR_WORD2
+    0x00000000, // CB_COLOR2_CLEAR_WORD3
+    0x00000000, // CB_COLOR3_BASE
+    0x00000000, // CB_COLOR3_PITCH
+    0x00000000, // CB_COLOR3_SLICE
+    0x00000000, // CB_COLOR3_VIEW
+    0x00000000, // CB_COLOR3_INFO
+    0x00000000, // CB_COLOR3_ATTRIB
+    0x00000000, // CB_COLOR3_DIM
+    0x00000000, // CB_COLOR3_CMASK
+    0x00000000, // CB_COLOR3_CMASK_SLICE
+    0x00000000, // CB_COLOR3_FMASK
+    0x00000000, // CB_COLOR3_FMASK_SLICE
+    0x00000000, // CB_COLOR3_CLEAR_WORD0
+    0x00000000, // CB_COLOR3_CLEAR_WORD1
+    0x00000000, // CB_COLOR3_CLEAR_WORD2
+    0x00000000, // CB_COLOR3_CLEAR_WORD3
+    0x00000000, // CB_COLOR4_BASE
+    0x00000000, // CB_COLOR4_PITCH
+    0x00000000, // CB_COLOR4_SLICE
+    0x00000000, // CB_COLOR4_VIEW
+    0x00000000, // CB_COLOR4_INFO
+    0x00000000, // CB_COLOR4_ATTRIB
+    0x00000000, // CB_COLOR4_DIM
+    0x00000000, // CB_COLOR4_CMASK
+    0x00000000, // CB_COLOR4_CMASK_SLICE
+    0x00000000, // CB_COLOR4_FMASK
+    0x00000000, // CB_COLOR4_FMASK_SLICE
+    0x00000000, // CB_COLOR4_CLEAR_WORD0
+    0x00000000, // CB_COLOR4_CLEAR_WORD1
+    0x00000000, // CB_COLOR4_CLEAR_WORD2
+    0x00000000, // CB_COLOR4_CLEAR_WORD3
+    0x00000000, // CB_COLOR5_BASE
+    0x00000000, // CB_COLOR5_PITCH
+    0x00000000, // CB_COLOR5_SLICE
+    0x00000000, // CB_COLOR5_VIEW
+    0x00000000, // CB_COLOR5_INFO
+    0x00000000, // CB_COLOR5_ATTRIB
+    0x00000000, // CB_COLOR5_DIM
+    0x00000000, // CB_COLOR5_CMASK
+    0x00000000, // CB_COLOR5_CMASK_SLICE
+    0x00000000, // CB_COLOR5_FMASK
+    0x00000000, // CB_COLOR5_FMASK_SLICE
+    0x00000000, // CB_COLOR5_CLEAR_WORD0
+    0x00000000, // CB_COLOR5_CLEAR_WORD1
+    0x00000000, // CB_COLOR5_CLEAR_WORD2
+    0x00000000, // CB_COLOR5_CLEAR_WORD3
+    0x00000000, // CB_COLOR6_BASE
+    0x00000000, // CB_COLOR6_PITCH
+    0x00000000, // CB_COLOR6_SLICE
+    0x00000000, // CB_COLOR6_VIEW
+    0x00000000, // CB_COLOR6_INFO
+    0x00000000, // CB_COLOR6_ATTRIB
+    0x00000000, // CB_COLOR6_DIM
+    0x00000000, // CB_COLOR6_CMASK
+    0x00000000, // CB_COLOR6_CMASK_SLICE
+    0x00000000, // CB_COLOR6_FMASK
+    0x00000000, // CB_COLOR6_FMASK_SLICE
+    0x00000000, // CB_COLOR6_CLEAR_WORD0
+    0x00000000, // CB_COLOR6_CLEAR_WORD1
+    0x00000000, // CB_COLOR6_CLEAR_WORD2
+    0x00000000, // CB_COLOR6_CLEAR_WORD3
+    0x00000000, // CB_COLOR7_BASE
+    0x00000000, // CB_COLOR7_PITCH
+    0x00000000, // CB_COLOR7_SLICE
+    0x00000000, // CB_COLOR7_VIEW
+    0x00000000, // CB_COLOR7_INFO
+    0x00000000, // CB_COLOR7_ATTRIB
+    0x00000000, // CB_COLOR7_DIM
+    0x00000000, // CB_COLOR7_CMASK
+    0x00000000, // CB_COLOR7_CMASK_SLICE
+    0x00000000, // CB_COLOR7_FMASK
+    0x00000000, // CB_COLOR7_FMASK_SLICE
+    0x00000000, // CB_COLOR7_CLEAR_WORD0
+    0x00000000, // CB_COLOR7_CLEAR_WORD1
+    0x00000000, // CB_COLOR7_CLEAR_WORD2
+    0x00000000, // CB_COLOR7_CLEAR_WORD3
+    0x00000000, // CB_COLOR8_BASE
+    0x00000000, // CB_COLOR8_PITCH
+    0x00000000, // CB_COLOR8_SLICE
+    0x00000000, // CB_COLOR8_VIEW
+    0x00000000, // CB_COLOR8_INFO
+    0x00000000, // CB_COLOR8_ATTRIB
+    0x00000000, // CB_COLOR8_DIM
+    0x00000000, // CB_COLOR9_BASE
+    0x00000000, // CB_COLOR9_PITCH
+    0x00000000, // CB_COLOR9_SLICE
+    0x00000000, // CB_COLOR9_VIEW
+    0x00000000, // CB_COLOR9_INFO
+    0x00000000, // CB_COLOR9_ATTRIB
+    0x00000000, // CB_COLOR9_DIM
+    0x00000000, // CB_COLOR10_BASE
+    0x00000000, // CB_COLOR10_PITCH
+    0x00000000, // CB_COLOR10_SLICE
+    0x00000000, // CB_COLOR10_VIEW
+    0x00000000, // CB_COLOR10_INFO
+    0x00000000, // CB_COLOR10_ATTRIB
+    0x00000000, // CB_COLOR10_DIM
+    0x00000000, // CB_COLOR11_BASE
+    0x00000000, // CB_COLOR11_PITCH
+    0x00000000, // CB_COLOR11_SLICE
+    0x00000000, // CB_COLOR11_VIEW
+    0x00000000, // CB_COLOR11_INFO
+    0x00000000, // CB_COLOR11_ATTRIB
+    0x00000000, // CB_COLOR11_DIM
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_15
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_15
+};
+static const struct cs_extent_def SECT_CONTEXT_defs[] =
+{
+    {SECT_CONTEXT_def_1, 0x0000a000, 488 },
+    {SECT_CONTEXT_def_2, 0x0000a1f5, 6 },
+    {SECT_CONTEXT_def_3, 0x0000a200, 55 },
+    {SECT_CONTEXT_def_4, 0x0000a23a, 99 },
+    {SECT_CONTEXT_def_5, 0x0000a29e, 5 },
+    {SECT_CONTEXT_def_6, 0x0000a2a5, 56 },
+    {SECT_CONTEXT_def_7, 0x0000a2de, 290 },
+    { 0, 0, 0 }
+};
+static const u32 SECT_CLEAR_def_1[] =
+{
+    0xffffffff, // SQ_TEX_SAMPLER_CLEAR
+    0xffffffff, // SQ_TEX_RESOURCE_CLEAR
+    0xffffffff, // SQ_LOOP_BOOL_CLEAR
+};
+static const struct cs_extent_def SECT_CLEAR_defs[] =
+{
+    {SECT_CLEAR_def_1, 0x0000ffc0, 3 },
+    { 0, 0, 0 }
+};
+static const u32 SECT_CTRLCONST_def_1[] =
+{
+    0x00000000, // SQ_VTX_BASE_VTX_LOC
+    0x00000000, // SQ_VTX_START_INST_LOC
+};
+static const struct cs_extent_def SECT_CTRLCONST_defs[] =
+{
+    {SECT_CTRLCONST_def_1, 0x0000f3fc, 2 },
+    { 0, 0, 0 }
+};
+struct cs_section_def cayman_cs_data[] = {
+    { SECT_CONTEXT_defs, SECT_CONTEXT },
+    { SECT_CLEAR_defs, SECT_CLEAR },
+    { SECT_CTRLCONST_defs, SECT_CTRLCONST },
+    { 0, SECT_NONE }
+};
diff --git a/drivers/gpu/drm/radeon/clearstate_defs.h b/drivers/gpu/drm/radeon/clearstate_defs.h
new file mode 100644
index 000000000000..3eda707d7388
--- /dev/null
+++ b/drivers/gpu/drm/radeon/clearstate_defs.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef CLEARSTATE_DEFS_H
+#define CLEARSTATE_DEFS_H
+
+enum section_id {
+    SECT_NONE,
+    SECT_CONTEXT,
+    SECT_CLEAR,
+    SECT_CTRLCONST
+};
+
+struct cs_extent_def {
+    const unsigned int *extent;
+    const unsigned int reg_index;
+    const unsigned int reg_count;
+};
+
+struct cs_section_def {
+    const struct cs_extent_def *section;
+    const enum section_id id;
+};
+
+#endif
diff --git a/drivers/gpu/drm/radeon/clearstate_evergreen.h b/drivers/gpu/drm/radeon/clearstate_evergreen.h
new file mode 100644
index 000000000000..4791d856b7fd
--- /dev/null
+++ b/drivers/gpu/drm/radeon/clearstate_evergreen.h
@@ -0,0 +1,1080 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+static const u32 SECT_CONTEXT_def_1[] =
+{
+    0x00000000, // DB_RENDER_CONTROL
+    0x00000000, // DB_COUNT_CONTROL
+    0x00000000, // DB_DEPTH_VIEW
+    0x00000000, // DB_RENDER_OVERRIDE
+    0x00000000, // DB_RENDER_OVERRIDE2
+    0x00000000, // DB_HTILE_DATA_BASE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_STENCIL_CLEAR
+    0x00000000, // DB_DEPTH_CLEAR
+    0x00000000, // PA_SC_SCREEN_SCISSOR_TL
+    0x40004000, // PA_SC_SCREEN_SCISSOR_BR
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_Z_INFO
+    0x00000000, // DB_STENCIL_INFO
+    0x00000000, // DB_Z_READ_BASE
+    0x00000000, // DB_STENCIL_READ_BASE
+    0x00000000, // DB_Z_WRITE_BASE
+    0x00000000, // DB_STENCIL_WRITE_BASE
+    0x00000000, // DB_DEPTH_SIZE
+    0x00000000, // DB_DEPTH_SLICE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_PS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_VS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_GS_15
+    0x00000000, // PA_SC_WINDOW_OFFSET
+    0x80000000, // PA_SC_WINDOW_SCISSOR_TL
+    0x40004000, // PA_SC_WINDOW_SCISSOR_BR
+    0x0000ffff, // PA_SC_CLIPRECT_RULE
+    0x00000000, // PA_SC_CLIPRECT_0_TL
+    0x40004000, // PA_SC_CLIPRECT_0_BR
+    0x00000000, // PA_SC_CLIPRECT_1_TL
+    0x40004000, // PA_SC_CLIPRECT_1_BR
+    0x00000000, // PA_SC_CLIPRECT_2_TL
+    0x40004000, // PA_SC_CLIPRECT_2_BR
+    0x00000000, // PA_SC_CLIPRECT_3_TL
+    0x40004000, // PA_SC_CLIPRECT_3_BR
+    0xaa99aaaa, // PA_SC_EDGERULE
+    0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET
+    0xffffffff, // CB_TARGET_MASK
+    0xffffffff, // CB_SHADER_MASK
+    0x80000000, // PA_SC_GENERIC_SCISSOR_TL
+    0x40004000, // PA_SC_GENERIC_SCISSOR_BR
+    0x00000000, // COHER_DEST_BASE_0
+    0x00000000, // COHER_DEST_BASE_1
+    0x80000000, // PA_SC_VPORT_SCISSOR_0_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_0_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_1_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_1_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_2_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_2_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_3_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_3_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_4_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_4_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_5_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_5_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_6_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_6_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_7_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_7_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_8_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_8_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_9_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_9_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_10_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_10_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_11_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_11_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_12_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_12_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_13_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_13_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_14_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_14_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_15_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_15_BR
+    0x00000000, // PA_SC_VPORT_ZMIN_0
+    0x3f800000, // PA_SC_VPORT_ZMAX_0
+    0x00000000, // PA_SC_VPORT_ZMIN_1
+    0x3f800000, // PA_SC_VPORT_ZMAX_1
+    0x00000000, // PA_SC_VPORT_ZMIN_2
+    0x3f800000, // PA_SC_VPORT_ZMAX_2
+    0x00000000, // PA_SC_VPORT_ZMIN_3
+    0x3f800000, // PA_SC_VPORT_ZMAX_3
+    0x00000000, // PA_SC_VPORT_ZMIN_4
+    0x3f800000, // PA_SC_VPORT_ZMAX_4
+    0x00000000, // PA_SC_VPORT_ZMIN_5
+    0x3f800000, // PA_SC_VPORT_ZMAX_5
+    0x00000000, // PA_SC_VPORT_ZMIN_6
+    0x3f800000, // PA_SC_VPORT_ZMAX_6
+    0x00000000, // PA_SC_VPORT_ZMIN_7
+    0x3f800000, // PA_SC_VPORT_ZMAX_7
+    0x00000000, // PA_SC_VPORT_ZMIN_8
+    0x3f800000, // PA_SC_VPORT_ZMAX_8
+    0x00000000, // PA_SC_VPORT_ZMIN_9
+    0x3f800000, // PA_SC_VPORT_ZMAX_9
+    0x00000000, // PA_SC_VPORT_ZMIN_10
+    0x3f800000, // PA_SC_VPORT_ZMAX_10
+    0x00000000, // PA_SC_VPORT_ZMIN_11
+    0x3f800000, // PA_SC_VPORT_ZMAX_11
+    0x00000000, // PA_SC_VPORT_ZMIN_12
+    0x3f800000, // PA_SC_VPORT_ZMAX_12
+    0x00000000, // PA_SC_VPORT_ZMIN_13
+    0x3f800000, // PA_SC_VPORT_ZMAX_13
+    0x00000000, // PA_SC_VPORT_ZMIN_14
+    0x3f800000, // PA_SC_VPORT_ZMAX_14
+    0x00000000, // PA_SC_VPORT_ZMIN_15
+    0x3f800000, // PA_SC_VPORT_ZMAX_15
+    0x00000000, // SX_MISC
+    0x00000000, // SX_SURFACE_SYNC
+    0x00000000, // CP_PERFMON_CNTX_CNTL
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_VTX_SEMANTIC_0
+    0x00000000, // SQ_VTX_SEMANTIC_1
+    0x00000000, // SQ_VTX_SEMANTIC_2
+    0x00000000, // SQ_VTX_SEMANTIC_3
+    0x00000000, // SQ_VTX_SEMANTIC_4
+    0x00000000, // SQ_VTX_SEMANTIC_5
+    0x00000000, // SQ_VTX_SEMANTIC_6
+    0x00000000, // SQ_VTX_SEMANTIC_7
+    0x00000000, // SQ_VTX_SEMANTIC_8
+    0x00000000, // SQ_VTX_SEMANTIC_9
+    0x00000000, // SQ_VTX_SEMANTIC_10
+    0x00000000, // SQ_VTX_SEMANTIC_11
+    0x00000000, // SQ_VTX_SEMANTIC_12
+    0x00000000, // SQ_VTX_SEMANTIC_13
+    0x00000000, // SQ_VTX_SEMANTIC_14
+    0x00000000, // SQ_VTX_SEMANTIC_15
+    0x00000000, // SQ_VTX_SEMANTIC_16
+    0x00000000, // SQ_VTX_SEMANTIC_17
+    0x00000000, // SQ_VTX_SEMANTIC_18
+    0x00000000, // SQ_VTX_SEMANTIC_19
+    0x00000000, // SQ_VTX_SEMANTIC_20
+    0x00000000, // SQ_VTX_SEMANTIC_21
+    0x00000000, // SQ_VTX_SEMANTIC_22
+    0x00000000, // SQ_VTX_SEMANTIC_23
+    0x00000000, // SQ_VTX_SEMANTIC_24
+    0x00000000, // SQ_VTX_SEMANTIC_25
+    0x00000000, // SQ_VTX_SEMANTIC_26
+    0x00000000, // SQ_VTX_SEMANTIC_27
+    0x00000000, // SQ_VTX_SEMANTIC_28
+    0x00000000, // SQ_VTX_SEMANTIC_29
+    0x00000000, // SQ_VTX_SEMANTIC_30
+    0x00000000, // SQ_VTX_SEMANTIC_31
+    0xffffffff, // VGT_MAX_VTX_INDX
+    0x00000000, // VGT_MIN_VTX_INDX
+    0x00000000, // VGT_INDX_OFFSET
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX
+    0x00000000, // SX_ALPHA_TEST_CONTROL
+    0x00000000, // CB_BLEND_RED
+    0x00000000, // CB_BLEND_GREEN
+    0x00000000, // CB_BLEND_BLUE
+    0x00000000, // CB_BLEND_ALPHA
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_STENCILREFMASK
+    0x00000000, // DB_STENCILREFMASK_BF
+    0x00000000, // SX_ALPHA_REF
+    0x00000000, // PA_CL_VPORT_XSCALE
+    0x00000000, // PA_CL_VPORT_XOFFSET
+    0x00000000, // PA_CL_VPORT_YSCALE
+    0x00000000, // PA_CL_VPORT_YOFFSET
+    0x00000000, // PA_CL_VPORT_ZSCALE
+    0x00000000, // PA_CL_VPORT_ZOFFSET
+    0x00000000, // PA_CL_VPORT_XSCALE_1
+    0x00000000, // PA_CL_VPORT_XOFFSET_1
+    0x00000000, // PA_CL_VPORT_YSCALE_1
+    0x00000000, // PA_CL_VPORT_YOFFSET_1
+    0x00000000, // PA_CL_VPORT_ZSCALE_1
+    0x00000000, // PA_CL_VPORT_ZOFFSET_1
+    0x00000000, // PA_CL_VPORT_XSCALE_2
+    0x00000000, // PA_CL_VPORT_XOFFSET_2
+    0x00000000, // PA_CL_VPORT_YSCALE_2
+    0x00000000, // PA_CL_VPORT_YOFFSET_2
+    0x00000000, // PA_CL_VPORT_ZSCALE_2
+    0x00000000, // PA_CL_VPORT_ZOFFSET_2
+    0x00000000, // PA_CL_VPORT_XSCALE_3
+    0x00000000, // PA_CL_VPORT_XOFFSET_3
+    0x00000000, // PA_CL_VPORT_YSCALE_3
+    0x00000000, // PA_CL_VPORT_YOFFSET_3
+    0x00000000, // PA_CL_VPORT_ZSCALE_3
+    0x00000000, // PA_CL_VPORT_ZOFFSET_3
+    0x00000000, // PA_CL_VPORT_XSCALE_4
+    0x00000000, // PA_CL_VPORT_XOFFSET_4
+    0x00000000, // PA_CL_VPORT_YSCALE_4
+    0x00000000, // PA_CL_VPORT_YOFFSET_4
+    0x00000000, // PA_CL_VPORT_ZSCALE_4
+    0x00000000, // PA_CL_VPORT_ZOFFSET_4
+    0x00000000, // PA_CL_VPORT_XSCALE_5
+    0x00000000, // PA_CL_VPORT_XOFFSET_5
+    0x00000000, // PA_CL_VPORT_YSCALE_5
+    0x00000000, // PA_CL_VPORT_YOFFSET_5
+    0x00000000, // PA_CL_VPORT_ZSCALE_5
+    0x00000000, // PA_CL_VPORT_ZOFFSET_5
+    0x00000000, // PA_CL_VPORT_XSCALE_6
+    0x00000000, // PA_CL_VPORT_XOFFSET_6
+    0x00000000, // PA_CL_VPORT_YSCALE_6
+    0x00000000, // PA_CL_VPORT_YOFFSET_6
+    0x00000000, // PA_CL_VPORT_ZSCALE_6
+    0x00000000, // PA_CL_VPORT_ZOFFSET_6
+    0x00000000, // PA_CL_VPORT_XSCALE_7
+    0x00000000, // PA_CL_VPORT_XOFFSET_7
+    0x00000000, // PA_CL_VPORT_YSCALE_7
+    0x00000000, // PA_CL_VPORT_YOFFSET_7
+    0x00000000, // PA_CL_VPORT_ZSCALE_7
+    0x00000000, // PA_CL_VPORT_ZOFFSET_7
+    0x00000000, // PA_CL_VPORT_XSCALE_8
+    0x00000000, // PA_CL_VPORT_XOFFSET_8
+    0x00000000, // PA_CL_VPORT_YSCALE_8
+    0x00000000, // PA_CL_VPORT_YOFFSET_8
+    0x00000000, // PA_CL_VPORT_ZSCALE_8
+    0x00000000, // PA_CL_VPORT_ZOFFSET_8
+    0x00000000, // PA_CL_VPORT_XSCALE_9
+    0x00000000, // PA_CL_VPORT_XOFFSET_9
+    0x00000000, // PA_CL_VPORT_YSCALE_9
+    0x00000000, // PA_CL_VPORT_YOFFSET_9
+    0x00000000, // PA_CL_VPORT_ZSCALE_9
+    0x00000000, // PA_CL_VPORT_ZOFFSET_9
+    0x00000000, // PA_CL_VPORT_XSCALE_10
+    0x00000000, // PA_CL_VPORT_XOFFSET_10
+    0x00000000, // PA_CL_VPORT_YSCALE_10
+    0x00000000, // PA_CL_VPORT_YOFFSET_10
+    0x00000000, // PA_CL_VPORT_ZSCALE_10
+    0x00000000, // PA_CL_VPORT_ZOFFSET_10
+    0x00000000, // PA_CL_VPORT_XSCALE_11
+    0x00000000, // PA_CL_VPORT_XOFFSET_11
+    0x00000000, // PA_CL_VPORT_YSCALE_11
+    0x00000000, // PA_CL_VPORT_YOFFSET_11
+    0x00000000, // PA_CL_VPORT_ZSCALE_11
+    0x00000000, // PA_CL_VPORT_ZOFFSET_11
+    0x00000000, // PA_CL_VPORT_XSCALE_12
+    0x00000000, // PA_CL_VPORT_XOFFSET_12
+    0x00000000, // PA_CL_VPORT_YSCALE_12
+    0x00000000, // PA_CL_VPORT_YOFFSET_12
+    0x00000000, // PA_CL_VPORT_ZSCALE_12
+    0x00000000, // PA_CL_VPORT_ZOFFSET_12
+    0x00000000, // PA_CL_VPORT_XSCALE_13
+    0x00000000, // PA_CL_VPORT_XOFFSET_13
+    0x00000000, // PA_CL_VPORT_YSCALE_13
+    0x00000000, // PA_CL_VPORT_YOFFSET_13
+    0x00000000, // PA_CL_VPORT_ZSCALE_13
+    0x00000000, // PA_CL_VPORT_ZOFFSET_13
+    0x00000000, // PA_CL_VPORT_XSCALE_14
+    0x00000000, // PA_CL_VPORT_XOFFSET_14
+    0x00000000, // PA_CL_VPORT_YSCALE_14
+    0x00000000, // PA_CL_VPORT_YOFFSET_14
+    0x00000000, // PA_CL_VPORT_ZSCALE_14
+    0x00000000, // PA_CL_VPORT_ZOFFSET_14
+    0x00000000, // PA_CL_VPORT_XSCALE_15
+    0x00000000, // PA_CL_VPORT_XOFFSET_15
+    0x00000000, // PA_CL_VPORT_YSCALE_15
+    0x00000000, // PA_CL_VPORT_YOFFSET_15
+    0x00000000, // PA_CL_VPORT_ZSCALE_15
+    0x00000000, // PA_CL_VPORT_ZOFFSET_15
+    0x00000000, // PA_CL_UCP_0_X
+    0x00000000, // PA_CL_UCP_0_Y
+    0x00000000, // PA_CL_UCP_0_Z
+    0x00000000, // PA_CL_UCP_0_W
+    0x00000000, // PA_CL_UCP_1_X
+    0x00000000, // PA_CL_UCP_1_Y
+    0x00000000, // PA_CL_UCP_1_Z
+    0x00000000, // PA_CL_UCP_1_W
+    0x00000000, // PA_CL_UCP_2_X
+    0x00000000, // PA_CL_UCP_2_Y
+    0x00000000, // PA_CL_UCP_2_Z
+    0x00000000, // PA_CL_UCP_2_W
+    0x00000000, // PA_CL_UCP_3_X
+    0x00000000, // PA_CL_UCP_3_Y
+    0x00000000, // PA_CL_UCP_3_Z
+    0x00000000, // PA_CL_UCP_3_W
+    0x00000000, // PA_CL_UCP_4_X
+    0x00000000, // PA_CL_UCP_4_Y
+    0x00000000, // PA_CL_UCP_4_Z
+    0x00000000, // PA_CL_UCP_4_W
+    0x00000000, // PA_CL_UCP_5_X
+    0x00000000, // PA_CL_UCP_5_Y
+    0x00000000, // PA_CL_UCP_5_Z
+    0x00000000, // PA_CL_UCP_5_W
+    0x00000000, // SPI_VS_OUT_ID_0
+    0x00000000, // SPI_VS_OUT_ID_1
+    0x00000000, // SPI_VS_OUT_ID_2
+    0x00000000, // SPI_VS_OUT_ID_3
+    0x00000000, // SPI_VS_OUT_ID_4
+    0x00000000, // SPI_VS_OUT_ID_5
+    0x00000000, // SPI_VS_OUT_ID_6
+    0x00000000, // SPI_VS_OUT_ID_7
+    0x00000000, // SPI_VS_OUT_ID_8
+    0x00000000, // SPI_VS_OUT_ID_9
+    0x00000000, // SPI_PS_INPUT_CNTL_0
+    0x00000000, // SPI_PS_INPUT_CNTL_1
+    0x00000000, // SPI_PS_INPUT_CNTL_2
+    0x00000000, // SPI_PS_INPUT_CNTL_3
+    0x00000000, // SPI_PS_INPUT_CNTL_4
+    0x00000000, // SPI_PS_INPUT_CNTL_5
+    0x00000000, // SPI_PS_INPUT_CNTL_6
+    0x00000000, // SPI_PS_INPUT_CNTL_7
+    0x00000000, // SPI_PS_INPUT_CNTL_8
+    0x00000000, // SPI_PS_INPUT_CNTL_9
+    0x00000000, // SPI_PS_INPUT_CNTL_10
+    0x00000000, // SPI_PS_INPUT_CNTL_11
+    0x00000000, // SPI_PS_INPUT_CNTL_12
+    0x00000000, // SPI_PS_INPUT_CNTL_13
+    0x00000000, // SPI_PS_INPUT_CNTL_14
+    0x00000000, // SPI_PS_INPUT_CNTL_15
+    0x00000000, // SPI_PS_INPUT_CNTL_16
+    0x00000000, // SPI_PS_INPUT_CNTL_17
+    0x00000000, // SPI_PS_INPUT_CNTL_18
+    0x00000000, // SPI_PS_INPUT_CNTL_19
+    0x00000000, // SPI_PS_INPUT_CNTL_20
+    0x00000000, // SPI_PS_INPUT_CNTL_21
+    0x00000000, // SPI_PS_INPUT_CNTL_22
+    0x00000000, // SPI_PS_INPUT_CNTL_23
+    0x00000000, // SPI_PS_INPUT_CNTL_24
+    0x00000000, // SPI_PS_INPUT_CNTL_25
+    0x00000000, // SPI_PS_INPUT_CNTL_26
+    0x00000000, // SPI_PS_INPUT_CNTL_27
+    0x00000000, // SPI_PS_INPUT_CNTL_28
+    0x00000000, // SPI_PS_INPUT_CNTL_29
+    0x00000000, // SPI_PS_INPUT_CNTL_30
+    0x00000000, // SPI_PS_INPUT_CNTL_31
+    0x00000000, // SPI_VS_OUT_CONFIG
+    0x00000001, // SPI_THREAD_GROUPING
+    0x00000000, // SPI_PS_IN_CONTROL_0
+    0x00000000, // SPI_PS_IN_CONTROL_1
+    0x00000000, // SPI_INTERP_CONTROL_0
+    0x00000000, // SPI_INPUT_Z
+    0x00000000, // SPI_FOG_CNTL
+    0x00000000, // SPI_BARYC_CNTL
+    0x00000000, // SPI_PS_IN_CONTROL_2
+    0x00000000, // SPI_COMPUTE_INPUT_CNTL
+    0x00000000, // SPI_COMPUTE_NUM_THREAD_X
+    0x00000000, // SPI_COMPUTE_NUM_THREAD_Y
+    0x00000000, // SPI_COMPUTE_NUM_THREAD_Z
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // GDS_ADDR_BASE
+    0x00003fff, // GDS_ADDR_SIZE
+    0x00000001, // GDS_ORDERED_WAVE_PER_SE
+    0x00000000, // GDS_APPEND_CONSUME_UAV0
+    0x00000000, // GDS_APPEND_CONSUME_UAV1
+    0x00000000, // GDS_APPEND_CONSUME_UAV2
+    0x00000000, // GDS_APPEND_CONSUME_UAV3
+    0x00000000, // GDS_APPEND_CONSUME_UAV4
+    0x00000000, // GDS_APPEND_CONSUME_UAV5
+    0x00000000, // GDS_APPEND_CONSUME_UAV6
+    0x00000000, // GDS_APPEND_CONSUME_UAV7
+    0x00000000, // GDS_APPEND_CONSUME_UAV8
+    0x00000000, // GDS_APPEND_CONSUME_UAV9
+    0x00000000, // GDS_APPEND_CONSUME_UAV10
+    0x00000000, // GDS_APPEND_CONSUME_UAV11
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_BLEND0_CONTROL
+    0x00000000, // CB_BLEND1_CONTROL
+    0x00000000, // CB_BLEND2_CONTROL
+    0x00000000, // CB_BLEND3_CONTROL
+    0x00000000, // CB_BLEND4_CONTROL
+    0x00000000, // CB_BLEND5_CONTROL
+    0x00000000, // CB_BLEND6_CONTROL
+    0x00000000, // CB_BLEND7_CONTROL
+};
+static const u32 SECT_CONTEXT_def_2[] =
+{
+    0x00000000, // PA_CL_POINT_X_RAD
+    0x00000000, // PA_CL_POINT_Y_RAD
+    0x00000000, // PA_CL_POINT_SIZE
+    0x00000000, // PA_CL_POINT_CULL_RAD
+    0x00000000, // VGT_DMA_BASE_HI
+    0x00000000, // VGT_DMA_BASE
+};
+static const u32 SECT_CONTEXT_def_3[] =
+{
+    0x00000000, // DB_DEPTH_CONTROL
+    0, // HOLE
+    0x00000000, // CB_COLOR_CONTROL
+    0x00000200, // DB_SHADER_CONTROL
+    0x00000000, // PA_CL_CLIP_CNTL
+    0x00000000, // PA_SU_SC_MODE_CNTL
+    0x00000000, // PA_CL_VTE_CNTL
+    0x00000000, // PA_CL_VS_OUT_CNTL
+    0x00000000, // PA_CL_NANINF_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_SCALE
+    0x00000000, // PA_SU_PRIM_FILTER_CNTL
+    0x00000000, // SQ_LSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_HSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_DYN_GPR_RESOURCE_LIMIT_1
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_PS
+    0x00000000, // SQ_PGM_RESOURCES_PS
+    0x00000000, // SQ_PGM_RESOURCES_2_PS
+    0x00000000, // SQ_PGM_EXPORTS_PS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_VS
+    0x00000000, // SQ_PGM_RESOURCES_VS
+    0x00000000, // SQ_PGM_RESOURCES_2_VS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_GS
+    0x00000000, // SQ_PGM_RESOURCES_GS
+    0x00000000, // SQ_PGM_RESOURCES_2_GS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_ES
+    0x00000000, // SQ_PGM_RESOURCES_ES
+    0x00000000, // SQ_PGM_RESOURCES_2_ES
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_FS
+    0x00000000, // SQ_PGM_RESOURCES_FS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_HS
+    0x00000000, // SQ_PGM_RESOURCES_HS
+    0x00000000, // SQ_PGM_RESOURCES_2_HS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_PGM_START_LS
+    0x00000000, // SQ_PGM_RESOURCES_LS
+    0x00000000, // SQ_PGM_RESOURCES_2_LS
+};
+static const u32 SECT_CONTEXT_def_4[] =
+{
+    0x00000000, // SQ_LDS_ALLOC
+    0x00000000, // SQ_LDS_ALLOC_PS
+    0x00000000, // SQ_VTX_SEMANTIC_CLEAR
+    0, // HOLE
+    0x00000000, // SQ_THREAD_TRACE_CTRL
+    0, // HOLE
+    0x00000000, // SQ_ESGS_RING_ITEMSIZE
+    0x00000000, // SQ_GSVS_RING_ITEMSIZE
+    0x00000000, // SQ_ESTMP_RING_ITEMSIZE
+    0x00000000, // SQ_GSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_VSTMP_RING_ITEMSIZE
+    0x00000000, // SQ_PSTMP_RING_ITEMSIZE
+    0, // HOLE
+    0x00000000, // SQ_GS_VERT_ITEMSIZE
+    0x00000000, // SQ_GS_VERT_ITEMSIZE_1
+    0x00000000, // SQ_GS_VERT_ITEMSIZE_2
+    0x00000000, // SQ_GS_VERT_ITEMSIZE_3
+    0x00000000, // SQ_GSVS_RING_OFFSET_1
+    0x00000000, // SQ_GSVS_RING_OFFSET_2
+    0x00000000, // SQ_GSVS_RING_OFFSET_3
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_PS_15
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_VS_15
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_GS_15
+    0x00000000, // PA_SU_POINT_SIZE
+    0x00000000, // PA_SU_POINT_MINMAX
+    0x00000000, // PA_SU_LINE_CNTL
+    0x00000000, // PA_SC_LINE_STIPPLE
+    0x00000000, // VGT_OUTPUT_PATH_CNTL
+    0x00000000, // VGT_HOS_CNTL
+    0x00000000, // VGT_HOS_MAX_TESS_LEVEL
+    0x00000000, // VGT_HOS_MIN_TESS_LEVEL
+    0x00000000, // VGT_HOS_REUSE_DEPTH
+    0x00000000, // VGT_GROUP_PRIM_TYPE
+    0x00000000, // VGT_GROUP_FIRST_DECR
+    0x00000000, // VGT_GROUP_DECR
+    0x00000000, // VGT_GROUP_VECT_0_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_CNTL
+    0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL
+    0x00000000, // VGT_GS_MODE
+    0, // HOLE
+    0x00000000, // PA_SC_MODE_CNTL_0
+    0x00000000, // PA_SC_MODE_CNTL_1
+    0x00000000, // VGT_ENHANCE
+    0x00000000, // VGT_GS_PER_ES
+    0x00000000, // VGT_ES_PER_GS
+    0x00000000, // VGT_GS_PER_VS
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_GS_OUT_PRIM_TYPE
+};
+static const u32 SECT_CONTEXT_def_5[] =
+{
+    0x00000000, // VGT_DMA_MAX_SIZE
+    0x00000000, // VGT_DMA_INDEX_TYPE
+    0, // HOLE
+    0x00000000, // VGT_PRIMITIVEID_EN
+    0x00000000, // VGT_DMA_NUM_INSTANCES
+};
+static const u32 SECT_CONTEXT_def_6[] =
+{
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_INSTANCE_STEP_RATE_0
+    0x00000000, // VGT_INSTANCE_STEP_RATE_1
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_REUSE_OFF
+    0x00000000, // VGT_VTX_CNT_EN
+    0x00000000, // DB_HTILE_SURFACE
+    0x00000000, // DB_SRESULTS_COMPARE_STATE0
+    0x00000000, // DB_SRESULTS_COMPARE_STATE1
+    0x00000000, // DB_PRELOAD_CONTROL
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_0
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_0
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_1
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_1
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_2
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_2
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_3
+    0x00000000, // VGT_STRMOUT_BUFFER_BASE_3
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_0
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_1
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_2
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_3
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+    0, // HOLE
+    0x00000000, // VGT_GS_MAX_VERT_OUT
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_0
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_1
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_2
+    0x00000000, // VGT_STRMOUT_BASE_OFFSET_HI_3
+    0x00000000, // VGT_SHADER_STAGES_EN
+    0x00000000, // VGT_LS_HS_CONFIG
+    0x00000000, // VGT_LS_SIZE
+    0x00000000, // VGT_HS_SIZE
+    0x00000000, // VGT_LS_HS_ALLOC
+    0x00000000, // VGT_HS_PATCH_CONST
+    0x00000000, // VGT_TF_PARAM
+    0x00000000, // DB_ALPHA_TO_MASK
+};
+static const u32 SECT_CONTEXT_def_7[] =
+{
+    0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL
+    0x00000000, // PA_SU_POLY_OFFSET_CLAMP
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET
+    0x00000000, // VGT_GS_INSTANCE_CNT
+    0x00000000, // VGT_STRMOUT_CONFIG
+    0x00000000, // VGT_STRMOUT_BUFFER_CONFIG
+    0x00000000, // CB_IMMED0_BASE
+    0x00000000, // CB_IMMED1_BASE
+    0x00000000, // CB_IMMED2_BASE
+    0x00000000, // CB_IMMED3_BASE
+    0x00000000, // CB_IMMED4_BASE
+    0x00000000, // CB_IMMED5_BASE
+    0x00000000, // CB_IMMED6_BASE
+    0x00000000, // CB_IMMED7_BASE
+    0x00000000, // CB_IMMED8_BASE
+    0x00000000, // CB_IMMED9_BASE
+    0x00000000, // CB_IMMED10_BASE
+    0x00000000, // CB_IMMED11_BASE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00001000, // PA_SC_LINE_CNTL
+    0x00000000, // PA_SC_AA_CONFIG
+    0x00000005, // PA_SU_VTX_CNTL
+    0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_VERT_DISC_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_4
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_5
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_6
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_7
+    0xffffffff, // PA_SC_AA_MASK
+    0x00000000, // CB_CLRCMP_CONTROL
+    0x00000000, // CB_CLRCMP_SRC
+    0x00000000, // CB_CLRCMP_DST
+    0x00000000, // CB_CLRCMP_MSK
+    0, // HOLE
+    0, // HOLE
+    0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL
+    0x00000010, // VGT_OUT_DEALLOC_CNTL
+    0x00000000, // CB_COLOR0_BASE
+    0x00000000, // CB_COLOR0_PITCH
+    0x00000000, // CB_COLOR0_SLICE
+    0x00000000, // CB_COLOR0_VIEW
+    0x00000000, // CB_COLOR0_INFO
+    0x00000000, // CB_COLOR0_ATTRIB
+    0x00000000, // CB_COLOR0_DIM
+    0x00000000, // CB_COLOR0_CMASK
+    0x00000000, // CB_COLOR0_CMASK_SLICE
+    0x00000000, // CB_COLOR0_FMASK
+    0x00000000, // CB_COLOR0_FMASK_SLICE
+    0x00000000, // CB_COLOR0_CLEAR_WORD0
+    0x00000000, // CB_COLOR0_CLEAR_WORD1
+    0x00000000, // CB_COLOR0_CLEAR_WORD2
+    0x00000000, // CB_COLOR0_CLEAR_WORD3
+    0x00000000, // CB_COLOR1_BASE
+    0x00000000, // CB_COLOR1_PITCH
+    0x00000000, // CB_COLOR1_SLICE
+    0x00000000, // CB_COLOR1_VIEW
+    0x00000000, // CB_COLOR1_INFO
+    0x00000000, // CB_COLOR1_ATTRIB
+    0x00000000, // CB_COLOR1_DIM
+    0x00000000, // CB_COLOR1_CMASK
+    0x00000000, // CB_COLOR1_CMASK_SLICE
+    0x00000000, // CB_COLOR1_FMASK
+    0x00000000, // CB_COLOR1_FMASK_SLICE
+    0x00000000, // CB_COLOR1_CLEAR_WORD0
+    0x00000000, // CB_COLOR1_CLEAR_WORD1
+    0x00000000, // CB_COLOR1_CLEAR_WORD2
+    0x00000000, // CB_COLOR1_CLEAR_WORD3
+    0x00000000, // CB_COLOR2_BASE
+    0x00000000, // CB_COLOR2_PITCH
+    0x00000000, // CB_COLOR2_SLICE
+    0x00000000, // CB_COLOR2_VIEW
+    0x00000000, // CB_COLOR2_INFO
+    0x00000000, // CB_COLOR2_ATTRIB
+    0x00000000, // CB_COLOR2_DIM
+    0x00000000, // CB_COLOR2_CMASK
+    0x00000000, // CB_COLOR2_CMASK_SLICE
+    0x00000000, // CB_COLOR2_FMASK
+    0x00000000, // CB_COLOR2_FMASK_SLICE
+    0x00000000, // CB_COLOR2_CLEAR_WORD0
+    0x00000000, // CB_COLOR2_CLEAR_WORD1
+    0x00000000, // CB_COLOR2_CLEAR_WORD2
+    0x00000000, // CB_COLOR2_CLEAR_WORD3
+    0x00000000, // CB_COLOR3_BASE
+    0x00000000, // CB_COLOR3_PITCH
+    0x00000000, // CB_COLOR3_SLICE
+    0x00000000, // CB_COLOR3_VIEW
+    0x00000000, // CB_COLOR3_INFO
+    0x00000000, // CB_COLOR3_ATTRIB
+    0x00000000, // CB_COLOR3_DIM
+    0x00000000, // CB_COLOR3_CMASK
+    0x00000000, // CB_COLOR3_CMASK_SLICE
+    0x00000000, // CB_COLOR3_FMASK
+    0x00000000, // CB_COLOR3_FMASK_SLICE
+    0x00000000, // CB_COLOR3_CLEAR_WORD0
+    0x00000000, // CB_COLOR3_CLEAR_WORD1
+    0x00000000, // CB_COLOR3_CLEAR_WORD2
+    0x00000000, // CB_COLOR3_CLEAR_WORD3
+    0x00000000, // CB_COLOR4_BASE
+    0x00000000, // CB_COLOR4_PITCH
+    0x00000000, // CB_COLOR4_SLICE
+    0x00000000, // CB_COLOR4_VIEW
+    0x00000000, // CB_COLOR4_INFO
+    0x00000000, // CB_COLOR4_ATTRIB
+    0x00000000, // CB_COLOR4_DIM
+    0x00000000, // CB_COLOR4_CMASK
+    0x00000000, // CB_COLOR4_CMASK_SLICE
+    0x00000000, // CB_COLOR4_FMASK
+    0x00000000, // CB_COLOR4_FMASK_SLICE
+    0x00000000, // CB_COLOR4_CLEAR_WORD0
+    0x00000000, // CB_COLOR4_CLEAR_WORD1
+    0x00000000, // CB_COLOR4_CLEAR_WORD2
+    0x00000000, // CB_COLOR4_CLEAR_WORD3
+    0x00000000, // CB_COLOR5_BASE
+    0x00000000, // CB_COLOR5_PITCH
+    0x00000000, // CB_COLOR5_SLICE
+    0x00000000, // CB_COLOR5_VIEW
+    0x00000000, // CB_COLOR5_INFO
+    0x00000000, // CB_COLOR5_ATTRIB
+    0x00000000, // CB_COLOR5_DIM
+    0x00000000, // CB_COLOR5_CMASK
+    0x00000000, // CB_COLOR5_CMASK_SLICE
+    0x00000000, // CB_COLOR5_FMASK
+    0x00000000, // CB_COLOR5_FMASK_SLICE
+    0x00000000, // CB_COLOR5_CLEAR_WORD0
+    0x00000000, // CB_COLOR5_CLEAR_WORD1
+    0x00000000, // CB_COLOR5_CLEAR_WORD2
+    0x00000000, // CB_COLOR5_CLEAR_WORD3
+    0x00000000, // CB_COLOR6_BASE
+    0x00000000, // CB_COLOR6_PITCH
+    0x00000000, // CB_COLOR6_SLICE
+    0x00000000, // CB_COLOR6_VIEW
+    0x00000000, // CB_COLOR6_INFO
+    0x00000000, // CB_COLOR6_ATTRIB
+    0x00000000, // CB_COLOR6_DIM
+    0x00000000, // CB_COLOR6_CMASK
+    0x00000000, // CB_COLOR6_CMASK_SLICE
+    0x00000000, // CB_COLOR6_FMASK
+    0x00000000, // CB_COLOR6_FMASK_SLICE
+    0x00000000, // CB_COLOR6_CLEAR_WORD0
+    0x00000000, // CB_COLOR6_CLEAR_WORD1
+    0x00000000, // CB_COLOR6_CLEAR_WORD2
+    0x00000000, // CB_COLOR6_CLEAR_WORD3
+    0x00000000, // CB_COLOR7_BASE
+    0x00000000, // CB_COLOR7_PITCH
+    0x00000000, // CB_COLOR7_SLICE
+    0x00000000, // CB_COLOR7_VIEW
+    0x00000000, // CB_COLOR7_INFO
+    0x00000000, // CB_COLOR7_ATTRIB
+    0x00000000, // CB_COLOR7_DIM
+    0x00000000, // CB_COLOR7_CMASK
+    0x00000000, // CB_COLOR7_CMASK_SLICE
+    0x00000000, // CB_COLOR7_FMASK
+    0x00000000, // CB_COLOR7_FMASK_SLICE
+    0x00000000, // CB_COLOR7_CLEAR_WORD0
+    0x00000000, // CB_COLOR7_CLEAR_WORD1
+    0x00000000, // CB_COLOR7_CLEAR_WORD2
+    0x00000000, // CB_COLOR7_CLEAR_WORD3
+    0x00000000, // CB_COLOR8_BASE
+    0x00000000, // CB_COLOR8_PITCH
+    0x00000000, // CB_COLOR8_SLICE
+    0x00000000, // CB_COLOR8_VIEW
+    0x00000000, // CB_COLOR8_INFO
+    0x00000000, // CB_COLOR8_ATTRIB
+    0x00000000, // CB_COLOR8_DIM
+    0x00000000, // CB_COLOR9_BASE
+    0x00000000, // CB_COLOR9_PITCH
+    0x00000000, // CB_COLOR9_SLICE
+    0x00000000, // CB_COLOR9_VIEW
+    0x00000000, // CB_COLOR9_INFO
+    0x00000000, // CB_COLOR9_ATTRIB
+    0x00000000, // CB_COLOR9_DIM
+    0x00000000, // CB_COLOR10_BASE
+    0x00000000, // CB_COLOR10_PITCH
+    0x00000000, // CB_COLOR10_SLICE
+    0x00000000, // CB_COLOR10_VIEW
+    0x00000000, // CB_COLOR10_INFO
+    0x00000000, // CB_COLOR10_ATTRIB
+    0x00000000, // CB_COLOR10_DIM
+    0x00000000, // CB_COLOR11_BASE
+    0x00000000, // CB_COLOR11_PITCH
+    0x00000000, // CB_COLOR11_SLICE
+    0x00000000, // CB_COLOR11_VIEW
+    0x00000000, // CB_COLOR11_INFO
+    0x00000000, // CB_COLOR11_ATTRIB
+    0x00000000, // CB_COLOR11_DIM
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_HS_15
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_0
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_1
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_2
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_3
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_4
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_5
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_6
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_7
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_8
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_9
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_10
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_11
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_12
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_13
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_14
+    0x00000000, // SQ_ALU_CONST_CACHE_LS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_HS_15
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_0
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_1
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_2
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_3
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_4
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_5
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_6
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_7
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_8
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_9
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_10
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_11
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_12
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_13
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_14
+    0x00000000, // SQ_ALU_CONST_BUFFER_SIZE_LS_15
+};
+static const struct cs_extent_def SECT_CONTEXT_defs[] =
+{
+    {SECT_CONTEXT_def_1, 0x0000a000, 488 },
+    {SECT_CONTEXT_def_2, 0x0000a1f5, 6 },
+    {SECT_CONTEXT_def_3, 0x0000a200, 55 },
+    {SECT_CONTEXT_def_4, 0x0000a23a, 98 },
+    {SECT_CONTEXT_def_5, 0x0000a29e, 5 },
+    {SECT_CONTEXT_def_6, 0x0000a2a5, 56 },
+    {SECT_CONTEXT_def_7, 0x0000a2de, 290 },
+    { 0, 0, 0 }
+};
+static const u32 SECT_CLEAR_def_1[] =
+{
+    0xffffffff, // SQ_TEX_SAMPLER_CLEAR
+    0xffffffff, // SQ_TEX_RESOURCE_CLEAR
+    0xffffffff, // SQ_LOOP_BOOL_CLEAR
+};
+static const struct cs_extent_def SECT_CLEAR_defs[] =
+{
+    {SECT_CLEAR_def_1, 0x0000ffc0, 3 },
+    { 0, 0, 0 }
+};
+static const u32 SECT_CTRLCONST_def_1[] =
+{
+    0x00000000, // SQ_VTX_BASE_VTX_LOC
+    0x00000000, // SQ_VTX_START_INST_LOC
+};
+static const struct cs_extent_def SECT_CTRLCONST_defs[] =
+{
+    {SECT_CTRLCONST_def_1, 0x0000f3fc, 2 },
+    { 0, 0, 0 }
+};
+struct cs_section_def evergreen_cs_data[] = {
+    { SECT_CONTEXT_defs, SECT_CONTEXT },
+    { SECT_CLEAR_defs, SECT_CLEAR },
+    { SECT_CTRLCONST_defs, SECT_CTRLCONST },
+    { 0, SECT_NONE }
+};
diff --git a/drivers/gpu/drm/radeon/clearstate_si.h b/drivers/gpu/drm/radeon/clearstate_si.h
new file mode 100644
index 000000000000..b994cb2a35a0
--- /dev/null
+++ b/drivers/gpu/drm/radeon/clearstate_si.h
@@ -0,0 +1,941 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+static const u32 si_SECT_CONTEXT_def_1[] =
+{
+    0x00000000, // DB_RENDER_CONTROL
+    0x00000000, // DB_COUNT_CONTROL
+    0x00000000, // DB_DEPTH_VIEW
+    0x00000000, // DB_RENDER_OVERRIDE
+    0x00000000, // DB_RENDER_OVERRIDE2
+    0x00000000, // DB_HTILE_DATA_BASE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_DEPTH_BOUNDS_MIN
+    0x00000000, // DB_DEPTH_BOUNDS_MAX
+    0x00000000, // DB_STENCIL_CLEAR
+    0x00000000, // DB_DEPTH_CLEAR
+    0x00000000, // PA_SC_SCREEN_SCISSOR_TL
+    0x40004000, // PA_SC_SCREEN_SCISSOR_BR
+    0, // HOLE
+    0x00000000, // DB_DEPTH_INFO
+    0x00000000, // DB_Z_INFO
+    0x00000000, // DB_STENCIL_INFO
+    0x00000000, // DB_Z_READ_BASE
+    0x00000000, // DB_STENCIL_READ_BASE
+    0x00000000, // DB_Z_WRITE_BASE
+    0x00000000, // DB_STENCIL_WRITE_BASE
+    0x00000000, // DB_DEPTH_SIZE
+    0x00000000, // DB_DEPTH_SLICE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // TA_BC_BASE_ADDR
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // COHER_DEST_BASE_2
+    0x00000000, // COHER_DEST_BASE_3
+    0x00000000, // PA_SC_WINDOW_OFFSET
+    0x80000000, // PA_SC_WINDOW_SCISSOR_TL
+    0x40004000, // PA_SC_WINDOW_SCISSOR_BR
+    0x0000ffff, // PA_SC_CLIPRECT_RULE
+    0x00000000, // PA_SC_CLIPRECT_0_TL
+    0x40004000, // PA_SC_CLIPRECT_0_BR
+    0x00000000, // PA_SC_CLIPRECT_1_TL
+    0x40004000, // PA_SC_CLIPRECT_1_BR
+    0x00000000, // PA_SC_CLIPRECT_2_TL
+    0x40004000, // PA_SC_CLIPRECT_2_BR
+    0x00000000, // PA_SC_CLIPRECT_3_TL
+    0x40004000, // PA_SC_CLIPRECT_3_BR
+    0xaa99aaaa, // PA_SC_EDGERULE
+    0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET
+    0xffffffff, // CB_TARGET_MASK
+    0xffffffff, // CB_SHADER_MASK
+    0x80000000, // PA_SC_GENERIC_SCISSOR_TL
+    0x40004000, // PA_SC_GENERIC_SCISSOR_BR
+    0x00000000, // COHER_DEST_BASE_0
+    0x00000000, // COHER_DEST_BASE_1
+    0x80000000, // PA_SC_VPORT_SCISSOR_0_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_0_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_1_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_1_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_2_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_2_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_3_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_3_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_4_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_4_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_5_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_5_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_6_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_6_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_7_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_7_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_8_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_8_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_9_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_9_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_10_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_10_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_11_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_11_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_12_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_12_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_13_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_13_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_14_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_14_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_15_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_15_BR
+    0x00000000, // PA_SC_VPORT_ZMIN_0
+    0x3f800000, // PA_SC_VPORT_ZMAX_0
+    0x00000000, // PA_SC_VPORT_ZMIN_1
+    0x3f800000, // PA_SC_VPORT_ZMAX_1
+    0x00000000, // PA_SC_VPORT_ZMIN_2
+    0x3f800000, // PA_SC_VPORT_ZMAX_2
+    0x00000000, // PA_SC_VPORT_ZMIN_3
+    0x3f800000, // PA_SC_VPORT_ZMAX_3
+    0x00000000, // PA_SC_VPORT_ZMIN_4
+    0x3f800000, // PA_SC_VPORT_ZMAX_4
+    0x00000000, // PA_SC_VPORT_ZMIN_5
+    0x3f800000, // PA_SC_VPORT_ZMAX_5
+    0x00000000, // PA_SC_VPORT_ZMIN_6
+    0x3f800000, // PA_SC_VPORT_ZMAX_6
+    0x00000000, // PA_SC_VPORT_ZMIN_7
+    0x3f800000, // PA_SC_VPORT_ZMAX_7
+    0x00000000, // PA_SC_VPORT_ZMIN_8
+    0x3f800000, // PA_SC_VPORT_ZMAX_8
+    0x00000000, // PA_SC_VPORT_ZMIN_9
+    0x3f800000, // PA_SC_VPORT_ZMAX_9
+    0x00000000, // PA_SC_VPORT_ZMIN_10
+    0x3f800000, // PA_SC_VPORT_ZMAX_10
+    0x00000000, // PA_SC_VPORT_ZMIN_11
+    0x3f800000, // PA_SC_VPORT_ZMAX_11
+    0x00000000, // PA_SC_VPORT_ZMIN_12
+    0x3f800000, // PA_SC_VPORT_ZMAX_12
+    0x00000000, // PA_SC_VPORT_ZMIN_13
+    0x3f800000, // PA_SC_VPORT_ZMAX_13
+    0x00000000, // PA_SC_VPORT_ZMIN_14
+    0x3f800000, // PA_SC_VPORT_ZMAX_14
+    0x00000000, // PA_SC_VPORT_ZMIN_15
+    0x3f800000, // PA_SC_VPORT_ZMAX_15
+};
+static const u32 si_SECT_CONTEXT_def_2[] =
+{
+    0x00000000, // CP_PERFMON_CNTX_CNTL
+    0x00000000, // CP_RINGID
+    0x00000000, // CP_VMID
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0xffffffff, // VGT_MAX_VTX_INDX
+    0x00000000, // VGT_MIN_VTX_INDX
+    0x00000000, // VGT_INDX_OFFSET
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX
+    0, // HOLE
+    0x00000000, // CB_BLEND_RED
+    0x00000000, // CB_BLEND_GREEN
+    0x00000000, // CB_BLEND_BLUE
+    0x00000000, // CB_BLEND_ALPHA
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_STENCIL_CONTROL
+    0x00000000, // DB_STENCILREFMASK
+    0x00000000, // DB_STENCILREFMASK_BF
+    0, // HOLE
+    0x00000000, // PA_CL_VPORT_XSCALE
+    0x00000000, // PA_CL_VPORT_XOFFSET
+    0x00000000, // PA_CL_VPORT_YSCALE
+    0x00000000, // PA_CL_VPORT_YOFFSET
+    0x00000000, // PA_CL_VPORT_ZSCALE
+    0x00000000, // PA_CL_VPORT_ZOFFSET
+    0x00000000, // PA_CL_VPORT_XSCALE_1
+    0x00000000, // PA_CL_VPORT_XOFFSET_1
+    0x00000000, // PA_CL_VPORT_YSCALE_1
+    0x00000000, // PA_CL_VPORT_YOFFSET_1
+    0x00000000, // PA_CL_VPORT_ZSCALE_1
+    0x00000000, // PA_CL_VPORT_ZOFFSET_1
+    0x00000000, // PA_CL_VPORT_XSCALE_2
+    0x00000000, // PA_CL_VPORT_XOFFSET_2
+    0x00000000, // PA_CL_VPORT_YSCALE_2
+    0x00000000, // PA_CL_VPORT_YOFFSET_2
+    0x00000000, // PA_CL_VPORT_ZSCALE_2
+    0x00000000, // PA_CL_VPORT_ZOFFSET_2
+    0x00000000, // PA_CL_VPORT_XSCALE_3
+    0x00000000, // PA_CL_VPORT_XOFFSET_3
+    0x00000000, // PA_CL_VPORT_YSCALE_3
+    0x00000000, // PA_CL_VPORT_YOFFSET_3
+    0x00000000, // PA_CL_VPORT_ZSCALE_3
+    0x00000000, // PA_CL_VPORT_ZOFFSET_3
+    0x00000000, // PA_CL_VPORT_XSCALE_4
+    0x00000000, // PA_CL_VPORT_XOFFSET_4
+    0x00000000, // PA_CL_VPORT_YSCALE_4
+    0x00000000, // PA_CL_VPORT_YOFFSET_4
+    0x00000000, // PA_CL_VPORT_ZSCALE_4
+    0x00000000, // PA_CL_VPORT_ZOFFSET_4
+    0x00000000, // PA_CL_VPORT_XSCALE_5
+    0x00000000, // PA_CL_VPORT_XOFFSET_5
+    0x00000000, // PA_CL_VPORT_YSCALE_5
+    0x00000000, // PA_CL_VPORT_YOFFSET_5
+    0x00000000, // PA_CL_VPORT_ZSCALE_5
+    0x00000000, // PA_CL_VPORT_ZOFFSET_5
+    0x00000000, // PA_CL_VPORT_XSCALE_6
+    0x00000000, // PA_CL_VPORT_XOFFSET_6
+    0x00000000, // PA_CL_VPORT_YSCALE_6
+    0x00000000, // PA_CL_VPORT_YOFFSET_6
+    0x00000000, // PA_CL_VPORT_ZSCALE_6
+    0x00000000, // PA_CL_VPORT_ZOFFSET_6
+    0x00000000, // PA_CL_VPORT_XSCALE_7
+    0x00000000, // PA_CL_VPORT_XOFFSET_7
+    0x00000000, // PA_CL_VPORT_YSCALE_7
+    0x00000000, // PA_CL_VPORT_YOFFSET_7
+    0x00000000, // PA_CL_VPORT_ZSCALE_7
+    0x00000000, // PA_CL_VPORT_ZOFFSET_7
+    0x00000000, // PA_CL_VPORT_XSCALE_8
+    0x00000000, // PA_CL_VPORT_XOFFSET_8
+    0x00000000, // PA_CL_VPORT_YSCALE_8
+    0x00000000, // PA_CL_VPORT_YOFFSET_8
+    0x00000000, // PA_CL_VPORT_ZSCALE_8
+    0x00000000, // PA_CL_VPORT_ZOFFSET_8
+    0x00000000, // PA_CL_VPORT_XSCALE_9
+    0x00000000, // PA_CL_VPORT_XOFFSET_9
+    0x00000000, // PA_CL_VPORT_YSCALE_9
+    0x00000000, // PA_CL_VPORT_YOFFSET_9
+    0x00000000, // PA_CL_VPORT_ZSCALE_9
+    0x00000000, // PA_CL_VPORT_ZOFFSET_9
+    0x00000000, // PA_CL_VPORT_XSCALE_10
+    0x00000000, // PA_CL_VPORT_XOFFSET_10
+    0x00000000, // PA_CL_VPORT_YSCALE_10
+    0x00000000, // PA_CL_VPORT_YOFFSET_10
+    0x00000000, // PA_CL_VPORT_ZSCALE_10
+    0x00000000, // PA_CL_VPORT_ZOFFSET_10
+    0x00000000, // PA_CL_VPORT_XSCALE_11
+    0x00000000, // PA_CL_VPORT_XOFFSET_11
+    0x00000000, // PA_CL_VPORT_YSCALE_11
+    0x00000000, // PA_CL_VPORT_YOFFSET_11
+    0x00000000, // PA_CL_VPORT_ZSCALE_11
+    0x00000000, // PA_CL_VPORT_ZOFFSET_11
+    0x00000000, // PA_CL_VPORT_XSCALE_12
+    0x00000000, // PA_CL_VPORT_XOFFSET_12
+    0x00000000, // PA_CL_VPORT_YSCALE_12
+    0x00000000, // PA_CL_VPORT_YOFFSET_12
+    0x00000000, // PA_CL_VPORT_ZSCALE_12
+    0x00000000, // PA_CL_VPORT_ZOFFSET_12
+    0x00000000, // PA_CL_VPORT_XSCALE_13
+    0x00000000, // PA_CL_VPORT_XOFFSET_13
+    0x00000000, // PA_CL_VPORT_YSCALE_13
+    0x00000000, // PA_CL_VPORT_YOFFSET_13
+    0x00000000, // PA_CL_VPORT_ZSCALE_13
+    0x00000000, // PA_CL_VPORT_ZOFFSET_13
+    0x00000000, // PA_CL_VPORT_XSCALE_14
+    0x00000000, // PA_CL_VPORT_XOFFSET_14
+    0x00000000, // PA_CL_VPORT_YSCALE_14
+    0x00000000, // PA_CL_VPORT_YOFFSET_14
+    0x00000000, // PA_CL_VPORT_ZSCALE_14
+    0x00000000, // PA_CL_VPORT_ZOFFSET_14
+    0x00000000, // PA_CL_VPORT_XSCALE_15
+    0x00000000, // PA_CL_VPORT_XOFFSET_15
+    0x00000000, // PA_CL_VPORT_YSCALE_15
+    0x00000000, // PA_CL_VPORT_YOFFSET_15
+    0x00000000, // PA_CL_VPORT_ZSCALE_15
+    0x00000000, // PA_CL_VPORT_ZOFFSET_15
+    0x00000000, // PA_CL_UCP_0_X
+    0x00000000, // PA_CL_UCP_0_Y
+    0x00000000, // PA_CL_UCP_0_Z
+    0x00000000, // PA_CL_UCP_0_W
+    0x00000000, // PA_CL_UCP_1_X
+    0x00000000, // PA_CL_UCP_1_Y
+    0x00000000, // PA_CL_UCP_1_Z
+    0x00000000, // PA_CL_UCP_1_W
+    0x00000000, // PA_CL_UCP_2_X
+    0x00000000, // PA_CL_UCP_2_Y
+    0x00000000, // PA_CL_UCP_2_Z
+    0x00000000, // PA_CL_UCP_2_W
+    0x00000000, // PA_CL_UCP_3_X
+    0x00000000, // PA_CL_UCP_3_Y
+    0x00000000, // PA_CL_UCP_3_Z
+    0x00000000, // PA_CL_UCP_3_W
+    0x00000000, // PA_CL_UCP_4_X
+    0x00000000, // PA_CL_UCP_4_Y
+    0x00000000, // PA_CL_UCP_4_Z
+    0x00000000, // PA_CL_UCP_4_W
+    0x00000000, // PA_CL_UCP_5_X
+    0x00000000, // PA_CL_UCP_5_Y
+    0x00000000, // PA_CL_UCP_5_Z
+    0x00000000, // PA_CL_UCP_5_W
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SPI_PS_INPUT_CNTL_0
+    0x00000000, // SPI_PS_INPUT_CNTL_1
+    0x00000000, // SPI_PS_INPUT_CNTL_2
+    0x00000000, // SPI_PS_INPUT_CNTL_3
+    0x00000000, // SPI_PS_INPUT_CNTL_4
+    0x00000000, // SPI_PS_INPUT_CNTL_5
+    0x00000000, // SPI_PS_INPUT_CNTL_6
+    0x00000000, // SPI_PS_INPUT_CNTL_7
+    0x00000000, // SPI_PS_INPUT_CNTL_8
+    0x00000000, // SPI_PS_INPUT_CNTL_9
+    0x00000000, // SPI_PS_INPUT_CNTL_10
+    0x00000000, // SPI_PS_INPUT_CNTL_11
+    0x00000000, // SPI_PS_INPUT_CNTL_12
+    0x00000000, // SPI_PS_INPUT_CNTL_13
+    0x00000000, // SPI_PS_INPUT_CNTL_14
+    0x00000000, // SPI_PS_INPUT_CNTL_15
+    0x00000000, // SPI_PS_INPUT_CNTL_16
+    0x00000000, // SPI_PS_INPUT_CNTL_17
+    0x00000000, // SPI_PS_INPUT_CNTL_18
+    0x00000000, // SPI_PS_INPUT_CNTL_19
+    0x00000000, // SPI_PS_INPUT_CNTL_20
+    0x00000000, // SPI_PS_INPUT_CNTL_21
+    0x00000000, // SPI_PS_INPUT_CNTL_22
+    0x00000000, // SPI_PS_INPUT_CNTL_23
+    0x00000000, // SPI_PS_INPUT_CNTL_24
+    0x00000000, // SPI_PS_INPUT_CNTL_25
+    0x00000000, // SPI_PS_INPUT_CNTL_26
+    0x00000000, // SPI_PS_INPUT_CNTL_27
+    0x00000000, // SPI_PS_INPUT_CNTL_28
+    0x00000000, // SPI_PS_INPUT_CNTL_29
+    0x00000000, // SPI_PS_INPUT_CNTL_30
+    0x00000000, // SPI_PS_INPUT_CNTL_31
+    0x00000000, // SPI_VS_OUT_CONFIG
+    0, // HOLE
+    0x00000000, // SPI_PS_INPUT_ENA
+    0x00000000, // SPI_PS_INPUT_ADDR
+    0x00000000, // SPI_INTERP_CONTROL_0
+    0x00000002, // SPI_PS_IN_CONTROL
+    0, // HOLE
+    0x00000000, // SPI_BARYC_CNTL
+    0, // HOLE
+    0x00000000, // SPI_TMPRING_SIZE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SPI_WAVE_MGMT_1
+    0x00000000, // SPI_WAVE_MGMT_2
+    0x00000000, // SPI_SHADER_POS_FORMAT
+    0x00000000, // SPI_SHADER_Z_FORMAT
+    0x00000000, // SPI_SHADER_COL_FORMAT
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_BLEND0_CONTROL
+    0x00000000, // CB_BLEND1_CONTROL
+    0x00000000, // CB_BLEND2_CONTROL
+    0x00000000, // CB_BLEND3_CONTROL
+    0x00000000, // CB_BLEND4_CONTROL
+    0x00000000, // CB_BLEND5_CONTROL
+    0x00000000, // CB_BLEND6_CONTROL
+    0x00000000, // CB_BLEND7_CONTROL
+};
+static const u32 si_SECT_CONTEXT_def_3[] =
+{
+    0x00000000, // PA_CL_POINT_X_RAD
+    0x00000000, // PA_CL_POINT_Y_RAD
+    0x00000000, // PA_CL_POINT_SIZE
+    0x00000000, // PA_CL_POINT_CULL_RAD
+    0x00000000, // VGT_DMA_BASE_HI
+    0x00000000, // VGT_DMA_BASE
+};
+static const u32 si_SECT_CONTEXT_def_4[] =
+{
+    0x00000000, // DB_DEPTH_CONTROL
+    0x00000000, // DB_EQAA
+    0x00000000, // CB_COLOR_CONTROL
+    0x00000000, // DB_SHADER_CONTROL
+    0x00090000, // PA_CL_CLIP_CNTL
+    0x00000004, // PA_SU_SC_MODE_CNTL
+    0x00000000, // PA_CL_VTE_CNTL
+    0x00000000, // PA_CL_VS_OUT_CNTL
+    0x00000000, // PA_CL_NANINF_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_SCALE
+    0x00000000, // PA_SU_PRIM_FILTER_CNTL
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // PA_SU_POINT_SIZE
+    0x00000000, // PA_SU_POINT_MINMAX
+    0x00000000, // PA_SU_LINE_CNTL
+    0x00000000, // PA_SC_LINE_STIPPLE
+    0x00000000, // VGT_OUTPUT_PATH_CNTL
+    0x00000000, // VGT_HOS_CNTL
+    0x00000000, // VGT_HOS_MAX_TESS_LEVEL
+    0x00000000, // VGT_HOS_MIN_TESS_LEVEL
+    0x00000000, // VGT_HOS_REUSE_DEPTH
+    0x00000000, // VGT_GROUP_PRIM_TYPE
+    0x00000000, // VGT_GROUP_FIRST_DECR
+    0x00000000, // VGT_GROUP_DECR
+    0x00000000, // VGT_GROUP_VECT_0_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_CNTL
+    0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL
+    0x00000000, // VGT_GS_MODE
+    0, // HOLE
+    0x00000000, // PA_SC_MODE_CNTL_0
+    0x00000000, // PA_SC_MODE_CNTL_1
+    0x00000000, // VGT_ENHANCE
+    0x00000100, // VGT_GS_PER_ES
+    0x00000080, // VGT_ES_PER_GS
+    0x00000002, // VGT_GS_PER_VS
+    0x00000000, // VGT_GSVS_RING_OFFSET_1
+    0x00000000, // VGT_GSVS_RING_OFFSET_2
+    0x00000000, // VGT_GSVS_RING_OFFSET_3
+    0x00000000, // VGT_GS_OUT_PRIM_TYPE
+    0x00000000, // IA_ENHANCE
+};
+static const u32 si_SECT_CONTEXT_def_5[] =
+{
+    0x00000000, // VGT_PRIMITIVEID_EN
+};
+static const u32 si_SECT_CONTEXT_def_6[] =
+{
+    0x00000000, // VGT_PRIMITIVEID_RESET
+};
+static const u32 si_SECT_CONTEXT_def_7[] =
+{
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_INSTANCE_STEP_RATE_0
+    0x00000000, // VGT_INSTANCE_STEP_RATE_1
+    0x000000ff, // IA_MULTI_VGT_PARAM
+    0x00000000, // VGT_ESGS_RING_ITEMSIZE
+    0x00000000, // VGT_GSVS_RING_ITEMSIZE
+    0x00000000, // VGT_REUSE_OFF
+    0x00000000, // VGT_VTX_CNT_EN
+    0x00000000, // DB_HTILE_SURFACE
+    0x00000000, // DB_SRESULTS_COMPARE_STATE0
+    0x00000000, // DB_SRESULTS_COMPARE_STATE1
+    0x00000000, // DB_PRELOAD_CONTROL
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_0
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_1
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_2
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_3
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+    0, // HOLE
+    0x00000000, // VGT_GS_MAX_VERT_OUT
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_SHADER_STAGES_EN
+    0x00000000, // VGT_LS_HS_CONFIG
+    0x00000000, // VGT_GS_VERT_ITEMSIZE
+    0x00000000, // VGT_GS_VERT_ITEMSIZE_1
+    0x00000000, // VGT_GS_VERT_ITEMSIZE_2
+    0x00000000, // VGT_GS_VERT_ITEMSIZE_3
+    0x00000000, // VGT_TF_PARAM
+    0x00000000, // DB_ALPHA_TO_MASK
+    0, // HOLE
+    0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL
+    0x00000000, // PA_SU_POLY_OFFSET_CLAMP
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET
+    0x00000000, // VGT_GS_INSTANCE_CNT
+    0x00000000, // VGT_STRMOUT_CONFIG
+    0x00000000, // VGT_STRMOUT_BUFFER_CONFIG
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // PA_SC_CENTROID_PRIORITY_0
+    0x00000000, // PA_SC_CENTROID_PRIORITY_1
+    0x00001000, // PA_SC_LINE_CNTL
+    0x00000000, // PA_SC_AA_CONFIG
+    0x00000005, // PA_SU_VTX_CNTL
+    0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_VERT_DISC_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+    0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0
+    0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL
+    0x00000010, // VGT_OUT_DEALLOC_CNTL
+    0x00000000, // CB_COLOR0_BASE
+    0x00000000, // CB_COLOR0_PITCH
+    0x00000000, // CB_COLOR0_SLICE
+    0x00000000, // CB_COLOR0_VIEW
+    0x00000000, // CB_COLOR0_INFO
+    0x00000000, // CB_COLOR0_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR0_CMASK
+    0x00000000, // CB_COLOR0_CMASK_SLICE
+    0x00000000, // CB_COLOR0_FMASK
+    0x00000000, // CB_COLOR0_FMASK_SLICE
+    0x00000000, // CB_COLOR0_CLEAR_WORD0
+    0x00000000, // CB_COLOR0_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR1_BASE
+    0x00000000, // CB_COLOR1_PITCH
+    0x00000000, // CB_COLOR1_SLICE
+    0x00000000, // CB_COLOR1_VIEW
+    0x00000000, // CB_COLOR1_INFO
+    0x00000000, // CB_COLOR1_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR1_CMASK
+    0x00000000, // CB_COLOR1_CMASK_SLICE
+    0x00000000, // CB_COLOR1_FMASK
+    0x00000000, // CB_COLOR1_FMASK_SLICE
+    0x00000000, // CB_COLOR1_CLEAR_WORD0
+    0x00000000, // CB_COLOR1_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR2_BASE
+    0x00000000, // CB_COLOR2_PITCH
+    0x00000000, // CB_COLOR2_SLICE
+    0x00000000, // CB_COLOR2_VIEW
+    0x00000000, // CB_COLOR2_INFO
+    0x00000000, // CB_COLOR2_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR2_CMASK
+    0x00000000, // CB_COLOR2_CMASK_SLICE
+    0x00000000, // CB_COLOR2_FMASK
+    0x00000000, // CB_COLOR2_FMASK_SLICE
+    0x00000000, // CB_COLOR2_CLEAR_WORD0
+    0x00000000, // CB_COLOR2_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR3_BASE
+    0x00000000, // CB_COLOR3_PITCH
+    0x00000000, // CB_COLOR3_SLICE
+    0x00000000, // CB_COLOR3_VIEW
+    0x00000000, // CB_COLOR3_INFO
+    0x00000000, // CB_COLOR3_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR3_CMASK
+    0x00000000, // CB_COLOR3_CMASK_SLICE
+    0x00000000, // CB_COLOR3_FMASK
+    0x00000000, // CB_COLOR3_FMASK_SLICE
+    0x00000000, // CB_COLOR3_CLEAR_WORD0
+    0x00000000, // CB_COLOR3_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR4_BASE
+    0x00000000, // CB_COLOR4_PITCH
+    0x00000000, // CB_COLOR4_SLICE
+    0x00000000, // CB_COLOR4_VIEW
+    0x00000000, // CB_COLOR4_INFO
+    0x00000000, // CB_COLOR4_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR4_CMASK
+    0x00000000, // CB_COLOR4_CMASK_SLICE
+    0x00000000, // CB_COLOR4_FMASK
+    0x00000000, // CB_COLOR4_FMASK_SLICE
+    0x00000000, // CB_COLOR4_CLEAR_WORD0
+    0x00000000, // CB_COLOR4_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR5_BASE
+    0x00000000, // CB_COLOR5_PITCH
+    0x00000000, // CB_COLOR5_SLICE
+    0x00000000, // CB_COLOR5_VIEW
+    0x00000000, // CB_COLOR5_INFO
+    0x00000000, // CB_COLOR5_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR5_CMASK
+    0x00000000, // CB_COLOR5_CMASK_SLICE
+    0x00000000, // CB_COLOR5_FMASK
+    0x00000000, // CB_COLOR5_FMASK_SLICE
+    0x00000000, // CB_COLOR5_CLEAR_WORD0
+    0x00000000, // CB_COLOR5_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR6_BASE
+    0x00000000, // CB_COLOR6_PITCH
+    0x00000000, // CB_COLOR6_SLICE
+    0x00000000, // CB_COLOR6_VIEW
+    0x00000000, // CB_COLOR6_INFO
+    0x00000000, // CB_COLOR6_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR6_CMASK
+    0x00000000, // CB_COLOR6_CMASK_SLICE
+    0x00000000, // CB_COLOR6_FMASK
+    0x00000000, // CB_COLOR6_FMASK_SLICE
+    0x00000000, // CB_COLOR6_CLEAR_WORD0
+    0x00000000, // CB_COLOR6_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR7_BASE
+    0x00000000, // CB_COLOR7_PITCH
+    0x00000000, // CB_COLOR7_SLICE
+    0x00000000, // CB_COLOR7_VIEW
+    0x00000000, // CB_COLOR7_INFO
+    0x00000000, // CB_COLOR7_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR7_CMASK
+    0x00000000, // CB_COLOR7_CMASK_SLICE
+    0x00000000, // CB_COLOR7_FMASK
+    0x00000000, // CB_COLOR7_FMASK_SLICE
+    0x00000000, // CB_COLOR7_CLEAR_WORD0
+    0x00000000, // CB_COLOR7_CLEAR_WORD1
+};
+static const struct cs_extent_def si_SECT_CONTEXT_defs[] =
+{
+    {si_SECT_CONTEXT_def_1, 0x0000a000, 212 },
+    {si_SECT_CONTEXT_def_2, 0x0000a0d8, 272 },
+    {si_SECT_CONTEXT_def_3, 0x0000a1f5, 6 },
+    {si_SECT_CONTEXT_def_4, 0x0000a200, 157 },
+    {si_SECT_CONTEXT_def_5, 0x0000a2a1, 1 },
+    {si_SECT_CONTEXT_def_6, 0x0000a2a3, 1 },
+    {si_SECT_CONTEXT_def_7, 0x0000a2a5, 233 },
+    { 0, 0, 0 }
+};
+static const struct cs_section_def si_cs_data[] = {
+    { si_SECT_CONTEXT_defs, SECT_CONTEXT },
+    { 0, SECT_NONE }
+};
diff --git a/drivers/gpu/drm/radeon/cypress_dpm.c b/drivers/gpu/drm/radeon/cypress_dpm.c
new file mode 100644
index 000000000000..5ada922e5cec
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cypress_dpm.c
@@ -0,0 +1,2176 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "evergreend.h"
+#include "r600_dpm.h"
+#include "cypress_dpm.h"
+#include "atom.h"
+
+#define SMC_RAM_END 0x8000
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define MC_CG_SEQ_DRAMCONF_S0       0x05
+#define MC_CG_SEQ_DRAMCONF_S1       0x06
+#define MC_CG_SEQ_YCLK_SUSPEND      0x04
+#define MC_CG_SEQ_YCLK_RESUME       0x0a
+
+struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps);
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
+struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
+
+static void cypress_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                                 bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp, bif;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        if (enable) {
+                if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+                    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+                        if (!pi->boot_in_gen2) {
+                                bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
+                                bif |= CG_CLIENT_REQ(0xd);
+                                WREG32(CG_BIF_REQ_AND_RSP, bif);
+
+                                tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                                tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
+                                tmp |= LC_GEN2_EN_STRAP;
+
+                                tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                                udelay(10);
+                                tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                        }
+                }
+        } else {
+                if (!pi->boot_in_gen2) {
+                        tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                        tmp &= ~LC_GEN2_EN_STRAP;
+                }
+                if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
+                    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
+                        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+        }
+}
+
+static void cypress_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                             bool enable)
+{
+        cypress_enable_bif_dynamic_pcie_gen2(rdev, enable);
+
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
+}
+
+#if 0
+static int cypress_enter_ulp_state(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (pi->gfx_clock_gating) {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+
+                RREG32(GB_ADDR_CONFIG);
+        }
+
+        WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
+                 ~HOST_SMC_MSG_MASK);
+
+        udelay(7000);
+
+        return 0;
+}
+#endif
+
+static void cypress_gfx_clock_gating_enable(struct radeon_device *rdev,
+                                            bool enable)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (enable) {
+                if (eg_pi->light_sleep) {
+                        WREG32(GRBM_GFX_INDEX, 0xC0000000);
+
+                        WREG32_CG(CG_CGLS_TILE_0, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_1, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_2, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_3, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_4, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_5, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_6, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_7, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_8, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_9, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_10, 0xFFFFFFFF);
+                        WREG32_CG(CG_CGLS_TILE_11, 0xFFFFFFFF);
+
+                        WREG32_P(SCLK_PWRMGT_CNTL, DYN_LIGHT_SLEEP_EN, ~DYN_LIGHT_SLEEP_EN);
+                }
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+        } else {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+                RREG32(GB_ADDR_CONFIG);
+
+                if (eg_pi->light_sleep) {
+                        WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_LIGHT_SLEEP_EN);
+
+                        WREG32(GRBM_GFX_INDEX, 0xC0000000);
+
+                        WREG32_CG(CG_CGLS_TILE_0, 0);
+                        WREG32_CG(CG_CGLS_TILE_1, 0);
+                        WREG32_CG(CG_CGLS_TILE_2, 0);
+                        WREG32_CG(CG_CGLS_TILE_3, 0);
+                        WREG32_CG(CG_CGLS_TILE_4, 0);
+                        WREG32_CG(CG_CGLS_TILE_5, 0);
+                        WREG32_CG(CG_CGLS_TILE_6, 0);
+                        WREG32_CG(CG_CGLS_TILE_7, 0);
+                        WREG32_CG(CG_CGLS_TILE_8, 0);
+                        WREG32_CG(CG_CGLS_TILE_9, 0);
+                        WREG32_CG(CG_CGLS_TILE_10, 0);
+                        WREG32_CG(CG_CGLS_TILE_11, 0);
+                }
+        }
+}
+
+static void cypress_mg_clock_gating_enable(struct radeon_device *rdev,
+                                           bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (enable) {
+                u32 cgts_sm_ctrl_reg;
+
+                if (rdev->family == CHIP_CEDAR)
+                        cgts_sm_ctrl_reg = CEDAR_MGCGCGTSSMCTRL_DFLT;
+                else if (rdev->family == CHIP_REDWOOD)
+                        cgts_sm_ctrl_reg = REDWOOD_MGCGCGTSSMCTRL_DFLT;
+                else
+                        cgts_sm_ctrl_reg = CYPRESS_MGCGCGTSSMCTRL_DFLT;
+
+                WREG32(GRBM_GFX_INDEX, 0xC0000000);
+
+                WREG32_CG(CG_CGTT_LOCAL_0, CYPRESS_MGCGTTLOCAL0_DFLT);
+                WREG32_CG(CG_CGTT_LOCAL_1, CYPRESS_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF);
+                WREG32_CG(CG_CGTT_LOCAL_2, CYPRESS_MGCGTTLOCAL2_DFLT);
+                WREG32_CG(CG_CGTT_LOCAL_3, CYPRESS_MGCGTTLOCAL3_DFLT);
+
+                if (pi->mgcgtssm)
+                        WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
+
+                if (eg_pi->mcls) {
+                        WREG32_P(MC_CITF_MISC_RD_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(MC_CITF_MISC_WR_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(MC_CITF_MISC_VM_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(MC_HUB_MISC_HUB_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(MC_HUB_MISC_VM_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(MC_HUB_MISC_SIP_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(MC_XPB_CLK_GAT, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                        WREG32_P(VM_L2_CG, MEM_LS_ENABLE, ~MEM_LS_ENABLE);
+                }
+        } else {
+                WREG32(GRBM_GFX_INDEX, 0xC0000000);
+
+                WREG32_CG(CG_CGTT_LOCAL_0, 0xFFFFFFFF);
+                WREG32_CG(CG_CGTT_LOCAL_1, 0xFFFFFFFF);
+                WREG32_CG(CG_CGTT_LOCAL_2, 0xFFFFFFFF);
+                WREG32_CG(CG_CGTT_LOCAL_3, 0xFFFFFFFF);
+
+                if (pi->mgcgtssm)
+                        WREG32(CGTS_SM_CTRL_REG, 0x81f44bc0);
+        }
+}
+
+void cypress_enable_spread_spectrum(struct radeon_device *rdev,
+                                    bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (enable) {
+                if (pi->sclk_ss)
+                        WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
+
+                if (pi->mclk_ss)
+                        WREG32_P(MPLL_CNTL_MODE, SS_SSEN, ~SS_SSEN);
+        } else {
+                WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
+                WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
+                WREG32_P(MPLL_CNTL_MODE, 0, ~SS_SSEN);
+                WREG32_P(MPLL_CNTL_MODE, 0, ~SS_DSMODE_EN);
+        }
+}
+
+void cypress_start_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+}
+
+void cypress_enable_sclk_control(struct radeon_device *rdev,
+                                 bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
+        else
+                WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
+}
+
+void cypress_enable_mclk_control(struct radeon_device *rdev,
+                                 bool enable)
+{
+        if (enable)
+                WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
+        else
+                WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
+}
+
+int cypress_notify_smc_display_change(struct radeon_device *rdev,
+                                      bool has_display)
+{
+        PPSMC_Msg msg = has_display ?
+                (PPSMC_Msg)PPSMC_MSG_HasDisplay : (PPSMC_Msg)PPSMC_MSG_NoDisplay;
+
+        if (rv770_send_msg_to_smc(rdev, msg) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return 0;
+}
+
+void cypress_program_response_times(struct radeon_device *rdev)
+{
+        u32 reference_clock;
+        u32 mclk_switch_limit;
+
+        reference_clock = radeon_get_xclk(rdev);
+        mclk_switch_limit = (460 * reference_clock) / 100;
+
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_mclk_switch_lim,
+                                      mclk_switch_limit);
+
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
+
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
+
+        rv770_program_response_times(rdev);
+
+        if (ASIC_IS_LOMBOK(rdev))
+                rv770_write_smc_soft_register(rdev,
+                                              RV770_SMC_SOFT_REGISTER_is_asic_lombok, 1);
+
+}
+
+static int cypress_pcie_performance_request(struct radeon_device *rdev,
+                                            u8 perf_req, bool advertise)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 tmp;
+
+        udelay(10);
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) && (tmp & LC_CURRENT_DATA_RATE))
+                return 0;
+
+#if defined(CONFIG_ACPI)
+        if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
+            (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
+                eg_pi->pcie_performance_request_registered = true;
+                return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
+        } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
+                   eg_pi->pcie_performance_request_registered) {
+                eg_pi->pcie_performance_request_registered = false;
+                return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
+        }
+#endif
+
+        return 0;
+}
+
+void cypress_advertise_gen2_capability(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp;
+
+#if defined(CONFIG_ACPI)
+        radeon_acpi_pcie_notify_device_ready(rdev);
+#endif
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+
+        if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
+                pi->pcie_gen2 = true;
+        else
+                pi->pcie_gen2 = false;
+
+        if (!pi->pcie_gen2)
+                cypress_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
+
+}
+
+static enum radeon_pcie_gen cypress_get_maximum_link_speed(struct radeon_ps *radeon_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+
+        if (state->high.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
+                return 1;
+        return 0;
+}
+
+void cypress_notify_link_speed_change_after_state_change(struct radeon_device *rdev,
+                                                         struct radeon_ps *radeon_new_state,
+                                                         struct radeon_ps *radeon_current_state)
+{
+        enum radeon_pcie_gen pcie_link_speed_target =
+                cypress_get_maximum_link_speed(radeon_new_state);
+        enum radeon_pcie_gen pcie_link_speed_current =
+                cypress_get_maximum_link_speed(radeon_current_state);
+        u8 request;
+
+        if (pcie_link_speed_target < pcie_link_speed_current) {
+                if (pcie_link_speed_target == RADEON_PCIE_GEN1)
+                        request = PCIE_PERF_REQ_PECI_GEN1;
+                else if (pcie_link_speed_target == RADEON_PCIE_GEN2)
+                        request = PCIE_PERF_REQ_PECI_GEN2;
+                else
+                        request = PCIE_PERF_REQ_PECI_GEN3;
+
+                cypress_pcie_performance_request(rdev, request, false);
+        }
+}
+
+void cypress_notify_link_speed_change_before_state_change(struct radeon_device *rdev,
+                                                          struct radeon_ps *radeon_new_state,
+                                                          struct radeon_ps *radeon_current_state)
+{
+        enum radeon_pcie_gen pcie_link_speed_target =
+                cypress_get_maximum_link_speed(radeon_new_state);
+        enum radeon_pcie_gen pcie_link_speed_current =
+                cypress_get_maximum_link_speed(radeon_current_state);
+        u8 request;
+
+        if (pcie_link_speed_target > pcie_link_speed_current) {
+                if (pcie_link_speed_target == RADEON_PCIE_GEN1)
+                        request = PCIE_PERF_REQ_PECI_GEN1;
+                else if (pcie_link_speed_target == RADEON_PCIE_GEN2)
+                        request = PCIE_PERF_REQ_PECI_GEN2;
+                else
+                        request = PCIE_PERF_REQ_PECI_GEN3;
+
+                cypress_pcie_performance_request(rdev, request, false);
+        }
+}
+
+static int cypress_populate_voltage_value(struct radeon_device *rdev,
+                                          struct atom_voltage_table *table,
+                                          u16 value, RV770_SMC_VOLTAGE_VALUE *voltage)
+{
+        unsigned int i;
+
+        for (i = 0; i < table->count; i++) {
+                if (value <= table->entries[i].value) {
+                        voltage->index = (u8)i;
+                        voltage->value = cpu_to_be16(table->entries[i].value);
+                        break;
+                }
+        }
+
+        if (i == table->count)
+                return -EINVAL;
+
+        return 0;
+}
+
+u8 cypress_get_strobe_mode_settings(struct radeon_device *rdev, u32 mclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 result = 0;
+        bool strobe_mode = false;
+
+        if (pi->mem_gddr5) {
+                if (mclk <= pi->mclk_strobe_mode_threshold)
+                        strobe_mode = true;
+                result = cypress_get_mclk_frequency_ratio(rdev, mclk, strobe_mode);
+
+                if (strobe_mode)
+                        result |= SMC_STROBE_ENABLE;
+        }
+
+        return result;
+}
+
+u32 cypress_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf)
+{
+        u32 ref_clk = rdev->clock.mpll.reference_freq;
+        u32 vco = clkf * ref_clk;
+
+        /* 100 Mhz ref clk */
+        if (ref_clk == 10000) {
+                if (vco > 500000)
+                        return 0xC6;
+                if (vco > 400000)
+                        return 0x9D;
+                if (vco > 330000)
+                        return 0x6C;
+                if (vco > 250000)
+                        return 0x2B;
+                if (vco >  160000)
+                        return 0x5B;
+                if (vco > 120000)
+                        return 0x0A;
+                return 0x4B;
+        }
+
+        /* 27 Mhz ref clk */
+        if (vco > 250000)
+                return 0x8B;
+        if (vco > 200000)
+                return 0xCC;
+        if (vco > 150000)
+                return 0x9B;
+        return 0x6B;
+}
+
+static int cypress_populate_mclk_value(struct radeon_device *rdev,
+                                       u32 engine_clock, u32 memory_clock,
+                                       RV7XX_SMC_MCLK_VALUE *mclk,
+                                       bool strobe_mode, bool dll_state_on)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        u32 mpll_ad_func_cntl =
+                pi->clk_regs.rv770.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl =
+                pi->clk_regs.rv770.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_dq_func_cntl_2;
+        u32 mclk_pwrmgt_cntl =
+                pi->clk_regs.rv770.mclk_pwrmgt_cntl;
+        u32 dll_cntl =
+                pi->clk_regs.rv770.dll_cntl;
+        u32 mpll_ss1 = pi->clk_regs.rv770.mpll_ss1;
+        u32 mpll_ss2 = pi->clk_regs.rv770.mpll_ss2;
+        struct atom_clock_dividers dividers;
+        u32 ibias;
+        u32 dll_speed;
+        int ret;
+        u32 mc_seq_misc7;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                             memory_clock, strobe_mode, &dividers);
+        if (ret)
+                return ret;
+
+        if (!strobe_mode) {
+                mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
+
+                if(mc_seq_misc7 & 0x8000000)
+                        dividers.post_div = 1;
+        }
+
+        ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
+
+        mpll_ad_func_cntl &= ~(CLKR_MASK |
+                               YCLK_POST_DIV_MASK |
+                               CLKF_MASK |
+                               CLKFRAC_MASK |
+                               IBIAS_MASK);
+        mpll_ad_func_cntl |= CLKR(dividers.ref_div);
+        mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
+        mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
+        mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
+        mpll_ad_func_cntl |= IBIAS(ibias);
+
+        if (dividers.vco_mode)
+                mpll_ad_func_cntl_2 |= VCO_MODE;
+        else
+                mpll_ad_func_cntl_2 &= ~VCO_MODE;
+
+        if (pi->mem_gddr5) {
+                mpll_dq_func_cntl &= ~(CLKR_MASK |
+                                       YCLK_POST_DIV_MASK |
+                                       CLKF_MASK |
+                                       CLKFRAC_MASK |
+                                       IBIAS_MASK);
+                mpll_dq_func_cntl |= CLKR(dividers.ref_div);
+                mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
+                mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
+                mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
+                mpll_dq_func_cntl |= IBIAS(ibias);
+
+                if (strobe_mode)
+                        mpll_dq_func_cntl &= ~PDNB;
+                else
+                        mpll_dq_func_cntl |= PDNB;
+
+                if (dividers.vco_mode)
+                        mpll_dq_func_cntl_2 |= VCO_MODE;
+                else
+                        mpll_dq_func_cntl_2 &= ~VCO_MODE;
+        }
+
+        if (pi->mclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = memory_clock * dividers.post_div;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
+                        u32 reference_clock = rdev->clock.mpll.reference_freq;
+                        u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
+                        u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
+                        u32 clk_v = ss.percentage *
+                                (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
+
+                        mpll_ss1 &= ~CLKV_MASK;
+                        mpll_ss1 |= CLKV(clk_v);
+
+                        mpll_ss2 &= ~CLKS_MASK;
+                        mpll_ss2 |= CLKS(clk_s);
+                }
+        }
+
+        dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
+                                        memory_clock);
+
+        mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
+        mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
+        if (dll_state_on)
+                mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
+                                     MRDCKA1_PDNB |
+                                     MRDCKB0_PDNB |
+                                     MRDCKB1_PDNB |
+                                     MRDCKC0_PDNB |
+                                     MRDCKC1_PDNB |
+                                     MRDCKD0_PDNB |
+                                     MRDCKD1_PDNB);
+        else
+                mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
+                                      MRDCKA1_PDNB |
+                                      MRDCKB0_PDNB |
+                                      MRDCKB1_PDNB |
+                                      MRDCKC0_PDNB |
+                                      MRDCKC1_PDNB |
+                                      MRDCKD0_PDNB |
+                                      MRDCKD1_PDNB);
+
+        mclk->mclk770.mclk_value = cpu_to_be32(memory_clock);
+        mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+        mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
+        mclk->mclk770.vMPLL_SS = cpu_to_be32(mpll_ss1);
+        mclk->mclk770.vMPLL_SS2 = cpu_to_be32(mpll_ss2);
+
+        return 0;
+}
+
+u8 cypress_get_mclk_frequency_ratio(struct radeon_device *rdev,
+                                    u32 memory_clock, bool strobe_mode)
+{
+        u8 mc_para_index;
+
+        if (rdev->family >= CHIP_BARTS) {
+                if (strobe_mode) {
+                        if (memory_clock < 10000)
+                                mc_para_index = 0x00;
+                        else if (memory_clock > 47500)
+                                mc_para_index = 0x0f;
+                        else
+                                mc_para_index = (u8)((memory_clock - 10000) / 2500);
+                } else {
+                        if (memory_clock < 65000)
+                                mc_para_index = 0x00;
+                        else if (memory_clock > 135000)
+                                mc_para_index = 0x0f;
+                        else
+                                mc_para_index = (u8)((memory_clock - 60000) / 5000);
+                }
+        } else {
+                if (strobe_mode) {
+                        if (memory_clock < 10000)
+                                mc_para_index = 0x00;
+                        else if (memory_clock > 47500)
+                                mc_para_index = 0x0f;
+                        else
+                                mc_para_index = (u8)((memory_clock - 10000) / 2500);
+                } else {
+                        if (memory_clock < 40000)
+                                mc_para_index = 0x00;
+                        else if (memory_clock > 115000)
+                                mc_para_index = 0x0f;
+                        else
+                                mc_para_index = (u8)((memory_clock - 40000) / 5000);
+                }
+        }
+        return mc_para_index;
+}
+
+static int cypress_populate_mvdd_value(struct radeon_device *rdev,
+                                       u32 mclk,
+                                       RV770_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (!pi->mvdd_control) {
+                voltage->index = eg_pi->mvdd_high_index;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+                return 0;
+        }
+
+        if (mclk <= pi->mvdd_split_frequency) {
+                voltage->index = eg_pi->mvdd_low_index;
+                voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
+        } else {
+                voltage->index = eg_pi->mvdd_high_index;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+        }
+
+        return 0;
+}
+
+int cypress_convert_power_level_to_smc(struct radeon_device *rdev,
+                                       struct rv7xx_pl *pl,
+                                       RV770_SMC_HW_PERFORMANCE_LEVEL *level,
+                                       u8 watermark_level)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        int ret;
+        bool dll_state_on;
+
+        level->gen2PCIE = pi->pcie_gen2 ?
+                ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
+        level->gen2XSP  = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0;
+        level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0;
+        level->displayWatermark = watermark_level;
+
+        ret = rv740_populate_sclk_value(rdev, pl->sclk, &level->sclk);
+        if (ret)
+                return ret;
+
+        level->mcFlags =  0;
+        if (pi->mclk_stutter_mode_threshold &&
+            (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
+            !eg_pi->uvd_enabled) {
+                level->mcFlags |= SMC_MC_STUTTER_EN;
+                if (eg_pi->sclk_deep_sleep)
+                        level->stateFlags |= PPSMC_STATEFLAG_AUTO_PULSE_SKIP;
+                else
+                        level->stateFlags &= ~PPSMC_STATEFLAG_AUTO_PULSE_SKIP;
+        }
+
+        if (pi->mem_gddr5) {
+                if (pl->mclk > pi->mclk_edc_enable_threshold)
+                        level->mcFlags |= SMC_MC_EDC_RD_FLAG;
+
+                if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
+                        level->mcFlags |= SMC_MC_EDC_WR_FLAG;
+
+                level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
+
+                if (level->strobeMode & SMC_STROBE_ENABLE) {
+                        if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
+                            ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
+                                dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
+                        else
+                                dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
+                } else
+                        dll_state_on = eg_pi->dll_default_on;
+
+                ret = cypress_populate_mclk_value(rdev,
+                                                  pl->sclk,
+                                                  pl->mclk,
+                                                  &level->mclk,
+                                                  (level->strobeMode & SMC_STROBE_ENABLE) != 0,
+                                                  dll_state_on);
+        } else {
+                ret = cypress_populate_mclk_value(rdev,
+                                                  pl->sclk,
+                                                  pl->mclk,
+                                                  &level->mclk,
+                                                  true,
+                                                  true);
+        }
+        if (ret)
+                return ret;
+
+        ret = cypress_populate_voltage_value(rdev,
+                                             &eg_pi->vddc_voltage_table,
+                                             pl->vddc,
+                                             &level->vddc);
+        if (ret)
+                return ret;
+
+        if (eg_pi->vddci_control) {
+                ret = cypress_populate_voltage_value(rdev,
+                                                     &eg_pi->vddci_voltage_table,
+                                                     pl->vddci,
+                                                     &level->vddci);
+                if (ret)
+                        return ret;
+        }
+
+        ret = cypress_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
+
+        return ret;
+}
+
+static int cypress_convert_power_state_to_smc(struct radeon_device *rdev,
+                                              struct radeon_ps *radeon_state,
+                                              RV770_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        int ret;
+
+        if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
+                smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        ret = cypress_convert_power_level_to_smc(rdev,
+                                                 &state->low,
+                                                 &smc_state->levels[0],
+                                                 PPSMC_DISPLAY_WATERMARK_LOW);
+        if (ret)
+                return ret;
+
+        ret = cypress_convert_power_level_to_smc(rdev,
+                                                 &state->medium,
+                                                 &smc_state->levels[1],
+                                                 PPSMC_DISPLAY_WATERMARK_LOW);
+        if (ret)
+                return ret;
+
+        ret = cypress_convert_power_level_to_smc(rdev,
+                                                 &state->high,
+                                                 &smc_state->levels[2],
+                                                 PPSMC_DISPLAY_WATERMARK_HIGH);
+        if (ret)
+                return ret;
+
+        smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1;
+        smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2;
+        smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3;
+
+        if (eg_pi->dynamic_ac_timing) {
+                smc_state->levels[0].ACIndex = 2;
+                smc_state->levels[1].ACIndex = 3;
+                smc_state->levels[2].ACIndex = 4;
+        } else {
+                smc_state->levels[0].ACIndex = 0;
+                smc_state->levels[1].ACIndex = 0;
+                smc_state->levels[2].ACIndex = 0;
+        }
+
+        rv770_populate_smc_sp(rdev, radeon_state, smc_state);
+
+        return rv770_populate_smc_t(rdev, radeon_state, smc_state);
+}
+
+static void cypress_convert_mc_registers(struct evergreen_mc_reg_entry *entry,
+                                         SMC_Evergreen_MCRegisterSet *data,
+                                         u32 num_entries, u32 valid_flag)
+{
+        u32 i, j;
+
+        for (i = 0, j = 0; j < num_entries; j++) {
+                if (valid_flag & (1 << j)) {
+                        data->value[i] = cpu_to_be32(entry->mc_data[j]);
+                        i++;
+                }
+        }
+}
+
+static void cypress_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
+                                                      struct rv7xx_pl *pl,
+                                                      SMC_Evergreen_MCRegisterSet *mc_reg_table_data)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 i = 0;
+
+        for (i = 0; i < eg_pi->mc_reg_table.num_entries; i++) {
+                if (pl->mclk <=
+                    eg_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
+                        break;
+        }
+
+        if ((i == eg_pi->mc_reg_table.num_entries) && (i > 0))
+                --i;
+
+        cypress_convert_mc_registers(&eg_pi->mc_reg_table.mc_reg_table_entry[i],
+                                     mc_reg_table_data,
+                                     eg_pi->mc_reg_table.last,
+                                     eg_pi->mc_reg_table.valid_flag);
+}
+
+static void cypress_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
+                                                struct radeon_ps *radeon_state,
+                                                SMC_Evergreen_MCRegisters *mc_reg_table)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+
+        cypress_convert_mc_reg_table_entry_to_smc(rdev,
+                                                  &state->low,
+                                                  &mc_reg_table->data[2]);
+        cypress_convert_mc_reg_table_entry_to_smc(rdev,
+                                                  &state->medium,
+                                                  &mc_reg_table->data[3]);
+        cypress_convert_mc_reg_table_entry_to_smc(rdev,
+                                                  &state->high,
+                                                  &mc_reg_table->data[4]);
+}
+
+int cypress_upload_sw_state(struct radeon_device *rdev,
+                            struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u16 address = pi->state_table_start +
+                offsetof(RV770_SMC_STATETABLE, driverState);
+        RV770_SMC_SWSTATE state = { 0 };
+        int ret;
+
+        ret = cypress_convert_power_state_to_smc(rdev, radeon_new_state, &state);
+        if (ret)
+                return ret;
+
+        return rv770_copy_bytes_to_smc(rdev, address, (u8 *)&state,
+                                    sizeof(RV770_SMC_SWSTATE),
+                                    pi->sram_end);
+}
+
+int cypress_upload_mc_reg_table(struct radeon_device *rdev,
+                                struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        SMC_Evergreen_MCRegisters mc_reg_table = { 0 };
+        u16 address;
+
+        cypress_convert_mc_reg_table_to_smc(rdev, radeon_new_state, &mc_reg_table);
+
+        address = eg_pi->mc_reg_table_start +
+                (u16)offsetof(SMC_Evergreen_MCRegisters, data[2]);
+
+        return rv770_copy_bytes_to_smc(rdev, address,
+                                       (u8 *)&mc_reg_table.data[2],
+                                       sizeof(SMC_Evergreen_MCRegisterSet) * 3,
+                                       pi->sram_end);
+}
+
+u32 cypress_calculate_burst_time(struct radeon_device *rdev,
+                                 u32 engine_clock, u32 memory_clock)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 multiplier = pi->mem_gddr5 ? 1 : 2;
+        u32 result = (4 * multiplier * engine_clock) / (memory_clock / 2);
+        u32 burst_time;
+
+        if (result <= 4)
+                burst_time = 0;
+        else if (result < 8)
+                burst_time = result - 4;
+        else {
+                burst_time = result / 2 ;
+                if (burst_time > 18)
+                        burst_time = 18;
+        }
+
+        return burst_time;
+}
+
+void cypress_program_memory_timing_parameters(struct radeon_device *rdev,
+                                              struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
+        u32 mc_arb_burst_time = RREG32(MC_ARB_BURST_TIME);
+
+        mc_arb_burst_time &= ~(STATE1_MASK | STATE2_MASK | STATE3_MASK);
+
+        mc_arb_burst_time |= STATE1(cypress_calculate_burst_time(rdev,
+                                                                 new_state->low.sclk,
+                                                                 new_state->low.mclk));
+        mc_arb_burst_time |= STATE2(cypress_calculate_burst_time(rdev,
+                                                                 new_state->medium.sclk,
+                                                                 new_state->medium.mclk));
+        mc_arb_burst_time |= STATE3(cypress_calculate_burst_time(rdev,
+                                                                 new_state->high.sclk,
+                                                                 new_state->high.mclk));
+
+        rv730_program_memory_timing_parameters(rdev, radeon_new_state);
+
+        WREG32(MC_ARB_BURST_TIME, mc_arb_burst_time);
+}
+
+static void cypress_populate_mc_reg_addresses(struct radeon_device *rdev,
+                                              SMC_Evergreen_MCRegisters *mc_reg_table)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 i, j;
+
+        for (i = 0, j = 0; j < eg_pi->mc_reg_table.last; j++) {
+                if (eg_pi->mc_reg_table.valid_flag & (1 << j)) {
+                        mc_reg_table->address[i].s0 =
+                                cpu_to_be16(eg_pi->mc_reg_table.mc_reg_address[j].s0);
+                        mc_reg_table->address[i].s1 =
+                                cpu_to_be16(eg_pi->mc_reg_table.mc_reg_address[j].s1);
+                        i++;
+                }
+        }
+
+        mc_reg_table->last = (u8)i;
+}
+
+static void cypress_set_mc_reg_address_table(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 i = 0;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RAS_TIMING_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RAS_TIMING >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_CAS_TIMING_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_CAS_TIMING >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC_TIMING_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC_TIMING >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC_TIMING2_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC_TIMING2 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RD_CTL_D0_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RD_CTL_D0 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RD_CTL_D1_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RD_CTL_D1 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_WR_CTL_D0_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_WR_CTL_D0 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_WR_CTL_D1_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_WR_CTL_D1 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_PMG_CMD_EMRS >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_PMG_CMD_MRS >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_PMG_CMD_MRS1 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC1 >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC1 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_RESERVE_M >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_RESERVE_M >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.mc_reg_address[i].s0 = MC_SEQ_MISC3 >> 2;
+        eg_pi->mc_reg_table.mc_reg_address[i].s1 = MC_SEQ_MISC3 >> 2;
+        i++;
+
+        eg_pi->mc_reg_table.last = (u8)i;
+}
+
+static void cypress_retrieve_ac_timing_for_one_entry(struct radeon_device *rdev,
+                                                     struct evergreen_mc_reg_entry *entry)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 i;
+
+        for (i = 0; i < eg_pi->mc_reg_table.last; i++)
+                entry->mc_data[i] =
+                        RREG32(eg_pi->mc_reg_table.mc_reg_address[i].s1 << 2);
+
+}
+
+static void cypress_retrieve_ac_timing_for_all_ranges(struct radeon_device *rdev,
+                                                      struct atom_memory_clock_range_table *range_table)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 i, j;
+
+        for (i = 0; i < range_table->num_entries; i++) {
+                eg_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max =
+                        range_table->mclk[i];
+                radeon_atom_set_ac_timing(rdev, range_table->mclk[i]);
+                cypress_retrieve_ac_timing_for_one_entry(rdev,
+                                                         &eg_pi->mc_reg_table.mc_reg_table_entry[i]);
+        }
+
+        eg_pi->mc_reg_table.num_entries = range_table->num_entries;
+        eg_pi->mc_reg_table.valid_flag = 0;
+
+        for (i = 0; i < eg_pi->mc_reg_table.last; i++) {
+                for (j = 1; j < range_table->num_entries; j++) {
+                        if (eg_pi->mc_reg_table.mc_reg_table_entry[j-1].mc_data[i] !=
+                            eg_pi->mc_reg_table.mc_reg_table_entry[j].mc_data[i]) {
+                                eg_pi->mc_reg_table.valid_flag |= (1 << i);
+                                break;
+                        }
+                }
+        }
+}
+
+static int cypress_initialize_mc_reg_table(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 module_index = rv770_get_memory_module_index(rdev);
+        struct atom_memory_clock_range_table range_table = { 0 };
+        int ret;
+
+        ret = radeon_atom_get_mclk_range_table(rdev,
+                                               pi->mem_gddr5,
+                                               module_index, &range_table);
+        if (ret)
+                return ret;
+
+        cypress_retrieve_ac_timing_for_all_ranges(rdev, &range_table);
+
+        return 0;
+}
+
+static void cypress_wait_for_mc_sequencer(struct radeon_device *rdev, u8 value)
+{
+        u32 i, j;
+        u32 channels = 2;
+
+        if ((rdev->family == CHIP_CYPRESS) ||
+            (rdev->family == CHIP_HEMLOCK))
+                channels = 4;
+        else if (rdev->family == CHIP_CEDAR)
+                channels = 1;
+
+        for (i = 0; i < channels; i++) {
+                if ((rdev->family == CHIP_CYPRESS) ||
+                    (rdev->family == CHIP_HEMLOCK)) {
+                        WREG32_P(MC_CONFIG_MCD, MC_RD_ENABLE_MCD(i), ~MC_RD_ENABLE_MCD_MASK);
+                        WREG32_P(MC_CG_CONFIG_MCD, MC_RD_ENABLE_MCD(i), ~MC_RD_ENABLE_MCD_MASK);
+                } else {
+                        WREG32_P(MC_CONFIG, MC_RD_ENABLE(i), ~MC_RD_ENABLE_MASK);
+                        WREG32_P(MC_CG_CONFIG, MC_RD_ENABLE(i), ~MC_RD_ENABLE_MASK);
+                }
+                for (j = 0; j < rdev->usec_timeout; j++) {
+                        if (((RREG32(MC_SEQ_CG) & CG_SEQ_RESP_MASK) >> CG_SEQ_RESP_SHIFT) == value)
+                                break;
+                        udelay(1);
+                }
+        }
+}
+
+static void cypress_force_mc_use_s1(struct radeon_device *rdev,
+                                    struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
+        u32 strobe_mode;
+        u32 mc_seq_cg;
+        int i;
+
+        if (RREG32(MC_SEQ_STATUS_M) & PMG_PWRSTATE)
+                return;
+
+        radeon_atom_set_ac_timing(rdev, boot_state->low.mclk);
+        radeon_mc_wait_for_idle(rdev);
+
+        if ((rdev->family == CHIP_CYPRESS) ||
+            (rdev->family == CHIP_HEMLOCK)) {
+                WREG32(MC_CONFIG_MCD, 0xf);
+                WREG32(MC_CG_CONFIG_MCD, 0xf);
+        } else {
+                WREG32(MC_CONFIG, 0xf);
+                WREG32(MC_CG_CONFIG, 0xf);
+        }
+
+        for (i = 0; i < rdev->num_crtc; i++)
+                radeon_wait_for_vblank(rdev, i);
+
+        WREG32(MC_SEQ_CG, MC_CG_SEQ_YCLK_SUSPEND);
+        cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_SUSPEND);
+
+        strobe_mode = cypress_get_strobe_mode_settings(rdev,
+                                                       boot_state->low.mclk);
+
+        mc_seq_cg = CG_SEQ_REQ(MC_CG_SEQ_DRAMCONF_S1);
+        mc_seq_cg |= SEQ_CG_RESP(strobe_mode);
+        WREG32(MC_SEQ_CG, mc_seq_cg);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(MC_SEQ_STATUS_M) & PMG_PWRSTATE)
+                        break;
+                udelay(1);
+        }
+
+        mc_seq_cg &= ~CG_SEQ_REQ_MASK;
+        mc_seq_cg |= CG_SEQ_REQ(MC_CG_SEQ_YCLK_RESUME);
+        WREG32(MC_SEQ_CG, mc_seq_cg);
+
+        cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_RESUME);
+}
+
+static void cypress_copy_ac_timing_from_s1_to_s0(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 value;
+        u32 i;
+
+        for (i = 0; i < eg_pi->mc_reg_table.last; i++) {
+                value = RREG32(eg_pi->mc_reg_table.mc_reg_address[i].s1 << 2);
+                WREG32(eg_pi->mc_reg_table.mc_reg_address[i].s0 << 2, value);
+        }
+}
+
+static void cypress_force_mc_use_s0(struct radeon_device *rdev,
+                                    struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
+        u32 strobe_mode;
+        u32 mc_seq_cg;
+        int i;
+
+        cypress_copy_ac_timing_from_s1_to_s0(rdev);
+        radeon_mc_wait_for_idle(rdev);
+
+        if ((rdev->family == CHIP_CYPRESS) ||
+            (rdev->family == CHIP_HEMLOCK)) {
+                WREG32(MC_CONFIG_MCD, 0xf);
+                WREG32(MC_CG_CONFIG_MCD, 0xf);
+        } else {
+                WREG32(MC_CONFIG, 0xf);
+                WREG32(MC_CG_CONFIG, 0xf);
+        }
+
+        for (i = 0; i < rdev->num_crtc; i++)
+                radeon_wait_for_vblank(rdev, i);
+
+        WREG32(MC_SEQ_CG, MC_CG_SEQ_YCLK_SUSPEND);
+        cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_SUSPEND);
+
+        strobe_mode = cypress_get_strobe_mode_settings(rdev,
+                                                       boot_state->low.mclk);
+
+        mc_seq_cg = CG_SEQ_REQ(MC_CG_SEQ_DRAMCONF_S0);
+        mc_seq_cg |= SEQ_CG_RESP(strobe_mode);
+        WREG32(MC_SEQ_CG, mc_seq_cg);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (!(RREG32(MC_SEQ_STATUS_M) & PMG_PWRSTATE))
+                        break;
+                udelay(1);
+        }
+
+        mc_seq_cg &= ~CG_SEQ_REQ_MASK;
+        mc_seq_cg |= CG_SEQ_REQ(MC_CG_SEQ_YCLK_RESUME);
+        WREG32(MC_SEQ_CG, mc_seq_cg);
+
+        cypress_wait_for_mc_sequencer(rdev, MC_CG_SEQ_YCLK_RESUME);
+}
+
+static int cypress_populate_initial_mvdd_value(struct radeon_device *rdev,
+                                               RV770_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        voltage->index = eg_pi->mvdd_high_index;
+        voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+
+        return 0;
+}
+
+int cypress_populate_smc_initial_state(struct radeon_device *rdev,
+                                       struct radeon_ps *radeon_initial_state,
+                                       RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_ps *initial_state = rv770_get_ps(radeon_initial_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 a_t;
+
+        table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2);
+        table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl);
+        table->initialState.levels[0].mclk.mclk770.vDLL_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.dll_cntl);
+
+        table->initialState.levels[0].mclk.mclk770.vMPLL_SS =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ss1);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ss2);
+
+        table->initialState.levels[0].mclk.mclk770.mclk_value =
+                cpu_to_be32(initial_state->low.mclk);
+
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2);
+
+        table->initialState.levels[0].sclk.sclk_value =
+                cpu_to_be32(initial_state->low.sclk);
+
+        table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
+
+        table->initialState.levels[0].ACIndex = 0;
+
+        cypress_populate_voltage_value(rdev,
+                                       &eg_pi->vddc_voltage_table,
+                                       initial_state->low.vddc,
+                                       &table->initialState.levels[0].vddc);
+
+        if (eg_pi->vddci_control)
+                cypress_populate_voltage_value(rdev,
+                                               &eg_pi->vddci_voltage_table,
+                                               initial_state->low.vddci,
+                                               &table->initialState.levels[0].vddci);
+
+        cypress_populate_initial_mvdd_value(rdev,
+                                            &table->initialState.levels[0].mvdd);
+
+        a_t = CG_R(0xffff) | CG_L(0);
+        table->initialState.levels[0].aT = cpu_to_be32(a_t);
+
+        table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
+
+
+        if (pi->boot_in_gen2)
+                table->initialState.levels[0].gen2PCIE = 1;
+        else
+                table->initialState.levels[0].gen2PCIE = 0;
+        if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
+                table->initialState.levels[0].gen2XSP = 1;
+        else
+                table->initialState.levels[0].gen2XSP = 0;
+
+        if (pi->mem_gddr5) {
+                table->initialState.levels[0].strobeMode =
+                        cypress_get_strobe_mode_settings(rdev,
+                                                         initial_state->low.mclk);
+
+                if (initial_state->low.mclk > pi->mclk_edc_enable_threshold)
+                        table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
+                else
+                        table->initialState.levels[0].mcFlags =  0;
+        }
+
+        table->initialState.levels[1] = table->initialState.levels[0];
+        table->initialState.levels[2] = table->initialState.levels[0];
+
+        table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        return 0;
+}
+
+int cypress_populate_smc_acpi_state(struct radeon_device *rdev,
+                                    RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 mpll_ad_func_cntl =
+                pi->clk_regs.rv770.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl =
+                pi->clk_regs.rv770.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_dq_func_cntl_2;
+        u32 spll_func_cntl =
+                pi->clk_regs.rv770.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 =
+                pi->clk_regs.rv770.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 =
+                pi->clk_regs.rv770.cg_spll_func_cntl_3;
+        u32 mclk_pwrmgt_cntl =
+                pi->clk_regs.rv770.mclk_pwrmgt_cntl;
+        u32 dll_cntl =
+                pi->clk_regs.rv770.dll_cntl;
+
+        table->ACPIState = table->initialState;
+
+        table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+        if (pi->acpi_vddc) {
+                cypress_populate_voltage_value(rdev,
+                                               &eg_pi->vddc_voltage_table,
+                                               pi->acpi_vddc,
+                                               &table->ACPIState.levels[0].vddc);
+                if (pi->pcie_gen2) {
+                        if (pi->acpi_pcie_gen2)
+                                table->ACPIState.levels[0].gen2PCIE = 1;
+                        else
+                                table->ACPIState.levels[0].gen2PCIE = 0;
+                } else
+                        table->ACPIState.levels[0].gen2PCIE = 0;
+                if (pi->acpi_pcie_gen2)
+                        table->ACPIState.levels[0].gen2XSP = 1;
+                else
+                        table->ACPIState.levels[0].gen2XSP = 0;
+        } else {
+                cypress_populate_voltage_value(rdev,
+                                               &eg_pi->vddc_voltage_table,
+                                               pi->min_vddc_in_table,
+                                               &table->ACPIState.levels[0].vddc);
+                table->ACPIState.levels[0].gen2PCIE = 0;
+        }
+
+        if (eg_pi->acpi_vddci) {
+                if (eg_pi->vddci_control) {
+                        cypress_populate_voltage_value(rdev,
+                                                       &eg_pi->vddci_voltage_table,
+                                                       eg_pi->acpi_vddci,
+                                                       &table->ACPIState.levels[0].vddci);
+                }
+        }
+
+        mpll_ad_func_cntl &= ~PDNB;
+
+        mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
+
+        if (pi->mem_gddr5)
+                mpll_dq_func_cntl &= ~PDNB;
+        mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
+
+        mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
+                             MRDCKA1_RESET |
+                             MRDCKB0_RESET |
+                             MRDCKB1_RESET |
+                             MRDCKC0_RESET |
+                             MRDCKC1_RESET |
+                             MRDCKD0_RESET |
+                             MRDCKD1_RESET);
+
+        mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
+                              MRDCKA1_PDNB |
+                              MRDCKB0_PDNB |
+                              MRDCKB1_PDNB |
+                              MRDCKC0_PDNB |
+                              MRDCKC1_PDNB |
+                              MRDCKD0_PDNB |
+                              MRDCKD1_PDNB);
+
+        dll_cntl |= (MRDCKA0_BYPASS |
+                     MRDCKA1_BYPASS |
+                     MRDCKB0_BYPASS |
+                     MRDCKB1_BYPASS |
+                     MRDCKC0_BYPASS |
+                     MRDCKC1_BYPASS |
+                     MRDCKD0_BYPASS |
+                     MRDCKD1_BYPASS);
+
+        /* evergreen only */
+        if (rdev->family <= CHIP_HEMLOCK)
+                spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN;
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL =
+                cpu_to_be32(mpll_ad_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 =
+                cpu_to_be32(mpll_ad_func_cntl_2);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL =
+                cpu_to_be32(mpll_dq_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 =
+                cpu_to_be32(mpll_dq_func_cntl_2);
+        table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(mclk_pwrmgt_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
+
+        table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0;
+
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(spll_func_cntl);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(spll_func_cntl_2);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(spll_func_cntl_3);
+
+        table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+        cypress_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
+
+        if (eg_pi->dynamic_ac_timing)
+                table->ACPIState.levels[0].ACIndex = 1;
+
+        table->ACPIState.levels[1] = table->ACPIState.levels[0];
+        table->ACPIState.levels[2] = table->ACPIState.levels[0];
+
+        return 0;
+}
+
+static void cypress_trim_voltage_table_to_fit_state_table(struct radeon_device *rdev,
+                                                          struct atom_voltage_table *voltage_table)
+{
+        unsigned int i, diff;
+
+        if (voltage_table->count <= MAX_NO_VREG_STEPS)
+                return;
+
+        diff = voltage_table->count - MAX_NO_VREG_STEPS;
+
+        for (i= 0; i < MAX_NO_VREG_STEPS; i++)
+                voltage_table->entries[i] = voltage_table->entries[i + diff];
+
+        voltage_table->count = MAX_NO_VREG_STEPS;
+}
+
+int cypress_construct_voltage_tables(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        int ret;
+
+        ret = radeon_atom_get_voltage_table(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0,
+                                            &eg_pi->vddc_voltage_table);
+        if (ret)
+                return ret;
+
+        if (eg_pi->vddc_voltage_table.count > MAX_NO_VREG_STEPS)
+                cypress_trim_voltage_table_to_fit_state_table(rdev,
+                                                              &eg_pi->vddc_voltage_table);
+
+        if (eg_pi->vddci_control) {
+                ret = radeon_atom_get_voltage_table(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0,
+                                                    &eg_pi->vddci_voltage_table);
+                if (ret)
+                        return ret;
+
+                if (eg_pi->vddci_voltage_table.count > MAX_NO_VREG_STEPS)
+                        cypress_trim_voltage_table_to_fit_state_table(rdev,
+                                                                      &eg_pi->vddci_voltage_table);
+        }
+
+        return 0;
+}
+
+static void cypress_populate_smc_voltage_table(struct radeon_device *rdev,
+                                               struct atom_voltage_table *voltage_table,
+                                               RV770_SMC_STATETABLE *table)
+{
+        unsigned int i;
+
+        for (i = 0; i < voltage_table->count; i++) {
+                table->highSMIO[i] = 0;
+                table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
+        }
+}
+
+int cypress_populate_smc_voltage_tables(struct radeon_device *rdev,
+                                        RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        unsigned char i;
+
+        if (eg_pi->vddc_voltage_table.count) {
+                cypress_populate_smc_voltage_table(rdev,
+                                                   &eg_pi->vddc_voltage_table,
+                                                   table);
+
+                table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0;
+                table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] =
+                        cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
+
+                for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
+                        if (pi->max_vddc_in_table <=
+                            eg_pi->vddc_voltage_table.entries[i].value) {
+                                table->maxVDDCIndexInPPTable = i;
+                                break;
+                        }
+                }
+        }
+
+        if (eg_pi->vddci_voltage_table.count) {
+                cypress_populate_smc_voltage_table(rdev,
+                                                   &eg_pi->vddci_voltage_table,
+                                                   table);
+
+                table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDCI] = 0;
+                table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDCI] =
+                        cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
+        }
+
+        return 0;
+}
+
+static u32 cypress_get_mclk_split_point(struct atom_memory_info *memory_info)
+{
+        if ((memory_info->mem_type == MEM_TYPE_GDDR3) ||
+            (memory_info->mem_type == MEM_TYPE_DDR3))
+                return 30000;
+
+        return 0;
+}
+
+int cypress_get_mvdd_configuration(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u8 module_index;
+        struct atom_memory_info memory_info;
+        u32 tmp = RREG32(GENERAL_PWRMGT);
+
+        if (!(tmp & BACKBIAS_PAD_EN)) {
+                eg_pi->mvdd_high_index = 0;
+                eg_pi->mvdd_low_index = 1;
+                pi->mvdd_control = false;
+                return 0;
+        }
+
+        if (tmp & BACKBIAS_VALUE)
+                eg_pi->mvdd_high_index = 1;
+        else
+                eg_pi->mvdd_high_index = 0;
+
+        eg_pi->mvdd_low_index =
+                (eg_pi->mvdd_high_index == 0) ? 1 : 0;
+
+        module_index = rv770_get_memory_module_index(rdev);
+
+        if (radeon_atom_get_memory_info(rdev, module_index, &memory_info)) {
+                pi->mvdd_control = false;
+                return 0;
+        }
+
+        pi->mvdd_split_frequency =
+                cypress_get_mclk_split_point(&memory_info);
+
+        if (pi->mvdd_split_frequency == 0) {
+                pi->mvdd_control = false;
+                return 0;
+        }
+
+        return 0;
+}
+
+static int cypress_init_smc_table(struct radeon_device *rdev,
+                                  struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        RV770_SMC_STATETABLE *table = &pi->smc_statetable;
+        int ret;
+
+        memset(table, 0, sizeof(RV770_SMC_STATETABLE));
+
+        cypress_populate_smc_voltage_tables(rdev, table);
+
+        switch (rdev->pm.int_thermal_type) {
+        case THERMAL_TYPE_EVERGREEN:
+        case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
+                break;
+        case THERMAL_TYPE_NONE:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
+                break;
+        default:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
+                break;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+        if (pi->mem_gddr5)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+        ret = cypress_populate_smc_initial_state(rdev, radeon_boot_state, table);
+        if (ret)
+                return ret;
+
+        ret = cypress_populate_smc_acpi_state(rdev, table);
+        if (ret)
+                return ret;
+
+        table->driverState = table->initialState;
+
+        return rv770_copy_bytes_to_smc(rdev,
+                                       pi->state_table_start,
+                                       (u8 *)table, sizeof(RV770_SMC_STATETABLE),
+                                       pi->sram_end);
+}
+
+int cypress_populate_mc_reg_table(struct radeon_device *rdev,
+                                  struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
+        SMC_Evergreen_MCRegisters mc_reg_table = { 0 };
+
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_seq_index, 1);
+
+        cypress_populate_mc_reg_addresses(rdev, &mc_reg_table);
+
+        cypress_convert_mc_reg_table_entry_to_smc(rdev,
+                                                  &boot_state->low,
+                                                  &mc_reg_table.data[0]);
+
+        cypress_convert_mc_registers(&eg_pi->mc_reg_table.mc_reg_table_entry[0],
+                                     &mc_reg_table.data[1], eg_pi->mc_reg_table.last,
+                                     eg_pi->mc_reg_table.valid_flag);
+
+        cypress_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, &mc_reg_table);
+
+        return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
+                                       (u8 *)&mc_reg_table, sizeof(SMC_Evergreen_MCRegisters),
+                                       pi->sram_end);
+}
+
+int cypress_get_table_locations(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION +
+                                        EVERGREEN_SMC_FIRMWARE_HEADER_stateTable,
+                                        &tmp, pi->sram_end);
+        if (ret)
+                return ret;
+
+        pi->state_table_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION +
+                                        EVERGREEN_SMC_FIRMWARE_HEADER_softRegisters,
+                                        &tmp, pi->sram_end);
+        if (ret)
+                return ret;
+
+        pi->soft_regs_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION +
+                                        EVERGREEN_SMC_FIRMWARE_HEADER_mcRegisterTable,
+                                        &tmp, pi->sram_end);
+        if (ret)
+                return ret;
+
+        eg_pi->mc_reg_table_start = (u16)tmp;
+
+        return 0;
+}
+
+void cypress_enable_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
+
+        tmp &= ~(DISP1_GAP_MASK | DISP2_GAP_MASK);
+        tmp |= (DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE) |
+                DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE));
+
+        tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
+        tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK) |
+                DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+}
+
+static void cypress_program_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp, pipe;
+        int i;
+
+        tmp = RREG32(CG_DISPLAY_GAP_CNTL) & ~(DISP1_GAP_MASK | DISP2_GAP_MASK);
+        if (rdev->pm.dpm.new_active_crtc_count > 0)
+                tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
+        else
+                tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE);
+
+        if (rdev->pm.dpm.new_active_crtc_count > 1)
+                tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
+        else
+                tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE);
+
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+
+        tmp = RREG32(DCCG_DISP_SLOW_SELECT_REG);
+        pipe = (tmp & DCCG_DISP1_SLOW_SELECT_MASK) >> DCCG_DISP1_SLOW_SELECT_SHIFT;
+
+        if ((rdev->pm.dpm.new_active_crtc_count > 0) &&
+            (!(rdev->pm.dpm.new_active_crtcs & (1 << pipe)))) {
+                /* find the first active crtc */
+                for (i = 0; i < rdev->num_crtc; i++) {
+                        if (rdev->pm.dpm.new_active_crtcs & (1 << i))
+                                break;
+                }
+                if (i == rdev->num_crtc)
+                        pipe = 0;
+                else
+                        pipe = i;
+
+                tmp &= ~DCCG_DISP1_SLOW_SELECT_MASK;
+                tmp |= DCCG_DISP1_SLOW_SELECT(pipe);
+                WREG32(DCCG_DISP_SLOW_SELECT_REG, tmp);
+        }
+
+        cypress_notify_smc_display_change(rdev, rdev->pm.dpm.new_active_crtc_count > 0);
+}
+
+void cypress_dpm_setup_asic(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        rv740_read_clock_registers(rdev);
+        rv770_read_voltage_smio_registers(rdev);
+        rv770_get_max_vddc(rdev);
+        rv770_get_memory_type(rdev);
+
+        if (eg_pi->pcie_performance_request)
+                eg_pi->pcie_performance_request_registered = false;
+
+        if (eg_pi->pcie_performance_request)
+                cypress_advertise_gen2_capability(rdev);
+
+        rv770_get_pcie_gen2_status(rdev);
+
+        rv770_enable_acpi_pm(rdev);
+}
+
+int cypress_dpm_enable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        if (pi->gfx_clock_gating)
+                rv770_restore_cgcg(rdev);
+
+        if (rv770_dpm_enabled(rdev))
+                return -EINVAL;
+
+        if (pi->voltage_control) {
+                rv770_enable_voltage_control(rdev, true);
+                ret = cypress_construct_voltage_tables(rdev);
+                if (ret) {
+                        DRM_ERROR("cypress_construct_voltage_tables failed\n");
+                        return ret;
+                }
+        }
+
+        if (pi->mvdd_control) {
+                ret = cypress_get_mvdd_configuration(rdev);
+                if (ret) {
+                        DRM_ERROR("cypress_get_mvdd_configuration failed\n");
+                        return ret;
+                }
+        }
+
+        if (eg_pi->dynamic_ac_timing) {
+                cypress_set_mc_reg_address_table(rdev);
+                cypress_force_mc_use_s0(rdev, boot_ps);
+                ret = cypress_initialize_mc_reg_table(rdev);
+                if (ret)
+                        eg_pi->dynamic_ac_timing = false;
+                cypress_force_mc_use_s1(rdev, boot_ps);
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv770_enable_backbias(rdev, true);
+
+        if (pi->dynamic_ss)
+                cypress_enable_spread_spectrum(rdev, true);
+
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, true);
+
+        rv770_setup_bsp(rdev);
+        rv770_program_git(rdev);
+        rv770_program_tp(rdev);
+        rv770_program_tpp(rdev);
+        rv770_program_sstp(rdev);
+        rv770_program_engine_speed_parameters(rdev);
+        cypress_enable_display_gap(rdev);
+        rv770_program_vc(rdev);
+
+        if (pi->dynamic_pcie_gen2)
+                cypress_enable_dynamic_pcie_gen2(rdev, true);
+
+        ret = rv770_upload_firmware(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_upload_firmware failed\n");
+                return ret;
+        }
+
+        ret = cypress_get_table_locations(rdev);
+        if (ret) {
+                DRM_ERROR("cypress_get_table_locations failed\n");
+                return ret;
+        }
+        ret = cypress_init_smc_table(rdev, boot_ps);
+        if (ret) {
+                DRM_ERROR("cypress_init_smc_table failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = cypress_populate_mc_reg_table(rdev, boot_ps);
+                if (ret) {
+                        DRM_ERROR("cypress_populate_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+
+        cypress_program_response_times(rdev);
+
+        r7xx_start_smc(rdev);
+
+        ret = cypress_notify_smc_display_change(rdev, false);
+        if (ret) {
+                DRM_ERROR("cypress_notify_smc_display_change failed\n");
+                return ret;
+        }
+        cypress_enable_sclk_control(rdev, true);
+
+        if (eg_pi->memory_transition)
+                cypress_enable_mclk_control(rdev, true);
+
+        cypress_start_dpm(rdev);
+
+        if (pi->gfx_clock_gating)
+                cypress_gfx_clock_gating_enable(rdev, true);
+
+        if (pi->mg_clock_gating)
+                cypress_mg_clock_gating_enable(rdev, true);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                PPSMC_Result result;
+
+                ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+                result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
+
+                if (result != PPSMC_Result_OK)
+                        DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
+        }
+
+        rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+
+        return 0;
+}
+
+void cypress_dpm_disable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+
+        if (!rv770_dpm_enabled(rdev))
+                return;
+
+        rv770_clear_vc(rdev);
+
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, false);
+
+        if (pi->dynamic_pcie_gen2)
+                cypress_enable_dynamic_pcie_gen2(rdev, false);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        if (pi->gfx_clock_gating)
+                cypress_gfx_clock_gating_enable(rdev, false);
+
+        if (pi->mg_clock_gating)
+                cypress_mg_clock_gating_enable(rdev, false);
+
+        rv770_stop_dpm(rdev);
+        r7xx_stop_smc(rdev);
+
+        cypress_enable_spread_spectrum(rdev, false);
+
+        if (eg_pi->dynamic_ac_timing)
+                cypress_force_mc_use_s1(rdev, boot_ps);
+
+        rv770_reset_smio_status(rdev);
+}
+
+int cypress_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
+        struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
+        int ret;
+
+        ret = rv770_restrict_performance_levels_before_switch(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n");
+                return ret;
+        }
+        if (eg_pi->pcie_performance_request)
+                cypress_notify_link_speed_change_before_state_change(rdev, new_ps, old_ps);
+
+        rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+        ret = rv770_halt_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_halt_smc failed\n");
+                return ret;
+        }
+        ret = cypress_upload_sw_state(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("cypress_upload_sw_state failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = cypress_upload_mc_reg_table(rdev, new_ps);
+                if (ret) {
+                        DRM_ERROR("cypress_upload_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+
+        cypress_program_memory_timing_parameters(rdev, new_ps);
+
+        ret = rv770_resume_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_resume_smc failed\n");
+                return ret;
+        }
+        ret = rv770_set_sw_state(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_set_sw_state failed\n");
+                return ret;
+        }
+        rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+
+        if (eg_pi->pcie_performance_request)
+                cypress_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps);
+
+        ret = rv770_unrestrict_performance_levels_after_switch(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_unrestrict_performance_levels_after_switch failed\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+void cypress_dpm_reset_asic(struct radeon_device *rdev)
+{
+        rv770_restrict_performance_levels_before_switch(rdev);
+        rv770_set_boot_state(rdev);
+}
+
+void cypress_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+        cypress_program_display_gap(rdev);
+}
+
+int cypress_dpm_init(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi;
+        struct evergreen_power_info *eg_pi;
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        uint16_t data_offset, size;
+        uint8_t frev, crev;
+        struct atom_clock_dividers dividers;
+        int ret;
+
+        eg_pi = kzalloc(sizeof(struct evergreen_power_info), GFP_KERNEL);
+        if (eg_pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = eg_pi;
+        pi = &eg_pi->rv7xx;
+
+        rv770_get_max_vddc(rdev);
+
+        eg_pi->ulv.supported = false;
+        pi->acpi_vddc = 0;
+        eg_pi->acpi_vddci = 0;
+        pi->min_vddc_in_table = 0;
+        pi->max_vddc_in_table = 0;
+
+        ret = rv7xx_parse_power_table(rdev);
+        if (ret)
+                return ret;
+
+        if (rdev->pm.dpm.voltage_response_time == 0)
+                rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+        if (rdev->pm.dpm.backbias_response_time == 0)
+                rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->ref_div = dividers.ref_div + 1;
+        else
+                pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        pi->mclk_strobe_mode_threshold = 40000;
+        pi->mclk_edc_enable_threshold = 40000;
+        eg_pi->mclk_edc_wr_enable_threshold = 40000;
+
+        pi->rlp = RV770_RLP_DFLT;
+        pi->rmp = RV770_RMP_DFLT;
+        pi->lhp = RV770_LHP_DFLT;
+        pi->lmp = RV770_LMP_DFLT;
+
+        pi->voltage_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
+
+        pi->mvdd_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
+
+        eg_pi->vddci_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                pi->sclk_ss = true;
+                pi->mclk_ss = true;
+                pi->dynamic_ss = true;
+        } else {
+                pi->sclk_ss = false;
+                pi->mclk_ss = false;
+                pi->dynamic_ss = true;
+        }
+
+        pi->asi = RV770_ASI_DFLT;
+        pi->pasi = CYPRESS_HASI_DFLT;
+        pi->vrc = CYPRESS_VRC_DFLT;
+
+        pi->power_gating = false;
+
+        if ((rdev->family == CHIP_CYPRESS) ||
+            (rdev->family == CHIP_HEMLOCK))
+                pi->gfx_clock_gating = false;
+        else
+                pi->gfx_clock_gating = true;
+
+        pi->mg_clock_gating = true;
+        pi->mgcgtssm = true;
+        eg_pi->ls_clock_gating = false;
+        eg_pi->sclk_deep_sleep = false;
+
+        pi->dynamic_pcie_gen2 = true;
+
+        if (pi->gfx_clock_gating &&
+            (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
+                pi->thermal_protection = true;
+        else
+                pi->thermal_protection = false;
+
+        pi->display_gap = true;
+
+        if (rdev->flags & RADEON_IS_MOBILITY)
+                pi->dcodt = true;
+        else
+                pi->dcodt = false;
+
+        pi->ulps = true;
+
+        eg_pi->dynamic_ac_timing = true;
+        eg_pi->abm = true;
+        eg_pi->mcls = true;
+        eg_pi->light_sleep = true;
+        eg_pi->memory_transition = true;
+#if defined(CONFIG_ACPI)
+        eg_pi->pcie_performance_request =
+                radeon_acpi_is_pcie_performance_request_supported(rdev);
+#else
+        eg_pi->pcie_performance_request = false;
+#endif
+
+        if ((rdev->family == CHIP_CYPRESS) ||
+            (rdev->family == CHIP_HEMLOCK) ||
+            (rdev->family == CHIP_JUNIPER))
+                eg_pi->dll_default_on = true;
+        else
+                eg_pi->dll_default_on = false;
+
+        eg_pi->sclk_deep_sleep = false;
+        pi->mclk_stutter_mode_threshold = 0;
+
+        pi->sram_end = SMC_RAM_END;
+
+        return 0;
+}
+
+void cypress_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+}
diff --git a/drivers/gpu/drm/radeon/cypress_dpm.h b/drivers/gpu/drm/radeon/cypress_dpm.h
new file mode 100644
index 000000000000..4c3f18c69f4f
--- /dev/null
+++ b/drivers/gpu/drm/radeon/cypress_dpm.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __CYPRESS_DPM_H__
+#define __CYPRESS_DPM_H__
+
+#include "rv770_dpm.h"
+#include "evergreen_smc.h"
+
+struct evergreen_mc_reg_entry {
+        u32 mclk_max;
+        u32 mc_data[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct evergreen_mc_reg_table {
+        u8 last;
+        u8 num_entries;
+        u16 valid_flag;
+        struct evergreen_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+        SMC_Evergreen_MCRegisterAddress mc_reg_address[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct evergreen_ulv_param {
+        bool supported;
+        struct rv7xx_pl *pl;
+};
+
+struct evergreen_arb_registers {
+        u32 mc_arb_dram_timing;
+        u32 mc_arb_dram_timing2;
+        u32 mc_arb_rfsh_rate;
+        u32 mc_arb_burst_time;
+};
+
+struct at {
+        u32 rlp;
+        u32 rmp;
+        u32 lhp;
+        u32 lmp;
+};
+
+struct evergreen_power_info {
+        /* must be first! */
+        struct rv7xx_power_info rv7xx;
+        /* flags */
+        bool vddci_control;
+        bool dynamic_ac_timing;
+        bool abm;
+        bool mcls;
+        bool light_sleep;
+        bool memory_transition;
+        bool pcie_performance_request;
+        bool pcie_performance_request_registered;
+        bool sclk_deep_sleep;
+        bool dll_default_on;
+        bool ls_clock_gating;
+        bool smu_uvd_hs;
+        bool uvd_enabled;
+        /* stored values */
+        u16 acpi_vddci;
+        u8 mvdd_high_index;
+        u8 mvdd_low_index;
+        u32 mclk_edc_wr_enable_threshold;
+        struct evergreen_mc_reg_table mc_reg_table;
+        struct atom_voltage_table vddc_voltage_table;
+        struct atom_voltage_table vddci_voltage_table;
+        struct evergreen_arb_registers bootup_arb_registers;
+        struct evergreen_ulv_param ulv;
+        struct at ats[2];
+        /* smc offsets */
+        u16 mc_reg_table_start;
+        struct radeon_ps current_rps;
+        struct rv7xx_ps current_ps;
+        struct radeon_ps requested_rps;
+        struct rv7xx_ps requested_ps;
+};
+
+#define CYPRESS_HASI_DFLT                               400000
+#define CYPRESS_MGCGTTLOCAL0_DFLT                       0x00000000
+#define CYPRESS_MGCGTTLOCAL1_DFLT                       0x00000000
+#define CYPRESS_MGCGTTLOCAL2_DFLT                       0x00000000
+#define CYPRESS_MGCGTTLOCAL3_DFLT                       0x00000000
+#define CYPRESS_MGCGCGTSSMCTRL_DFLT                     0x81944bc0
+#define REDWOOD_MGCGCGTSSMCTRL_DFLT                     0x6e944040
+#define CEDAR_MGCGCGTSSMCTRL_DFLT                       0x46944040
+#define CYPRESS_VRC_DFLT                                0xC00033
+
+#define PCIE_PERF_REQ_REMOVE_REGISTRY   0
+#define PCIE_PERF_REQ_FORCE_LOWPOWER    1
+#define PCIE_PERF_REQ_PECI_GEN1         2
+#define PCIE_PERF_REQ_PECI_GEN2         3
+#define PCIE_PERF_REQ_PECI_GEN3         4
+
+int cypress_convert_power_level_to_smc(struct radeon_device *rdev,
+                                       struct rv7xx_pl *pl,
+                                       RV770_SMC_HW_PERFORMANCE_LEVEL *level,
+                                       u8 watermark_level);
+int cypress_populate_smc_acpi_state(struct radeon_device *rdev,
+                                    RV770_SMC_STATETABLE *table);
+int cypress_populate_smc_voltage_tables(struct radeon_device *rdev,
+                                        RV770_SMC_STATETABLE *table);
+int cypress_populate_smc_initial_state(struct radeon_device *rdev,
+                                       struct radeon_ps *radeon_initial_state,
+                                       RV770_SMC_STATETABLE *table);
+u32 cypress_calculate_burst_time(struct radeon_device *rdev,
+                                 u32 engine_clock, u32 memory_clock);
+void cypress_notify_link_speed_change_before_state_change(struct radeon_device *rdev,
+                                                          struct radeon_ps *radeon_new_state,
+                                                          struct radeon_ps *radeon_current_state);
+int cypress_upload_sw_state(struct radeon_device *rdev,
+                            struct radeon_ps *radeon_new_state);
+int cypress_upload_mc_reg_table(struct radeon_device *rdev,
+                                struct radeon_ps *radeon_new_state);
+void cypress_program_memory_timing_parameters(struct radeon_device *rdev,
+                                              struct radeon_ps *radeon_new_state);
+void cypress_notify_link_speed_change_after_state_change(struct radeon_device *rdev,
+                                                         struct radeon_ps *radeon_new_state,
+                                                         struct radeon_ps *radeon_current_state);
+int cypress_construct_voltage_tables(struct radeon_device *rdev);
+int cypress_get_mvdd_configuration(struct radeon_device *rdev);
+void cypress_enable_spread_spectrum(struct radeon_device *rdev,
+                                    bool enable);
+void cypress_enable_display_gap(struct radeon_device *rdev);
+int cypress_get_table_locations(struct radeon_device *rdev);
+int cypress_populate_mc_reg_table(struct radeon_device *rdev,
+                                  struct radeon_ps *radeon_boot_state);
+void cypress_program_response_times(struct radeon_device *rdev);
+int cypress_notify_smc_display_change(struct radeon_device *rdev,
+                                      bool has_display);
+void cypress_enable_sclk_control(struct radeon_device *rdev,
+                                 bool enable);
+void cypress_enable_mclk_control(struct radeon_device *rdev,
+                                 bool enable);
+void cypress_start_dpm(struct radeon_device *rdev);
+void cypress_advertise_gen2_capability(struct radeon_device *rdev);
+u32 cypress_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf);
+u8 cypress_get_mclk_frequency_ratio(struct radeon_device *rdev,
+                                    u32 memory_clock, bool strobe_mode);
+u8 cypress_get_strobe_mode_settings(struct radeon_device *rdev, u32 mclk);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 0f89ce3d02b9..2e1de4fd2975 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -33,9 +33,7 @@
 #include "avivod.h"
 #include "evergreen_reg.h"
 #include "evergreen_blit_shaders.h"
-
-#define EVERGREEN_PFP_UCODE_SIZE 1120
-#define EVERGREEN_PM4_UCODE_SIZE 1376
+#include "radeon_ucode.h"
 
 static const u32 crtc_offsets[6] =
 {
@@ -47,9 +45,98 @@ static const u32 crtc_offsets[6] =
         EVERGREEN_CRTC5_REGISTER_OFFSET
 };
 
+#include "clearstate_evergreen.h"
+
+static u32 sumo_rlc_save_restore_register_list[] =
+{
+        0x98fc,
+        0x9830,
+        0x9834,
+        0x9838,
+        0x9870,
+        0x9874,
+        0x8a14,
+        0x8b24,
+        0x8bcc,
+        0x8b10,
+        0x8d00,
+        0x8d04,
+        0x8c00,
+        0x8c04,
+        0x8c08,
+        0x8c0c,
+        0x8d8c,
+        0x8c20,
+        0x8c24,
+        0x8c28,
+        0x8c18,
+        0x8c1c,
+        0x8cf0,
+        0x8e2c,
+        0x8e38,
+        0x8c30,
+        0x9508,
+        0x9688,
+        0x9608,
+        0x960c,
+        0x9610,
+        0x9614,
+        0x88c4,
+        0x88d4,
+        0xa008,
+        0x900c,
+        0x9100,
+        0x913c,
+        0x98f8,
+        0x98f4,
+        0x9b7c,
+        0x3f8c,
+        0x8950,
+        0x8954,
+        0x8a18,
+        0x8b28,
+        0x9144,
+        0x9148,
+        0x914c,
+        0x3f90,
+        0x3f94,
+        0x915c,
+        0x9160,
+        0x9178,
+        0x917c,
+        0x9180,
+        0x918c,
+        0x9190,
+        0x9194,
+        0x9198,
+        0x919c,
+        0x91a8,
+        0x91ac,
+        0x91b0,
+        0x91b4,
+        0x91b8,
+        0x91c4,
+        0x91c8,
+        0x91cc,
+        0x91d0,
+        0x91d4,
+        0x91e0,
+        0x91e4,
+        0x91ec,
+        0x91f0,
+        0x91f4,
+        0x9200,
+        0x9204,
+        0x929c,
+        0x9150,
+        0x802c,
+};
+static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
+
 static void evergreen_gpu_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
+void evergreen_program_aspm(struct radeon_device *rdev);
 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
                                      int ring, u32 cp_int_cntl);
 
@@ -2036,7 +2123,8 @@ static void evergreen_program_watermarks(struct radeon_device *rdev,
                                          u32 lb_size, u32 num_heads)
 {
         struct drm_display_mode *mode = &radeon_crtc->base.mode;
-        struct evergreen_wm_params wm;
+        struct evergreen_wm_params wm_low, wm_high;
+        u32 dram_channels;
         u32 pixel_period;
         u32 line_time = 0;
         u32 latency_watermark_a = 0, latency_watermark_b = 0;
@@ -2052,39 +2140,81 @@ static void evergreen_program_watermarks(struct radeon_device *rdev,
                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
                 priority_a_cnt = 0;
                 priority_b_cnt = 0;
+                dram_channels = evergreen_get_number_of_dram_channels(rdev);
+
+                /* watermark for high clocks */
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        wm_high.yclk =
+                                radeon_dpm_get_mclk(rdev, false) * 10;
+                        wm_high.sclk =
+                                radeon_dpm_get_sclk(rdev, false) * 10;
+                } else {
+                        wm_high.yclk = rdev->pm.current_mclk * 10;
+                        wm_high.sclk = rdev->pm.current_sclk * 10;
+                }
 
-                wm.yclk = rdev->pm.current_mclk * 10;
-                wm.sclk = rdev->pm.current_sclk * 10;
-                wm.disp_clk = mode->clock;
-                wm.src_width = mode->crtc_hdisplay;
-                wm.active_time = mode->crtc_hdisplay * pixel_period;
-                wm.blank_time = line_time - wm.active_time;
-                wm.interlaced = false;
+                wm_high.disp_clk = mode->clock;
+                wm_high.src_width = mode->crtc_hdisplay;
+                wm_high.active_time = mode->crtc_hdisplay * pixel_period;
+                wm_high.blank_time = line_time - wm_high.active_time;
+                wm_high.interlaced = false;
                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-                        wm.interlaced = true;
-                wm.vsc = radeon_crtc->vsc;
-                wm.vtaps = 1;
+                        wm_high.interlaced = true;
+                wm_high.vsc = radeon_crtc->vsc;
+                wm_high.vtaps = 1;
                 if (radeon_crtc->rmx_type != RMX_OFF)
-                        wm.vtaps = 2;
-                wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
-                wm.lb_size = lb_size;
-                wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
-                wm.num_heads = num_heads;
+                        wm_high.vtaps = 2;
+                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
+                wm_high.lb_size = lb_size;
+                wm_high.dram_channels = dram_channels;
+                wm_high.num_heads = num_heads;
+
+                /* watermark for low clocks */
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        wm_low.yclk =
+                                radeon_dpm_get_mclk(rdev, true) * 10;
+                        wm_low.sclk =
+                                radeon_dpm_get_sclk(rdev, true) * 10;
+                } else {
+                        wm_low.yclk = rdev->pm.current_mclk * 10;
+                        wm_low.sclk = rdev->pm.current_sclk * 10;
+                }
+
+                wm_low.disp_clk = mode->clock;
+                wm_low.src_width = mode->crtc_hdisplay;
+                wm_low.active_time = mode->crtc_hdisplay * pixel_period;
+                wm_low.blank_time = line_time - wm_low.active_time;
+                wm_low.interlaced = false;
+                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                        wm_low.interlaced = true;
+                wm_low.vsc = radeon_crtc->vsc;
+                wm_low.vtaps = 1;
+                if (radeon_crtc->rmx_type != RMX_OFF)
+                        wm_low.vtaps = 2;
+                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
+                wm_low.lb_size = lb_size;
+                wm_low.dram_channels = dram_channels;
+                wm_low.num_heads = num_heads;
 
                 /* set for high clocks */
-                latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
+                latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
                 /* set for low clocks */
-                /* wm.yclk = low clk; wm.sclk = low clk */
-                latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
+                latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
 
                 /* possibly force display priority to high */
                 /* should really do this at mode validation time... */
-                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
-                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
-                    !evergreen_check_latency_hiding(&wm) ||
+                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
+                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
+                    !evergreen_check_latency_hiding(&wm_high) ||
                     (rdev->disp_priority == 2)) {
-                        DRM_DEBUG_KMS("force priority to high\n");
+                        DRM_DEBUG_KMS("force priority a to high\n");
                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
+                }
+                if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
+                    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
+                    !evergreen_check_latency_hiding(&wm_low) ||
+                    (rdev->disp_priority == 2)) {
+                        DRM_DEBUG_KMS("force priority b to high\n");
                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
                 }
 
@@ -2137,6 +2267,10 @@ static void evergreen_program_watermarks(struct radeon_device *rdev,
         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
 
+        /* save values for DPM */
+        radeon_crtc->line_time = line_time;
+        radeon_crtc->wm_high = latency_watermark_a;
+        radeon_crtc->wm_low = latency_watermark_b;
 }
 
 /**
@@ -3120,10 +3254,8 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
                 u32 efuse_straps_4;
                 u32 efuse_straps_3;
 
-                WREG32(RCU_IND_INDEX, 0x204);
-                efuse_straps_4 = RREG32(RCU_IND_DATA);
-                WREG32(RCU_IND_INDEX, 0x203);
-                efuse_straps_3 = RREG32(RCU_IND_DATA);
+                efuse_straps_4 = RREG32_RCU(0x204);
+                efuse_straps_3 = RREG32_RCU(0x203);
                 tmp = (((efuse_straps_4 & 0xf) << 4) |
                       ((efuse_straps_3 & 0xf0000000) >> 28));
         } else {
@@ -3727,6 +3859,262 @@ bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *rin
         return radeon_ring_test_lockup(rdev, ring);
 }
 
+/*
+ * RLC
+ */
+#define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
+#define RLC_CLEAR_STATE_END_MARKER          0x00000001
+
+void sumo_rlc_fini(struct radeon_device *rdev)
+{
+        int r;
+
+        /* save restore block */
+        if (rdev->rlc.save_restore_obj) {
+                r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
+                if (unlikely(r != 0))
+                        dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
+                radeon_bo_unpin(rdev->rlc.save_restore_obj);
+                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
+
+                radeon_bo_unref(&rdev->rlc.save_restore_obj);
+                rdev->rlc.save_restore_obj = NULL;
+        }
+
+        /* clear state block */
+        if (rdev->rlc.clear_state_obj) {
+                r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
+                if (unlikely(r != 0))
+                        dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
+                radeon_bo_unpin(rdev->rlc.clear_state_obj);
+                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
+
+                radeon_bo_unref(&rdev->rlc.clear_state_obj);
+                rdev->rlc.clear_state_obj = NULL;
+        }
+}
+
+int sumo_rlc_init(struct radeon_device *rdev)
+{
+        u32 *src_ptr;
+        volatile u32 *dst_ptr;
+        u32 dws, data, i, j, k, reg_num;
+        u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
+        u64 reg_list_mc_addr;
+        struct cs_section_def *cs_data;
+        int r;
+
+        src_ptr = rdev->rlc.reg_list;
+        dws = rdev->rlc.reg_list_size;
+        cs_data = rdev->rlc.cs_data;
+
+        /* save restore block */
+        if (rdev->rlc.save_restore_obj == NULL) {
+                r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
+                                     RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
+                        return r;
+                }
+        }
+
+        r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
+        if (unlikely(r != 0)) {
+                sumo_rlc_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
+                          &rdev->rlc.save_restore_gpu_addr);
+        if (r) {
+                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
+                dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
+                sumo_rlc_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
+        if (r) {
+                dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
+                sumo_rlc_fini(rdev);
+                return r;
+        }
+        /* write the sr buffer */
+        dst_ptr = rdev->rlc.sr_ptr;
+        /* format:
+         * dw0: (reg2 << 16) | reg1
+         * dw1: reg1 save space
+         * dw2: reg2 save space
+         */
+        for (i = 0; i < dws; i++) {
+                data = src_ptr[i] >> 2;
+                i++;
+                if (i < dws)
+                        data |= (src_ptr[i] >> 2) << 16;
+                j = (((i - 1) * 3) / 2);
+                dst_ptr[j] = data;
+        }
+        j = ((i * 3) / 2);
+        dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
+
+        radeon_bo_kunmap(rdev->rlc.save_restore_obj);
+        radeon_bo_unreserve(rdev->rlc.save_restore_obj);
+
+        /* clear state block */
+        reg_list_num = 0;
+        dws = 0;
+        for (i = 0; cs_data[i].section != NULL; i++) {
+                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
+                        reg_list_num++;
+                        dws += cs_data[i].section[j].reg_count;
+                }
+        }
+        reg_list_blk_index = (3 * reg_list_num + 2);
+        dws += reg_list_blk_index;
+
+        if (rdev->rlc.clear_state_obj == NULL) {
+                r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
+                                     RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
+                        sumo_rlc_fini(rdev);
+                        return r;
+                }
+        }
+        r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
+        if (unlikely(r != 0)) {
+                sumo_rlc_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
+                          &rdev->rlc.clear_state_gpu_addr);
+        if (r) {
+
+                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
+                dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
+                sumo_rlc_fini(rdev);
+                return r;
+        }
+        r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
+        if (r) {
+                dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
+                sumo_rlc_fini(rdev);
+                return r;
+        }
+        /* set up the cs buffer */
+        dst_ptr = rdev->rlc.cs_ptr;
+        reg_list_hdr_blk_index = 0;
+        reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
+        data = upper_32_bits(reg_list_mc_addr);
+        dst_ptr[reg_list_hdr_blk_index] = data;
+        reg_list_hdr_blk_index++;
+        for (i = 0; cs_data[i].section != NULL; i++) {
+                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
+                        reg_num = cs_data[i].section[j].reg_count;
+                        data = reg_list_mc_addr & 0xffffffff;
+                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        reg_list_hdr_blk_index++;
+
+                        data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
+                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        reg_list_hdr_blk_index++;
+
+                        data = 0x08000000 | (reg_num * 4);
+                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        reg_list_hdr_blk_index++;
+
+                        for (k = 0; k < reg_num; k++) {
+                                data = cs_data[i].section[j].extent[k];
+                                dst_ptr[reg_list_blk_index + k] = data;
+                        }
+                        reg_list_mc_addr += reg_num * 4;
+                        reg_list_blk_index += reg_num;
+                }
+        }
+        dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
+
+        radeon_bo_kunmap(rdev->rlc.clear_state_obj);
+        radeon_bo_unreserve(rdev->rlc.clear_state_obj);
+
+        return 0;
+}
+
+static void evergreen_rlc_start(struct radeon_device *rdev)
+{
+        u32 mask = RLC_ENABLE;
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
+        }
+
+        WREG32(RLC_CNTL, mask);
+}
+
+int evergreen_rlc_resume(struct radeon_device *rdev)
+{
+        u32 i;
+        const __be32 *fw_data;
+
+        if (!rdev->rlc_fw)
+                return -EINVAL;
+
+        r600_rlc_stop(rdev);
+
+        WREG32(RLC_HB_CNTL, 0);
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                if (rdev->family == CHIP_ARUBA) {
+                        u32 always_on_bitmap =
+                                3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
+                        /* find out the number of active simds */
+                        u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
+                        tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
+                        tmp = hweight32(~tmp);
+                        if (tmp == rdev->config.cayman.max_simds_per_se) {
+                                WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
+                                WREG32(TN_RLC_LB_PARAMS, 0x00601004);
+                                WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
+                                WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
+                                WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
+                        }
+                } else {
+                        WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
+                        WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
+                }
+                WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
+                WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
+        } else {
+                WREG32(RLC_HB_BASE, 0);
+                WREG32(RLC_HB_RPTR, 0);
+                WREG32(RLC_HB_WPTR, 0);
+                WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
+                WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
+        }
+        WREG32(RLC_MC_CNTL, 0);
+        WREG32(RLC_UCODE_CNTL, 0);
+
+        fw_data = (const __be32 *)rdev->rlc_fw->data;
+        if (rdev->family >= CHIP_ARUBA) {
+                for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
+                        WREG32(RLC_UCODE_ADDR, i);
+                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
+                }
+        } else if (rdev->family >= CHIP_CAYMAN) {
+                for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
+                        WREG32(RLC_UCODE_ADDR, i);
+                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
+                }
+        } else {
+                for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
+                        WREG32(RLC_UCODE_ADDR, i);
+                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
+                }
+        }
+        WREG32(RLC_UCODE_ADDR, 0);
+
+        evergreen_rlc_start(rdev);
+
+        return 0;
+}
+
 /* Interrupts */
 
 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
@@ -3805,6 +4193,7 @@ int evergreen_irq_set(struct radeon_device *rdev)
         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
         u32 dma_cntl, dma_cntl1 = 0;
+        u32 thermal_int = 0;
 
         if (!rdev->irq.installed) {
                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -3824,6 +4213,12 @@ int evergreen_irq_set(struct radeon_device *rdev)
         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
+        if (rdev->family == CHIP_ARUBA)
+                thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
+                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
+        else
+                thermal_int = RREG32(CG_THERMAL_INT) &
+                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
 
         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
@@ -3869,6 +4264,11 @@ int evergreen_irq_set(struct radeon_device *rdev)
                 }
         }
 
+        if (rdev->irq.dpm_thermal) {
+                DRM_DEBUG("dpm thermal\n");
+                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
+        }
+
         if (rdev->irq.crtc_vblank_int[0] ||
             atomic_read(&rdev->irq.pflip[0])) {
                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
@@ -3990,6 +4390,10 @@ int evergreen_irq_set(struct radeon_device *rdev)
         WREG32(DC_HPD4_INT_CONTROL, hpd4);
         WREG32(DC_HPD5_INT_CONTROL, hpd5);
         WREG32(DC_HPD6_INT_CONTROL, hpd6);
+        if (rdev->family == CHIP_ARUBA)
+                WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
+        else
+                WREG32(CG_THERMAL_INT, thermal_int);
 
         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
@@ -4181,6 +4585,7 @@ int evergreen_irq_process(struct radeon_device *rdev)
         u32 ring_index;
         bool queue_hotplug = false;
         bool queue_hdmi = false;
+        bool queue_thermal = false;
 
         if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
@@ -4502,6 +4907,16 @@ restart_ih:
                         DRM_DEBUG("IH: DMA trap\n");
                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
                         break;
+                case 230: /* thermal low to high */
+                        DRM_DEBUG("IH: thermal low to high\n");
+                        rdev->pm.dpm.thermal.high_to_low = false;
+                        queue_thermal = true;
+                        break;
+                case 231: /* thermal high to low */
+                        DRM_DEBUG("IH: thermal high to low\n");
+                        rdev->pm.dpm.thermal.high_to_low = true;
+                        queue_thermal = true;
+                        break;
                 case 233: /* GUI IDLE */
                         DRM_DEBUG("IH: GUI idle\n");
                         break;
@@ -4524,6 +4939,8 @@ restart_ih:
                 schedule_work(&rdev->hotplug_work);
         if (queue_hdmi)
                 schedule_work(&rdev->audio_work);
+        if (queue_thermal && rdev->pm.dpm_enabled)
+                schedule_work(&rdev->pm.dpm.thermal.work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         atomic_set(&rdev->ih.lock, 0);
@@ -4680,6 +5097,8 @@ static int evergreen_startup(struct radeon_device *rdev)
 
         /* enable pcie gen2 link */
         evergreen_pcie_gen2_enable(rdev);
+        /* enable aspm */
+        evergreen_program_aspm(rdev);
 
         if (ASIC_IS_DCE5(rdev)) {
                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
@@ -4725,6 +5144,18 @@ static int evergreen_startup(struct radeon_device *rdev)
                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
         }
 
+        /* allocate rlc buffers */
+        if (rdev->flags & RADEON_IS_IGP) {
+                rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
+                rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
+                rdev->rlc.cs_data = evergreen_cs_data;
+                r = sumo_rlc_init(rdev);
+                if (r) {
+                        DRM_ERROR("Failed to init rlc BOs!\n");
+                        return r;
+                }
+        }
+
         /* allocate wb buffer */
         r = radeon_wb_init(rdev);
         if (r)
@@ -4956,6 +5387,8 @@ int evergreen_init(struct radeon_device *rdev)
                 r700_cp_fini(rdev);
                 r600_dma_fini(rdev);
                 r600_irq_fini(rdev);
+                if (rdev->flags & RADEON_IS_IGP)
+                        sumo_rlc_fini(rdev);
                 radeon_wb_fini(rdev);
                 radeon_ib_pool_fini(rdev);
                 radeon_irq_kms_fini(rdev);
@@ -4984,6 +5417,8 @@ void evergreen_fini(struct radeon_device *rdev)
         r700_cp_fini(rdev);
         r600_dma_fini(rdev);
         r600_irq_fini(rdev);
+        if (rdev->flags & RADEON_IS_IGP)
+                sumo_rlc_fini(rdev);
         radeon_wb_fini(rdev);
         radeon_ib_pool_fini(rdev);
         radeon_irq_kms_fini(rdev);
@@ -5061,3 +5496,150 @@ void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
         }
 }
+
+void evergreen_program_aspm(struct radeon_device *rdev)
+{
+        u32 data, orig;
+        u32 pcie_lc_cntl, pcie_lc_cntl_old;
+        bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
+        /* fusion_platform = true
+         * if the system is a fusion system
+         * (APU or DGPU in a fusion system).
+         * todo: check if the system is a fusion platform.
+         */
+        bool fusion_platform = false;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        switch (rdev->family) {
+        case CHIP_CYPRESS:
+        case CHIP_HEMLOCK:
+        case CHIP_JUNIPER:
+        case CHIP_REDWOOD:
+        case CHIP_CEDAR:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+        case CHIP_PALM:
+        case CHIP_ARUBA:
+                disable_l0s = true;
+                break;
+        default:
+                disable_l0s = false;
+                break;
+        }
+
+        if (rdev->flags & RADEON_IS_IGP)
+                fusion_platform = true; /* XXX also dGPUs in a fusion system */
+
+        data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
+        if (fusion_platform)
+                data &= ~MULTI_PIF;
+        else
+                data |= MULTI_PIF;
+        if (data != orig)
+                WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
+
+        data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
+        if (fusion_platform)
+                data &= ~MULTI_PIF;
+        else
+                data |= MULTI_PIF;
+        if (data != orig)
+                WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
+
+        pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+        pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
+        if (!disable_l0s) {
+                if (rdev->family >= CHIP_BARTS)
+                        pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
+                else
+                        pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
+        }
+
+        if (!disable_l1) {
+                if (rdev->family >= CHIP_BARTS)
+                        pcie_lc_cntl |= LC_L1_INACTIVITY(7);
+                else
+                        pcie_lc_cntl |= LC_L1_INACTIVITY(8);
+
+                if (!disable_plloff_in_l1) {
+                        data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
+                        if (data != orig)
+                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
+
+                        data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
+                        if (data != orig)
+                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
+
+                        data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
+                        if (data != orig)
+                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
+
+                        data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
+                        if (data != orig)
+                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
+
+                        if (rdev->family >= CHIP_BARTS) {
+                                data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
+                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
+                                data |= PLL_RAMP_UP_TIME_0(4);
+                                if (data != orig)
+                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
+
+                                data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
+                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
+                                data |= PLL_RAMP_UP_TIME_1(4);
+                                if (data != orig)
+                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
+
+                                data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
+                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
+                                data |= PLL_RAMP_UP_TIME_0(4);
+                                if (data != orig)
+                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
+
+                                data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
+                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
+                                data |= PLL_RAMP_UP_TIME_1(4);
+                                if (data != orig)
+                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
+                        }
+
+                        data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
+                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
+                        data |= LC_DYN_LANES_PWR_STATE(3);
+                        if (data != orig)
+                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
+
+                        if (rdev->family >= CHIP_BARTS) {
+                                data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
+                                data &= ~LS2_EXIT_TIME_MASK;
+                                data |= LS2_EXIT_TIME(1);
+                                if (data != orig)
+                                        WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
+
+                                data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
+                                data &= ~LS2_EXIT_TIME_MASK;
+                                data |= LS2_EXIT_TIME(1);
+                                if (data != orig)
+                                        WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
+                        }
+                }
+        }
+
+        /* evergreen parts only */
+        if (rdev->family < CHIP_BARTS)
+                pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
+
+        if (pcie_lc_cntl != pcie_lc_cntl_old)
+                WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
+}
diff --git a/drivers/gpu/drm/radeon/evergreen_hdmi.c b/drivers/gpu/drm/radeon/evergreen_hdmi.c
index ed7c8a768092..b9c6f7675e59 100644
--- a/drivers/gpu/drm/radeon/evergreen_hdmi.c
+++ b/drivers/gpu/drm/radeon/evergreen_hdmi.c
@@ -128,14 +128,7 @@ static void evergreen_hdmi_update_avi_infoframe(struct drm_encoder *encoder,
         struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
         uint32_t offset = dig->afmt->offset;
         uint8_t *frame = buffer + 3;
-
-        /* Our header values (type, version, length) should be alright, Intel
-         * is using the same. Checksum function also seems to be OK, it works
-         * fine for audio infoframe. However calculated value is always lower
-         * by 2 in comparison to fglrx. It breaks displaying anything in case
-         * of TVs that strictly check the checksum. Hack it manually here to
-         * workaround this issue. */
-        frame[0x0] += 2;
+        uint8_t *header = buffer;
 
         WREG32(AFMT_AVI_INFO0 + offset,
                 frame[0x0] | (frame[0x1] << 8) | (frame[0x2] << 16) | (frame[0x3] << 24));
@@ -144,7 +137,7 @@ static void evergreen_hdmi_update_avi_infoframe(struct drm_encoder *encoder,
         WREG32(AFMT_AVI_INFO2 + offset,
                 frame[0x8] | (frame[0x9] << 8) | (frame[0xA] << 16) | (frame[0xB] << 24));
         WREG32(AFMT_AVI_INFO3 + offset,
-                frame[0xC] | (frame[0xD] << 8));
+                frame[0xC] | (frame[0xD] << 8) | (header[1] << 24));
 }
 
 static void evergreen_audio_set_dto(struct drm_encoder *encoder, u32 clock)
diff --git a/drivers/gpu/drm/radeon/evergreen_reg.h b/drivers/gpu/drm/radeon/evergreen_reg.h
index 881aba23c477..8a4e641f0e3c 100644
--- a/drivers/gpu/drm/radeon/evergreen_reg.h
+++ b/drivers/gpu/drm/radeon/evergreen_reg.h
@@ -24,7 +24,16 @@
 #ifndef __EVERGREEN_REG_H__
 #define __EVERGREEN_REG_H__
 
+/* trinity */
+#define TN_SMC_IND_INDEX_0                              0x200
+#define TN_SMC_IND_DATA_0                               0x204
+
 /* evergreen */
+#define EVERGREEN_PIF_PHY0_INDEX                        0x8
+#define EVERGREEN_PIF_PHY0_DATA                         0xc
+#define EVERGREEN_PIF_PHY1_INDEX                        0x10
+#define EVERGREEN_PIF_PHY1_DATA                         0x14
+
 #define EVERGREEN_VGA_MEMORY_BASE_ADDRESS               0x310
 #define EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH          0x324
 #define EVERGREEN_D3VGA_CONTROL                         0x3e0
@@ -40,6 +49,9 @@
 #define EVERGREEN_AUDIO_PLL1_DIV                        0x5b4
 #define EVERGREEN_AUDIO_PLL1_UNK                        0x5bc
 
+#define EVERGREEN_CG_IND_ADDR                           0x8f8
+#define EVERGREEN_CG_IND_DATA                           0x8fc
+
 #define EVERGREEN_AUDIO_ENABLE                          0x5e78
 #define EVERGREEN_AUDIO_VENDOR_ID                       0x5ec0
 
diff --git a/drivers/gpu/drm/radeon/evergreen_smc.h b/drivers/gpu/drm/radeon/evergreen_smc.h
new file mode 100644
index 000000000000..76ada8cfe902
--- /dev/null
+++ b/drivers/gpu/drm/radeon/evergreen_smc.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __EVERGREEN_SMC_H__
+#define __EVERGREEN_SMC_H__
+
+#include "rv770_smc.h"
+
+#pragma pack(push, 1)
+
+#define SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE 16
+
+struct SMC_Evergreen_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMC_Evergreen_MCRegisterAddress SMC_Evergreen_MCRegisterAddress;
+
+
+struct SMC_Evergreen_MCRegisterSet
+{
+    uint32_t value[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMC_Evergreen_MCRegisterSet SMC_Evergreen_MCRegisterSet;
+
+struct SMC_Evergreen_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMC_Evergreen_MCRegisterAddress     address[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+    SMC_Evergreen_MCRegisterSet         data[5];
+};
+
+typedef struct SMC_Evergreen_MCRegisters SMC_Evergreen_MCRegisters;
+
+#define EVERGREEN_SMC_FIRMWARE_HEADER_LOCATION 0x100
+
+#define EVERGREEN_SMC_FIRMWARE_HEADER_softRegisters   0x0
+#define EVERGREEN_SMC_FIRMWARE_HEADER_stateTable      0xC
+#define EVERGREEN_SMC_FIRMWARE_HEADER_mcRegisterTable 0x20
+
+
+#pragma pack(pop)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 75c05631146d..a7baf67aef6c 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -48,6 +48,293 @@
 #define SUMO_GB_ADDR_CONFIG_GOLDEN           0x02010002
 #define SUMO2_GB_ADDR_CONFIG_GOLDEN          0x02010002
 
+/* pm registers */
+#define SMC_MSG                                         0x20c
+#define         HOST_SMC_MSG(x)                         ((x) << 0)
+#define         HOST_SMC_MSG_MASK                       (0xff << 0)
+#define         HOST_SMC_MSG_SHIFT                      0
+#define         HOST_SMC_RESP(x)                        ((x) << 8)
+#define         HOST_SMC_RESP_MASK                      (0xff << 8)
+#define         HOST_SMC_RESP_SHIFT                     8
+#define         SMC_HOST_MSG(x)                         ((x) << 16)
+#define         SMC_HOST_MSG_MASK                       (0xff << 16)
+#define         SMC_HOST_MSG_SHIFT                      16
+#define         SMC_HOST_RESP(x)                        ((x) << 24)
+#define         SMC_HOST_RESP_MASK                      (0xff << 24)
+#define         SMC_HOST_RESP_SHIFT                     24
+
+#define DCCG_DISP_SLOW_SELECT_REG                       0x4fc
+#define         DCCG_DISP1_SLOW_SELECT(x)               ((x) << 0)
+#define         DCCG_DISP1_SLOW_SELECT_MASK             (7 << 0)
+#define         DCCG_DISP1_SLOW_SELECT_SHIFT            0
+#define         DCCG_DISP2_SLOW_SELECT(x)               ((x) << 4)
+#define         DCCG_DISP2_SLOW_SELECT_MASK             (7 << 4)
+#define         DCCG_DISP2_SLOW_SELECT_SHIFT            4
+
+#define CG_SPLL_FUNC_CNTL                               0x600
+#define         SPLL_RESET                              (1 << 0)
+#define         SPLL_SLEEP                              (1 << 1)
+#define         SPLL_BYPASS_EN                          (1 << 3)
+#define         SPLL_REF_DIV(x)                         ((x) << 4)
+#define         SPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         SPLL_PDIV_A(x)                          ((x) << 20)
+#define         SPLL_PDIV_A_MASK                        (0x7f << 20)
+#define CG_SPLL_FUNC_CNTL_2                             0x604
+#define         SCLK_MUX_SEL(x)                         ((x) << 0)
+#define         SCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_SPLL_FUNC_CNTL_3                             0x608
+#define         SPLL_FB_DIV(x)                          ((x) << 0)
+#define         SPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         SPLL_DITHEN                             (1 << 28)
+
+#define MPLL_CNTL_MODE                                  0x61c
+#       define SS_SSEN                                  (1 << 24)
+#       define SS_DSMODE_EN                             (1 << 25)
+
+#define MPLL_AD_FUNC_CNTL                               0x624
+#define         CLKF(x)                                 ((x) << 0)
+#define         CLKF_MASK                               (0x7f << 0)
+#define         CLKR(x)                                 ((x) << 7)
+#define         CLKR_MASK                               (0x1f << 7)
+#define         CLKFRAC(x)                              ((x) << 12)
+#define         CLKFRAC_MASK                            (0x1f << 12)
+#define         YCLK_POST_DIV(x)                        ((x) << 17)
+#define         YCLK_POST_DIV_MASK                      (3 << 17)
+#define         IBIAS(x)                                ((x) << 20)
+#define         IBIAS_MASK                              (0x3ff << 20)
+#define         RESET                                   (1 << 30)
+#define         PDNB                                    (1 << 31)
+#define MPLL_AD_FUNC_CNTL_2                             0x628
+#define         BYPASS                                  (1 << 19)
+#define         BIAS_GEN_PDNB                           (1 << 24)
+#define         RESET_EN                                (1 << 25)
+#define         VCO_MODE                                (1 << 29)
+#define MPLL_DQ_FUNC_CNTL                               0x62c
+#define MPLL_DQ_FUNC_CNTL_2                             0x630
+
+#define GENERAL_PWRMGT                                  0x63c
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define ENABLE_GEN2PCIE                          (1 << 4)
+#       define ENABLE_GEN2XSP                           (1 << 5)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (3 << 6)
+#       define SW_SMIO_INDEX_SHIFT                      6
+#       define LOW_VOLT_D2_ACPI                         (1 << 8)
+#       define LOW_VOLT_D3_ACPI                         (1 << 9)
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define BACKBIAS_PAD_EN                          (1 << 18)
+#       define BACKBIAS_VALUE                           (1 << 19)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#       define AC_DC_SW                                 (1 << 24)
+
+#define SCLK_PWRMGT_CNTL                                  0x644
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_LOW_D1                                (1 << 1)
+#       define FIR_RESET                                  (1 << 4)
+#       define FIR_FORCE_TREND_SEL                        (1 << 5)
+#       define FIR_TREND_MODE                             (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 7)
+#       define GFX_CLK_FORCE_ON                           (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                        (1 << 9)
+#       define GFX_CLK_FORCE_OFF                          (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                        (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                        (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                        (1 << 13)
+#       define DYN_LIGHT_SLEEP_EN                         (1 << 14)
+#define MCLK_PWRMGT_CNTL                                0x648
+#       define DLL_SPEED(x)                             ((x) << 0)
+#       define DLL_SPEED_MASK                           (0x1f << 0)
+#       define MPLL_PWRMGT_OFF                          (1 << 5)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCKA0_PDNB                             (1 << 8)
+#       define MRDCKA1_PDNB                             (1 << 9)
+#       define MRDCKB0_PDNB                             (1 << 10)
+#       define MRDCKB1_PDNB                             (1 << 11)
+#       define MRDCKC0_PDNB                             (1 << 12)
+#       define MRDCKC1_PDNB                             (1 << 13)
+#       define MRDCKD0_PDNB                             (1 << 14)
+#       define MRDCKD1_PDNB                             (1 << 15)
+#       define MRDCKA0_RESET                            (1 << 16)
+#       define MRDCKA1_RESET                            (1 << 17)
+#       define MRDCKB0_RESET                            (1 << 18)
+#       define MRDCKB1_RESET                            (1 << 19)
+#       define MRDCKC0_RESET                            (1 << 20)
+#       define MRDCKC1_RESET                            (1 << 21)
+#       define MRDCKD0_RESET                            (1 << 22)
+#       define MRDCKD1_RESET                            (1 << 23)
+#       define DLL_READY_READ                           (1 << 24)
+#       define USE_DISPLAY_GAP                          (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                (1 << 26)
+#       define MPLL_TURNOFF_D2                          (1 << 28)
+#define DLL_CNTL                                        0x64c
+#       define MRDCKA0_BYPASS                           (1 << 24)
+#       define MRDCKA1_BYPASS                           (1 << 25)
+#       define MRDCKB0_BYPASS                           (1 << 26)
+#       define MRDCKB1_BYPASS                           (1 << 27)
+#       define MRDCKC0_BYPASS                           (1 << 28)
+#       define MRDCKC1_BYPASS                           (1 << 29)
+#       define MRDCKD0_BYPASS                           (1 << 30)
+#       define MRDCKD1_BYPASS                           (1 << 31)
+
+#define CG_AT                                           0x6d4
+#       define CG_R(x)                                  ((x) << 0)
+#       define CG_R_MASK                                (0xffff << 0)
+#       define CG_L(x)                                  ((x) << 16)
+#       define CG_L_MASK                                (0xffff << 16)
+
+#define CG_DISPLAY_GAP_CNTL                               0x714
+#       define DISP1_GAP(x)                               ((x) << 0)
+#       define DISP1_GAP_MASK                             (3 << 0)
+#       define DISP2_GAP(x)                               ((x) << 2)
+#       define DISP2_GAP_MASK                             (3 << 2)
+#       define VBI_TIMER_COUNT(x)                         ((x) << 4)
+#       define VBI_TIMER_COUNT_MASK                       (0x3fff << 4)
+#       define VBI_TIMER_UNIT(x)                          ((x) << 20)
+#       define VBI_TIMER_UNIT_MASK                        (7 << 20)
+#       define DISP1_GAP_MCHG(x)                          ((x) << 24)
+#       define DISP1_GAP_MCHG_MASK                        (3 << 24)
+#       define DISP2_GAP_MCHG(x)                          ((x) << 26)
+#       define DISP2_GAP_MCHG_MASK                        (3 << 26)
+
+#define CG_BIF_REQ_AND_RSP                              0x7f4
+#define         CG_CLIENT_REQ(x)                        ((x) << 0)
+#define         CG_CLIENT_REQ_MASK                      (0xff << 0)
+#define         CG_CLIENT_REQ_SHIFT                     0
+#define         CG_CLIENT_RESP(x)                       ((x) << 8)
+#define         CG_CLIENT_RESP_MASK                     (0xff << 8)
+#define         CG_CLIENT_RESP_SHIFT                    8
+#define         CLIENT_CG_REQ(x)                        ((x) << 16)
+#define         CLIENT_CG_REQ_MASK                      (0xff << 16)
+#define         CLIENT_CG_REQ_SHIFT                     16
+#define         CLIENT_CG_RESP(x)                       ((x) << 24)
+#define         CLIENT_CG_RESP_MASK                     (0xff << 24)
+#define         CLIENT_CG_RESP_SHIFT                    24
+
+#define CG_SPLL_SPREAD_SPECTRUM                         0x790
+#define         SSEN                                    (1 << 0)
+#define CG_SPLL_SPREAD_SPECTRUM_2                       0x794
+
+#define MPLL_SS1                                        0x85c
+#define         CLKV(x)                                 ((x) << 0)
+#define         CLKV_MASK                               (0x3ffffff << 0)
+#define MPLL_SS2                                        0x860
+#define         CLKS(x)                                 ((x) << 0)
+#define         CLKS_MASK                               (0xfff << 0)
+
+#define CG_IND_ADDR                                     0x8f8
+#define CG_IND_DATA                                     0x8fc
+/* CGIND regs */
+#define CG_CGTT_LOCAL_0                                 0x00
+#define CG_CGTT_LOCAL_1                                 0x01
+#define CG_CGTT_LOCAL_2                                 0x02
+#define CG_CGTT_LOCAL_3                                 0x03
+#define CG_CGLS_TILE_0                                  0x20
+#define CG_CGLS_TILE_1                                  0x21
+#define CG_CGLS_TILE_2                                  0x22
+#define CG_CGLS_TILE_3                                  0x23
+#define CG_CGLS_TILE_4                                  0x24
+#define CG_CGLS_TILE_5                                  0x25
+#define CG_CGLS_TILE_6                                  0x26
+#define CG_CGLS_TILE_7                                  0x27
+#define CG_CGLS_TILE_8                                  0x28
+#define CG_CGLS_TILE_9                                  0x29
+#define CG_CGLS_TILE_10                                 0x2a
+#define CG_CGLS_TILE_11                                 0x2b
+
+#define VM_L2_CG                                        0x15c0
+
+#define MC_CONFIG                                       0x2000
+
+#define MC_CONFIG_MCD                                   0x20a0
+#define MC_CG_CONFIG_MCD                                0x20a4
+#define         MC_RD_ENABLE_MCD(x)                     ((x) << 8)
+#define         MC_RD_ENABLE_MCD_MASK                   (7 << 8)
+
+#define MC_HUB_MISC_HUB_CG                              0x20b8
+#define MC_HUB_MISC_VM_CG                               0x20bc
+#define MC_HUB_MISC_SIP_CG                              0x20c0
+
+#define MC_XPB_CLK_GAT                                  0x2478
+
+#define MC_CG_CONFIG                                    0x25bc
+#define         MC_RD_ENABLE(x)                         ((x) << 4)
+#define         MC_RD_ENABLE_MASK                       (3 << 4)
+
+#define MC_CITF_MISC_RD_CG                              0x2648
+#define MC_CITF_MISC_WR_CG                              0x264c
+#define MC_CITF_MISC_VM_CG                              0x2650
+#       define MEM_LS_ENABLE                            (1 << 19)
+
+#define MC_ARB_BURST_TIME                               0x2808
+#define         STATE0(x)                               ((x) << 0)
+#define         STATE0_MASK                             (0x1f << 0)
+#define         STATE1(x)                               ((x) << 5)
+#define         STATE1_MASK                             (0x1f << 5)
+#define         STATE2(x)                               ((x) << 10)
+#define         STATE2_MASK                             (0x1f << 10)
+#define         STATE3(x)                               ((x) << 15)
+#define         STATE3_MASK                             (0x1f << 15)
+
+#define MC_SEQ_RAS_TIMING                               0x28a0
+#define MC_SEQ_CAS_TIMING                               0x28a4
+#define MC_SEQ_MISC_TIMING                              0x28a8
+#define MC_SEQ_MISC_TIMING2                             0x28ac
+
+#define MC_SEQ_RD_CTL_D0                                0x28b4
+#define MC_SEQ_RD_CTL_D1                                0x28b8
+#define MC_SEQ_WR_CTL_D0                                0x28bc
+#define MC_SEQ_WR_CTL_D1                                0x28c0
+
+#define MC_SEQ_STATUS_M                                 0x29f4
+#       define PMG_PWRSTATE                             (1 << 16)
+
+#define MC_SEQ_MISC1                                    0x2a04
+#define MC_SEQ_RESERVE_M                                0x2a08
+#define MC_PMG_CMD_EMRS                                 0x2a0c
+
+#define MC_SEQ_MISC3                                    0x2a2c
+
+#define MC_SEQ_MISC5                                    0x2a54
+#define MC_SEQ_MISC6                                    0x2a58
+
+#define MC_SEQ_MISC7                                    0x2a64
+
+#define MC_SEQ_CG                                       0x2a68
+#define         CG_SEQ_REQ(x)                           ((x) << 0)
+#define         CG_SEQ_REQ_MASK                         (0xff << 0)
+#define         CG_SEQ_REQ_SHIFT                        0
+#define         CG_SEQ_RESP(x)                          ((x) << 8)
+#define         CG_SEQ_RESP_MASK                        (0xff << 8)
+#define         CG_SEQ_RESP_SHIFT                       8
+#define         SEQ_CG_REQ(x)                           ((x) << 16)
+#define         SEQ_CG_REQ_MASK                         (0xff << 16)
+#define         SEQ_CG_REQ_SHIFT                        16
+#define         SEQ_CG_RESP(x)                          ((x) << 24)
+#define         SEQ_CG_RESP_MASK                        (0xff << 24)
+#define         SEQ_CG_RESP_SHIFT                       24
+#define MC_SEQ_RAS_TIMING_LP                            0x2a6c
+#define MC_SEQ_CAS_TIMING_LP                            0x2a70
+#define MC_SEQ_MISC_TIMING_LP                           0x2a74
+#define MC_SEQ_MISC_TIMING2_LP                          0x2a78
+#define MC_SEQ_WR_CTL_D0_LP                             0x2a7c
+#define MC_SEQ_WR_CTL_D1_LP                             0x2a80
+#define MC_SEQ_PMG_CMD_EMRS_LP                          0x2a84
+#define MC_SEQ_PMG_CMD_MRS_LP                           0x2a88
+
+#define MC_PMG_CMD_MRS                                  0x2aac
+
+#define MC_SEQ_RD_CTL_D0_LP                             0x2b1c
+#define MC_SEQ_RD_CTL_D1_LP                             0x2b20
+
+#define MC_PMG_CMD_MRS1                                 0x2b44
+#define MC_SEQ_PMG_CMD_MRS1_LP                          0x2b48
+
+#define CGTS_SM_CTRL_REG                                0x9150
+
 /* Registers */
 
 #define RCU_IND_INDEX                                   0x100
@@ -90,6 +377,34 @@
 #define CG_VCLK_STATUS                                  0x61c
 #define CG_SCRATCH1                                     0x820
 
+#define RLC_CNTL                                        0x3f00
+#       define RLC_ENABLE                               (1 << 0)
+#       define GFX_POWER_GATING_ENABLE                  (1 << 7)
+#       define GFX_POWER_GATING_SRC                     (1 << 8)
+#       define DYN_PER_SIMD_PG_ENABLE                   (1 << 27)
+#       define LB_CNT_SPIM_ACTIVE                       (1 << 30)
+#       define LOAD_BALANCE_ENABLE                      (1 << 31)
+
+#define RLC_HB_BASE                                       0x3f10
+#define RLC_HB_CNTL                                       0x3f0c
+#define RLC_HB_RPTR                                       0x3f20
+#define RLC_HB_WPTR                                       0x3f1c
+#define RLC_HB_WPTR_LSB_ADDR                              0x3f14
+#define RLC_HB_WPTR_MSB_ADDR                              0x3f18
+#define RLC_MC_CNTL                                       0x3f44
+#define RLC_UCODE_CNTL                                    0x3f48
+#define RLC_UCODE_ADDR                                    0x3f2c
+#define RLC_UCODE_DATA                                    0x3f30
+
+/* new for TN */
+#define TN_RLC_SAVE_AND_RESTORE_BASE                      0x3f10
+#define TN_RLC_LB_CNTR_MAX                                0x3f14
+#define TN_RLC_LB_CNTR_INIT                               0x3f18
+#define TN_RLC_CLEAR_STATE_RESTORE_BASE                   0x3f20
+#define TN_RLC_LB_INIT_SIMD_MASK                          0x3fe4
+#define TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK                 0x3fe8
+#define TN_RLC_LB_PARAMS                                  0x3fec
+
 #define GRBM_GFX_INDEX                                  0x802C
 #define         INSTANCE_INDEX(x)                       ((x) << 0)
 #define         SE_INDEX(x)                             ((x) << 16)
@@ -503,6 +818,30 @@
 #define CG_THERMAL_CTRL                                 0x72c
 #define         TOFFSET_MASK                            0x00003FE0
 #define         TOFFSET_SHIFT                           5
+#define         DIG_THERM_DPM(x)                        ((x) << 14)
+#define         DIG_THERM_DPM_MASK                      0x003FC000
+#define         DIG_THERM_DPM_SHIFT                     14
+
+#define CG_THERMAL_INT                                  0x734
+#define         DIG_THERM_INTH(x)                       ((x) << 8)
+#define         DIG_THERM_INTH_MASK                     0x0000FF00
+#define         DIG_THERM_INTH_SHIFT                    8
+#define         DIG_THERM_INTL(x)                       ((x) << 16)
+#define         DIG_THERM_INTL_MASK                     0x00FF0000
+#define         DIG_THERM_INTL_SHIFT                    16
+#define         THERM_INT_MASK_HIGH                     (1 << 24)
+#define         THERM_INT_MASK_LOW                      (1 << 25)
+
+#define TN_CG_THERMAL_INT_CTRL                          0x738
+#define         TN_DIG_THERM_INTH(x)                    ((x) << 0)
+#define         TN_DIG_THERM_INTH_MASK                  0x000000FF
+#define         TN_DIG_THERM_INTH_SHIFT                 0
+#define         TN_DIG_THERM_INTL(x)                    ((x) << 8)
+#define         TN_DIG_THERM_INTL_MASK                  0x0000FF00
+#define         TN_DIG_THERM_INTL_SHIFT                 8
+#define         TN_THERM_INT_MASK_HIGH                  (1 << 24)
+#define         TN_THERM_INT_MASK_LOW                   (1 << 25)
+
 #define CG_MULT_THERMAL_STATUS                          0x740
 #define         ASIC_T(x)                               ((x) << 16)
 #define         ASIC_T_MASK                             0x07FF0000
@@ -510,6 +849,7 @@
 #define CG_TS0_STATUS                                   0x760
 #define         TS0_ADC_DOUT_MASK                       0x000003FF
 #define         TS0_ADC_DOUT_SHIFT                      0
+
 /* APU */
 #define CG_THERMAL_STATUS                               0x678
 
@@ -992,7 +1332,48 @@
 #define DMA_PACKET_CONSTANT_FILL                0xd
 #define DMA_PACKET_NOP                          0xf
 
-/* PCIE link stuff */
+/* PIF PHY0 indirect regs */
+#define PB0_PIF_CNTL                                      0x10
+#       define LS2_EXIT_TIME(x)                           ((x) << 17)
+#       define LS2_EXIT_TIME_MASK                         (0x7 << 17)
+#       define LS2_EXIT_TIME_SHIFT                        17
+#define PB0_PIF_PAIRING                                   0x11
+#       define MULTI_PIF                                  (1 << 25)
+#define PB0_PIF_PWRDOWN_0                                 0x12
+#       define PLL_POWER_STATE_IN_TXS2_0(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_0_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_0_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_0(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_0_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_0_SHIFT             10
+#       define PLL_RAMP_UP_TIME_0(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_0_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_0_SHIFT                   24
+#define PB0_PIF_PWRDOWN_1                                 0x13
+#       define PLL_POWER_STATE_IN_TXS2_1(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_1_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_1_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_1(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_1_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_1_SHIFT             10
+#       define PLL_RAMP_UP_TIME_1(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_1_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_1_SHIFT                   24
+/* PIF PHY1 indirect regs */
+#define PB1_PIF_CNTL                                      0x10
+#define PB1_PIF_PAIRING                                   0x11
+#define PB1_PIF_PWRDOWN_0                                 0x12
+#define PB1_PIF_PWRDOWN_1                                 0x13
+/* PCIE PORT indirect regs */
+#define PCIE_LC_CNTL                                      0xa0
+#       define LC_L0S_INACTIVITY(x)                       ((x) << 8)
+#       define LC_L0S_INACTIVITY_MASK                     (0xf << 8)
+#       define LC_L0S_INACTIVITY_SHIFT                    8
+#       define LC_L1_INACTIVITY(x)                        ((x) << 12)
+#       define LC_L1_INACTIVITY_MASK                      (0xf << 12)
+#       define LC_L1_INACTIVITY_SHIFT                     12
+#       define LC_PMI_TO_L1_DIS                           (1 << 16)
+#       define LC_ASPM_TO_L1_DIS                          (1 << 24)
 #define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
 #define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
 #       define LC_LINK_WIDTH_SHIFT                        0
@@ -1012,6 +1393,9 @@
 #       define LC_SHORT_RECONFIG_EN                       (1 << 11)
 #       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
 #       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#       define LC_DYN_LANES_PWR_STATE(x)                  ((x) << 21)
+#       define LC_DYN_LANES_PWR_STATE_MASK                (0x3 << 21)
+#       define LC_DYN_LANES_PWR_STATE_SHIFT               21
 #define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
 #       define LC_GEN2_EN_STRAP                           (1 << 0)
 #       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
@@ -1020,6 +1404,9 @@
 #       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
 #       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
 #       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_HW_VOLTAGE_IF_CONTROL(x)                ((x) << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_MASK              (3 << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_SHIFT             12
 #       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
 #       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
 #       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 84583302b081..f30127cb30ef 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -33,6 +33,135 @@
 #include "atom.h"
 #include "ni_reg.h"
 #include "cayman_blit_shaders.h"
+#include "radeon_ucode.h"
+#include "clearstate_cayman.h"
+
+static u32 tn_rlc_save_restore_register_list[] =
+{
+        0x98fc,
+        0x98f0,
+        0x9834,
+        0x9838,
+        0x9870,
+        0x9874,
+        0x8a14,
+        0x8b24,
+        0x8bcc,
+        0x8b10,
+        0x8c30,
+        0x8d00,
+        0x8d04,
+        0x8c00,
+        0x8c04,
+        0x8c10,
+        0x8c14,
+        0x8d8c,
+        0x8cf0,
+        0x8e38,
+        0x9508,
+        0x9688,
+        0x9608,
+        0x960c,
+        0x9610,
+        0x9614,
+        0x88c4,
+        0x8978,
+        0x88d4,
+        0x900c,
+        0x9100,
+        0x913c,
+        0x90e8,
+        0x9354,
+        0xa008,
+        0x98f8,
+        0x9148,
+        0x914c,
+        0x3f94,
+        0x98f4,
+        0x9b7c,
+        0x3f8c,
+        0x8950,
+        0x8954,
+        0x8a18,
+        0x8b28,
+        0x9144,
+        0x3f90,
+        0x915c,
+        0x9160,
+        0x9178,
+        0x917c,
+        0x9180,
+        0x918c,
+        0x9190,
+        0x9194,
+        0x9198,
+        0x919c,
+        0x91a8,
+        0x91ac,
+        0x91b0,
+        0x91b4,
+        0x91b8,
+        0x91c4,
+        0x91c8,
+        0x91cc,
+        0x91d0,
+        0x91d4,
+        0x91e0,
+        0x91e4,
+        0x91ec,
+        0x91f0,
+        0x91f4,
+        0x9200,
+        0x9204,
+        0x929c,
+        0x8030,
+        0x9150,
+        0x9a60,
+        0x920c,
+        0x9210,
+        0x9228,
+        0x922c,
+        0x9244,
+        0x9248,
+        0x91e8,
+        0x9294,
+        0x9208,
+        0x9224,
+        0x9240,
+        0x9220,
+        0x923c,
+        0x9258,
+        0x9744,
+        0xa200,
+        0xa204,
+        0xa208,
+        0xa20c,
+        0x8d58,
+        0x9030,
+        0x9034,
+        0x9038,
+        0x903c,
+        0x9040,
+        0x9654,
+        0x897c,
+        0xa210,
+        0xa214,
+        0x9868,
+        0xa02c,
+        0x9664,
+        0x9698,
+        0x949c,
+        0x8e10,
+        0x8e18,
+        0x8c50,
+        0x8c58,
+        0x8c60,
+        0x8c68,
+        0x89b4,
+        0x9830,
+        0x802c,
+};
+static u32 tn_rlc_save_restore_register_list_size = ARRAY_SIZE(tn_rlc_save_restore_register_list);
 
 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
@@ -44,36 +173,29 @@ extern void evergreen_irq_suspend(struct radeon_device *rdev);
 extern int evergreen_mc_init(struct radeon_device *rdev);
 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
-extern void si_rlc_fini(struct radeon_device *rdev);
-extern int si_rlc_init(struct radeon_device *rdev);
-
-#define EVERGREEN_PFP_UCODE_SIZE 1120
-#define EVERGREEN_PM4_UCODE_SIZE 1376
-#define EVERGREEN_RLC_UCODE_SIZE 768
-#define BTC_MC_UCODE_SIZE 6024
-
-#define CAYMAN_PFP_UCODE_SIZE 2176
-#define CAYMAN_PM4_UCODE_SIZE 2176
-#define CAYMAN_RLC_UCODE_SIZE 1024
-#define CAYMAN_MC_UCODE_SIZE 6037
-
-#define ARUBA_RLC_UCODE_SIZE 1536
+extern void evergreen_program_aspm(struct radeon_device *rdev);
+extern void sumo_rlc_fini(struct radeon_device *rdev);
+extern int sumo_rlc_init(struct radeon_device *rdev);
 
 /* Firmware Names */
 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
+MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
+MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
+MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
+MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
@@ -566,6 +688,7 @@ int ni_init_microcode(struct radeon_device *rdev)
         const char *chip_name;
         const char *rlc_chip_name;
         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
+        size_t smc_req_size = 0;
         char fw_name[30];
         int err;
 
@@ -586,6 +709,7 @@ int ni_init_microcode(struct radeon_device *rdev)
                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_TURKS:
                 chip_name = "TURKS";
@@ -594,6 +718,7 @@ int ni_init_microcode(struct radeon_device *rdev)
                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_CAICOS:
                 chip_name = "CAICOS";
@@ -602,6 +727,7 @@ int ni_init_microcode(struct radeon_device *rdev)
                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_CAYMAN:
                 chip_name = "CAYMAN";
@@ -610,6 +736,7 @@ int ni_init_microcode(struct radeon_device *rdev)
                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_ARUBA:
                 chip_name = "ARUBA";
@@ -672,6 +799,20 @@ int ni_init_microcode(struct radeon_device *rdev)
                         err = -EINVAL;
                 }
         }
+
+        if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
+                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
+                err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
+                if (err)
+                        goto out;
+                if (rdev->smc_fw->size != smc_req_size) {
+                        printk(KERN_ERR
+                               "ni_mc: Bogus length %zu in firmware \"%s\"\n",
+                               rdev->mc_fw->size, fw_name);
+                        err = -EINVAL;
+                }
+        }
+
 out:
         platform_device_unregister(pdev);
 
@@ -692,6 +833,14 @@ out:
         return err;
 }
 
+int tn_get_temp(struct radeon_device *rdev)
+{
+        u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
+        int actual_temp = (temp / 8) - 49;
+
+        return actual_temp * 1000;
+}
+
 /*
  * Core functions
  */
@@ -1027,6 +1176,16 @@ static void cayman_gpu_init(struct radeon_device *rdev)
         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
 
         udelay(50);
+
+        /* set clockgating golden values on TN */
+        if (rdev->family == CHIP_ARUBA) {
+                tmp = RREG32_CG(CG_CGTT_LOCAL_0);
+                tmp &= ~0x00380000;
+                WREG32_CG(CG_CGTT_LOCAL_0, tmp);
+                tmp = RREG32_CG(CG_CGTT_LOCAL_1);
+                tmp &= ~0x0e000000;
+                WREG32_CG(CG_CGTT_LOCAL_1, tmp);
+        }
 }
 
 /*
@@ -1928,6 +2087,8 @@ static int cayman_startup(struct radeon_device *rdev)
 
         /* enable pcie gen2 link */
         evergreen_pcie_gen2_enable(rdev);
+        /* enable aspm */
+        evergreen_program_aspm(rdev);
 
         if (rdev->flags & RADEON_IS_IGP) {
                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
@@ -1972,7 +2133,10 @@ static int cayman_startup(struct radeon_device *rdev)
 
         /* allocate rlc buffers */
         if (rdev->flags & RADEON_IS_IGP) {
-                r = si_rlc_init(rdev);
+                rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
+                rdev->rlc.reg_list_size = tn_rlc_save_restore_register_list_size;
+                rdev->rlc.cs_data = cayman_cs_data;
+                r = sumo_rlc_init(rdev);
                 if (r) {
                         DRM_ERROR("Failed to init rlc BOs!\n");
                         return r;
@@ -2229,7 +2393,7 @@ int cayman_init(struct radeon_device *rdev)
                 cayman_dma_fini(rdev);
                 r600_irq_fini(rdev);
                 if (rdev->flags & RADEON_IS_IGP)
-                        si_rlc_fini(rdev);
+                        sumo_rlc_fini(rdev);
                 radeon_wb_fini(rdev);
                 radeon_ib_pool_fini(rdev);
                 radeon_vm_manager_fini(rdev);
@@ -2260,7 +2424,7 @@ void cayman_fini(struct radeon_device *rdev)
         cayman_dma_fini(rdev);
         r600_irq_fini(rdev);
         if (rdev->flags & RADEON_IS_IGP)
-                si_rlc_fini(rdev);
+                sumo_rlc_fini(rdev);
         radeon_wb_fini(rdev);
         radeon_vm_manager_fini(rdev);
         radeon_ib_pool_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/ni_dpm.c b/drivers/gpu/drm/radeon/ni_dpm.c
new file mode 100644
index 000000000000..a4cb99c2da85
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni_dpm.c
@@ -0,0 +1,4332 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "nid.h"
+#include "r600_dpm.h"
+#include "ni_dpm.h"
+#include "atom.h"
+#include <linux/math64.h>
+#include <linux/seq_file.h>
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define SMC_RAM_END 0xC000
+
+static const struct ni_cac_weights cac_weights_cayman_xt =
+{
+        0x15,
+        0x2,
+        0x19,
+        0x2,
+        0x8,
+        0x14,
+        0x2,
+        0x16,
+        0xE,
+        0x17,
+        0x13,
+        0x2B,
+        0x10,
+        0x7,
+        0x5,
+        0x5,
+        0x5,
+        0x2,
+        0x3,
+        0x9,
+        0x10,
+        0x10,
+        0x2B,
+        0xA,
+        0x9,
+        0x4,
+        0xD,
+        0xD,
+        0x3E,
+        0x18,
+        0x14,
+        0,
+        0x3,
+        0x3,
+        0x5,
+        0,
+        0x2,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0x1CC,
+        0,
+        0x164,
+        1,
+        1,
+        1,
+        1,
+        12,
+        12,
+        12,
+        0x12,
+        0x1F,
+        132,
+        5,
+        7,
+        0,
+        { 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0 },
+        true
+};
+
+static const struct ni_cac_weights cac_weights_cayman_pro =
+{
+        0x16,
+        0x4,
+        0x10,
+        0x2,
+        0xA,
+        0x16,
+        0x2,
+        0x18,
+        0x10,
+        0x1A,
+        0x16,
+        0x2D,
+        0x12,
+        0xA,
+        0x6,
+        0x6,
+        0x6,
+        0x2,
+        0x4,
+        0xB,
+        0x11,
+        0x11,
+        0x2D,
+        0xC,
+        0xC,
+        0x7,
+        0x10,
+        0x10,
+        0x3F,
+        0x1A,
+        0x16,
+        0,
+        0x7,
+        0x4,
+        0x6,
+        1,
+        0x2,
+        0x1,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0x30,
+        0,
+        0x1CF,
+        0,
+        0x166,
+        1,
+        1,
+        1,
+        1,
+        12,
+        12,
+        12,
+        0x15,
+        0x1F,
+        132,
+        6,
+        6,
+        0,
+        { 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0 },
+        true
+};
+
+static const struct ni_cac_weights cac_weights_cayman_le =
+{
+        0x7,
+        0xE,
+        0x1,
+        0xA,
+        0x1,
+        0x3F,
+        0x2,
+        0x18,
+        0x10,
+        0x1A,
+        0x1,
+        0x3F,
+        0x1,
+        0xE,
+        0x6,
+        0x6,
+        0x6,
+        0x2,
+        0x4,
+        0x9,
+        0x1A,
+        0x1A,
+        0x2C,
+        0xA,
+        0x11,
+        0x8,
+        0x19,
+        0x19,
+        0x1,
+        0x1,
+        0x1A,
+        0,
+        0x8,
+        0x5,
+        0x8,
+        0x1,
+        0x3,
+        0x1,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0x38,
+        0x38,
+        0x239,
+        0x3,
+        0x18A,
+        1,
+        1,
+        1,
+        1,
+        12,
+        12,
+        12,
+        0x15,
+        0x22,
+        132,
+        6,
+        6,
+        0,
+        { 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0 },
+        true
+};
+
+#define NISLANDS_MGCG_SEQUENCE  300
+
+static const u32 cayman_cgcg_cgls_default[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
+
+static const u32 cayman_cgcg_cgls_disable[] =
+{
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00000644, 0x000f7902, 0x001f4180,
+        0x00000644, 0x000f3802, 0x001f4180
+};
+#define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
+
+static const u32 cayman_cgcg_cgls_enable[] =
+{
+        0x00000644, 0x000f7882, 0x001f4080,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000020, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000021, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000022, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000023, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000024, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000025, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000026, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000027, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000028, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000029, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002a, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x0000002b, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff
+};
+#define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
+
+static const u32 cayman_mgcg_default[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00003fc4, 0xc0000000, 0xffffffff,
+        0x00005448, 0x00000100, 0xffffffff,
+        0x000055e4, 0x00000100, 0xffffffff,
+        0x0000160c, 0x00000100, 0xffffffff,
+        0x00008984, 0x06000100, 0xffffffff,
+        0x0000c164, 0x00000100, 0xffffffff,
+        0x00008a18, 0x00000100, 0xffffffff,
+        0x0000897c, 0x06000100, 0xffffffff,
+        0x00008b28, 0x00000100, 0xffffffff,
+        0x00009144, 0x00800200, 0xffffffff,
+        0x00009a60, 0x00000100, 0xffffffff,
+        0x00009868, 0x00000100, 0xffffffff,
+        0x00008d58, 0x00000100, 0xffffffff,
+        0x00009510, 0x00000100, 0xffffffff,
+        0x0000949c, 0x00000100, 0xffffffff,
+        0x00009654, 0x00000100, 0xffffffff,
+        0x00009030, 0x00000100, 0xffffffff,
+        0x00009034, 0x00000100, 0xffffffff,
+        0x00009038, 0x00000100, 0xffffffff,
+        0x0000903c, 0x00000100, 0xffffffff,
+        0x00009040, 0x00000100, 0xffffffff,
+        0x0000a200, 0x00000100, 0xffffffff,
+        0x0000a204, 0x00000100, 0xffffffff,
+        0x0000a208, 0x00000100, 0xffffffff,
+        0x0000a20c, 0x00000100, 0xffffffff,
+        0x00009744, 0x00000100, 0xffffffff,
+        0x00003f80, 0x00000100, 0xffffffff,
+        0x0000a210, 0x00000100, 0xffffffff,
+        0x0000a214, 0x00000100, 0xffffffff,
+        0x000004d8, 0x00000100, 0xffffffff,
+        0x00009664, 0x00000100, 0xffffffff,
+        0x00009698, 0x00000100, 0xffffffff,
+        0x000004d4, 0x00000200, 0xffffffff,
+        0x000004d0, 0x00000000, 0xffffffff,
+        0x000030cc, 0x00000104, 0xffffffff,
+        0x0000d0c0, 0x00000100, 0xffffffff,
+        0x0000d8c0, 0x00000100, 0xffffffff,
+        0x0000802c, 0x40000000, 0xffffffff,
+        0x00003fc4, 0x40000000, 0xffffffff,
+        0x0000915c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091b0, 0x00070000, 0xffffffff,
+        0x000091b4, 0x00030002, 0xffffffff,
+        0x000091b8, 0x00050004, 0xffffffff,
+        0x000091c4, 0x00010006, 0xffffffff,
+        0x000091c8, 0x00090008, 0xffffffff,
+        0x000091cc, 0x00070000, 0xffffffff,
+        0x000091d0, 0x00030002, 0xffffffff,
+        0x000091d4, 0x00050004, 0xffffffff,
+        0x000091e0, 0x00010006, 0xffffffff,
+        0x000091e4, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x000091ec, 0x00070000, 0xffffffff,
+        0x000091f0, 0x00030002, 0xffffffff,
+        0x000091f4, 0x00050004, 0xffffffff,
+        0x00009200, 0x00010006, 0xffffffff,
+        0x00009204, 0x00090008, 0xffffffff,
+        0x00009208, 0x00070000, 0xffffffff,
+        0x0000920c, 0x00030002, 0xffffffff,
+        0x00009210, 0x00050004, 0xffffffff,
+        0x0000921c, 0x00010006, 0xffffffff,
+        0x00009220, 0x00090008, 0xffffffff,
+        0x00009224, 0x00070000, 0xffffffff,
+        0x00009228, 0x00030002, 0xffffffff,
+        0x0000922c, 0x00050004, 0xffffffff,
+        0x00009238, 0x00010006, 0xffffffff,
+        0x0000923c, 0x00090008, 0xffffffff,
+        0x00009240, 0x00070000, 0xffffffff,
+        0x00009244, 0x00030002, 0xffffffff,
+        0x00009248, 0x00050004, 0xffffffff,
+        0x00009254, 0x00010006, 0xffffffff,
+        0x00009258, 0x00090008, 0xffffffff,
+        0x0000925c, 0x00070000, 0xffffffff,
+        0x00009260, 0x00030002, 0xffffffff,
+        0x00009264, 0x00050004, 0xffffffff,
+        0x00009270, 0x00010006, 0xffffffff,
+        0x00009274, 0x00090008, 0xffffffff,
+        0x00009278, 0x00070000, 0xffffffff,
+        0x0000927c, 0x00030002, 0xffffffff,
+        0x00009280, 0x00050004, 0xffffffff,
+        0x0000928c, 0x00010006, 0xffffffff,
+        0x00009290, 0x00090008, 0xffffffff,
+        0x000092a8, 0x00070000, 0xffffffff,
+        0x000092ac, 0x00030002, 0xffffffff,
+        0x000092b0, 0x00050004, 0xffffffff,
+        0x000092bc, 0x00010006, 0xffffffff,
+        0x000092c0, 0x00090008, 0xffffffff,
+        0x000092c4, 0x00070000, 0xffffffff,
+        0x000092c8, 0x00030002, 0xffffffff,
+        0x000092cc, 0x00050004, 0xffffffff,
+        0x000092d8, 0x00010006, 0xffffffff,
+        0x000092dc, 0x00090008, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff,
+        0x0000802c, 0x40010000, 0xffffffff,
+        0x00003fc4, 0x40010000, 0xffffffff,
+        0x0000915c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091b0, 0x00070000, 0xffffffff,
+        0x000091b4, 0x00030002, 0xffffffff,
+        0x000091b8, 0x00050004, 0xffffffff,
+        0x000091c4, 0x00010006, 0xffffffff,
+        0x000091c8, 0x00090008, 0xffffffff,
+        0x000091cc, 0x00070000, 0xffffffff,
+        0x000091d0, 0x00030002, 0xffffffff,
+        0x000091d4, 0x00050004, 0xffffffff,
+        0x000091e0, 0x00010006, 0xffffffff,
+        0x000091e4, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x000091ec, 0x00070000, 0xffffffff,
+        0x000091f0, 0x00030002, 0xffffffff,
+        0x000091f4, 0x00050004, 0xffffffff,
+        0x00009200, 0x00010006, 0xffffffff,
+        0x00009204, 0x00090008, 0xffffffff,
+        0x00009208, 0x00070000, 0xffffffff,
+        0x0000920c, 0x00030002, 0xffffffff,
+        0x00009210, 0x00050004, 0xffffffff,
+        0x0000921c, 0x00010006, 0xffffffff,
+        0x00009220, 0x00090008, 0xffffffff,
+        0x00009224, 0x00070000, 0xffffffff,
+        0x00009228, 0x00030002, 0xffffffff,
+        0x0000922c, 0x00050004, 0xffffffff,
+        0x00009238, 0x00010006, 0xffffffff,
+        0x0000923c, 0x00090008, 0xffffffff,
+        0x00009240, 0x00070000, 0xffffffff,
+        0x00009244, 0x00030002, 0xffffffff,
+        0x00009248, 0x00050004, 0xffffffff,
+        0x00009254, 0x00010006, 0xffffffff,
+        0x00009258, 0x00090008, 0xffffffff,
+        0x0000925c, 0x00070000, 0xffffffff,
+        0x00009260, 0x00030002, 0xffffffff,
+        0x00009264, 0x00050004, 0xffffffff,
+        0x00009270, 0x00010006, 0xffffffff,
+        0x00009274, 0x00090008, 0xffffffff,
+        0x00009278, 0x00070000, 0xffffffff,
+        0x0000927c, 0x00030002, 0xffffffff,
+        0x00009280, 0x00050004, 0xffffffff,
+        0x0000928c, 0x00010006, 0xffffffff,
+        0x00009290, 0x00090008, 0xffffffff,
+        0x000092a8, 0x00070000, 0xffffffff,
+        0x000092ac, 0x00030002, 0xffffffff,
+        0x000092b0, 0x00050004, 0xffffffff,
+        0x000092bc, 0x00010006, 0xffffffff,
+        0x000092c0, 0x00090008, 0xffffffff,
+        0x000092c4, 0x00070000, 0xffffffff,
+        0x000092c8, 0x00030002, 0xffffffff,
+        0x000092cc, 0x00050004, 0xffffffff,
+        0x000092d8, 0x00010006, 0xffffffff,
+        0x000092dc, 0x00090008, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff,
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00003fc4, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000010, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000011, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000012, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000013, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000014, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000015, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000016, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000017, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000018, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000019, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001a, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x0000001b, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff
+};
+#define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
+
+static const u32 cayman_mgcg_disable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xffffffff,
+        0x00009150, 0x00600000, 0xffffffff
+};
+#define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
+
+static const u32 cayman_mgcg_enable[] =
+{
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x000008f8, 0x00000002, 0xffffffff,
+        0x000008fc, 0x00600000, 0xffffffff,
+        0x000008f8, 0x00000003, 0xffffffff,
+        0x000008fc, 0x00000000, 0xffffffff,
+        0x00009150, 0x96944200, 0xffffffff
+};
+
+#define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
+
+#define NISLANDS_SYSLS_SEQUENCE  100
+
+static const u32 cayman_sysls_default[] =
+{
+        /* Register,   Value,     Mask bits */
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x0000d8bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x00002f50, 0x00000404, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00008dfc, 0x00000000, 0xffffffff
+};
+#define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
+
+static const u32 cayman_sysls_disable[] =
+{
+        /* Register,   Value,     Mask bits */
+        0x0000d0c0, 0x00000000, 0xffffffff,
+        0x0000d8c0, 0x00000000, 0xffffffff,
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x0000d8bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x00041401, 0xffffffff,
+        0x0000264c, 0x00040400, 0xffffffff,
+        0x00002648, 0x00040400, 0xffffffff,
+        0x00002650, 0x00040400, 0xffffffff,
+        0x000020b8, 0x00040400, 0xffffffff,
+        0x000020bc, 0x00040400, 0xffffffff,
+        0x000020c0, 0x00040c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680000, 0xffffffff,
+        0x00002f50, 0x00000404, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x000064ec, 0x00007ffd, 0xffffffff,
+        0x00000c7c, 0x0000ff00, 0xffffffff,
+        0x00008dfc, 0x0000007f, 0xffffffff
+};
+#define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
+
+static const u32 cayman_sysls_enable[] =
+{
+        /* Register,   Value,     Mask bits */
+        0x000055e8, 0x00000001, 0xffffffff,
+        0x0000d0bc, 0x00000100, 0xffffffff,
+        0x0000d8bc, 0x00000100, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x00002f50, 0x00000903, 0xffffffff,
+        0x000004c8, 0x00000000, 0xffffffff,
+        0x000064ec, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00008dfc, 0x00000000, 0xffffffff
+};
+#define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
+
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
+struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
+
+struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
+{
+        struct ni_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+struct ni_ps *ni_get_ps(struct radeon_ps *rps)
+{
+        struct ni_ps *ps = rps->ps_priv;
+
+        return ps;
+}
+
+static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
+                                                     u16 v, s32 t,
+                                                     u32 ileakage,
+                                                     u32 *leakage)
+{
+        s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
+
+        i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
+        vddc = div64_s64(drm_int2fixp(v), 1000);
+        temperature = div64_s64(drm_int2fixp(t), 1000);
+
+        kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
+                          drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
+        kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
+                          drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
+
+        leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
+
+        *leakage = drm_fixp2int(leakage_w * 1000);
+}
+
+static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
+                                             const struct ni_leakage_coeffients *coeff,
+                                             u16 v,
+                                             s32 t,
+                                             u32 i_leakage,
+                                             u32 *leakage)
+{
+        ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
+}
+
+static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
+                                        struct radeon_ps *rps)
+{
+        struct ni_ps *ps = ni_get_ps(rps);
+        struct radeon_clock_and_voltage_limits *max_limits;
+        bool disable_mclk_switching;
+        u32 mclk, sclk;
+        u16 vddc, vddci;
+        int i;
+
+        if (rdev->pm.dpm.new_active_crtc_count > 1)
+                disable_mclk_switching = true;
+        else
+                disable_mclk_switching = false;
+
+        if (rdev->pm.dpm.ac_power)
+                max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
+        else
+                max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
+
+        if (rdev->pm.dpm.ac_power == false) {
+                for (i = 0; i < ps->performance_level_count; i++) {
+                        if (ps->performance_levels[i].mclk > max_limits->mclk)
+                                ps->performance_levels[i].mclk = max_limits->mclk;
+                        if (ps->performance_levels[i].sclk > max_limits->sclk)
+                                ps->performance_levels[i].sclk = max_limits->sclk;
+                        if (ps->performance_levels[i].vddc > max_limits->vddc)
+                                ps->performance_levels[i].vddc = max_limits->vddc;
+                        if (ps->performance_levels[i].vddci > max_limits->vddci)
+                                ps->performance_levels[i].vddci = max_limits->vddci;
+                }
+        }
+
+        /* XXX validate the min clocks required for display */
+
+        if (disable_mclk_switching) {
+                mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
+                sclk = ps->performance_levels[0].sclk;
+                vddc = ps->performance_levels[0].vddc;
+                vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
+        } else {
+                sclk = ps->performance_levels[0].sclk;
+                mclk = ps->performance_levels[0].mclk;
+                vddc = ps->performance_levels[0].vddc;
+                vddci = ps->performance_levels[0].vddci;
+        }
+
+        /* adjusted low state */
+        ps->performance_levels[0].sclk = sclk;
+        ps->performance_levels[0].mclk = mclk;
+        ps->performance_levels[0].vddc = vddc;
+        ps->performance_levels[0].vddci = vddci;
+
+        btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
+                                  &ps->performance_levels[0].sclk,
+                                  &ps->performance_levels[0].mclk);
+
+        for (i = 1; i < ps->performance_level_count; i++) {
+                if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
+                        ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
+                if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
+                        ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
+        }
+
+        if (disable_mclk_switching) {
+                mclk = ps->performance_levels[0].mclk;
+                for (i = 1; i < ps->performance_level_count; i++) {
+                        if (mclk < ps->performance_levels[i].mclk)
+                                mclk = ps->performance_levels[i].mclk;
+                }
+                for (i = 0; i < ps->performance_level_count; i++) {
+                        ps->performance_levels[i].mclk = mclk;
+                        ps->performance_levels[i].vddci = vddci;
+                }
+        } else {
+                for (i = 1; i < ps->performance_level_count; i++) {
+                        if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
+                                ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
+                        if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
+                                ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
+                }
+        }
+
+        for (i = 1; i < ps->performance_level_count; i++)
+                btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
+                                          &ps->performance_levels[i].sclk,
+                                          &ps->performance_levels[i].mclk);
+
+        for (i = 0; i < ps->performance_level_count; i++)
+                btc_adjust_clock_combinations(rdev, max_limits,
+                                              &ps->performance_levels[i]);
+
+        for (i = 0; i < ps->performance_level_count; i++) {
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+                                                   ps->performance_levels[i].sclk,
+                                                   max_limits->vddc,  &ps->performance_levels[i].vddc);
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+                                                   ps->performance_levels[i].mclk,
+                                                   max_limits->vddci, &ps->performance_levels[i].vddci);
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+                                                   ps->performance_levels[i].mclk,
+                                                   max_limits->vddc,  &ps->performance_levels[i].vddc);
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
+                                                   rdev->clock.current_dispclk,
+                                                   max_limits->vddc,  &ps->performance_levels[i].vddc);
+        }
+
+        for (i = 0; i < ps->performance_level_count; i++) {
+                btc_apply_voltage_delta_rules(rdev,
+                                              max_limits->vddc, max_limits->vddci,
+                                              &ps->performance_levels[i].vddc,
+                                              &ps->performance_levels[i].vddci);
+        }
+
+        ps->dc_compatible = true;
+        for (i = 0; i < ps->performance_level_count; i++) {
+                if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
+                        ps->dc_compatible = false;
+
+                if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
+                        ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
+        }
+}
+
+static void ni_cg_clockgating_default(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *ps = NULL;
+
+        ps = (const u32 *)&cayman_cgcg_cgls_default;
+        count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
+
+        btc_program_mgcg_hw_sequence(rdev, ps, count);
+}
+
+static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
+                                      bool enable)
+{
+        u32 count;
+        const u32 *ps = NULL;
+
+        if (enable) {
+                ps = (const u32 *)&cayman_cgcg_cgls_enable;
+                count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
+        } else {
+                ps = (const u32 *)&cayman_cgcg_cgls_disable;
+                count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
+        }
+
+        btc_program_mgcg_hw_sequence(rdev, ps, count);
+}
+
+static void ni_mg_clockgating_default(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *ps = NULL;
+
+        ps = (const u32 *)&cayman_mgcg_default;
+        count = CAYMAN_MGCG_DEFAULT_LENGTH;
+
+        btc_program_mgcg_hw_sequence(rdev, ps, count);
+}
+
+static void ni_mg_clockgating_enable(struct radeon_device *rdev,
+                                     bool enable)
+{
+        u32 count;
+        const u32 *ps = NULL;
+
+        if (enable) {
+                ps = (const u32 *)&cayman_mgcg_enable;
+                count = CAYMAN_MGCG_ENABLE_LENGTH;
+        } else {
+                ps = (const u32 *)&cayman_mgcg_disable;
+                count = CAYMAN_MGCG_DISABLE_LENGTH;
+        }
+
+        btc_program_mgcg_hw_sequence(rdev, ps, count);
+}
+
+static void ni_ls_clockgating_default(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *ps = NULL;
+
+        ps = (const u32 *)&cayman_sysls_default;
+        count = CAYMAN_SYSLS_DEFAULT_LENGTH;
+
+        btc_program_mgcg_hw_sequence(rdev, ps, count);
+}
+
+static void ni_ls_clockgating_enable(struct radeon_device *rdev,
+                                     bool enable)
+{
+        u32 count;
+        const u32 *ps = NULL;
+
+        if (enable) {
+                ps = (const u32 *)&cayman_sysls_enable;
+                count = CAYMAN_SYSLS_ENABLE_LENGTH;
+        } else {
+                ps = (const u32 *)&cayman_sysls_disable;
+                count = CAYMAN_SYSLS_DISABLE_LENGTH;
+        }
+
+        btc_program_mgcg_hw_sequence(rdev, ps, count);
+
+}
+
+static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
+                                                             struct radeon_clock_voltage_dependency_table *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 i;
+
+        if (table) {
+                for (i = 0; i < table->count; i++) {
+                        if (0xff01 == table->entries[i].v) {
+                                if (pi->max_vddc == 0)
+                                        return -EINVAL;
+                                table->entries[i].v = pi->max_vddc;
+                        }
+                }
+        }
+        return 0;
+}
+
+static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
+{
+        int ret = 0;
+
+        ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
+                                                                &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
+
+        ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
+                                                                &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
+        return ret;
+}
+
+static void ni_stop_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
+}
+
+#if 0
+static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
+                                        bool ac_power)
+{
+        if (ac_power)
+                return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
+                        0 : -EINVAL;
+
+        return 0;
+}
+#endif
+
+static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
+                                                      PPSMC_Msg msg, u32 parameter)
+{
+        WREG32(SMC_SCRATCH0, parameter);
+        return rv770_send_msg_to_smc(rdev, msg);
+}
+
+static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static int ni_unrestrict_performance_levels_after_switch(struct radeon_device *rdev)
+{
+        if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static void ni_stop_smc(struct radeon_device *rdev)
+{
+        u32 tmp;
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
+                if (tmp != 1)
+                        break;
+                udelay(1);
+        }
+
+        udelay(100);
+
+        r7xx_stop_smc(rdev);
+}
+
+static int ni_process_firmware_header(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        pi->state_table_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        pi->soft_regs_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        eg_pi->mc_reg_table_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        ni_pi->fan_table_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        ni_pi->arb_table_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        ni_pi->cac_table_start = (u16)tmp;
+
+        ret = rv770_read_smc_sram_dword(rdev,
+                                        NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                        NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
+                                        &tmp, pi->sram_end);
+
+        if (ret)
+                return ret;
+
+        ni_pi->spll_table_start = (u16)tmp;
+
+
+        return ret;
+}
+
+static void ni_read_clock_registers(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+
+        ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
+        ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
+        ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
+        ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
+        ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
+        ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
+        ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
+        ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
+        ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
+        ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
+        ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
+        ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
+        ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
+        ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
+}
+
+#if 0
+static int ni_enter_ulp_state(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (pi->gfx_clock_gating) {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+                RREG32(GB_ADDR_CONFIG);
+        }
+
+        WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
+                 ~HOST_SMC_MSG_MASK);
+
+        udelay(25000);
+
+        return 0;
+}
+#endif
+
+static void ni_program_response_times(struct radeon_device *rdev)
+{
+        u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
+        u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
+        u32 reference_clock;
+
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
+
+        voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
+        backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
+
+        if (voltage_response_time == 0)
+                voltage_response_time = 1000;
+
+        if (backbias_response_time == 0)
+                backbias_response_time = 1000;
+
+        acpi_delay_time = 15000;
+        vbi_time_out = 100000;
+
+        reference_clock = radeon_get_xclk(rdev);
+
+        vddc_dly = (voltage_response_time  * reference_clock) / 1600;
+        bb_dly   = (backbias_response_time * reference_clock) / 1600;
+        acpi_dly = (acpi_delay_time * reference_clock) / 1600;
+        vbi_dly  = (vbi_time_out * reference_clock) / 1600;
+
+        mclk_switch_limit = (460 * reference_clock) / 100;
+
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
+}
+
+static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
+                                          struct atom_voltage_table *voltage_table,
+                                          NISLANDS_SMC_STATETABLE *table)
+{
+        unsigned int i;
+
+        for (i = 0; i < voltage_table->count; i++) {
+                table->highSMIO[i] = 0;
+                table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
+        }
+}
+
+static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
+                                           NISLANDS_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        unsigned char i;
+
+        if (eg_pi->vddc_voltage_table.count) {
+                ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
+                table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
+                table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
+                        cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
+
+                for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
+                        if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
+                                table->maxVDDCIndexInPPTable = i;
+                                break;
+                        }
+                }
+        }
+
+        if (eg_pi->vddci_voltage_table.count) {
+                ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
+
+                table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
+                table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
+                        cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
+        }
+}
+
+static int ni_populate_voltage_value(struct radeon_device *rdev,
+                                     struct atom_voltage_table *table,
+                                     u16 value,
+                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        unsigned int i;
+
+        for (i = 0; i < table->count; i++) {
+                if (value <= table->entries[i].value) {
+                        voltage->index = (u8)i;
+                        voltage->value = cpu_to_be16(table->entries[i].value);
+                        break;
+                }
+        }
+
+        if (i >= table->count)
+                return -EINVAL;
+
+        return 0;
+}
+
+static void ni_populate_mvdd_value(struct radeon_device *rdev,
+                                   u32 mclk,
+                                   NISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (!pi->mvdd_control) {
+                voltage->index = eg_pi->mvdd_high_index;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+                return;
+        }
+
+        if (mclk <= pi->mvdd_split_frequency) {
+                voltage->index = eg_pi->mvdd_low_index;
+                voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
+        } else {
+                voltage->index = eg_pi->mvdd_high_index;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+        }
+}
+
+static int ni_get_std_voltage_value(struct radeon_device *rdev,
+                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage,
+                                    u16 *std_voltage)
+{
+        if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
+            ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
+                *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
+        else
+                *std_voltage = be16_to_cpu(voltage->value);
+
+        return 0;
+}
+
+static void ni_populate_std_voltage_value(struct radeon_device *rdev,
+                                          u16 value, u8 index,
+                                          NISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        voltage->index = index;
+        voltage->value = cpu_to_be16(value);
+}
+
+static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
+{
+        u32 xclk_period;
+        u32 xclk = radeon_get_xclk(rdev);
+        u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
+
+        xclk_period = (1000000000UL / xclk);
+        xclk_period /= 10000UL;
+
+        return tmp * xclk_period;
+}
+
+static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
+{
+        return (power_in_watts * scaling_factor) << 2;
+}
+
+static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
+                                          struct radeon_ps *radeon_state,
+                                          u32 near_tdp_limit)
+{
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 power_boost_limit = 0;
+        int ret;
+
+        if (ni_pi->enable_power_containment &&
+            ni_pi->use_power_boost_limit) {
+                NISLANDS_SMC_VOLTAGE_VALUE vddc;
+                u16 std_vddc_med;
+                u16 std_vddc_high;
+                u64 tmp, n, d;
+
+                if (state->performance_level_count < 3)
+                        return 0;
+
+                ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                                state->performance_levels[state->performance_level_count - 2].vddc,
+                                                &vddc);
+                if (ret)
+                        return 0;
+
+                ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
+                if (ret)
+                        return 0;
+
+                ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                                state->performance_levels[state->performance_level_count - 1].vddc,
+                                                &vddc);
+                if (ret)
+                        return 0;
+
+                ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
+                if (ret)
+                        return 0;
+
+                n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
+                d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
+                tmp = div64_u64(n, d);
+
+                if (tmp >> 32)
+                        return 0;
+                power_boost_limit = (u32)tmp;
+        }
+
+        return power_boost_limit;
+}
+
+static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
+                                            bool adjust_polarity,
+                                            u32 tdp_adjustment,
+                                            u32 *tdp_limit,
+                                            u32 *near_tdp_limit)
+{
+        if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
+                return -EINVAL;
+
+        if (adjust_polarity) {
+                *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
+                *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
+        } else {
+                *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
+                *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
+        }
+
+        return 0;
+}
+
+static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
+                                      struct radeon_ps *radeon_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+
+        if (ni_pi->enable_power_containment) {
+                NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
+                u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
+                u32 tdp_limit;
+                u32 near_tdp_limit;
+                u32 power_boost_limit;
+                int ret;
+
+                if (scaling_factor == 0)
+                        return -EINVAL;
+
+                memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
+
+                ret = ni_calculate_adjusted_tdp_limits(rdev,
+                                                       false, /* ??? */
+                                                       rdev->pm.dpm.tdp_adjustment,
+                                                       &tdp_limit,
+                                                       &near_tdp_limit);
+                if (ret)
+                        return ret;
+
+                power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
+                                                                   near_tdp_limit);
+
+                smc_table->dpm2Params.TDPLimit =
+                        cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
+                smc_table->dpm2Params.NearTDPLimit =
+                        cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
+                smc_table->dpm2Params.SafePowerLimit =
+                        cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
+                                                           scaling_factor));
+                smc_table->dpm2Params.PowerBoostLimit =
+                        cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
+
+                ret = rv770_copy_bytes_to_smc(rdev,
+                                              (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
+                                                    offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
+                                              (u8 *)(&smc_table->dpm2Params.TDPLimit),
+                                              sizeof(u32) * 4, pi->sram_end);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
+                                u32 arb_freq_src, u32 arb_freq_dest)
+{
+        u32 mc_arb_dram_timing;
+        u32 mc_arb_dram_timing2;
+        u32 burst_time;
+        u32 mc_cg_config;
+
+        switch (arb_freq_src) {
+        case MC_CG_ARB_FREQ_F0:
+                mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
+                mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+                burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
+                break;
+        case MC_CG_ARB_FREQ_F1:
+                mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
+                mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
+                burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
+                break;
+        case MC_CG_ARB_FREQ_F2:
+                mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
+                mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
+                burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
+                break;
+        case MC_CG_ARB_FREQ_F3:
+                mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
+                mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
+                burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        switch (arb_freq_dest) {
+        case MC_CG_ARB_FREQ_F0:
+                WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
+                WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
+                WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
+                break;
+        case MC_CG_ARB_FREQ_F1:
+                WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
+                WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
+                WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
+                break;
+        case MC_CG_ARB_FREQ_F2:
+                WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
+                WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
+                WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
+                break;
+        case MC_CG_ARB_FREQ_F3:
+                WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
+                WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
+                WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
+        WREG32(MC_CG_CONFIG, mc_cg_config);
+        WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
+
+        return 0;
+}
+
+static int ni_init_arb_table_index(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
+                                        &tmp, pi->sram_end);
+        if (ret)
+                return ret;
+
+        tmp &= 0x00FFFFFF;
+        tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
+
+        return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
+                                          tmp, pi->sram_end);
+}
+
+static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
+{
+        return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
+}
+
+static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
+                                        &tmp, pi->sram_end);
+        if (ret)
+                return ret;
+
+        tmp = (tmp >> 24) & 0xff;
+
+        if (tmp == MC_CG_ARB_FREQ_F0)
+                return 0;
+
+        return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
+}
+
+static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
+                                                struct rv7xx_pl *pl,
+                                                SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
+{
+        u32 dram_timing;
+        u32 dram_timing2;
+
+        arb_regs->mc_arb_rfsh_rate =
+                (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
+
+
+        radeon_atom_set_engine_dram_timings(rdev,
+                                            pl->sclk,
+                                            pl->mclk);
+
+        dram_timing = RREG32(MC_ARB_DRAM_TIMING);
+        dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+
+        arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
+        arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
+
+        return 0;
+}
+
+static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
+                                                  struct radeon_ps *radeon_state,
+                                                  unsigned int first_arb_set)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
+        int i, ret = 0;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
+                if (ret)
+                        break;
+
+                ret = rv770_copy_bytes_to_smc(rdev,
+                                              (u16)(ni_pi->arb_table_start +
+                                                    offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
+                                                    sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
+                                              (u8 *)&arb_regs,
+                                              (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
+                                              pi->sram_end);
+                if (ret)
+                        break;
+        }
+        return ret;
+}
+
+static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
+                                               struct radeon_ps *radeon_new_state)
+{
+        return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
+                                                      NISLANDS_DRIVER_STATE_ARB_INDEX);
+}
+
+static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
+                                           struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        voltage->index = eg_pi->mvdd_high_index;
+        voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+}
+
+static int ni_populate_smc_initial_state(struct radeon_device *rdev,
+                                         struct radeon_ps *radeon_initial_state,
+                                         NISLANDS_SMC_STATETABLE *table)
+{
+        struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 reg;
+        int ret;
+
+        table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
+                cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
+        table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
+                cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
+        table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
+                cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
+        table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
+                cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
+        table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
+        table->initialState.levels[0].mclk.vDLL_CNTL =
+                cpu_to_be32(ni_pi->clock_registers.dll_cntl);
+        table->initialState.levels[0].mclk.vMPLL_SS =
+                cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
+        table->initialState.levels[0].mclk.vMPLL_SS2 =
+                cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
+        table->initialState.levels[0].mclk.mclk_value =
+                cpu_to_be32(initial_state->performance_levels[0].mclk);
+
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
+                cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
+                cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
+                cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
+        table->initialState.levels[0].sclk.sclk_value =
+                cpu_to_be32(initial_state->performance_levels[0].sclk);
+        table->initialState.levels[0].arbRefreshState =
+                NISLANDS_INITIAL_STATE_ARB_INDEX;
+
+        table->initialState.levels[0].ACIndex = 0;
+
+        ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                        initial_state->performance_levels[0].vddc,
+                                        &table->initialState.levels[0].vddc);
+        if (!ret) {
+                u16 std_vddc;
+
+                ret = ni_get_std_voltage_value(rdev,
+                                               &table->initialState.levels[0].vddc,
+                                               &std_vddc);
+                if (!ret)
+                        ni_populate_std_voltage_value(rdev, std_vddc,
+                                                      table->initialState.levels[0].vddc.index,
+                                                      &table->initialState.levels[0].std_vddc);
+        }
+
+        if (eg_pi->vddci_control)
+                ni_populate_voltage_value(rdev,
+                                          &eg_pi->vddci_voltage_table,
+                                          initial_state->performance_levels[0].vddci,
+                                          &table->initialState.levels[0].vddci);
+
+        ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
+
+        reg = CG_R(0xffff) | CG_L(0);
+        table->initialState.levels[0].aT = cpu_to_be32(reg);
+
+        table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
+
+        if (pi->boot_in_gen2)
+                table->initialState.levels[0].gen2PCIE = 1;
+        else
+                table->initialState.levels[0].gen2PCIE = 0;
+
+        if (pi->mem_gddr5) {
+                table->initialState.levels[0].strobeMode =
+                        cypress_get_strobe_mode_settings(rdev,
+                                                         initial_state->performance_levels[0].mclk);
+
+                if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
+                        table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
+                else
+                        table->initialState.levels[0].mcFlags =  0;
+        }
+
+        table->initialState.levelCount = 1;
+
+        table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        table->initialState.levels[0].dpm2.MaxPS = 0;
+        table->initialState.levels[0].dpm2.NearTDPDec = 0;
+        table->initialState.levels[0].dpm2.AboveSafeInc = 0;
+        table->initialState.levels[0].dpm2.BelowSafeInc = 0;
+
+        reg = MIN_POWER_MASK | MAX_POWER_MASK;
+        table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
+
+        reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+        table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
+
+        return 0;
+}
+
+static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
+                                      NISLANDS_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
+        u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
+        u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
+        u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
+        u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
+        u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
+        u32 reg;
+        int ret;
+
+        table->ACPIState = table->initialState;
+
+        table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+        if (pi->acpi_vddc) {
+                ret = ni_populate_voltage_value(rdev,
+                                                &eg_pi->vddc_voltage_table,
+                                                pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
+                if (!ret) {
+                        u16 std_vddc;
+
+                        ret = ni_get_std_voltage_value(rdev,
+                                                       &table->ACPIState.levels[0].vddc, &std_vddc);
+                        if (!ret)
+                                ni_populate_std_voltage_value(rdev, std_vddc,
+                                                              table->ACPIState.levels[0].vddc.index,
+                                                              &table->ACPIState.levels[0].std_vddc);
+                }
+
+                if (pi->pcie_gen2) {
+                        if (pi->acpi_pcie_gen2)
+                                table->ACPIState.levels[0].gen2PCIE = 1;
+                        else
+                                table->ACPIState.levels[0].gen2PCIE = 0;
+                } else {
+                        table->ACPIState.levels[0].gen2PCIE = 0;
+                }
+        } else {
+                ret = ni_populate_voltage_value(rdev,
+                                                &eg_pi->vddc_voltage_table,
+                                                pi->min_vddc_in_table,
+                                                &table->ACPIState.levels[0].vddc);
+                if (!ret) {
+                        u16 std_vddc;
+
+                        ret = ni_get_std_voltage_value(rdev,
+                                                       &table->ACPIState.levels[0].vddc,
+                                                       &std_vddc);
+                        if (!ret)
+                                ni_populate_std_voltage_value(rdev, std_vddc,
+                                                              table->ACPIState.levels[0].vddc.index,
+                                                              &table->ACPIState.levels[0].std_vddc);
+                }
+                table->ACPIState.levels[0].gen2PCIE = 0;
+        }
+
+        if (eg_pi->acpi_vddci) {
+                if (eg_pi->vddci_control)
+                        ni_populate_voltage_value(rdev,
+                                                  &eg_pi->vddci_voltage_table,
+                                                  eg_pi->acpi_vddci,
+                                                  &table->ACPIState.levels[0].vddci);
+        }
+
+
+        mpll_ad_func_cntl &= ~PDNB;
+
+        mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
+
+        if (pi->mem_gddr5)
+                mpll_dq_func_cntl &= ~PDNB;
+        mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
+
+
+        mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
+                             MRDCKA1_RESET |
+                             MRDCKB0_RESET |
+                             MRDCKB1_RESET |
+                             MRDCKC0_RESET |
+                             MRDCKC1_RESET |
+                             MRDCKD0_RESET |
+                             MRDCKD1_RESET);
+
+        mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
+                              MRDCKA1_PDNB |
+                              MRDCKB0_PDNB |
+                              MRDCKB1_PDNB |
+                              MRDCKC0_PDNB |
+                              MRDCKC1_PDNB |
+                              MRDCKD0_PDNB |
+                              MRDCKD1_PDNB);
+
+        dll_cntl |= (MRDCKA0_BYPASS |
+                     MRDCKA1_BYPASS |
+                     MRDCKB0_BYPASS |
+                     MRDCKB1_BYPASS |
+                     MRDCKC0_BYPASS |
+                     MRDCKC1_BYPASS |
+                     MRDCKD0_BYPASS |
+                     MRDCKD1_BYPASS);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+        table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+        table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
+
+        table->ACPIState.levels[0].mclk.mclk_value = 0;
+
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
+
+        table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+        ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
+
+        if (eg_pi->dynamic_ac_timing)
+                table->ACPIState.levels[0].ACIndex = 1;
+
+        table->ACPIState.levels[0].dpm2.MaxPS = 0;
+        table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
+        table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
+        table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
+
+        reg = MIN_POWER_MASK | MAX_POWER_MASK;
+        table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
+
+        reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+        table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
+
+        return 0;
+}
+
+static int ni_init_smc_table(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        int ret;
+        struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
+        NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
+
+        memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
+
+        ni_populate_smc_voltage_tables(rdev, table);
+
+        switch (rdev->pm.int_thermal_type) {
+        case THERMAL_TYPE_NI:
+        case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
+                break;
+        case THERMAL_TYPE_NONE:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
+                break;
+        default:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
+                break;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+        if (pi->mem_gddr5)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+        ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
+        if (ret)
+                return ret;
+
+        ret = ni_populate_smc_acpi_state(rdev, table);
+        if (ret)
+                return ret;
+
+        table->driverState = table->initialState;
+
+        table->ULVState = table->initialState;
+
+        ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
+                                                     NISLANDS_INITIAL_STATE_ARB_INDEX);
+        if (ret)
+                return ret;
+
+        return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
+                                       sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
+}
+
+static int ni_calculate_sclk_params(struct radeon_device *rdev,
+                                    u32 engine_clock,
+                                    NISLANDS_SMC_SCLK_VALUE *sclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
+        u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
+        u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
+        u64 tmp;
+        u32 reference_clock = rdev->clock.spll.reference_freq;
+        u32 reference_divider;
+        u32 fbdiv;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             engine_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        reference_divider = 1 + dividers.ref_div;
+
+
+        tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
+        do_div(tmp, reference_clock);
+        fbdiv = (u32) tmp;
+
+        spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
+        spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
+        spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(2);
+
+        spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
+        spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
+        spll_func_cntl_3 |= SPLL_DITHEN;
+
+        if (pi->sclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = engine_clock * dividers.post_div;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+                        u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+                        u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
+
+                        cg_spll_spread_spectrum &= ~CLK_S_MASK;
+                        cg_spll_spread_spectrum |= CLK_S(clk_s);
+                        cg_spll_spread_spectrum |= SSEN;
+
+                        cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
+                        cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
+                }
+        }
+
+        sclk->sclk_value = engine_clock;
+        sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
+        sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
+        sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
+        sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
+        sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
+        sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
+
+        return 0;
+}
+
+static int ni_populate_sclk_value(struct radeon_device *rdev,
+                                  u32 engine_clock,
+                                  NISLANDS_SMC_SCLK_VALUE *sclk)
+{
+        NISLANDS_SMC_SCLK_VALUE sclk_tmp;
+        int ret;
+
+        ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
+        if (!ret) {
+                sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
+                sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
+                sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
+                sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
+                sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
+                sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
+                sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
+        }
+
+        return ret;
+}
+
+static int ni_init_smc_spll_table(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
+        NISLANDS_SMC_SCLK_VALUE sclk_params;
+        u32 fb_div;
+        u32 p_div;
+        u32 clk_s;
+        u32 clk_v;
+        u32 sclk = 0;
+        int i, ret;
+        u32 tmp;
+
+        if (ni_pi->spll_table_start == 0)
+                return -EINVAL;
+
+        spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
+        if (spll_table == NULL)
+                return -ENOMEM;
+
+        for (i = 0; i < 256; i++) {
+                ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
+                if (ret)
+                        break;
+
+                p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
+                fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
+                clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
+                clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
+
+                fb_div &= ~0x00001FFF;
+                fb_div >>= 1;
+                clk_v >>= 6;
+
+                if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
+                        ret = -EINVAL;
+
+                if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
+                        ret = -EINVAL;
+
+                if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
+                        ret = -EINVAL;
+
+                if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
+                        ret = -EINVAL;
+
+                if (ret)
+                        break;
+
+                tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
+                        ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
+                spll_table->freq[i] = cpu_to_be32(tmp);
+
+                tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
+                        ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
+                spll_table->ss[i] = cpu_to_be32(tmp);
+
+                sclk += 512;
+        }
+
+        if (!ret)
+                ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
+                                              sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
+
+        kfree(spll_table);
+
+        return ret;
+}
+
+static int ni_populate_mclk_value(struct radeon_device *rdev,
+                                  u32 engine_clock,
+                                  u32 memory_clock,
+                                  NISLANDS_SMC_MCLK_VALUE *mclk,
+                                  bool strobe_mode,
+                                  bool dll_state_on)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
+        u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
+        u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
+        u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
+        u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
+        struct atom_clock_dividers dividers;
+        u32 ibias;
+        u32 dll_speed;
+        int ret;
+        u32 mc_seq_misc7;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                             memory_clock, strobe_mode, &dividers);
+        if (ret)
+                return ret;
+
+        if (!strobe_mode) {
+                mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
+
+                if (mc_seq_misc7 & 0x8000000)
+                        dividers.post_div = 1;
+        }
+
+        ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
+
+        mpll_ad_func_cntl &= ~(CLKR_MASK |
+                               YCLK_POST_DIV_MASK |
+                               CLKF_MASK |
+                               CLKFRAC_MASK |
+                               IBIAS_MASK);
+        mpll_ad_func_cntl |= CLKR(dividers.ref_div);
+        mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
+        mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
+        mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
+        mpll_ad_func_cntl |= IBIAS(ibias);
+
+        if (dividers.vco_mode)
+                mpll_ad_func_cntl_2 |= VCO_MODE;
+        else
+                mpll_ad_func_cntl_2 &= ~VCO_MODE;
+
+        if (pi->mem_gddr5) {
+                mpll_dq_func_cntl &= ~(CLKR_MASK |
+                                       YCLK_POST_DIV_MASK |
+                                       CLKF_MASK |
+                                       CLKFRAC_MASK |
+                                       IBIAS_MASK);
+                mpll_dq_func_cntl |= CLKR(dividers.ref_div);
+                mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
+                mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
+                mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
+                mpll_dq_func_cntl |= IBIAS(ibias);
+
+                if (strobe_mode)
+                        mpll_dq_func_cntl &= ~PDNB;
+                else
+                        mpll_dq_func_cntl |= PDNB;
+
+                if (dividers.vco_mode)
+                        mpll_dq_func_cntl_2 |= VCO_MODE;
+                else
+                        mpll_dq_func_cntl_2 &= ~VCO_MODE;
+        }
+
+        if (pi->mclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = memory_clock * dividers.post_div;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
+                        u32 reference_clock = rdev->clock.mpll.reference_freq;
+                        u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
+                        u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
+                        u32 clk_v = ss.percentage *
+                                (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
+
+                        mpll_ss1 &= ~CLKV_MASK;
+                        mpll_ss1 |= CLKV(clk_v);
+
+                        mpll_ss2 &= ~CLKS_MASK;
+                        mpll_ss2 |= CLKS(clk_s);
+                }
+        }
+
+        dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
+                                        memory_clock);
+
+        mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
+        mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
+        if (dll_state_on)
+                mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
+                                     MRDCKA1_PDNB |
+                                     MRDCKB0_PDNB |
+                                     MRDCKB1_PDNB |
+                                     MRDCKC0_PDNB |
+                                     MRDCKC1_PDNB |
+                                     MRDCKD0_PDNB |
+                                     MRDCKD1_PDNB);
+        else
+                mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
+                                      MRDCKA1_PDNB |
+                                      MRDCKB0_PDNB |
+                                      MRDCKB1_PDNB |
+                                      MRDCKC0_PDNB |
+                                      MRDCKC1_PDNB |
+                                      MRDCKD0_PDNB |
+                                      MRDCKD1_PDNB);
+
+
+        mclk->mclk_value = cpu_to_be32(memory_clock);
+        mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+        mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
+        mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
+        mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
+
+        return 0;
+}
+
+static void ni_populate_smc_sp(struct radeon_device *rdev,
+                               struct radeon_ps *radeon_state,
+                               NISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct ni_ps *ps = ni_get_ps(radeon_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+
+        for (i = 0; i < ps->performance_level_count - 1; i++)
+                smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
+
+        smc_state->levels[ps->performance_level_count - 1].bSP =
+                cpu_to_be32(pi->psp);
+}
+
+static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
+                                         struct rv7xx_pl *pl,
+                                         NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        int ret;
+        bool dll_state_on;
+        u16 std_vddc;
+        u32 tmp = RREG32(DC_STUTTER_CNTL);
+
+        level->gen2PCIE = pi->pcie_gen2 ?
+                ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
+
+        ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
+        if (ret)
+                return ret;
+
+        level->mcFlags =  0;
+        if (pi->mclk_stutter_mode_threshold &&
+            (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
+            !eg_pi->uvd_enabled &&
+            (tmp & DC_STUTTER_ENABLE_A) &&
+            (tmp & DC_STUTTER_ENABLE_B))
+                level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
+
+        if (pi->mem_gddr5) {
+                if (pl->mclk > pi->mclk_edc_enable_threshold)
+                        level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
+                if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
+                        level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
+
+                level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
+
+                if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
+                        if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
+                            ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
+                                dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
+                        else
+                                dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
+                } else {
+                        dll_state_on = false;
+                        if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
+                                level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
+                }
+
+                ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
+                                             &level->mclk,
+                                             (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
+                                             dll_state_on);
+        } else
+                ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
+
+        if (ret)
+                return ret;
+
+        ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                        pl->vddc, &level->vddc);
+        if (ret)
+                return ret;
+
+        ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
+        if (ret)
+                return ret;
+
+        ni_populate_std_voltage_value(rdev, std_vddc,
+                                      level->vddc.index, &level->std_vddc);
+
+        if (eg_pi->vddci_control) {
+                ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
+                                                pl->vddci, &level->vddci);
+                if (ret)
+                        return ret;
+        }
+
+        ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
+
+        return ret;
+}
+
+static int ni_populate_smc_t(struct radeon_device *rdev,
+                             struct radeon_ps *radeon_state,
+                             NISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        u32 a_t;
+        u32 t_l, t_h;
+        u32 high_bsp;
+        int i, ret;
+
+        if (state->performance_level_count >= 9)
+                return -EINVAL;
+
+        if (state->performance_level_count < 2) {
+                a_t = CG_R(0xffff) | CG_L(0);
+                smc_state->levels[0].aT = cpu_to_be32(a_t);
+                return 0;
+        }
+
+        smc_state->levels[0].aT = cpu_to_be32(0);
+
+        for (i = 0; i <= state->performance_level_count - 2; i++) {
+                if (eg_pi->uvd_enabled)
+                        ret = r600_calculate_at(
+                                1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
+                                100 * R600_AH_DFLT,
+                                state->performance_levels[i + 1].sclk,
+                                state->performance_levels[i].sclk,
+                                &t_l,
+                                &t_h);
+                else
+                        ret = r600_calculate_at(
+                                1000 * (i + 1),
+                                100 * R600_AH_DFLT,
+                                state->performance_levels[i + 1].sclk,
+                                state->performance_levels[i].sclk,
+                                &t_l,
+                                &t_h);
+
+                if (ret) {
+                        t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
+                        t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
+                }
+
+                a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
+                a_t |= CG_R(t_l * pi->bsp / 20000);
+                smc_state->levels[i].aT = cpu_to_be32(a_t);
+
+                high_bsp = (i == state->performance_level_count - 2) ?
+                        pi->pbsp : pi->bsp;
+
+                a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
+                smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
+        }
+
+        return 0;
+}
+
+static int ni_populate_power_containment_values(struct radeon_device *rdev,
+                                                struct radeon_ps *radeon_state,
+                                                NISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        u32 prev_sclk;
+        u32 max_sclk;
+        u32 min_sclk;
+        int i, ret;
+        u32 tdp_limit;
+        u32 near_tdp_limit;
+        u32 power_boost_limit;
+        u8 max_ps_percent;
+
+        if (ni_pi->enable_power_containment == false)
+                return 0;
+
+        if (state->performance_level_count == 0)
+                return -EINVAL;
+
+        if (smc_state->levelCount != state->performance_level_count)
+                return -EINVAL;
+
+        ret = ni_calculate_adjusted_tdp_limits(rdev,
+                                               false, /* ??? */
+                                               rdev->pm.dpm.tdp_adjustment,
+                                               &tdp_limit,
+                                               &near_tdp_limit);
+        if (ret)
+                return ret;
+
+        power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
+
+        ret = rv770_write_smc_sram_dword(rdev,
+                                         pi->state_table_start +
+                                         offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
+                                         offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
+                                         ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
+                                         pi->sram_end);
+        if (ret)
+                power_boost_limit = 0;
+
+        smc_state->levels[0].dpm2.MaxPS = 0;
+        smc_state->levels[0].dpm2.NearTDPDec = 0;
+        smc_state->levels[0].dpm2.AboveSafeInc = 0;
+        smc_state->levels[0].dpm2.BelowSafeInc = 0;
+        smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
+
+        for (i = 1; i < state->performance_level_count; i++) {
+                prev_sclk = state->performance_levels[i-1].sclk;
+                max_sclk  = state->performance_levels[i].sclk;
+                max_ps_percent = (i != (state->performance_level_count - 1)) ?
+                        NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
+
+                if (max_sclk < prev_sclk)
+                        return -EINVAL;
+
+                if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
+                        min_sclk = max_sclk;
+                else if (1 == i)
+                        min_sclk = prev_sclk;
+                else
+                        min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
+
+                if (min_sclk < state->performance_levels[0].sclk)
+                        min_sclk = state->performance_levels[0].sclk;
+
+                if (min_sclk == 0)
+                        return -EINVAL;
+
+                smc_state->levels[i].dpm2.MaxPS =
+                        (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
+                smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
+                smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
+                smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
+                smc_state->levels[i].stateFlags |=
+                        ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
+                        PPSMC_STATEFLAG_POWERBOOST : 0;
+        }
+
+        return 0;
+}
+
+static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
+                                         struct radeon_ps *radeon_state,
+                                         NISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        u32 sq_power_throttle;
+        u32 sq_power_throttle2;
+        bool enable_sq_ramping = ni_pi->enable_sq_ramping;
+        int i;
+
+        if (state->performance_level_count == 0)
+                return -EINVAL;
+
+        if (smc_state->levelCount != state->performance_level_count)
+                return -EINVAL;
+
+        if (rdev->pm.dpm.sq_ramping_threshold == 0)
+                return -EINVAL;
+
+        if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
+                enable_sq_ramping = false;
+
+        if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
+                enable_sq_ramping = false;
+
+        if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
+                enable_sq_ramping = false;
+
+        if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
+                enable_sq_ramping = false;
+
+        if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO <= (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
+                enable_sq_ramping = false;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                sq_power_throttle  = 0;
+                sq_power_throttle2 = 0;
+
+                if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
+                    enable_sq_ramping) {
+                        sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
+                        sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
+                        sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
+                        sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
+                        sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
+                } else {
+                        sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
+                        sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+                }
+
+                smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
+                smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
+        }
+
+        return 0;
+}
+
+static int ni_enable_power_containment(struct radeon_device *rdev,
+                                       struct radeon_ps *radeon_new_state,
+                                       bool enable)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        PPSMC_Result smc_result;
+        int ret = 0;
+
+        if (ni_pi->enable_power_containment) {
+                if (enable) {
+                        if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
+                                smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
+                                if (smc_result != PPSMC_Result_OK) {
+                                        ret = -EINVAL;
+                                        ni_pi->pc_enabled = false;
+                                } else {
+                                        ni_pi->pc_enabled = true;
+                                }
+                        }
+                } else {
+                        smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
+                        if (smc_result != PPSMC_Result_OK)
+                                ret = -EINVAL;
+                        ni_pi->pc_enabled = false;
+                }
+        }
+
+        return ret;
+}
+
+static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
+                                         struct radeon_ps *radeon_state,
+                                         NISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        int i, ret;
+        u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
+
+        if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
+                smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        smc_state->levelCount = 0;
+
+        if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
+                return -EINVAL;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
+                                                    &smc_state->levels[i]);
+                smc_state->levels[i].arbRefreshState =
+                        (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
+
+                if (ret)
+                        return ret;
+
+                if (ni_pi->enable_power_containment)
+                        smc_state->levels[i].displayWatermark =
+                                (state->performance_levels[i].sclk < threshold) ?
+                                PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
+                else
+                        smc_state->levels[i].displayWatermark = (i < 2) ?
+                                PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
+
+                if (eg_pi->dynamic_ac_timing)
+                        smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
+                else
+                        smc_state->levels[i].ACIndex = 0;
+
+                smc_state->levelCount++;
+        }
+
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
+                                      cpu_to_be32(threshold / 512));
+
+        ni_populate_smc_sp(rdev, radeon_state, smc_state);
+
+        ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
+        if (ret)
+                ni_pi->enable_power_containment = false;
+
+        ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
+        if (ret)
+                ni_pi->enable_sq_ramping = false;
+
+        return ni_populate_smc_t(rdev, radeon_state, smc_state);
+}
+
+static int ni_upload_sw_state(struct radeon_device *rdev,
+                              struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u16 address = pi->state_table_start +
+                offsetof(NISLANDS_SMC_STATETABLE, driverState);
+        u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
+                ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
+        int ret;
+        NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
+
+        if (smc_state == NULL)
+                return -ENOMEM;
+
+        ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
+        if (ret)
+                goto done;
+
+        ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
+
+done:
+        kfree(smc_state);
+
+        return ret;
+}
+
+static int ni_set_mc_special_registers(struct radeon_device *rdev,
+                                       struct ni_mc_reg_table *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 i, j, k;
+        u32 temp_reg;
+
+        for (i = 0, j = table->last; i < table->last; i++) {
+                switch (table->mc_reg_address[i].s1) {
+                case MC_SEQ_MISC1 >> 2:
+                        if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+                        temp_reg = RREG32(MC_PMG_CMD_EMRS);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++)
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        ((temp_reg & 0xffff0000)) |
+                                        ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+                        j++;
+                        if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+
+                        temp_reg = RREG32(MC_PMG_CMD_MRS);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+                        for(k = 0; k < table->num_entries; k++) {
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        (temp_reg & 0xffff0000) |
+                                        (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+                                if (!pi->mem_gddr5)
+                                        table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+                        }
+                        j++;
+                        if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+                        break;
+                case MC_SEQ_RESERVE_M >> 2:
+                        temp_reg = RREG32(MC_PMG_CMD_MRS1);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++)
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        (temp_reg & 0xffff0000) |
+                                        (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+                        j++;
+                        if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        table->last = j;
+
+        return 0;
+}
+
+static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
+{
+        bool result = true;
+
+        switch (in_reg) {
+        case  MC_SEQ_RAS_TIMING >> 2:
+                *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_CAS_TIMING >> 2:
+                *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_MISC_TIMING >> 2:
+                *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_MISC_TIMING2 >> 2:
+                *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
+                break;
+        case MC_SEQ_RD_CTL_D0 >> 2:
+                *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
+                break;
+        case MC_SEQ_RD_CTL_D1 >> 2:
+                *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_D0 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_D1 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
+                break;
+        case MC_PMG_CMD_EMRS >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS1 >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+                break;
+        case MC_SEQ_PMG_TIMING >> 2:
+                *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS2 >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
+                break;
+        default:
+                result = false;
+                break;
+        }
+
+        return result;
+}
+
+static void ni_set_valid_flag(struct ni_mc_reg_table *table)
+{
+        u8 i, j;
+
+        for (i = 0; i < table->last; i++) {
+                for (j = 1; j < table->num_entries; j++) {
+                        if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
+                                table->valid_flag |= 1 << i;
+                                break;
+                        }
+                }
+        }
+}
+
+static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
+{
+        u32 i;
+        u16 address;
+
+        for (i = 0; i < table->last; i++)
+                table->mc_reg_address[i].s0 =
+                        ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
+                        address : table->mc_reg_address[i].s1;
+}
+
+static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
+                                      struct ni_mc_reg_table *ni_table)
+{
+        u8 i, j;
+
+        if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                return -EINVAL;
+        if (table->num_entries > MAX_AC_TIMING_ENTRIES)
+                return -EINVAL;
+
+        for (i = 0; i < table->last; i++)
+                ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+        ni_table->last = table->last;
+
+        for (i = 0; i < table->num_entries; i++) {
+                ni_table->mc_reg_table_entry[i].mclk_max =
+                        table->mc_reg_table_entry[i].mclk_max;
+                for (j = 0; j < table->last; j++)
+                        ni_table->mc_reg_table_entry[i].mc_data[j] =
+                                table->mc_reg_table_entry[i].mc_data[j];
+        }
+        ni_table->num_entries = table->num_entries;
+
+        return 0;
+}
+
+static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        int ret;
+        struct atom_mc_reg_table *table;
+        struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
+        u8 module_index = rv770_get_memory_module_index(rdev);
+
+        table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
+        if (!table)
+                return -ENOMEM;
+
+        WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
+        WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
+        WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
+        WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
+        WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
+        WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
+        WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
+        WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
+        WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
+        WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
+        WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
+        WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
+        WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
+
+        ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
+
+        if (ret)
+                goto init_mc_done;
+
+        ret = ni_copy_vbios_mc_reg_table(table, ni_table);
+
+        if (ret)
+                goto init_mc_done;
+
+        ni_set_s0_mc_reg_index(ni_table);
+
+        ret = ni_set_mc_special_registers(rdev, ni_table);
+
+        if (ret)
+                goto init_mc_done;
+
+        ni_set_valid_flag(ni_table);
+
+init_mc_done:
+        kfree(table);
+
+        return ret;
+}
+
+static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
+                                         SMC_NIslands_MCRegisters *mc_reg_table)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 i, j;
+
+        for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
+                if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
+                        if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                break;
+                        mc_reg_table->address[i].s0 =
+                                cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
+                        mc_reg_table->address[i].s1 =
+                                cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
+                        i++;
+                }
+        }
+        mc_reg_table->last = (u8)i;
+}
+
+
+static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
+                                    SMC_NIslands_MCRegisterSet *data,
+                                    u32 num_entries, u32 valid_flag)
+{
+        u32 i, j;
+
+        for (i = 0, j = 0; j < num_entries; j++) {
+                if (valid_flag & (1 << j)) {
+                        data->value[i] = cpu_to_be32(entry->mc_data[j]);
+                        i++;
+                }
+        }
+}
+
+static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
+                                                 struct rv7xx_pl *pl,
+                                                 SMC_NIslands_MCRegisterSet *mc_reg_table_data)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 i = 0;
+
+        for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
+                if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
+                        break;
+        }
+
+        if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
+                --i;
+
+        ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
+                                mc_reg_table_data,
+                                ni_pi->mc_reg_table.last,
+                                ni_pi->mc_reg_table.valid_flag);
+}
+
+static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
+                                           struct radeon_ps *radeon_state,
+                                           SMC_NIslands_MCRegisters *mc_reg_table)
+{
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        int i;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                ni_convert_mc_reg_table_entry_to_smc(rdev,
+                                                     &state->performance_levels[i],
+                                                     &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
+        }
+}
+
+static int ni_populate_mc_reg_table(struct radeon_device *rdev,
+                                    struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
+        SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
+
+        memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
+
+        rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
+
+        ni_populate_mc_reg_addresses(rdev, mc_reg_table);
+
+        ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
+                                             &mc_reg_table->data[0]);
+
+        ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
+                                &mc_reg_table->data[1],
+                                ni_pi->mc_reg_table.last,
+                                ni_pi->mc_reg_table.valid_flag);
+
+        ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
+
+        return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
+                                       (u8 *)mc_reg_table,
+                                       sizeof(SMC_NIslands_MCRegisters),
+                                       pi->sram_end);
+}
+
+static int ni_upload_mc_reg_table(struct radeon_device *rdev,
+                                  struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
+        SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
+        u16 address;
+
+        memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
+
+        ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
+
+        address = eg_pi->mc_reg_table_start +
+                (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
+
+        return rv770_copy_bytes_to_smc(rdev, address,
+                                       (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
+                                       sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
+                                       pi->sram_end);
+}
+
+static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
+                                                   PP_NIslands_CACTABLES *cac_tables)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 leakage = 0;
+        unsigned int i, j, table_size;
+        s32 t;
+        u32 smc_leakage, max_leakage = 0;
+        u32 scaling_factor;
+
+        table_size = eg_pi->vddc_voltage_table.count;
+
+        if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
+                table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
+
+        scaling_factor = ni_get_smc_power_scaling_factor(rdev);
+
+        for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
+                for (j = 0; j < table_size; j++) {
+                        t = (1000 * ((i + 1) * 8));
+
+                        if (t < ni_pi->cac_data.leakage_minimum_temperature)
+                                t = ni_pi->cac_data.leakage_minimum_temperature;
+
+                        ni_calculate_leakage_for_v_and_t(rdev,
+                                                         &ni_pi->cac_data.leakage_coefficients,
+                                                         eg_pi->vddc_voltage_table.entries[j].value,
+                                                         t,
+                                                         ni_pi->cac_data.i_leakage,
+                                                         &leakage);
+
+                        smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
+                        if (smc_leakage > max_leakage)
+                                max_leakage = smc_leakage;
+
+                        cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
+                }
+        }
+
+        for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
+                for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
+                        cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
+        }
+        return 0;
+}
+
+static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
+                                            PP_NIslands_CACTABLES *cac_tables)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_cac_leakage_table *leakage_table =
+                &rdev->pm.dpm.dyn_state.cac_leakage_table;
+        u32 i, j, table_size;
+        u32 smc_leakage, max_leakage = 0;
+        u32 scaling_factor;
+
+        if (!leakage_table)
+                return -EINVAL;
+
+        table_size = leakage_table->count;
+
+        if (eg_pi->vddc_voltage_table.count != table_size)
+                table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
+                        eg_pi->vddc_voltage_table.count : leakage_table->count;
+
+        if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
+                table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
+
+        if (table_size == 0)
+                return -EINVAL;
+
+        scaling_factor = ni_get_smc_power_scaling_factor(rdev);
+
+        for (j = 0; j < table_size; j++) {
+                smc_leakage = leakage_table->entries[j].leakage;
+
+                if (smc_leakage > max_leakage)
+                        max_leakage = smc_leakage;
+
+                for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
+                        cac_tables->cac_lkge_lut[i][j] =
+                                cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
+        }
+
+        for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
+                for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
+                        cac_tables->cac_lkge_lut[i][j] =
+                                cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
+        }
+        return 0;
+}
+
+static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        PP_NIslands_CACTABLES *cac_tables = NULL;
+        int i, ret;
+        u32 reg;
+
+        if (ni_pi->enable_cac == false)
+                return 0;
+
+        cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
+        if (!cac_tables)
+                return -ENOMEM;
+
+        reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
+        reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
+                TID_UNIT(ni_pi->cac_weights->tid_unit));
+        WREG32(CG_CAC_CTRL, reg);
+
+        for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
+                ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
+
+        for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
+                cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
+
+        ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
+        ni_pi->cac_data.pwr_const = 0;
+        ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
+        ni_pi->cac_data.bif_cac_value = 0;
+        ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
+        ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
+        ni_pi->cac_data.allow_ovrflw = 0;
+        ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
+        ni_pi->cac_data.num_win_tdp = 0;
+        ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
+
+        if (ni_pi->driver_calculate_cac_leakage)
+                ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
+        else
+                ret = ni_init_simplified_leakage_table(rdev, cac_tables);
+
+        if (ret)
+                goto done_free;
+
+        cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
+        cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
+        cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
+        cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
+        cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
+        cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
+        cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
+        cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
+        cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
+
+        ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
+                                      sizeof(PP_NIslands_CACTABLES), pi->sram_end);
+
+done_free:
+        if (ret) {
+                ni_pi->enable_cac = false;
+                ni_pi->enable_power_containment = false;
+        }
+
+        kfree(cac_tables);
+
+        return 0;
+}
+
+static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        u32 reg;
+
+        if (!ni_pi->enable_cac ||
+            !ni_pi->cac_configuration_required)
+                return 0;
+
+        if (ni_pi->cac_weights == NULL)
+                return -EINVAL;
+
+        reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
+                                                      WEIGHT_TCP_SIG1_MASK |
+                                                      WEIGHT_TA_SIG_MASK);
+        reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
+                WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
+                WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
+        WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
+                                                      WEIGHT_TCC_EN1_MASK |
+                                                      WEIGHT_TCC_EN2_MASK);
+        reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
+                WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
+                WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
+        WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
+                                                      WEIGHT_CB_EN1_MASK |
+                                                      WEIGHT_CB_EN2_MASK |
+                                                      WEIGHT_CB_EN3_MASK);
+        reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
+                WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
+                WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
+                WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
+        WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
+                                                      WEIGHT_DB_SIG1_MASK |
+                                                      WEIGHT_DB_SIG2_MASK |
+                                                      WEIGHT_DB_SIG3_MASK);
+        reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
+                WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
+                WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
+                WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
+        WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
+                                                      WEIGHT_SXM_SIG1_MASK |
+                                                      WEIGHT_SXM_SIG2_MASK |
+                                                      WEIGHT_SXS_SIG0_MASK |
+                                                      WEIGHT_SXS_SIG1_MASK);
+        reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
+                WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
+                WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
+                WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
+                WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
+        WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
+                                                      WEIGHT_XBR_1_MASK |
+                                                      WEIGHT_XBR_2_MASK |
+                                                      WEIGHT_SPI_SIG0_MASK);
+        reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
+                WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
+                WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
+                WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
+        WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
+                                                      WEIGHT_SPI_SIG2_MASK |
+                                                      WEIGHT_SPI_SIG3_MASK |
+                                                      WEIGHT_SPI_SIG4_MASK |
+                                                      WEIGHT_SPI_SIG5_MASK);
+        reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
+                WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
+                WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
+                WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
+                WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
+        WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
+                                                      WEIGHT_LDS_SIG1_MASK |
+                                                      WEIGHT_SC_MASK);
+        reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
+                WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
+                WEIGHT_SC(ni_pi->cac_weights->weight_sc));
+        WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
+                                                      WEIGHT_CP_MASK |
+                                                      WEIGHT_PA_SIG0_MASK |
+                                                      WEIGHT_PA_SIG1_MASK |
+                                                      WEIGHT_VGT_SIG0_MASK);
+        reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
+                WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
+                WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
+                WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
+                WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
+        WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
+                                                      WEIGHT_VGT_SIG2_MASK |
+                                                      WEIGHT_DC_SIG0_MASK |
+                                                      WEIGHT_DC_SIG1_MASK |
+                                                      WEIGHT_DC_SIG2_MASK);
+        reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
+                WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
+                WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
+                WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
+                WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
+        WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
+                                                      WEIGHT_UVD_SIG0_MASK |
+                                                      WEIGHT_UVD_SIG1_MASK |
+                                                      WEIGHT_SPARE0_MASK |
+                                                      WEIGHT_SPARE1_MASK);
+        reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
+                WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
+                WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
+                WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
+                WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
+        WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
+                                                      WEIGHT_SQ_VSP0_MASK);
+        reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
+                WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
+        WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
+        reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
+        WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
+
+        reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
+                                                        OVR_VAL_SPARE_0_MASK |
+                                                        OVR_MODE_SPARE_1_MASK |
+                                                        OVR_VAL_SPARE_1_MASK);
+        reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
+                OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
+                OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
+                OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
+        WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
+
+        reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
+                                           VSP0_MASK |
+                                           GPR_MASK);
+        reg |= (VSP(ni_pi->cac_weights->vsp) |
+                VSP0(ni_pi->cac_weights->vsp0) |
+                GPR(ni_pi->cac_weights->gpr));
+        WREG32(SQ_CAC_THRESHOLD, reg);
+
+        reg = (MCDW_WR_ENABLE |
+               MCDX_WR_ENABLE |
+               MCDY_WR_ENABLE |
+               MCDZ_WR_ENABLE |
+               INDEX(0x09D4));
+        WREG32(MC_CG_CONFIG, reg);
+
+        reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
+               WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
+               ALLOW_OVERFLOW);
+        WREG32(MC_CG_DATAPORT, reg);
+
+        return 0;
+}
+
+static int ni_enable_smc_cac(struct radeon_device *rdev,
+                             struct radeon_ps *radeon_new_state,
+                             bool enable)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        int ret = 0;
+        PPSMC_Result smc_result;
+
+        if (ni_pi->enable_cac) {
+                if (enable) {
+                        if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
+                                smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
+
+                                if (ni_pi->support_cac_long_term_average) {
+                                        smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
+                                        if (PPSMC_Result_OK != smc_result)
+                                                ni_pi->support_cac_long_term_average = false;
+                                }
+
+                                smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
+                                if (PPSMC_Result_OK != smc_result)
+                                        ret = -EINVAL;
+
+                                ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
+                        }
+                } else if (ni_pi->cac_enabled) {
+                        smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
+
+                        ni_pi->cac_enabled = false;
+
+                        if (ni_pi->support_cac_long_term_average) {
+                                smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
+                                if (PPSMC_Result_OK != smc_result)
+                                        ni_pi->support_cac_long_term_average = false;
+                        }
+                }
+        }
+
+        return ret;
+}
+
+static int ni_pcie_performance_request(struct radeon_device *rdev,
+                                       u8 perf_req, bool advertise)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+#if defined(CONFIG_ACPI)
+        if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
+            (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
+                if (eg_pi->pcie_performance_request_registered == false)
+                        radeon_acpi_pcie_notify_device_ready(rdev);
+                eg_pi->pcie_performance_request_registered = true;
+                return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
+        } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
+                   eg_pi->pcie_performance_request_registered) {
+                eg_pi->pcie_performance_request_registered = false;
+                return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
+        }
+#endif
+        return 0;
+}
+
+static int ni_advertise_gen2_capability(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+
+        if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
+                pi->pcie_gen2 = true;
+        else
+                pi->pcie_gen2 = false;
+
+        if (!pi->pcie_gen2)
+                ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
+
+        return 0;
+}
+
+static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                            bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp, bif;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+
+        if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+                if (enable) {
+                        if (!pi->boot_in_gen2) {
+                                bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
+                                bif |= CG_CLIENT_REQ(0xd);
+                                WREG32(CG_BIF_REQ_AND_RSP, bif);
+                        }
+                        tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                        tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
+                        tmp |= LC_GEN2_EN_STRAP;
+
+                        tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
+                        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                        udelay(10);
+                        tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
+                        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                } else {
+                        if (!pi->boot_in_gen2) {
+                                bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
+                                bif |= CG_CLIENT_REQ(0xd);
+                                WREG32(CG_BIF_REQ_AND_RSP, bif);
+
+                                tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                                tmp &= ~LC_GEN2_EN_STRAP;
+                        }
+                        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+                }
+        }
+}
+
+static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                        bool enable)
+{
+        ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
+
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
+}
+
+void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                           struct radeon_ps *new_ps,
+                                           struct radeon_ps *old_ps)
+{
+        struct ni_ps *new_state = ni_get_ps(new_ps);
+        struct ni_ps *current_state = ni_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
+            current_state->performance_levels[current_state->performance_level_count - 1].sclk)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                          struct radeon_ps *new_ps,
+                                          struct radeon_ps *old_ps)
+{
+        struct ni_ps *new_state = ni_get_ps(new_ps);
+        struct ni_ps *current_state = ni_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
+            current_state->performance_levels[current_state->performance_level_count - 1].sclk)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+void ni_dpm_setup_asic(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        ni_read_clock_registers(rdev);
+        btc_read_arb_registers(rdev);
+        rv770_get_memory_type(rdev);
+        if (eg_pi->pcie_performance_request)
+                ni_advertise_gen2_capability(rdev);
+        rv770_get_pcie_gen2_status(rdev);
+        rv770_enable_acpi_pm(rdev);
+}
+
+void ni_update_current_ps(struct radeon_device *rdev,
+                          struct radeon_ps *rps)
+{
+        struct ni_ps *new_ps = ni_get_ps(rps);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+
+        eg_pi->current_rps = *rps;
+        ni_pi->current_ps = *new_ps;
+        eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
+}
+
+void ni_update_requested_ps(struct radeon_device *rdev,
+                            struct radeon_ps *rps)
+{
+        struct ni_ps *new_ps = ni_get_ps(rps);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+
+        eg_pi->requested_rps = *rps;
+        ni_pi->requested_ps = *new_ps;
+        eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
+}
+
+int ni_dpm_enable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        if (pi->gfx_clock_gating)
+                ni_cg_clockgating_default(rdev);
+        if (btc_dpm_enabled(rdev))
+                return -EINVAL;
+        if (pi->mg_clock_gating)
+                ni_mg_clockgating_default(rdev);
+        if (eg_pi->ls_clock_gating)
+                ni_ls_clockgating_default(rdev);
+        if (pi->voltage_control) {
+                rv770_enable_voltage_control(rdev, true);
+                ret = cypress_construct_voltage_tables(rdev);
+                if (ret) {
+                        DRM_ERROR("cypress_construct_voltage_tables failed\n");
+                        return ret;
+                }
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = ni_initialize_mc_reg_table(rdev);
+                if (ret)
+                        eg_pi->dynamic_ac_timing = false;
+        }
+        if (pi->dynamic_ss)
+                cypress_enable_spread_spectrum(rdev, true);
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, true);
+        rv770_setup_bsp(rdev);
+        rv770_program_git(rdev);
+        rv770_program_tp(rdev);
+        rv770_program_tpp(rdev);
+        rv770_program_sstp(rdev);
+        cypress_enable_display_gap(rdev);
+        rv770_program_vc(rdev);
+        if (pi->dynamic_pcie_gen2)
+                ni_enable_dynamic_pcie_gen2(rdev, true);
+        ret = rv770_upload_firmware(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_upload_firmware failed\n");
+                return ret;
+        }
+        ret = ni_process_firmware_header(rdev);
+        if (ret) {
+                DRM_ERROR("ni_process_firmware_header failed\n");
+                return ret;
+        }
+        ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
+        if (ret) {
+                DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
+                return ret;
+        }
+        ret = ni_init_smc_table(rdev);
+        if (ret) {
+                DRM_ERROR("ni_init_smc_table failed\n");
+                return ret;
+        }
+        ret = ni_init_smc_spll_table(rdev);
+        if (ret) {
+                DRM_ERROR("ni_init_smc_spll_table failed\n");
+                return ret;
+        }
+        ret = ni_init_arb_table_index(rdev);
+        if (ret) {
+                DRM_ERROR("ni_init_arb_table_index failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = ni_populate_mc_reg_table(rdev, boot_ps);
+                if (ret) {
+                        DRM_ERROR("ni_populate_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+        ret = ni_initialize_smc_cac_tables(rdev);
+        if (ret) {
+                DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
+                return ret;
+        }
+        ret = ni_initialize_hardware_cac_manager(rdev);
+        if (ret) {
+                DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
+                return ret;
+        }
+        ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
+        if (ret) {
+                DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
+                return ret;
+        }
+        ni_program_response_times(rdev);
+        r7xx_start_smc(rdev);
+        ret = cypress_notify_smc_display_change(rdev, false);
+        if (ret) {
+                DRM_ERROR("cypress_notify_smc_display_change failed\n");
+                return ret;
+        }
+        cypress_enable_sclk_control(rdev, true);
+        if (eg_pi->memory_transition)
+                cypress_enable_mclk_control(rdev, true);
+        cypress_start_dpm(rdev);
+        if (pi->gfx_clock_gating)
+                ni_gfx_clockgating_enable(rdev, true);
+        if (pi->mg_clock_gating)
+                ni_mg_clockgating_enable(rdev, true);
+        if (eg_pi->ls_clock_gating)
+                ni_ls_clockgating_enable(rdev, true);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                PPSMC_Result result;
+
+                ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, 0xff * 1000);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+                result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
+
+                if (result != PPSMC_Result_OK)
+                        DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
+        }
+
+        rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+
+        ni_update_current_ps(rdev, boot_ps);
+
+        return 0;
+}
+
+void ni_dpm_disable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+
+        if (!btc_dpm_enabled(rdev))
+                return;
+        rv770_clear_vc(rdev);
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, false);
+        ni_enable_power_containment(rdev, boot_ps, false);
+        ni_enable_smc_cac(rdev, boot_ps, false);
+        cypress_enable_spread_spectrum(rdev, false);
+        rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
+        if (pi->dynamic_pcie_gen2)
+                ni_enable_dynamic_pcie_gen2(rdev, false);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        if (pi->gfx_clock_gating)
+                ni_gfx_clockgating_enable(rdev, false);
+        if (pi->mg_clock_gating)
+                ni_mg_clockgating_enable(rdev, false);
+        if (eg_pi->ls_clock_gating)
+                ni_ls_clockgating_enable(rdev, false);
+        ni_stop_dpm(rdev);
+        btc_reset_to_default(rdev);
+        ni_stop_smc(rdev);
+        ni_force_switch_to_arb_f0(rdev);
+
+        ni_update_current_ps(rdev, boot_ps);
+}
+
+static int ni_power_control_set_level(struct radeon_device *rdev)
+{
+        struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
+        int ret;
+
+        ret = ni_restrict_performance_levels_before_switch(rdev);
+        if (ret)
+                return ret;
+        ret = rv770_halt_smc(rdev);
+        if (ret)
+                return ret;
+        ret = ni_populate_smc_tdp_limits(rdev, new_ps);
+        if (ret)
+                return ret;
+        ret = rv770_resume_smc(rdev);
+        if (ret)
+                return ret;
+        ret = rv770_set_sw_state(rdev);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
+        struct radeon_ps *new_ps = &requested_ps;
+
+        ni_update_requested_ps(rdev, new_ps);
+
+        ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
+
+        return 0;
+}
+
+int ni_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = &eg_pi->requested_rps;
+        struct radeon_ps *old_ps = &eg_pi->current_rps;
+        int ret;
+
+        ret = ni_restrict_performance_levels_before_switch(rdev);
+        if (ret) {
+                DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
+                return ret;
+        }
+        ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+        ret = ni_enable_power_containment(rdev, new_ps, false);
+        if (ret) {
+                DRM_ERROR("ni_enable_power_containment failed\n");
+                return ret;
+        }
+        ret = ni_enable_smc_cac(rdev, new_ps, false);
+        if (ret) {
+                DRM_ERROR("ni_enable_smc_cac failed\n");
+                return ret;
+        }
+        ret = rv770_halt_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_halt_smc failed\n");
+                return ret;
+        }
+        if (eg_pi->smu_uvd_hs)
+                btc_notify_uvd_to_smc(rdev, new_ps);
+        ret = ni_upload_sw_state(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("ni_upload_sw_state failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = ni_upload_mc_reg_table(rdev, new_ps);
+                if (ret) {
+                        DRM_ERROR("ni_upload_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+        ret = ni_program_memory_timing_parameters(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("ni_program_memory_timing_parameters failed\n");
+                return ret;
+        }
+        ret = rv770_resume_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_resume_smc failed\n");
+                return ret;
+        }
+        ret = rv770_set_sw_state(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_set_sw_state failed\n");
+                return ret;
+        }
+        ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+        ret = ni_enable_smc_cac(rdev, new_ps, true);
+        if (ret) {
+                DRM_ERROR("ni_enable_smc_cac failed\n");
+                return ret;
+        }
+        ret = ni_enable_power_containment(rdev, new_ps, true);
+        if (ret) {
+                DRM_ERROR("ni_enable_power_containment failed\n");
+                return ret;
+        }
+
+        /* update tdp */
+        ret = ni_power_control_set_level(rdev);
+        if (ret) {
+                DRM_ERROR("ni_power_control_set_level failed\n");
+                return ret;
+        }
+
+        ret = ni_unrestrict_performance_levels_after_switch(rdev);
+        if (ret) {
+                DRM_ERROR("ni_unrestrict_performance_levels_after_switch failed\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+void ni_dpm_post_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = &eg_pi->requested_rps;
+
+        ni_update_current_ps(rdev, new_ps);
+}
+
+void ni_dpm_reset_asic(struct radeon_device *rdev)
+{
+        ni_restrict_performance_levels_before_switch(rdev);
+        rv770_set_boot_state(rdev);
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                          struct radeon_ps *rps,
+                                          struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+                                          u8 table_rev)
+{
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+        } else if (r600_is_uvd_state(rps->class, rps->class2)) {
+                rps->vclk = RV770_DEFAULT_VCLK_FREQ;
+                rps->dclk = RV770_DEFAULT_DCLK_FREQ;
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+                rdev->pm.dpm.boot_ps = rps;
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
+                                      struct radeon_ps *rps, int index,
+                                      union pplib_clock_info *clock_info)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_ps *ps = ni_get_ps(rps);
+        u16 vddc;
+        struct rv7xx_pl *pl = &ps->performance_levels[index];
+
+        ps->performance_level_count = index + 1;
+
+        pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
+        pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
+        pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
+        pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
+
+        pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
+        pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
+        pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
+
+        /* patch up vddc if necessary */
+        if (pl->vddc == 0xff01) {
+                if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
+                        pl->vddc = vddc;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
+                pi->acpi_vddc = pl->vddc;
+                eg_pi->acpi_vddci = pl->vddci;
+                if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
+                        pi->acpi_pcie_gen2 = true;
+                else
+                        pi->acpi_pcie_gen2 = false;
+        }
+
+        if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
+                eg_pi->ulv.supported = true;
+                eg_pi->ulv.pl = pl;
+        }
+
+        if (pi->min_vddc_in_table > pl->vddc)
+                pi->min_vddc_in_table = pl->vddc;
+
+        if (pi->max_vddc_in_table < pl->vddc)
+                pi->max_vddc_in_table = pl->vddc;
+
+        /* patch up boot state */
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                u16 vddc, vddci, mvdd;
+                radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
+                pl->mclk = rdev->clock.default_mclk;
+                pl->sclk = rdev->clock.default_sclk;
+                pl->vddc = vddc;
+                pl->vddci = vddci;
+        }
+
+        if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
+            ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
+        }
+}
+
+static int ni_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j;
+        union pplib_clock_info *clock_info;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        struct ni_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  power_info->pplib.ucNumStates, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+
+        for (i = 0; i < power_info->pplib.ucNumStates; i++) {
+                power_state = (union pplib_power_state *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usStateArrayOffset) +
+                         i * power_info->pplib.ucStateEntrySize);
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
+                         (power_state->v1.ucNonClockStateIndex *
+                          power_info->pplib.ucNonClockSize));
+                if (power_info->pplib.ucStateEntrySize - 1) {
+                        ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
+                        if (ps == NULL) {
+                                kfree(rdev->pm.dpm.ps);
+                                return -ENOMEM;
+                        }
+                        rdev->pm.dpm.ps[i].ps_priv = ps;
+                        ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                                         non_clock_info,
+                                                         power_info->pplib.ucNonClockSize);
+                        for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
+                                clock_info = (union pplib_clock_info *)
+                                        (mode_info->atom_context->bios + data_offset +
+                                         le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
+                                         (power_state->v1.ucClockStateIndices[j] *
+                                          power_info->pplib.ucClockInfoSize));
+                                ni_parse_pplib_clock_info(rdev,
+                                                          &rdev->pm.dpm.ps[i], j,
+                                                          clock_info);
+                        }
+                }
+        }
+        rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
+        return 0;
+}
+
+int ni_dpm_init(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi;
+        struct evergreen_power_info *eg_pi;
+        struct ni_power_info *ni_pi;
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        u16 data_offset, size;
+        u8 frev, crev;
+        struct atom_clock_dividers dividers;
+        int ret;
+
+        ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
+        if (ni_pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = ni_pi;
+        eg_pi = &ni_pi->eg;
+        pi = &eg_pi->rv7xx;
+
+        rv770_get_max_vddc(rdev);
+
+        eg_pi->ulv.supported = false;
+        pi->acpi_vddc = 0;
+        eg_pi->acpi_vddci = 0;
+        pi->min_vddc_in_table = 0;
+        pi->max_vddc_in_table = 0;
+
+        ret = ni_parse_power_table(rdev);
+        if (ret)
+                return ret;
+        ret = r600_parse_extended_power_table(rdev);
+        if (ret)
+                return ret;
+
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
+                kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
+        if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
+                r600_free_extended_power_table(rdev);
+                return -ENOMEM;
+        }
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
+
+        ni_patch_dependency_tables_based_on_leakage(rdev);
+
+        if (rdev->pm.dpm.voltage_response_time == 0)
+                rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+        if (rdev->pm.dpm.backbias_response_time == 0)
+                rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->ref_div = dividers.ref_div + 1;
+        else
+                pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        pi->rlp = RV770_RLP_DFLT;
+        pi->rmp = RV770_RMP_DFLT;
+        pi->lhp = RV770_LHP_DFLT;
+        pi->lmp = RV770_LMP_DFLT;
+
+        eg_pi->ats[0].rlp = RV770_RLP_DFLT;
+        eg_pi->ats[0].rmp = RV770_RMP_DFLT;
+        eg_pi->ats[0].lhp = RV770_LHP_DFLT;
+        eg_pi->ats[0].lmp = RV770_LMP_DFLT;
+
+        eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
+        eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
+        eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
+        eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
+
+        eg_pi->smu_uvd_hs = true;
+
+        if (rdev->pdev->device == 0x6707) {
+                pi->mclk_strobe_mode_threshold = 55000;
+                pi->mclk_edc_enable_threshold = 55000;
+                eg_pi->mclk_edc_wr_enable_threshold = 55000;
+        } else {
+                pi->mclk_strobe_mode_threshold = 40000;
+                pi->mclk_edc_enable_threshold = 40000;
+                eg_pi->mclk_edc_wr_enable_threshold = 40000;
+        }
+        ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
+
+        pi->voltage_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
+
+        pi->mvdd_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
+
+        eg_pi->vddci_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                pi->sclk_ss = true;
+                pi->mclk_ss = true;
+                pi->dynamic_ss = true;
+        } else {
+                pi->sclk_ss = false;
+                pi->mclk_ss = false;
+                pi->dynamic_ss = true;
+        }
+
+        pi->asi = RV770_ASI_DFLT;
+        pi->pasi = CYPRESS_HASI_DFLT;
+        pi->vrc = CYPRESS_VRC_DFLT;
+
+        pi->power_gating = false;
+
+        pi->gfx_clock_gating = true;
+
+        pi->mg_clock_gating = true;
+        pi->mgcgtssm = true;
+        eg_pi->ls_clock_gating = false;
+        eg_pi->sclk_deep_sleep = false;
+
+        pi->dynamic_pcie_gen2 = true;
+
+        if (pi->gfx_clock_gating &&
+            (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
+                pi->thermal_protection = true;
+        else
+                pi->thermal_protection = false;
+
+        pi->display_gap = true;
+
+        pi->dcodt = true;
+
+        pi->ulps = true;
+
+        eg_pi->dynamic_ac_timing = true;
+        eg_pi->abm = true;
+        eg_pi->mcls = true;
+        eg_pi->light_sleep = true;
+        eg_pi->memory_transition = true;
+#if defined(CONFIG_ACPI)
+        eg_pi->pcie_performance_request =
+                radeon_acpi_is_pcie_performance_request_supported(rdev);
+#else
+        eg_pi->pcie_performance_request = false;
+#endif
+
+        eg_pi->dll_default_on = false;
+
+        eg_pi->sclk_deep_sleep = false;
+
+        pi->mclk_stutter_mode_threshold = 0;
+
+        pi->sram_end = SMC_RAM_END;
+
+        rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
+        rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
+        rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
+        rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
+        rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
+        rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
+        rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
+        rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
+
+        ni_pi->cac_data.leakage_coefficients.at = 516;
+        ni_pi->cac_data.leakage_coefficients.bt = 18;
+        ni_pi->cac_data.leakage_coefficients.av = 51;
+        ni_pi->cac_data.leakage_coefficients.bv = 2957;
+
+        switch (rdev->pdev->device) {
+        case 0x6700:
+        case 0x6701:
+        case 0x6702:
+        case 0x6703:
+        case 0x6718:
+                ni_pi->cac_weights = &cac_weights_cayman_xt;
+                break;
+        case 0x6705:
+        case 0x6719:
+        case 0x671D:
+        case 0x671C:
+        default:
+                ni_pi->cac_weights = &cac_weights_cayman_pro;
+                break;
+        case 0x6704:
+        case 0x6706:
+        case 0x6707:
+        case 0x6708:
+        case 0x6709:
+                ni_pi->cac_weights = &cac_weights_cayman_le;
+                break;
+        }
+
+        if (ni_pi->cac_weights->enable_power_containment_by_default) {
+                ni_pi->enable_power_containment = true;
+                ni_pi->enable_cac = true;
+                ni_pi->enable_sq_ramping = true;
+        } else {
+                ni_pi->enable_power_containment = false;
+                ni_pi->enable_cac = false;
+                ni_pi->enable_sq_ramping = false;
+        }
+
+        ni_pi->driver_calculate_cac_leakage = false;
+        ni_pi->cac_configuration_required = true;
+
+        if (ni_pi->cac_configuration_required) {
+                ni_pi->support_cac_long_term_average = true;
+                ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
+                ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
+        } else {
+                ni_pi->support_cac_long_term_average = false;
+                ni_pi->lta_window_size = 0;
+                ni_pi->lts_truncate = 0;
+        }
+
+        ni_pi->use_power_boost_limit = true;
+
+        return 0;
+}
+
+void ni_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+        kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
+        r600_free_extended_power_table(rdev);
+}
+
+void ni_dpm_print_power_state(struct radeon_device *rdev,
+                              struct radeon_ps *rps)
+{
+        struct ni_ps *ps = ni_get_ps(rps);
+        struct rv7xx_pl *pl;
+        int i;
+
+        r600_dpm_print_class_info(rps->class, rps->class2);
+        r600_dpm_print_cap_info(rps->caps);
+        printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+        for (i = 0; i < ps->performance_level_count; i++) {
+                pl = &ps->performance_levels[i];
+                if (rdev->family >= CHIP_TAHITI)
+                        printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
+                               i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
+                else
+                        printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+                               i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+        }
+        r600_dpm_print_ps_status(rdev, rps);
+}
+
+void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                    struct seq_file *m)
+{
+        struct radeon_ps *rps = rdev->pm.dpm.current_ps;
+        struct ni_ps *ps = ni_get_ps(rps);
+        struct rv7xx_pl *pl;
+        u32 current_index =
+                (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
+                CURRENT_STATE_INDEX_SHIFT;
+
+        if (current_index >= ps->performance_level_count) {
+                seq_printf(m, "invalid dpm profile %d\n", current_index);
+        } else {
+                pl = &ps->performance_levels[current_index];
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+                           current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+        }
+}
+
+u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
+
+        if (low)
+                return requested_state->performance_levels[0].sclk;
+        else
+                return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
+}
+
+u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
+
+        if (low)
+                return requested_state->performance_levels[0].mclk;
+        else
+                return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
+}
+
diff --git a/drivers/gpu/drm/radeon/ni_dpm.h b/drivers/gpu/drm/radeon/ni_dpm.h
new file mode 100644
index 000000000000..ac1c7abf2c67
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni_dpm.h
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __NI_DPM_H__
+#define __NI_DPM_H__
+
+#include "cypress_dpm.h"
+#include "btc_dpm.h"
+#include "nislands_smc.h"
+
+struct ni_clock_registers {
+        u32 cg_spll_func_cntl;
+        u32 cg_spll_func_cntl_2;
+        u32 cg_spll_func_cntl_3;
+        u32 cg_spll_func_cntl_4;
+        u32 cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2;
+        u32 mclk_pwrmgt_cntl;
+        u32 dll_cntl;
+        u32 mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2;
+        u32 mpll_ss1;
+        u32 mpll_ss2;
+};
+
+struct ni_mc_reg_entry {
+        u32 mclk_max;
+        u32 mc_data[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct ni_mc_reg_table {
+        u8 last;
+        u8 num_entries;
+        u16 valid_flag;
+        struct ni_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+        SMC_NIslands_MCRegisterAddress mc_reg_address[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+#define NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT 2
+
+enum ni_dc_cac_level
+{
+        NISLANDS_DCCAC_LEVEL_0 = 0,
+        NISLANDS_DCCAC_LEVEL_1,
+        NISLANDS_DCCAC_LEVEL_2,
+        NISLANDS_DCCAC_LEVEL_3,
+        NISLANDS_DCCAC_LEVEL_4,
+        NISLANDS_DCCAC_LEVEL_5,
+        NISLANDS_DCCAC_LEVEL_6,
+        NISLANDS_DCCAC_LEVEL_7,
+        NISLANDS_DCCAC_MAX_LEVELS
+};
+
+struct ni_leakage_coeffients
+{
+        u32 at;
+        u32 bt;
+        u32 av;
+        u32 bv;
+        s32 t_slope;
+        s32 t_intercept;
+        u32 t_ref;
+};
+
+struct ni_cac_data
+{
+        struct ni_leakage_coeffients leakage_coefficients;
+        u32 i_leakage;
+        s32 leakage_minimum_temperature;
+        u32 pwr_const;
+        u32 dc_cac_value;
+        u32 bif_cac_value;
+        u32 lkge_pwr;
+        u8 mc_wr_weight;
+        u8 mc_rd_weight;
+        u8 allow_ovrflw;
+        u8 num_win_tdp;
+        u8 l2num_win_tdp;
+        u8 lts_truncate_n;
+};
+
+struct ni_cac_weights
+{
+        u32 weight_tcp_sig0;
+        u32 weight_tcp_sig1;
+        u32 weight_ta_sig;
+        u32 weight_tcc_en0;
+        u32 weight_tcc_en1;
+        u32 weight_tcc_en2;
+        u32 weight_cb_en0;
+        u32 weight_cb_en1;
+        u32 weight_cb_en2;
+        u32 weight_cb_en3;
+        u32 weight_db_sig0;
+        u32 weight_db_sig1;
+        u32 weight_db_sig2;
+        u32 weight_db_sig3;
+        u32 weight_sxm_sig0;
+        u32 weight_sxm_sig1;
+        u32 weight_sxm_sig2;
+        u32 weight_sxs_sig0;
+        u32 weight_sxs_sig1;
+        u32 weight_xbr_0;
+        u32 weight_xbr_1;
+        u32 weight_xbr_2;
+        u32 weight_spi_sig0;
+        u32 weight_spi_sig1;
+        u32 weight_spi_sig2;
+        u32 weight_spi_sig3;
+        u32 weight_spi_sig4;
+        u32 weight_spi_sig5;
+        u32 weight_lds_sig0;
+        u32 weight_lds_sig1;
+        u32 weight_sc;
+        u32 weight_bif;
+        u32 weight_cp;
+        u32 weight_pa_sig0;
+        u32 weight_pa_sig1;
+        u32 weight_vgt_sig0;
+        u32 weight_vgt_sig1;
+        u32 weight_vgt_sig2;
+        u32 weight_dc_sig0;
+        u32 weight_dc_sig1;
+        u32 weight_dc_sig2;
+        u32 weight_dc_sig3;
+        u32 weight_uvd_sig0;
+        u32 weight_uvd_sig1;
+        u32 weight_spare0;
+        u32 weight_spare1;
+        u32 weight_sq_vsp;
+        u32 weight_sq_vsp0;
+        u32 weight_sq_gpr;
+        u32 ovr_mode_spare_0;
+        u32 ovr_val_spare_0;
+        u32 ovr_mode_spare_1;
+        u32 ovr_val_spare_1;
+        u32 vsp;
+        u32 vsp0;
+        u32 gpr;
+        u8 mc_read_weight;
+        u8 mc_write_weight;
+        u32 tid_cnt;
+        u32 tid_unit;
+        u32 l2_lta_window_size;
+        u32 lts_truncate;
+        u32 dc_cac[NISLANDS_DCCAC_MAX_LEVELS];
+        u32 pcie_cac[SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES];
+        bool enable_power_containment_by_default;
+};
+
+struct ni_ps {
+        u16 performance_level_count;
+        bool dc_compatible;
+        struct rv7xx_pl performance_levels[NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
+};
+
+struct ni_power_info {
+        /* must be first! */
+        struct evergreen_power_info eg;
+        struct ni_clock_registers clock_registers;
+        struct ni_mc_reg_table mc_reg_table;
+        u32 mclk_rtt_mode_threshold;
+        /* flags */
+        bool use_power_boost_limit;
+        bool support_cac_long_term_average;
+        bool cac_enabled;
+        bool cac_configuration_required;
+        bool driver_calculate_cac_leakage;
+        bool pc_enabled;
+        bool enable_power_containment;
+        bool enable_cac;
+        bool enable_sq_ramping;
+        /* smc offsets */
+        u16 arb_table_start;
+        u16 fan_table_start;
+        u16 cac_table_start;
+        u16 spll_table_start;
+        /* CAC stuff */
+        struct ni_cac_data cac_data;
+        u32 dc_cac_table[NISLANDS_DCCAC_MAX_LEVELS];
+        const struct ni_cac_weights *cac_weights;
+        u8 lta_window_size;
+        u8 lts_truncate;
+        struct ni_ps current_ps;
+        struct ni_ps requested_ps;
+        /* scratch structs */
+        SMC_NIslands_MCRegisters smc_mc_reg_table;
+        NISLANDS_SMC_STATETABLE smc_statetable;
+};
+
+#define NISLANDS_INITIAL_STATE_ARB_INDEX    0
+#define NISLANDS_ACPI_STATE_ARB_INDEX       1
+#define NISLANDS_ULV_STATE_ARB_INDEX        2
+#define NISLANDS_DRIVER_STATE_ARB_INDEX     3
+
+#define NISLANDS_DPM2_MAX_PULSE_SKIP        256
+
+#define NISLANDS_DPM2_NEAR_TDP_DEC          10
+#define NISLANDS_DPM2_ABOVE_SAFE_INC        5
+#define NISLANDS_DPM2_BELOW_SAFE_INC        20
+
+#define NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT            80
+
+#define NISLANDS_DPM2_MAXPS_PERCENT_H                   90
+#define NISLANDS_DPM2_MAXPS_PERCENT_M                   0
+
+#define NISLANDS_DPM2_SQ_RAMP_MAX_POWER                 0x3FFF
+#define NISLANDS_DPM2_SQ_RAMP_MIN_POWER                 0x12
+#define NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA           0x15
+#define NISLANDS_DPM2_SQ_RAMP_STI_SIZE                  0x1E
+#define NISLANDS_DPM2_SQ_RAMP_LTI_RATIO                 0xF
+
+int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
+                                u32 arb_freq_src, u32 arb_freq_dest);
+void ni_update_current_ps(struct radeon_device *rdev,
+                          struct radeon_ps *rps);
+void ni_update_requested_ps(struct radeon_device *rdev,
+                            struct radeon_ps *rps);
+
+void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                           struct radeon_ps *new_ps,
+                                           struct radeon_ps *old_ps);
+void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                          struct radeon_ps *new_ps,
+                                          struct radeon_ps *old_ps);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index e226faf16fea..fe24a93542ec 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -489,6 +489,571 @@
 #       define CACHE_FLUSH_AND_INV_EVENT_TS                     (0x14 << 0)
 #       define CACHE_FLUSH_AND_INV_EVENT                        (0x16 << 0)
 
+/* TN SMU registers */
+#define TN_CURRENT_GNB_TEMP                             0x1F390
+
+/* pm registers */
+#define SMC_MSG                                         0x20c
+#define         HOST_SMC_MSG(x)                         ((x) << 0)
+#define         HOST_SMC_MSG_MASK                       (0xff << 0)
+#define         HOST_SMC_MSG_SHIFT                      0
+#define         HOST_SMC_RESP(x)                        ((x) << 8)
+#define         HOST_SMC_RESP_MASK                      (0xff << 8)
+#define         HOST_SMC_RESP_SHIFT                     8
+#define         SMC_HOST_MSG(x)                         ((x) << 16)
+#define         SMC_HOST_MSG_MASK                       (0xff << 16)
+#define         SMC_HOST_MSG_SHIFT                      16
+#define         SMC_HOST_RESP(x)                        ((x) << 24)
+#define         SMC_HOST_RESP_MASK                      (0xff << 24)
+#define         SMC_HOST_RESP_SHIFT                     24
+
+#define CG_SPLL_FUNC_CNTL                               0x600
+#define         SPLL_RESET                              (1 << 0)
+#define         SPLL_SLEEP                              (1 << 1)
+#define         SPLL_BYPASS_EN                          (1 << 3)
+#define         SPLL_REF_DIV(x)                         ((x) << 4)
+#define         SPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         SPLL_PDIV_A(x)                          ((x) << 20)
+#define         SPLL_PDIV_A_MASK                        (0x7f << 20)
+#define         SPLL_PDIV_A_SHIFT                       20
+#define CG_SPLL_FUNC_CNTL_2                             0x604
+#define         SCLK_MUX_SEL(x)                         ((x) << 0)
+#define         SCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_SPLL_FUNC_CNTL_3                             0x608
+#define         SPLL_FB_DIV(x)                          ((x) << 0)
+#define         SPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         SPLL_FB_DIV_SHIFT                       0
+#define         SPLL_DITHEN                             (1 << 28)
+
+#define MPLL_CNTL_MODE                                  0x61c
+#       define SS_SSEN                                  (1 << 24)
+#       define SS_DSMODE_EN                             (1 << 25)
+
+#define MPLL_AD_FUNC_CNTL                               0x624
+#define         CLKF(x)                                 ((x) << 0)
+#define         CLKF_MASK                               (0x7f << 0)
+#define         CLKR(x)                                 ((x) << 7)
+#define         CLKR_MASK                               (0x1f << 7)
+#define         CLKFRAC(x)                              ((x) << 12)
+#define         CLKFRAC_MASK                            (0x1f << 12)
+#define         YCLK_POST_DIV(x)                        ((x) << 17)
+#define         YCLK_POST_DIV_MASK                      (3 << 17)
+#define         IBIAS(x)                                ((x) << 20)
+#define         IBIAS_MASK                              (0x3ff << 20)
+#define         RESET                                   (1 << 30)
+#define         PDNB                                    (1 << 31)
+#define MPLL_AD_FUNC_CNTL_2                             0x628
+#define         BYPASS                                  (1 << 19)
+#define         BIAS_GEN_PDNB                           (1 << 24)
+#define         RESET_EN                                (1 << 25)
+#define         VCO_MODE                                (1 << 29)
+#define MPLL_DQ_FUNC_CNTL                               0x62c
+#define MPLL_DQ_FUNC_CNTL_2                             0x630
+
+#define GENERAL_PWRMGT                                  0x63c
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define ENABLE_GEN2PCIE                          (1 << 4)
+#       define ENABLE_GEN2XSP                           (1 << 5)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (3 << 6)
+#       define SW_SMIO_INDEX_SHIFT                      6
+#       define LOW_VOLT_D2_ACPI                         (1 << 8)
+#       define LOW_VOLT_D3_ACPI                         (1 << 9)
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define BACKBIAS_PAD_EN                          (1 << 18)
+#       define BACKBIAS_VALUE                           (1 << 19)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#       define AC_DC_SW                                 (1 << 24)
+
+#define SCLK_PWRMGT_CNTL                                  0x644
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_LOW_D1                                (1 << 1)
+#       define FIR_RESET                                  (1 << 4)
+#       define FIR_FORCE_TREND_SEL                        (1 << 5)
+#       define FIR_TREND_MODE                             (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 7)
+#       define GFX_CLK_FORCE_ON                           (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                        (1 << 9)
+#       define GFX_CLK_FORCE_OFF                          (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                        (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                        (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                        (1 << 13)
+#       define DYN_LIGHT_SLEEP_EN                         (1 << 14)
+#define MCLK_PWRMGT_CNTL                                0x648
+#       define DLL_SPEED(x)                             ((x) << 0)
+#       define DLL_SPEED_MASK                           (0x1f << 0)
+#       define MPLL_PWRMGT_OFF                          (1 << 5)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCKA0_PDNB                             (1 << 8)
+#       define MRDCKA1_PDNB                             (1 << 9)
+#       define MRDCKB0_PDNB                             (1 << 10)
+#       define MRDCKB1_PDNB                             (1 << 11)
+#       define MRDCKC0_PDNB                             (1 << 12)
+#       define MRDCKC1_PDNB                             (1 << 13)
+#       define MRDCKD0_PDNB                             (1 << 14)
+#       define MRDCKD1_PDNB                             (1 << 15)
+#       define MRDCKA0_RESET                            (1 << 16)
+#       define MRDCKA1_RESET                            (1 << 17)
+#       define MRDCKB0_RESET                            (1 << 18)
+#       define MRDCKB1_RESET                            (1 << 19)
+#       define MRDCKC0_RESET                            (1 << 20)
+#       define MRDCKC1_RESET                            (1 << 21)
+#       define MRDCKD0_RESET                            (1 << 22)
+#       define MRDCKD1_RESET                            (1 << 23)
+#       define DLL_READY_READ                           (1 << 24)
+#       define USE_DISPLAY_GAP                          (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                (1 << 26)
+#       define MPLL_TURNOFF_D2                          (1 << 28)
+#define DLL_CNTL                                        0x64c
+#       define MRDCKA0_BYPASS                           (1 << 24)
+#       define MRDCKA1_BYPASS                           (1 << 25)
+#       define MRDCKB0_BYPASS                           (1 << 26)
+#       define MRDCKB1_BYPASS                           (1 << 27)
+#       define MRDCKC0_BYPASS                           (1 << 28)
+#       define MRDCKC1_BYPASS                           (1 << 29)
+#       define MRDCKD0_BYPASS                           (1 << 30)
+#       define MRDCKD1_BYPASS                           (1 << 31)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                  0x66c
+#       define CURRENT_STATE_INDEX_MASK                   (0xf << 4)
+#       define CURRENT_STATE_INDEX_SHIFT                  4
+
+#define CG_AT                                           0x6d4
+#       define CG_R(x)                                  ((x) << 0)
+#       define CG_R_MASK                                (0xffff << 0)
+#       define CG_L(x)                                  ((x) << 16)
+#       define CG_L_MASK                                (0xffff << 16)
+
+#define CG_BIF_REQ_AND_RSP                              0x7f4
+#define         CG_CLIENT_REQ(x)                        ((x) << 0)
+#define         CG_CLIENT_REQ_MASK                      (0xff << 0)
+#define         CG_CLIENT_REQ_SHIFT                     0
+#define         CG_CLIENT_RESP(x)                       ((x) << 8)
+#define         CG_CLIENT_RESP_MASK                     (0xff << 8)
+#define         CG_CLIENT_RESP_SHIFT                    8
+#define         CLIENT_CG_REQ(x)                        ((x) << 16)
+#define         CLIENT_CG_REQ_MASK                      (0xff << 16)
+#define         CLIENT_CG_REQ_SHIFT                     16
+#define         CLIENT_CG_RESP(x)                       ((x) << 24)
+#define         CLIENT_CG_RESP_MASK                     (0xff << 24)
+#define         CLIENT_CG_RESP_SHIFT                    24
+
+#define CG_SPLL_SPREAD_SPECTRUM                         0x790
+#define         SSEN                                    (1 << 0)
+#define         CLK_S(x)                                ((x) << 4)
+#define         CLK_S_MASK                              (0xfff << 4)
+#define         CLK_S_SHIFT                             4
+#define CG_SPLL_SPREAD_SPECTRUM_2                       0x794
+#define         CLK_V(x)                                ((x) << 0)
+#define         CLK_V_MASK                              (0x3ffffff << 0)
+#define         CLK_V_SHIFT                             0
+
+#define SMC_SCRATCH0                                    0x81c
+
+#define CG_SPLL_FUNC_CNTL_4                             0x850
+
+#define MPLL_SS1                                        0x85c
+#define         CLKV(x)                                 ((x) << 0)
+#define         CLKV_MASK                               (0x3ffffff << 0)
+#define MPLL_SS2                                        0x860
+#define         CLKS(x)                                 ((x) << 0)
+#define         CLKS_MASK                               (0xfff << 0)
+
+#define CG_CAC_CTRL                                     0x88c
+#define         TID_CNT(x)                              ((x) << 0)
+#define         TID_CNT_MASK                            (0x3fff << 0)
+#define         TID_UNIT(x)                             ((x) << 14)
+#define         TID_UNIT_MASK                           (0xf << 14)
+
+#define CG_IND_ADDR                                     0x8f8
+#define CG_IND_DATA                                     0x8fc
+/* CGIND regs */
+#define CG_CGTT_LOCAL_0                                 0x00
+#define CG_CGTT_LOCAL_1                                 0x01
+
+#define MC_CG_CONFIG                                    0x25bc
+#define         MCDW_WR_ENABLE                          (1 << 0)
+#define         MCDX_WR_ENABLE                          (1 << 1)
+#define         MCDY_WR_ENABLE                          (1 << 2)
+#define         MCDZ_WR_ENABLE                          (1 << 3)
+#define         MC_RD_ENABLE(x)                         ((x) << 4)
+#define         MC_RD_ENABLE_MASK                       (3 << 4)
+#define         INDEX(x)                                ((x) << 6)
+#define         INDEX_MASK                              (0xfff << 6)
+#define         INDEX_SHIFT                             6
+
+#define MC_ARB_CAC_CNTL                                 0x2750
+#define         ENABLE                                  (1 << 0)
+#define         READ_WEIGHT(x)                          ((x) << 1)
+#define         READ_WEIGHT_MASK                        (0x3f << 1)
+#define         READ_WEIGHT_SHIFT                       1
+#define         WRITE_WEIGHT(x)                         ((x) << 7)
+#define         WRITE_WEIGHT_MASK                       (0x3f << 7)
+#define         WRITE_WEIGHT_SHIFT                      7
+#define         ALLOW_OVERFLOW                          (1 << 13)
+
+#define MC_ARB_DRAM_TIMING                              0x2774
+#define MC_ARB_DRAM_TIMING2                             0x2778
+
+#define MC_ARB_RFSH_RATE                                0x27b0
+#define         POWERMODE0(x)                           ((x) << 0)
+#define         POWERMODE0_MASK                         (0xff << 0)
+#define         POWERMODE0_SHIFT                        0
+#define         POWERMODE1(x)                           ((x) << 8)
+#define         POWERMODE1_MASK                         (0xff << 8)
+#define         POWERMODE1_SHIFT                        8
+#define         POWERMODE2(x)                           ((x) << 16)
+#define         POWERMODE2_MASK                         (0xff << 16)
+#define         POWERMODE2_SHIFT                        16
+#define         POWERMODE3(x)                           ((x) << 24)
+#define         POWERMODE3_MASK                         (0xff << 24)
+#define         POWERMODE3_SHIFT                        24
+
+#define MC_ARB_CG                                       0x27e8
+#define         CG_ARB_REQ(x)                           ((x) << 0)
+#define         CG_ARB_REQ_MASK                         (0xff << 0)
+#define         CG_ARB_REQ_SHIFT                        0
+#define         CG_ARB_RESP(x)                          ((x) << 8)
+#define         CG_ARB_RESP_MASK                        (0xff << 8)
+#define         CG_ARB_RESP_SHIFT                       8
+#define         ARB_CG_REQ(x)                           ((x) << 16)
+#define         ARB_CG_REQ_MASK                         (0xff << 16)
+#define         ARB_CG_REQ_SHIFT                        16
+#define         ARB_CG_RESP(x)                          ((x) << 24)
+#define         ARB_CG_RESP_MASK                        (0xff << 24)
+#define         ARB_CG_RESP_SHIFT                       24
+
+#define MC_ARB_DRAM_TIMING_1                            0x27f0
+#define MC_ARB_DRAM_TIMING_2                            0x27f4
+#define MC_ARB_DRAM_TIMING_3                            0x27f8
+#define MC_ARB_DRAM_TIMING2_1                           0x27fc
+#define MC_ARB_DRAM_TIMING2_2                           0x2800
+#define MC_ARB_DRAM_TIMING2_3                           0x2804
+#define MC_ARB_BURST_TIME                               0x2808
+#define         STATE0(x)                               ((x) << 0)
+#define         STATE0_MASK                             (0x1f << 0)
+#define         STATE0_SHIFT                            0
+#define         STATE1(x)                               ((x) << 5)
+#define         STATE1_MASK                             (0x1f << 5)
+#define         STATE1_SHIFT                            5
+#define         STATE2(x)                               ((x) << 10)
+#define         STATE2_MASK                             (0x1f << 10)
+#define         STATE2_SHIFT                            10
+#define         STATE3(x)                               ((x) << 15)
+#define         STATE3_MASK                             (0x1f << 15)
+#define         STATE3_SHIFT                            15
+
+#define MC_CG_DATAPORT                                  0x2884
+
+#define MC_SEQ_RAS_TIMING                               0x28a0
+#define MC_SEQ_CAS_TIMING                               0x28a4
+#define MC_SEQ_MISC_TIMING                              0x28a8
+#define MC_SEQ_MISC_TIMING2                             0x28ac
+#define MC_SEQ_PMG_TIMING                               0x28b0
+#define MC_SEQ_RD_CTL_D0                                0x28b4
+#define MC_SEQ_RD_CTL_D1                                0x28b8
+#define MC_SEQ_WR_CTL_D0                                0x28bc
+#define MC_SEQ_WR_CTL_D1                                0x28c0
+
+#define MC_SEQ_MISC0                                    0x2a00
+#define         MC_SEQ_MISC0_GDDR5_SHIFT                28
+#define         MC_SEQ_MISC0_GDDR5_MASK                 0xf0000000
+#define         MC_SEQ_MISC0_GDDR5_VALUE                5
+#define MC_SEQ_MISC1                                    0x2a04
+#define MC_SEQ_RESERVE_M                                0x2a08
+#define MC_PMG_CMD_EMRS                                 0x2a0c
+
+#define MC_SEQ_MISC3                                    0x2a2c
+
+#define MC_SEQ_MISC5                                    0x2a54
+#define MC_SEQ_MISC6                                    0x2a58
+
+#define MC_SEQ_MISC7                                    0x2a64
+
+#define MC_SEQ_RAS_TIMING_LP                            0x2a6c
+#define MC_SEQ_CAS_TIMING_LP                            0x2a70
+#define MC_SEQ_MISC_TIMING_LP                           0x2a74
+#define MC_SEQ_MISC_TIMING2_LP                          0x2a78
+#define MC_SEQ_WR_CTL_D0_LP                             0x2a7c
+#define MC_SEQ_WR_CTL_D1_LP                             0x2a80
+#define MC_SEQ_PMG_CMD_EMRS_LP                          0x2a84
+#define MC_SEQ_PMG_CMD_MRS_LP                           0x2a88
+
+#define MC_PMG_CMD_MRS                                  0x2aac
+
+#define MC_SEQ_RD_CTL_D0_LP                             0x2b1c
+#define MC_SEQ_RD_CTL_D1_LP                             0x2b20
+
+#define MC_PMG_CMD_MRS1                                 0x2b44
+#define MC_SEQ_PMG_CMD_MRS1_LP                          0x2b48
+#define MC_SEQ_PMG_TIMING_LP                            0x2b4c
+
+#define MC_PMG_CMD_MRS2                                 0x2b5c
+#define MC_SEQ_PMG_CMD_MRS2_LP                          0x2b60
+
+#define LB_SYNC_RESET_SEL                               0x6b28
+#define         LB_SYNC_RESET_SEL_MASK                  (3 << 0)
+#define         LB_SYNC_RESET_SEL_SHIFT                 0
+
+#define DC_STUTTER_CNTL                                 0x6b30
+#define         DC_STUTTER_ENABLE_A                     (1 << 0)
+#define         DC_STUTTER_ENABLE_B                     (1 << 1)
+
+#define SQ_CAC_THRESHOLD                                0x8e4c
+#define         VSP(x)                                  ((x) << 0)
+#define         VSP_MASK                                (0xff << 0)
+#define         VSP_SHIFT                               0
+#define         VSP0(x)                                 ((x) << 8)
+#define         VSP0_MASK                               (0xff << 8)
+#define         VSP0_SHIFT                              8
+#define         GPR(x)                                  ((x) << 16)
+#define         GPR_MASK                                (0xff << 16)
+#define         GPR_SHIFT                               16
+
+#define SQ_POWER_THROTTLE                               0x8e58
+#define         MIN_POWER(x)                            ((x) << 0)
+#define         MIN_POWER_MASK                          (0x3fff << 0)
+#define         MIN_POWER_SHIFT                         0
+#define         MAX_POWER(x)                            ((x) << 16)
+#define         MAX_POWER_MASK                          (0x3fff << 16)
+#define         MAX_POWER_SHIFT                         0
+#define SQ_POWER_THROTTLE2                              0x8e5c
+#define         MAX_POWER_DELTA(x)                      ((x) << 0)
+#define         MAX_POWER_DELTA_MASK                    (0x3fff << 0)
+#define         MAX_POWER_DELTA_SHIFT                   0
+#define         STI_SIZE(x)                             ((x) << 16)
+#define         STI_SIZE_MASK                           (0x3ff << 16)
+#define         STI_SIZE_SHIFT                          16
+#define         LTI_RATIO(x)                            ((x) << 27)
+#define         LTI_RATIO_MASK                          (0xf << 27)
+#define         LTI_RATIO_SHIFT                         27
+
+/* CG indirect registers */
+#define CG_CAC_REGION_1_WEIGHT_0                        0x83
+#define         WEIGHT_TCP_SIG0(x)                      ((x) << 0)
+#define         WEIGHT_TCP_SIG0_MASK                    (0x3f << 0)
+#define         WEIGHT_TCP_SIG0_SHIFT                   0
+#define         WEIGHT_TCP_SIG1(x)                      ((x) << 6)
+#define         WEIGHT_TCP_SIG1_MASK                    (0x3f << 6)
+#define         WEIGHT_TCP_SIG1_SHIFT                   6
+#define         WEIGHT_TA_SIG(x)                        ((x) << 12)
+#define         WEIGHT_TA_SIG_MASK                      (0x3f << 12)
+#define         WEIGHT_TA_SIG_SHIFT                     12
+#define CG_CAC_REGION_1_WEIGHT_1                        0x84
+#define         WEIGHT_TCC_EN0(x)                       ((x) << 0)
+#define         WEIGHT_TCC_EN0_MASK                     (0x3f << 0)
+#define         WEIGHT_TCC_EN0_SHIFT                    0
+#define         WEIGHT_TCC_EN1(x)                       ((x) << 6)
+#define         WEIGHT_TCC_EN1_MASK                     (0x3f << 6)
+#define         WEIGHT_TCC_EN1_SHIFT                    6
+#define         WEIGHT_TCC_EN2(x)                       ((x) << 12)
+#define         WEIGHT_TCC_EN2_MASK                     (0x3f << 12)
+#define         WEIGHT_TCC_EN2_SHIFT                    12
+#define         WEIGHT_TCC_EN3(x)                       ((x) << 18)
+#define         WEIGHT_TCC_EN3_MASK                     (0x3f << 18)
+#define         WEIGHT_TCC_EN3_SHIFT                    18
+#define CG_CAC_REGION_2_WEIGHT_0                        0x85
+#define         WEIGHT_CB_EN0(x)                        ((x) << 0)
+#define         WEIGHT_CB_EN0_MASK                      (0x3f << 0)
+#define         WEIGHT_CB_EN0_SHIFT                     0
+#define         WEIGHT_CB_EN1(x)                        ((x) << 6)
+#define         WEIGHT_CB_EN1_MASK                      (0x3f << 6)
+#define         WEIGHT_CB_EN1_SHIFT                     6
+#define         WEIGHT_CB_EN2(x)                        ((x) << 12)
+#define         WEIGHT_CB_EN2_MASK                      (0x3f << 12)
+#define         WEIGHT_CB_EN2_SHIFT                     12
+#define         WEIGHT_CB_EN3(x)                        ((x) << 18)
+#define         WEIGHT_CB_EN3_MASK                      (0x3f << 18)
+#define         WEIGHT_CB_EN3_SHIFT                     18
+#define CG_CAC_REGION_2_WEIGHT_1                        0x86
+#define         WEIGHT_DB_SIG0(x)                       ((x) << 0)
+#define         WEIGHT_DB_SIG0_MASK                     (0x3f << 0)
+#define         WEIGHT_DB_SIG0_SHIFT                    0
+#define         WEIGHT_DB_SIG1(x)                       ((x) << 6)
+#define         WEIGHT_DB_SIG1_MASK                     (0x3f << 6)
+#define         WEIGHT_DB_SIG1_SHIFT                    6
+#define         WEIGHT_DB_SIG2(x)                       ((x) << 12)
+#define         WEIGHT_DB_SIG2_MASK                     (0x3f << 12)
+#define         WEIGHT_DB_SIG2_SHIFT                    12
+#define         WEIGHT_DB_SIG3(x)                       ((x) << 18)
+#define         WEIGHT_DB_SIG3_MASK                     (0x3f << 18)
+#define         WEIGHT_DB_SIG3_SHIFT                    18
+#define CG_CAC_REGION_2_WEIGHT_2                        0x87
+#define         WEIGHT_SXM_SIG0(x)                      ((x) << 0)
+#define         WEIGHT_SXM_SIG0_MASK                    (0x3f << 0)
+#define         WEIGHT_SXM_SIG0_SHIFT                   0
+#define         WEIGHT_SXM_SIG1(x)                      ((x) << 6)
+#define         WEIGHT_SXM_SIG1_MASK                    (0x3f << 6)
+#define         WEIGHT_SXM_SIG1_SHIFT                   6
+#define         WEIGHT_SXM_SIG2(x)                      ((x) << 12)
+#define         WEIGHT_SXM_SIG2_MASK                    (0x3f << 12)
+#define         WEIGHT_SXM_SIG2_SHIFT                   12
+#define         WEIGHT_SXS_SIG0(x)                      ((x) << 18)
+#define         WEIGHT_SXS_SIG0_MASK                    (0x3f << 18)
+#define         WEIGHT_SXS_SIG0_SHIFT                   18
+#define         WEIGHT_SXS_SIG1(x)                      ((x) << 24)
+#define         WEIGHT_SXS_SIG1_MASK                    (0x3f << 24)
+#define         WEIGHT_SXS_SIG1_SHIFT                   24
+#define CG_CAC_REGION_3_WEIGHT_0                        0x88
+#define         WEIGHT_XBR_0(x)                         ((x) << 0)
+#define         WEIGHT_XBR_0_MASK                       (0x3f << 0)
+#define         WEIGHT_XBR_0_SHIFT                      0
+#define         WEIGHT_XBR_1(x)                         ((x) << 6)
+#define         WEIGHT_XBR_1_MASK                       (0x3f << 6)
+#define         WEIGHT_XBR_1_SHIFT                      6
+#define         WEIGHT_XBR_2(x)                         ((x) << 12)
+#define         WEIGHT_XBR_2_MASK                       (0x3f << 12)
+#define         WEIGHT_XBR_2_SHIFT                      12
+#define         WEIGHT_SPI_SIG0(x)                      ((x) << 18)
+#define         WEIGHT_SPI_SIG0_MASK                    (0x3f << 18)
+#define         WEIGHT_SPI_SIG0_SHIFT                   18
+#define CG_CAC_REGION_3_WEIGHT_1                        0x89
+#define         WEIGHT_SPI_SIG1(x)                      ((x) << 0)
+#define         WEIGHT_SPI_SIG1_MASK                    (0x3f << 0)
+#define         WEIGHT_SPI_SIG1_SHIFT                   0
+#define         WEIGHT_SPI_SIG2(x)                      ((x) << 6)
+#define         WEIGHT_SPI_SIG2_MASK                    (0x3f << 6)
+#define         WEIGHT_SPI_SIG2_SHIFT                   6
+#define         WEIGHT_SPI_SIG3(x)                      ((x) << 12)
+#define         WEIGHT_SPI_SIG3_MASK                    (0x3f << 12)
+#define         WEIGHT_SPI_SIG3_SHIFT                   12
+#define         WEIGHT_SPI_SIG4(x)                      ((x) << 18)
+#define         WEIGHT_SPI_SIG4_MASK                    (0x3f << 18)
+#define         WEIGHT_SPI_SIG4_SHIFT                   18
+#define         WEIGHT_SPI_SIG5(x)                      ((x) << 24)
+#define         WEIGHT_SPI_SIG5_MASK                    (0x3f << 24)
+#define         WEIGHT_SPI_SIG5_SHIFT                   24
+#define CG_CAC_REGION_4_WEIGHT_0                        0x8a
+#define         WEIGHT_LDS_SIG0(x)                      ((x) << 0)
+#define         WEIGHT_LDS_SIG0_MASK                    (0x3f << 0)
+#define         WEIGHT_LDS_SIG0_SHIFT                   0
+#define         WEIGHT_LDS_SIG1(x)                      ((x) << 6)
+#define         WEIGHT_LDS_SIG1_MASK                    (0x3f << 6)
+#define         WEIGHT_LDS_SIG1_SHIFT                   6
+#define         WEIGHT_SC(x)                            ((x) << 24)
+#define         WEIGHT_SC_MASK                          (0x3f << 24)
+#define         WEIGHT_SC_SHIFT                         24
+#define CG_CAC_REGION_4_WEIGHT_1                        0x8b
+#define         WEIGHT_BIF(x)                           ((x) << 0)
+#define         WEIGHT_BIF_MASK                         (0x3f << 0)
+#define         WEIGHT_BIF_SHIFT                        0
+#define         WEIGHT_CP(x)                            ((x) << 6)
+#define         WEIGHT_CP_MASK                          (0x3f << 6)
+#define         WEIGHT_CP_SHIFT                         6
+#define         WEIGHT_PA_SIG0(x)                       ((x) << 12)
+#define         WEIGHT_PA_SIG0_MASK                     (0x3f << 12)
+#define         WEIGHT_PA_SIG0_SHIFT                    12
+#define         WEIGHT_PA_SIG1(x)                       ((x) << 18)
+#define         WEIGHT_PA_SIG1_MASK                     (0x3f << 18)
+#define         WEIGHT_PA_SIG1_SHIFT                    18
+#define         WEIGHT_VGT_SIG0(x)                      ((x) << 24)
+#define         WEIGHT_VGT_SIG0_MASK                    (0x3f << 24)
+#define         WEIGHT_VGT_SIG0_SHIFT                   24
+#define CG_CAC_REGION_4_WEIGHT_2                        0x8c
+#define         WEIGHT_VGT_SIG1(x)                      ((x) << 0)
+#define         WEIGHT_VGT_SIG1_MASK                    (0x3f << 0)
+#define         WEIGHT_VGT_SIG1_SHIFT                   0
+#define         WEIGHT_VGT_SIG2(x)                      ((x) << 6)
+#define         WEIGHT_VGT_SIG2_MASK                    (0x3f << 6)
+#define         WEIGHT_VGT_SIG2_SHIFT                   6
+#define         WEIGHT_DC_SIG0(x)                       ((x) << 12)
+#define         WEIGHT_DC_SIG0_MASK                     (0x3f << 12)
+#define         WEIGHT_DC_SIG0_SHIFT                    12
+#define         WEIGHT_DC_SIG1(x)                       ((x) << 18)
+#define         WEIGHT_DC_SIG1_MASK                     (0x3f << 18)
+#define         WEIGHT_DC_SIG1_SHIFT                    18
+#define         WEIGHT_DC_SIG2(x)                       ((x) << 24)
+#define         WEIGHT_DC_SIG2_MASK                     (0x3f << 24)
+#define         WEIGHT_DC_SIG2_SHIFT                    24
+#define CG_CAC_REGION_4_WEIGHT_3                        0x8d
+#define         WEIGHT_DC_SIG3(x)                       ((x) << 0)
+#define         WEIGHT_DC_SIG3_MASK                     (0x3f << 0)
+#define         WEIGHT_DC_SIG3_SHIFT                    0
+#define         WEIGHT_UVD_SIG0(x)                      ((x) << 6)
+#define         WEIGHT_UVD_SIG0_MASK                    (0x3f << 6)
+#define         WEIGHT_UVD_SIG0_SHIFT                   6
+#define         WEIGHT_UVD_SIG1(x)                      ((x) << 12)
+#define         WEIGHT_UVD_SIG1_MASK                    (0x3f << 12)
+#define         WEIGHT_UVD_SIG1_SHIFT                   12
+#define         WEIGHT_SPARE0(x)                        ((x) << 18)
+#define         WEIGHT_SPARE0_MASK                      (0x3f << 18)
+#define         WEIGHT_SPARE0_SHIFT                     18
+#define         WEIGHT_SPARE1(x)                        ((x) << 24)
+#define         WEIGHT_SPARE1_MASK                      (0x3f << 24)
+#define         WEIGHT_SPARE1_SHIFT                     24
+#define CG_CAC_REGION_5_WEIGHT_0                        0x8e
+#define         WEIGHT_SQ_VSP(x)                        ((x) << 0)
+#define         WEIGHT_SQ_VSP_MASK                      (0x3fff << 0)
+#define         WEIGHT_SQ_VSP_SHIFT                     0
+#define         WEIGHT_SQ_VSP0(x)                       ((x) << 14)
+#define         WEIGHT_SQ_VSP0_MASK                     (0x3fff << 14)
+#define         WEIGHT_SQ_VSP0_SHIFT                    14
+#define CG_CAC_REGION_4_OVERRIDE_4                      0xab
+#define         OVR_MODE_SPARE_0(x)                     ((x) << 16)
+#define         OVR_MODE_SPARE_0_MASK                   (0x1 << 16)
+#define         OVR_MODE_SPARE_0_SHIFT                  16
+#define         OVR_VAL_SPARE_0(x)                      ((x) << 17)
+#define         OVR_VAL_SPARE_0_MASK                    (0x1 << 17)
+#define         OVR_VAL_SPARE_0_SHIFT                   17
+#define         OVR_MODE_SPARE_1(x)                     ((x) << 18)
+#define         OVR_MODE_SPARE_1_MASK                   (0x3f << 18)
+#define         OVR_MODE_SPARE_1_SHIFT                  18
+#define         OVR_VAL_SPARE_1(x)                      ((x) << 19)
+#define         OVR_VAL_SPARE_1_MASK                    (0x3f << 19)
+#define         OVR_VAL_SPARE_1_SHIFT                   19
+#define CG_CAC_REGION_5_WEIGHT_1                        0xb7
+#define         WEIGHT_SQ_GPR(x)                        ((x) << 0)
+#define         WEIGHT_SQ_GPR_MASK                      (0x3fff << 0)
+#define         WEIGHT_SQ_GPR_SHIFT                     0
+#define         WEIGHT_SQ_LDS(x)                        ((x) << 14)
+#define         WEIGHT_SQ_LDS_MASK                      (0x3fff << 14)
+#define         WEIGHT_SQ_LDS_SHIFT                     14
+
+/* PCIE link stuff */
+#define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 1)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 6)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_HW_VOLTAGE_IF_CONTROL(x)                ((x) << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_MASK              (3 << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_SHIFT             12
+#       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+#define MM_CFGREGS_CNTL                                   0x544c
+#       define MM_WR_TO_CFG_EN                            (1 << 3)
+#define LINK_CNTL2                                        0x88 /* F0 */
+#       define TARGET_LINK_SPEED_MASK                     (0xf << 0)
+#       define SELECTABLE_DEEMPHASIS                      (1 << 6)
+
 /*
  * UVD
  */
diff --git a/drivers/gpu/drm/radeon/nislands_smc.h b/drivers/gpu/drm/radeon/nislands_smc.h
new file mode 100644
index 000000000000..3cf8fc0d83f4
--- /dev/null
+++ b/drivers/gpu/drm/radeon/nislands_smc.h
@@ -0,0 +1,329 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __NISLANDS_SMC_H__
+#define __NISLANDS_SMC_H__
+
+#pragma pack(push, 1)
+
+#define NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16
+
+struct PP_NIslands_Dpm2PerfLevel
+{
+    uint8_t     MaxPS;
+    uint8_t     TgtAct;
+    uint8_t     MaxPS_StepInc;
+    uint8_t     MaxPS_StepDec;
+    uint8_t     PSST;
+    uint8_t     NearTDPDec;
+    uint8_t     AboveSafeInc;
+    uint8_t     BelowSafeInc;
+    uint8_t     PSDeltaLimit;
+    uint8_t     PSDeltaWin;
+    uint8_t     Reserved[6];
+};
+
+typedef struct PP_NIslands_Dpm2PerfLevel PP_NIslands_Dpm2PerfLevel;
+
+struct PP_NIslands_DPM2Parameters
+{
+    uint32_t    TDPLimit;
+    uint32_t    NearTDPLimit;
+    uint32_t    SafePowerLimit;
+    uint32_t    PowerBoostLimit;
+};
+typedef struct PP_NIslands_DPM2Parameters PP_NIslands_DPM2Parameters;
+
+struct NISLANDS_SMC_SCLK_VALUE
+{
+    uint32_t        vCG_SPLL_FUNC_CNTL;
+    uint32_t        vCG_SPLL_FUNC_CNTL_2;
+    uint32_t        vCG_SPLL_FUNC_CNTL_3;
+    uint32_t        vCG_SPLL_FUNC_CNTL_4;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM_2;
+    uint32_t        sclk_value;
+};
+
+typedef struct NISLANDS_SMC_SCLK_VALUE NISLANDS_SMC_SCLK_VALUE;
+
+struct NISLANDS_SMC_MCLK_VALUE
+{
+    uint32_t        vMPLL_FUNC_CNTL;
+    uint32_t        vMPLL_FUNC_CNTL_1;
+    uint32_t        vMPLL_FUNC_CNTL_2;
+    uint32_t        vMPLL_AD_FUNC_CNTL;
+    uint32_t        vMPLL_AD_FUNC_CNTL_2;
+    uint32_t        vMPLL_DQ_FUNC_CNTL;
+    uint32_t        vMPLL_DQ_FUNC_CNTL_2;
+    uint32_t        vMCLK_PWRMGT_CNTL;
+    uint32_t        vDLL_CNTL;
+    uint32_t        vMPLL_SS;
+    uint32_t        vMPLL_SS2;
+    uint32_t        mclk_value;
+};
+
+typedef struct NISLANDS_SMC_MCLK_VALUE NISLANDS_SMC_MCLK_VALUE;
+
+struct NISLANDS_SMC_VOLTAGE_VALUE
+{
+    uint16_t             value;
+    uint8_t              index;
+    uint8_t              padding;
+};
+
+typedef struct NISLANDS_SMC_VOLTAGE_VALUE NISLANDS_SMC_VOLTAGE_VALUE;
+
+struct NISLANDS_SMC_HW_PERFORMANCE_LEVEL
+{
+    uint8_t                     arbValue;
+    uint8_t                     ACIndex;
+    uint8_t                     displayWatermark;
+    uint8_t                     gen2PCIE;
+    uint8_t                     reserved1;
+    uint8_t                     reserved2;
+    uint8_t                     strobeMode;
+    uint8_t                     mcFlags;
+    uint32_t                    aT;
+    uint32_t                    bSP;
+    NISLANDS_SMC_SCLK_VALUE     sclk;
+    NISLANDS_SMC_MCLK_VALUE     mclk;
+    NISLANDS_SMC_VOLTAGE_VALUE  vddc;
+    NISLANDS_SMC_VOLTAGE_VALUE  mvdd;
+    NISLANDS_SMC_VOLTAGE_VALUE  vddci;
+    NISLANDS_SMC_VOLTAGE_VALUE  std_vddc;
+    uint32_t                    powergate_en;
+    uint8_t                     hUp;
+    uint8_t                     hDown;
+    uint8_t                     stateFlags;
+    uint8_t                     arbRefreshState;
+    uint32_t                    SQPowerThrottle;
+    uint32_t                    SQPowerThrottle_2;
+    uint32_t                    reserved[2];
+    PP_NIslands_Dpm2PerfLevel   dpm2;
+};
+
+#define NISLANDS_SMC_STROBE_RATIO    0x0F
+#define NISLANDS_SMC_STROBE_ENABLE   0x10
+
+#define NISLANDS_SMC_MC_EDC_RD_FLAG  0x01
+#define NISLANDS_SMC_MC_EDC_WR_FLAG  0x02
+#define NISLANDS_SMC_MC_RTT_ENABLE   0x04
+#define NISLANDS_SMC_MC_STUTTER_EN   0x08
+
+typedef struct NISLANDS_SMC_HW_PERFORMANCE_LEVEL NISLANDS_SMC_HW_PERFORMANCE_LEVEL;
+
+struct NISLANDS_SMC_SWSTATE
+{
+    uint8_t                             flags;
+    uint8_t                             levelCount;
+    uint8_t                             padding2;
+    uint8_t                             padding3;
+    NISLANDS_SMC_HW_PERFORMANCE_LEVEL   levels[1];
+};
+
+typedef struct NISLANDS_SMC_SWSTATE NISLANDS_SMC_SWSTATE;
+
+#define NISLANDS_SMC_VOLTAGEMASK_VDDC  0
+#define NISLANDS_SMC_VOLTAGEMASK_MVDD  1
+#define NISLANDS_SMC_VOLTAGEMASK_VDDCI 2
+#define NISLANDS_SMC_VOLTAGEMASK_MAX   4
+
+struct NISLANDS_SMC_VOLTAGEMASKTABLE
+{
+    uint8_t  highMask[NISLANDS_SMC_VOLTAGEMASK_MAX];
+    uint32_t lowMask[NISLANDS_SMC_VOLTAGEMASK_MAX];
+};
+
+typedef struct NISLANDS_SMC_VOLTAGEMASKTABLE NISLANDS_SMC_VOLTAGEMASKTABLE;
+
+#define NISLANDS_MAX_NO_VREG_STEPS 32
+
+struct NISLANDS_SMC_STATETABLE
+{
+    uint8_t                             thermalProtectType;
+    uint8_t                             systemFlags;
+    uint8_t                             maxVDDCIndexInPPTable;
+    uint8_t                             extraFlags;
+    uint8_t                             highSMIO[NISLANDS_MAX_NO_VREG_STEPS];
+    uint32_t                            lowSMIO[NISLANDS_MAX_NO_VREG_STEPS];
+    NISLANDS_SMC_VOLTAGEMASKTABLE       voltageMaskTable;
+    PP_NIslands_DPM2Parameters          dpm2Params;
+    NISLANDS_SMC_SWSTATE                initialState;
+    NISLANDS_SMC_SWSTATE                ACPIState;
+    NISLANDS_SMC_SWSTATE                ULVState;
+    NISLANDS_SMC_SWSTATE                driverState;
+    NISLANDS_SMC_HW_PERFORMANCE_LEVEL   dpmLevels[NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1];
+};
+
+typedef struct NISLANDS_SMC_STATETABLE NISLANDS_SMC_STATETABLE;
+
+#define NI_SMC_SOFT_REGISTERS_START        0x108
+
+#define NI_SMC_SOFT_REGISTER_mclk_chg_timeout        0x0
+#define NI_SMC_SOFT_REGISTER_delay_bbias             0xC
+#define NI_SMC_SOFT_REGISTER_delay_vreg              0x10
+#define NI_SMC_SOFT_REGISTER_delay_acpi              0x2C
+#define NI_SMC_SOFT_REGISTER_seq_index               0x64
+#define NI_SMC_SOFT_REGISTER_mvdd_chg_time           0x68
+#define NI_SMC_SOFT_REGISTER_mclk_switch_lim         0x78
+#define NI_SMC_SOFT_REGISTER_watermark_threshold     0x80
+#define NI_SMC_SOFT_REGISTER_mc_block_delay          0x84
+#define NI_SMC_SOFT_REGISTER_uvd_enabled             0x98
+
+#define SMC_NISLANDS_MC_TPP_CAC_NUM_OF_ENTRIES 16
+#define SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES 16
+#define SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES 4
+
+struct SMC_NISLANDS_MC_TPP_CAC_TABLE
+{
+    uint32_t    tpp[SMC_NISLANDS_MC_TPP_CAC_NUM_OF_ENTRIES];
+    uint32_t    cacValue[SMC_NISLANDS_MC_TPP_CAC_NUM_OF_ENTRIES];
+};
+
+typedef struct SMC_NISLANDS_MC_TPP_CAC_TABLE SMC_NISLANDS_MC_TPP_CAC_TABLE;
+
+
+struct PP_NIslands_CACTABLES
+{
+    uint32_t                cac_bif_lut[SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES];
+    uint32_t                cac_lkge_lut[SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES][SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES];
+
+    uint32_t                pwr_const;
+
+    uint32_t                dc_cacValue;
+    uint32_t                bif_cacValue;
+    uint32_t                lkge_pwr;
+
+    uint8_t                 cac_width;
+    uint8_t                 window_size_p2;
+
+    uint8_t                 num_drop_lsb;
+    uint8_t                 padding_0;
+
+    uint32_t                last_power;
+
+    uint8_t                 AllowOvrflw;
+    uint8_t                 MCWrWeight;
+    uint8_t                 MCRdWeight;
+    uint8_t                 padding_1[9];
+
+    uint8_t                 enableWinAvg;
+    uint8_t                 numWin_TDP;
+    uint8_t                 l2numWin_TDP;
+    uint8_t                 WinIndex;
+
+    uint32_t                dynPwr_TDP[4];
+    uint32_t                lkgePwr_TDP[4];
+    uint32_t                power_TDP[4];
+    uint32_t                avg_dynPwr_TDP;
+    uint32_t                avg_lkgePwr_TDP;
+    uint32_t                avg_power_TDP;
+    uint32_t                lts_power_TDP;
+    uint8_t                 lts_truncate_n;
+    uint8_t                 padding_2[7];
+};
+
+typedef struct PP_NIslands_CACTABLES PP_NIslands_CACTABLES;
+
+#define SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE 32
+#define SMC_NISLANDS_MC_REGISTER_ARRAY_SET_COUNT 20
+
+struct SMC_NIslands_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMC_NIslands_MCRegisterAddress SMC_NIslands_MCRegisterAddress;
+
+
+struct SMC_NIslands_MCRegisterSet
+{
+    uint32_t value[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMC_NIslands_MCRegisterSet SMC_NIslands_MCRegisterSet;
+
+struct SMC_NIslands_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMC_NIslands_MCRegisterAddress      address[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+    SMC_NIslands_MCRegisterSet          data[SMC_NISLANDS_MC_REGISTER_ARRAY_SET_COUNT];
+};
+
+typedef struct SMC_NIslands_MCRegisters SMC_NIslands_MCRegisters;
+
+struct SMC_NIslands_MCArbDramTimingRegisterSet
+{
+    uint32_t mc_arb_dram_timing;
+    uint32_t mc_arb_dram_timing2;
+    uint8_t  mc_arb_rfsh_rate;
+    uint8_t  padding[3];
+};
+
+typedef struct SMC_NIslands_MCArbDramTimingRegisterSet SMC_NIslands_MCArbDramTimingRegisterSet;
+
+struct SMC_NIslands_MCArbDramTimingRegisters
+{
+    uint8_t                                     arb_current;
+    uint8_t                                     reserved[3];
+    SMC_NIslands_MCArbDramTimingRegisterSet     data[20];
+};
+
+typedef struct SMC_NIslands_MCArbDramTimingRegisters SMC_NIslands_MCArbDramTimingRegisters;
+
+struct SMC_NISLANDS_SPLL_DIV_TABLE
+{
+    uint32_t    freq[256];
+    uint32_t    ss[256];
+};
+
+#define SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK  0x01ffffff
+#define SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT 0
+#define SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK   0xfe000000
+#define SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT  25
+#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK   0x000fffff
+#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT  0
+#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK   0xfff00000
+#define SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT  20
+
+typedef struct SMC_NISLANDS_SPLL_DIV_TABLE SMC_NISLANDS_SPLL_DIV_TABLE;
+
+#define NISLANDS_SMC_FIRMWARE_HEADER_LOCATION 0x100
+
+#define NISLANDS_SMC_FIRMWARE_HEADER_version                   0x0
+#define NISLANDS_SMC_FIRMWARE_HEADER_flags                     0x4
+#define NISLANDS_SMC_FIRMWARE_HEADER_softRegisters             0x8
+#define NISLANDS_SMC_FIRMWARE_HEADER_stateTable                0xC
+#define NISLANDS_SMC_FIRMWARE_HEADER_fanTable                  0x10
+#define NISLANDS_SMC_FIRMWARE_HEADER_cacTable                  0x14
+#define NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable           0x20
+#define NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable 0x2C
+#define NISLANDS_SMC_FIRMWARE_HEADER_spllTable                 0x30
+
+#pragma pack(pop)
+
+#endif
+
diff --git a/drivers/gpu/drm/radeon/ppsmc.h b/drivers/gpu/drm/radeon/ppsmc.h
new file mode 100644
index 000000000000..8fb1113a8fd7
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ppsmc.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef PP_SMC_H
+#define PP_SMC_H
+
+#pragma pack(push, 1)
+
+#define PPSMC_SWSTATE_FLAG_DC                           0x01
+#define PPSMC_SWSTATE_FLAG_UVD                          0x02
+#define PPSMC_SWSTATE_FLAG_VCE                          0x04
+#define PPSMC_SWSTATE_FLAG_PCIE_X1                      0x08
+
+#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL             0x00
+#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL             0x01
+#define PPSMC_THERMAL_PROTECT_TYPE_NONE                 0xff
+
+#define PPSMC_SYSTEMFLAG_GPIO_DC                        0x01
+#define PPSMC_SYSTEMFLAG_STEPVDDC                       0x02
+#define PPSMC_SYSTEMFLAG_GDDR5                          0x04
+#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP               0x08
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT                  0x10
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG           0x20
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_PROG_GPIO        0x40
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK              0x07
+#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK     0x08
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE   0x00
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE  0x01
+#define PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH      0x02
+
+#define PPSMC_DISPLAY_WATERMARK_LOW                     0
+#define PPSMC_DISPLAY_WATERMARK_HIGH                    1
+
+#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP    0x01
+#define PPSMC_STATEFLAG_POWERBOOST         0x02
+#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20
+#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS   0x40
+
+#define PPSMC_Result_OK             ((uint8_t)0x01)
+#define PPSMC_Result_Failed         ((uint8_t)0xFF)
+
+typedef uint8_t PPSMC_Result;
+
+#define PPSMC_MSG_Halt                      ((uint8_t)0x10)
+#define PPSMC_MSG_Resume                    ((uint8_t)0x11)
+#define PPSMC_MSG_ZeroLevelsDisabled        ((uint8_t)0x13)
+#define PPSMC_MSG_OneLevelsDisabled         ((uint8_t)0x14)
+#define PPSMC_MSG_TwoLevelsDisabled         ((uint8_t)0x15)
+#define PPSMC_MSG_EnableThermalInterrupt    ((uint8_t)0x16)
+#define PPSMC_MSG_RunningOnAC               ((uint8_t)0x17)
+#define PPSMC_MSG_SwitchToSwState           ((uint8_t)0x20)
+#define PPSMC_MSG_SwitchToInitialState      ((uint8_t)0x40)
+#define PPSMC_MSG_NoForcedLevel             ((uint8_t)0x41)
+#define PPSMC_MSG_SwitchToMinimumPower      ((uint8_t)0x51)
+#define PPSMC_MSG_ResumeFromMinimumPower    ((uint8_t)0x52)
+#define PPSMC_MSG_EnableCac                 ((uint8_t)0x53)
+#define PPSMC_MSG_DisableCac                ((uint8_t)0x54)
+#define PPSMC_TDPClampingActive             ((uint8_t)0x59)
+#define PPSMC_TDPClampingInactive           ((uint8_t)0x5A)
+#define PPSMC_MSG_NoDisplay                 ((uint8_t)0x5D)
+#define PPSMC_MSG_HasDisplay                ((uint8_t)0x5E)
+#define PPSMC_MSG_UVDPowerOFF               ((uint8_t)0x60)
+#define PPSMC_MSG_UVDPowerON                ((uint8_t)0x61)
+#define PPSMC_MSG_EnableULV                 ((uint8_t)0x62)
+#define PPSMC_MSG_DisableULV                ((uint8_t)0x63)
+#define PPSMC_MSG_EnterULV                  ((uint8_t)0x64)
+#define PPSMC_MSG_ExitULV                   ((uint8_t)0x65)
+#define PPSMC_CACLongTermAvgEnable          ((uint8_t)0x6E)
+#define PPSMC_CACLongTermAvgDisable         ((uint8_t)0x6F)
+#define PPSMC_MSG_CollectCAC_PowerCorreln   ((uint8_t)0x7A)
+#define PPSMC_FlushDataCache                ((uint8_t)0x80)
+#define PPSMC_MSG_SetEnabledLevels          ((uint8_t)0x82)
+#define PPSMC_MSG_SetForcedLevels           ((uint8_t)0x83)
+#define PPSMC_MSG_ResetToDefaults           ((uint8_t)0x84)
+#define PPSMC_MSG_EnableDTE                 ((uint8_t)0x87)
+#define PPSMC_MSG_DisableDTE                ((uint8_t)0x88)
+#define PPSMC_MSG_ThrottleOVRDSCLKDS        ((uint8_t)0x96)
+#define PPSMC_MSG_CancelThrottleOVRDSCLKDS  ((uint8_t)0x97)
+
+/* TN */
+#define PPSMC_MSG_DPM_Config                ((uint32_t) 0x102)
+#define PPSMC_MSG_DPM_ForceState            ((uint32_t) 0x104)
+#define PPSMC_MSG_PG_SIMD_Config            ((uint32_t) 0x108)
+#define PPSMC_MSG_DCE_RemoveVoltageAdjustment   ((uint32_t) 0x11d)
+#define PPSMC_MSG_DCE_AllowVoltageAdjustment    ((uint32_t) 0x11e)
+#define PPSMC_MSG_UVD_DPM_Config            ((uint32_t) 0x124)
+
+
+typedef uint16_t PPSMC_Msg;
+
+#pragma pack(pop)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 6948eb88c2b7..2d3655f7f41e 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -38,18 +38,7 @@
 #include "r600d.h"
 #include "atom.h"
 #include "avivod.h"
-
-#define PFP_UCODE_SIZE 576
-#define PM4_UCODE_SIZE 1792
-#define RLC_UCODE_SIZE 768
-#define R700_PFP_UCODE_SIZE 848
-#define R700_PM4_UCODE_SIZE 1360
-#define R700_RLC_UCODE_SIZE 1024
-#define EVERGREEN_PFP_UCODE_SIZE 1120
-#define EVERGREEN_PM4_UCODE_SIZE 1376
-#define EVERGREEN_RLC_UCODE_SIZE 768
-#define CAYMAN_RLC_UCODE_SIZE 1024
-#define ARUBA_RLC_UCODE_SIZE 1536
+#include "radeon_ucode.h"
 
 /* Firmware Names */
 MODULE_FIRMWARE("radeon/R600_pfp.bin");
@@ -68,24 +57,32 @@ MODULE_FIRMWARE("radeon/RS780_pfp.bin");
 MODULE_FIRMWARE("radeon/RS780_me.bin");
 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
 MODULE_FIRMWARE("radeon/RV770_me.bin");
+MODULE_FIRMWARE("radeon/RV770_smc.bin");
 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
 MODULE_FIRMWARE("radeon/RV730_me.bin");
+MODULE_FIRMWARE("radeon/RV730_smc.bin");
+MODULE_FIRMWARE("radeon/RV740_smc.bin");
 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
 MODULE_FIRMWARE("radeon/RV710_me.bin");
+MODULE_FIRMWARE("radeon/RV710_smc.bin");
 MODULE_FIRMWARE("radeon/R600_rlc.bin");
 MODULE_FIRMWARE("radeon/R700_rlc.bin");
 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
+MODULE_FIRMWARE("radeon/CEDAR_smc.bin");
 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
+MODULE_FIRMWARE("radeon/REDWOOD_smc.bin");
 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
+MODULE_FIRMWARE("radeon/JUNIPER_smc.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
+MODULE_FIRMWARE("radeon/CYPRESS_smc.bin");
 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
 MODULE_FIRMWARE("radeon/PALM_me.bin");
 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
@@ -108,6 +105,7 @@ static void r600_gpu_init(struct radeon_device *rdev);
 void r600_fini(struct radeon_device *rdev);
 void r600_irq_disable(struct radeon_device *rdev);
 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
+extern int evergreen_rlc_resume(struct radeon_device *rdev);
 
 /**
  * r600_get_xclk - get the xclk
@@ -2149,7 +2147,8 @@ int r600_init_microcode(struct radeon_device *rdev)
         struct platform_device *pdev;
         const char *chip_name;
         const char *rlc_chip_name;
-        size_t pfp_req_size, me_req_size, rlc_req_size;
+        const char *smc_chip_name = "RV770";
+        size_t pfp_req_size, me_req_size, rlc_req_size, smc_req_size = 0;
         char fw_name[30];
         int err;
 
@@ -2195,32 +2194,51 @@ int r600_init_microcode(struct radeon_device *rdev)
         case CHIP_RV770:
                 chip_name = "RV770";
                 rlc_chip_name = "R700";
+                smc_chip_name = "RV770";
+                smc_req_size = ALIGN(RV770_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_RV730:
-        case CHIP_RV740:
                 chip_name = "RV730";
                 rlc_chip_name = "R700";
+                smc_chip_name = "RV730";
+                smc_req_size = ALIGN(RV730_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_RV710:
                 chip_name = "RV710";
                 rlc_chip_name = "R700";
+                smc_chip_name = "RV710";
+                smc_req_size = ALIGN(RV710_SMC_UCODE_SIZE, 4);
+                break;
+        case CHIP_RV740:
+                chip_name = "RV730";
+                rlc_chip_name = "R700";
+                smc_chip_name = "RV740";
+                smc_req_size = ALIGN(RV740_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_CEDAR:
                 chip_name = "CEDAR";
                 rlc_chip_name = "CEDAR";
+                smc_chip_name = "CEDAR";
+                smc_req_size = ALIGN(CEDAR_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_REDWOOD:
                 chip_name = "REDWOOD";
                 rlc_chip_name = "REDWOOD";
+                smc_chip_name = "REDWOOD";
+                smc_req_size = ALIGN(REDWOOD_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_JUNIPER:
                 chip_name = "JUNIPER";
                 rlc_chip_name = "JUNIPER";
+                smc_chip_name = "JUNIPER";
+                smc_req_size = ALIGN(JUNIPER_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_CYPRESS:
         case CHIP_HEMLOCK:
                 chip_name = "CYPRESS";
                 rlc_chip_name = "CYPRESS";
+                smc_chip_name = "CYPRESS";
+                smc_req_size = ALIGN(CYPRESS_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_PALM:
                 chip_name = "PALM";
@@ -2246,9 +2264,9 @@ int r600_init_microcode(struct radeon_device *rdev)
                 me_req_size = R700_PM4_UCODE_SIZE * 4;
                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
         } else {
-                pfp_req_size = PFP_UCODE_SIZE * 4;
-                me_req_size = PM4_UCODE_SIZE * 12;
-                rlc_req_size = RLC_UCODE_SIZE * 4;
+                pfp_req_size = R600_PFP_UCODE_SIZE * 4;
+                me_req_size = R600_PM4_UCODE_SIZE * 12;
+                rlc_req_size = R600_RLC_UCODE_SIZE * 4;
         }
 
         DRM_INFO("Loading %s Microcode\n", chip_name);
@@ -2287,6 +2305,19 @@ int r600_init_microcode(struct radeon_device *rdev)
                 err = -EINVAL;
         }
 
+        if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) {
+                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", smc_chip_name);
+                err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
+                if (err)
+                        goto out;
+                if (rdev->smc_fw->size != smc_req_size) {
+                        printk(KERN_ERR
+                               "smc: Bogus length %zu in firmware \"%s\"\n",
+                               rdev->smc_fw->size, fw_name);
+                        err = -EINVAL;
+                }
+        }
+
 out:
         platform_device_unregister(pdev);
 
@@ -2301,6 +2332,8 @@ out:
                 rdev->me_fw = NULL;
                 release_firmware(rdev->rlc_fw);
                 rdev->rlc_fw = NULL;
+                release_firmware(rdev->smc_fw);
+                rdev->smc_fw = NULL;
         }
         return err;
 }
@@ -2331,13 +2364,13 @@ static int r600_cp_load_microcode(struct radeon_device *rdev)
 
         fw_data = (const __be32 *)rdev->me_fw->data;
         WREG32(CP_ME_RAM_WADDR, 0);
-        for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
+        for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++)
                 WREG32(CP_ME_RAM_DATA,
                        be32_to_cpup(fw_data++));
 
         fw_data = (const __be32 *)rdev->pfp_fw->data;
         WREG32(CP_PFP_UCODE_ADDR, 0);
-        for (i = 0; i < PFP_UCODE_SIZE; i++)
+        for (i = 0; i < R600_PFP_UCODE_SIZE; i++)
                 WREG32(CP_PFP_UCODE_DATA,
                        be32_to_cpup(fw_data++));
 
@@ -3789,7 +3822,7 @@ static void r600_rlc_start(struct radeon_device *rdev)
         WREG32(RLC_CNTL, RLC_ENABLE);
 }
 
-static int r600_rlc_init(struct radeon_device *rdev)
+static int r600_rlc_resume(struct radeon_device *rdev)
 {
         u32 i;
         const __be32 *fw_data;
@@ -3801,45 +3834,22 @@ static int r600_rlc_init(struct radeon_device *rdev)
 
         WREG32(RLC_HB_CNTL, 0);
 
-        if (rdev->family == CHIP_ARUBA) {
-                WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
-                WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
-        }
-        if (rdev->family <= CHIP_CAYMAN) {
-                WREG32(RLC_HB_BASE, 0);
-                WREG32(RLC_HB_RPTR, 0);
-                WREG32(RLC_HB_WPTR, 0);
-        }
-        if (rdev->family <= CHIP_CAICOS) {
-                WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
-                WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
-        }
+        WREG32(RLC_HB_BASE, 0);
+        WREG32(RLC_HB_RPTR, 0);
+        WREG32(RLC_HB_WPTR, 0);
+        WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
+        WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
         WREG32(RLC_MC_CNTL, 0);
         WREG32(RLC_UCODE_CNTL, 0);
 
         fw_data = (const __be32 *)rdev->rlc_fw->data;
-        if (rdev->family >= CHIP_ARUBA) {
-                for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
-                        WREG32(RLC_UCODE_ADDR, i);
-                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
-                }
-        } else if (rdev->family >= CHIP_CAYMAN) {
-                for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
-                        WREG32(RLC_UCODE_ADDR, i);
-                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
-                }
-        } else if (rdev->family >= CHIP_CEDAR) {
-                for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
-                        WREG32(RLC_UCODE_ADDR, i);
-                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
-                }
-        } else if (rdev->family >= CHIP_RV770) {
+        if (rdev->family >= CHIP_RV770) {
                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
                         WREG32(RLC_UCODE_ADDR, i);
                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
                 }
         } else {
-                for (i = 0; i < RLC_UCODE_SIZE; i++) {
+                for (i = 0; i < R600_RLC_UCODE_SIZE; i++) {
                         WREG32(RLC_UCODE_ADDR, i);
                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
                 }
@@ -3947,7 +3957,10 @@ int r600_irq_init(struct radeon_device *rdev)
         r600_disable_interrupts(rdev);
 
         /* init rlc */
-        ret = r600_rlc_init(rdev);
+        if (rdev->family >= CHIP_CEDAR)
+                ret = evergreen_rlc_resume(rdev);
+        else
+                ret = r600_rlc_resume(rdev);
         if (ret) {
                 r600_ih_ring_fini(rdev);
                 return ret;
@@ -4028,6 +4041,7 @@ int r600_irq_set(struct radeon_device *rdev)
         u32 hdmi0, hdmi1;
         u32 d1grph = 0, d2grph = 0;
         u32 dma_cntl;
+        u32 thermal_int = 0;
 
         if (!rdev->irq.installed) {
                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -4062,8 +4076,21 @@ int r600_irq_set(struct radeon_device *rdev)
                 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
                 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
         }
+
         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
 
+        if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
+                thermal_int = RREG32(CG_THERMAL_INT) &
+                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
+        } else if (rdev->family >= CHIP_RV770) {
+                thermal_int = RREG32(RV770_CG_THERMAL_INT) &
+                        ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
+        }
+        if (rdev->irq.dpm_thermal) {
+                DRM_DEBUG("dpm thermal\n");
+                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
+        }
+
         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
                 DRM_DEBUG("r600_irq_set: sw int\n");
                 cp_int_cntl |= RB_INT_ENABLE;
@@ -4145,6 +4172,11 @@ int r600_irq_set(struct radeon_device *rdev)
                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
         }
+        if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
+                WREG32(CG_THERMAL_INT, thermal_int);
+        } else if (rdev->family >= CHIP_RV770) {
+                WREG32(RV770_CG_THERMAL_INT, thermal_int);
+        }
 
         return 0;
 }
@@ -4336,6 +4368,7 @@ int r600_irq_process(struct radeon_device *rdev)
         u32 ring_index;
         bool queue_hotplug = false;
         bool queue_hdmi = false;
+        bool queue_thermal = false;
 
         if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
@@ -4503,6 +4536,16 @@ restart_ih:
                         DRM_DEBUG("IH: DMA trap\n");
                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
                         break;
+                case 230: /* thermal low to high */
+                        DRM_DEBUG("IH: thermal low to high\n");
+                        rdev->pm.dpm.thermal.high_to_low = false;
+                        queue_thermal = true;
+                        break;
+                case 231: /* thermal high to low */
+                        DRM_DEBUG("IH: thermal high to low\n");
+                        rdev->pm.dpm.thermal.high_to_low = true;
+                        queue_thermal = true;
+                        break;
                 case 233: /* GUI IDLE */
                         DRM_DEBUG("IH: GUI idle\n");
                         break;
@@ -4519,6 +4562,8 @@ restart_ih:
                 schedule_work(&rdev->hotplug_work);
         if (queue_hdmi)
                 schedule_work(&rdev->audio_work);
+        if (queue_thermal && rdev->pm.dpm_enabled)
+                schedule_work(&rdev->pm.dpm.thermal.work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         atomic_set(&rdev->ih.lock, 0);
diff --git a/drivers/gpu/drm/radeon/r600_dpm.c b/drivers/gpu/drm/radeon/r600_dpm.c
new file mode 100644
index 000000000000..76368c04f809
--- /dev/null
+++ b/drivers/gpu/drm/radeon/r600_dpm.c
@@ -0,0 +1,1024 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "r600d.h"
+#include "r600_dpm.h"
+#include "atom.h"
+
+const u32 r600_utc[R600_PM_NUMBER_OF_TC] =
+{
+        R600_UTC_DFLT_00,
+        R600_UTC_DFLT_01,
+        R600_UTC_DFLT_02,
+        R600_UTC_DFLT_03,
+        R600_UTC_DFLT_04,
+        R600_UTC_DFLT_05,
+        R600_UTC_DFLT_06,
+        R600_UTC_DFLT_07,
+        R600_UTC_DFLT_08,
+        R600_UTC_DFLT_09,
+        R600_UTC_DFLT_10,
+        R600_UTC_DFLT_11,
+        R600_UTC_DFLT_12,
+        R600_UTC_DFLT_13,
+        R600_UTC_DFLT_14,
+};
+
+const u32 r600_dtc[R600_PM_NUMBER_OF_TC] =
+{
+        R600_DTC_DFLT_00,
+        R600_DTC_DFLT_01,
+        R600_DTC_DFLT_02,
+        R600_DTC_DFLT_03,
+        R600_DTC_DFLT_04,
+        R600_DTC_DFLT_05,
+        R600_DTC_DFLT_06,
+        R600_DTC_DFLT_07,
+        R600_DTC_DFLT_08,
+        R600_DTC_DFLT_09,
+        R600_DTC_DFLT_10,
+        R600_DTC_DFLT_11,
+        R600_DTC_DFLT_12,
+        R600_DTC_DFLT_13,
+        R600_DTC_DFLT_14,
+};
+
+void r600_dpm_print_class_info(u32 class, u32 class2)
+{
+        printk("\tui class: ");
+        switch (class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
+        case ATOM_PPLIB_CLASSIFICATION_UI_NONE:
+        default:
+                printk("none\n");
+                break;
+        case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
+                printk("battery\n");
+                break;
+        case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED:
+                printk("balanced\n");
+                break;
+        case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
+                printk("performance\n");
+                break;
+        }
+        printk("\tinternal class: ");
+        if (((class & ~ATOM_PPLIB_CLASSIFICATION_UI_MASK) == 0) &&
+            (class2 == 0))
+                printk("none");
+        else {
+                if (class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+                        printk("boot ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
+                        printk("thermal ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE)
+                        printk("limited_pwr ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_REST)
+                        printk("rest ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_FORCED)
+                        printk("forced ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE)
+                        printk("3d_perf ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_OVERDRIVETEMPLATE)
+                        printk("ovrdrv ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                        printk("uvd ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_3DLOW)
+                        printk("3d_low ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_ACPI)
+                        printk("acpi ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE)
+                        printk("uvd_hd2 ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE)
+                        printk("uvd_hd ");
+                if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE)
+                        printk("uvd_sd ");
+                if (class2 & ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2)
+                        printk("limited_pwr2 ");
+                if (class2 & ATOM_PPLIB_CLASSIFICATION2_ULV)
+                        printk("ulv ");
+                if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC)
+                        printk("uvd_mvc ");
+        }
+        printk("\n");
+}
+
+void r600_dpm_print_cap_info(u32 caps)
+{
+        printk("\tcaps: ");
+        if (caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY)
+                printk("single_disp ");
+        if (caps & ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK)
+                printk("video ");
+        if (caps & ATOM_PPLIB_DISALLOW_ON_DC)
+                printk("no_dc ");
+        printk("\n");
+}
+
+void r600_dpm_print_ps_status(struct radeon_device *rdev,
+                              struct radeon_ps *rps)
+{
+        printk("\tstatus: ");
+        if (rps == rdev->pm.dpm.current_ps)
+                printk("c ");
+        if (rps == rdev->pm.dpm.requested_ps)
+                printk("r ");
+        if (rps == rdev->pm.dpm.boot_ps)
+                printk("b ");
+        printk("\n");
+}
+
+void r600_calculate_u_and_p(u32 i, u32 r_c, u32 p_b,
+                            u32 *p, u32 *u)
+{
+        u32 b_c = 0;
+        u32 i_c;
+        u32 tmp;
+
+        i_c = (i * r_c) / 100;
+        tmp = i_c >> p_b;
+
+        while (tmp) {
+                b_c++;
+                tmp >>= 1;
+        }
+
+        *u = (b_c + 1) / 2;
+        *p = i_c / (1 << (2 * (*u)));
+}
+
+int r600_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th)
+{
+        u32 k, a, ah, al;
+        u32 t1;
+
+        if ((fl == 0) || (fh == 0) || (fl > fh))
+                return -EINVAL;
+
+        k = (100 * fh) / fl;
+        t1 = (t * (k - 100));
+        a = (1000 * (100 * h + t1)) / (10000 + (t1 / 100));
+        a = (a + 5) / 10;
+        ah = ((a * t) + 5000) / 10000;
+        al = a - ah;
+
+        *th = t - ah;
+        *tl = t + al;
+
+        return 0;
+}
+
+void r600_gfx_clockgating_enable(struct radeon_device *rdev, bool enable)
+{
+        int i;
+
+        if (enable) {
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+        } else {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+
+                WREG32(CG_RLC_REQ_AND_RSP, 0x2);
+
+                for (i = 0; i < rdev->usec_timeout; i++) {
+                        if (((RREG32(CG_RLC_REQ_AND_RSP) & CG_RLC_RSP_TYPE_MASK) >> CG_RLC_RSP_TYPE_SHIFT) == 1)
+                                break;
+                        udelay(1);
+                }
+
+                WREG32(CG_RLC_REQ_AND_RSP, 0x0);
+
+                WREG32(GRBM_PWR_CNTL, 0x1);
+                RREG32(GRBM_PWR_CNTL);
+        }
+}
+
+void r600_dynamicpm_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
+}
+
+void r600_enable_thermal_protection(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+        else
+                WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+}
+
+void r600_enable_acpi_pm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
+}
+
+void r600_enable_dynamic_pcie_gen2(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
+}
+
+bool r600_dynamicpm_enabled(struct radeon_device *rdev)
+{
+        if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
+                return true;
+        else
+                return false;
+}
+
+void r600_enable_sclk_control(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, 0, ~SCLK_PWRMGT_OFF);
+        else
+                WREG32_P(GENERAL_PWRMGT, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
+}
+
+void r600_enable_mclk_control(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
+        else
+                WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
+}
+
+void r600_enable_spll_bypass(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(CG_SPLL_FUNC_CNTL, SPLL_BYPASS_EN, ~SPLL_BYPASS_EN);
+        else
+                WREG32_P(CG_SPLL_FUNC_CNTL, 0, ~SPLL_BYPASS_EN);
+}
+
+void r600_wait_for_spll_change(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(CG_SPLL_FUNC_CNTL) & SPLL_CHG_STATUS)
+                        break;
+                udelay(1);
+        }
+}
+
+void r600_set_bsp(struct radeon_device *rdev, u32 u, u32 p)
+{
+        WREG32(CG_BSP, BSP(p) | BSU(u));
+}
+
+void r600_set_at(struct radeon_device *rdev,
+                 u32 l_to_m, u32 m_to_h,
+                 u32 h_to_m, u32 m_to_l)
+{
+        WREG32(CG_RT, FLS(l_to_m) | FMS(m_to_h));
+        WREG32(CG_LT, FHS(h_to_m) | FMS(m_to_l));
+}
+
+void r600_set_tc(struct radeon_device *rdev,
+                 u32 index, u32 u_t, u32 d_t)
+{
+        WREG32(CG_FFCT_0 + (index * 4), UTC_0(u_t) | DTC_0(d_t));
+}
+
+void r600_select_td(struct radeon_device *rdev,
+                    enum r600_td td)
+{
+        if (td == R600_TD_AUTO)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
+        else
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
+        if (td == R600_TD_UP)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
+        if (td == R600_TD_DOWN)
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
+}
+
+void r600_set_vrc(struct radeon_device *rdev, u32 vrv)
+{
+        WREG32(CG_FTV, vrv);
+}
+
+void r600_set_tpu(struct radeon_device *rdev, u32 u)
+{
+        WREG32_P(CG_TPC, TPU(u), ~TPU_MASK);
+}
+
+void r600_set_tpc(struct radeon_device *rdev, u32 c)
+{
+        WREG32_P(CG_TPC, TPCC(c), ~TPCC_MASK);
+}
+
+void r600_set_sstu(struct radeon_device *rdev, u32 u)
+{
+        WREG32_P(CG_SSP, CG_SSTU(u), ~CG_SSTU_MASK);
+}
+
+void r600_set_sst(struct radeon_device *rdev, u32 t)
+{
+        WREG32_P(CG_SSP, CG_SST(t), ~CG_SST_MASK);
+}
+
+void r600_set_git(struct radeon_device *rdev, u32 t)
+{
+        WREG32_P(CG_GIT, CG_GICST(t), ~CG_GICST_MASK);
+}
+
+void r600_set_fctu(struct radeon_device *rdev, u32 u)
+{
+        WREG32_P(CG_FC_T, FC_TU(u), ~FC_TU_MASK);
+}
+
+void r600_set_fct(struct radeon_device *rdev, u32 t)
+{
+        WREG32_P(CG_FC_T, FC_T(t), ~FC_T_MASK);
+}
+
+void r600_set_ctxcgtt3d_rphc(struct radeon_device *rdev, u32 p)
+{
+        WREG32_P(CG_CTX_CGTT3D_R, PHC(p), ~PHC_MASK);
+}
+
+void r600_set_ctxcgtt3d_rsdc(struct radeon_device *rdev, u32 s)
+{
+        WREG32_P(CG_CTX_CGTT3D_R, SDC(s), ~SDC_MASK);
+}
+
+void r600_set_vddc3d_oorsu(struct radeon_device *rdev, u32 u)
+{
+        WREG32_P(CG_VDDC3D_OOR, SU(u), ~SU_MASK);
+}
+
+void r600_set_vddc3d_oorphc(struct radeon_device *rdev, u32 p)
+{
+        WREG32_P(CG_VDDC3D_OOR, PHC(p), ~PHC_MASK);
+}
+
+void r600_set_vddc3d_oorsdc(struct radeon_device *rdev, u32 s)
+{
+        WREG32_P(CG_VDDC3D_OOR, SDC(s), ~SDC_MASK);
+}
+
+void r600_set_mpll_lock_time(struct radeon_device *rdev, u32 lock_time)
+{
+        WREG32_P(MPLL_TIME, MPLL_LOCK_TIME(lock_time), ~MPLL_LOCK_TIME_MASK);
+}
+
+void r600_set_mpll_reset_time(struct radeon_device *rdev, u32 reset_time)
+{
+        WREG32_P(MPLL_TIME, MPLL_RESET_TIME(reset_time), ~MPLL_RESET_TIME_MASK);
+}
+
+void r600_engine_clock_entry_enable(struct radeon_device *rdev,
+                                    u32 index, bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2),
+                         STEP_0_SPLL_ENTRY_VALID, ~STEP_0_SPLL_ENTRY_VALID);
+        else
+                WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2),
+                         0, ~STEP_0_SPLL_ENTRY_VALID);
+}
+
+void r600_engine_clock_entry_enable_pulse_skipping(struct radeon_device *rdev,
+                                                   u32 index, bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2),
+                         STEP_0_SPLL_STEP_ENABLE, ~STEP_0_SPLL_STEP_ENABLE);
+        else
+                WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2),
+                         0, ~STEP_0_SPLL_STEP_ENABLE);
+}
+
+void r600_engine_clock_entry_enable_post_divider(struct radeon_device *rdev,
+                                                 u32 index, bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2),
+                         STEP_0_POST_DIV_EN, ~STEP_0_POST_DIV_EN);
+        else
+                WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART2 + (index * 4 * 2),
+                         0, ~STEP_0_POST_DIV_EN);
+}
+
+void r600_engine_clock_entry_set_post_divider(struct radeon_device *rdev,
+                                              u32 index, u32 divider)
+{
+        WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2),
+                 STEP_0_SPLL_POST_DIV(divider), ~STEP_0_SPLL_POST_DIV_MASK);
+}
+
+void r600_engine_clock_entry_set_reference_divider(struct radeon_device *rdev,
+                                                   u32 index, u32 divider)
+{
+        WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2),
+                 STEP_0_SPLL_REF_DIV(divider), ~STEP_0_SPLL_REF_DIV_MASK);
+}
+
+void r600_engine_clock_entry_set_feedback_divider(struct radeon_device *rdev,
+                                                  u32 index, u32 divider)
+{
+        WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2),
+                 STEP_0_SPLL_FB_DIV(divider), ~STEP_0_SPLL_FB_DIV_MASK);
+}
+
+void r600_engine_clock_entry_set_step_time(struct radeon_device *rdev,
+                                           u32 index, u32 step_time)
+{
+        WREG32_P(SCLK_FREQ_SETTING_STEP_0_PART1 + (index * 4 * 2),
+                 STEP_0_SPLL_STEP_TIME(step_time), ~STEP_0_SPLL_STEP_TIME_MASK);
+}
+
+void r600_vid_rt_set_ssu(struct radeon_device *rdev, u32 u)
+{
+        WREG32_P(VID_RT, SSTU(u), ~SSTU_MASK);
+}
+
+void r600_vid_rt_set_vru(struct radeon_device *rdev, u32 u)
+{
+        WREG32_P(VID_RT, VID_CRTU(u), ~VID_CRTU_MASK);
+}
+
+void r600_vid_rt_set_vrt(struct radeon_device *rdev, u32 rt)
+{
+        WREG32_P(VID_RT, VID_CRT(rt), ~VID_CRT_MASK);
+}
+
+void r600_voltage_control_enable_pins(struct radeon_device *rdev,
+                                      u64 mask)
+{
+        WREG32(LOWER_GPIO_ENABLE, mask & 0xffffffff);
+        WREG32(UPPER_GPIO_ENABLE, upper_32_bits(mask));
+}
+
+
+void r600_voltage_control_program_voltages(struct radeon_device *rdev,
+                                           enum r600_power_level index, u64 pins)
+{
+        u32 tmp, mask;
+        u32 ix = 3 - (3 & index);
+
+        WREG32(CTXSW_VID_LOWER_GPIO_CNTL + (ix * 4), pins & 0xffffffff);
+
+        mask = 7 << (3 * ix);
+        tmp = RREG32(VID_UPPER_GPIO_CNTL);
+        tmp = (tmp & ~mask) | ((pins >> (32 - (3 * ix))) & mask);
+        WREG32(VID_UPPER_GPIO_CNTL, tmp);
+}
+
+void r600_voltage_control_deactivate_static_control(struct radeon_device *rdev,
+                                                    u64 mask)
+{
+        u32 gpio;
+
+        gpio = RREG32(GPIOPAD_MASK);
+        gpio &= ~mask;
+        WREG32(GPIOPAD_MASK, gpio);
+
+        gpio = RREG32(GPIOPAD_EN);
+        gpio &= ~mask;
+        WREG32(GPIOPAD_EN, gpio);
+
+        gpio = RREG32(GPIOPAD_A);
+        gpio &= ~mask;
+        WREG32(GPIOPAD_A, gpio);
+}
+
+void r600_power_level_enable(struct radeon_device *rdev,
+                             enum r600_power_level index, bool enable)
+{
+        u32 ix = 3 - (3 & index);
+
+        if (enable)
+                WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), CTXSW_FREQ_STATE_ENABLE,
+                         ~CTXSW_FREQ_STATE_ENABLE);
+        else
+                WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), 0,
+                         ~CTXSW_FREQ_STATE_ENABLE);
+}
+
+void r600_power_level_set_voltage_index(struct radeon_device *rdev,
+                                        enum r600_power_level index, u32 voltage_index)
+{
+        u32 ix = 3 - (3 & index);
+
+        WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4),
+                 CTXSW_FREQ_VIDS_CFG_INDEX(voltage_index), ~CTXSW_FREQ_VIDS_CFG_INDEX_MASK);
+}
+
+void r600_power_level_set_mem_clock_index(struct radeon_device *rdev,
+                                          enum r600_power_level index, u32 mem_clock_index)
+{
+        u32 ix = 3 - (3 & index);
+
+        WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4),
+                 CTXSW_FREQ_MCLK_CFG_INDEX(mem_clock_index), ~CTXSW_FREQ_MCLK_CFG_INDEX_MASK);
+}
+
+void r600_power_level_set_eng_clock_index(struct radeon_device *rdev,
+                                          enum r600_power_level index, u32 eng_clock_index)
+{
+        u32 ix = 3 - (3 & index);
+
+        WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4),
+                 CTXSW_FREQ_SCLK_CFG_INDEX(eng_clock_index), ~CTXSW_FREQ_SCLK_CFG_INDEX_MASK);
+}
+
+void r600_power_level_set_watermark_id(struct radeon_device *rdev,
+                                       enum r600_power_level index,
+                                       enum r600_display_watermark watermark_id)
+{
+        u32 ix = 3 - (3 & index);
+        u32 tmp = 0;
+
+        if (watermark_id == R600_DISPLAY_WATERMARK_HIGH)
+                tmp = CTXSW_FREQ_DISPLAY_WATERMARK;
+        WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), tmp, ~CTXSW_FREQ_DISPLAY_WATERMARK);
+}
+
+void r600_power_level_set_pcie_gen2(struct radeon_device *rdev,
+                                    enum r600_power_level index, bool compatible)
+{
+        u32 ix = 3 - (3 & index);
+        u32 tmp = 0;
+
+        if (compatible)
+                tmp = CTXSW_FREQ_GEN2PCIE_VOLT;
+        WREG32_P(CTXSW_PROFILE_INDEX + (ix * 4), tmp, ~CTXSW_FREQ_GEN2PCIE_VOLT);
+}
+
+enum r600_power_level r600_power_level_get_current_index(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK;
+        tmp >>= CURRENT_PROFILE_INDEX_SHIFT;
+        return tmp;
+}
+
+enum r600_power_level r600_power_level_get_target_index(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_PROFILE_INDEX_MASK;
+        tmp >>= TARGET_PROFILE_INDEX_SHIFT;
+        return tmp;
+}
+
+void r600_power_level_set_enter_index(struct radeon_device *rdev,
+                                      enum r600_power_level index)
+{
+        WREG32_P(TARGET_AND_CURRENT_PROFILE_INDEX, DYN_PWR_ENTER_INDEX(index),
+                 ~DYN_PWR_ENTER_INDEX_MASK);
+}
+
+void r600_wait_for_power_level_unequal(struct radeon_device *rdev,
+                                       enum r600_power_level index)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (r600_power_level_get_target_index(rdev) != index)
+                        break;
+                udelay(1);
+        }
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (r600_power_level_get_current_index(rdev) != index)
+                        break;
+                udelay(1);
+        }
+}
+
+void r600_wait_for_power_level(struct radeon_device *rdev,
+                               enum r600_power_level index)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (r600_power_level_get_target_index(rdev) == index)
+                        break;
+                udelay(1);
+        }
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (r600_power_level_get_current_index(rdev) == index)
+                        break;
+                udelay(1);
+        }
+}
+
+void r600_start_dpm(struct radeon_device *rdev)
+{
+        r600_enable_sclk_control(rdev, false);
+        r600_enable_mclk_control(rdev, false);
+
+        r600_dynamicpm_enable(rdev, true);
+
+        radeon_wait_for_vblank(rdev, 0);
+        radeon_wait_for_vblank(rdev, 1);
+
+        r600_enable_spll_bypass(rdev, true);
+        r600_wait_for_spll_change(rdev);
+        r600_enable_spll_bypass(rdev, false);
+        r600_wait_for_spll_change(rdev);
+
+        r600_enable_spll_bypass(rdev, true);
+        r600_wait_for_spll_change(rdev);
+        r600_enable_spll_bypass(rdev, false);
+        r600_wait_for_spll_change(rdev);
+
+        r600_enable_sclk_control(rdev, true);
+        r600_enable_mclk_control(rdev, true);
+}
+
+void r600_stop_dpm(struct radeon_device *rdev)
+{
+        r600_dynamicpm_enable(rdev, false);
+}
+
+int r600_dpm_pre_set_power_state(struct radeon_device *rdev)
+{
+        return 0;
+}
+
+void r600_dpm_post_set_power_state(struct radeon_device *rdev)
+{
+
+}
+
+bool r600_is_uvd_state(u32 class, u32 class2)
+{
+        if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                return true;
+        if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE)
+                return true;
+        if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE)
+                return true;
+        if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE)
+                return true;
+        if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC)
+                return true;
+        return false;
+}
+
+int r600_set_thermal_temperature_range(struct radeon_device *rdev,
+                                       int min_temp, int max_temp)
+{
+        int low_temp = 0 * 1000;
+        int high_temp = 255 * 1000;
+
+        if (low_temp < min_temp)
+                low_temp = min_temp;
+        if (high_temp > max_temp)
+                high_temp = max_temp;
+        if (high_temp < low_temp) {
+                DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
+                return -EINVAL;
+        }
+
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
+        WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
+
+        rdev->pm.dpm.thermal.min_temp = low_temp;
+        rdev->pm.dpm.thermal.max_temp = high_temp;
+
+        return 0;
+}
+
+bool r600_is_internal_thermal_sensor(enum radeon_int_thermal_type sensor)
+{
+        switch (sensor) {
+        case THERMAL_TYPE_RV6XX:
+        case THERMAL_TYPE_RV770:
+        case THERMAL_TYPE_EVERGREEN:
+        case THERMAL_TYPE_SUMO:
+        case THERMAL_TYPE_NI:
+        case THERMAL_TYPE_SI:
+                return true;
+        case THERMAL_TYPE_ADT7473_WITH_INTERNAL:
+        case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
+                return false; /* need special handling */
+        case THERMAL_TYPE_NONE:
+        case THERMAL_TYPE_EXTERNAL:
+        case THERMAL_TYPE_EXTERNAL_GPIO:
+        default:
+                return false;
+        }
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE4 pplib4;
+        struct _ATOM_PPLIB_POWERPLAYTABLE5 pplib5;
+};
+
+union fan_info {
+        struct _ATOM_PPLIB_FANTABLE fan;
+        struct _ATOM_PPLIB_FANTABLE2 fan2;
+};
+
+static int r600_parse_clk_voltage_dep_table(struct radeon_clock_voltage_dependency_table *radeon_table,
+                                            ATOM_PPLIB_Clock_Voltage_Dependency_Table *atom_table)
+{
+        u32 size = atom_table->ucNumEntries *
+                sizeof(struct radeon_clock_voltage_dependency_entry);
+        int i;
+
+        radeon_table->entries = kzalloc(size, GFP_KERNEL);
+        if (!radeon_table->entries)
+                return -ENOMEM;
+
+        for (i = 0; i < atom_table->ucNumEntries; i++) {
+                radeon_table->entries[i].clk = le16_to_cpu(atom_table->entries[i].usClockLow) |
+                        (atom_table->entries[i].ucClockHigh << 16);
+                radeon_table->entries[i].v = le16_to_cpu(atom_table->entries[i].usVoltage);
+        }
+        radeon_table->count = atom_table->ucNumEntries;
+
+        return 0;
+}
+
+/* sizeof(ATOM_PPLIB_EXTENDEDHEADER) */
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2 12
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3 14
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4 16
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5 18
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6 20
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7 22
+
+int r600_parse_extended_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        union power_info *power_info;
+        union fan_info *fan_info;
+        ATOM_PPLIB_Clock_Voltage_Dependency_Table *dep_table;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        int ret, i;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        /* fan table */
+        if (le16_to_cpu(power_info->pplib.usTableSize) >=
+            sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE3)) {
+                if (power_info->pplib3.usFanTableOffset) {
+                        fan_info = (union fan_info *)(mode_info->atom_context->bios + data_offset +
+                                                      le16_to_cpu(power_info->pplib3.usFanTableOffset));
+                        rdev->pm.dpm.fan.t_hyst = fan_info->fan.ucTHyst;
+                        rdev->pm.dpm.fan.t_min = le16_to_cpu(fan_info->fan.usTMin);
+                        rdev->pm.dpm.fan.t_med = le16_to_cpu(fan_info->fan.usTMed);
+                        rdev->pm.dpm.fan.t_high = le16_to_cpu(fan_info->fan.usTHigh);
+                        rdev->pm.dpm.fan.pwm_min = le16_to_cpu(fan_info->fan.usPWMMin);
+                        rdev->pm.dpm.fan.pwm_med = le16_to_cpu(fan_info->fan.usPWMMed);
+                        rdev->pm.dpm.fan.pwm_high = le16_to_cpu(fan_info->fan.usPWMHigh);
+                        if (fan_info->fan.ucFanTableFormat >= 2)
+                                rdev->pm.dpm.fan.t_max = le16_to_cpu(fan_info->fan2.usTMax);
+                        else
+                                rdev->pm.dpm.fan.t_max = 10900;
+                        rdev->pm.dpm.fan.cycle_delay = 100000;
+                        rdev->pm.dpm.fan.ucode_fan_control = true;
+                }
+        }
+
+        /* clock dependancy tables, shedding tables */
+        if (le16_to_cpu(power_info->pplib.usTableSize) >=
+            sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE4)) {
+                if (power_info->pplib4.usVddcDependencyOnSCLKOffset) {
+                        dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib4.usVddcDependencyOnSCLKOffset));
+                        ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+                                                               dep_table);
+                        if (ret)
+                                return ret;
+                }
+                if (power_info->pplib4.usVddciDependencyOnMCLKOffset) {
+                        dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib4.usVddciDependencyOnMCLKOffset));
+                        ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+                                                               dep_table);
+                        if (ret) {
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+                                return ret;
+                        }
+                }
+                if (power_info->pplib4.usVddcDependencyOnMCLKOffset) {
+                        dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib4.usVddcDependencyOnMCLKOffset));
+                        ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+                                                               dep_table);
+                        if (ret) {
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
+                                return ret;
+                        }
+                }
+                if (power_info->pplib4.usMaxClockVoltageOnDCOffset) {
+                        ATOM_PPLIB_Clock_Voltage_Limit_Table *clk_v =
+                                (ATOM_PPLIB_Clock_Voltage_Limit_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib4.usMaxClockVoltageOnDCOffset));
+                        if (clk_v->ucNumEntries) {
+                                rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk =
+                                        le16_to_cpu(clk_v->entries[0].usSclkLow) |
+                                        (clk_v->entries[0].ucSclkHigh << 16);
+                                rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk =
+                                        le16_to_cpu(clk_v->entries[0].usMclkLow) |
+                                        (clk_v->entries[0].ucMclkHigh << 16);
+                                rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc =
+                                        le16_to_cpu(clk_v->entries[0].usVddc);
+                                rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddci =
+                                        le16_to_cpu(clk_v->entries[0].usVddci);
+                        }
+                }
+                if (power_info->pplib4.usVddcPhaseShedLimitsTableOffset) {
+                        ATOM_PPLIB_PhaseSheddingLimits_Table *psl =
+                                (ATOM_PPLIB_PhaseSheddingLimits_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib4.usVddcPhaseShedLimitsTableOffset));
+
+                        rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries =
+                                kzalloc(psl->ucNumEntries *
+                                        sizeof(struct radeon_phase_shedding_limits_entry),
+                                        GFP_KERNEL);
+                        if (!rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries) {
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries);
+                                return -ENOMEM;
+                        }
+
+                        for (i = 0; i < psl->ucNumEntries; i++) {
+                                rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].sclk =
+                                        le16_to_cpu(psl->entries[i].usSclkLow) |
+                                        (psl->entries[i].ucSclkHigh << 16);
+                                rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].mclk =
+                                        le16_to_cpu(psl->entries[i].usMclkLow) |
+                                        (psl->entries[i].ucMclkHigh << 16);
+                                rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].voltage =
+                                        le16_to_cpu(psl->entries[i].usVoltage);
+                        }
+                        rdev->pm.dpm.dyn_state.phase_shedding_limits_table.count =
+                                psl->ucNumEntries;
+                }
+        }
+
+        /* cac data */
+        if (le16_to_cpu(power_info->pplib.usTableSize) >=
+            sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE5)) {
+                rdev->pm.dpm.tdp_limit = le32_to_cpu(power_info->pplib5.ulTDPLimit);
+                rdev->pm.dpm.near_tdp_limit = le32_to_cpu(power_info->pplib5.ulNearTDPLimit);
+                rdev->pm.dpm.near_tdp_limit_adjusted = rdev->pm.dpm.near_tdp_limit;
+                rdev->pm.dpm.tdp_od_limit = le16_to_cpu(power_info->pplib5.usTDPODLimit);
+                if (rdev->pm.dpm.tdp_od_limit)
+                        rdev->pm.dpm.power_control = true;
+                else
+                        rdev->pm.dpm.power_control = false;
+                rdev->pm.dpm.tdp_adjustment = 0;
+                rdev->pm.dpm.sq_ramping_threshold = le32_to_cpu(power_info->pplib5.ulSQRampingThreshold);
+                rdev->pm.dpm.cac_leakage = le32_to_cpu(power_info->pplib5.ulCACLeakage);
+                rdev->pm.dpm.load_line_slope = le16_to_cpu(power_info->pplib5.usLoadLineSlope);
+                if (power_info->pplib5.usCACLeakageTableOffset) {
+                        ATOM_PPLIB_CAC_Leakage_Table *cac_table =
+                                (ATOM_PPLIB_CAC_Leakage_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib5.usCACLeakageTableOffset));
+                        u32 size = cac_table->ucNumEntries * sizeof(struct radeon_cac_leakage_table);
+                        rdev->pm.dpm.dyn_state.cac_leakage_table.entries = kzalloc(size, GFP_KERNEL);
+                        if (!rdev->pm.dpm.dyn_state.cac_leakage_table.entries) {
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries);
+                                return -ENOMEM;
+                        }
+                        for (i = 0; i < cac_table->ucNumEntries; i++) {
+                                rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc =
+                                        le16_to_cpu(cac_table->entries[i].usVddc);
+                                rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage =
+                                        le32_to_cpu(cac_table->entries[i].ulLeakageValue);
+                        }
+                        rdev->pm.dpm.dyn_state.cac_leakage_table.count = cac_table->ucNumEntries;
+                }
+        }
+
+        /* ppm table */
+        if (le16_to_cpu(power_info->pplib.usTableSize) >=
+            sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE3)) {
+                ATOM_PPLIB_EXTENDEDHEADER *ext_hdr = (ATOM_PPLIB_EXTENDEDHEADER *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib3.usExtendendedHeaderOffset));
+                if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5) &&
+                    ext_hdr->usPPMTableOffset) {
+                        ATOM_PPLIB_PPM_Table *ppm = (ATOM_PPLIB_PPM_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(ext_hdr->usPPMTableOffset));
+                        rdev->pm.dpm.dyn_state.ppm_table =
+                                kzalloc(sizeof(struct radeon_ppm_table), GFP_KERNEL);
+                        if (!rdev->pm.dpm.dyn_state.ppm_table) {
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries);
+                                kfree(rdev->pm.dpm.dyn_state.cac_leakage_table.entries);
+                                return -ENOMEM;
+                        }
+                        rdev->pm.dpm.dyn_state.ppm_table->ppm_design = ppm->ucPpmDesign;
+                        rdev->pm.dpm.dyn_state.ppm_table->cpu_core_number =
+                                le16_to_cpu(ppm->usCpuCoreNumber);
+                        rdev->pm.dpm.dyn_state.ppm_table->platform_tdp =
+                                le32_to_cpu(ppm->ulPlatformTDP);
+                        rdev->pm.dpm.dyn_state.ppm_table->small_ac_platform_tdp =
+                                le32_to_cpu(ppm->ulSmallACPlatformTDP);
+                        rdev->pm.dpm.dyn_state.ppm_table->platform_tdc =
+                                le32_to_cpu(ppm->ulPlatformTDC);
+                        rdev->pm.dpm.dyn_state.ppm_table->small_ac_platform_tdc =
+                                le32_to_cpu(ppm->ulSmallACPlatformTDC);
+                        rdev->pm.dpm.dyn_state.ppm_table->apu_tdp =
+                                le32_to_cpu(ppm->ulApuTDP);
+                        rdev->pm.dpm.dyn_state.ppm_table->dgpu_tdp =
+                                le32_to_cpu(ppm->ulDGpuTDP);
+                        rdev->pm.dpm.dyn_state.ppm_table->dgpu_ulv_power =
+                                le32_to_cpu(ppm->ulDGpuUlvPower);
+                        rdev->pm.dpm.dyn_state.ppm_table->tj_max =
+                                le32_to_cpu(ppm->ulTjmax);
+                }
+        }
+
+        return 0;
+}
+
+void r600_free_extended_power_table(struct radeon_device *rdev)
+{
+        if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries)
+                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+        if (rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries)
+                kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
+        if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries)
+                kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries);
+        if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries)
+                kfree(rdev->pm.dpm.dyn_state.cac_leakage_table.entries);
+        if (rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries)
+                kfree(rdev->pm.dpm.dyn_state.phase_shedding_limits_table.entries);
+        if (rdev->pm.dpm.dyn_state.ppm_table)
+                kfree(rdev->pm.dpm.dyn_state.ppm_table);
+}
+
+enum radeon_pcie_gen r600_get_pcie_gen_support(struct radeon_device *rdev,
+                                               u32 sys_mask,
+                                               enum radeon_pcie_gen asic_gen,
+                                               enum radeon_pcie_gen default_gen)
+{
+        switch (asic_gen) {
+        case RADEON_PCIE_GEN1:
+                return RADEON_PCIE_GEN1;
+        case RADEON_PCIE_GEN2:
+                return RADEON_PCIE_GEN2;
+        case RADEON_PCIE_GEN3:
+                return RADEON_PCIE_GEN3;
+        default:
+                if ((sys_mask & DRM_PCIE_SPEED_80) && (default_gen == RADEON_PCIE_GEN3))
+                        return RADEON_PCIE_GEN3;
+                else if ((sys_mask & DRM_PCIE_SPEED_50) && (default_gen == RADEON_PCIE_GEN2))
+                        return RADEON_PCIE_GEN2;
+                else
+                        return RADEON_PCIE_GEN1;
+        }
+        return RADEON_PCIE_GEN1;
+}
diff --git a/drivers/gpu/drm/radeon/r600_dpm.h b/drivers/gpu/drm/radeon/r600_dpm.h
new file mode 100644
index 000000000000..a95ab214289b
--- /dev/null
+++ b/drivers/gpu/drm/radeon/r600_dpm.h
@@ -0,0 +1,226 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __R600_DPM_H__
+#define __R600_DPM_H__
+
+#define R600_ASI_DFLT                                10000
+#define R600_BSP_DFLT                                0x41EB
+#define R600_BSU_DFLT                                0x2
+#define R600_AH_DFLT                                 5
+#define R600_RLP_DFLT                                25
+#define R600_RMP_DFLT                                65
+#define R600_LHP_DFLT                                40
+#define R600_LMP_DFLT                                15
+#define R600_TD_DFLT                                 0
+#define R600_UTC_DFLT_00                             0x24
+#define R600_UTC_DFLT_01                             0x22
+#define R600_UTC_DFLT_02                             0x22
+#define R600_UTC_DFLT_03                             0x22
+#define R600_UTC_DFLT_04                             0x22
+#define R600_UTC_DFLT_05                             0x22
+#define R600_UTC_DFLT_06                             0x22
+#define R600_UTC_DFLT_07                             0x22
+#define R600_UTC_DFLT_08                             0x22
+#define R600_UTC_DFLT_09                             0x22
+#define R600_UTC_DFLT_10                             0x22
+#define R600_UTC_DFLT_11                             0x22
+#define R600_UTC_DFLT_12                             0x22
+#define R600_UTC_DFLT_13                             0x22
+#define R600_UTC_DFLT_14                             0x22
+#define R600_DTC_DFLT_00                             0x24
+#define R600_DTC_DFLT_01                             0x22
+#define R600_DTC_DFLT_02                             0x22
+#define R600_DTC_DFLT_03                             0x22
+#define R600_DTC_DFLT_04                             0x22
+#define R600_DTC_DFLT_05                             0x22
+#define R600_DTC_DFLT_06                             0x22
+#define R600_DTC_DFLT_07                             0x22
+#define R600_DTC_DFLT_08                             0x22
+#define R600_DTC_DFLT_09                             0x22
+#define R600_DTC_DFLT_10                             0x22
+#define R600_DTC_DFLT_11                             0x22
+#define R600_DTC_DFLT_12                             0x22
+#define R600_DTC_DFLT_13                             0x22
+#define R600_DTC_DFLT_14                             0x22
+#define R600_VRC_DFLT                                0x0000C003
+#define R600_VOLTAGERESPONSETIME_DFLT                1000
+#define R600_BACKBIASRESPONSETIME_DFLT               1000
+#define R600_VRU_DFLT                                0x3
+#define R600_SPLLSTEPTIME_DFLT                       0x1000
+#define R600_SPLLSTEPUNIT_DFLT                       0x3
+#define R600_TPU_DFLT                                0
+#define R600_TPC_DFLT                                0x200
+#define R600_SSTU_DFLT                               0
+#define R600_SST_DFLT                                0x00C8
+#define R600_GICST_DFLT                              0x200
+#define R600_FCT_DFLT                                0x0400
+#define R600_FCTU_DFLT                               0
+#define R600_CTXCGTT3DRPHC_DFLT                      0x20
+#define R600_CTXCGTT3DRSDC_DFLT                      0x40
+#define R600_VDDC3DOORPHC_DFLT                       0x100
+#define R600_VDDC3DOORSDC_DFLT                       0x7
+#define R600_VDDC3DOORSU_DFLT                        0
+#define R600_MPLLLOCKTIME_DFLT                       100
+#define R600_MPLLRESETTIME_DFLT                      150
+#define R600_VCOSTEPPCT_DFLT                          20
+#define R600_ENDINGVCOSTEPPCT_DFLT                    5
+#define R600_REFERENCEDIVIDER_DFLT                    4
+
+#define R600_PM_NUMBER_OF_TC 15
+#define R600_PM_NUMBER_OF_SCLKS 20
+#define R600_PM_NUMBER_OF_MCLKS 4
+#define R600_PM_NUMBER_OF_VOLTAGE_LEVELS 4
+#define R600_PM_NUMBER_OF_ACTIVITY_LEVELS 3
+
+/* XXX are these ok? */
+#define R600_TEMP_RANGE_MIN (90 * 1000)
+#define R600_TEMP_RANGE_MAX (120 * 1000)
+
+enum r600_power_level {
+        R600_POWER_LEVEL_LOW = 0,
+        R600_POWER_LEVEL_MEDIUM = 1,
+        R600_POWER_LEVEL_HIGH = 2,
+        R600_POWER_LEVEL_CTXSW = 3,
+};
+
+enum r600_td {
+        R600_TD_AUTO,
+        R600_TD_UP,
+        R600_TD_DOWN,
+};
+
+enum r600_display_watermark {
+        R600_DISPLAY_WATERMARK_LOW = 0,
+        R600_DISPLAY_WATERMARK_HIGH = 1,
+};
+
+enum r600_display_gap
+{
+    R600_PM_DISPLAY_GAP_VBLANK_OR_WM = 0,
+    R600_PM_DISPLAY_GAP_VBLANK       = 1,
+    R600_PM_DISPLAY_GAP_WATERMARK    = 2,
+    R600_PM_DISPLAY_GAP_IGNORE       = 3,
+};
+
+extern const u32 r600_utc[R600_PM_NUMBER_OF_TC];
+extern const u32 r600_dtc[R600_PM_NUMBER_OF_TC];
+
+void r600_dpm_print_class_info(u32 class, u32 class2);
+void r600_dpm_print_cap_info(u32 caps);
+void r600_dpm_print_ps_status(struct radeon_device *rdev,
+                              struct radeon_ps *rps);
+bool r600_is_uvd_state(u32 class, u32 class2);
+void r600_calculate_u_and_p(u32 i, u32 r_c, u32 p_b,
+                            u32 *p, u32 *u);
+int r600_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th);
+void r600_gfx_clockgating_enable(struct radeon_device *rdev, bool enable);
+void r600_dynamicpm_enable(struct radeon_device *rdev, bool enable);
+void r600_enable_thermal_protection(struct radeon_device *rdev, bool enable);
+void r600_enable_acpi_pm(struct radeon_device *rdev);
+void r600_enable_dynamic_pcie_gen2(struct radeon_device *rdev, bool enable);
+bool r600_dynamicpm_enabled(struct radeon_device *rdev);
+void r600_enable_sclk_control(struct radeon_device *rdev, bool enable);
+void r600_enable_mclk_control(struct radeon_device *rdev, bool enable);
+void r600_enable_spll_bypass(struct radeon_device *rdev, bool enable);
+void r600_wait_for_spll_change(struct radeon_device *rdev);
+void r600_set_bsp(struct radeon_device *rdev, u32 u, u32 p);
+void r600_set_at(struct radeon_device *rdev,
+                 u32 l_to_m, u32 m_to_h,
+                 u32 h_to_m, u32 m_to_l);
+void r600_set_tc(struct radeon_device *rdev, u32 index, u32 u_t, u32 d_t);
+void r600_select_td(struct radeon_device *rdev, enum r600_td td);
+void r600_set_vrc(struct radeon_device *rdev, u32 vrv);
+void r600_set_tpu(struct radeon_device *rdev, u32 u);
+void r600_set_tpc(struct radeon_device *rdev, u32 c);
+void r600_set_sstu(struct radeon_device *rdev, u32 u);
+void r600_set_sst(struct radeon_device *rdev, u32 t);
+void r600_set_git(struct radeon_device *rdev, u32 t);
+void r600_set_fctu(struct radeon_device *rdev, u32 u);
+void r600_set_fct(struct radeon_device *rdev, u32 t);
+void r600_set_ctxcgtt3d_rphc(struct radeon_device *rdev, u32 p);
+void r600_set_ctxcgtt3d_rsdc(struct radeon_device *rdev, u32 s);
+void r600_set_vddc3d_oorsu(struct radeon_device *rdev, u32 u);
+void r600_set_vddc3d_oorphc(struct radeon_device *rdev, u32 p);
+void r600_set_vddc3d_oorsdc(struct radeon_device *rdev, u32 s);
+void r600_set_mpll_lock_time(struct radeon_device *rdev, u32 lock_time);
+void r600_set_mpll_reset_time(struct radeon_device *rdev, u32 reset_time);
+void r600_engine_clock_entry_enable(struct radeon_device *rdev,
+                                    u32 index, bool enable);
+void r600_engine_clock_entry_enable_pulse_skipping(struct radeon_device *rdev,
+                                                   u32 index, bool enable);
+void r600_engine_clock_entry_enable_post_divider(struct radeon_device *rdev,
+                                                 u32 index, bool enable);
+void r600_engine_clock_entry_set_post_divider(struct radeon_device *rdev,
+                                              u32 index, u32 divider);
+void r600_engine_clock_entry_set_reference_divider(struct radeon_device *rdev,
+                                                   u32 index, u32 divider);
+void r600_engine_clock_entry_set_feedback_divider(struct radeon_device *rdev,
+                                                  u32 index, u32 divider);
+void r600_engine_clock_entry_set_step_time(struct radeon_device *rdev,
+                                           u32 index, u32 step_time);
+void r600_vid_rt_set_ssu(struct radeon_device *rdev, u32 u);
+void r600_vid_rt_set_vru(struct radeon_device *rdev, u32 u);
+void r600_vid_rt_set_vrt(struct radeon_device *rdev, u32 rt);
+void r600_voltage_control_enable_pins(struct radeon_device *rdev,
+                                      u64 mask);
+void r600_voltage_control_program_voltages(struct radeon_device *rdev,
+                                           enum r600_power_level index, u64 pins);
+void r600_voltage_control_deactivate_static_control(struct radeon_device *rdev,
+                                                    u64 mask);
+void r600_power_level_enable(struct radeon_device *rdev,
+                             enum r600_power_level index, bool enable);
+void r600_power_level_set_voltage_index(struct radeon_device *rdev,
+                                        enum r600_power_level index, u32 voltage_index);
+void r600_power_level_set_mem_clock_index(struct radeon_device *rdev,
+                                          enum r600_power_level index, u32 mem_clock_index);
+void r600_power_level_set_eng_clock_index(struct radeon_device *rdev,
+                                          enum r600_power_level index, u32 eng_clock_index);
+void r600_power_level_set_watermark_id(struct radeon_device *rdev,
+                                       enum r600_power_level index,
+                                       enum r600_display_watermark watermark_id);
+void r600_power_level_set_pcie_gen2(struct radeon_device *rdev,
+                                    enum r600_power_level index, bool compatible);
+enum r600_power_level r600_power_level_get_current_index(struct radeon_device *rdev);
+enum r600_power_level r600_power_level_get_target_index(struct radeon_device *rdev);
+void r600_power_level_set_enter_index(struct radeon_device *rdev,
+                                      enum r600_power_level index);
+void r600_wait_for_power_level_unequal(struct radeon_device *rdev,
+                                       enum r600_power_level index);
+void r600_wait_for_power_level(struct radeon_device *rdev,
+                               enum r600_power_level index);
+void r600_start_dpm(struct radeon_device *rdev);
+void r600_stop_dpm(struct radeon_device *rdev);
+
+int r600_set_thermal_temperature_range(struct radeon_device *rdev,
+                                       int min_temp, int max_temp);
+bool r600_is_internal_thermal_sensor(enum radeon_int_thermal_type sensor);
+
+int r600_parse_extended_power_table(struct radeon_device *rdev);
+void r600_free_extended_power_table(struct radeon_device *rdev);
+
+enum radeon_pcie_gen r600_get_pcie_gen_support(struct radeon_device *rdev,
+                                               u32 sys_mask,
+                                               enum radeon_pcie_gen asic_gen,
+                                               enum radeon_pcie_gen default_gen);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/r600_hdmi.c b/drivers/gpu/drm/radeon/r600_hdmi.c
index 456750a0daa5..e73b2a73494a 100644
--- a/drivers/gpu/drm/radeon/r600_hdmi.c
+++ b/drivers/gpu/drm/radeon/r600_hdmi.c
@@ -133,14 +133,7 @@ static void r600_hdmi_update_avi_infoframe(struct drm_encoder *encoder,
         struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
         uint32_t offset = dig->afmt->offset;
         uint8_t *frame = buffer + 3;
-
-        /* Our header values (type, version, length) should be alright, Intel
-         * is using the same. Checksum function also seems to be OK, it works
-         * fine for audio infoframe. However calculated value is always lower
-         * by 2 in comparison to fglrx. It breaks displaying anything in case
-         * of TVs that strictly check the checksum. Hack it manually here to
-         * workaround this issue. */
-        frame[0x0] += 2;
+        uint8_t *header = buffer;
 
         WREG32(HDMI0_AVI_INFO0 + offset,
                 frame[0x0] | (frame[0x1] << 8) | (frame[0x2] << 16) | (frame[0x3] << 24));
@@ -149,7 +142,7 @@ static void r600_hdmi_update_avi_infoframe(struct drm_encoder *encoder,
         WREG32(HDMI0_AVI_INFO2 + offset,
                 frame[0x8] | (frame[0x9] << 8) | (frame[0xA] << 16) | (frame[0xB] << 24));
         WREG32(HDMI0_AVI_INFO3 + offset,
-                frame[0xC] | (frame[0xD] << 8));
+                frame[0xC] | (frame[0xD] << 8) | (header[1] << 24));
 }
 
 /*
diff --git a/drivers/gpu/drm/radeon/r600_reg.h b/drivers/gpu/drm/radeon/r600_reg.h
index 909219b1bf80..3ef202629e7e 100644
--- a/drivers/gpu/drm/radeon/r600_reg.h
+++ b/drivers/gpu/drm/radeon/r600_reg.h
@@ -31,6 +31,12 @@
 #define R600_PCIE_PORT_INDEX                0x0038
 #define R600_PCIE_PORT_DATA                 0x003c
 
+#define R600_RCU_INDEX                      0x0100
+#define R600_RCU_DATA                       0x0104
+
+#define R600_UVD_CTX_INDEX                  0xf4a0
+#define R600_UVD_CTX_DATA                   0xf4a4
+
 #define R600_MC_VM_FB_LOCATION                  0x2180
 #define         R600_MC_FB_BASE_MASK                    0x0000FFFF
 #define         R600_MC_FB_BASE_SHIFT                   0
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index 79df558f8c40..f1b3084d8f51 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -302,10 +302,25 @@
 #define GRBM_SOFT_RESET                                 0x8020
 #define         SOFT_RESET_CP                                   (1<<0)
 
+#define CG_THERMAL_CTRL                                 0x7F0
+#define         DIG_THERM_DPM(x)                        ((x) << 12)
+#define         DIG_THERM_DPM_MASK                      0x000FF000
+#define         DIG_THERM_DPM_SHIFT                     12
 #define CG_THERMAL_STATUS                               0x7F4
 #define         ASIC_T(x)                               ((x) << 0)
 #define         ASIC_T_MASK                             0x1FF
 #define         ASIC_T_SHIFT                            0
+#define CG_THERMAL_INT                                  0x7F8
+#define         DIG_THERM_INTH(x)                       ((x) << 8)
+#define         DIG_THERM_INTH_MASK                     0x0000FF00
+#define         DIG_THERM_INTH_SHIFT                    8
+#define         DIG_THERM_INTL(x)                       ((x) << 16)
+#define         DIG_THERM_INTL_MASK                     0x00FF0000
+#define         DIG_THERM_INTL_SHIFT                    16
+#define         THERM_INT_MASK_HIGH                     (1 << 24)
+#define         THERM_INT_MASK_LOW                      (1 << 25)
+
+#define RV770_CG_THERMAL_INT                            0x734
 
 #define HDP_HOST_PATH_CNTL                              0x2C00
 #define HDP_NONSURFACE_BASE                             0x2C04
@@ -684,10 +699,6 @@
 #define RLC_UCODE_ADDR                                    0x3f2c
 #define RLC_UCODE_DATA                                    0x3f30
 
-/* new for TN */
-#define TN_RLC_SAVE_AND_RESTORE_BASE                      0x3f10
-#define TN_RLC_CLEAR_STATE_RESTORE_BASE                   0x3f20
-
 #define SRBM_SOFT_RESET                                   0xe60
 #       define SOFT_RESET_DMA                             (1 << 12)
 #       define SOFT_RESET_RLC                             (1 << 13)
@@ -1148,6 +1159,219 @@
 #       define AFMT_AZ_FORMAT_WTRIG_ACK      (1 << 29)
 #       define AFMT_AZ_AUDIO_ENABLE_CHG_ACK  (1 << 30)
 
+/* Power management */
+#define CG_SPLL_FUNC_CNTL                                 0x600
+#       define SPLL_RESET                                (1 << 0)
+#       define SPLL_SLEEP                                (1 << 1)
+#       define SPLL_REF_DIV(x)                           ((x) << 2)
+#       define SPLL_REF_DIV_MASK                         (7 << 2)
+#       define SPLL_FB_DIV(x)                            ((x) << 5)
+#       define SPLL_FB_DIV_MASK                          (0xff << 5)
+#       define SPLL_PULSEEN                              (1 << 13)
+#       define SPLL_PULSENUM(x)                          ((x) << 14)
+#       define SPLL_PULSENUM_MASK                        (3 << 14)
+#       define SPLL_SW_HILEN(x)                          ((x) << 16)
+#       define SPLL_SW_HILEN_MASK                        (0xf << 16)
+#       define SPLL_SW_LOLEN(x)                          ((x) << 20)
+#       define SPLL_SW_LOLEN_MASK                        (0xf << 20)
+#       define SPLL_DIVEN                                (1 << 24)
+#       define SPLL_BYPASS_EN                            (1 << 25)
+#       define SPLL_CHG_STATUS                           (1 << 29)
+#       define SPLL_CTLREQ                               (1 << 30)
+#       define SPLL_CTLACK                               (1 << 31)
+
+#define GENERAL_PWRMGT                                    0x618
+#       define GLOBAL_PWRMGT_EN                           (1 << 0)
+#       define STATIC_PM_EN                               (1 << 1)
+#       define MOBILE_SU                                  (1 << 2)
+#       define THERMAL_PROTECTION_DIS                     (1 << 3)
+#       define THERMAL_PROTECTION_TYPE                    (1 << 4)
+#       define ENABLE_GEN2PCIE                            (1 << 5)
+#       define SW_GPIO_INDEX(x)                           ((x) << 6)
+#       define SW_GPIO_INDEX_MASK                         (3 << 6)
+#       define LOW_VOLT_D2_ACPI                           (1 << 8)
+#       define LOW_VOLT_D3_ACPI                           (1 << 9)
+#       define VOLT_PWRMGT_EN                             (1 << 10)
+#define CG_TPC                                            0x61c
+#       define TPCC(x)                                    ((x) << 0)
+#       define TPCC_MASK                                  (0x7fffff << 0)
+#       define TPU(x)                                     ((x) << 23)
+#       define TPU_MASK                                   (0x1f << 23)
+#define SCLK_PWRMGT_CNTL                                  0x620
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_TURNOFF                               (1 << 1)
+#       define SPLL_TURNOFF                               (1 << 2)
+#       define SU_SCLK_USE_BCLK                           (1 << 3)
+#       define DYNAMIC_GFX_ISLAND_PWR_DOWN                (1 << 4)
+#       define DYNAMIC_GFX_ISLAND_PWR_LP                  (1 << 5)
+#       define CLK_TURN_ON_STAGGER                        (1 << 6)
+#       define CLK_TURN_OFF_STAGGER                       (1 << 7)
+#       define FIR_FORCE_TREND_SEL                        (1 << 8)
+#       define FIR_TREND_MODE                             (1 << 9)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 10)
+#       define VDDC3D_TURNOFF_D1                          (1 << 11)
+#       define VDDC3D_TURNOFF_D2                          (1 << 12)
+#       define VDDC3D_TURNOFF_D3                          (1 << 13)
+#       define SPLL_TURNOFF_D2                            (1 << 14)
+#       define SCLK_LOW_D1                                (1 << 15)
+#       define DYN_GFX_CLK_OFF_MC_EN                      (1 << 16)
+#define MCLK_PWRMGT_CNTL                                  0x624
+#       define MPLL_PWRMGT_OFF                            (1 << 0)
+#       define YCLK_TURNOFF                               (1 << 1)
+#       define MPLL_TURNOFF                               (1 << 2)
+#       define SU_MCLK_USE_BCLK                           (1 << 3)
+#       define DLL_READY                                  (1 << 4)
+#       define MC_BUSY                                    (1 << 5)
+#       define MC_INT_CNTL                                (1 << 7)
+#       define MRDCKA_SLEEP                               (1 << 8)
+#       define MRDCKB_SLEEP                               (1 << 9)
+#       define MRDCKC_SLEEP                               (1 << 10)
+#       define MRDCKD_SLEEP                               (1 << 11)
+#       define MRDCKE_SLEEP                               (1 << 12)
+#       define MRDCKF_SLEEP                               (1 << 13)
+#       define MRDCKG_SLEEP                               (1 << 14)
+#       define MRDCKH_SLEEP                               (1 << 15)
+#       define MRDCKA_RESET                               (1 << 16)
+#       define MRDCKB_RESET                               (1 << 17)
+#       define MRDCKC_RESET                               (1 << 18)
+#       define MRDCKD_RESET                               (1 << 19)
+#       define MRDCKE_RESET                               (1 << 20)
+#       define MRDCKF_RESET                               (1 << 21)
+#       define MRDCKG_RESET                               (1 << 22)
+#       define MRDCKH_RESET                               (1 << 23)
+#       define DLL_READY_READ                             (1 << 24)
+#       define USE_DISPLAY_GAP                            (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                  (1 << 26)
+#       define USE_DISPLAY_GAP_CTXSW                      (1 << 27)
+#       define MPLL_TURNOFF_D2                            (1 << 28)
+#       define USE_DISPLAY_URGENT_CTXSW                   (1 << 29)
+
+#define MPLL_TIME                                         0x634
+#       define MPLL_LOCK_TIME(x)                          ((x) << 0)
+#       define MPLL_LOCK_TIME_MASK                        (0xffff << 0)
+#       define MPLL_RESET_TIME(x)                         ((x) << 16)
+#       define MPLL_RESET_TIME_MASK                       (0xffff << 16)
+
+#define SCLK_FREQ_SETTING_STEP_0_PART1                    0x648
+#       define STEP_0_SPLL_POST_DIV(x)                    ((x) << 0)
+#       define STEP_0_SPLL_POST_DIV_MASK                  (0xff << 0)
+#       define STEP_0_SPLL_FB_DIV(x)                      ((x) << 8)
+#       define STEP_0_SPLL_FB_DIV_MASK                    (0xff << 8)
+#       define STEP_0_SPLL_REF_DIV(x)                     ((x) << 16)
+#       define STEP_0_SPLL_REF_DIV_MASK                   (7 << 16)
+#       define STEP_0_SPLL_STEP_TIME(x)                   ((x) << 19)
+#       define STEP_0_SPLL_STEP_TIME_MASK                 (0x1fff << 19)
+#define SCLK_FREQ_SETTING_STEP_0_PART2                    0x64c
+#       define STEP_0_PULSE_HIGH_CNT(x)                   ((x) << 0)
+#       define STEP_0_PULSE_HIGH_CNT_MASK                 (0x1ff << 0)
+#       define STEP_0_POST_DIV_EN                         (1 << 9)
+#       define STEP_0_SPLL_STEP_ENABLE                    (1 << 30)
+#       define STEP_0_SPLL_ENTRY_VALID                    (1 << 31)
+
+#define VID_RT                                            0x6f8
+#       define VID_CRT(x)                                 ((x) << 0)
+#       define VID_CRT_MASK                               (0x1fff << 0)
+#       define VID_CRTU(x)                                ((x) << 13)
+#       define VID_CRTU_MASK                              (7 << 13)
+#       define SSTU(x)                                    ((x) << 16)
+#       define SSTU_MASK                                  (7 << 16)
+#define CTXSW_PROFILE_INDEX                               0x6fc
+#       define CTXSW_FREQ_VIDS_CFG_INDEX(x)               ((x) << 0)
+#       define CTXSW_FREQ_VIDS_CFG_INDEX_MASK             (3 << 0)
+#       define CTXSW_FREQ_VIDS_CFG_INDEX_SHIFT            0
+#       define CTXSW_FREQ_MCLK_CFG_INDEX(x)               ((x) << 2)
+#       define CTXSW_FREQ_MCLK_CFG_INDEX_MASK             (3 << 2)
+#       define CTXSW_FREQ_MCLK_CFG_INDEX_SHIFT            2
+#       define CTXSW_FREQ_SCLK_CFG_INDEX(x)               ((x) << 4)
+#       define CTXSW_FREQ_SCLK_CFG_INDEX_MASK             (0x1f << 4)
+#       define CTXSW_FREQ_SCLK_CFG_INDEX_SHIFT            4
+#       define CTXSW_FREQ_STATE_SPLL_RESET_EN             (1 << 9)
+#       define CTXSW_FREQ_STATE_ENABLE                    (1 << 10)
+#       define CTXSW_FREQ_DISPLAY_WATERMARK               (1 << 11)
+#       define CTXSW_FREQ_GEN2PCIE_VOLT                   (1 << 12)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                  0x70c
+#       define TARGET_PROFILE_INDEX_MASK                  (3 << 0)
+#       define TARGET_PROFILE_INDEX_SHIFT                 0
+#       define CURRENT_PROFILE_INDEX_MASK                 (3 << 2)
+#       define CURRENT_PROFILE_INDEX_SHIFT                2
+#       define DYN_PWR_ENTER_INDEX(x)                     ((x) << 4)
+#       define DYN_PWR_ENTER_INDEX_MASK                   (3 << 4)
+#       define DYN_PWR_ENTER_INDEX_SHIFT                  4
+#       define CURR_MCLK_INDEX_MASK                       (3 << 6)
+#       define CURR_MCLK_INDEX_SHIFT                      6
+#       define CURR_SCLK_INDEX_MASK                       (0x1f << 8)
+#       define CURR_SCLK_INDEX_SHIFT                      8
+#       define CURR_VID_INDEX_MASK                        (3 << 13)
+#       define CURR_VID_INDEX_SHIFT                       13
+
+#define LOWER_GPIO_ENABLE                                 0x710
+#define UPPER_GPIO_ENABLE                                 0x714
+#define CTXSW_VID_LOWER_GPIO_CNTL                         0x718
+
+#define VID_UPPER_GPIO_CNTL                               0x740
+#define CG_CTX_CGTT3D_R                                   0x744
+#       define PHC(x)                                     ((x) << 0)
+#       define PHC_MASK                                   (0x1ff << 0)
+#       define SDC(x)                                     ((x) << 9)
+#       define SDC_MASK                                   (0x3fff << 9)
+#define CG_VDDC3D_OOR                                     0x748
+#       define SU(x)                                      ((x) << 23)
+#       define SU_MASK                                    (0xf << 23)
+#define CG_FTV                                            0x74c
+#define CG_FFCT_0                                         0x750
+#       define UTC_0(x)                                   ((x) << 0)
+#       define UTC_0_MASK                                 (0x3ff << 0)
+#       define DTC_0(x)                                   ((x) << 10)
+#       define DTC_0_MASK                                 (0x3ff << 10)
+
+#define CG_BSP                                            0x78c
+#       define BSP(x)                                     ((x) << 0)
+#       define BSP_MASK                                   (0xffff << 0)
+#       define BSU(x)                                     ((x) << 16)
+#       define BSU_MASK                                   (0xf << 16)
+#define CG_RT                                             0x790
+#       define FLS(x)                                     ((x) << 0)
+#       define FLS_MASK                                   (0xffff << 0)
+#       define FMS(x)                                     ((x) << 16)
+#       define FMS_MASK                                   (0xffff << 16)
+#define CG_LT                                             0x794
+#       define FHS(x)                                     ((x) << 0)
+#       define FHS_MASK                                   (0xffff << 0)
+#define CG_GIT                                            0x798
+#       define CG_GICST(x)                                ((x) << 0)
+#       define CG_GICST_MASK                              (0xffff << 0)
+#       define CG_GIPOT(x)                                ((x) << 16)
+#       define CG_GIPOT_MASK                              (0xffff << 16)
+
+#define CG_SSP                                            0x7a8
+#       define CG_SST(x)                                  ((x) << 0)
+#       define CG_SST_MASK                                (0xffff << 0)
+#       define CG_SSTU(x)                                 ((x) << 16)
+#       define CG_SSTU_MASK                               (0xf << 16)
+
+#define CG_RLC_REQ_AND_RSP                                0x7c4
+#       define RLC_CG_REQ_TYPE_MASK                       0xf
+#       define RLC_CG_REQ_TYPE_SHIFT                      0
+#       define CG_RLC_RSP_TYPE_MASK                       0xf0
+#       define CG_RLC_RSP_TYPE_SHIFT                      4
+
+#define CG_FC_T                                           0x7cc
+#       define FC_T(x)                                    ((x) << 0)
+#       define FC_T_MASK                                  (0xffff << 0)
+#       define FC_TU(x)                                   ((x) << 16)
+#       define FC_TU_MASK                                 (0x1f << 16)
+
+#define GPIOPAD_MASK                                      0x1798
+#define GPIOPAD_A                                         0x179c
+#define GPIOPAD_EN                                        0x17a0
+
+#define GRBM_PWR_CNTL                                     0x800c
+#       define REQ_TYPE_MASK                              0xf
+#       define REQ_TYPE_SHIFT                             0
+#       define RSP_TYPE_MASK                              0xf0
+#       define RSP_TYPE_SHIFT                             4
+
 /*
  * UVD
  */
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 142ce6cc69f5..f51807f04f65 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -96,6 +96,7 @@ extern int radeon_pcie_gen2;
 extern int radeon_msi;
 extern int radeon_lockup_timeout;
 extern int radeon_fastfb;
+extern int radeon_dpm;
 
 /*
  * Copy from radeon_drv.h so we don't have to include both and have conflicting
@@ -150,6 +151,13 @@ extern int radeon_fastfb;
 #define RADEON_RESET_MC                         (1 << 10)
 #define RADEON_RESET_DISPLAY                    (1 << 11)
 
+/* max cursor sizes (in pixels) */
+#define CURSOR_WIDTH 64
+#define CURSOR_HEIGHT 64
+
+#define CIK_CURSOR_WIDTH 128
+#define CIK_CURSOR_HEIGHT 128
+
 /*
  * Errata workarounds.
  */
@@ -192,6 +200,7 @@ struct radeon_clock {
         uint32_t default_mclk;
         uint32_t default_sclk;
         uint32_t default_dispclk;
+        uint32_t current_dispclk;
         uint32_t dp_extclk;
         uint32_t max_pixel_clock;
 };
@@ -211,13 +220,51 @@ int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
                                    u32 clock,
                                    bool strobe_mode,
                                    struct atom_clock_dividers *dividers);
+int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev,
+                                        u32 clock,
+                                        bool strobe_mode,
+                                        struct atom_mpll_param *mpll_param);
 void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type);
+int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev,
+                                          u16 voltage_level, u8 voltage_type,
+                                          u32 *gpio_value, u32 *gpio_mask);
+void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev,
+                                         u32 eng_clock, u32 mem_clock);
+int radeon_atom_get_voltage_step(struct radeon_device *rdev,
+                                 u8 voltage_type, u16 *voltage_step);
+int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
+                             u16 voltage_id, u16 *voltage);
+int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev,
+                                                      u16 *voltage,
+                                                      u16 leakage_idx);
+int radeon_atom_round_to_true_voltage(struct radeon_device *rdev,
+                                      u8 voltage_type,
+                                      u16 nominal_voltage,
+                                      u16 *true_voltage);
+int radeon_atom_get_min_voltage(struct radeon_device *rdev,
+                                u8 voltage_type, u16 *min_voltage);
+int radeon_atom_get_max_voltage(struct radeon_device *rdev,
+                                u8 voltage_type, u16 *max_voltage);
+int radeon_atom_get_voltage_table(struct radeon_device *rdev,
+                                  u8 voltage_type, u8 voltage_mode,
+                                  struct atom_voltage_table *voltage_table);
+bool radeon_atom_is_voltage_gpio(struct radeon_device *rdev,
+                                 u8 voltage_type, u8 voltage_mode);
+void radeon_atom_update_memory_dll(struct radeon_device *rdev,
+                                   u32 mem_clock);
+void radeon_atom_set_ac_timing(struct radeon_device *rdev,
+                               u32 mem_clock);
+int radeon_atom_init_mc_reg_table(struct radeon_device *rdev,
+                                  u8 module_index,
+                                  struct atom_mc_reg_table *reg_table);
+int radeon_atom_get_memory_info(struct radeon_device *rdev,
+                                u8 module_index, struct atom_memory_info *mem_info);
+int radeon_atom_get_mclk_range_table(struct radeon_device *rdev,
+                                     bool gddr5, u8 module_index,
+                                     struct atom_memory_clock_range_table *mclk_range_table);
+int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
+                             u16 voltage_id, u16 *voltage);
 void rs690_pm_info(struct radeon_device *rdev);
-extern int rv6xx_get_temp(struct radeon_device *rdev);
-extern int rv770_get_temp(struct radeon_device *rdev);
-extern int evergreen_get_temp(struct radeon_device *rdev);
-extern int sumo_get_temp(struct radeon_device *rdev);
-extern int si_get_temp(struct radeon_device *rdev);
 extern void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
                                     unsigned *bankh, unsigned *mtaspect,
                                     unsigned *tile_split);
@@ -549,6 +596,20 @@ struct radeon_scratch {
 int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg);
 void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);
 
+/*
+ * GPU doorbell structures, functions & helpers
+ */
+struct radeon_doorbell {
+        u32                     num_pages;
+        bool                    free[1024];
+        /* doorbell mmio */
+        resource_size_t                 base;
+        resource_size_t                 size;
+        void __iomem                    *ptr;
+};
+
+int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
+void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
 
 /*
  * IRQS.
@@ -600,10 +661,21 @@ struct evergreen_irq_stat_regs {
         u32 afmt_status6;
 };
 
+struct cik_irq_stat_regs {
+        u32 disp_int;
+        u32 disp_int_cont;
+        u32 disp_int_cont2;
+        u32 disp_int_cont3;
+        u32 disp_int_cont4;
+        u32 disp_int_cont5;
+        u32 disp_int_cont6;
+};
+
 union radeon_irq_stat_regs {
         struct r500_irq_stat_regs r500;
         struct r600_irq_stat_regs r600;
         struct evergreen_irq_stat_regs evergreen;
+        struct cik_irq_stat_regs cik;
 };
 
 #define RADEON_MAX_HPD_PINS 6
@@ -620,6 +692,7 @@ struct radeon_irq {
         bool                            hpd[RADEON_MAX_HPD_PINS];
         bool                            afmt[RADEON_MAX_AFMT_BLOCKS];
         union radeon_irq_stat_regs      stat_regs;
+        bool                            dpm_thermal;
 };
 
 int radeon_irq_kms_init(struct radeon_device *rdev);
@@ -677,6 +750,22 @@ struct radeon_ring {
         u32                     idx;
         u64                     last_semaphore_signal_addr;
         u64                     last_semaphore_wait_addr;
+        /* for CIK queues */
+        u32 me;
+        u32 pipe;
+        u32 queue;
+        struct radeon_bo        *mqd_obj;
+        u32 doorbell_page_num;
+        u32 doorbell_offset;
+        unsigned                wptr_offs;
+};
+
+struct radeon_mec {
+        struct radeon_bo        *hpd_eop_obj;
+        u64                     hpd_eop_gpu_addr;
+        u32 num_pipe;
+        u32 num_mec;
+        u32 num_queue;
 };
 
 /*
@@ -778,15 +867,22 @@ struct r600_blit {
 };
 
 /*
- * SI RLC stuff
+ * RLC stuff
  */
-struct si_rlc {
+#include "clearstate_defs.h"
+
+struct radeon_rlc {
         /* for power gating */
         struct radeon_bo        *save_restore_obj;
         uint64_t                save_restore_gpu_addr;
+        volatile uint32_t       *sr_ptr;
+        u32                     *reg_list;
+        u32                     reg_list_size;
         /* for clear state */
         struct radeon_bo        *clear_state_obj;
         uint64_t                clear_state_gpu_addr;
+        volatile uint32_t       *cs_ptr;
+        struct cs_section_def   *cs_data;
 };
 
 int radeon_ib_get(struct radeon_device *rdev, int ring,
@@ -883,6 +979,7 @@ struct radeon_cs_parser {
         u32                     cs_flags;
         u32                     ring;
         s32                     priority;
+        struct ww_acquire_ctx   ticket;
 };
 
 extern int radeon_cs_finish_pages(struct radeon_cs_parser *p);
@@ -934,6 +1031,8 @@ struct radeon_wb {
 #define CAYMAN_WB_DMA1_RPTR_OFFSET   2304
 #define R600_WB_UVD_RPTR_OFFSET  2560
 #define R600_WB_EVENT_OFFSET     3072
+#define CIK_WB_CP1_WPTR_OFFSET     3328
+#define CIK_WB_CP2_WPTR_OFFSET     3584
 
 /**
  * struct radeon_pm - power management datas
@@ -958,6 +1057,7 @@ struct radeon_wb {
 enum radeon_pm_method {
         PM_METHOD_PROFILE,
         PM_METHOD_DYNPM,
+        PM_METHOD_DPM,
 };
 
 enum radeon_dynpm_state {
@@ -983,11 +1083,23 @@ enum radeon_voltage_type {
 };
 
 enum radeon_pm_state_type {
+        /* not used for dpm */
         POWER_STATE_TYPE_DEFAULT,
         POWER_STATE_TYPE_POWERSAVE,
+        /* user selectable states */
         POWER_STATE_TYPE_BATTERY,
         POWER_STATE_TYPE_BALANCED,
         POWER_STATE_TYPE_PERFORMANCE,
+        /* internal states */
+        POWER_STATE_TYPE_INTERNAL_UVD,
+        POWER_STATE_TYPE_INTERNAL_UVD_SD,
+        POWER_STATE_TYPE_INTERNAL_UVD_HD,
+        POWER_STATE_TYPE_INTERNAL_UVD_HD2,
+        POWER_STATE_TYPE_INTERNAL_UVD_MVC,
+        POWER_STATE_TYPE_INTERNAL_BOOT,
+        POWER_STATE_TYPE_INTERNAL_THERMAL,
+        POWER_STATE_TYPE_INTERNAL_ACPI,
+        POWER_STATE_TYPE_INTERNAL_ULV,
 };
 
 enum radeon_pm_profile_type {
@@ -1016,12 +1128,17 @@ struct radeon_pm_profile {
 
 enum radeon_int_thermal_type {
         THERMAL_TYPE_NONE,
+        THERMAL_TYPE_EXTERNAL,
+        THERMAL_TYPE_EXTERNAL_GPIO,
         THERMAL_TYPE_RV6XX,
         THERMAL_TYPE_RV770,
+        THERMAL_TYPE_ADT7473_WITH_INTERNAL,
         THERMAL_TYPE_EVERGREEN,
         THERMAL_TYPE_SUMO,
         THERMAL_TYPE_NI,
         THERMAL_TYPE_SI,
+        THERMAL_TYPE_EMC2103_WITH_INTERNAL,
+        THERMAL_TYPE_CI,
 };
 
 struct radeon_voltage {
@@ -1075,6 +1192,193 @@ struct radeon_power_state {
  */
 #define RADEON_MODE_OVERCLOCK_MARGIN 500 /* 5 MHz */
 
+enum radeon_dpm_auto_throttle_src {
+        RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL,
+        RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL
+};
+
+enum radeon_dpm_event_src {
+        RADEON_DPM_EVENT_SRC_ANALOG = 0,
+        RADEON_DPM_EVENT_SRC_EXTERNAL = 1,
+        RADEON_DPM_EVENT_SRC_DIGITAL = 2,
+        RADEON_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
+        RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
+};
+
+struct radeon_ps {
+        u32 caps; /* vbios flags */
+        u32 class; /* vbios flags */
+        u32 class2; /* vbios flags */
+        /* UVD clocks */
+        u32 vclk;
+        u32 dclk;
+        /* asic priv */
+        void *ps_priv;
+};
+
+struct radeon_dpm_thermal {
+        /* thermal interrupt work */
+        struct work_struct work;
+        /* low temperature threshold */
+        int                min_temp;
+        /* high temperature threshold */
+        int                max_temp;
+        /* was interrupt low to high or high to low */
+        bool               high_to_low;
+};
+
+enum radeon_clk_action
+{
+        RADEON_SCLK_UP = 1,
+        RADEON_SCLK_DOWN
+};
+
+struct radeon_blacklist_clocks
+{
+        u32 sclk;
+        u32 mclk;
+        enum radeon_clk_action action;
+};
+
+struct radeon_clock_and_voltage_limits {
+        u32 sclk;
+        u32 mclk;
+        u32 vddc;
+        u32 vddci;
+};
+
+struct radeon_clock_array {
+        u32 count;
+        u32 *values;
+};
+
+struct radeon_clock_voltage_dependency_entry {
+        u32 clk;
+        u16 v;
+};
+
+struct radeon_clock_voltage_dependency_table {
+        u32 count;
+        struct radeon_clock_voltage_dependency_entry *entries;
+};
+
+struct radeon_cac_leakage_entry {
+        u16 vddc;
+        u32 leakage;
+};
+
+struct radeon_cac_leakage_table {
+        u32 count;
+        struct radeon_cac_leakage_entry *entries;
+};
+
+struct radeon_phase_shedding_limits_entry {
+        u16 voltage;
+        u32 sclk;
+        u32 mclk;
+};
+
+struct radeon_phase_shedding_limits_table {
+        u32 count;
+        struct radeon_phase_shedding_limits_entry *entries;
+};
+
+struct radeon_ppm_table {
+        u8 ppm_design;
+        u16 cpu_core_number;
+        u32 platform_tdp;
+        u32 small_ac_platform_tdp;
+        u32 platform_tdc;
+        u32 small_ac_platform_tdc;
+        u32 apu_tdp;
+        u32 dgpu_tdp;
+        u32 dgpu_ulv_power;
+        u32 tj_max;
+};
+
+struct radeon_dpm_dynamic_state {
+        struct radeon_clock_voltage_dependency_table vddc_dependency_on_sclk;
+        struct radeon_clock_voltage_dependency_table vddci_dependency_on_mclk;
+        struct radeon_clock_voltage_dependency_table vddc_dependency_on_mclk;
+        struct radeon_clock_voltage_dependency_table vddc_dependency_on_dispclk;
+        struct radeon_clock_array valid_sclk_values;
+        struct radeon_clock_array valid_mclk_values;
+        struct radeon_clock_and_voltage_limits max_clock_voltage_on_dc;
+        struct radeon_clock_and_voltage_limits max_clock_voltage_on_ac;
+        u32 mclk_sclk_ratio;
+        u32 sclk_mclk_delta;
+        u16 vddc_vddci_delta;
+        u16 min_vddc_for_pcie_gen2;
+        struct radeon_cac_leakage_table cac_leakage_table;
+        struct radeon_phase_shedding_limits_table phase_shedding_limits_table;
+        struct radeon_ppm_table *ppm_table;
+};
+
+struct radeon_dpm_fan {
+        u16 t_min;
+        u16 t_med;
+        u16 t_high;
+        u16 pwm_min;
+        u16 pwm_med;
+        u16 pwm_high;
+        u8 t_hyst;
+        u32 cycle_delay;
+        u16 t_max;
+        bool ucode_fan_control;
+};
+
+enum radeon_pcie_gen {
+        RADEON_PCIE_GEN1 = 0,
+        RADEON_PCIE_GEN2 = 1,
+        RADEON_PCIE_GEN3 = 2,
+        RADEON_PCIE_GEN_INVALID = 0xffff
+};
+
+struct radeon_dpm {
+        struct radeon_ps        *ps;
+        /* number of valid power states */
+        int                     num_ps;
+        /* current power state that is active */
+        struct radeon_ps        *current_ps;
+        /* requested power state */
+        struct radeon_ps        *requested_ps;
+        /* boot up power state */
+        struct radeon_ps        *boot_ps;
+        /* default uvd power state */
+        struct radeon_ps        *uvd_ps;
+        enum radeon_pm_state_type state;
+        enum radeon_pm_state_type user_state;
+        u32                     platform_caps;
+        u32                     voltage_response_time;
+        u32                     backbias_response_time;
+        void                    *priv;
+        u32                     new_active_crtcs;
+        int                     new_active_crtc_count;
+        u32                     current_active_crtcs;
+        int                     current_active_crtc_count;
+        struct radeon_dpm_dynamic_state dyn_state;
+        struct radeon_dpm_fan fan;
+        u32 tdp_limit;
+        u32 near_tdp_limit;
+        u32 near_tdp_limit_adjusted;
+        u32 sq_ramping_threshold;
+        u32 cac_leakage;
+        u16 tdp_od_limit;
+        u32 tdp_adjustment;
+        u16 load_line_slope;
+        bool power_control;
+        bool ac_power;
+        /* special states active */
+        bool                    thermal_active;
+        bool                    uvd_active;
+        /* thermal handling */
+        struct radeon_dpm_thermal thermal;
+};
+
+void radeon_dpm_enable_power_state(struct radeon_device *rdev,
+                                    enum radeon_pm_state_type dpm_state);
+
+
 struct radeon_pm {
         struct mutex            mutex;
         /* write locked while reprogramming mclk */
@@ -1128,6 +1432,9 @@ struct radeon_pm {
         /* internal thermal controller on rv6xx+ */
         enum radeon_int_thermal_type int_thermal_type;
         struct device           *int_hwmon_dev;
+        /* dpm */
+        bool                    dpm_enabled;
+        struct radeon_dpm       dpm;
 };
 
 int radeon_pm_get_type_index(struct radeon_device *rdev,
@@ -1266,6 +1573,10 @@ struct radeon_asic {
                 int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp);
                 bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp);
                 void (*vm_flush)(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
+
+                u32 (*get_rptr)(struct radeon_device *rdev, struct radeon_ring *ring);
+                u32 (*get_wptr)(struct radeon_device *rdev, struct radeon_ring *ring);
+                void (*set_wptr)(struct radeon_device *rdev, struct radeon_ring *ring);
         } ring[RADEON_NUM_RINGS];
         /* irqs */
         struct {
@@ -1325,7 +1636,7 @@ struct radeon_asic {
                 bool (*sense)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
                 void (*set_polarity)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
         } hpd;
-        /* power management */
+        /* static power management */
         struct {
                 void (*misc)(struct radeon_device *rdev);
                 void (*prepare)(struct radeon_device *rdev);
@@ -1340,7 +1651,24 @@ struct radeon_asic {
                 void (*set_pcie_lanes)(struct radeon_device *rdev, int lanes);
                 void (*set_clock_gating)(struct radeon_device *rdev, int enable);
                 int (*set_uvd_clocks)(struct radeon_device *rdev, u32 vclk, u32 dclk);
+                int (*get_temperature)(struct radeon_device *rdev);
         } pm;
+        /* dynamic power management */
+        struct {
+                int (*init)(struct radeon_device *rdev);
+                void (*setup_asic)(struct radeon_device *rdev);
+                int (*enable)(struct radeon_device *rdev);
+                void (*disable)(struct radeon_device *rdev);
+                int (*pre_set_power_state)(struct radeon_device *rdev);
+                int (*set_power_state)(struct radeon_device *rdev);
+                void (*post_set_power_state)(struct radeon_device *rdev);
+                void (*display_configuration_changed)(struct radeon_device *rdev);
+                void (*fini)(struct radeon_device *rdev);
+                u32 (*get_sclk)(struct radeon_device *rdev, bool low);
+                u32 (*get_mclk)(struct radeon_device *rdev, bool low);
+                void (*print_power_state)(struct radeon_device *rdev, struct radeon_ps *ps);
+                void (*debugfs_print_current_performance_level)(struct radeon_device *rdev, struct seq_file *m);
+        } dpm;
         /* pageflipping */
         struct {
                 void (*pre_page_flip)(struct radeon_device *rdev, int crtc);
@@ -1505,6 +1833,36 @@ struct si_asic {
         uint32_t tile_mode_array[32];
 };
 
+struct cik_asic {
+        unsigned max_shader_engines;
+        unsigned max_tile_pipes;
+        unsigned max_cu_per_sh;
+        unsigned max_sh_per_se;
+        unsigned max_backends_per_se;
+        unsigned max_texture_channel_caches;
+        unsigned max_gprs;
+        unsigned max_gs_threads;
+        unsigned max_hw_contexts;
+        unsigned sc_prim_fifo_size_frontend;
+        unsigned sc_prim_fifo_size_backend;
+        unsigned sc_hiz_tile_fifo_size;
+        unsigned sc_earlyz_tile_fifo_size;
+
+        unsigned num_tile_pipes;
+        unsigned num_backends_per_se;
+        unsigned backend_disable_mask_per_asic;
+        unsigned backend_map;
+        unsigned num_texture_channel_caches;
+        unsigned mem_max_burst_length_bytes;
+        unsigned mem_row_size_in_kb;
+        unsigned shader_engine_tile_size;
+        unsigned num_gpus;
+        unsigned multi_gpu_tile_size;
+
+        unsigned tile_config;
+        uint32_t tile_mode_array[32];
+};
+
 union radeon_asic_config {
         struct r300_asic        r300;
         struct r100_asic        r100;
@@ -1513,6 +1871,7 @@ union radeon_asic_config {
         struct evergreen_asic   evergreen;
         struct cayman_asic      cayman;
         struct si_asic          si;
+        struct cik_asic         cik;
 };
 
 /*
@@ -1657,6 +2016,7 @@ struct radeon_device {
         struct radeon_gart              gart;
         struct radeon_mode_info         mode_info;
         struct radeon_scratch           scratch;
+        struct radeon_doorbell          doorbell;
         struct radeon_mman              mman;
         struct radeon_fence_driver      fence_drv[RADEON_NUM_RINGS];
         wait_queue_head_t               fence_queue;
@@ -1684,13 +2044,18 @@ struct radeon_device {
         const struct firmware *mc_fw;   /* NI MC firmware */
         const struct firmware *ce_fw;   /* SI CE firmware */
         const struct firmware *uvd_fw;  /* UVD firmware */
+        const struct firmware *mec_fw;  /* CIK MEC firmware */
+        const struct firmware *sdma_fw; /* CIK SDMA firmware */
+        const struct firmware *smc_fw;  /* SMC firmware */
         struct r600_blit r600_blit;
         struct r600_vram_scratch vram_scratch;
         int msi_enabled; /* msi enabled */
         struct r600_ih ih; /* r6/700 interrupt ring */
-        struct si_rlc rlc;
+        struct radeon_rlc rlc;
+        struct radeon_mec mec;
         struct work_struct hotplug_work;
         struct work_struct audio_work;
+        struct work_struct reset_work;
         int num_crtc; /* number of crtcs */
         struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
         bool audio_enabled;
@@ -1727,6 +2092,9 @@ void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v,
 u32 r100_io_rreg(struct radeon_device *rdev, u32 reg);
 void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v);
 
+u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset);
+void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v);
+
 /*
  * Cast helper
  */
@@ -1754,6 +2122,18 @@ void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v);
 #define WREG32_PCIE(reg, v) rv370_pcie_wreg(rdev, (reg), (v))
 #define RREG32_PCIE_PORT(reg) rdev->pciep_rreg(rdev, (reg))
 #define WREG32_PCIE_PORT(reg, v) rdev->pciep_wreg(rdev, (reg), (v))
+#define RREG32_SMC(reg) tn_smc_rreg(rdev, (reg))
+#define WREG32_SMC(reg, v) tn_smc_wreg(rdev, (reg), (v))
+#define RREG32_RCU(reg) r600_rcu_rreg(rdev, (reg))
+#define WREG32_RCU(reg, v) r600_rcu_wreg(rdev, (reg), (v))
+#define RREG32_CG(reg) eg_cg_rreg(rdev, (reg))
+#define WREG32_CG(reg, v) eg_cg_wreg(rdev, (reg), (v))
+#define RREG32_PIF_PHY0(reg) eg_pif_phy0_rreg(rdev, (reg))
+#define WREG32_PIF_PHY0(reg, v) eg_pif_phy0_wreg(rdev, (reg), (v))
+#define RREG32_PIF_PHY1(reg) eg_pif_phy1_rreg(rdev, (reg))
+#define WREG32_PIF_PHY1(reg, v) eg_pif_phy1_wreg(rdev, (reg), (v))
+#define RREG32_UVD_CTX(reg) r600_uvd_ctx_rreg(rdev, (reg))
+#define WREG32_UVD_CTX(reg, v) r600_uvd_ctx_wreg(rdev, (reg), (v))
 #define WREG32_P(reg, val, mask)                                \
         do {                                                    \
                 uint32_t tmp_ = RREG32(reg);                    \
@@ -1774,6 +2154,9 @@ void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v);
 #define RREG32_IO(reg) r100_io_rreg(rdev, (reg))
 #define WREG32_IO(reg, v) r100_io_wreg(rdev, (reg), (v))
 
+#define RDOORBELL32(offset) cik_mm_rdoorbell(rdev, (offset))
+#define WDOORBELL32(offset, v) cik_mm_wdoorbell(rdev, (offset), (v))
+
 /*
  * Indirect registers accessor
  */
@@ -1792,6 +2175,96 @@ static inline void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uin
         WREG32(RADEON_PCIE_DATA, (v));
 }
 
+static inline u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(TN_SMC_IND_INDEX_0, (reg));
+        r = RREG32(TN_SMC_IND_DATA_0);
+        return r;
+}
+
+static inline void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(TN_SMC_IND_INDEX_0, (reg));
+        WREG32(TN_SMC_IND_DATA_0, (v));
+}
+
+static inline u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(R600_RCU_INDEX, ((reg) & 0x1fff));
+        r = RREG32(R600_RCU_DATA);
+        return r;
+}
+
+static inline void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(R600_RCU_INDEX, ((reg) & 0x1fff));
+        WREG32(R600_RCU_DATA, (v));
+}
+
+static inline u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
+        r = RREG32(EVERGREEN_CG_IND_DATA);
+        return r;
+}
+
+static inline void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
+        WREG32(EVERGREEN_CG_IND_DATA, (v));
+}
+
+static inline u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
+        r = RREG32(EVERGREEN_PIF_PHY0_DATA);
+        return r;
+}
+
+static inline void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
+        WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
+}
+
+static inline u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
+        r = RREG32(EVERGREEN_PIF_PHY1_DATA);
+        return r;
+}
+
+static inline void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
+        WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
+}
+
+static inline u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg)
+{
+        u32 r;
+
+        WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff));
+        r = RREG32(R600_UVD_CTX_DATA);
+        return r;
+}
+
+static inline void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v)
+{
+        WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff));
+        WREG32(R600_UVD_CTX_DATA, (v));
+}
+
 void r100_pll_errata_after_index(struct radeon_device *rdev);
 
 
@@ -1840,6 +2313,16 @@ void r100_pll_errata_after_index(struct radeon_device *rdev);
                              (rdev->flags & RADEON_IS_IGP))
 #define ASIC_IS_DCE64(rdev) ((rdev->family == CHIP_OLAND))
 #define ASIC_IS_NODCE(rdev) ((rdev->family == CHIP_HAINAN))
+#define ASIC_IS_DCE8(rdev) ((rdev->family >= CHIP_BONAIRE))
+
+#define ASIC_IS_LOMBOK(rdev) ((rdev->ddev->pdev->device == 0x6849) || \
+                              (rdev->ddev->pdev->device == 0x6850) || \
+                              (rdev->ddev->pdev->device == 0x6858) || \
+                              (rdev->ddev->pdev->device == 0x6859) || \
+                              (rdev->ddev->pdev->device == 0x6840) || \
+                              (rdev->ddev->pdev->device == 0x6841) || \
+                              (rdev->ddev->pdev->device == 0x6842) || \
+                              (rdev->ddev->pdev->device == 0x6843))
 
 /*
  * BIOS helpers.
@@ -1892,6 +2375,9 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)].ib_parse((rdev), (ib))
 #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)].is_lockup((rdev), (cp))
 #define radeon_ring_vm_flush(rdev, r, vm) (rdev)->asic->ring[(r)].vm_flush((rdev), (r), (vm))
+#define radeon_ring_get_rptr(rdev, r) (rdev)->asic->ring[(r)->idx].get_rptr((rdev), (r))
+#define radeon_ring_get_wptr(rdev, r) (rdev)->asic->ring[(r)->idx].get_wptr((rdev), (r))
+#define radeon_ring_set_wptr(rdev, r) (rdev)->asic->ring[(r)->idx].set_wptr((rdev), (r))
 #define radeon_irq_set(rdev) (rdev)->asic->irq.set((rdev))
 #define radeon_irq_process(rdev) (rdev)->asic->irq.process((rdev))
 #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->display.get_vblank_counter((rdev), (crtc))
@@ -1915,6 +2401,7 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_set_pcie_lanes(rdev, l) (rdev)->asic->pm.set_pcie_lanes((rdev), (l))
 #define radeon_set_clock_gating(rdev, e) (rdev)->asic->pm.set_clock_gating((rdev), (e))
 #define radeon_set_uvd_clocks(rdev, v, d) (rdev)->asic->pm.set_uvd_clocks((rdev), (v), (d))
+#define radeon_get_temperature(rdev) (rdev)->asic->pm.get_temperature((rdev))
 #define radeon_set_surface_reg(rdev, r, f, p, o, s) ((rdev)->asic->surface.set_reg((rdev), (r), (f), (p), (o), (s)))
 #define radeon_clear_surface_reg(rdev, r) ((rdev)->asic->surface.clear_reg((rdev), (r)))
 #define radeon_bandwidth_update(rdev) (rdev)->asic->display.bandwidth_update((rdev))
@@ -1935,6 +2422,19 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_mc_wait_for_idle(rdev) (rdev)->asic->mc_wait_for_idle((rdev))
 #define radeon_get_xclk(rdev) (rdev)->asic->get_xclk((rdev))
 #define radeon_get_gpu_clock_counter(rdev) (rdev)->asic->get_gpu_clock_counter((rdev))
+#define radeon_dpm_init(rdev) rdev->asic->dpm.init((rdev))
+#define radeon_dpm_setup_asic(rdev) rdev->asic->dpm.setup_asic((rdev))
+#define radeon_dpm_enable(rdev) rdev->asic->dpm.enable((rdev))
+#define radeon_dpm_disable(rdev) rdev->asic->dpm.disable((rdev))
+#define radeon_dpm_pre_set_power_state(rdev) rdev->asic->dpm.pre_set_power_state((rdev))
+#define radeon_dpm_set_power_state(rdev) rdev->asic->dpm.set_power_state((rdev))
+#define radeon_dpm_post_set_power_state(rdev) rdev->asic->dpm.post_set_power_state((rdev))
+#define radeon_dpm_display_configuration_changed(rdev) rdev->asic->dpm.display_configuration_changed((rdev))
+#define radeon_dpm_fini(rdev) rdev->asic->dpm.fini((rdev))
+#define radeon_dpm_get_sclk(rdev, l) rdev->asic->dpm.get_sclk((rdev), (l))
+#define radeon_dpm_get_mclk(rdev, l) rdev->asic->dpm.get_mclk((rdev), (l))
+#define radeon_dpm_print_power_state(rdev, ps) rdev->asic->dpm.print_power_state((rdev), (ps))
+#define radeon_dpm_debugfs_print_current_performance_level(rdev, m) rdev->asic->dpm.debugfs_print_current_performance_level((rdev), (m))
 
 /* Common functions */
 /* AGP */
@@ -2054,6 +2554,10 @@ extern int ni_mc_load_microcode(struct radeon_device *rdev);
 #if defined(CONFIG_ACPI)
 extern int radeon_acpi_init(struct radeon_device *rdev);
 extern void radeon_acpi_fini(struct radeon_device *rdev);
+extern bool radeon_acpi_is_pcie_performance_request_supported(struct radeon_device *rdev);
+extern int radeon_acpi_pcie_performance_request(struct radeon_device *rdev,
+                                                u8 perf_req, bool advertise);
+extern int radeon_acpi_pcie_notify_device_ready(struct radeon_device *rdev);
 #else
 static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; }
 static inline void radeon_acpi_fini(struct radeon_device *rdev) { }
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.c b/drivers/gpu/drm/radeon/radeon_acpi.c
index 196d28d99570..10f98c7742d8 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.c
+++ b/drivers/gpu/drm/radeon/radeon_acpi.c
@@ -78,6 +78,22 @@ struct atcs_verify_interface {
         u32 function_bits;      /* supported functions bit vector */
 } __packed;
 
+#define ATCS_VALID_FLAGS_MASK   0x3
+
+struct atcs_pref_req_input {
+        u16 size;               /* structure size in bytes (includes size field) */
+        u16 client_id;          /* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */
+        u16 valid_flags_mask;   /* valid flags mask */
+        u16 flags;              /* flags */
+        u8 req_type;            /* request type */
+        u8 perf_req;            /* performance request */
+} __packed;
+
+struct atcs_pref_req_output {
+        u16 size;               /* structure size in bytes (includes size field) */
+        u8 ret_val;             /* return value */
+} __packed;
+
 /* Call the ATIF method
  */
 /**
@@ -506,6 +522,135 @@ out:
 }
 
 /**
+ * radeon_acpi_is_pcie_performance_request_supported
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Check if the ATCS pcie_perf_req and pcie_dev_rdy methods
+ * are supported (all asics).
+ * returns true if supported, false if not.
+ */
+bool radeon_acpi_is_pcie_performance_request_supported(struct radeon_device *rdev)
+{
+        struct radeon_atcs *atcs = &rdev->atcs;
+
+        if (atcs->functions.pcie_perf_req && atcs->functions.pcie_dev_rdy)
+                return true;
+
+        return false;
+}
+
+/**
+ * radeon_acpi_pcie_notify_device_ready
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Executes the PCIE_DEVICE_READY_NOTIFICATION method
+ * (all asics).
+ * returns 0 on success, error on failure.
+ */
+int radeon_acpi_pcie_notify_device_ready(struct radeon_device *rdev)
+{
+        acpi_handle handle;
+        union acpi_object *info;
+        struct radeon_atcs *atcs = &rdev->atcs;
+
+        /* Get the device handle */
+        handle = DEVICE_ACPI_HANDLE(&rdev->pdev->dev);
+        if (!handle)
+                return -EINVAL;
+
+        if (!atcs->functions.pcie_dev_rdy)
+                return -EINVAL;
+
+        info = radeon_atcs_call(handle, ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION, NULL);
+        if (!info)
+                return -EIO;
+
+        kfree(info);
+
+        return 0;
+}
+
+/**
+ * radeon_acpi_pcie_performance_request
+ *
+ * @rdev: radeon_device pointer
+ * @perf_req: requested perf level (pcie gen speed)
+ * @advertise: set advertise caps flag if set
+ *
+ * Executes the PCIE_PERFORMANCE_REQUEST method to
+ * change the pcie gen speed (all asics).
+ * returns 0 on success, error on failure.
+ */
+int radeon_acpi_pcie_performance_request(struct radeon_device *rdev,
+                                         u8 perf_req, bool advertise)
+{
+        acpi_handle handle;
+        union acpi_object *info;
+        struct radeon_atcs *atcs = &rdev->atcs;
+        struct atcs_pref_req_input atcs_input;
+        struct atcs_pref_req_output atcs_output;
+        struct acpi_buffer params;
+        size_t size;
+        u32 retry = 3;
+
+        /* Get the device handle */
+        handle = DEVICE_ACPI_HANDLE(&rdev->pdev->dev);
+        if (!handle)
+                return -EINVAL;
+
+        if (!atcs->functions.pcie_perf_req)
+                return -EINVAL;
+
+        atcs_input.size = sizeof(struct atcs_pref_req_input);
+        /* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */
+        atcs_input.client_id = rdev->pdev->devfn | (rdev->pdev->bus->number << 8);
+        atcs_input.valid_flags_mask = ATCS_VALID_FLAGS_MASK;
+        atcs_input.flags = ATCS_WAIT_FOR_COMPLETION;
+        if (advertise)
+                atcs_input.flags |= ATCS_ADVERTISE_CAPS;
+        atcs_input.req_type = ATCS_PCIE_LINK_SPEED;
+        atcs_input.perf_req = perf_req;
+
+        params.length = sizeof(struct atcs_pref_req_input);
+        params.pointer = &atcs_input;
+
+        while (retry--) {
+                info = radeon_atcs_call(handle, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST, &params);
+                if (!info)
+                        return -EIO;
+
+                memset(&atcs_output, 0, sizeof(atcs_output));
+
+                size = *(u16 *) info->buffer.pointer;
+                if (size < 3) {
+                        DRM_INFO("ATCS buffer is too small: %zu\n", size);
+                        kfree(info);
+                        return -EINVAL;
+                }
+                size = min(sizeof(atcs_output), size);
+
+                memcpy(&atcs_output, info->buffer.pointer, size);
+
+                kfree(info);
+
+                switch (atcs_output.ret_val) {
+                case ATCS_REQUEST_REFUSED:
+                default:
+                        return -EINVAL;
+                case ATCS_REQUEST_COMPLETE:
+                        return 0;
+                case ATCS_REQUEST_IN_PROGRESS:
+                        udelay(10);
+                        break;
+                }
+        }
+
+        return 0;
+}
+
+/**
  * radeon_acpi_event - handle notify events
  *
  * @nb: notifier block
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index a2802b47ee95..a5b244dc50ca 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -126,7 +126,11 @@ static void radeon_register_accessor_init(struct radeon_device *rdev)
                 rdev->mc_rreg = &rs780_mc_rreg;
                 rdev->mc_wreg = &rs780_mc_wreg;
         }
-        if (rdev->family >= CHIP_R600) {
+
+        if (rdev->family >= CHIP_BONAIRE) {
+                rdev->pciep_rreg = &cik_pciep_rreg;
+                rdev->pciep_wreg = &cik_pciep_wreg;
+        } else if (rdev->family >= CHIP_R600) {
                 rdev->pciep_rreg = &r600_pciep_rreg;
                 rdev->pciep_wreg = &r600_pciep_wreg;
         }
@@ -192,6 +196,9 @@ static struct radeon_asic r100_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -268,6 +275,9 @@ static struct radeon_asic r200_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -344,6 +354,9 @@ static struct radeon_asic r300_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -420,6 +433,9 @@ static struct radeon_asic r300_asic_pcie = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -496,6 +512,9 @@ static struct radeon_asic r420_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -572,6 +591,9 @@ static struct radeon_asic rs400_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -648,6 +670,9 @@ static struct radeon_asic rs600_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -726,6 +751,9 @@ static struct radeon_asic rs690_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -804,6 +832,9 @@ static struct radeon_asic rv515_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -880,6 +911,9 @@ static struct radeon_asic r520_asic = {
                         .ring_test = &r100_ring_test,
                         .ib_test = &r100_ib_test,
                         .is_lockup = &r100_gpu_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -957,6 +991,9 @@ static struct radeon_asic r600_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &r600_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &r600_dma_ring_ib_execute,
@@ -966,6 +1003,9 @@ static struct radeon_asic r600_asic = {
                         .ring_test = &r600_dma_ring_test,
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &r600_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1012,6 +1052,115 @@ static struct radeon_asic r600_asic = {
                 .get_pcie_lanes = &r600_get_pcie_lanes,
                 .set_pcie_lanes = &r600_set_pcie_lanes,
                 .set_clock_gating = NULL,
+                .get_temperature = &rv6xx_get_temp,
+        },
+        .pflip = {
+                .pre_page_flip = &rs600_pre_page_flip,
+                .page_flip = &rs600_page_flip,
+                .post_page_flip = &rs600_post_page_flip,
+        },
+};
+
+static struct radeon_asic rv6xx_asic = {
+        .init = &r600_init,
+        .fini = &r600_fini,
+        .suspend = &r600_suspend,
+        .resume = &r600_resume,
+        .vga_set_state = &r600_vga_set_state,
+        .asic_reset = &r600_asic_reset,
+        .ioctl_wait_idle = r600_ioctl_wait_idle,
+        .gui_idle = &r600_gui_idle,
+        .mc_wait_for_idle = &r600_mc_wait_for_idle,
+        .get_xclk = &r600_get_xclk,
+        .get_gpu_clock_counter = &r600_get_gpu_clock_counter,
+        .gart = {
+                .tlb_flush = &r600_pcie_gart_tlb_flush,
+                .set_page = &rs600_gart_set_page,
+        },
+        .ring = {
+                [RADEON_RING_TYPE_GFX_INDEX] = {
+                        .ib_execute = &r600_ring_ib_execute,
+                        .emit_fence = &r600_fence_ring_emit,
+                        .emit_semaphore = &r600_semaphore_ring_emit,
+                        .cs_parse = &r600_cs_parse,
+                        .ring_test = &r600_ring_test,
+                        .ib_test = &r600_ib_test,
+                        .is_lockup = &r600_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [R600_RING_TYPE_DMA_INDEX] = {
+                        .ib_execute = &r600_dma_ring_ib_execute,
+                        .emit_fence = &r600_dma_fence_ring_emit,
+                        .emit_semaphore = &r600_dma_semaphore_ring_emit,
+                        .cs_parse = &r600_dma_cs_parse,
+                        .ring_test = &r600_dma_ring_test,
+                        .ib_test = &r600_dma_ib_test,
+                        .is_lockup = &r600_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                }
+        },
+        .irq = {
+                .set = &r600_irq_set,
+                .process = &r600_irq_process,
+        },
+        .display = {
+                .bandwidth_update = &rv515_bandwidth_update,
+                .get_vblank_counter = &rs600_get_vblank_counter,
+                .wait_for_vblank = &avivo_wait_for_vblank,
+                .set_backlight_level = &atombios_set_backlight_level,
+                .get_backlight_level = &atombios_get_backlight_level,
+        },
+        .copy = {
+                .blit = &r600_copy_blit,
+                .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+                .dma = &r600_copy_dma,
+                .dma_ring_index = R600_RING_TYPE_DMA_INDEX,
+                .copy = &r600_copy_dma,
+                .copy_ring_index = R600_RING_TYPE_DMA_INDEX,
+        },
+        .surface = {
+                .set_reg = r600_set_surface_reg,
+                .clear_reg = r600_clear_surface_reg,
+        },
+        .hpd = {
+                .init = &r600_hpd_init,
+                .fini = &r600_hpd_fini,
+                .sense = &r600_hpd_sense,
+                .set_polarity = &r600_hpd_set_polarity,
+        },
+        .pm = {
+                .misc = &r600_pm_misc,
+                .prepare = &rs600_pm_prepare,
+                .finish = &rs600_pm_finish,
+                .init_profile = &r600_pm_init_profile,
+                .get_dynpm_state = &r600_pm_get_dynpm_state,
+                .get_engine_clock = &radeon_atom_get_engine_clock,
+                .set_engine_clock = &radeon_atom_set_engine_clock,
+                .get_memory_clock = &radeon_atom_get_memory_clock,
+                .set_memory_clock = &radeon_atom_set_memory_clock,
+                .get_pcie_lanes = &r600_get_pcie_lanes,
+                .set_pcie_lanes = &r600_set_pcie_lanes,
+                .set_clock_gating = NULL,
+                .get_temperature = &rv6xx_get_temp,
+        },
+        .dpm = {
+                .init = &rv6xx_dpm_init,
+                .setup_asic = &rv6xx_setup_asic,
+                .enable = &rv6xx_dpm_enable,
+                .disable = &rv6xx_dpm_disable,
+                .pre_set_power_state = &r600_dpm_pre_set_power_state,
+                .set_power_state = &rv6xx_dpm_set_power_state,
+                .post_set_power_state = &r600_dpm_post_set_power_state,
+                .display_configuration_changed = &rv6xx_dpm_display_configuration_changed,
+                .fini = &rv6xx_dpm_fini,
+                .get_sclk = &rv6xx_dpm_get_sclk,
+                .get_mclk = &rv6xx_dpm_get_mclk,
+                .print_power_state = &rv6xx_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &rv6xx_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &rs600_pre_page_flip,
@@ -1045,6 +1194,9 @@ static struct radeon_asic rs780_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &r600_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &r600_dma_ring_ib_execute,
@@ -1054,6 +1206,9 @@ static struct radeon_asic rs780_asic = {
                         .ring_test = &r600_dma_ring_test,
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &r600_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1100,6 +1255,21 @@ static struct radeon_asic rs780_asic = {
                 .get_pcie_lanes = NULL,
                 .set_pcie_lanes = NULL,
                 .set_clock_gating = NULL,
+                .get_temperature = &rv6xx_get_temp,
+        },
+        .dpm = {
+                .init = &rs780_dpm_init,
+                .setup_asic = &rs780_dpm_setup_asic,
+                .enable = &rs780_dpm_enable,
+                .disable = &rs780_dpm_disable,
+                .pre_set_power_state = &r600_dpm_pre_set_power_state,
+                .set_power_state = &rs780_dpm_set_power_state,
+                .post_set_power_state = &r600_dpm_post_set_power_state,
+                .display_configuration_changed = &rs780_dpm_display_configuration_changed,
+                .fini = &rs780_dpm_fini,
+                .get_sclk = &rs780_dpm_get_sclk,
+                .get_mclk = &rs780_dpm_get_mclk,
+                .print_power_state = &rs780_dpm_print_power_state,
         },
         .pflip = {
                 .pre_page_flip = &rs600_pre_page_flip,
@@ -1133,6 +1303,9 @@ static struct radeon_asic rv770_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &r600_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &r600_dma_ring_ib_execute,
@@ -1142,6 +1315,9 @@ static struct radeon_asic rv770_asic = {
                         .ring_test = &r600_dma_ring_test,
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &r600_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1151,6 +1327,9 @@ static struct radeon_asic rv770_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1198,6 +1377,22 @@ static struct radeon_asic rv770_asic = {
                 .set_pcie_lanes = &r600_set_pcie_lanes,
                 .set_clock_gating = &radeon_atom_set_clock_gating,
                 .set_uvd_clocks = &rv770_set_uvd_clocks,
+                .get_temperature = &rv770_get_temp,
+        },
+        .dpm = {
+                .init = &rv770_dpm_init,
+                .setup_asic = &rv770_dpm_setup_asic,
+                .enable = &rv770_dpm_enable,
+                .disable = &rv770_dpm_disable,
+                .pre_set_power_state = &r600_dpm_pre_set_power_state,
+                .set_power_state = &rv770_dpm_set_power_state,
+                .post_set_power_state = &r600_dpm_post_set_power_state,
+                .display_configuration_changed = &rv770_dpm_display_configuration_changed,
+                .fini = &rv770_dpm_fini,
+                .get_sclk = &rv770_dpm_get_sclk,
+                .get_mclk = &rv770_dpm_get_mclk,
+                .print_power_state = &rv770_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &rv770_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &rs600_pre_page_flip,
@@ -1231,6 +1426,9 @@ static struct radeon_asic evergreen_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &evergreen_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &evergreen_dma_ring_ib_execute,
@@ -1240,6 +1438,9 @@ static struct radeon_asic evergreen_asic = {
                         .ring_test = &r600_dma_ring_test,
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &evergreen_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1249,6 +1450,9 @@ static struct radeon_asic evergreen_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1296,6 +1500,22 @@ static struct radeon_asic evergreen_asic = {
                 .set_pcie_lanes = &r600_set_pcie_lanes,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &evergreen_set_uvd_clocks,
+                .get_temperature = &evergreen_get_temp,
+        },
+        .dpm = {
+                .init = &cypress_dpm_init,
+                .setup_asic = &cypress_dpm_setup_asic,
+                .enable = &cypress_dpm_enable,
+                .disable = &cypress_dpm_disable,
+                .pre_set_power_state = &r600_dpm_pre_set_power_state,
+                .set_power_state = &cypress_dpm_set_power_state,
+                .post_set_power_state = &r600_dpm_post_set_power_state,
+                .display_configuration_changed = &cypress_dpm_display_configuration_changed,
+                .fini = &cypress_dpm_fini,
+                .get_sclk = &rv770_dpm_get_sclk,
+                .get_mclk = &rv770_dpm_get_mclk,
+                .print_power_state = &rv770_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &rv770_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &evergreen_pre_page_flip,
@@ -1329,6 +1549,9 @@ static struct radeon_asic sumo_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &evergreen_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &evergreen_dma_ring_ib_execute,
@@ -1338,6 +1561,9 @@ static struct radeon_asic sumo_asic = {
                         .ring_test = &r600_dma_ring_test,
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &evergreen_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1347,6 +1573,9 @@ static struct radeon_asic sumo_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1394,6 +1623,22 @@ static struct radeon_asic sumo_asic = {
                 .set_pcie_lanes = NULL,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &sumo_set_uvd_clocks,
+                .get_temperature = &sumo_get_temp,
+        },
+        .dpm = {
+                .init = &sumo_dpm_init,
+                .setup_asic = &sumo_dpm_setup_asic,
+                .enable = &sumo_dpm_enable,
+                .disable = &sumo_dpm_disable,
+                .pre_set_power_state = &sumo_dpm_pre_set_power_state,
+                .set_power_state = &sumo_dpm_set_power_state,
+                .post_set_power_state = &sumo_dpm_post_set_power_state,
+                .display_configuration_changed = &sumo_dpm_display_configuration_changed,
+                .fini = &sumo_dpm_fini,
+                .get_sclk = &sumo_dpm_get_sclk,
+                .get_mclk = &sumo_dpm_get_mclk,
+                .print_power_state = &sumo_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &sumo_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &evergreen_pre_page_flip,
@@ -1427,6 +1672,9 @@ static struct radeon_asic btc_asic = {
                         .ring_test = &r600_ring_test,
                         .ib_test = &r600_ib_test,
                         .is_lockup = &evergreen_gfx_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &evergreen_dma_ring_ib_execute,
@@ -1436,6 +1684,9 @@ static struct radeon_asic btc_asic = {
                         .ring_test = &r600_dma_ring_test,
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &evergreen_dma_is_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1445,6 +1696,9 @@ static struct radeon_asic btc_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1492,6 +1746,22 @@ static struct radeon_asic btc_asic = {
                 .set_pcie_lanes = &r600_set_pcie_lanes,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &evergreen_set_uvd_clocks,
+                .get_temperature = &evergreen_get_temp,
+        },
+        .dpm = {
+                .init = &btc_dpm_init,
+                .setup_asic = &btc_dpm_setup_asic,
+                .enable = &btc_dpm_enable,
+                .disable = &btc_dpm_disable,
+                .pre_set_power_state = &btc_dpm_pre_set_power_state,
+                .set_power_state = &btc_dpm_set_power_state,
+                .post_set_power_state = &btc_dpm_post_set_power_state,
+                .display_configuration_changed = &cypress_dpm_display_configuration_changed,
+                .fini = &btc_dpm_fini,
+                .get_sclk = &btc_dpm_get_sclk,
+                .get_mclk = &btc_dpm_get_mclk,
+                .print_power_state = &rv770_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &rv770_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &evergreen_pre_page_flip,
@@ -1533,6 +1803,9 @@ static struct radeon_asic cayman_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &cayman_gfx_is_lockup,
                         .vm_flush = &cayman_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_CP1_INDEX] = {
                         .ib_execute = &cayman_ring_ib_execute,
@@ -1544,6 +1817,9 @@ static struct radeon_asic cayman_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &cayman_gfx_is_lockup,
                         .vm_flush = &cayman_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_CP2_INDEX] = {
                         .ib_execute = &cayman_ring_ib_execute,
@@ -1555,6 +1831,9 @@ static struct radeon_asic cayman_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &cayman_gfx_is_lockup,
                         .vm_flush = &cayman_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &cayman_dma_ring_ib_execute,
@@ -1566,6 +1845,9 @@ static struct radeon_asic cayman_asic = {
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &cayman_dma_is_lockup,
                         .vm_flush = &cayman_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_DMA1_INDEX] = {
                         .ib_execute = &cayman_dma_ring_ib_execute,
@@ -1577,6 +1859,9 @@ static struct radeon_asic cayman_asic = {
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &cayman_dma_is_lockup,
                         .vm_flush = &cayman_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1586,6 +1871,9 @@ static struct radeon_asic cayman_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1633,6 +1921,22 @@ static struct radeon_asic cayman_asic = {
                 .set_pcie_lanes = &r600_set_pcie_lanes,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &evergreen_set_uvd_clocks,
+                .get_temperature = &evergreen_get_temp,
+        },
+        .dpm = {
+                .init = &ni_dpm_init,
+                .setup_asic = &ni_dpm_setup_asic,
+                .enable = &ni_dpm_enable,
+                .disable = &ni_dpm_disable,
+                .pre_set_power_state = &ni_dpm_pre_set_power_state,
+                .set_power_state = &ni_dpm_set_power_state,
+                .post_set_power_state = &ni_dpm_post_set_power_state,
+                .display_configuration_changed = &cypress_dpm_display_configuration_changed,
+                .fini = &ni_dpm_fini,
+                .get_sclk = &ni_dpm_get_sclk,
+                .get_mclk = &ni_dpm_get_mclk,
+                .print_power_state = &ni_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &ni_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &evergreen_pre_page_flip,
@@ -1674,6 +1978,9 @@ static struct radeon_asic trinity_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &cayman_gfx_is_lockup,
                         .vm_flush = &cayman_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_CP1_INDEX] = {
                         .ib_execute = &cayman_ring_ib_execute,
@@ -1685,6 +1992,9 @@ static struct radeon_asic trinity_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &cayman_gfx_is_lockup,
                         .vm_flush = &cayman_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_CP2_INDEX] = {
                         .ib_execute = &cayman_ring_ib_execute,
@@ -1696,6 +2006,9 @@ static struct radeon_asic trinity_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &cayman_gfx_is_lockup,
                         .vm_flush = &cayman_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &cayman_dma_ring_ib_execute,
@@ -1707,6 +2020,9 @@ static struct radeon_asic trinity_asic = {
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &cayman_dma_is_lockup,
                         .vm_flush = &cayman_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_DMA1_INDEX] = {
                         .ib_execute = &cayman_dma_ring_ib_execute,
@@ -1718,6 +2034,9 @@ static struct radeon_asic trinity_asic = {
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &cayman_dma_is_lockup,
                         .vm_flush = &cayman_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1727,6 +2046,9 @@ static struct radeon_asic trinity_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1772,6 +2094,22 @@ static struct radeon_asic trinity_asic = {
                 .set_pcie_lanes = NULL,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &sumo_set_uvd_clocks,
+                .get_temperature = &tn_get_temp,
+        },
+        .dpm = {
+                .init = &trinity_dpm_init,
+                .setup_asic = &trinity_dpm_setup_asic,
+                .enable = &trinity_dpm_enable,
+                .disable = &trinity_dpm_disable,
+                .pre_set_power_state = &trinity_dpm_pre_set_power_state,
+                .set_power_state = &trinity_dpm_set_power_state,
+                .post_set_power_state = &trinity_dpm_post_set_power_state,
+                .display_configuration_changed = &trinity_dpm_display_configuration_changed,
+                .fini = &trinity_dpm_fini,
+                .get_sclk = &trinity_dpm_get_sclk,
+                .get_mclk = &trinity_dpm_get_mclk,
+                .print_power_state = &trinity_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &trinity_dpm_debugfs_print_current_performance_level,
         },
         .pflip = {
                 .pre_page_flip = &evergreen_pre_page_flip,
@@ -1813,6 +2151,9 @@ static struct radeon_asic si_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &si_gfx_is_lockup,
                         .vm_flush = &si_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_CP1_INDEX] = {
                         .ib_execute = &si_ring_ib_execute,
@@ -1824,6 +2165,9 @@ static struct radeon_asic si_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &si_gfx_is_lockup,
                         .vm_flush = &si_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_CP2_INDEX] = {
                         .ib_execute = &si_ring_ib_execute,
@@ -1835,6 +2179,9 @@ static struct radeon_asic si_asic = {
                         .ib_test = &r600_ib_test,
                         .is_lockup = &si_gfx_is_lockup,
                         .vm_flush = &si_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_DMA_INDEX] = {
                         .ib_execute = &cayman_dma_ring_ib_execute,
@@ -1846,6 +2193,9 @@ static struct radeon_asic si_asic = {
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &si_dma_is_lockup,
                         .vm_flush = &si_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [CAYMAN_RING_TYPE_DMA1_INDEX] = {
                         .ib_execute = &cayman_dma_ring_ib_execute,
@@ -1857,6 +2207,9 @@ static struct radeon_asic si_asic = {
                         .ib_test = &r600_dma_ib_test,
                         .is_lockup = &si_dma_is_lockup,
                         .vm_flush = &si_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 },
                 [R600_RING_TYPE_UVD_INDEX] = {
                         .ib_execute = &r600_uvd_ib_execute,
@@ -1866,6 +2219,9 @@ static struct radeon_asic si_asic = {
                         .ring_test = &r600_uvd_ring_test,
                         .ib_test = &r600_uvd_ib_test,
                         .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
                 }
         },
         .irq = {
@@ -1911,6 +2267,332 @@ static struct radeon_asic si_asic = {
                 .set_pcie_lanes = &r600_set_pcie_lanes,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &si_set_uvd_clocks,
+                .get_temperature = &si_get_temp,
+        },
+        .dpm = {
+                .init = &si_dpm_init,
+                .setup_asic = &si_dpm_setup_asic,
+                .enable = &si_dpm_enable,
+                .disable = &si_dpm_disable,
+                .pre_set_power_state = &si_dpm_pre_set_power_state,
+                .set_power_state = &si_dpm_set_power_state,
+                .post_set_power_state = &si_dpm_post_set_power_state,
+                .display_configuration_changed = &si_dpm_display_configuration_changed,
+                .fini = &si_dpm_fini,
+                .get_sclk = &ni_dpm_get_sclk,
+                .get_mclk = &ni_dpm_get_mclk,
+                .print_power_state = &ni_dpm_print_power_state,
+                .debugfs_print_current_performance_level = &si_dpm_debugfs_print_current_performance_level,
+        },
+        .pflip = {
+                .pre_page_flip = &evergreen_pre_page_flip,
+                .page_flip = &evergreen_page_flip,
+                .post_page_flip = &evergreen_post_page_flip,
+        },
+};
+
+static struct radeon_asic ci_asic = {
+        .init = &cik_init,
+        .fini = &cik_fini,
+        .suspend = &cik_suspend,
+        .resume = &cik_resume,
+        .asic_reset = &cik_asic_reset,
+        .vga_set_state = &r600_vga_set_state,
+        .ioctl_wait_idle = NULL,
+        .gui_idle = &r600_gui_idle,
+        .mc_wait_for_idle = &evergreen_mc_wait_for_idle,
+        .get_xclk = &cik_get_xclk,
+        .get_gpu_clock_counter = &cik_get_gpu_clock_counter,
+        .gart = {
+                .tlb_flush = &cik_pcie_gart_tlb_flush,
+                .set_page = &rs600_gart_set_page,
+        },
+        .vm = {
+                .init = &cik_vm_init,
+                .fini = &cik_vm_fini,
+                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
+                .set_page = &cik_vm_set_page,
+        },
+        .ring = {
+                [RADEON_RING_TYPE_GFX_INDEX] = {
+                        .ib_execute = &cik_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_fence_gfx_ring_emit,
+                        .emit_semaphore = &cik_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_ring_test,
+                        .ib_test = &cik_ib_test,
+                        .is_lockup = &cik_gfx_is_lockup,
+                        .vm_flush = &cik_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [CAYMAN_RING_TYPE_CP1_INDEX] = {
+                        .ib_execute = &cik_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_fence_compute_ring_emit,
+                        .emit_semaphore = &cik_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_ring_test,
+                        .ib_test = &cik_ib_test,
+                        .is_lockup = &cik_gfx_is_lockup,
+                        .vm_flush = &cik_vm_flush,
+                        .get_rptr = &cik_compute_ring_get_rptr,
+                        .get_wptr = &cik_compute_ring_get_wptr,
+                        .set_wptr = &cik_compute_ring_set_wptr,
+                },
+                [CAYMAN_RING_TYPE_CP2_INDEX] = {
+                        .ib_execute = &cik_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_fence_compute_ring_emit,
+                        .emit_semaphore = &cik_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_ring_test,
+                        .ib_test = &cik_ib_test,
+                        .is_lockup = &cik_gfx_is_lockup,
+                        .vm_flush = &cik_vm_flush,
+                        .get_rptr = &cik_compute_ring_get_rptr,
+                        .get_wptr = &cik_compute_ring_get_wptr,
+                        .set_wptr = &cik_compute_ring_set_wptr,
+                },
+                [R600_RING_TYPE_DMA_INDEX] = {
+                        .ib_execute = &cik_sdma_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_sdma_fence_ring_emit,
+                        .emit_semaphore = &cik_sdma_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_sdma_ring_test,
+                        .ib_test = &cik_sdma_ib_test,
+                        .is_lockup = &cik_sdma_is_lockup,
+                        .vm_flush = &cik_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [CAYMAN_RING_TYPE_DMA1_INDEX] = {
+                        .ib_execute = &cik_sdma_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_sdma_fence_ring_emit,
+                        .emit_semaphore = &cik_sdma_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_sdma_ring_test,
+                        .ib_test = &cik_sdma_ib_test,
+                        .is_lockup = &cik_sdma_is_lockup,
+                        .vm_flush = &cik_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [R600_RING_TYPE_UVD_INDEX] = {
+                        .ib_execute = &r600_uvd_ib_execute,
+                        .emit_fence = &r600_uvd_fence_emit,
+                        .emit_semaphore = &cayman_uvd_semaphore_emit,
+                        .cs_parse = &radeon_uvd_cs_parse,
+                        .ring_test = &r600_uvd_ring_test,
+                        .ib_test = &r600_uvd_ib_test,
+                        .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                }
+        },
+        .irq = {
+                .set = &cik_irq_set,
+                .process = &cik_irq_process,
+        },
+        .display = {
+                .bandwidth_update = &dce8_bandwidth_update,
+                .get_vblank_counter = &evergreen_get_vblank_counter,
+                .wait_for_vblank = &dce4_wait_for_vblank,
+        },
+        .copy = {
+                .blit = NULL,
+                .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+                .dma = &cik_copy_dma,
+                .dma_ring_index = R600_RING_TYPE_DMA_INDEX,
+                .copy = &cik_copy_dma,
+                .copy_ring_index = R600_RING_TYPE_DMA_INDEX,
+        },
+        .surface = {
+                .set_reg = r600_set_surface_reg,
+                .clear_reg = r600_clear_surface_reg,
+        },
+        .hpd = {
+                .init = &evergreen_hpd_init,
+                .fini = &evergreen_hpd_fini,
+                .sense = &evergreen_hpd_sense,
+                .set_polarity = &evergreen_hpd_set_polarity,
+        },
+        .pm = {
+                .misc = &evergreen_pm_misc,
+                .prepare = &evergreen_pm_prepare,
+                .finish = &evergreen_pm_finish,
+                .init_profile = &sumo_pm_init_profile,
+                .get_dynpm_state = &r600_pm_get_dynpm_state,
+                .get_engine_clock = &radeon_atom_get_engine_clock,
+                .set_engine_clock = &radeon_atom_set_engine_clock,
+                .get_memory_clock = &radeon_atom_get_memory_clock,
+                .set_memory_clock = &radeon_atom_set_memory_clock,
+                .get_pcie_lanes = NULL,
+                .set_pcie_lanes = NULL,
+                .set_clock_gating = NULL,
+                .set_uvd_clocks = &cik_set_uvd_clocks,
+        },
+        .pflip = {
+                .pre_page_flip = &evergreen_pre_page_flip,
+                .page_flip = &evergreen_page_flip,
+                .post_page_flip = &evergreen_post_page_flip,
+        },
+};
+
+static struct radeon_asic kv_asic = {
+        .init = &cik_init,
+        .fini = &cik_fini,
+        .suspend = &cik_suspend,
+        .resume = &cik_resume,
+        .asic_reset = &cik_asic_reset,
+        .vga_set_state = &r600_vga_set_state,
+        .ioctl_wait_idle = NULL,
+        .gui_idle = &r600_gui_idle,
+        .mc_wait_for_idle = &evergreen_mc_wait_for_idle,
+        .get_xclk = &cik_get_xclk,
+        .get_gpu_clock_counter = &cik_get_gpu_clock_counter,
+        .gart = {
+                .tlb_flush = &cik_pcie_gart_tlb_flush,
+                .set_page = &rs600_gart_set_page,
+        },
+        .vm = {
+                .init = &cik_vm_init,
+                .fini = &cik_vm_fini,
+                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
+                .set_page = &cik_vm_set_page,
+        },
+        .ring = {
+                [RADEON_RING_TYPE_GFX_INDEX] = {
+                        .ib_execute = &cik_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_fence_gfx_ring_emit,
+                        .emit_semaphore = &cik_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_ring_test,
+                        .ib_test = &cik_ib_test,
+                        .is_lockup = &cik_gfx_is_lockup,
+                        .vm_flush = &cik_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [CAYMAN_RING_TYPE_CP1_INDEX] = {
+                        .ib_execute = &cik_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_fence_compute_ring_emit,
+                        .emit_semaphore = &cik_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_ring_test,
+                        .ib_test = &cik_ib_test,
+                        .is_lockup = &cik_gfx_is_lockup,
+                        .vm_flush = &cik_vm_flush,
+                        .get_rptr = &cik_compute_ring_get_rptr,
+                        .get_wptr = &cik_compute_ring_get_wptr,
+                        .set_wptr = &cik_compute_ring_set_wptr,
+                },
+                [CAYMAN_RING_TYPE_CP2_INDEX] = {
+                        .ib_execute = &cik_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_fence_compute_ring_emit,
+                        .emit_semaphore = &cik_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_ring_test,
+                        .ib_test = &cik_ib_test,
+                        .is_lockup = &cik_gfx_is_lockup,
+                        .vm_flush = &cik_vm_flush,
+                        .get_rptr = &cik_compute_ring_get_rptr,
+                        .get_wptr = &cik_compute_ring_get_wptr,
+                        .set_wptr = &cik_compute_ring_set_wptr,
+                },
+                [R600_RING_TYPE_DMA_INDEX] = {
+                        .ib_execute = &cik_sdma_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_sdma_fence_ring_emit,
+                        .emit_semaphore = &cik_sdma_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_sdma_ring_test,
+                        .ib_test = &cik_sdma_ib_test,
+                        .is_lockup = &cik_sdma_is_lockup,
+                        .vm_flush = &cik_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [CAYMAN_RING_TYPE_DMA1_INDEX] = {
+                        .ib_execute = &cik_sdma_ring_ib_execute,
+                        .ib_parse = &cik_ib_parse,
+                        .emit_fence = &cik_sdma_fence_ring_emit,
+                        .emit_semaphore = &cik_sdma_semaphore_ring_emit,
+                        .cs_parse = NULL,
+                        .ring_test = &cik_sdma_ring_test,
+                        .ib_test = &cik_sdma_ib_test,
+                        .is_lockup = &cik_sdma_is_lockup,
+                        .vm_flush = &cik_dma_vm_flush,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                },
+                [R600_RING_TYPE_UVD_INDEX] = {
+                        .ib_execute = &r600_uvd_ib_execute,
+                        .emit_fence = &r600_uvd_fence_emit,
+                        .emit_semaphore = &cayman_uvd_semaphore_emit,
+                        .cs_parse = &radeon_uvd_cs_parse,
+                        .ring_test = &r600_uvd_ring_test,
+                        .ib_test = &r600_uvd_ib_test,
+                        .is_lockup = &radeon_ring_test_lockup,
+                        .get_rptr = &radeon_ring_generic_get_rptr,
+                        .get_wptr = &radeon_ring_generic_get_wptr,
+                        .set_wptr = &radeon_ring_generic_set_wptr,
+                }
+        },
+        .irq = {
+                .set = &cik_irq_set,
+                .process = &cik_irq_process,
+        },
+        .display = {
+                .bandwidth_update = &dce8_bandwidth_update,
+                .get_vblank_counter = &evergreen_get_vblank_counter,
+                .wait_for_vblank = &dce4_wait_for_vblank,
+        },
+        .copy = {
+                .blit = NULL,
+                .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+                .dma = &cik_copy_dma,
+                .dma_ring_index = R600_RING_TYPE_DMA_INDEX,
+                .copy = &cik_copy_dma,
+                .copy_ring_index = R600_RING_TYPE_DMA_INDEX,
+        },
+        .surface = {
+                .set_reg = r600_set_surface_reg,
+                .clear_reg = r600_clear_surface_reg,
+        },
+        .hpd = {
+                .init = &evergreen_hpd_init,
+                .fini = &evergreen_hpd_fini,
+                .sense = &evergreen_hpd_sense,
+                .set_polarity = &evergreen_hpd_set_polarity,
+        },
+        .pm = {
+                .misc = &evergreen_pm_misc,
+                .prepare = &evergreen_pm_prepare,
+                .finish = &evergreen_pm_finish,
+                .init_profile = &sumo_pm_init_profile,
+                .get_dynpm_state = &r600_pm_get_dynpm_state,
+                .get_engine_clock = &radeon_atom_get_engine_clock,
+                .set_engine_clock = &radeon_atom_set_engine_clock,
+                .get_memory_clock = &radeon_atom_get_memory_clock,
+                .set_memory_clock = &radeon_atom_set_memory_clock,
+                .get_pcie_lanes = NULL,
+                .set_pcie_lanes = NULL,
+                .set_clock_gating = NULL,
+                .set_uvd_clocks = &cik_set_uvd_clocks,
         },
         .pflip = {
                 .pre_page_flip = &evergreen_pre_page_flip,
@@ -1999,16 +2681,15 @@ int radeon_asic_init(struct radeon_device *rdev)
                 rdev->asic = &r520_asic;
                 break;
         case CHIP_R600:
+                rdev->asic = &r600_asic;
+                break;
         case CHIP_RV610:
         case CHIP_RV630:
         case CHIP_RV620:
         case CHIP_RV635:
         case CHIP_RV670:
-                rdev->asic = &r600_asic;
-                if (rdev->family == CHIP_R600)
-                        rdev->has_uvd = false;
-                else
-                        rdev->has_uvd = true;
+                rdev->asic = &rv6xx_asic;
+                rdev->has_uvd = true;
                 break;
         case CHIP_RS780:
         case CHIP_RS880:
@@ -2082,6 +2763,19 @@ int radeon_asic_init(struct radeon_device *rdev)
                 else
                         rdev->has_uvd = true;
                 break;
+        case CHIP_BONAIRE:
+                rdev->asic = &ci_asic;
+                rdev->num_crtc = 6;
+                break;
+        case CHIP_KAVERI:
+        case CHIP_KABINI:
+                rdev->asic = &kv_asic;
+                /* set num crtcs */
+                if (rdev->family == CHIP_KAVERI)
+                        rdev->num_crtc = 4;
+                else
+                        rdev->num_crtc = 2;
+                break;
         default:
                 /* FIXME: not supported yet */
                 return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index a72759ede753..6822c7aeacaa 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -47,6 +47,12 @@ u8 atombios_get_backlight_level(struct radeon_encoder *radeon_encoder);
 void radeon_legacy_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level);
 u8 radeon_legacy_get_backlight_level(struct radeon_encoder *radeon_encoder);
 
+u32 radeon_ring_generic_get_rptr(struct radeon_device *rdev,
+                                 struct radeon_ring *ring);
+u32 radeon_ring_generic_get_wptr(struct radeon_device *rdev,
+                                 struct radeon_ring *ring);
+void radeon_ring_generic_set_wptr(struct radeon_device *rdev,
+                                  struct radeon_ring *ring);
 
 /*
  * r100,rv100,rs100,rv200,rs200
@@ -395,6 +401,35 @@ void r600_kms_blit_copy(struct radeon_device *rdev,
 int r600_mc_wait_for_idle(struct radeon_device *rdev);
 u32 r600_get_xclk(struct radeon_device *rdev);
 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev);
+int rv6xx_get_temp(struct radeon_device *rdev);
+int r600_dpm_pre_set_power_state(struct radeon_device *rdev);
+void r600_dpm_post_set_power_state(struct radeon_device *rdev);
+/* rv6xx dpm */
+int rv6xx_dpm_init(struct radeon_device *rdev);
+int rv6xx_dpm_enable(struct radeon_device *rdev);
+void rv6xx_dpm_disable(struct radeon_device *rdev);
+int rv6xx_dpm_set_power_state(struct radeon_device *rdev);
+void rv6xx_setup_asic(struct radeon_device *rdev);
+void rv6xx_dpm_display_configuration_changed(struct radeon_device *rdev);
+void rv6xx_dpm_fini(struct radeon_device *rdev);
+u32 rv6xx_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 rv6xx_dpm_get_mclk(struct radeon_device *rdev, bool low);
+void rv6xx_dpm_print_power_state(struct radeon_device *rdev,
+                                 struct radeon_ps *ps);
+void rv6xx_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                       struct seq_file *m);
+/* rs780 dpm */
+int rs780_dpm_init(struct radeon_device *rdev);
+int rs780_dpm_enable(struct radeon_device *rdev);
+void rs780_dpm_disable(struct radeon_device *rdev);
+int rs780_dpm_set_power_state(struct radeon_device *rdev);
+void rs780_dpm_setup_asic(struct radeon_device *rdev);
+void rs780_dpm_display_configuration_changed(struct radeon_device *rdev);
+void rs780_dpm_fini(struct radeon_device *rdev);
+u32 rs780_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 rs780_dpm_get_mclk(struct radeon_device *rdev, bool low);
+void rs780_dpm_print_power_state(struct radeon_device *rdev,
+                                 struct radeon_ps *ps);
 
 /* uvd */
 int r600_uvd_init(struct radeon_device *rdev);
@@ -428,6 +463,21 @@ int rv770_copy_dma(struct radeon_device *rdev,
 u32 rv770_get_xclk(struct radeon_device *rdev);
 int rv770_uvd_resume(struct radeon_device *rdev);
 int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
+int rv770_get_temp(struct radeon_device *rdev);
+/* rv7xx pm */
+int rv770_dpm_init(struct radeon_device *rdev);
+int rv770_dpm_enable(struct radeon_device *rdev);
+void rv770_dpm_disable(struct radeon_device *rdev);
+int rv770_dpm_set_power_state(struct radeon_device *rdev);
+void rv770_dpm_setup_asic(struct radeon_device *rdev);
+void rv770_dpm_display_configuration_changed(struct radeon_device *rdev);
+void rv770_dpm_fini(struct radeon_device *rdev);
+u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low);
+void rv770_dpm_print_power_state(struct radeon_device *rdev,
+                                 struct radeon_ps *ps);
+void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                       struct seq_file *m);
 
 /*
  * evergreen
@@ -482,6 +532,41 @@ int evergreen_copy_dma(struct radeon_device *rdev,
                        struct radeon_fence **fence);
 void evergreen_hdmi_enable(struct drm_encoder *encoder, bool enable);
 void evergreen_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode *mode);
+int evergreen_get_temp(struct radeon_device *rdev);
+int sumo_get_temp(struct radeon_device *rdev);
+int tn_get_temp(struct radeon_device *rdev);
+int cypress_dpm_init(struct radeon_device *rdev);
+void cypress_dpm_setup_asic(struct radeon_device *rdev);
+int cypress_dpm_enable(struct radeon_device *rdev);
+void cypress_dpm_disable(struct radeon_device *rdev);
+int cypress_dpm_set_power_state(struct radeon_device *rdev);
+void cypress_dpm_display_configuration_changed(struct radeon_device *rdev);
+void cypress_dpm_fini(struct radeon_device *rdev);
+int btc_dpm_init(struct radeon_device *rdev);
+void btc_dpm_setup_asic(struct radeon_device *rdev);
+int btc_dpm_enable(struct radeon_device *rdev);
+void btc_dpm_disable(struct radeon_device *rdev);
+int btc_dpm_pre_set_power_state(struct radeon_device *rdev);
+int btc_dpm_set_power_state(struct radeon_device *rdev);
+void btc_dpm_post_set_power_state(struct radeon_device *rdev);
+void btc_dpm_fini(struct radeon_device *rdev);
+u32 btc_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 btc_dpm_get_mclk(struct radeon_device *rdev, bool low);
+int sumo_dpm_init(struct radeon_device *rdev);
+int sumo_dpm_enable(struct radeon_device *rdev);
+void sumo_dpm_disable(struct radeon_device *rdev);
+int sumo_dpm_pre_set_power_state(struct radeon_device *rdev);
+int sumo_dpm_set_power_state(struct radeon_device *rdev);
+void sumo_dpm_post_set_power_state(struct radeon_device *rdev);
+void sumo_dpm_setup_asic(struct radeon_device *rdev);
+void sumo_dpm_display_configuration_changed(struct radeon_device *rdev);
+void sumo_dpm_fini(struct radeon_device *rdev);
+u32 sumo_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 sumo_dpm_get_mclk(struct radeon_device *rdev, bool low);
+void sumo_dpm_print_power_state(struct radeon_device *rdev,
+                                struct radeon_ps *ps);
+void sumo_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                      struct seq_file *m);
 
 /*
  * cayman
@@ -516,6 +601,36 @@ bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
 void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 
+int ni_dpm_init(struct radeon_device *rdev);
+void ni_dpm_setup_asic(struct radeon_device *rdev);
+int ni_dpm_enable(struct radeon_device *rdev);
+void ni_dpm_disable(struct radeon_device *rdev);
+int ni_dpm_pre_set_power_state(struct radeon_device *rdev);
+int ni_dpm_set_power_state(struct radeon_device *rdev);
+void ni_dpm_post_set_power_state(struct radeon_device *rdev);
+void ni_dpm_fini(struct radeon_device *rdev);
+u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low);
+void ni_dpm_print_power_state(struct radeon_device *rdev,
+                              struct radeon_ps *ps);
+void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                    struct seq_file *m);
+int trinity_dpm_init(struct radeon_device *rdev);
+int trinity_dpm_enable(struct radeon_device *rdev);
+void trinity_dpm_disable(struct radeon_device *rdev);
+int trinity_dpm_pre_set_power_state(struct radeon_device *rdev);
+int trinity_dpm_set_power_state(struct radeon_device *rdev);
+void trinity_dpm_post_set_power_state(struct radeon_device *rdev);
+void trinity_dpm_setup_asic(struct radeon_device *rdev);
+void trinity_dpm_display_configuration_changed(struct radeon_device *rdev);
+void trinity_dpm_fini(struct radeon_device *rdev);
+u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low);
+u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low);
+void trinity_dpm_print_power_state(struct radeon_device *rdev,
+                                   struct radeon_ps *ps);
+void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                         struct seq_file *m);
+
 /* DCE6 - SI */
 void dce6_bandwidth_update(struct radeon_device *rdev);
 
@@ -552,5 +667,80 @@ void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 u32 si_get_xclk(struct radeon_device *rdev);
 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev);
 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
+int si_get_temp(struct radeon_device *rdev);
+int si_dpm_init(struct radeon_device *rdev);
+void si_dpm_setup_asic(struct radeon_device *rdev);
+int si_dpm_enable(struct radeon_device *rdev);
+void si_dpm_disable(struct radeon_device *rdev);
+int si_dpm_pre_set_power_state(struct radeon_device *rdev);
+int si_dpm_set_power_state(struct radeon_device *rdev);
+void si_dpm_post_set_power_state(struct radeon_device *rdev);
+void si_dpm_fini(struct radeon_device *rdev);
+void si_dpm_display_configuration_changed(struct radeon_device *rdev);
+void si_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                    struct seq_file *m);
+
+/* DCE8 - CIK */
+void dce8_bandwidth_update(struct radeon_device *rdev);
+
+/*
+ * cik
+ */
+uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev);
+u32 cik_get_xclk(struct radeon_device *rdev);
+uint32_t cik_pciep_rreg(struct radeon_device *rdev, uint32_t reg);
+void cik_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
+int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
+int cik_uvd_resume(struct radeon_device *rdev);
+void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
+                              struct radeon_fence *fence);
+void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
+                                  struct radeon_ring *ring,
+                                  struct radeon_semaphore *semaphore,
+                                  bool emit_wait);
+void cik_sdma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
+int cik_copy_dma(struct radeon_device *rdev,
+                 uint64_t src_offset, uint64_t dst_offset,
+                 unsigned num_gpu_pages,
+                 struct radeon_fence **fence);
+int cik_sdma_ring_test(struct radeon_device *rdev, struct radeon_ring *ring);
+int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring);
+bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
+void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
+                             struct radeon_fence *fence);
+void cik_fence_compute_ring_emit(struct radeon_device *rdev,
+                                 struct radeon_fence *fence);
+void cik_semaphore_ring_emit(struct radeon_device *rdev,
+                             struct radeon_ring *cp,
+                             struct radeon_semaphore *semaphore,
+                             bool emit_wait);
+void cik_pcie_gart_tlb_flush(struct radeon_device *rdev);
+int cik_init(struct radeon_device *rdev);
+void cik_fini(struct radeon_device *rdev);
+int cik_suspend(struct radeon_device *rdev);
+int cik_resume(struct radeon_device *rdev);
+bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp);
+int cik_asic_reset(struct radeon_device *rdev);
+void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
+int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring);
+int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring);
+int cik_irq_set(struct radeon_device *rdev);
+int cik_irq_process(struct radeon_device *rdev);
+int cik_vm_init(struct radeon_device *rdev);
+void cik_vm_fini(struct radeon_device *rdev);
+void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
+void cik_vm_set_page(struct radeon_device *rdev,
+                     struct radeon_ib *ib,
+                     uint64_t pe,
+                     uint64_t addr, unsigned count,
+                     uint32_t incr, uint32_t flags);
+void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
+int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
+u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
+                              struct radeon_ring *ring);
+u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
+                              struct radeon_ring *ring);
+void cik_compute_ring_set_wptr(struct radeon_device *rdev,
+                               struct radeon_ring *ring);
 
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index dea6f63c9724..b1777d10d0b5 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -56,10 +56,6 @@ extern void
 radeon_add_legacy_encoder(struct drm_device *dev, uint32_t encoder_enum,
                           uint32_t supported_device);
 
-/* local */
-static int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
-                                    u16 voltage_id, u16 *voltage);
-
 union atom_supported_devices {
         struct _ATOM_SUPPORTED_DEVICES_INFO info;
         struct _ATOM_SUPPORTED_DEVICES_INFO_2 info_2;
@@ -1247,6 +1243,7 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                         }
                         rdev->clock.dp_extclk =
                                 le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq);
+                        rdev->clock.current_dispclk = rdev->clock.default_dispclk;
                 }
                 *dcpll = *p1pll;
 
@@ -1269,6 +1266,7 @@ union igp_info {
         struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
         struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
         struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8;
 };
 
 bool radeon_atombios_sideport_present(struct radeon_device *rdev)
@@ -1438,6 +1436,22 @@ static void radeon_atombios_get_igp_ss_overrides(struct radeon_device *rdev,
                                 break;
                         }
                         break;
+                case 8:
+                        switch (id) {
+                        case ASIC_INTERNAL_SS_ON_TMDS:
+                                percentage = le16_to_cpu(igp_info->info_8.usDVISSPercentage);
+                                rate = le16_to_cpu(igp_info->info_8.usDVISSpreadRateIn10Hz);
+                                break;
+                        case ASIC_INTERNAL_SS_ON_HDMI:
+                                percentage = le16_to_cpu(igp_info->info_8.usHDMISSPercentage);
+                                rate = le16_to_cpu(igp_info->info_8.usHDMISSpreadRateIn10Hz);
+                                break;
+                        case ASIC_INTERNAL_SS_ON_LVDS:
+                                percentage = le16_to_cpu(igp_info->info_8.usLvdsSSPercentage);
+                                rate = le16_to_cpu(igp_info->info_8.usLvdsSSpreadRateIn10Hz);
+                                break;
+                        }
+                        break;
                 default:
                         DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
                         break;
@@ -1499,6 +1513,10 @@ bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev,
                                                 le16_to_cpu(ss_info->info_2.asSpreadSpectrum[i].usSpreadSpectrumPercentage);
                                         ss->type = ss_info->info_2.asSpreadSpectrum[i].ucSpreadSpectrumMode;
                                         ss->rate = le16_to_cpu(ss_info->info_2.asSpreadSpectrum[i].usSpreadRateIn10Hz);
+                                        if ((crev == 2) &&
+                                            ((id == ASIC_INTERNAL_ENGINE_SS) ||
+                                             (id == ASIC_INTERNAL_MEMORY_SS)))
+                                                ss->rate /= 100;
                                         return true;
                                 }
                         }
@@ -1513,6 +1531,9 @@ bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev,
                                                 le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadSpectrumPercentage);
                                         ss->type = ss_info->info_3.asSpreadSpectrum[i].ucSpreadSpectrumMode;
                                         ss->rate = le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadRateIn10Hz);
+                                        if ((id == ASIC_INTERNAL_ENGINE_SS) ||
+                                            (id == ASIC_INTERNAL_MEMORY_SS))
+                                                ss->rate /= 100;
                                         if (rdev->flags & RADEON_IS_IGP)
                                                 radeon_atombios_get_igp_ss_overrides(rdev, ss, id);
                                         return true;
@@ -1927,6 +1948,7 @@ static const char *pp_lib_thermal_controller_names[] = {
         "Northern Islands",
         "Southern Islands",
         "lm96163",
+        "Sea Islands",
 };
 
 union power_info {
@@ -1944,6 +1966,7 @@ union pplib_clock_info {
         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
         struct _ATOM_PPLIB_SI_CLOCK_INFO si;
+        struct _ATOM_PPLIB_CI_CLOCK_INFO ci;
 };
 
 union pplib_power_state {
@@ -2209,6 +2232,11 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r
                                  (controller->ucFanParameters &
                                   ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
                         rdev->pm.int_thermal_type = THERMAL_TYPE_SI;
+                } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_CISLANDS) {
+                        DRM_INFO("Internal thermal controller %s fan control\n",
+                                 (controller->ucFanParameters &
+                                  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
+                        rdev->pm.int_thermal_type = THERMAL_TYPE_CI;
                 } else if ((controller->ucType ==
                             ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) ||
                            (controller->ucType ==
@@ -2241,8 +2269,8 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r
         }
 }
 
-static void radeon_atombios_get_default_voltages(struct radeon_device *rdev,
-                                                 u16 *vddc, u16 *vddci)
+void radeon_atombios_get_default_voltages(struct radeon_device *rdev,
+                                          u16 *vddc, u16 *vddci, u16 *mvdd)
 {
         struct radeon_mode_info *mode_info = &rdev->mode_info;
         int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
@@ -2252,6 +2280,7 @@ static void radeon_atombios_get_default_voltages(struct radeon_device *rdev,
 
         *vddc = 0;
         *vddci = 0;
+        *mvdd = 0;
 
         if (atom_parse_data_header(mode_info->atom_context, index, NULL,
                                    &frev, &crev, &data_offset)) {
@@ -2259,8 +2288,10 @@ static void radeon_atombios_get_default_voltages(struct radeon_device *rdev,
                         (union firmware_info *)(mode_info->atom_context->bios +
                                                 data_offset);
                 *vddc = le16_to_cpu(firmware_info->info_14.usBootUpVDDCVoltage);
-                if ((frev == 2) && (crev >= 2))
+                if ((frev == 2) && (crev >= 2)) {
                         *vddci = le16_to_cpu(firmware_info->info_22.usBootUpVDDCIVoltage);
+                        *mvdd = le16_to_cpu(firmware_info->info_22.usBootUpMVDDCVoltage);
+                }
         }
 }
 
@@ -2271,9 +2302,9 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde
         int j;
         u32 misc = le32_to_cpu(non_clock_info->ulCapsAndSettings);
         u32 misc2 = le16_to_cpu(non_clock_info->usClassification);
-        u16 vddc, vddci;
+        u16 vddc, vddci, mvdd;
 
-        radeon_atombios_get_default_voltages(rdev, &vddc, &vddci);
+        radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
 
         rdev->pm.power_state[state_index].misc = misc;
         rdev->pm.power_state[state_index].misc2 = misc2;
@@ -2316,7 +2347,13 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde
                         rdev->pm.default_vddc = rdev->pm.power_state[state_index].clock_info[0].voltage.voltage;
                         rdev->pm.default_vddci = rdev->pm.power_state[state_index].clock_info[0].voltage.vddci;
                 } else {
-                        /* patch the table values with the default slck/mclk from firmware info */
+                        u16 max_vddci = 0;
+
+                        if (ASIC_IS_DCE4(rdev))
+                                radeon_atom_get_max_voltage(rdev,
+                                                            SET_VOLTAGE_TYPE_ASIC_VDDCI,
+                                                            &max_vddci);
+                        /* patch the table values with the default sclk/mclk from firmware info */
                         for (j = 0; j < mode_index; j++) {
                                 rdev->pm.power_state[state_index].clock_info[j].mclk =
                                         rdev->clock.default_mclk;
@@ -2325,6 +2362,9 @@ static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rde
                                 if (vddc)
                                         rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
                                                 vddc;
+                                if (max_vddci)
+                                        rdev->pm.power_state[state_index].clock_info[j].voltage.vddci =
+                                                max_vddci;
                         }
                 }
         }
@@ -2347,6 +2387,15 @@ static bool radeon_atombios_parse_pplib_clock_info(struct radeon_device *rdev,
                         sclk |= clock_info->rs780.ucLowEngineClockHigh << 16;
                         rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
                 }
+        } else if (rdev->family >= CHIP_BONAIRE) {
+                sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
+                sclk |= clock_info->ci.ucEngineClockHigh << 16;
+                mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
+                mclk |= clock_info->ci.ucMemoryClockHigh << 16;
+                rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
+                rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
+                rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
+                        VOLTAGE_NONE;
         } else if (rdev->family >= CHIP_TAHITI) {
                 sclk = le16_to_cpu(clock_info->si.usEngineClockLow);
                 sclk |= clock_info->si.ucEngineClockHigh << 16;
@@ -2667,6 +2716,8 @@ union get_clock_dividers {
         struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3 v3;
         struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 v4;
         struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5 v5;
+        struct _COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6 v6_in;
+        struct _COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 v6_out;
 };
 
 int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
@@ -2699,7 +2750,8 @@ int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
                 break;
         case 2:
         case 3:
-                /* r6xx, r7xx, evergreen, ni */
+        case 5:
+                /* r6xx, r7xx, evergreen, ni, si */
                 if (rdev->family <= CHIP_RV770) {
                         args.v2.ucAction = clock_type;
                         args.v2.ulClock = cpu_to_le32(clock);   /* 10 khz */
@@ -2732,6 +2784,9 @@ int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
                                 dividers->vco_mode = (args.v3.ucCntlFlag &
                                                       ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0;
                         } else {
+                                /* for SI we use ComputeMemoryClockParam for memory plls */
+                                if (rdev->family >= CHIP_TAHITI)
+                                        return -EINVAL;
                                 args.v5.ulClockParams = cpu_to_le32((clock_type << 24) | clock);
                                 if (strobe_mode)
                                         args.v5.ucInputFlag = ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN;
@@ -2757,9 +2812,76 @@ int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
 
                 atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 
-                dividers->post_div = args.v4.ucPostDiv;
+                dividers->post_divider = dividers->post_div = args.v4.ucPostDiv;
                 dividers->real_clock = le32_to_cpu(args.v4.ulClock);
                 break;
+        case 6:
+                /* CI */
+                /* COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, COMPUTE_GPUCLK_INPUT_FLAG_SCLK */
+                args.v6_in.ulClock.ulComputeClockFlag = clock_type;
+                args.v6_in.ulClock.ulClockFreq = cpu_to_le32(clock);    /* 10 khz */
+
+                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+                dividers->whole_fb_div = le16_to_cpu(args.v6_out.ulFbDiv.usFbDiv);
+                dividers->frac_fb_div = le16_to_cpu(args.v6_out.ulFbDiv.usFbDivFrac);
+                dividers->ref_div = args.v6_out.ucPllRefDiv;
+                dividers->post_div = args.v6_out.ucPllPostDiv;
+                dividers->flags = args.v6_out.ucPllCntlFlag;
+                dividers->real_clock = le32_to_cpu(args.v6_out.ulClock.ulClock);
+                dividers->post_divider = args.v6_out.ulClock.ucPostDiv;
+                break;
+        default:
+                return -EINVAL;
+        }
+        return 0;
+}
+
+int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev,
+                                        u32 clock,
+                                        bool strobe_mode,
+                                        struct atom_mpll_param *mpll_param)
+{
+        COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 args;
+        int index = GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam);
+        u8 frev, crev;
+
+        memset(&args, 0, sizeof(args));
+        memset(mpll_param, 0, sizeof(struct atom_mpll_param));
+
+        if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
+                return -EINVAL;
+
+        switch (frev) {
+        case 2:
+                switch (crev) {
+                case 1:
+                        /* SI */
+                        args.ulClock = cpu_to_le32(clock);      /* 10 khz */
+                        args.ucInputFlag = 0;
+                        if (strobe_mode)
+                                args.ucInputFlag |= MPLL_INPUT_FLAG_STROBE_MODE_EN;
+
+                        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+                        mpll_param->clkfrac = le16_to_cpu(args.ulFbDiv.usFbDivFrac);
+                        mpll_param->clkf = le16_to_cpu(args.ulFbDiv.usFbDiv);
+                        mpll_param->post_div = args.ucPostDiv;
+                        mpll_param->dll_speed = args.ucDllSpeed;
+                        mpll_param->bwcntl = args.ucBWCntl;
+                        mpll_param->vco_mode =
+                                (args.ucPllCntlFlag & MPLL_CNTL_FLAG_VCO_MODE_MASK) ? 1 : 0;
+                        mpll_param->yclk_sel =
+                                (args.ucPllCntlFlag & MPLL_CNTL_FLAG_BYPASS_DQ_PLL) ? 1 : 0;
+                        mpll_param->qdr =
+                                (args.ucPllCntlFlag & MPLL_CNTL_FLAG_QDR_ENABLE) ? 1 : 0;
+                        mpll_param->half_rate =
+                                (args.ucPllCntlFlag & MPLL_CNTL_FLAG_AD_HALF_RATE) ? 1 : 0;
+                        break;
+                default:
+                        return -EINVAL;
+                }
+                break;
         default:
                 return -EINVAL;
         }
@@ -2819,6 +2941,48 @@ void radeon_atom_set_memory_clock(struct radeon_device *rdev,
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
 
+void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev,
+                                         u32 eng_clock, u32 mem_clock)
+{
+        SET_ENGINE_CLOCK_PS_ALLOCATION args;
+        int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings);
+        u32 tmp;
+
+        memset(&args, 0, sizeof(args));
+
+        tmp = eng_clock & SET_CLOCK_FREQ_MASK;
+        tmp |= (COMPUTE_ENGINE_PLL_PARAM << 24);
+
+        args.ulTargetEngineClock = cpu_to_le32(tmp);
+        if (mem_clock)
+                args.sReserved.ulClock = cpu_to_le32(mem_clock & SET_CLOCK_FREQ_MASK);
+
+        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+}
+
+void radeon_atom_update_memory_dll(struct radeon_device *rdev,
+                                   u32 mem_clock)
+{
+        u32 args;
+        int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings);
+
+        args = cpu_to_le32(mem_clock);  /* 10 khz */
+
+        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+}
+
+void radeon_atom_set_ac_timing(struct radeon_device *rdev,
+                               u32 mem_clock)
+{
+        SET_MEMORY_CLOCK_PS_ALLOCATION args;
+        int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings);
+        u32 tmp = mem_clock | (COMPUTE_MEMORY_PLL_PARAM << 24);
+
+        args.ulTargetMemoryClock = cpu_to_le32(tmp);    /* 10 khz */
+
+        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+}
+
 union set_voltage {
         struct _SET_VOLTAGE_PS_ALLOCATION alloc;
         struct _SET_VOLTAGE_PARAMETERS v1;
@@ -2863,8 +3027,8 @@ void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 v
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
 
-static int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
-                                    u16 voltage_id, u16 *voltage)
+int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
+                             u16 voltage_id, u16 *voltage)
 {
         union set_voltage args;
         int index = GetIndexIntoMasterTable(COMMAND, SetVoltage);
@@ -2902,6 +3066,695 @@ static int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
         return 0;
 }
 
+int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev,
+                                                      u16 *voltage,
+                                                      u16 leakage_idx)
+{
+        return radeon_atom_get_max_vddc(rdev, VOLTAGE_TYPE_VDDC, leakage_idx, voltage);
+}
+
+int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev,
+                                          u16 voltage_level, u8 voltage_type,
+                                          u32 *gpio_value, u32 *gpio_mask)
+{
+        union set_voltage args;
+        int index = GetIndexIntoMasterTable(COMMAND, SetVoltage);
+        u8 frev, crev;
+
+        if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
+                return -EINVAL;
+
+        switch (crev) {
+        case 1:
+                return -EINVAL;
+        case 2:
+                args.v2.ucVoltageType = voltage_type;
+                args.v2.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_GET_GPIOMASK;
+                args.v2.usVoltageLevel = cpu_to_le16(voltage_level);
+
+                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+                *gpio_mask = le32_to_cpu(*(u32 *)&args.v2);
+
+                args.v2.ucVoltageType = voltage_type;
+                args.v2.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_GET_GPIOVAL;
+                args.v2.usVoltageLevel = cpu_to_le16(voltage_level);
+
+                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+
+                *gpio_value = le32_to_cpu(*(u32 *)&args.v2);
+                break;
+        default:
+                DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+union voltage_object_info {
+        struct _ATOM_VOLTAGE_OBJECT_INFO v1;
+        struct _ATOM_VOLTAGE_OBJECT_INFO_V2 v2;
+        struct _ATOM_VOLTAGE_OBJECT_INFO_V3_1 v3;
+};
+
+union voltage_object {
+        struct _ATOM_VOLTAGE_OBJECT v1;
+        struct _ATOM_VOLTAGE_OBJECT_V2 v2;
+        union _ATOM_VOLTAGE_OBJECT_V3 v3;
+};
+
+static ATOM_VOLTAGE_OBJECT *atom_lookup_voltage_object_v1(ATOM_VOLTAGE_OBJECT_INFO *v1,
+                                                          u8 voltage_type)
+{
+        u32 size = le16_to_cpu(v1->sHeader.usStructureSize);
+        u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO, asVoltageObj[0]);
+        u8 *start = (u8 *)v1;
+
+        while (offset < size) {
+                ATOM_VOLTAGE_OBJECT *vo = (ATOM_VOLTAGE_OBJECT *)(start + offset);
+                if (vo->ucVoltageType == voltage_type)
+                        return vo;
+                offset += offsetof(ATOM_VOLTAGE_OBJECT, asFormula.ucVIDAdjustEntries) +
+                        vo->asFormula.ucNumOfVoltageEntries;
+        }
+        return NULL;
+}
+
+static ATOM_VOLTAGE_OBJECT_V2 *atom_lookup_voltage_object_v2(ATOM_VOLTAGE_OBJECT_INFO_V2 *v2,
+                                                             u8 voltage_type)
+{
+        u32 size = le16_to_cpu(v2->sHeader.usStructureSize);
+        u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V2, asVoltageObj[0]);
+        u8 *start = (u8*)v2;
+
+        while (offset < size) {
+                ATOM_VOLTAGE_OBJECT_V2 *vo = (ATOM_VOLTAGE_OBJECT_V2 *)(start + offset);
+                if (vo->ucVoltageType == voltage_type)
+                        return vo;
+                offset += offsetof(ATOM_VOLTAGE_OBJECT_V2, asFormula.asVIDAdjustEntries) +
+                        (vo->asFormula.ucNumOfVoltageEntries * sizeof(VOLTAGE_LUT_ENTRY));
+        }
+        return NULL;
+}
+
+static ATOM_VOLTAGE_OBJECT_V3 *atom_lookup_voltage_object_v3(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *v3,
+                                                             u8 voltage_type, u8 voltage_mode)
+{
+        u32 size = le16_to_cpu(v3->sHeader.usStructureSize);
+        u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]);
+        u8 *start = (u8*)v3;
+
+        while (offset < size) {
+                ATOM_VOLTAGE_OBJECT_V3 *vo = (ATOM_VOLTAGE_OBJECT_V3 *)(start + offset);
+                if ((vo->asGpioVoltageObj.sHeader.ucVoltageType == voltage_type) &&
+                    (vo->asGpioVoltageObj.sHeader.ucVoltageMode == voltage_mode))
+                        return vo;
+                offset += le16_to_cpu(vo->asGpioVoltageObj.sHeader.usSize);
+        }
+        return NULL;
+}
+
+bool
+radeon_atom_is_voltage_gpio(struct radeon_device *rdev,
+                            u8 voltage_type, u8 voltage_mode)
+{
+        int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+        u8 frev, crev;
+        u16 data_offset, size;
+        union voltage_object_info *voltage_info;
+        union voltage_object *voltage_object = NULL;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                voltage_info = (union voltage_object_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+
+                switch (frev) {
+                case 1:
+                case 2:
+                        switch (crev) {
+                        case 1:
+                                voltage_object = (union voltage_object *)
+                                        atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type);
+                                if (voltage_object &&
+                                    (voltage_object->v1.asControl.ucVoltageControlId == VOLTAGE_CONTROLLED_BY_GPIO))
+                                        return true;
+                                break;
+                        case 2:
+                                voltage_object = (union voltage_object *)
+                                        atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type);
+                                if (voltage_object &&
+                                    (voltage_object->v2.asControl.ucVoltageControlId == VOLTAGE_CONTROLLED_BY_GPIO))
+                                        return true;
+                                break;
+                        default:
+                                DRM_ERROR("unknown voltage object table\n");
+                                return false;
+                        }
+                        break;
+                case 3:
+                        switch (crev) {
+                        case 1:
+                                if (atom_lookup_voltage_object_v3(&voltage_info->v3,
+                                                                  voltage_type, voltage_mode))
+                                        return true;
+                                break;
+                        default:
+                                DRM_ERROR("unknown voltage object table\n");
+                                return false;
+                        }
+                        break;
+                default:
+                        DRM_ERROR("unknown voltage object table\n");
+                        return false;
+                }
+
+        }
+        return false;
+}
+
+int radeon_atom_get_max_voltage(struct radeon_device *rdev,
+                                u8 voltage_type, u16 *max_voltage)
+{
+        int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+        u8 frev, crev;
+        u16 data_offset, size;
+        union voltage_object_info *voltage_info;
+        union voltage_object *voltage_object = NULL;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                voltage_info = (union voltage_object_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+
+                switch (crev) {
+                case 1:
+                        voltage_object = (union voltage_object *)
+                                atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type);
+                        if (voltage_object) {
+                                ATOM_VOLTAGE_FORMULA *formula =
+                                        &voltage_object->v1.asFormula;
+                                if (formula->ucFlag & 1)
+                                        *max_voltage =
+                                                le16_to_cpu(formula->usVoltageBaseLevel) +
+                                                formula->ucNumOfVoltageEntries / 2 *
+                                                le16_to_cpu(formula->usVoltageStep);
+                                else
+                                        *max_voltage =
+                                                le16_to_cpu(formula->usVoltageBaseLevel) +
+                                                (formula->ucNumOfVoltageEntries - 1) *
+                                                le16_to_cpu(formula->usVoltageStep);
+                                return 0;
+                        }
+                        break;
+                case 2:
+                        voltage_object = (union voltage_object *)
+                                atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type);
+                        if (voltage_object) {
+                                ATOM_VOLTAGE_FORMULA_V2 *formula =
+                                        &voltage_object->v2.asFormula;
+                                if (formula->ucNumOfVoltageEntries) {
+                                        *max_voltage =
+                                                le16_to_cpu(formula->asVIDAdjustEntries[
+                                                                    formula->ucNumOfVoltageEntries - 1
+                                                                    ].usVoltageValue);
+                                        return 0;
+                                }
+                        }
+                        break;
+                default:
+                        DRM_ERROR("unknown voltage object table\n");
+                        return -EINVAL;
+                }
+
+        }
+        return -EINVAL;
+}
+
+int radeon_atom_get_min_voltage(struct radeon_device *rdev,
+                                u8 voltage_type, u16 *min_voltage)
+{
+        int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+        u8 frev, crev;
+        u16 data_offset, size;
+        union voltage_object_info *voltage_info;
+        union voltage_object *voltage_object = NULL;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                voltage_info = (union voltage_object_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+
+                switch (crev) {
+                case 1:
+                        voltage_object = (union voltage_object *)
+                                atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type);
+                        if (voltage_object) {
+                                ATOM_VOLTAGE_FORMULA *formula =
+                                        &voltage_object->v1.asFormula;
+                                *min_voltage =
+                                        le16_to_cpu(formula->usVoltageBaseLevel);
+                                return 0;
+                        }
+                        break;
+                case 2:
+                        voltage_object = (union voltage_object *)
+                                atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type);
+                        if (voltage_object) {
+                                ATOM_VOLTAGE_FORMULA_V2 *formula =
+                                        &voltage_object->v2.asFormula;
+                                if (formula->ucNumOfVoltageEntries) {
+                                        *min_voltage =
+                                                le16_to_cpu(formula->asVIDAdjustEntries[
+                                                                    0
+                                                                    ].usVoltageValue);
+                                        return 0;
+                                }
+                        }
+                        break;
+                default:
+                        DRM_ERROR("unknown voltage object table\n");
+                        return -EINVAL;
+                }
+
+        }
+        return -EINVAL;
+}
+
+int radeon_atom_get_voltage_step(struct radeon_device *rdev,
+                                 u8 voltage_type, u16 *voltage_step)
+{
+        int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+        u8 frev, crev;
+        u16 data_offset, size;
+        union voltage_object_info *voltage_info;
+        union voltage_object *voltage_object = NULL;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                voltage_info = (union voltage_object_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+
+                switch (crev) {
+                case 1:
+                        voltage_object = (union voltage_object *)
+                                atom_lookup_voltage_object_v1(&voltage_info->v1, voltage_type);
+                        if (voltage_object) {
+                                ATOM_VOLTAGE_FORMULA *formula =
+                                        &voltage_object->v1.asFormula;
+                                if (formula->ucFlag & 1)
+                                        *voltage_step =
+                                                (le16_to_cpu(formula->usVoltageStep) + 1) / 2;
+                                else
+                                        *voltage_step =
+                                                le16_to_cpu(formula->usVoltageStep);
+                                return 0;
+                        }
+                        break;
+                case 2:
+                        return -EINVAL;
+                default:
+                        DRM_ERROR("unknown voltage object table\n");
+                        return -EINVAL;
+                }
+
+        }
+        return -EINVAL;
+}
+
+int radeon_atom_round_to_true_voltage(struct radeon_device *rdev,
+                                      u8 voltage_type,
+                                      u16 nominal_voltage,
+                                      u16 *true_voltage)
+{
+        u16 min_voltage, max_voltage, voltage_step;
+
+        if (radeon_atom_get_max_voltage(rdev, voltage_type, &max_voltage))
+                return -EINVAL;
+        if (radeon_atom_get_min_voltage(rdev, voltage_type, &min_voltage))
+                return -EINVAL;
+        if (radeon_atom_get_voltage_step(rdev, voltage_type, &voltage_step))
+                return -EINVAL;
+
+        if (nominal_voltage <= min_voltage)
+                *true_voltage = min_voltage;
+        else if (nominal_voltage >= max_voltage)
+                *true_voltage = max_voltage;
+        else
+                *true_voltage = min_voltage +
+                        ((nominal_voltage - min_voltage) / voltage_step) *
+                        voltage_step;
+
+        return 0;
+}
+
+int radeon_atom_get_voltage_table(struct radeon_device *rdev,
+                                  u8 voltage_type, u8 voltage_mode,
+                                  struct atom_voltage_table *voltage_table)
+{
+        int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+        u8 frev, crev;
+        u16 data_offset, size;
+        int i, ret;
+        union voltage_object_info *voltage_info;
+        union voltage_object *voltage_object = NULL;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                voltage_info = (union voltage_object_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+
+                switch (frev) {
+                case 1:
+                case 2:
+                        switch (crev) {
+                        case 1:
+                                DRM_ERROR("old table version %d, %d\n", frev, crev);
+                                return -EINVAL;
+                        case 2:
+                                voltage_object = (union voltage_object *)
+                                        atom_lookup_voltage_object_v2(&voltage_info->v2, voltage_type);
+                                if (voltage_object) {
+                                        ATOM_VOLTAGE_FORMULA_V2 *formula =
+                                                &voltage_object->v2.asFormula;
+                                        if (formula->ucNumOfVoltageEntries > MAX_VOLTAGE_ENTRIES)
+                                                return -EINVAL;
+                                        for (i = 0; i < formula->ucNumOfVoltageEntries; i++) {
+                                                voltage_table->entries[i].value =
+                                                        le16_to_cpu(formula->asVIDAdjustEntries[i].usVoltageValue);
+                                                ret = radeon_atom_get_voltage_gpio_settings(rdev,
+                                                                                            voltage_table->entries[i].value,
+                                                                                            voltage_type,
+                                                                                            &voltage_table->entries[i].smio_low,
+                                                                                            &voltage_table->mask_low);
+                                                if (ret)
+                                                        return ret;
+                                        }
+                                        voltage_table->count = formula->ucNumOfVoltageEntries;
+                                        return 0;
+                                }
+                                break;
+                        default:
+                                DRM_ERROR("unknown voltage object table\n");
+                                return -EINVAL;
+                        }
+                        break;
+                case 3:
+                        switch (crev) {
+                        case 1:
+                                voltage_object = (union voltage_object *)
+                                        atom_lookup_voltage_object_v3(&voltage_info->v3,
+                                                                      voltage_type, voltage_mode);
+                                if (voltage_object) {
+                                        ATOM_GPIO_VOLTAGE_OBJECT_V3 *gpio =
+                                                &voltage_object->v3.asGpioVoltageObj;
+                                        if (gpio->ucGpioEntryNum > MAX_VOLTAGE_ENTRIES)
+                                                return -EINVAL;
+                                        for (i = 0; i < gpio->ucGpioEntryNum; i++) {
+                                                voltage_table->entries[i].value =
+                                                        le16_to_cpu(gpio->asVolGpioLut[i].usVoltageValue);
+                                                voltage_table->entries[i].smio_low =
+                                                        le32_to_cpu(gpio->asVolGpioLut[i].ulVoltageId);
+                                        }
+                                        voltage_table->mask_low = le32_to_cpu(gpio->ulGpioMaskVal);
+                                        voltage_table->count = gpio->ucGpioEntryNum;
+                                        voltage_table->phase_delay = gpio->ucPhaseDelay;
+                                        return 0;
+                                }
+                                break;
+                        default:
+                                DRM_ERROR("unknown voltage object table\n");
+                                return -EINVAL;
+                        }
+                        break;
+                default:
+                        DRM_ERROR("unknown voltage object table\n");
+                        return -EINVAL;
+                }
+        }
+        return -EINVAL;
+}
+
+union vram_info {
+        struct _ATOM_VRAM_INFO_V3 v1_3;
+        struct _ATOM_VRAM_INFO_V4 v1_4;
+        struct _ATOM_VRAM_INFO_HEADER_V2_1 v2_1;
+};
+
+int radeon_atom_get_memory_info(struct radeon_device *rdev,
+                                u8 module_index, struct atom_memory_info *mem_info)
+{
+        int index = GetIndexIntoMasterTable(DATA, VRAM_Info);
+        u8 frev, crev, i;
+        u16 data_offset, size;
+        union vram_info *vram_info;
+        u8 *p;
+
+        memset(mem_info, 0, sizeof(struct atom_memory_info));
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                vram_info = (union vram_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+                switch (frev) {
+                case 1:
+                        switch (crev) {
+                        case 3:
+                                /* r6xx */
+                                if (module_index < vram_info->v1_3.ucNumOfVRAMModule) {
+                                        ATOM_VRAM_MODULE_V3 *vram_module =
+                                                (ATOM_VRAM_MODULE_V3 *)vram_info->v1_3.aVramInfo;
+                                        p = (u8 *)vram_info->v1_3.aVramInfo;
+
+                                        for (i = 0; i < module_index; i++) {
+                                                vram_module = (ATOM_VRAM_MODULE_V3 *)p;
+                                                if (le16_to_cpu(vram_module->usSize) == 0)
+                                                        return -EINVAL;
+                                                p += le16_to_cpu(vram_module->usSize);
+                                        }
+                                        mem_info->mem_vendor = vram_module->asMemory.ucMemoryVenderID & 0xf;
+                                        mem_info->mem_type = vram_module->asMemory.ucMemoryType & 0xf0;
+                                } else
+                                        return -EINVAL;
+                                break;
+                        case 4:
+                                /* r7xx, evergreen */
+                                if (module_index < vram_info->v1_4.ucNumOfVRAMModule) {
+                                        ATOM_VRAM_MODULE_V4 *vram_module =
+                                                (ATOM_VRAM_MODULE_V4 *)vram_info->v1_4.aVramInfo;
+                                        p = (u8 *)vram_info->v1_4.aVramInfo;
+
+                                        for (i = 0; i < module_index; i++) {
+                                                vram_module = (ATOM_VRAM_MODULE_V4 *)p;
+                                                if (le16_to_cpu(vram_module->usModuleSize) == 0)
+                                                        return -EINVAL;
+                                                p += le16_to_cpu(vram_module->usModuleSize);
+                                        }
+                                        mem_info->mem_vendor = vram_module->ucMemoryVenderID & 0xf;
+                                        mem_info->mem_type = vram_module->ucMemoryType & 0xf0;
+                                } else
+                                        return -EINVAL;
+                                break;
+                        default:
+                                DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                                return -EINVAL;
+                        }
+                        break;
+                case 2:
+                        switch (crev) {
+                        case 1:
+                                /* ni */
+                                if (module_index < vram_info->v2_1.ucNumOfVRAMModule) {
+                                        ATOM_VRAM_MODULE_V7 *vram_module =
+                                                (ATOM_VRAM_MODULE_V7 *)vram_info->v2_1.aVramInfo;
+                                        p = (u8 *)vram_info->v2_1.aVramInfo;
+
+                                        for (i = 0; i < module_index; i++) {
+                                                vram_module = (ATOM_VRAM_MODULE_V7 *)p;
+                                                if (le16_to_cpu(vram_module->usModuleSize) == 0)
+                                                        return -EINVAL;
+                                                p += le16_to_cpu(vram_module->usModuleSize);
+                                        }
+                                        mem_info->mem_vendor = vram_module->ucMemoryVenderID & 0xf;
+                                        mem_info->mem_type = vram_module->ucMemoryType & 0xf0;
+                                } else
+                                        return -EINVAL;
+                                break;
+                        default:
+                                DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                                return -EINVAL;
+                        }
+                        break;
+                default:
+                        DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                        return -EINVAL;
+                }
+                return 0;
+        }
+        return -EINVAL;
+}
+
+int radeon_atom_get_mclk_range_table(struct radeon_device *rdev,
+                                     bool gddr5, u8 module_index,
+                                     struct atom_memory_clock_range_table *mclk_range_table)
+{
+        int index = GetIndexIntoMasterTable(DATA, VRAM_Info);
+        u8 frev, crev, i;
+        u16 data_offset, size;
+        union vram_info *vram_info;
+        u32 mem_timing_size = gddr5 ?
+                sizeof(ATOM_MEMORY_TIMING_FORMAT_V2) : sizeof(ATOM_MEMORY_TIMING_FORMAT);
+        u8 *p;
+
+        memset(mclk_range_table, 0, sizeof(struct atom_memory_clock_range_table));
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                vram_info = (union vram_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+                switch (frev) {
+                case 1:
+                        switch (crev) {
+                        case 3:
+                                DRM_ERROR("old table version %d, %d\n", frev, crev);
+                                return -EINVAL;
+                        case 4:
+                                /* r7xx, evergreen */
+                                if (module_index < vram_info->v1_4.ucNumOfVRAMModule) {
+                                        ATOM_VRAM_MODULE_V4 *vram_module =
+                                                (ATOM_VRAM_MODULE_V4 *)vram_info->v1_4.aVramInfo;
+                                        ATOM_MEMORY_TIMING_FORMAT *format;
+                                        p = (u8 *)vram_info->v1_4.aVramInfo;
+
+                                        for (i = 0; i < module_index; i++) {
+                                                vram_module = (ATOM_VRAM_MODULE_V4 *)p;
+                                                if (le16_to_cpu(vram_module->usModuleSize) == 0)
+                                                        return -EINVAL;
+                                                p += le16_to_cpu(vram_module->usModuleSize);
+                                        }
+                                        mclk_range_table->num_entries = (u8)
+                                                ((vram_module->usModuleSize - offsetof(ATOM_VRAM_MODULE_V4, asMemTiming)) /
+                                                 mem_timing_size);
+                                        p = (u8 *)vram_module->asMemTiming;
+                                        for (i = 0; i < mclk_range_table->num_entries; i++) {
+                                                format = (ATOM_MEMORY_TIMING_FORMAT *)p;
+                                                mclk_range_table->mclk[i] = le32_to_cpu(format->ulClkRange);
+                                                p += mem_timing_size;
+                                        }
+                                } else
+                                        return -EINVAL;
+                                break;
+                        default:
+                                DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                                return -EINVAL;
+                        }
+                        break;
+                case 2:
+                        DRM_ERROR("new table version %d, %d\n", frev, crev);
+                        return -EINVAL;
+                default:
+                        DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                        return -EINVAL;
+                }
+                return 0;
+        }
+        return -EINVAL;
+}
+
+#define MEM_ID_MASK           0xff000000
+#define MEM_ID_SHIFT          24
+#define CLOCK_RANGE_MASK      0x00ffffff
+#define CLOCK_RANGE_SHIFT     0
+#define LOW_NIBBLE_MASK       0xf
+#define DATA_EQU_PREV         0
+#define DATA_FROM_TABLE       4
+
+int radeon_atom_init_mc_reg_table(struct radeon_device *rdev,
+                                  u8 module_index,
+                                  struct atom_mc_reg_table *reg_table)
+{
+        int index = GetIndexIntoMasterTable(DATA, VRAM_Info);
+        u8 frev, crev, num_entries, t_mem_id, num_ranges = 0;
+        u32 i = 0, j;
+        u16 data_offset, size;
+        union vram_info *vram_info;
+
+        memset(reg_table, 0, sizeof(struct atom_mc_reg_table));
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                vram_info = (union vram_info *)
+                        (rdev->mode_info.atom_context->bios + data_offset);
+                switch (frev) {
+                case 1:
+                        DRM_ERROR("old table version %d, %d\n", frev, crev);
+                        return -EINVAL;
+                case 2:
+                        switch (crev) {
+                        case 1:
+                                if (module_index < vram_info->v2_1.ucNumOfVRAMModule) {
+                                        ATOM_INIT_REG_BLOCK *reg_block =
+                                                (ATOM_INIT_REG_BLOCK *)
+                                                ((u8 *)vram_info + le16_to_cpu(vram_info->v2_1.usMemClkPatchTblOffset));
+                                        ATOM_MEMORY_SETTING_DATA_BLOCK *reg_data =
+                                                (ATOM_MEMORY_SETTING_DATA_BLOCK *)
+                                                ((u8 *)reg_block + (2 * sizeof(u16)) +
+                                                 le16_to_cpu(reg_block->usRegIndexTblSize));
+                                        num_entries = (u8)((le16_to_cpu(reg_block->usRegIndexTblSize)) /
+                                                           sizeof(ATOM_INIT_REG_INDEX_FORMAT)) - 1;
+                                        if (num_entries > VBIOS_MC_REGISTER_ARRAY_SIZE)
+                                                return -EINVAL;
+                                        while (!(reg_block->asRegIndexBuf[i].ucPreRegDataLength & ACCESS_PLACEHOLDER) &&
+                                              (i < num_entries)) {
+                                                reg_table->mc_reg_address[i].s1 =
+                                                        (u16)(le16_to_cpu(reg_block->asRegIndexBuf[i].usRegIndex));
+                                                reg_table->mc_reg_address[i].pre_reg_data =
+                                                        (u8)(reg_block->asRegIndexBuf[i].ucPreRegDataLength);
+                                                i++;
+                                        }
+                                        reg_table->last = i;
+                                        while ((*(u32 *)reg_data != END_OF_REG_DATA_BLOCK) &&
+                                               (num_ranges < VBIOS_MAX_AC_TIMING_ENTRIES)) {
+                                                t_mem_id = (u8)((*(u32 *)reg_data & MEM_ID_MASK) >> MEM_ID_SHIFT);
+                                                if (module_index == t_mem_id) {
+                                                        reg_table->mc_reg_table_entry[num_ranges].mclk_max =
+                                                                (u32)((*(u32 *)reg_data & CLOCK_RANGE_MASK) >> CLOCK_RANGE_SHIFT);
+                                                        for (i = 0, j = 1; i < reg_table->last; i++) {
+                                                                if ((reg_table->mc_reg_address[i].pre_reg_data & LOW_NIBBLE_MASK) == DATA_FROM_TABLE) {
+                                                                        reg_table->mc_reg_table_entry[num_ranges].mc_data[i] =
+                                                                                (u32)*((u32 *)reg_data + j);
+                                                                        j++;
+                                                                } else if ((reg_table->mc_reg_address[i].pre_reg_data & LOW_NIBBLE_MASK) == DATA_EQU_PREV) {
+                                                                        reg_table->mc_reg_table_entry[num_ranges].mc_data[i] =
+                                                                                reg_table->mc_reg_table_entry[num_ranges].mc_data[i - 1];
+                                                                }
+                                                        }
+                                                        num_ranges++;
+                                                }
+                                                reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *)
+                                                        ((u8 *)reg_data + le16_to_cpu(reg_block->usRegDataBlkSize));
+                                        }
+                                        if (*(u32 *)reg_data != END_OF_REG_DATA_BLOCK)
+                                                return -EINVAL;
+                                        reg_table->num_entries = num_ranges;
+                                } else
+                                        return -EINVAL;
+                                break;
+                        default:
+                                DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                                return -EINVAL;
+                        }
+                        break;
+                default:
+                        DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+                        return -EINVAL;
+                }
+                return 0;
+        }
+        return -EINVAL;
+}
+
 void radeon_atom_initialize_bios_scratch_regs(struct drm_device *dev)
 {
         struct radeon_device *rdev = dev->dev_private;
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 7e265a58141f..13a130fb3517 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -106,7 +106,7 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
                 radeon_bo_list_add_object(&p->relocs[i].lobj,
                                           &p->validated);
         }
-        return radeon_bo_list_validate(&p->validated, p->ring);
+        return radeon_bo_list_validate(&p->ticket, &p->validated, p->ring);
 }
 
 static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
@@ -314,15 +314,17 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
  * If error is set than unvalidate buffer, otherwise just free memory
  * used by parsing context.
  **/
-static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
+static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff)
 {
         unsigned i;
 
         if (!error) {
-                ttm_eu_fence_buffer_objects(&parser->validated,
+                ttm_eu_fence_buffer_objects(&parser->ticket,
+                                            &parser->validated,
                                             parser->ib.fence);
-        } else {
-                ttm_eu_backoff_reservation(&parser->validated);
+        } else if (backoff) {
+                ttm_eu_backoff_reservation(&parser->ticket,
+                                           &parser->validated);
         }
 
         if (parser->relocs != NULL) {
@@ -535,7 +537,7 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
         r = radeon_cs_parser_init(&parser, data);
         if (r) {
                 DRM_ERROR("Failed to initialize parser !\n");
-                radeon_cs_parser_fini(&parser, r);
+                radeon_cs_parser_fini(&parser, r, false);
                 up_read(&rdev->exclusive_lock);
                 r = radeon_cs_handle_lockup(rdev, r);
                 return r;
@@ -544,12 +546,13 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
         if (r) {
                 if (r != -ERESTARTSYS)
                         DRM_ERROR("Failed to parse relocation %d!\n", r);
-                radeon_cs_parser_fini(&parser, r);
+                radeon_cs_parser_fini(&parser, r, false);
                 up_read(&rdev->exclusive_lock);
                 r = radeon_cs_handle_lockup(rdev, r);
                 return r;
         }
 
+        /* XXX pick SD/HD/MVC */
         if (parser.ring == R600_RING_TYPE_UVD_INDEX)
                 radeon_uvd_note_usage(rdev);
 
@@ -562,7 +565,7 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 goto out;
         }
 out:
-        radeon_cs_parser_fini(&parser, r);
+        radeon_cs_parser_fini(&parser, r, true);
         up_read(&rdev->exclusive_lock);
         r = radeon_cs_handle_lockup(rdev, r);
         return r;
diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c
index b097d5b4ff39..9630e8d95fb4 100644
--- a/drivers/gpu/drm/radeon/radeon_cursor.c
+++ b/drivers/gpu/drm/radeon/radeon_cursor.c
@@ -27,9 +27,6 @@
 #include <drm/radeon_drm.h>
 #include "radeon.h"
 
-#define CURSOR_WIDTH 64
-#define CURSOR_HEIGHT 64
-
 static void radeon_lock_cursor(struct drm_crtc *crtc, bool lock)
 {
         struct radeon_device *rdev = crtc->dev->dev_private;
@@ -167,7 +164,8 @@ int radeon_crtc_cursor_set(struct drm_crtc *crtc,
                 goto unpin;
         }
 
-        if ((width > CURSOR_WIDTH) || (height > CURSOR_HEIGHT)) {
+        if ((width > radeon_crtc->max_cursor_width) ||
+            (height > radeon_crtc->max_cursor_height)) {
                 DRM_ERROR("bad cursor width or height %d x %d\n", width, height);
                 return -EINVAL;
         }
@@ -233,11 +231,11 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc,
         DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
 
         if (x < 0) {
-                xorigin = min(-x, CURSOR_WIDTH - 1);
+                xorigin = min(-x, radeon_crtc->max_cursor_width - 1);
                 x = 0;
         }
         if (y < 0) {
-                yorigin = min(-y, CURSOR_HEIGHT - 1);
+                yorigin = min(-y, radeon_crtc->max_cursor_height - 1);
                 y = 0;
         }
 
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index b0dc0b6cb4e0..82335e38ec4f 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -95,6 +95,9 @@ static const char radeon_family_name[][16] = {
         "VERDE",
         "OLAND",
         "HAINAN",
+        "BONAIRE",
+        "KAVERI",
+        "KABINI",
         "LAST",
 };
 
@@ -229,6 +232,94 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
 }
 
 /*
+ * GPU doorbell aperture helpers function.
+ */
+/**
+ * radeon_doorbell_init - Init doorbell driver information.
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Init doorbell driver information (CIK)
+ * Returns 0 on success, error on failure.
+ */
+int radeon_doorbell_init(struct radeon_device *rdev)
+{
+        int i;
+
+        /* doorbell bar mapping */
+        rdev->doorbell.base = pci_resource_start(rdev->pdev, 2);
+        rdev->doorbell.size = pci_resource_len(rdev->pdev, 2);
+
+        /* limit to 4 MB for now */
+        if (rdev->doorbell.size > (4 * 1024 * 1024))
+                rdev->doorbell.size = 4 * 1024 * 1024;
+
+        rdev->doorbell.ptr = ioremap(rdev->doorbell.base, rdev->doorbell.size);
+        if (rdev->doorbell.ptr == NULL) {
+                return -ENOMEM;
+        }
+        DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)rdev->doorbell.base);
+        DRM_INFO("doorbell mmio size: %u\n", (unsigned)rdev->doorbell.size);
+
+        rdev->doorbell.num_pages = rdev->doorbell.size / PAGE_SIZE;
+
+        for (i = 0; i < rdev->doorbell.num_pages; i++) {
+                rdev->doorbell.free[i] = true;
+        }
+        return 0;
+}
+
+/**
+ * radeon_doorbell_fini - Tear down doorbell driver information.
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Tear down doorbell driver information (CIK)
+ */
+void radeon_doorbell_fini(struct radeon_device *rdev)
+{
+        iounmap(rdev->doorbell.ptr);
+        rdev->doorbell.ptr = NULL;
+}
+
+/**
+ * radeon_doorbell_get - Allocate a doorbell page
+ *
+ * @rdev: radeon_device pointer
+ * @doorbell: doorbell page number
+ *
+ * Allocate a doorbell page for use by the driver (all asics).
+ * Returns 0 on success or -EINVAL on failure.
+ */
+int radeon_doorbell_get(struct radeon_device *rdev, u32 *doorbell)
+{
+        int i;
+
+        for (i = 0; i < rdev->doorbell.num_pages; i++) {
+                if (rdev->doorbell.free[i]) {
+                        rdev->doorbell.free[i] = false;
+                        *doorbell = i;
+                        return 0;
+                }
+        }
+        return -EINVAL;
+}
+
+/**
+ * radeon_doorbell_free - Free a doorbell page
+ *
+ * @rdev: radeon_device pointer
+ * @doorbell: doorbell page number
+ *
+ * Free a doorbell page allocated for use by the driver (all asics)
+ */
+void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell)
+{
+        if (doorbell < rdev->doorbell.num_pages)
+                rdev->doorbell.free[doorbell] = true;
+}
+
+/*
  * radeon_wb_*()
  * Writeback is the the method by which the the GPU updates special pages
  * in memory with the status of certain GPU events (fences, ring pointers,
@@ -1145,8 +1236,13 @@ int radeon_device_init(struct radeon_device *rdev,
         /* Registers mapping */
         /* TODO: block userspace mapping of io register */
         spin_lock_init(&rdev->mmio_idx_lock);
-        rdev->rmmio_base = pci_resource_start(rdev->pdev, 2);
-        rdev->rmmio_size = pci_resource_len(rdev->pdev, 2);
+        if (rdev->family >= CHIP_BONAIRE) {
+                rdev->rmmio_base = pci_resource_start(rdev->pdev, 5);
+                rdev->rmmio_size = pci_resource_len(rdev->pdev, 5);
+        } else {
+                rdev->rmmio_base = pci_resource_start(rdev->pdev, 2);
+                rdev->rmmio_size = pci_resource_len(rdev->pdev, 2);
+        }
         rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size);
         if (rdev->rmmio == NULL) {
                 return -ENOMEM;
@@ -1154,6 +1250,10 @@ int radeon_device_init(struct radeon_device *rdev,
         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)rdev->rmmio_base);
         DRM_INFO("register mmio size: %u\n", (unsigned)rdev->rmmio_size);
 
+        /* doorbell bar mapping */
+        if (rdev->family >= CHIP_BONAIRE)
+                radeon_doorbell_init(rdev);
+
         /* io port mapping */
         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
                 if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) {
@@ -1231,6 +1331,8 @@ void radeon_device_fini(struct radeon_device *rdev)
         rdev->rio_mem = NULL;
         iounmap(rdev->rmmio);
         rdev->rmmio = NULL;
+        if (rdev->family >= CHIP_BONAIRE)
+                radeon_doorbell_fini(rdev);
         radeon_debugfs_remove_files(rdev);
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index eb18bb7af1cc..c2b67b4e1ac2 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -153,7 +153,13 @@ static void dce5_crtc_load_lut(struct drm_crtc *crtc)
                 NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
         /* XXX match this to the depth of the crtc fmt block, move to modeset? */
         WREG32(0x6940 + radeon_crtc->crtc_offset, 0);
-
+        if (ASIC_IS_DCE8(rdev)) {
+                /* XXX this only needs to be programmed once per crtc at startup,
+                 * not sure where the best place for it is
+                 */
+                WREG32(CIK_ALPHA_CONTROL + radeon_crtc->crtc_offset,
+                       CIK_CURSOR_ALPHA_BLND_ENA);
+        }
 }
 
 static void legacy_crtc_load_lut(struct drm_crtc *crtc)
@@ -512,6 +518,14 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
         radeon_crtc->crtc_id = index;
         rdev->mode_info.crtcs[index] = radeon_crtc;
 
+        if (rdev->family >= CHIP_BONAIRE) {
+                radeon_crtc->max_cursor_width = CIK_CURSOR_WIDTH;
+                radeon_crtc->max_cursor_height = CIK_CURSOR_HEIGHT;
+        } else {
+                radeon_crtc->max_cursor_width = CURSOR_WIDTH;
+                radeon_crtc->max_cursor_height = CURSOR_HEIGHT;
+        }
+
 #if 0
         radeon_crtc->mode_set.crtc = &radeon_crtc->base;
         radeon_crtc->mode_set.connectors = (struct drm_connector **)(radeon_crtc + 1);
@@ -530,7 +544,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
                 radeon_legacy_init_crtc(dev, radeon_crtc);
 }
 
-static const char *encoder_names[37] = {
+static const char *encoder_names[38] = {
         "NONE",
         "INTERNAL_LVDS",
         "INTERNAL_TMDS1",
@@ -567,7 +581,8 @@ static const char *encoder_names[37] = {
         "INTERNAL_UNIPHY2",
         "NUTMEG",
         "TRAVIS",
-        "INTERNAL_VCE"
+        "INTERNAL_VCE",
+        "INTERNAL_UNIPHY3",
 };
 
 static const char *hpd_names[6] = {
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 094e7e5ea39e..e5419b350170 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -74,9 +74,10 @@
  *   2.31.0 - Add fastfb support for rs690
  *   2.32.0 - new info request for rings working
  *   2.33.0 - Add SI tiling mode array query
+ *   2.34.0 - Add CIK tiling mode array query
  */
 #define KMS_DRIVER_MAJOR        2
-#define KMS_DRIVER_MINOR        33
+#define KMS_DRIVER_MINOR        34
 #define KMS_DRIVER_PATCHLEVEL   0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
@@ -127,6 +128,7 @@ struct drm_gem_object *radeon_gem_prime_import_sg_table(struct drm_device *dev,
                                                         size_t size,
                                                         struct sg_table *sg);
 int radeon_gem_prime_pin(struct drm_gem_object *obj);
+void radeon_gem_prime_unpin(struct drm_gem_object *obj);
 void *radeon_gem_prime_vmap(struct drm_gem_object *obj);
 void radeon_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 extern long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd,
@@ -164,6 +166,7 @@ int radeon_pcie_gen2 = -1;
 int radeon_msi = -1;
 int radeon_lockup_timeout = 10000;
 int radeon_fastfb = 0;
+int radeon_dpm = -1;
 
 MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers");
 module_param_named(no_wb, radeon_no_wb, int, 0444);
@@ -219,6 +222,9 @@ module_param_named(lockup_timeout, radeon_lockup_timeout, int, 0444);
 MODULE_PARM_DESC(fastfb, "Direct FB access for IGP chips (0 = disable, 1 = enable)");
 module_param_named(fastfb, radeon_fastfb, int, 0444);
 
+MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)");
+module_param_named(dpm, radeon_dpm, int, 0444);
+
 static struct pci_device_id pciidlist[] = {
         radeon_PCI_IDS
 };
@@ -422,6 +428,7 @@ static struct drm_driver kms_driver = {
         .gem_prime_export = drm_gem_prime_export,
         .gem_prime_import = drm_gem_prime_import,
         .gem_prime_pin = radeon_gem_prime_pin,
+        .gem_prime_unpin = radeon_gem_prime_unpin,
         .gem_prime_get_sg_table = radeon_gem_prime_get_sg_table,
         .gem_prime_import_sg_table = radeon_gem_prime_import_sg_table,
         .gem_prime_vmap = radeon_gem_prime_vmap,
diff --git a/drivers/gpu/drm/radeon/radeon_family.h b/drivers/gpu/drm/radeon/radeon_family.h
index 36e9803b077d..3c8289083f9d 100644
--- a/drivers/gpu/drm/radeon/radeon_family.h
+++ b/drivers/gpu/drm/radeon/radeon_family.h
@@ -93,6 +93,9 @@ enum radeon_family {
         CHIP_VERDE,
         CHIP_OLAND,
         CHIP_HAINAN,
+        CHIP_BONAIRE,
+        CHIP_KAVERI,
+        CHIP_KABINI,
         CHIP_LAST,
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 5a99d433fc35..bcdefd1dcd43 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -82,6 +82,23 @@ static void radeon_hotplug_work_func(struct work_struct *work)
 }
 
 /**
+ * radeon_irq_reset_work_func - execute gpu reset
+ *
+ * @work: work struct
+ *
+ * Execute scheduled gpu reset (cayman+).
+ * This function is called when the irq handler
+ * thinks we need a gpu reset.
+ */
+static void radeon_irq_reset_work_func(struct work_struct *work)
+{
+        struct radeon_device *rdev = container_of(work, struct radeon_device,
+                                                  reset_work);
+
+        radeon_gpu_reset(rdev);
+}
+
+/**
  * radeon_driver_irq_preinstall_kms - drm irq preinstall callback
  *
  * @dev: drm dev pointer
@@ -99,6 +116,7 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
         /* Disable *all* interrupts */
         for (i = 0; i < RADEON_NUM_RINGS; i++)
                 atomic_set(&rdev->irq.ring_int[i], 0);
+        rdev->irq.dpm_thermal = false;
         for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
                 rdev->irq.hpd[i] = false;
         for (i = 0; i < RADEON_MAX_CRTCS; i++) {
@@ -146,6 +164,7 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
         /* Disable *all* interrupts */
         for (i = 0; i < RADEON_NUM_RINGS; i++)
                 atomic_set(&rdev->irq.ring_int[i], 0);
+        rdev->irq.dpm_thermal = false;
         for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
                 rdev->irq.hpd[i] = false;
         for (i = 0; i < RADEON_MAX_CRTCS; i++) {
@@ -243,6 +262,7 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
 
         INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
         INIT_WORK(&rdev->audio_work, r600_audio_update_hdmi);
+        INIT_WORK(&rdev->reset_work, radeon_irq_reset_work_func);
 
         spin_lock_init(&rdev->irq.lock);
         r = drm_vblank_init(rdev->ddev, rdev->num_crtc);
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 4f2d4f4c1dab..49ff3d1a6102 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -229,7 +229,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 *value = rdev->accel_working;
                 break;
         case RADEON_INFO_TILING_CONFIG:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        *value = rdev->config.cik.tile_config;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.tile_config;
                 else if (rdev->family >= CHIP_CAYMAN)
                         *value = rdev->config.cayman.tile_config;
@@ -281,7 +283,10 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                         *value = rdev->clock.spll.reference_freq * 10;
                 break;
         case RADEON_INFO_NUM_BACKENDS:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        *value = rdev->config.cik.max_backends_per_se *
+                                rdev->config.cik.max_shader_engines;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.max_backends_per_se *
                                 rdev->config.si.max_shader_engines;
                 else if (rdev->family >= CHIP_CAYMAN)
@@ -298,7 +303,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 }
                 break;
         case RADEON_INFO_NUM_TILE_PIPES:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        *value = rdev->config.cik.max_tile_pipes;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.max_tile_pipes;
                 else if (rdev->family >= CHIP_CAYMAN)
                         *value = rdev->config.cayman.max_tile_pipes;
@@ -316,7 +323,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 *value = 1;
                 break;
         case RADEON_INFO_BACKEND_MAP:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        return -EINVAL;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.backend_map;
                 else if (rdev->family >= CHIP_CAYMAN)
                         *value = rdev->config.cayman.backend_map;
@@ -343,7 +352,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 *value = RADEON_IB_VM_MAX_SIZE;
                 break;
         case RADEON_INFO_MAX_PIPES:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        *value = rdev->config.cik.max_cu_per_sh;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.max_cu_per_sh;
                 else if (rdev->family >= CHIP_CAYMAN)
                         *value = rdev->config.cayman.max_pipes_per_simd;
@@ -367,7 +378,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 value64 = radeon_get_gpu_clock_counter(rdev);
                 break;
         case RADEON_INFO_MAX_SE:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        *value = rdev->config.cik.max_shader_engines;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.max_shader_engines;
                 else if (rdev->family >= CHIP_CAYMAN)
                         *value = rdev->config.cayman.max_shader_engines;
@@ -377,7 +390,9 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                         *value = 1;
                 break;
         case RADEON_INFO_MAX_SH_PER_SE:
-                if (rdev->family >= CHIP_TAHITI)
+                if (rdev->family >= CHIP_BONAIRE)
+                        *value = rdev->config.cik.max_sh_per_se;
+                else if (rdev->family >= CHIP_TAHITI)
                         *value = rdev->config.si.max_sh_per_se;
                 else
                         return -EINVAL;
@@ -407,12 +422,16 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 }
                 break;
         case RADEON_INFO_SI_TILE_MODE_ARRAY:
-                if (rdev->family < CHIP_TAHITI) {
-                        DRM_DEBUG_KMS("tile mode array is si only!\n");
+                if (rdev->family >= CHIP_BONAIRE) {
+                        value = rdev->config.cik.tile_mode_array;
+                        value_size = sizeof(uint32_t)*32;
+                } else if (rdev->family >= CHIP_TAHITI) {
+                        value = rdev->config.si.tile_mode_array;
+                        value_size = sizeof(uint32_t)*32;
+                } else {
+                        DRM_DEBUG_KMS("tile mode array is si+ only!\n");
                         return -EINVAL;
                 }
-                value = rdev->config.si.tile_mode_array;
-                value_size = sizeof(uint32_t)*32;
                 break;
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->request);
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 69ad4fe224c1..b568cb19a7fa 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -307,6 +307,8 @@ struct radeon_crtc {
         uint64_t cursor_addr;
         int cursor_width;
         int cursor_height;
+        int max_cursor_width;
+        int max_cursor_height;
         uint32_t legacy_display_base_addr;
         uint32_t legacy_cursor_offset;
         enum radeon_rmx_type rmx_type;
@@ -329,6 +331,10 @@ struct radeon_crtc {
         u32 pll_flags;
         struct drm_encoder *encoder;
         struct drm_connector *connector;
+        /* for dpm */
+        u32 line_time;
+        u32 wm_low;
+        u32 wm_high;
 };
 
 struct radeon_encoder_primary_dac {
@@ -512,12 +518,99 @@ struct atom_clock_dividers {
         bool enable_dithen;
         u32 vco_mode;
         u32 real_clock;
+        /* added for CI */
+        u32 post_divider;
+        u32 flags;
+};
+
+struct atom_mpll_param {
+        union {
+                struct {
+#ifdef __BIG_ENDIAN
+                        u32 reserved : 8;
+                        u32 clkfrac : 12;
+                        u32 clkf : 12;
+#else
+                        u32 clkf : 12;
+                        u32 clkfrac : 12;
+                        u32 reserved : 8;
+#endif
+                };
+                u32 fb_div;
+        };
+        u32 post_div;
+        u32 bwcntl;
+        u32 dll_speed;
+        u32 vco_mode;
+        u32 yclk_sel;
+        u32 qdr;
+        u32 half_rate;
+};
+
+#define MEM_TYPE_GDDR5  0x50
+#define MEM_TYPE_GDDR4  0x40
+#define MEM_TYPE_GDDR3  0x30
+#define MEM_TYPE_DDR2   0x20
+#define MEM_TYPE_GDDR1  0x10
+#define MEM_TYPE_DDR3   0xb0
+#define MEM_TYPE_MASK   0xf0
+
+struct atom_memory_info {
+        u8 mem_vendor;
+        u8 mem_type;
+};
+
+#define MAX_AC_TIMING_ENTRIES 16
+
+struct atom_memory_clock_range_table
+{
+        u8 num_entries;
+        u8 rsv[3];
+        u32 mclk[MAX_AC_TIMING_ENTRIES];
+};
+
+#define VBIOS_MC_REGISTER_ARRAY_SIZE 32
+#define VBIOS_MAX_AC_TIMING_ENTRIES 20
+
+struct atom_mc_reg_entry {
+        u32 mclk_max;
+        u32 mc_data[VBIOS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct atom_mc_register_address {
+        u16 s1;
+        u8 pre_reg_data;
+};
+
+struct atom_mc_reg_table {
+        u8 last;
+        u8 num_entries;
+        struct atom_mc_reg_entry mc_reg_table_entry[VBIOS_MAX_AC_TIMING_ENTRIES];
+        struct atom_mc_register_address mc_reg_address[VBIOS_MC_REGISTER_ARRAY_SIZE];
+};
+
+#define MAX_VOLTAGE_ENTRIES 32
+
+struct atom_voltage_table_entry
+{
+        u16 value;
+        u32 smio_low;
+};
+
+struct atom_voltage_table
+{
+        u32 count;
+        u32 mask_low;
+        u32 phase_delay;
+        struct atom_voltage_table_entry entries[MAX_VOLTAGE_ENTRIES];
 };
 
 extern enum radeon_tv_std
 radeon_combios_get_tv_info(struct radeon_device *rdev);
 extern enum radeon_tv_std
 radeon_atombios_get_tv_info(struct radeon_device *rdev);
+extern void radeon_atombios_get_default_voltages(struct radeon_device *rdev,
+                                                 u16 *vddc, u16 *vddci, u16 *mvdd);
 
 extern struct drm_connector *
 radeon_get_connector_for_encoder(struct drm_encoder *encoder);
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 1424ccde2377..0219d263e2df 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -322,8 +322,8 @@ int radeon_bo_init(struct radeon_device *rdev)
 {
         /* Add an MTRR for the VRAM */
         if (!rdev->fastfb_working) {
-                rdev->mc.vram_mtrr = mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size,
-                        MTRR_TYPE_WRCOMB, 1);
+                rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base,
+                                                      rdev->mc.aper_size);
         }
         DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
                 rdev->mc.mc_vram_size >> 20,
@@ -336,6 +336,7 @@ int radeon_bo_init(struct radeon_device *rdev)
 void radeon_bo_fini(struct radeon_device *rdev)
 {
         radeon_ttm_fini(rdev);
+        arch_phys_wc_del(rdev->mc.vram_mtrr);
 }
 
 void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
@@ -348,14 +349,15 @@ void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
         }
 }
 
-int radeon_bo_list_validate(struct list_head *head, int ring)
+int radeon_bo_list_validate(struct ww_acquire_ctx *ticket,
+                            struct list_head *head, int ring)
 {
         struct radeon_bo_list *lobj;
         struct radeon_bo *bo;
         u32 domain;
         int r;
 
-        r = ttm_eu_reserve_buffers(head);
+        r = ttm_eu_reserve_buffers(ticket, head);
         if (unlikely(r != 0)) {
                 return r;
         }
@@ -398,7 +400,7 @@ int radeon_bo_get_surface_reg(struct radeon_bo *bo)
         int steal;
         int i;
 
-        BUG_ON(!radeon_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->tbo.resv->lock.base);
 
         if (!bo->tiling_flags)
                 return 0;
@@ -524,7 +526,8 @@ void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
                                 uint32_t *tiling_flags,
                                 uint32_t *pitch)
 {
-        BUG_ON(!radeon_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->tbo.resv->lock.base);
+
         if (tiling_flags)
                 *tiling_flags = bo->tiling_flags;
         if (pitch)
@@ -534,7 +537,8 @@ void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
 int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
                                 bool force_drop)
 {
-        BUG_ON(!radeon_bo_is_reserved(bo) && !force_drop);
+        if (!force_drop)
+                lockdep_assert_held(&bo->tbo.resv->lock.base);
 
         if (!(bo->tiling_flags & RADEON_TILING_SURFACE))
                 return 0;
@@ -617,26 +621,3 @@ int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)
         ttm_bo_unreserve(&bo->tbo);
         return r;
 }
-
-
-/**
- * radeon_bo_reserve - reserve bo
- * @bo:         bo structure
- * @no_intr:    don't return -ERESTARTSYS on pending signal
- *
- * Returns:
- * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
- * a signal. Release all buffer reservations and return to user-space.
- */
-int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr)
-{
-        int r;
-
-        r = ttm_bo_reserve(&bo->tbo, !no_intr, false, false, 0);
-        if (unlikely(r != 0)) {
-                if (r != -ERESTARTSYS)
-                        dev_err(bo->rdev->dev, "%p reserve failed\n", bo);
-                return r;
-        }
-        return 0;
-}
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index e2cb80a96b51..91519a5622b4 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -52,7 +52,27 @@ static inline unsigned radeon_mem_type_to_domain(u32 mem_type)
         return 0;
 }
 
-int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr);
+/**
+ * radeon_bo_reserve - reserve bo
+ * @bo:         bo structure
+ * @no_intr:    don't return -ERESTARTSYS on pending signal
+ *
+ * Returns:
+ * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
+ * a signal. Release all buffer reservations and return to user-space.
+ */
+static inline int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr)
+{
+        int r;
+
+        r = ttm_bo_reserve(&bo->tbo, !no_intr, false, false, 0);
+        if (unlikely(r != 0)) {
+                if (r != -ERESTARTSYS)
+                        dev_err(bo->rdev->dev, "%p reserve failed\n", bo);
+                return r;
+        }
+        return 0;
+}
 
 static inline void radeon_bo_unreserve(struct radeon_bo *bo)
 {
@@ -78,11 +98,6 @@ static inline unsigned long radeon_bo_size(struct radeon_bo *bo)
         return bo->tbo.num_pages << PAGE_SHIFT;
 }
 
-static inline bool radeon_bo_is_reserved(struct radeon_bo *bo)
-{
-        return ttm_bo_is_reserved(&bo->tbo);
-}
-
 static inline unsigned radeon_bo_ngpu_pages(struct radeon_bo *bo)
 {
         return (bo->tbo.num_pages << PAGE_SHIFT) / RADEON_GPU_PAGE_SIZE;
@@ -128,7 +143,8 @@ extern int radeon_bo_init(struct radeon_device *rdev);
 extern void radeon_bo_fini(struct radeon_device *rdev);
 extern void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
                                 struct list_head *head);
-extern int radeon_bo_list_validate(struct list_head *head, int ring);
+extern int radeon_bo_list_validate(struct ww_acquire_ctx *ticket,
+                                   struct list_head *head, int ring);
 extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
                                 struct vm_area_struct *vma);
 extern int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 788c64cb4b47..ebbdb477745a 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -388,7 +388,8 @@ static ssize_t radeon_get_pm_method(struct device *dev,
         int pm = rdev->pm.pm_method;
 
         return snprintf(buf, PAGE_SIZE, "%s\n",
-                        (pm == PM_METHOD_DYNPM) ? "dynpm" : "profile");
+                        (pm == PM_METHOD_DYNPM) ? "dynpm" :
+                        (pm == PM_METHOD_PROFILE) ? "profile" : "dpm");
 }
 
 static ssize_t radeon_set_pm_method(struct device *dev,
@@ -399,6 +400,11 @@ static ssize_t radeon_set_pm_method(struct device *dev,
         struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev));
         struct radeon_device *rdev = ddev->dev_private;
 
+        /* we don't support the legacy modes with dpm */
+        if (rdev->pm.pm_method == PM_METHOD_DPM) {
+                count = -EINVAL;
+                goto fail;
+        }
 
         if (strncmp("dynpm", buf, strlen("dynpm")) == 0) {
                 mutex_lock(&rdev->pm.mutex);
@@ -423,8 +429,48 @@ fail:
         return count;
 }
 
+static ssize_t radeon_get_dpm_state(struct device *dev,
+                                    struct device_attribute *attr,
+                                    char *buf)
+{
+        struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev));
+        struct radeon_device *rdev = ddev->dev_private;
+        enum radeon_pm_state_type pm = rdev->pm.dpm.user_state;
+
+        return snprintf(buf, PAGE_SIZE, "%s\n",
+                        (pm == POWER_STATE_TYPE_BATTERY) ? "battery" :
+                        (pm == POWER_STATE_TYPE_BALANCED) ? "balanced" : "performance");
+}
+
+static ssize_t radeon_set_dpm_state(struct device *dev,
+                                    struct device_attribute *attr,
+                                    const char *buf,
+                                    size_t count)
+{
+        struct drm_device *ddev = pci_get_drvdata(to_pci_dev(dev));
+        struct radeon_device *rdev = ddev->dev_private;
+
+        mutex_lock(&rdev->pm.mutex);
+        if (strncmp("battery", buf, strlen("battery")) == 0)
+                rdev->pm.dpm.user_state = POWER_STATE_TYPE_BATTERY;
+        else if (strncmp("balanced", buf, strlen("balanced")) == 0)
+                rdev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
+        else if (strncmp("performance", buf, strlen("performance")) == 0)
+                rdev->pm.dpm.user_state = POWER_STATE_TYPE_PERFORMANCE;
+        else {
+                mutex_unlock(&rdev->pm.mutex);
+                count = -EINVAL;
+                goto fail;
+        }
+        mutex_unlock(&rdev->pm.mutex);
+        radeon_pm_compute_clocks(rdev);
+fail:
+        return count;
+}
+
 static DEVICE_ATTR(power_profile, S_IRUGO | S_IWUSR, radeon_get_pm_profile, radeon_set_pm_profile);
 static DEVICE_ATTR(power_method, S_IRUGO | S_IWUSR, radeon_get_pm_method, radeon_set_pm_method);
+static DEVICE_ATTR(power_dpm_state, S_IRUGO | S_IWUSR, radeon_get_dpm_state, radeon_set_dpm_state);
 
 static ssize_t radeon_hwmon_show_temp(struct device *dev,
                                       struct device_attribute *attr,
@@ -434,27 +480,10 @@ static ssize_t radeon_hwmon_show_temp(struct device *dev,
         struct radeon_device *rdev = ddev->dev_private;
         int temp;
 
-        switch (rdev->pm.int_thermal_type) {
-        case THERMAL_TYPE_RV6XX:
-                temp = rv6xx_get_temp(rdev);
-                break;
-        case THERMAL_TYPE_RV770:
-                temp = rv770_get_temp(rdev);
-                break;
-        case THERMAL_TYPE_EVERGREEN:
-        case THERMAL_TYPE_NI:
-                temp = evergreen_get_temp(rdev);
-                break;
-        case THERMAL_TYPE_SUMO:
-                temp = sumo_get_temp(rdev);
-                break;
-        case THERMAL_TYPE_SI:
-                temp = si_get_temp(rdev);
-                break;
-        default:
+        if (rdev->asic->pm.get_temperature)
+                temp = radeon_get_temperature(rdev);
+        else
                 temp = 0;
-                break;
-        }
 
         return snprintf(buf, PAGE_SIZE, "%d\n", temp);
 }
@@ -492,8 +521,7 @@ static int radeon_hwmon_init(struct radeon_device *rdev)
         case THERMAL_TYPE_NI:
         case THERMAL_TYPE_SUMO:
         case THERMAL_TYPE_SI:
-                /* No support for TN yet */
-                if (rdev->family == CHIP_ARUBA)
+                if (rdev->asic->pm.get_temperature == NULL)
                         return err;
                 rdev->pm.int_hwmon_dev = hwmon_device_register(rdev->dev);
                 if (IS_ERR(rdev->pm.int_hwmon_dev)) {
@@ -526,7 +554,270 @@ static void radeon_hwmon_fini(struct radeon_device *rdev)
         }
 }
 
-void radeon_pm_suspend(struct radeon_device *rdev)
+static void radeon_dpm_thermal_work_handler(struct work_struct *work)
+{
+        struct radeon_device *rdev =
+                container_of(work, struct radeon_device,
+                             pm.dpm.thermal.work);
+        /* switch to the thermal state */
+        enum radeon_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL;
+
+        if (!rdev->pm.dpm_enabled)
+                return;
+
+        if (rdev->asic->pm.get_temperature) {
+                int temp = radeon_get_temperature(rdev);
+
+                if (temp < rdev->pm.dpm.thermal.min_temp)
+                        /* switch back the user state */
+                        dpm_state = rdev->pm.dpm.user_state;
+        } else {
+                if (rdev->pm.dpm.thermal.high_to_low)
+                        /* switch back the user state */
+                        dpm_state = rdev->pm.dpm.user_state;
+        }
+        radeon_dpm_enable_power_state(rdev, dpm_state);
+}
+
+static struct radeon_ps *radeon_dpm_pick_power_state(struct radeon_device *rdev,
+                                                     enum radeon_pm_state_type dpm_state)
+{
+        int i;
+        struct radeon_ps *ps;
+        u32 ui_class;
+
+restart_search:
+        /* balanced states don't exist at the moment */
+        if (dpm_state == POWER_STATE_TYPE_BALANCED)
+                dpm_state = POWER_STATE_TYPE_PERFORMANCE;
+
+        /* Pick the best power state based on current conditions */
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                ps = &rdev->pm.dpm.ps[i];
+                ui_class = ps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK;
+                switch (dpm_state) {
+                /* user states */
+                case POWER_STATE_TYPE_BATTERY:
+                        if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) {
+                                if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) {
+                                        if (rdev->pm.dpm.new_active_crtc_count < 2)
+                                                return ps;
+                                } else
+                                        return ps;
+                        }
+                        break;
+                case POWER_STATE_TYPE_BALANCED:
+                        if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BALANCED) {
+                                if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) {
+                                        if (rdev->pm.dpm.new_active_crtc_count < 2)
+                                                return ps;
+                                } else
+                                        return ps;
+                        }
+                        break;
+                case POWER_STATE_TYPE_PERFORMANCE:
+                        if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
+                                if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) {
+                                        if (rdev->pm.dpm.new_active_crtc_count < 2)
+                                                return ps;
+                                } else
+                                        return ps;
+                        }
+                        break;
+                /* internal states */
+                case POWER_STATE_TYPE_INTERNAL_UVD:
+                        return rdev->pm.dpm.uvd_ps;
+                case POWER_STATE_TYPE_INTERNAL_UVD_SD:
+                        if (ps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE)
+                                return ps;
+                        break;
+                case POWER_STATE_TYPE_INTERNAL_UVD_HD:
+                        if (ps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE)
+                                return ps;
+                        break;
+                case POWER_STATE_TYPE_INTERNAL_UVD_HD2:
+                        if (ps->class & ATOM_PPLIB_CLASSIFICATION_HD2STATE)
+                                return ps;
+                        break;
+                case POWER_STATE_TYPE_INTERNAL_UVD_MVC:
+                        if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC)
+                                return ps;
+                        break;
+                case POWER_STATE_TYPE_INTERNAL_BOOT:
+                        return rdev->pm.dpm.boot_ps;
+                case POWER_STATE_TYPE_INTERNAL_THERMAL:
+                        if (ps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
+                                return ps;
+                        break;
+                case POWER_STATE_TYPE_INTERNAL_ACPI:
+                        if (ps->class & ATOM_PPLIB_CLASSIFICATION_ACPI)
+                                return ps;
+                        break;
+                case POWER_STATE_TYPE_INTERNAL_ULV:
+                        if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV)
+                                return ps;
+                        break;
+                default:
+                        break;
+                }
+        }
+        /* use a fallback state if we didn't match */
+        switch (dpm_state) {
+        case POWER_STATE_TYPE_INTERNAL_UVD_SD:
+        case POWER_STATE_TYPE_INTERNAL_UVD_HD:
+        case POWER_STATE_TYPE_INTERNAL_UVD_HD2:
+        case POWER_STATE_TYPE_INTERNAL_UVD_MVC:
+                return rdev->pm.dpm.uvd_ps;
+        case POWER_STATE_TYPE_INTERNAL_THERMAL:
+                dpm_state = POWER_STATE_TYPE_INTERNAL_ACPI;
+                goto restart_search;
+        case POWER_STATE_TYPE_INTERNAL_ACPI:
+                dpm_state = POWER_STATE_TYPE_BATTERY;
+                goto restart_search;
+        case POWER_STATE_TYPE_BATTERY:
+                dpm_state = POWER_STATE_TYPE_PERFORMANCE;
+                goto restart_search;
+        default:
+                break;
+        }
+
+        return NULL;
+}
+
+static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev)
+{
+        int i;
+        struct radeon_ps *ps;
+        enum radeon_pm_state_type dpm_state;
+        int ret;
+
+        /* if dpm init failed */
+        if (!rdev->pm.dpm_enabled)
+                return;
+
+        if (rdev->pm.dpm.user_state != rdev->pm.dpm.state) {
+                /* add other state override checks here */
+                if ((!rdev->pm.dpm.thermal_active) &&
+                    (!rdev->pm.dpm.uvd_active))
+                        rdev->pm.dpm.state = rdev->pm.dpm.user_state;
+        }
+        dpm_state = rdev->pm.dpm.state;
+
+        ps = radeon_dpm_pick_power_state(rdev, dpm_state);
+        if (ps)
+                rdev->pm.dpm.requested_ps = ps;
+        else
+                return;
+
+        /* no need to reprogram if nothing changed unless we are on BTC+ */
+        if (rdev->pm.dpm.current_ps == rdev->pm.dpm.requested_ps) {
+                if ((rdev->family < CHIP_BARTS) || (rdev->flags & RADEON_IS_IGP)) {
+                        /* for pre-BTC and APUs if the num crtcs changed but state is the same,
+                         * all we need to do is update the display configuration.
+                         */
+                        if (rdev->pm.dpm.new_active_crtcs != rdev->pm.dpm.current_active_crtcs) {
+                                /* update display watermarks based on new power state */
+                                radeon_bandwidth_update(rdev);
+                                /* update displays */
+                                radeon_dpm_display_configuration_changed(rdev);
+                                rdev->pm.dpm.current_active_crtcs = rdev->pm.dpm.new_active_crtcs;
+                                rdev->pm.dpm.current_active_crtc_count = rdev->pm.dpm.new_active_crtc_count;
+                        }
+                        return;
+                } else {
+                        /* for BTC+ if the num crtcs hasn't changed and state is the same,
+                         * nothing to do, if the num crtcs is > 1 and state is the same,
+                         * update display configuration.
+                         */
+                        if (rdev->pm.dpm.new_active_crtcs ==
+                            rdev->pm.dpm.current_active_crtcs) {
+                                return;
+                        } else {
+                                if ((rdev->pm.dpm.current_active_crtc_count > 1) &&
+                                    (rdev->pm.dpm.new_active_crtc_count > 1)) {
+                                        /* update display watermarks based on new power state */
+                                        radeon_bandwidth_update(rdev);
+                                        /* update displays */
+                                        radeon_dpm_display_configuration_changed(rdev);
+                                        rdev->pm.dpm.current_active_crtcs = rdev->pm.dpm.new_active_crtcs;
+                                        rdev->pm.dpm.current_active_crtc_count = rdev->pm.dpm.new_active_crtc_count;
+                                        return;
+                                }
+                        }
+                }
+        }
+
+        printk("switching from power state:\n");
+        radeon_dpm_print_power_state(rdev, rdev->pm.dpm.current_ps);
+        printk("switching to power state:\n");
+        radeon_dpm_print_power_state(rdev, rdev->pm.dpm.requested_ps);
+
+        mutex_lock(&rdev->ddev->struct_mutex);
+        down_write(&rdev->pm.mclk_lock);
+        mutex_lock(&rdev->ring_lock);
+
+        ret = radeon_dpm_pre_set_power_state(rdev);
+        if (ret)
+                goto done;
+
+        /* update display watermarks based on new power state */
+        radeon_bandwidth_update(rdev);
+        /* update displays */
+        radeon_dpm_display_configuration_changed(rdev);
+
+        rdev->pm.dpm.current_active_crtcs = rdev->pm.dpm.new_active_crtcs;
+        rdev->pm.dpm.current_active_crtc_count = rdev->pm.dpm.new_active_crtc_count;
+
+        /* wait for the rings to drain */
+        for (i = 0; i < RADEON_NUM_RINGS; i++) {
+                struct radeon_ring *ring = &rdev->ring[i];
+                if (ring->ready)
+                        radeon_fence_wait_empty_locked(rdev, i);
+        }
+
+        /* program the new power state */
+        radeon_dpm_set_power_state(rdev);
+
+        /* update current power state */
+        rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps;
+
+        radeon_dpm_post_set_power_state(rdev);
+
+done:
+        mutex_unlock(&rdev->ring_lock);
+        up_write(&rdev->pm.mclk_lock);
+        mutex_unlock(&rdev->ddev->struct_mutex);
+}
+
+void radeon_dpm_enable_power_state(struct radeon_device *rdev,
+                                   enum radeon_pm_state_type dpm_state)
+{
+        if (!rdev->pm.dpm_enabled)
+                return;
+
+        mutex_lock(&rdev->pm.mutex);
+        switch (dpm_state) {
+        case POWER_STATE_TYPE_INTERNAL_THERMAL:
+                rdev->pm.dpm.thermal_active = true;
+                break;
+        case POWER_STATE_TYPE_INTERNAL_UVD:
+        case POWER_STATE_TYPE_INTERNAL_UVD_SD:
+        case POWER_STATE_TYPE_INTERNAL_UVD_HD:
+        case POWER_STATE_TYPE_INTERNAL_UVD_HD2:
+        case POWER_STATE_TYPE_INTERNAL_UVD_MVC:
+                rdev->pm.dpm.uvd_active = true;
+                break;
+        default:
+                rdev->pm.dpm.thermal_active = false;
+                rdev->pm.dpm.uvd_active = false;
+                break;
+        }
+        rdev->pm.dpm.state = dpm_state;
+        mutex_unlock(&rdev->pm.mutex);
+        radeon_pm_compute_clocks(rdev);
+}
+
+static void radeon_pm_suspend_old(struct radeon_device *rdev)
 {
         mutex_lock(&rdev->pm.mutex);
         if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
@@ -538,7 +829,26 @@ void radeon_pm_suspend(struct radeon_device *rdev)
         cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
 }
 
-void radeon_pm_resume(struct radeon_device *rdev)
+static void radeon_pm_suspend_dpm(struct radeon_device *rdev)
+{
+        mutex_lock(&rdev->pm.mutex);
+        /* disable dpm */
+        radeon_dpm_disable(rdev);
+        /* reset the power state */
+        rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps;
+        rdev->pm.dpm_enabled = false;
+        mutex_unlock(&rdev->pm.mutex);
+}
+
+void radeon_pm_suspend(struct radeon_device *rdev)
+{
+        if (rdev->pm.pm_method == PM_METHOD_DPM)
+                radeon_pm_suspend_dpm(rdev);
+        else
+                radeon_pm_suspend_old(rdev);
+}
+
+static void radeon_pm_resume_old(struct radeon_device *rdev)
 {
         /* set up the default clocks if the MC ucode is loaded */
         if ((rdev->family >= CHIP_BARTS) &&
@@ -573,12 +883,50 @@ void radeon_pm_resume(struct radeon_device *rdev)
         radeon_pm_compute_clocks(rdev);
 }
 
-int radeon_pm_init(struct radeon_device *rdev)
+static void radeon_pm_resume_dpm(struct radeon_device *rdev)
+{
+        int ret;
+
+        /* asic init will reset to the boot state */
+        mutex_lock(&rdev->pm.mutex);
+        rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps;
+        radeon_dpm_setup_asic(rdev);
+        ret = radeon_dpm_enable(rdev);
+        mutex_unlock(&rdev->pm.mutex);
+        if (ret) {
+                DRM_ERROR("radeon: dpm resume failed\n");
+                if ((rdev->family >= CHIP_BARTS) &&
+                    (rdev->family <= CHIP_CAYMAN) &&
+                    rdev->mc_fw) {
+                        if (rdev->pm.default_vddc)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
+                                                        SET_VOLTAGE_TYPE_ASIC_VDDC);
+                        if (rdev->pm.default_vddci)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddci,
+                                                        SET_VOLTAGE_TYPE_ASIC_VDDCI);
+                        if (rdev->pm.default_sclk)
+                                radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
+                        if (rdev->pm.default_mclk)
+                                radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
+                }
+        } else {
+                rdev->pm.dpm_enabled = true;
+                radeon_pm_compute_clocks(rdev);
+        }
+}
+
+void radeon_pm_resume(struct radeon_device *rdev)
+{
+        if (rdev->pm.pm_method == PM_METHOD_DPM)
+                radeon_pm_resume_dpm(rdev);
+        else
+                radeon_pm_resume_old(rdev);
+}
+
+static int radeon_pm_init_old(struct radeon_device *rdev)
 {
         int ret;
 
-        /* default to profile method */
-        rdev->pm.pm_method = PM_METHOD_PROFILE;
         rdev->pm.profile = PM_PROFILE_DEFAULT;
         rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
         rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
@@ -640,7 +988,142 @@ int radeon_pm_init(struct radeon_device *rdev)
         return 0;
 }
 
-void radeon_pm_fini(struct radeon_device *rdev)
+static void radeon_dpm_print_power_states(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                printk("== power state %d ==\n", i);
+                radeon_dpm_print_power_state(rdev, &rdev->pm.dpm.ps[i]);
+        }
+}
+
+static int radeon_pm_init_dpm(struct radeon_device *rdev)
+{
+        int ret;
+
+        /* default to performance state */
+        rdev->pm.dpm.state = POWER_STATE_TYPE_PERFORMANCE;
+        rdev->pm.dpm.user_state = POWER_STATE_TYPE_PERFORMANCE;
+        rdev->pm.default_sclk = rdev->clock.default_sclk;
+        rdev->pm.default_mclk = rdev->clock.default_mclk;
+        rdev->pm.current_sclk = rdev->clock.default_sclk;
+        rdev->pm.current_mclk = rdev->clock.default_mclk;
+        rdev->pm.int_thermal_type = THERMAL_TYPE_NONE;
+
+        if (rdev->bios && rdev->is_atom_bios)
+                radeon_atombios_get_power_modes(rdev);
+        else
+                return -EINVAL;
+
+        /* set up the internal thermal sensor if applicable */
+        ret = radeon_hwmon_init(rdev);
+        if (ret)
+                return ret;
+
+        INIT_WORK(&rdev->pm.dpm.thermal.work, radeon_dpm_thermal_work_handler);
+        mutex_lock(&rdev->pm.mutex);
+        radeon_dpm_init(rdev);
+        rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps;
+        radeon_dpm_print_power_states(rdev);
+        radeon_dpm_setup_asic(rdev);
+        ret = radeon_dpm_enable(rdev);
+        mutex_unlock(&rdev->pm.mutex);
+        if (ret) {
+                rdev->pm.dpm_enabled = false;
+                if ((rdev->family >= CHIP_BARTS) &&
+                    (rdev->family <= CHIP_CAYMAN) &&
+                    rdev->mc_fw) {
+                        if (rdev->pm.default_vddc)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
+                                                        SET_VOLTAGE_TYPE_ASIC_VDDC);
+                        if (rdev->pm.default_vddci)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddci,
+                                                        SET_VOLTAGE_TYPE_ASIC_VDDCI);
+                        if (rdev->pm.default_sclk)
+                                radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
+                        if (rdev->pm.default_mclk)
+                                radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
+                }
+                DRM_ERROR("radeon: dpm initialization failed\n");
+                return ret;
+        }
+        rdev->pm.dpm_enabled = true;
+        radeon_pm_compute_clocks(rdev);
+
+        if (rdev->pm.num_power_states > 1) {
+                ret = device_create_file(rdev->dev, &dev_attr_power_dpm_state);
+                if (ret)
+                        DRM_ERROR("failed to create device file for dpm state\n");
+                /* XXX: these are noops for dpm but are here for backwards compat */
+                ret = device_create_file(rdev->dev, &dev_attr_power_profile);
+                if (ret)
+                        DRM_ERROR("failed to create device file for power profile\n");
+                ret = device_create_file(rdev->dev, &dev_attr_power_method);
+                if (ret)
+                        DRM_ERROR("failed to create device file for power method\n");
+
+                if (radeon_debugfs_pm_init(rdev)) {
+                        DRM_ERROR("Failed to register debugfs file for dpm!\n");
+                }
+
+                DRM_INFO("radeon: dpm initialized\n");
+        }
+
+        return 0;
+}
+
+int radeon_pm_init(struct radeon_device *rdev)
+{
+        /* enable dpm on rv6xx+ */
+        switch (rdev->family) {
+        case CHIP_RV610:
+        case CHIP_RV630:
+        case CHIP_RV620:
+        case CHIP_RV635:
+        case CHIP_RV670:
+        case CHIP_RS780:
+        case CHIP_RS880:
+        case CHIP_RV770:
+        case CHIP_RV730:
+        case CHIP_RV710:
+        case CHIP_RV740:
+        case CHIP_CEDAR:
+        case CHIP_REDWOOD:
+        case CHIP_JUNIPER:
+        case CHIP_CYPRESS:
+        case CHIP_HEMLOCK:
+        case CHIP_PALM:
+        case CHIP_SUMO:
+        case CHIP_SUMO2:
+        case CHIP_BARTS:
+        case CHIP_TURKS:
+        case CHIP_CAICOS:
+        case CHIP_CAYMAN:
+        case CHIP_ARUBA:
+        case CHIP_TAHITI:
+        case CHIP_PITCAIRN:
+        case CHIP_VERDE:
+        case CHIP_OLAND:
+        case CHIP_HAINAN:
+                if (radeon_dpm == 1)
+                        rdev->pm.pm_method = PM_METHOD_DPM;
+                else
+                        rdev->pm.pm_method = PM_METHOD_PROFILE;
+                break;
+        default:
+                /* default to profile method */
+                rdev->pm.pm_method = PM_METHOD_PROFILE;
+                break;
+        }
+
+        if (rdev->pm.pm_method == PM_METHOD_DPM)
+                return radeon_pm_init_dpm(rdev);
+        else
+                return radeon_pm_init_old(rdev);
+}
+
+static void radeon_pm_fini_old(struct radeon_device *rdev)
 {
         if (rdev->pm.num_power_states > 1) {
                 mutex_lock(&rdev->pm.mutex);
@@ -668,7 +1151,35 @@ void radeon_pm_fini(struct radeon_device *rdev)
         radeon_hwmon_fini(rdev);
 }
 
-void radeon_pm_compute_clocks(struct radeon_device *rdev)
+static void radeon_pm_fini_dpm(struct radeon_device *rdev)
+{
+        if (rdev->pm.num_power_states > 1) {
+                mutex_lock(&rdev->pm.mutex);
+                radeon_dpm_disable(rdev);
+                mutex_unlock(&rdev->pm.mutex);
+
+                device_remove_file(rdev->dev, &dev_attr_power_dpm_state);
+                /* XXX backwards compat */
+                device_remove_file(rdev->dev, &dev_attr_power_profile);
+                device_remove_file(rdev->dev, &dev_attr_power_method);
+        }
+        radeon_dpm_fini(rdev);
+
+        if (rdev->pm.power_state)
+                kfree(rdev->pm.power_state);
+
+        radeon_hwmon_fini(rdev);
+}
+
+void radeon_pm_fini(struct radeon_device *rdev)
+{
+        if (rdev->pm.pm_method == PM_METHOD_DPM)
+                radeon_pm_fini_dpm(rdev);
+        else
+                radeon_pm_fini_old(rdev);
+}
+
+static void radeon_pm_compute_clocks_old(struct radeon_device *rdev)
 {
         struct drm_device *ddev = rdev->ddev;
         struct drm_crtc *crtc;
@@ -677,6 +1188,7 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev)
         if (rdev->pm.num_power_states < 2)
                 return;
 
+        INIT_WORK(&rdev->pm.dpm.thermal.work, radeon_dpm_thermal_work_handler);
         mutex_lock(&rdev->pm.mutex);
 
         rdev->pm.active_crtcs = 0;
@@ -739,6 +1251,46 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev)
         mutex_unlock(&rdev->pm.mutex);
 }
 
+static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev)
+{
+        struct drm_device *ddev = rdev->ddev;
+        struct drm_crtc *crtc;
+        struct radeon_crtc *radeon_crtc;
+
+        mutex_lock(&rdev->pm.mutex);
+
+        /* update active crtc counts */
+        rdev->pm.dpm.new_active_crtcs = 0;
+        rdev->pm.dpm.new_active_crtc_count = 0;
+        list_for_each_entry(crtc,
+                &ddev->mode_config.crtc_list, head) {
+                radeon_crtc = to_radeon_crtc(crtc);
+                if (crtc->enabled) {
+                        rdev->pm.dpm.new_active_crtcs |= (1 << radeon_crtc->crtc_id);
+                        rdev->pm.dpm.new_active_crtc_count++;
+                }
+        }
+
+        /* update battery/ac status */
+        if (power_supply_is_system_supplied() > 0)
+                rdev->pm.dpm.ac_power = true;
+        else
+                rdev->pm.dpm.ac_power = false;
+
+        radeon_dpm_change_power_state_locked(rdev);
+
+        mutex_unlock(&rdev->pm.mutex);
+
+}
+
+void radeon_pm_compute_clocks(struct radeon_device *rdev)
+{
+        if (rdev->pm.pm_method == PM_METHOD_DPM)
+                radeon_pm_compute_clocks_dpm(rdev);
+        else
+                radeon_pm_compute_clocks_old(rdev);
+}
+
 static bool radeon_pm_in_vbl(struct radeon_device *rdev)
 {
         int  crtc, vpos, hpos, vbl_status;
@@ -842,19 +1394,28 @@ static int radeon_debugfs_pm_info(struct seq_file *m, void *data)
         struct drm_device *dev = node->minor->dev;
         struct radeon_device *rdev = dev->dev_private;
 
-        seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk);
-        /* radeon_get_engine_clock is not reliable on APUs so just print the current clock */
-        if ((rdev->family >= CHIP_PALM) && (rdev->flags & RADEON_IS_IGP))
-                seq_printf(m, "current engine clock: %u0 kHz\n", rdev->pm.current_sclk);
-        else
-                seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev));
-        seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk);
-        if (rdev->asic->pm.get_memory_clock)
-                seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev));
-        if (rdev->pm.current_vddc)
-                seq_printf(m, "voltage: %u mV\n", rdev->pm.current_vddc);
-        if (rdev->asic->pm.get_pcie_lanes)
-                seq_printf(m, "PCIE lanes: %d\n", radeon_get_pcie_lanes(rdev));
+        if (rdev->pm.dpm_enabled) {
+                mutex_lock(&rdev->pm.mutex);
+                if (rdev->asic->dpm.debugfs_print_current_performance_level)
+                        radeon_dpm_debugfs_print_current_performance_level(rdev, m);
+                else
+                        seq_printf(m, "Debugfs support not implemented for this asic\n");
+                mutex_unlock(&rdev->pm.mutex);
+        } else {
+                seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk);
+                /* radeon_get_engine_clock is not reliable on APUs so just print the current clock */
+                if ((rdev->family >= CHIP_PALM) && (rdev->flags & RADEON_IS_IGP))
+                        seq_printf(m, "current engine clock: %u0 kHz\n", rdev->pm.current_sclk);
+                else
+                        seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev));
+                seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk);
+                if (rdev->asic->pm.get_memory_clock)
+                        seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev));
+                if (rdev->pm.current_vddc)
+                        seq_printf(m, "voltage: %u mV\n", rdev->pm.current_vddc);
+                if (rdev->asic->pm.get_pcie_lanes)
+                        seq_printf(m, "PCIE lanes: %d\n", radeon_get_pcie_lanes(rdev));
+        }
 
         return 0;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_prime.c b/drivers/gpu/drm/radeon/radeon_prime.c
index 4940af7e75e6..65b9eabd5a2f 100644
--- a/drivers/gpu/drm/radeon/radeon_prime.c
+++ b/drivers/gpu/drm/radeon/radeon_prime.c
@@ -88,11 +88,19 @@ int radeon_gem_prime_pin(struct drm_gem_object *obj)
 
         /* pin buffer into GTT */
         ret = radeon_bo_pin(bo, RADEON_GEM_DOMAIN_GTT, NULL);
-        if (ret) {
-                radeon_bo_unreserve(bo);
-                return ret;
-        }
         radeon_bo_unreserve(bo);
+        return ret;
+}
+
+void radeon_gem_prime_unpin(struct drm_gem_object *obj)
+{
+        struct radeon_bo *bo = gem_to_radeon_bo(obj);
+        int ret = 0;
 
-        return 0;
+        ret = radeon_bo_reserve(bo, false);
+        if (unlikely(ret != 0))
+                return;
+
+        radeon_bo_unpin(bo);
+        radeon_bo_unreserve(bo);
 }
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index 7e2c2b7cf188..62d54976d24e 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -57,6 +57,7 @@
 #include "evergreen_reg.h"
 #include "ni_reg.h"
 #include "si_reg.h"
+#include "cik_reg.h"
 
 #define RADEON_MC_AGP_LOCATION          0x014c
 #define         RADEON_MC_AGP_START_MASK        0x0000FFFF
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 82434018cbe8..5f1c51a776ed 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -357,6 +357,38 @@ bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
         }
 }
 
+u32 radeon_ring_generic_get_rptr(struct radeon_device *rdev,
+                                 struct radeon_ring *ring)
+{
+        u32 rptr;
+
+        if (rdev->wb.enabled && ring != &rdev->ring[R600_RING_TYPE_UVD_INDEX])
+                rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
+        else
+                rptr = RREG32(ring->rptr_reg);
+        rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
+
+        return rptr;
+}
+
+u32 radeon_ring_generic_get_wptr(struct radeon_device *rdev,
+                                 struct radeon_ring *ring)
+{
+        u32 wptr;
+
+        wptr = RREG32(ring->wptr_reg);
+        wptr = (wptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
+
+        return wptr;
+}
+
+void radeon_ring_generic_set_wptr(struct radeon_device *rdev,
+                                  struct radeon_ring *ring)
+{
+        WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask);
+        (void)RREG32(ring->wptr_reg);
+}
+
 /**
  * radeon_ring_free_size - update the free size
  *
@@ -367,13 +399,7 @@ bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
  */
 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
 {
-        u32 rptr;
-
-        if (rdev->wb.enabled && ring != &rdev->ring[R600_RING_TYPE_UVD_INDEX])
-                rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
-        else
-                rptr = RREG32(ring->rptr_reg);
-        ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
+        ring->rptr = radeon_ring_get_rptr(rdev, ring);
         /* This works because ring_size is a power of 2 */
         ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
         ring->ring_free_dw -= ring->wptr;
@@ -465,8 +491,7 @@ void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring)
                 radeon_ring_write(ring, ring->nop);
         }
         DRM_MEMORYBARRIER();
-        WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask);
-        (void)RREG32(ring->wptr_reg);
+        radeon_ring_set_wptr(rdev, ring);
 }
 
 /**
@@ -568,7 +593,6 @@ void radeon_ring_lockup_update(struct radeon_ring *ring)
 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
 {
         unsigned long cjiffies, elapsed;
-        uint32_t rptr;
 
         cjiffies = jiffies;
         if (!time_after(cjiffies, ring->last_activity)) {
@@ -576,8 +600,7 @@ bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *rin
                 radeon_ring_lockup_update(ring);
                 return false;
         }
-        rptr = RREG32(ring->rptr_reg);
-        ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
+        ring->rptr = radeon_ring_get_rptr(rdev, ring);
         if (ring->rptr != ring->last_rptr) {
                 /* CP is still working no lockup */
                 radeon_ring_lockup_update(ring);
@@ -804,9 +827,9 @@ static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
 
         radeon_ring_free_size(rdev, ring);
         count = (ring->ring_size / 4) - ring->ring_free_dw;
-        tmp = RREG32(ring->wptr_reg) >> ring->ptr_reg_shift;
+        tmp = radeon_ring_get_wptr(rdev, ring);
         seq_printf(m, "wptr(0x%04x): 0x%08x [%5d]\n", ring->wptr_reg, tmp, tmp);
-        tmp = RREG32(ring->rptr_reg) >> ring->ptr_reg_shift;
+        tmp = radeon_ring_get_rptr(rdev, ring);
         seq_printf(m, "rptr(0x%04x): 0x%08x [%5d]\n", ring->rptr_reg, tmp, tmp);
         if (ring->rptr_save_reg) {
                 seq_printf(m, "rptr next(0x%04x): 0x%08x\n", ring->rptr_save_reg,
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index bbed4af8d0bc..f4d6bcee9006 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -35,7 +35,6 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
 {
         struct radeon_bo *vram_obj = NULL;
         struct radeon_bo **gtt_obj = NULL;
-        struct radeon_fence *fence = NULL;
         uint64_t gtt_addr, vram_addr;
         unsigned i, n, size;
         int r, ring;
@@ -81,37 +80,38 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
         }
         r = radeon_bo_reserve(vram_obj, false);
         if (unlikely(r != 0))
-                goto out_cleanup;
+                goto out_unref;
         r = radeon_bo_pin(vram_obj, RADEON_GEM_DOMAIN_VRAM, &vram_addr);
         if (r) {
                 DRM_ERROR("Failed to pin VRAM object\n");
-                goto out_cleanup;
+                goto out_unres;
         }
         for (i = 0; i < n; i++) {
                 void *gtt_map, *vram_map;
                 void **gtt_start, **gtt_end;
                 void **vram_start, **vram_end;
+                struct radeon_fence *fence = NULL;
 
                 r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
                                      RADEON_GEM_DOMAIN_GTT, NULL, gtt_obj + i);
                 if (r) {
                         DRM_ERROR("Failed to create GTT object %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean;
                 }
 
                 r = radeon_bo_reserve(gtt_obj[i], false);
                 if (unlikely(r != 0))
-                        goto out_cleanup;
+                        goto out_lclean_unref;
                 r = radeon_bo_pin(gtt_obj[i], RADEON_GEM_DOMAIN_GTT, &gtt_addr);
                 if (r) {
                         DRM_ERROR("Failed to pin GTT object %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unres;
                 }
 
                 r = radeon_bo_kmap(gtt_obj[i], &gtt_map);
                 if (r) {
                         DRM_ERROR("Failed to map GTT object %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 for (gtt_start = gtt_map, gtt_end = gtt_map + size;
@@ -127,13 +127,13 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
                         r = radeon_copy_blit(rdev, gtt_addr, vram_addr, size / RADEON_GPU_PAGE_SIZE, &fence);
                 if (r) {
                         DRM_ERROR("Failed GTT->VRAM copy %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 r = radeon_fence_wait(fence, false);
                 if (r) {
                         DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 radeon_fence_unref(&fence);
@@ -141,7 +141,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
                 r = radeon_bo_kmap(vram_obj, &vram_map);
                 if (r) {
                         DRM_ERROR("Failed to map VRAM object after copy %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 for (gtt_start = gtt_map, gtt_end = gtt_map + size,
@@ -160,7 +160,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
                                           (vram_addr - rdev->mc.vram_start +
                                            (void*)gtt_start - gtt_map));
                                 radeon_bo_kunmap(vram_obj);
-                                goto out_cleanup;
+                                goto out_lclean_unpin;
                         }
                         *vram_start = vram_start;
                 }
@@ -173,13 +173,13 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
                         r = radeon_copy_blit(rdev, vram_addr, gtt_addr, size / RADEON_GPU_PAGE_SIZE, &fence);
                 if (r) {
                         DRM_ERROR("Failed VRAM->GTT copy %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 r = radeon_fence_wait(fence, false);
                 if (r) {
                         DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 radeon_fence_unref(&fence);
@@ -187,7 +187,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
                 r = radeon_bo_kmap(gtt_obj[i], &gtt_map);
                 if (r) {
                         DRM_ERROR("Failed to map GTT object after copy %d\n", i);
-                        goto out_cleanup;
+                        goto out_lclean_unpin;
                 }
 
                 for (gtt_start = gtt_map, gtt_end = gtt_map + size,
@@ -206,7 +206,7 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
                                           (gtt_addr - rdev->mc.gtt_start +
                                            (void*)vram_start - vram_map));
                                 radeon_bo_kunmap(gtt_obj[i]);
-                                goto out_cleanup;
+                                goto out_lclean_unpin;
                         }
                 }
 
@@ -214,31 +214,32 @@ static void radeon_do_test_moves(struct radeon_device *rdev, int flag)
 
                 DRM_INFO("Tested GTT->VRAM and VRAM->GTT copy for GTT offset 0x%llx\n",
                          gtt_addr - rdev->mc.gtt_start);
+                continue;
+
+out_lclean_unpin:
+                radeon_bo_unpin(gtt_obj[i]);
+out_lclean_unres:
+                radeon_bo_unreserve(gtt_obj[i]);
+out_lclean_unref:
+                radeon_bo_unref(&gtt_obj[i]);
+out_lclean:
+                for (--i; i >= 0; --i) {
+                        radeon_bo_unpin(gtt_obj[i]);
+                        radeon_bo_unreserve(gtt_obj[i]);
+                        radeon_bo_unref(&gtt_obj[i]);
+                }
+                if (fence)
+                        radeon_fence_unref(&fence);
+                break;
         }
 
+        radeon_bo_unpin(vram_obj);
+out_unres:
+        radeon_bo_unreserve(vram_obj);
+out_unref:
+        radeon_bo_unref(&vram_obj);
 out_cleanup:
-        if (vram_obj) {
-                if (radeon_bo_is_reserved(vram_obj)) {
-                        radeon_bo_unpin(vram_obj);
-                        radeon_bo_unreserve(vram_obj);
-                }
-                radeon_bo_unref(&vram_obj);
-        }
-        if (gtt_obj) {
-                for (i = 0; i < n; i++) {
-                        if (gtt_obj[i]) {
-                                if (radeon_bo_is_reserved(gtt_obj[i])) {
-                                        radeon_bo_unpin(gtt_obj[i]);
-                                        radeon_bo_unreserve(gtt_obj[i]);
-                                }
-                                radeon_bo_unref(&gtt_obj[i]);
-                        }
-                }
-                kfree(gtt_obj);
-        }
-        if (fence) {
-                radeon_fence_unref(&fence);
-        }
+        kfree(gtt_obj);
         if (r) {
                 printk(KERN_WARNING "Error while testing BO move.\n");
         }
diff --git a/drivers/gpu/drm/radeon/radeon_ucode.h b/drivers/gpu/drm/radeon/radeon_ucode.h
new file mode 100644
index 000000000000..d8b05f7bcf1a
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_ucode.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RADEON_UCODE_H__
+#define __RADEON_UCODE_H__
+
+/* CP */
+#define R600_PFP_UCODE_SIZE          576
+#define R600_PM4_UCODE_SIZE          1792
+#define R700_PFP_UCODE_SIZE          848
+#define R700_PM4_UCODE_SIZE          1360
+#define EVERGREEN_PFP_UCODE_SIZE     1120
+#define EVERGREEN_PM4_UCODE_SIZE     1376
+#define CAYMAN_PFP_UCODE_SIZE        2176
+#define CAYMAN_PM4_UCODE_SIZE        2176
+#define SI_PFP_UCODE_SIZE            2144
+#define SI_PM4_UCODE_SIZE            2144
+#define SI_CE_UCODE_SIZE             2144
+
+/* RLC */
+#define R600_RLC_UCODE_SIZE          768
+#define R700_RLC_UCODE_SIZE          1024
+#define EVERGREEN_RLC_UCODE_SIZE     768
+#define CAYMAN_RLC_UCODE_SIZE        1024
+#define ARUBA_RLC_UCODE_SIZE         1536
+#define SI_RLC_UCODE_SIZE            2048
+
+/* MC */
+#define BTC_MC_UCODE_SIZE            6024
+#define CAYMAN_MC_UCODE_SIZE         6037
+#define SI_MC_UCODE_SIZE             7769
+#define OLAND_MC_UCODE_SIZE          7863
+
+/* SMC */
+#define RV770_SMC_UCODE_START        0x0100
+#define RV770_SMC_UCODE_SIZE         0x410d
+#define RV770_SMC_INT_VECTOR_START   0xffc0
+#define RV770_SMC_INT_VECTOR_SIZE    0x0040
+
+#define RV730_SMC_UCODE_START        0x0100
+#define RV730_SMC_UCODE_SIZE         0x412c
+#define RV730_SMC_INT_VECTOR_START   0xffc0
+#define RV730_SMC_INT_VECTOR_SIZE    0x0040
+
+#define RV710_SMC_UCODE_START        0x0100
+#define RV710_SMC_UCODE_SIZE         0x3f1f
+#define RV710_SMC_INT_VECTOR_START   0xffc0
+#define RV710_SMC_INT_VECTOR_SIZE    0x0040
+
+#define RV740_SMC_UCODE_START        0x0100
+#define RV740_SMC_UCODE_SIZE         0x41c5
+#define RV740_SMC_INT_VECTOR_START   0xffc0
+#define RV740_SMC_INT_VECTOR_SIZE    0x0040
+
+#define CEDAR_SMC_UCODE_START        0x0100
+#define CEDAR_SMC_UCODE_SIZE         0x5d50
+#define CEDAR_SMC_INT_VECTOR_START   0xffc0
+#define CEDAR_SMC_INT_VECTOR_SIZE    0x0040
+
+#define REDWOOD_SMC_UCODE_START      0x0100
+#define REDWOOD_SMC_UCODE_SIZE       0x5f0a
+#define REDWOOD_SMC_INT_VECTOR_START 0xffc0
+#define REDWOOD_SMC_INT_VECTOR_SIZE  0x0040
+
+#define JUNIPER_SMC_UCODE_START      0x0100
+#define JUNIPER_SMC_UCODE_SIZE       0x5f1f
+#define JUNIPER_SMC_INT_VECTOR_START 0xffc0
+#define JUNIPER_SMC_INT_VECTOR_SIZE  0x0040
+
+#define CYPRESS_SMC_UCODE_START      0x0100
+#define CYPRESS_SMC_UCODE_SIZE       0x61f7
+#define CYPRESS_SMC_INT_VECTOR_START 0xffc0
+#define CYPRESS_SMC_INT_VECTOR_SIZE  0x0040
+
+#define BARTS_SMC_UCODE_START        0x0100
+#define BARTS_SMC_UCODE_SIZE         0x6107
+#define BARTS_SMC_INT_VECTOR_START   0xffc0
+#define BARTS_SMC_INT_VECTOR_SIZE    0x0040
+
+#define TURKS_SMC_UCODE_START        0x0100
+#define TURKS_SMC_UCODE_SIZE         0x605b
+#define TURKS_SMC_INT_VECTOR_START   0xffc0
+#define TURKS_SMC_INT_VECTOR_SIZE    0x0040
+
+#define CAICOS_SMC_UCODE_START       0x0100
+#define CAICOS_SMC_UCODE_SIZE        0x5fbd
+#define CAICOS_SMC_INT_VECTOR_START  0xffc0
+#define CAICOS_SMC_INT_VECTOR_SIZE   0x0040
+
+#define CAYMAN_SMC_UCODE_START       0x0100
+#define CAYMAN_SMC_UCODE_SIZE        0x79ec
+#define CAYMAN_SMC_INT_VECTOR_START  0xffc0
+#define CAYMAN_SMC_INT_VECTOR_SIZE   0x0040
+
+#define TAHITI_SMC_UCODE_START       0x10000
+#define TAHITI_SMC_UCODE_SIZE        0xf458
+
+#define PITCAIRN_SMC_UCODE_START     0x10000
+#define PITCAIRN_SMC_UCODE_SIZE      0xe9f4
+
+#define VERDE_SMC_UCODE_START        0x10000
+#define VERDE_SMC_UCODE_SIZE         0xebe4
+
+#define OLAND_SMC_UCODE_START        0x10000
+#define OLAND_SMC_UCODE_SIZE         0xe7b4
+
+#define HAINAN_SMC_UCODE_START       0x10000
+#define HAINAN_SMC_UCODE_SIZE        0xe67C
+
+#endif
diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c
index cad735dd02c6..41efcec28cd8 100644
--- a/drivers/gpu/drm/radeon/radeon_uvd.c
+++ b/drivers/gpu/drm/radeon/radeon_uvd.c
@@ -44,11 +44,13 @@
 #define FIRMWARE_CYPRESS        "radeon/CYPRESS_uvd.bin"
 #define FIRMWARE_SUMO           "radeon/SUMO_uvd.bin"
 #define FIRMWARE_TAHITI         "radeon/TAHITI_uvd.bin"
+#define FIRMWARE_BONAIRE        "radeon/BONAIRE_uvd.bin"
 
 MODULE_FIRMWARE(FIRMWARE_RV710);
 MODULE_FIRMWARE(FIRMWARE_CYPRESS);
 MODULE_FIRMWARE(FIRMWARE_SUMO);
 MODULE_FIRMWARE(FIRMWARE_TAHITI);
+MODULE_FIRMWARE(FIRMWARE_BONAIRE);
 
 static void radeon_uvd_idle_work_handler(struct work_struct *work);
 
@@ -100,6 +102,12 @@ int radeon_uvd_init(struct radeon_device *rdev)
                 fw_name = FIRMWARE_TAHITI;
                 break;
 
+        case CHIP_BONAIRE:
+        case CHIP_KABINI:
+        case CHIP_KAVERI:
+                fw_name = FIRMWARE_BONAIRE;
+                break;
+
         default:
                 return -EINVAL;
         }
@@ -542,6 +550,7 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
                                struct radeon_fence **fence)
 {
         struct ttm_validate_buffer tv;
+        struct ww_acquire_ctx ticket;
         struct list_head head;
         struct radeon_ib ib;
         uint64_t addr;
@@ -553,7 +562,7 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
         INIT_LIST_HEAD(&head);
         list_add(&tv.head, &head);
 
-        r = ttm_eu_reserve_buffers(&head);
+        r = ttm_eu_reserve_buffers(&ticket, &head);
         if (r)
                 return r;
 
@@ -561,16 +570,12 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
         radeon_uvd_force_into_uvd_segment(bo);
 
         r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-        if (r) {
-                ttm_eu_backoff_reservation(&head);
-                return r;
-        }
+        if (r) 
+                goto err;
 
         r = radeon_ib_get(rdev, ring, &ib, NULL, 16);
-        if (r) {
-                ttm_eu_backoff_reservation(&head);
-                return r;
-        }
+        if (r)
+                goto err;
 
         addr = radeon_bo_gpu_offset(bo);
         ib.ptr[0] = PACKET0(UVD_GPCOM_VCPU_DATA0, 0);
@@ -584,11 +589,9 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
         ib.length_dw = 16;
 
         r = radeon_ib_schedule(rdev, &ib, NULL);
-        if (r) {
-                ttm_eu_backoff_reservation(&head);
-                return r;
-        }
-        ttm_eu_fence_buffer_objects(&head, ib.fence);
+        if (r)
+                goto err;
+        ttm_eu_fence_buffer_objects(&ticket, &head, ib.fence);
 
         if (fence)
                 *fence = radeon_fence_ref(ib.fence);
@@ -596,6 +599,10 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
         radeon_ib_free(rdev, &ib);
         radeon_bo_unref(&bo);
         return 0;
+
+err:
+        ttm_eu_backoff_reservation(&ticket, &head);
+        return r;
 }
 
 /* multiple fence commands without any stream commands in between can
@@ -691,11 +698,19 @@ static void radeon_uvd_idle_work_handler(struct work_struct *work)
         struct radeon_device *rdev =
                 container_of(work, struct radeon_device, uvd.idle_work.work);
 
-        if (radeon_fence_count_emitted(rdev, R600_RING_TYPE_UVD_INDEX) == 0)
-                radeon_set_uvd_clocks(rdev, 0, 0);
-        else
+        if (radeon_fence_count_emitted(rdev, R600_RING_TYPE_UVD_INDEX) == 0) {
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        mutex_lock(&rdev->pm.mutex);
+                        rdev->pm.dpm.uvd_active = false;
+                        mutex_unlock(&rdev->pm.mutex);
+                        radeon_pm_compute_clocks(rdev);
+                } else {
+                        radeon_set_uvd_clocks(rdev, 0, 0);
+                }
+        } else {
                 schedule_delayed_work(&rdev->uvd.idle_work,
                                       msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS));
+        }
 }
 
 void radeon_uvd_note_usage(struct radeon_device *rdev)
@@ -703,8 +718,14 @@ void radeon_uvd_note_usage(struct radeon_device *rdev)
         bool set_clocks = !cancel_delayed_work_sync(&rdev->uvd.idle_work);
         set_clocks &= schedule_delayed_work(&rdev->uvd.idle_work,
                                             msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS));
-        if (set_clocks)
-                radeon_set_uvd_clocks(rdev, 53300, 40000);
+        if (set_clocks) {
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        /* XXX pick SD/HD/MVC */
+                        radeon_dpm_enable_power_state(rdev, POWER_STATE_TYPE_INTERNAL_UVD);
+                } else {
+                        radeon_set_uvd_clocks(rdev, 53300, 40000);
+                }
+        }
 }
 
 static unsigned radeon_uvd_calc_upll_post_div(unsigned vco_freq,
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index 55880d5962c3..d8ddfb34545d 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -248,13 +248,16 @@ struct rs690_watermark {
 };
 
 static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
-                                  struct radeon_crtc *crtc,
-                                  struct rs690_watermark *wm)
+                                         struct radeon_crtc *crtc,
+                                         struct rs690_watermark *wm,
+                                         bool low)
 {
         struct drm_display_mode *mode = &crtc->base.mode;
         fixed20_12 a, b, c;
         fixed20_12 pclk, request_fifo_depth, tolerable_latency, estimated_width;
         fixed20_12 consumption_time, line_time, chunk_time, read_delay_latency;
+        fixed20_12 sclk, core_bandwidth, max_bandwidth;
+        u32 selected_sclk;
 
         if (!crtc->base.enabled) {
                 /* FIXME: wouldn't it better to set priority mark to maximum */
@@ -262,6 +265,21 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
                 return;
         }
 
+        if (((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880)) &&
+            (rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
+                selected_sclk = radeon_dpm_get_sclk(rdev, low);
+        else
+                selected_sclk = rdev->pm.current_sclk;
+
+        /* sclk in Mhz */
+        a.full = dfixed_const(100);
+        sclk.full = dfixed_const(selected_sclk);
+        sclk.full = dfixed_div(sclk, a);
+
+        /* core_bandwidth = sclk(Mhz) * 16 */
+        a.full = dfixed_const(16);
+        core_bandwidth.full = dfixed_div(rdev->pm.sclk, a);
+
         if (crtc->vsc.full > dfixed_const(2))
                 wm->num_line_pair.full = dfixed_const(2);
         else
@@ -322,36 +340,36 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
         wm->active_time.full = dfixed_div(wm->active_time, a);
 
         /* Maximun bandwidth is the minimun bandwidth of all component */
-        rdev->pm.max_bandwidth = rdev->pm.core_bandwidth;
+        max_bandwidth = core_bandwidth;
         if (rdev->mc.igp_sideport_enabled) {
-                if (rdev->pm.max_bandwidth.full > rdev->pm.sideport_bandwidth.full &&
+                if (max_bandwidth.full > rdev->pm.sideport_bandwidth.full &&
                         rdev->pm.sideport_bandwidth.full)
-                        rdev->pm.max_bandwidth = rdev->pm.sideport_bandwidth;
+                        max_bandwidth = rdev->pm.sideport_bandwidth;
                 read_delay_latency.full = dfixed_const(370 * 800 * 1000);
                 read_delay_latency.full = dfixed_div(read_delay_latency,
                         rdev->pm.igp_sideport_mclk);
         } else {
-                if (rdev->pm.max_bandwidth.full > rdev->pm.k8_bandwidth.full &&
+                if (max_bandwidth.full > rdev->pm.k8_bandwidth.full &&
                         rdev->pm.k8_bandwidth.full)
-                        rdev->pm.max_bandwidth = rdev->pm.k8_bandwidth;
-                if (rdev->pm.max_bandwidth.full > rdev->pm.ht_bandwidth.full &&
+                        max_bandwidth = rdev->pm.k8_bandwidth;
+                if (max_bandwidth.full > rdev->pm.ht_bandwidth.full &&
                         rdev->pm.ht_bandwidth.full)
-                        rdev->pm.max_bandwidth = rdev->pm.ht_bandwidth;
+                        max_bandwidth = rdev->pm.ht_bandwidth;
                 read_delay_latency.full = dfixed_const(5000);
         }
 
         /* sclk = system clocks(ns) = 1000 / max_bandwidth / 16 */
         a.full = dfixed_const(16);
-        rdev->pm.sclk.full = dfixed_mul(rdev->pm.max_bandwidth, a);
+        sclk.full = dfixed_mul(max_bandwidth, a);
         a.full = dfixed_const(1000);
-        rdev->pm.sclk.full = dfixed_div(a, rdev->pm.sclk);
+        sclk.full = dfixed_div(a, sclk);
         /* Determine chunk time
          * ChunkTime = the time it takes the DCP to send one chunk of data
          * to the LB which consists of pipeline delay and inter chunk gap
          * sclk = system clock(ns)
          */
         a.full = dfixed_const(256 * 13);
-        chunk_time.full = dfixed_mul(rdev->pm.sclk, a);
+        chunk_time.full = dfixed_mul(sclk, a);
         a.full = dfixed_const(10);
         chunk_time.full = dfixed_div(chunk_time, a);
 
@@ -415,175 +433,200 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
         }
 }
 
-void rs690_bandwidth_update(struct radeon_device *rdev)
+static void rs690_compute_mode_priority(struct radeon_device *rdev,
+                                        struct rs690_watermark *wm0,
+                                        struct rs690_watermark *wm1,
+                                        struct drm_display_mode *mode0,
+                                        struct drm_display_mode *mode1,
+                                        u32 *d1mode_priority_a_cnt,
+                                        u32 *d2mode_priority_a_cnt)
 {
-        struct drm_display_mode *mode0 = NULL;
-        struct drm_display_mode *mode1 = NULL;
-        struct rs690_watermark wm0;
-        struct rs690_watermark wm1;
-        u32 tmp;
-        u32 d1mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1);
-        u32 d2mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1);
         fixed20_12 priority_mark02, priority_mark12, fill_rate;
         fixed20_12 a, b;
 
-        radeon_update_display_priority(rdev);
-
-        if (rdev->mode_info.crtcs[0]->base.enabled)
-                mode0 = &rdev->mode_info.crtcs[0]->base.mode;
-        if (rdev->mode_info.crtcs[1]->base.enabled)
-                mode1 = &rdev->mode_info.crtcs[1]->base.mode;
-        /*
-         * Set display0/1 priority up in the memory controller for
-         * modes if the user specifies HIGH for displaypriority
-         * option.
-         */
-        if ((rdev->disp_priority == 2) &&
-            ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740))) {
-                tmp = RREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER);
-                tmp &= C_000104_MC_DISP0R_INIT_LAT;
-                tmp &= C_000104_MC_DISP1R_INIT_LAT;
-                if (mode0)
-                        tmp |= S_000104_MC_DISP0R_INIT_LAT(1);
-                if (mode1)
-                        tmp |= S_000104_MC_DISP1R_INIT_LAT(1);
-                WREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER, tmp);
-        }
-        rs690_line_buffer_adjust(rdev, mode0, mode1);
-
-        if ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740))
-                WREG32(R_006C9C_DCP_CONTROL, 0);
-        if ((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880))
-                WREG32(R_006C9C_DCP_CONTROL, 2);
-
-        rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0);
-        rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1);
-
-        tmp = (wm0.lb_request_fifo_depth - 1);
-        tmp |= (wm1.lb_request_fifo_depth - 1) << 16;
-        WREG32(R_006D58_LB_MAX_REQ_OUTSTANDING, tmp);
+        *d1mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1);
+        *d2mode_priority_a_cnt = S_006548_D1MODE_PRIORITY_A_OFF(1);
 
         if (mode0 && mode1) {
-                if (dfixed_trunc(wm0.dbpp) > 64)
-                        a.full = dfixed_mul(wm0.dbpp, wm0.num_line_pair);
+                if (dfixed_trunc(wm0->dbpp) > 64)
+                        a.full = dfixed_mul(wm0->dbpp, wm0->num_line_pair);
                 else
-                        a.full = wm0.num_line_pair.full;
-                if (dfixed_trunc(wm1.dbpp) > 64)
-                        b.full = dfixed_mul(wm1.dbpp, wm1.num_line_pair);
+                        a.full = wm0->num_line_pair.full;
+                if (dfixed_trunc(wm1->dbpp) > 64)
+                        b.full = dfixed_mul(wm1->dbpp, wm1->num_line_pair);
                 else
-                        b.full = wm1.num_line_pair.full;
+                        b.full = wm1->num_line_pair.full;
                 a.full += b.full;
-                fill_rate.full = dfixed_div(wm0.sclk, a);
-                if (wm0.consumption_rate.full > fill_rate.full) {
-                        b.full = wm0.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm0.active_time);
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                fill_rate.full = dfixed_div(wm0->sclk, a);
+                if (wm0->consumption_rate.full > fill_rate.full) {
+                        b.full = wm0->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm0->active_time);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         a.full = a.full + b.full;
                         b.full = dfixed_const(16 * 1000);
                         priority_mark02.full = dfixed_div(a, b);
                 } else {
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark02.full = dfixed_div(a, b);
                 }
-                if (wm1.consumption_rate.full > fill_rate.full) {
-                        b.full = wm1.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm1.active_time);
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                if (wm1->consumption_rate.full > fill_rate.full) {
+                        b.full = wm1->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm1->active_time);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         a.full = a.full + b.full;
                         b.full = dfixed_const(16 * 1000);
                         priority_mark12.full = dfixed_div(a, b);
                 } else {
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark12.full = dfixed_div(a, b);
                 }
-                if (wm0.priority_mark.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark.full;
+                if (wm0->priority_mark.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark.full;
                 if (dfixed_trunc(priority_mark02) < 0)
                         priority_mark02.full = 0;
-                if (wm0.priority_mark_max.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark_max.full;
-                if (wm1.priority_mark.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark.full;
+                if (wm0->priority_mark_max.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark_max.full;
+                if (wm1->priority_mark.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark.full;
                 if (dfixed_trunc(priority_mark12) < 0)
                         priority_mark12.full = 0;
-                if (wm1.priority_mark_max.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark_max.full;
-                d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
-                d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
+                if (wm1->priority_mark_max.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark_max.full;
+                *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
+                *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
                 if (rdev->disp_priority == 2) {
-                        d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1);
-                        d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1);
+                        *d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1);
+                        *d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1);
                 }
         } else if (mode0) {
-                if (dfixed_trunc(wm0.dbpp) > 64)
-                        a.full = dfixed_mul(wm0.dbpp, wm0.num_line_pair);
+                if (dfixed_trunc(wm0->dbpp) > 64)
+                        a.full = dfixed_mul(wm0->dbpp, wm0->num_line_pair);
                 else
-                        a.full = wm0.num_line_pair.full;
-                fill_rate.full = dfixed_div(wm0.sclk, a);
-                if (wm0.consumption_rate.full > fill_rate.full) {
-                        b.full = wm0.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm0.active_time);
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = wm0->num_line_pair.full;
+                fill_rate.full = dfixed_div(wm0->sclk, a);
+                if (wm0->consumption_rate.full > fill_rate.full) {
+                        b.full = wm0->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm0->active_time);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         a.full = a.full + b.full;
                         b.full = dfixed_const(16 * 1000);
                         priority_mark02.full = dfixed_div(a, b);
                 } else {
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark02.full = dfixed_div(a, b);
                 }
-                if (wm0.priority_mark.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark.full;
+                if (wm0->priority_mark.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark.full;
                 if (dfixed_trunc(priority_mark02) < 0)
                         priority_mark02.full = 0;
-                if (wm0.priority_mark_max.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark_max.full;
-                d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
+                if (wm0->priority_mark_max.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark_max.full;
+                *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
                 if (rdev->disp_priority == 2)
-                        d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1);
+                        *d1mode_priority_a_cnt |= S_006548_D1MODE_PRIORITY_A_ALWAYS_ON(1);
         } else if (mode1) {
-                if (dfixed_trunc(wm1.dbpp) > 64)
-                        a.full = dfixed_mul(wm1.dbpp, wm1.num_line_pair);
+                if (dfixed_trunc(wm1->dbpp) > 64)
+                        a.full = dfixed_mul(wm1->dbpp, wm1->num_line_pair);
                 else
-                        a.full = wm1.num_line_pair.full;
-                fill_rate.full = dfixed_div(wm1.sclk, a);
-                if (wm1.consumption_rate.full > fill_rate.full) {
-                        b.full = wm1.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm1.active_time);
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = wm1->num_line_pair.full;
+                fill_rate.full = dfixed_div(wm1->sclk, a);
+                if (wm1->consumption_rate.full > fill_rate.full) {
+                        b.full = wm1->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm1->active_time);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         a.full = a.full + b.full;
                         b.full = dfixed_const(16 * 1000);
                         priority_mark12.full = dfixed_div(a, b);
                 } else {
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark12.full = dfixed_div(a, b);
                 }
-                if (wm1.priority_mark.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark.full;
+                if (wm1->priority_mark.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark.full;
                 if (dfixed_trunc(priority_mark12) < 0)
                         priority_mark12.full = 0;
-                if (wm1.priority_mark_max.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark_max.full;
-                d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
+                if (wm1->priority_mark_max.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark_max.full;
+                *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
                 if (rdev->disp_priority == 2)
-                        d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1);
+                        *d2mode_priority_a_cnt |= S_006D48_D2MODE_PRIORITY_A_ALWAYS_ON(1);
         }
+}
+
+void rs690_bandwidth_update(struct radeon_device *rdev)
+{
+        struct drm_display_mode *mode0 = NULL;
+        struct drm_display_mode *mode1 = NULL;
+        struct rs690_watermark wm0_high, wm0_low;
+        struct rs690_watermark wm1_high, wm1_low;
+        u32 tmp;
+        u32 d1mode_priority_a_cnt, d1mode_priority_b_cnt;
+        u32 d2mode_priority_a_cnt, d2mode_priority_b_cnt;
+
+        radeon_update_display_priority(rdev);
+
+        if (rdev->mode_info.crtcs[0]->base.enabled)
+                mode0 = &rdev->mode_info.crtcs[0]->base.mode;
+        if (rdev->mode_info.crtcs[1]->base.enabled)
+                mode1 = &rdev->mode_info.crtcs[1]->base.mode;
+        /*
+         * Set display0/1 priority up in the memory controller for
+         * modes if the user specifies HIGH for displaypriority
+         * option.
+         */
+        if ((rdev->disp_priority == 2) &&
+            ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740))) {
+                tmp = RREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER);
+                tmp &= C_000104_MC_DISP0R_INIT_LAT;
+                tmp &= C_000104_MC_DISP1R_INIT_LAT;
+                if (mode0)
+                        tmp |= S_000104_MC_DISP0R_INIT_LAT(1);
+                if (mode1)
+                        tmp |= S_000104_MC_DISP1R_INIT_LAT(1);
+                WREG32_MC(R_000104_MC_INIT_MISC_LAT_TIMER, tmp);
+        }
+        rs690_line_buffer_adjust(rdev, mode0, mode1);
+
+        if ((rdev->family == CHIP_RS690) || (rdev->family == CHIP_RS740))
+                WREG32(R_006C9C_DCP_CONTROL, 0);
+        if ((rdev->family == CHIP_RS780) || (rdev->family == CHIP_RS880))
+                WREG32(R_006C9C_DCP_CONTROL, 2);
+
+        rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_high, false);
+        rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_high, false);
+
+        rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_low, true);
+        rs690_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_low, true);
+
+        tmp = (wm0_high.lb_request_fifo_depth - 1);
+        tmp |= (wm1_high.lb_request_fifo_depth - 1) << 16;
+        WREG32(R_006D58_LB_MAX_REQ_OUTSTANDING, tmp);
+
+        rs690_compute_mode_priority(rdev,
+                                    &wm0_high, &wm1_high,
+                                    mode0, mode1,
+                                    &d1mode_priority_a_cnt, &d2mode_priority_a_cnt);
+        rs690_compute_mode_priority(rdev,
+                                    &wm0_low, &wm1_low,
+                                    mode0, mode1,
+                                    &d1mode_priority_b_cnt, &d2mode_priority_b_cnt);
 
         WREG32(R_006548_D1MODE_PRIORITY_A_CNT, d1mode_priority_a_cnt);
-        WREG32(R_00654C_D1MODE_PRIORITY_B_CNT, d1mode_priority_a_cnt);
+        WREG32(R_00654C_D1MODE_PRIORITY_B_CNT, d1mode_priority_b_cnt);
         WREG32(R_006D48_D2MODE_PRIORITY_A_CNT, d2mode_priority_a_cnt);
-        WREG32(R_006D4C_D2MODE_PRIORITY_B_CNT, d2mode_priority_a_cnt);
+        WREG32(R_006D4C_D2MODE_PRIORITY_B_CNT, d2mode_priority_b_cnt);
 }
 
 uint32_t rs690_mc_rreg(struct radeon_device *rdev, uint32_t reg)
diff --git a/drivers/gpu/drm/radeon/rs780_dpm.c b/drivers/gpu/drm/radeon/rs780_dpm.c
new file mode 100644
index 000000000000..bef832a62fee
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rs780_dpm.c
@@ -0,0 +1,963 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "rs780d.h"
+#include "r600_dpm.h"
+#include "rs780_dpm.h"
+#include "atom.h"
+
+static struct igp_ps *rs780_get_ps(struct radeon_ps *rps)
+{
+        struct igp_ps *ps = rps->ps_priv;
+
+        return ps;
+}
+
+static struct igp_power_info *rs780_get_pi(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+static void rs780_get_pm_mode_parameters(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+        struct radeon_mode_info *minfo = &rdev->mode_info;
+        struct drm_crtc *crtc;
+        struct radeon_crtc *radeon_crtc;
+        int i;
+
+        /* defaults */
+        pi->crtc_id = 0;
+        pi->refresh_rate = 60;
+
+        for (i = 0; i < rdev->num_crtc; i++) {
+                crtc = (struct drm_crtc *)minfo->crtcs[i];
+                if (crtc && crtc->enabled) {
+                        radeon_crtc = to_radeon_crtc(crtc);
+                        pi->crtc_id = radeon_crtc->crtc_id;
+                        if (crtc->mode.htotal && crtc->mode.vtotal)
+                                pi->refresh_rate =
+                                        (crtc->mode.clock * 1000) /
+                                        (crtc->mode.htotal * crtc->mode.vtotal);
+                        break;
+                }
+        }
+}
+
+static void rs780_voltage_scaling_enable(struct radeon_device *rdev, bool enable);
+
+static int rs780_initialize_dpm_power_state(struct radeon_device *rdev,
+                                            struct radeon_ps *boot_ps)
+{
+        struct atom_clock_dividers dividers;
+        struct igp_ps *default_state = rs780_get_ps(boot_ps);
+        int i, ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             default_state->sclk_low, false, &dividers);
+        if (ret)
+                return ret;
+
+        r600_engine_clock_entry_set_reference_divider(rdev, 0, dividers.ref_div);
+        r600_engine_clock_entry_set_feedback_divider(rdev, 0, dividers.fb_div);
+        r600_engine_clock_entry_set_post_divider(rdev, 0, dividers.post_div);
+
+        if (dividers.enable_post_div)
+                r600_engine_clock_entry_enable_post_divider(rdev, 0, true);
+        else
+                r600_engine_clock_entry_enable_post_divider(rdev, 0, false);
+
+        r600_engine_clock_entry_set_step_time(rdev, 0, R600_SST_DFLT);
+        r600_engine_clock_entry_enable_pulse_skipping(rdev, 0, false);
+
+        r600_engine_clock_entry_enable(rdev, 0, true);
+        for (i = 1; i < R600_PM_NUMBER_OF_SCLKS; i++)
+                r600_engine_clock_entry_enable(rdev, i, false);
+
+        r600_enable_mclk_control(rdev, false);
+        r600_voltage_control_enable_pins(rdev, 0);
+
+        return 0;
+}
+
+static int rs780_initialize_dpm_parameters(struct radeon_device *rdev,
+                                           struct radeon_ps *boot_ps)
+{
+        int ret = 0;
+        int i;
+
+        r600_set_bsp(rdev, R600_BSU_DFLT, R600_BSP_DFLT);
+
+        r600_set_at(rdev, 0, 0, 0, 0);
+
+        r600_set_git(rdev, R600_GICST_DFLT);
+
+        for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
+                r600_set_tc(rdev, i, 0, 0);
+
+        r600_select_td(rdev, R600_TD_DFLT);
+        r600_set_vrc(rdev, 0);
+
+        r600_set_tpu(rdev, R600_TPU_DFLT);
+        r600_set_tpc(rdev, R600_TPC_DFLT);
+
+        r600_set_sstu(rdev, R600_SSTU_DFLT);
+        r600_set_sst(rdev, R600_SST_DFLT);
+
+        r600_set_fctu(rdev, R600_FCTU_DFLT);
+        r600_set_fct(rdev, R600_FCT_DFLT);
+
+        r600_set_vddc3d_oorsu(rdev, R600_VDDC3DOORSU_DFLT);
+        r600_set_vddc3d_oorphc(rdev, R600_VDDC3DOORPHC_DFLT);
+        r600_set_vddc3d_oorsdc(rdev, R600_VDDC3DOORSDC_DFLT);
+        r600_set_ctxcgtt3d_rphc(rdev, R600_CTXCGTT3DRPHC_DFLT);
+        r600_set_ctxcgtt3d_rsdc(rdev, R600_CTXCGTT3DRSDC_DFLT);
+
+        r600_vid_rt_set_vru(rdev, R600_VRU_DFLT);
+        r600_vid_rt_set_vrt(rdev, R600_VOLTAGERESPONSETIME_DFLT);
+        r600_vid_rt_set_ssu(rdev, R600_SPLLSTEPUNIT_DFLT);
+
+        ret = rs780_initialize_dpm_power_state(rdev, boot_ps);
+
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW,     0);
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM,  0);
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_HIGH,    0);
+
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW,    0);
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, 0);
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_HIGH,   0);
+
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW,    0);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM, 0);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_HIGH,   0);
+
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,    R600_DISPLAY_WATERMARK_HIGH);
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM, R600_DISPLAY_WATERMARK_HIGH);
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_HIGH,   R600_DISPLAY_WATERMARK_HIGH);
+
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_CTXSW, false);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
+
+        r600_power_level_set_enter_index(rdev, R600_POWER_LEVEL_LOW);
+
+        r600_set_vrc(rdev, RS780_CGFTV_DFLT);
+
+        return ret;
+}
+
+static void rs780_start_dpm(struct radeon_device *rdev)
+{
+        r600_enable_sclk_control(rdev, false);
+        r600_enable_mclk_control(rdev, false);
+
+        r600_dynamicpm_enable(rdev, true);
+
+        radeon_wait_for_vblank(rdev, 0);
+        radeon_wait_for_vblank(rdev, 1);
+
+        r600_enable_spll_bypass(rdev, true);
+        r600_wait_for_spll_change(rdev);
+        r600_enable_spll_bypass(rdev, false);
+        r600_wait_for_spll_change(rdev);
+
+        r600_enable_spll_bypass(rdev, true);
+        r600_wait_for_spll_change(rdev);
+        r600_enable_spll_bypass(rdev, false);
+        r600_wait_for_spll_change(rdev);
+
+        r600_enable_sclk_control(rdev, true);
+}
+
+
+static void rs780_preset_ranges_slow_clk_fbdiv_en(struct radeon_device *rdev)
+{
+        WREG32_P(FVTHROT_SLOW_CLK_FEEDBACK_DIV_REG1, RANGE_SLOW_CLK_FEEDBACK_DIV_EN,
+                 ~RANGE_SLOW_CLK_FEEDBACK_DIV_EN);
+
+        WREG32_P(FVTHROT_SLOW_CLK_FEEDBACK_DIV_REG1,
+                 RANGE0_SLOW_CLK_FEEDBACK_DIV(RS780_SLOWCLKFEEDBACKDIV_DFLT),
+                 ~RANGE0_SLOW_CLK_FEEDBACK_DIV_MASK);
+}
+
+static void rs780_preset_starting_fbdiv(struct radeon_device *rdev)
+{
+        u32 fbdiv = (RREG32(CG_SPLL_FUNC_CNTL) & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
+
+        WREG32_P(FVTHROT_FBDIV_REG1, STARTING_FEEDBACK_DIV(fbdiv),
+                 ~STARTING_FEEDBACK_DIV_MASK);
+
+        WREG32_P(FVTHROT_FBDIV_REG2, FORCED_FEEDBACK_DIV(fbdiv),
+                 ~FORCED_FEEDBACK_DIV_MASK);
+
+        WREG32_P(FVTHROT_FBDIV_REG1, FORCE_FEEDBACK_DIV, ~FORCE_FEEDBACK_DIV);
+}
+
+static void rs780_voltage_scaling_init(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+        struct drm_device *dev = rdev->ddev;
+        u32 fv_throt_pwm_fb_div_range[3];
+        u32 fv_throt_pwm_range[4];
+
+        if (dev->pdev->device == 0x9614) {
+                fv_throt_pwm_fb_div_range[0] = RS780D_FVTHROTPWMFBDIVRANGEREG0_DFLT;
+                fv_throt_pwm_fb_div_range[1] = RS780D_FVTHROTPWMFBDIVRANGEREG1_DFLT;
+                fv_throt_pwm_fb_div_range[2] = RS780D_FVTHROTPWMFBDIVRANGEREG2_DFLT;
+        } else if ((dev->pdev->device == 0x9714) ||
+                   (dev->pdev->device == 0x9715)) {
+                fv_throt_pwm_fb_div_range[0] = RS880D_FVTHROTPWMFBDIVRANGEREG0_DFLT;
+                fv_throt_pwm_fb_div_range[1] = RS880D_FVTHROTPWMFBDIVRANGEREG1_DFLT;
+                fv_throt_pwm_fb_div_range[2] = RS880D_FVTHROTPWMFBDIVRANGEREG2_DFLT;
+        } else {
+                fv_throt_pwm_fb_div_range[0] = RS780_FVTHROTPWMFBDIVRANGEREG0_DFLT;
+                fv_throt_pwm_fb_div_range[1] = RS780_FVTHROTPWMFBDIVRANGEREG1_DFLT;
+                fv_throt_pwm_fb_div_range[2] = RS780_FVTHROTPWMFBDIVRANGEREG2_DFLT;
+        }
+
+        if (pi->pwm_voltage_control) {
+                fv_throt_pwm_range[0] = pi->min_voltage;
+                fv_throt_pwm_range[1] = pi->min_voltage;
+                fv_throt_pwm_range[2] = pi->max_voltage;
+                fv_throt_pwm_range[3] = pi->max_voltage;
+        } else {
+                fv_throt_pwm_range[0] = pi->invert_pwm_required ?
+                        RS780_FVTHROTPWMRANGE3_GPIO_DFLT : RS780_FVTHROTPWMRANGE0_GPIO_DFLT;
+                fv_throt_pwm_range[1] = pi->invert_pwm_required ?
+                        RS780_FVTHROTPWMRANGE2_GPIO_DFLT : RS780_FVTHROTPWMRANGE1_GPIO_DFLT;
+                fv_throt_pwm_range[2] = pi->invert_pwm_required ?
+                        RS780_FVTHROTPWMRANGE1_GPIO_DFLT : RS780_FVTHROTPWMRANGE2_GPIO_DFLT;
+                fv_throt_pwm_range[3] = pi->invert_pwm_required ?
+                        RS780_FVTHROTPWMRANGE0_GPIO_DFLT : RS780_FVTHROTPWMRANGE3_GPIO_DFLT;
+        }
+
+        WREG32_P(FVTHROT_PWM_CTRL_REG0,
+                 STARTING_PWM_HIGHTIME(pi->max_voltage),
+                 ~STARTING_PWM_HIGHTIME_MASK);
+
+        WREG32_P(FVTHROT_PWM_CTRL_REG0,
+                 NUMBER_OF_CYCLES_IN_PERIOD(pi->num_of_cycles_in_period),
+                 ~NUMBER_OF_CYCLES_IN_PERIOD_MASK);
+
+        WREG32_P(FVTHROT_PWM_CTRL_REG0, FORCE_STARTING_PWM_HIGHTIME,
+                 ~FORCE_STARTING_PWM_HIGHTIME);
+
+        if (pi->invert_pwm_required)
+                WREG32_P(FVTHROT_PWM_CTRL_REG0, INVERT_PWM_WAVEFORM, ~INVERT_PWM_WAVEFORM);
+        else
+                WREG32_P(FVTHROT_PWM_CTRL_REG0, 0, ~INVERT_PWM_WAVEFORM);
+
+        rs780_voltage_scaling_enable(rdev, true);
+
+        WREG32(FVTHROT_PWM_CTRL_REG1,
+               (MIN_PWM_HIGHTIME(pi->min_voltage) |
+                MAX_PWM_HIGHTIME(pi->max_voltage)));
+
+        WREG32(FVTHROT_PWM_US_REG0, RS780_FVTHROTPWMUSREG0_DFLT);
+        WREG32(FVTHROT_PWM_US_REG1, RS780_FVTHROTPWMUSREG1_DFLT);
+        WREG32(FVTHROT_PWM_DS_REG0, RS780_FVTHROTPWMDSREG0_DFLT);
+        WREG32(FVTHROT_PWM_DS_REG1, RS780_FVTHROTPWMDSREG1_DFLT);
+
+        WREG32_P(FVTHROT_PWM_FEEDBACK_DIV_REG1,
+                 RANGE0_PWM_FEEDBACK_DIV(fv_throt_pwm_fb_div_range[0]),
+                 ~RANGE0_PWM_FEEDBACK_DIV_MASK);
+
+        WREG32(FVTHROT_PWM_FEEDBACK_DIV_REG2,
+               (RANGE1_PWM_FEEDBACK_DIV(fv_throt_pwm_fb_div_range[1]) |
+                RANGE2_PWM_FEEDBACK_DIV(fv_throt_pwm_fb_div_range[2])));
+
+        WREG32(FVTHROT_PWM_FEEDBACK_DIV_REG3,
+               (RANGE0_PWM(fv_throt_pwm_range[1]) |
+                RANGE1_PWM(fv_throt_pwm_range[2])));
+        WREG32(FVTHROT_PWM_FEEDBACK_DIV_REG4,
+               (RANGE2_PWM(fv_throt_pwm_range[1]) |
+                RANGE3_PWM(fv_throt_pwm_range[2])));
+}
+
+static void rs780_clk_scaling_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(FVTHROT_CNTRL_REG, ENABLE_FV_THROT | ENABLE_FV_UPDATE,
+                         ~(ENABLE_FV_THROT | ENABLE_FV_UPDATE));
+        else
+                WREG32_P(FVTHROT_CNTRL_REG, 0,
+                         ~(ENABLE_FV_THROT | ENABLE_FV_UPDATE));
+}
+
+static void rs780_voltage_scaling_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(FVTHROT_CNTRL_REG, ENABLE_FV_THROT_IO, ~ENABLE_FV_THROT_IO);
+        else
+                WREG32_P(FVTHROT_CNTRL_REG, 0, ~ENABLE_FV_THROT_IO);
+}
+
+static void rs780_set_engine_clock_wfc(struct radeon_device *rdev)
+{
+        WREG32(FVTHROT_UTC0, RS780_FVTHROTUTC0_DFLT);
+        WREG32(FVTHROT_UTC1, RS780_FVTHROTUTC1_DFLT);
+        WREG32(FVTHROT_UTC2, RS780_FVTHROTUTC2_DFLT);
+        WREG32(FVTHROT_UTC3, RS780_FVTHROTUTC3_DFLT);
+        WREG32(FVTHROT_UTC4, RS780_FVTHROTUTC4_DFLT);
+
+        WREG32(FVTHROT_DTC0, RS780_FVTHROTDTC0_DFLT);
+        WREG32(FVTHROT_DTC1, RS780_FVTHROTDTC1_DFLT);
+        WREG32(FVTHROT_DTC2, RS780_FVTHROTDTC2_DFLT);
+        WREG32(FVTHROT_DTC3, RS780_FVTHROTDTC3_DFLT);
+        WREG32(FVTHROT_DTC4, RS780_FVTHROTDTC4_DFLT);
+}
+
+static void rs780_set_engine_clock_sc(struct radeon_device *rdev)
+{
+        WREG32_P(FVTHROT_FBDIV_REG2,
+                 FB_DIV_TIMER_VAL(RS780_FBDIVTIMERVAL_DFLT),
+                 ~FB_DIV_TIMER_VAL_MASK);
+
+        WREG32_P(FVTHROT_CNTRL_REG,
+                 REFRESH_RATE_DIVISOR(0) | MINIMUM_CIP(0xf),
+                 ~(REFRESH_RATE_DIVISOR_MASK | MINIMUM_CIP_MASK));
+}
+
+static void rs780_set_engine_clock_tdc(struct radeon_device *rdev)
+{
+        WREG32_P(FVTHROT_CNTRL_REG, 0, ~(FORCE_TREND_SEL | TREND_SEL_MODE));
+}
+
+static void rs780_set_engine_clock_ssc(struct radeon_device *rdev)
+{
+        WREG32(FVTHROT_FB_US_REG0, RS780_FVTHROTFBUSREG0_DFLT);
+        WREG32(FVTHROT_FB_US_REG1, RS780_FVTHROTFBUSREG1_DFLT);
+        WREG32(FVTHROT_FB_DS_REG0, RS780_FVTHROTFBDSREG0_DFLT);
+        WREG32(FVTHROT_FB_DS_REG1, RS780_FVTHROTFBDSREG1_DFLT);
+
+        WREG32_P(FVTHROT_FBDIV_REG1, MAX_FEEDBACK_STEP(1), ~MAX_FEEDBACK_STEP_MASK);
+}
+
+static void rs780_program_at(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+
+        WREG32(FVTHROT_TARGET_REG, 30000000 / pi->refresh_rate);
+        WREG32(FVTHROT_CB1, 1000000 * 5 / pi->refresh_rate);
+        WREG32(FVTHROT_CB2, 1000000 * 10 / pi->refresh_rate);
+        WREG32(FVTHROT_CB3, 1000000 * 30 / pi->refresh_rate);
+        WREG32(FVTHROT_CB4, 1000000 * 50 / pi->refresh_rate);
+}
+
+static void rs780_disable_vbios_powersaving(struct radeon_device *rdev)
+{
+        WREG32_P(CG_INTGFX_MISC, 0, ~0xFFF00000);
+}
+
+static void rs780_force_voltage_to_high(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+        struct igp_ps *current_state = rs780_get_ps(rdev->pm.dpm.current_ps);
+
+        if ((current_state->max_voltage == RS780_VDDC_LEVEL_HIGH) &&
+            (current_state->min_voltage == RS780_VDDC_LEVEL_HIGH))
+                return;
+
+        WREG32_P(GFX_MACRO_BYPASS_CNTL, SPLL_BYPASS_CNTL, ~SPLL_BYPASS_CNTL);
+
+        udelay(1);
+
+        WREG32_P(FVTHROT_PWM_CTRL_REG0,
+                 STARTING_PWM_HIGHTIME(pi->max_voltage),
+                 ~STARTING_PWM_HIGHTIME_MASK);
+
+        WREG32_P(FVTHROT_PWM_CTRL_REG0,
+                 FORCE_STARTING_PWM_HIGHTIME, ~FORCE_STARTING_PWM_HIGHTIME);
+
+        WREG32_P(FVTHROT_PWM_FEEDBACK_DIV_REG1, 0,
+                ~RANGE_PWM_FEEDBACK_DIV_EN);
+
+        udelay(1);
+
+        WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~SPLL_BYPASS_CNTL);
+}
+
+static int rs780_set_engine_clock_scaling(struct radeon_device *rdev,
+                                          struct radeon_ps *new_ps,
+                                          struct radeon_ps *old_ps)
+{
+        struct atom_clock_dividers min_dividers, max_dividers, current_max_dividers;
+        struct igp_ps *new_state = rs780_get_ps(new_ps);
+        struct igp_ps *old_state = rs780_get_ps(old_ps);
+        int ret;
+
+        if ((new_state->sclk_high == old_state->sclk_high) &&
+            (new_state->sclk_low == old_state->sclk_low))
+                return 0;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             new_state->sclk_low, false, &min_dividers);
+        if (ret)
+                return ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             new_state->sclk_high, false, &max_dividers);
+        if (ret)
+                return ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             old_state->sclk_high, false, &current_max_dividers);
+        if (ret)
+                return ret;
+
+        WREG32_P(GFX_MACRO_BYPASS_CNTL, SPLL_BYPASS_CNTL, ~SPLL_BYPASS_CNTL);
+
+        WREG32_P(FVTHROT_FBDIV_REG2, FORCED_FEEDBACK_DIV(max_dividers.fb_div),
+                 ~FORCED_FEEDBACK_DIV_MASK);
+        WREG32_P(FVTHROT_FBDIV_REG1, STARTING_FEEDBACK_DIV(max_dividers.fb_div),
+                 ~STARTING_FEEDBACK_DIV_MASK);
+        WREG32_P(FVTHROT_FBDIV_REG1, FORCE_FEEDBACK_DIV, ~FORCE_FEEDBACK_DIV);
+
+        udelay(100);
+
+        WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~SPLL_BYPASS_CNTL);
+
+        if (max_dividers.fb_div > min_dividers.fb_div) {
+                WREG32_P(FVTHROT_FBDIV_REG0,
+                         MIN_FEEDBACK_DIV(min_dividers.fb_div) |
+                         MAX_FEEDBACK_DIV(max_dividers.fb_div),
+                         ~(MIN_FEEDBACK_DIV_MASK | MAX_FEEDBACK_DIV_MASK));
+
+                WREG32_P(FVTHROT_FBDIV_REG1, 0, ~FORCE_FEEDBACK_DIV);
+        }
+
+        return 0;
+}
+
+static void rs780_set_engine_clock_spc(struct radeon_device *rdev,
+                                       struct radeon_ps *new_ps,
+                                       struct radeon_ps *old_ps)
+{
+        struct igp_ps *new_state = rs780_get_ps(new_ps);
+        struct igp_ps *old_state = rs780_get_ps(old_ps);
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+
+        if ((new_state->sclk_high == old_state->sclk_high) &&
+            (new_state->sclk_low == old_state->sclk_low))
+                return;
+
+        if (pi->crtc_id == 0)
+                WREG32_P(CG_INTGFX_MISC, 0, ~FVTHROT_VBLANK_SEL);
+        else
+                WREG32_P(CG_INTGFX_MISC, FVTHROT_VBLANK_SEL, ~FVTHROT_VBLANK_SEL);
+
+}
+
+static void rs780_activate_engine_clk_scaling(struct radeon_device *rdev,
+                                              struct radeon_ps *new_ps,
+                                              struct radeon_ps *old_ps)
+{
+        struct igp_ps *new_state = rs780_get_ps(new_ps);
+        struct igp_ps *old_state = rs780_get_ps(old_ps);
+
+        if ((new_state->sclk_high == old_state->sclk_high) &&
+            (new_state->sclk_low == old_state->sclk_low))
+                return;
+
+        rs780_clk_scaling_enable(rdev, true);
+}
+
+static u32 rs780_get_voltage_for_vddc_level(struct radeon_device *rdev,
+                                            enum rs780_vddc_level vddc)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+
+        if (vddc == RS780_VDDC_LEVEL_HIGH)
+                return pi->max_voltage;
+        else if (vddc == RS780_VDDC_LEVEL_LOW)
+                return pi->min_voltage;
+        else
+                return pi->max_voltage;
+}
+
+static void rs780_enable_voltage_scaling(struct radeon_device *rdev,
+                                         struct radeon_ps *new_ps)
+{
+        struct igp_ps *new_state = rs780_get_ps(new_ps);
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+        enum rs780_vddc_level vddc_high, vddc_low;
+
+        udelay(100);
+
+        if ((new_state->max_voltage == RS780_VDDC_LEVEL_HIGH) &&
+            (new_state->min_voltage == RS780_VDDC_LEVEL_HIGH))
+                return;
+
+        vddc_high = rs780_get_voltage_for_vddc_level(rdev,
+                                                     new_state->max_voltage);
+        vddc_low = rs780_get_voltage_for_vddc_level(rdev,
+                                                    new_state->min_voltage);
+
+        WREG32_P(GFX_MACRO_BYPASS_CNTL, SPLL_BYPASS_CNTL, ~SPLL_BYPASS_CNTL);
+
+        udelay(1);
+        if (vddc_high > vddc_low) {
+                WREG32_P(FVTHROT_PWM_FEEDBACK_DIV_REG1,
+                         RANGE_PWM_FEEDBACK_DIV_EN, ~RANGE_PWM_FEEDBACK_DIV_EN);
+
+                WREG32_P(FVTHROT_PWM_CTRL_REG0, 0, ~FORCE_STARTING_PWM_HIGHTIME);
+        } else if (vddc_high == vddc_low) {
+                if (pi->max_voltage != vddc_high) {
+                        WREG32_P(FVTHROT_PWM_CTRL_REG0,
+                                 STARTING_PWM_HIGHTIME(vddc_high),
+                                 ~STARTING_PWM_HIGHTIME_MASK);
+
+                        WREG32_P(FVTHROT_PWM_CTRL_REG0,
+                                 FORCE_STARTING_PWM_HIGHTIME,
+                                 ~FORCE_STARTING_PWM_HIGHTIME);
+                }
+        }
+
+        WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~SPLL_BYPASS_CNTL);
+}
+
+static void rs780_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                                     struct radeon_ps *new_ps,
+                                                     struct radeon_ps *old_ps)
+{
+        struct igp_ps *new_state = rs780_get_ps(new_ps);
+        struct igp_ps *current_state = rs780_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->sclk_high >= current_state->sclk_high)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+static void rs780_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                                    struct radeon_ps *new_ps,
+                                                    struct radeon_ps *old_ps)
+{
+        struct igp_ps *new_state = rs780_get_ps(new_ps);
+        struct igp_ps *current_state = rs780_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->sclk_high < current_state->sclk_high)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+int rs780_dpm_enable(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        rs780_get_pm_mode_parameters(rdev);
+        rs780_disable_vbios_powersaving(rdev);
+
+        if (r600_dynamicpm_enabled(rdev))
+                return -EINVAL;
+        ret = rs780_initialize_dpm_parameters(rdev, boot_ps);
+        if (ret)
+                return ret;
+        rs780_start_dpm(rdev);
+
+        rs780_preset_ranges_slow_clk_fbdiv_en(rdev);
+        rs780_preset_starting_fbdiv(rdev);
+        if (pi->voltage_control)
+                rs780_voltage_scaling_init(rdev);
+        rs780_clk_scaling_enable(rdev, true);
+        rs780_set_engine_clock_sc(rdev);
+        rs780_set_engine_clock_wfc(rdev);
+        rs780_program_at(rdev);
+        rs780_set_engine_clock_tdc(rdev);
+        rs780_set_engine_clock_ssc(rdev);
+
+        if (pi->gfx_clock_gating)
+                r600_gfx_clockgating_enable(rdev, true);
+
+        if (rdev->irq.installed && (rdev->pm.int_thermal_type == THERMAL_TYPE_RV6XX)) {
+                ret = r600_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+        }
+
+        return 0;
+}
+
+void rs780_dpm_disable(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+
+        r600_dynamicpm_enable(rdev, false);
+
+        rs780_clk_scaling_enable(rdev, false);
+        rs780_voltage_scaling_enable(rdev, false);
+
+        if (pi->gfx_clock_gating)
+                r600_gfx_clockgating_enable(rdev, false);
+
+        if (rdev->irq.installed &&
+            (rdev->pm.int_thermal_type == THERMAL_TYPE_RV6XX)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+}
+
+int rs780_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+        struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
+        struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
+        int ret;
+
+        rs780_get_pm_mode_parameters(rdev);
+
+        rs780_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+
+        if (pi->voltage_control) {
+                rs780_force_voltage_to_high(rdev);
+                mdelay(5);
+        }
+
+        ret = rs780_set_engine_clock_scaling(rdev, new_ps, old_ps);
+        if (ret)
+                return ret;
+        rs780_set_engine_clock_spc(rdev, new_ps, old_ps);
+
+        rs780_activate_engine_clk_scaling(rdev, new_ps, old_ps);
+
+        if (pi->voltage_control)
+                rs780_enable_voltage_scaling(rdev, new_ps);
+
+        rs780_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+
+        return 0;
+}
+
+void rs780_dpm_setup_asic(struct radeon_device *rdev)
+{
+
+}
+
+void rs780_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+        rs780_get_pm_mode_parameters(rdev);
+        rs780_program_at(rdev);
+}
+
+union igp_info {
+        struct _ATOM_INTEGRATED_SYSTEM_INFO info;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
+};
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void rs780_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                             struct radeon_ps *rps,
+                                             struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+                                             u8 table_rev)
+{
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+        } else if (r600_is_uvd_state(rps->class, rps->class2)) {
+                rps->vclk = RS780_DEFAULT_VCLK_FREQ;
+                rps->dclk = RS780_DEFAULT_DCLK_FREQ;
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+                rdev->pm.dpm.boot_ps = rps;
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void rs780_parse_pplib_clock_info(struct radeon_device *rdev,
+                                         struct radeon_ps *rps,
+                                         union pplib_clock_info *clock_info)
+{
+        struct igp_ps *ps = rs780_get_ps(rps);
+        u32 sclk;
+
+        sclk = le16_to_cpu(clock_info->rs780.usLowEngineClockLow);
+        sclk |= clock_info->rs780.ucLowEngineClockHigh << 16;
+        ps->sclk_low = sclk;
+        sclk = le16_to_cpu(clock_info->rs780.usHighEngineClockLow);
+        sclk |= clock_info->rs780.ucHighEngineClockHigh << 16;
+        ps->sclk_high = sclk;
+        switch (le16_to_cpu(clock_info->rs780.usVDDC)) {
+        case ATOM_PPLIB_RS780_VOLTAGE_NONE:
+        default:
+                ps->min_voltage = RS780_VDDC_LEVEL_UNKNOWN;
+                ps->max_voltage = RS780_VDDC_LEVEL_UNKNOWN;
+                break;
+        case ATOM_PPLIB_RS780_VOLTAGE_LOW:
+                ps->min_voltage = RS780_VDDC_LEVEL_LOW;
+                ps->max_voltage = RS780_VDDC_LEVEL_LOW;
+                break;
+        case ATOM_PPLIB_RS780_VOLTAGE_HIGH:
+                ps->min_voltage = RS780_VDDC_LEVEL_HIGH;
+                ps->max_voltage = RS780_VDDC_LEVEL_HIGH;
+                break;
+        case ATOM_PPLIB_RS780_VOLTAGE_VARIABLE:
+                ps->min_voltage = RS780_VDDC_LEVEL_LOW;
+                ps->max_voltage = RS780_VDDC_LEVEL_HIGH;
+                break;
+        }
+        ps->flags = le32_to_cpu(clock_info->rs780.ulFlags);
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                ps->sclk_low = rdev->clock.default_sclk;
+                ps->sclk_high = rdev->clock.default_sclk;
+                ps->min_voltage = RS780_VDDC_LEVEL_HIGH;
+                ps->max_voltage = RS780_VDDC_LEVEL_HIGH;
+        }
+}
+
+static int rs780_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i;
+        union pplib_clock_info *clock_info;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        struct igp_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  power_info->pplib.ucNumStates, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+
+        for (i = 0; i < power_info->pplib.ucNumStates; i++) {
+                power_state = (union pplib_power_state *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usStateArrayOffset) +
+                         i * power_info->pplib.ucStateEntrySize);
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
+                         (power_state->v1.ucNonClockStateIndex *
+                          power_info->pplib.ucNonClockSize));
+                if (power_info->pplib.ucStateEntrySize - 1) {
+                        clock_info = (union pplib_clock_info *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
+                                 (power_state->v1.ucClockStateIndices[0] *
+                                  power_info->pplib.ucClockInfoSize));
+                        ps = kzalloc(sizeof(struct igp_ps), GFP_KERNEL);
+                        if (ps == NULL) {
+                                kfree(rdev->pm.dpm.ps);
+                                return -ENOMEM;
+                        }
+                        rdev->pm.dpm.ps[i].ps_priv = ps;
+                        rs780_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                                         non_clock_info,
+                                                         power_info->pplib.ucNonClockSize);
+                        rs780_parse_pplib_clock_info(rdev,
+                                                     &rdev->pm.dpm.ps[i],
+                                                     clock_info);
+                }
+        }
+        rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
+        return 0;
+}
+
+int rs780_dpm_init(struct radeon_device *rdev)
+{
+        struct igp_power_info *pi;
+        int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
+        union igp_info *info;
+        u16 data_offset;
+        u8 frev, crev;
+        int ret;
+
+        pi = kzalloc(sizeof(struct igp_power_info), GFP_KERNEL);
+        if (pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = pi;
+
+        ret = rs780_parse_power_table(rdev);
+        if (ret)
+                return ret;
+
+        pi->voltage_control = false;
+        pi->gfx_clock_gating = true;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, NULL,
+                                   &frev, &crev, &data_offset)) {
+                info = (union igp_info *)(rdev->mode_info.atom_context->bios + data_offset);
+
+                /* Get various system informations from bios */
+                switch (crev) {
+                case 1:
+                        pi->num_of_cycles_in_period =
+                                info->info.ucNumberOfCyclesInPeriod;
+                        pi->num_of_cycles_in_period |=
+                                info->info.ucNumberOfCyclesInPeriodHi << 8;
+                        pi->invert_pwm_required =
+                                (pi->num_of_cycles_in_period & 0x8000) ? true : false;
+                        pi->boot_voltage = info->info.ucStartingPWM_HighTime;
+                        pi->max_voltage = info->info.ucMaxNBVoltage;
+                        pi->max_voltage |= info->info.ucMaxNBVoltageHigh << 8;
+                        pi->min_voltage = info->info.ucMinNBVoltage;
+                        pi->min_voltage |= info->info.ucMinNBVoltageHigh << 8;
+                        pi->inter_voltage_low =
+                                le16_to_cpu(info->info.usInterNBVoltageLow);
+                        pi->inter_voltage_high =
+                                le16_to_cpu(info->info.usInterNBVoltageHigh);
+                        pi->voltage_control = true;
+                        pi->bootup_uma_clk = info->info.usK8MemoryClock * 100;
+                        break;
+                case 2:
+                        pi->num_of_cycles_in_period =
+                                le16_to_cpu(info->info_2.usNumberOfCyclesInPeriod);
+                        pi->invert_pwm_required =
+                                (pi->num_of_cycles_in_period & 0x8000) ? true : false;
+                        pi->boot_voltage =
+                                le16_to_cpu(info->info_2.usBootUpNBVoltage);
+                        pi->max_voltage =
+                                le16_to_cpu(info->info_2.usMaxNBVoltage);
+                        pi->min_voltage =
+                                le16_to_cpu(info->info_2.usMinNBVoltage);
+                        pi->system_config =
+                                le32_to_cpu(info->info_2.ulSystemConfig);
+                        pi->pwm_voltage_control =
+                                (pi->system_config & 0x4) ? true : false;
+                        pi->voltage_control = true;
+                        pi->bootup_uma_clk = le32_to_cpu(info->info_2.ulBootUpUMAClock);
+                        break;
+                default:
+                        DRM_ERROR("No integrated system info for your GPU\n");
+                        return -EINVAL;
+                }
+                if (pi->min_voltage > pi->max_voltage)
+                        pi->voltage_control = false;
+                if (pi->pwm_voltage_control) {
+                        if ((pi->num_of_cycles_in_period == 0) ||
+                            (pi->max_voltage == 0) ||
+                            (pi->min_voltage == 0))
+                                pi->voltage_control = false;
+                } else {
+                        if ((pi->num_of_cycles_in_period == 0) ||
+                            (pi->max_voltage == 0))
+                                pi->voltage_control = false;
+                }
+
+                return 0;
+        }
+        radeon_dpm_fini(rdev);
+        return -EINVAL;
+}
+
+void rs780_dpm_print_power_state(struct radeon_device *rdev,
+                                 struct radeon_ps *rps)
+{
+        struct igp_ps *ps = rs780_get_ps(rps);
+
+        r600_dpm_print_class_info(rps->class, rps->class2);
+        r600_dpm_print_cap_info(rps->caps);
+        printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+        printk("\t\tpower level 0    sclk: %u vddc_index: %d\n",
+               ps->sclk_low, ps->min_voltage);
+        printk("\t\tpower level 1    sclk: %u vddc_index: %d\n",
+               ps->sclk_high, ps->max_voltage);
+        r600_dpm_print_ps_status(rdev, rps);
+}
+
+void rs780_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+}
+
+u32 rs780_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct igp_ps *requested_state = rs780_get_ps(rdev->pm.dpm.requested_ps);
+
+        if (low)
+                return requested_state->sclk_low;
+        else
+                return requested_state->sclk_high;
+}
+
+u32 rs780_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct igp_power_info *pi = rs780_get_pi(rdev);
+
+        return pi->bootup_uma_clk;
+}
diff --git a/drivers/gpu/drm/radeon/rs780_dpm.h b/drivers/gpu/drm/radeon/rs780_dpm.h
new file mode 100644
index 000000000000..47a40b14fa43
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rs780_dpm.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RS780_DPM_H__
+#define __RS780_DPM_H__
+
+enum rs780_vddc_level {
+        RS780_VDDC_LEVEL_UNKNOWN = 0,
+        RS780_VDDC_LEVEL_LOW = 1,
+        RS780_VDDC_LEVEL_HIGH = 2,
+};
+
+struct igp_power_info {
+        /* flags */
+        bool invert_pwm_required;
+        bool pwm_voltage_control;
+        bool voltage_control;
+        bool gfx_clock_gating;
+        /* stored values */
+        u32 system_config;
+        u32 bootup_uma_clk;
+        u16 max_voltage;
+        u16 min_voltage;
+        u16 boot_voltage;
+        u16 inter_voltage_low;
+        u16 inter_voltage_high;
+        u16 num_of_cycles_in_period;
+        /* variable */
+        int crtc_id;
+        int refresh_rate;
+};
+
+struct igp_ps {
+        enum rs780_vddc_level min_voltage;
+        enum rs780_vddc_level max_voltage;
+        u32 sclk_low;
+        u32 sclk_high;
+        u32 flags;
+};
+
+#define RS780_CGFTV_DFLT                 0x0303000f
+#define RS780_FBDIVTIMERVAL_DFLT         0x2710
+
+#define RS780_FVTHROTUTC0_DFLT   0x04010040
+#define RS780_FVTHROTUTC1_DFLT   0x04010040
+#define RS780_FVTHROTUTC2_DFLT   0x04010040
+#define RS780_FVTHROTUTC3_DFLT   0x04010040
+#define RS780_FVTHROTUTC4_DFLT   0x04010040
+
+#define RS780_FVTHROTDTC0_DFLT 0x04010040
+#define RS780_FVTHROTDTC1_DFLT 0x04010040
+#define RS780_FVTHROTDTC2_DFLT 0x04010040
+#define RS780_FVTHROTDTC3_DFLT 0x04010040
+#define RS780_FVTHROTDTC4_DFLT 0x04010040
+
+#define RS780_FVTHROTFBUSREG0_DFLT       0x00001001
+#define RS780_FVTHROTFBUSREG1_DFLT       0x00002002
+#define RS780_FVTHROTFBDSREG0_DFLT       0x00004001
+#define RS780_FVTHROTFBDSREG1_DFLT       0x00020010
+
+#define RS780_FVTHROTPWMUSREG0_DFLT      0x00002001
+#define RS780_FVTHROTPWMUSREG1_DFLT      0x00004003
+#define RS780_FVTHROTPWMDSREG0_DFLT      0x00002001
+#define RS780_FVTHROTPWMDSREG1_DFLT      0x00004003
+
+#define RS780_FVTHROTPWMFBDIVRANGEREG0_DFLT  0x37
+#define RS780_FVTHROTPWMFBDIVRANGEREG1_DFLT  0x4b
+#define RS780_FVTHROTPWMFBDIVRANGEREG2_DFLT  0x8b
+
+#define RS780D_FVTHROTPWMFBDIVRANGEREG0_DFLT  0x8b
+#define RS780D_FVTHROTPWMFBDIVRANGEREG1_DFLT  0x8c
+#define RS780D_FVTHROTPWMFBDIVRANGEREG2_DFLT  0xb5
+
+#define RS880D_FVTHROTPWMFBDIVRANGEREG0_DFLT  0x8d
+#define RS880D_FVTHROTPWMFBDIVRANGEREG1_DFLT  0x8e
+#define RS880D_FVTHROTPWMFBDIVRANGEREG2_DFLT  0xBa
+
+#define RS780_FVTHROTPWMRANGE0_GPIO_DFLT  0x1a
+#define RS780_FVTHROTPWMRANGE1_GPIO_DFLT  0x1a
+#define RS780_FVTHROTPWMRANGE2_GPIO_DFLT  0x0
+#define RS780_FVTHROTPWMRANGE3_GPIO_DFLT  0x0
+
+#define RS780_SLOWCLKFEEDBACKDIV_DFLT 110
+
+#define RS780_CGCLKGATING_DFLT           0x0000E204
+
+#define RS780_DEFAULT_VCLK_FREQ  53300 /* 10 khz */
+#define RS780_DEFAULT_DCLK_FREQ  40000 /* 10 khz */
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rs780d.h b/drivers/gpu/drm/radeon/rs780d.h
new file mode 100644
index 000000000000..b1142ed1c628
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rs780d.h
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RS780D_H__
+#define __RS780D_H__
+
+#define CG_SPLL_FUNC_CNTL                                 0x600
+#       define SPLL_RESET                                (1 << 0)
+#       define SPLL_SLEEP                                (1 << 1)
+#       define SPLL_REF_DIV(x)                           ((x) << 2)
+#       define SPLL_REF_DIV_MASK                         (7 << 2)
+#       define SPLL_FB_DIV(x)                            ((x) << 5)
+#       define SPLL_FB_DIV_MASK                          (0xff << 2)
+#       define SPLL_FB_DIV_SHIFT                         2
+#       define SPLL_PULSEEN                              (1 << 13)
+#       define SPLL_PULSENUM(x)                          ((x) << 14)
+#       define SPLL_PULSENUM_MASK                        (3 << 14)
+#       define SPLL_SW_HILEN(x)                          ((x) << 16)
+#       define SPLL_SW_HILEN_MASK                        (0xf << 16)
+#       define SPLL_SW_LOLEN(x)                          ((x) << 20)
+#       define SPLL_SW_LOLEN_MASK                        (0xf << 20)
+#       define SPLL_DIVEN                                (1 << 24)
+#       define SPLL_BYPASS_EN                            (1 << 25)
+#       define SPLL_CHG_STATUS                           (1 << 29)
+#       define SPLL_CTLREQ                               (1 << 30)
+#       define SPLL_CTLACK                               (1 << 31)
+
+/* RS780/RS880 PM */
+#define FVTHROT_CNTRL_REG                               0x3000
+#define         DONT_WAIT_FOR_FBDIV_WRAP                (1 << 0)
+#define         MINIMUM_CIP(x)                          ((x) << 1)
+#define         MINIMUM_CIP_SHIFT                       1
+#define         MINIMUM_CIP_MASK                        0x1fffffe
+#define         REFRESH_RATE_DIVISOR(x)                 ((x) << 25)
+#define         REFRESH_RATE_DIVISOR_SHIFT              25
+#define         REFRESH_RATE_DIVISOR_MASK               (0x3 << 25)
+#define         ENABLE_FV_THROT                         (1 << 27)
+#define         ENABLE_FV_UPDATE                        (1 << 28)
+#define         TREND_SEL_MODE                          (1 << 29)
+#define         FORCE_TREND_SEL                         (1 << 30)
+#define         ENABLE_FV_THROT_IO                      (1 << 31)
+#define FVTHROT_TARGET_REG                              0x3004
+#define         TARGET_IDLE_COUNT(x)                    ((x) << 0)
+#define         TARGET_IDLE_COUNT_MASK                  0xffffff
+#define         TARGET_IDLE_COUNT_SHIFT                 0
+#define FVTHROT_CB1                                     0x3008
+#define FVTHROT_CB2                                     0x300c
+#define FVTHROT_CB3                                     0x3010
+#define FVTHROT_CB4                                     0x3014
+#define FVTHROT_UTC0                                    0x3018
+#define FVTHROT_UTC1                                    0x301c
+#define FVTHROT_UTC2                                    0x3020
+#define FVTHROT_UTC3                                    0x3024
+#define FVTHROT_UTC4                                    0x3028
+#define FVTHROT_DTC0                                    0x302c
+#define FVTHROT_DTC1                                    0x3030
+#define FVTHROT_DTC2                                    0x3034
+#define FVTHROT_DTC3                                    0x3038
+#define FVTHROT_DTC4                                    0x303c
+#define FVTHROT_FBDIV_REG0                              0x3040
+#define         MIN_FEEDBACK_DIV(x)                     ((x) << 0)
+#define         MIN_FEEDBACK_DIV_MASK                   0xfff
+#define         MIN_FEEDBACK_DIV_SHIFT                  0
+#define         MAX_FEEDBACK_DIV(x)                     ((x) << 12)
+#define         MAX_FEEDBACK_DIV_MASK                   (0xfff << 12)
+#define         MAX_FEEDBACK_DIV_SHIFT                  12
+#define FVTHROT_FBDIV_REG1                              0x3044
+#define         MAX_FEEDBACK_STEP(x)                    ((x) << 0)
+#define         MAX_FEEDBACK_STEP_MASK                  0xfff
+#define         MAX_FEEDBACK_STEP_SHIFT                 0
+#define         STARTING_FEEDBACK_DIV(x)                ((x) << 12)
+#define         STARTING_FEEDBACK_DIV_MASK              (0xfff << 12)
+#define         STARTING_FEEDBACK_DIV_SHIFT             12
+#define         FORCE_FEEDBACK_DIV                      (1 << 24)
+#define FVTHROT_FBDIV_REG2                              0x3048
+#define         FORCED_FEEDBACK_DIV(x)                  ((x) << 0)
+#define         FORCED_FEEDBACK_DIV_MASK                0xfff
+#define         FORCED_FEEDBACK_DIV_SHIFT               0
+#define         FB_DIV_TIMER_VAL(x)                     ((x) << 12)
+#define         FB_DIV_TIMER_VAL_MASK                   (0xffff << 12)
+#define         FB_DIV_TIMER_VAL_SHIFT                  12
+#define FVTHROT_FB_US_REG0                              0x304c
+#define FVTHROT_FB_US_REG1                              0x3050
+#define FVTHROT_FB_DS_REG0                              0x3054
+#define FVTHROT_FB_DS_REG1                              0x3058
+#define FVTHROT_PWM_CTRL_REG0                           0x305c
+#define         STARTING_PWM_HIGHTIME(x)                ((x) << 0)
+#define         STARTING_PWM_HIGHTIME_MASK              0xfff
+#define         STARTING_PWM_HIGHTIME_SHIFT             0
+#define         NUMBER_OF_CYCLES_IN_PERIOD(x)           ((x) << 12)
+#define         NUMBER_OF_CYCLES_IN_PERIOD_MASK         (0xfff << 12)
+#define         NUMBER_OF_CYCLES_IN_PERIOD_SHIFT        12
+#define         FORCE_STARTING_PWM_HIGHTIME             (1 << 24)
+#define         INVERT_PWM_WAVEFORM                     (1 << 25)
+#define FVTHROT_PWM_CTRL_REG1                           0x3060
+#define         MIN_PWM_HIGHTIME(x)                     ((x) << 0)
+#define         MIN_PWM_HIGHTIME_MASK                   0xfff
+#define         MIN_PWM_HIGHTIME_SHIFT                  0
+#define         MAX_PWM_HIGHTIME(x)                     ((x) << 12)
+#define         MAX_PWM_HIGHTIME_MASK                   (0xfff << 12)
+#define         MAX_PWM_HIGHTIME_SHIFT                  12
+#define FVTHROT_PWM_US_REG0                             0x3064
+#define FVTHROT_PWM_US_REG1                             0x3068
+#define FVTHROT_PWM_DS_REG0                             0x306c
+#define FVTHROT_PWM_DS_REG1                             0x3070
+#define FVTHROT_STATUS_REG0                             0x3074
+#define         CURRENT_FEEDBACK_DIV_MASK               0xfff
+#define         CURRENT_FEEDBACK_DIV_SHIFT              0
+#define FVTHROT_STATUS_REG1                             0x3078
+#define FVTHROT_STATUS_REG2                             0x307c
+#define CG_INTGFX_MISC                                  0x3080
+#define         FVTHROT_VBLANK_SEL                      (1 << 9)
+#define FVTHROT_PWM_FEEDBACK_DIV_REG1                   0x308c
+#define         RANGE0_PWM_FEEDBACK_DIV(x)              ((x) << 0)
+#define         RANGE0_PWM_FEEDBACK_DIV_MASK            0xfff
+#define         RANGE0_PWM_FEEDBACK_DIV_SHIFT           0
+#define         RANGE_PWM_FEEDBACK_DIV_EN               (1 << 12)
+#define FVTHROT_PWM_FEEDBACK_DIV_REG2                   0x3090
+#define         RANGE1_PWM_FEEDBACK_DIV(x)              ((x) << 0)
+#define         RANGE1_PWM_FEEDBACK_DIV_MASK            0xfff
+#define         RANGE1_PWM_FEEDBACK_DIV_SHIFT           0
+#define         RANGE2_PWM_FEEDBACK_DIV(x)              ((x) << 12)
+#define         RANGE2_PWM_FEEDBACK_DIV_MASK            (0xfff << 12)
+#define         RANGE2_PWM_FEEDBACK_DIV_SHIFT           12
+#define FVTHROT_PWM_FEEDBACK_DIV_REG3                   0x3094
+#define         RANGE0_PWM(x)                           ((x) << 0)
+#define         RANGE0_PWM_MASK                         0xfff
+#define         RANGE0_PWM_SHIFT                        0
+#define         RANGE1_PWM(x)                           ((x) << 12)
+#define         RANGE1_PWM_MASK                         (0xfff << 12)
+#define         RANGE1_PWM_SHIFT                        12
+#define FVTHROT_PWM_FEEDBACK_DIV_REG4                   0x3098
+#define         RANGE2_PWM(x)                           ((x) << 0)
+#define         RANGE2_PWM_MASK                         0xfff
+#define         RANGE2_PWM_SHIFT                        0
+#define         RANGE3_PWM(x)                           ((x) << 12)
+#define         RANGE3_PWM_MASK                         (0xfff << 12)
+#define         RANGE3_PWM_SHIFT                        12
+#define FVTHROT_SLOW_CLK_FEEDBACK_DIV_REG1              0x30ac
+#define         RANGE0_SLOW_CLK_FEEDBACK_DIV(x)         ((x) << 0)
+#define         RANGE0_SLOW_CLK_FEEDBACK_DIV_MASK       0xfff
+#define         RANGE0_SLOW_CLK_FEEDBACK_DIV_SHIFT      0
+#define         RANGE_SLOW_CLK_FEEDBACK_DIV_EN          (1 << 12)
+
+#define GFX_MACRO_BYPASS_CNTL                           0x30c0
+#define         SPLL_BYPASS_CNTL                        (1 << 0)
+#define         UPLL_BYPASS_CNTL                        (1 << 1)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 21c7d7b26e55..8ea1573ae820 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -937,13 +937,16 @@ struct rv515_watermark {
 };
 
 static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev,
-                                  struct radeon_crtc *crtc,
-                                  struct rv515_watermark *wm)
+                                         struct radeon_crtc *crtc,
+                                         struct rv515_watermark *wm,
+                                         bool low)
 {
         struct drm_display_mode *mode = &crtc->base.mode;
         fixed20_12 a, b, c;
         fixed20_12 pclk, request_fifo_depth, tolerable_latency, estimated_width;
         fixed20_12 consumption_time, line_time, chunk_time, read_delay_latency;
+        fixed20_12 sclk;
+        u32 selected_sclk;
 
         if (!crtc->base.enabled) {
                 /* FIXME: wouldn't it better to set priority mark to maximum */
@@ -951,6 +954,18 @@ static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev,
                 return;
         }
 
+        /* rv6xx, rv7xx */
+        if ((rdev->family >= CHIP_RV610) &&
+            (rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
+                selected_sclk = radeon_dpm_get_sclk(rdev, low);
+        else
+                selected_sclk = rdev->pm.current_sclk;
+
+        /* sclk in Mhz */
+        a.full = dfixed_const(100);
+        sclk.full = dfixed_const(selected_sclk);
+        sclk.full = dfixed_div(sclk, a);
+
         if (crtc->vsc.full > dfixed_const(2))
                 wm->num_line_pair.full = dfixed_const(2);
         else
@@ -1016,7 +1031,7 @@ static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev,
          * sclk = system clock(Mhz)
          */
         a.full = dfixed_const(600 * 1000);
-        chunk_time.full = dfixed_div(a, rdev->pm.sclk);
+        chunk_time.full = dfixed_div(a, sclk);
         read_delay_latency.full = dfixed_const(1000);
 
         /* Determine the worst case latency
@@ -1077,152 +1092,177 @@ static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev,
         }
 }
 
-void rv515_bandwidth_avivo_update(struct radeon_device *rdev)
+static void rv515_compute_mode_priority(struct radeon_device *rdev,
+                                        struct rv515_watermark *wm0,
+                                        struct rv515_watermark *wm1,
+                                        struct drm_display_mode *mode0,
+                                        struct drm_display_mode *mode1,
+                                        u32 *d1mode_priority_a_cnt,
+                                        u32 *d2mode_priority_a_cnt)
 {
-        struct drm_display_mode *mode0 = NULL;
-        struct drm_display_mode *mode1 = NULL;
-        struct rv515_watermark wm0;
-        struct rv515_watermark wm1;
-        u32 tmp;
-        u32 d1mode_priority_a_cnt = MODE_PRIORITY_OFF;
-        u32 d2mode_priority_a_cnt = MODE_PRIORITY_OFF;
         fixed20_12 priority_mark02, priority_mark12, fill_rate;
         fixed20_12 a, b;
 
-        if (rdev->mode_info.crtcs[0]->base.enabled)
-                mode0 = &rdev->mode_info.crtcs[0]->base.mode;
-        if (rdev->mode_info.crtcs[1]->base.enabled)
-                mode1 = &rdev->mode_info.crtcs[1]->base.mode;
-        rs690_line_buffer_adjust(rdev, mode0, mode1);
-
-        rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0);
-        rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1);
-
-        tmp = wm0.lb_request_fifo_depth;
-        tmp |= wm1.lb_request_fifo_depth << 16;
-        WREG32(LB_MAX_REQ_OUTSTANDING, tmp);
+        *d1mode_priority_a_cnt = MODE_PRIORITY_OFF;
+        *d2mode_priority_a_cnt = MODE_PRIORITY_OFF;
 
         if (mode0 && mode1) {
-                if (dfixed_trunc(wm0.dbpp) > 64)
-                        a.full = dfixed_div(wm0.dbpp, wm0.num_line_pair);
+                if (dfixed_trunc(wm0->dbpp) > 64)
+                        a.full = dfixed_div(wm0->dbpp, wm0->num_line_pair);
                 else
-                        a.full = wm0.num_line_pair.full;
-                if (dfixed_trunc(wm1.dbpp) > 64)
-                        b.full = dfixed_div(wm1.dbpp, wm1.num_line_pair);
+                        a.full = wm0->num_line_pair.full;
+                if (dfixed_trunc(wm1->dbpp) > 64)
+                        b.full = dfixed_div(wm1->dbpp, wm1->num_line_pair);
                 else
-                        b.full = wm1.num_line_pair.full;
+                        b.full = wm1->num_line_pair.full;
                 a.full += b.full;
-                fill_rate.full = dfixed_div(wm0.sclk, a);
-                if (wm0.consumption_rate.full > fill_rate.full) {
-                        b.full = wm0.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm0.active_time);
+                fill_rate.full = dfixed_div(wm0->sclk, a);
+                if (wm0->consumption_rate.full > fill_rate.full) {
+                        b.full = wm0->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm0->active_time);
                         a.full = dfixed_const(16);
                         b.full = dfixed_div(b, a);
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         priority_mark02.full = a.full + b.full;
                 } else {
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark02.full = dfixed_div(a, b);
                 }
-                if (wm1.consumption_rate.full > fill_rate.full) {
-                        b.full = wm1.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm1.active_time);
+                if (wm1->consumption_rate.full > fill_rate.full) {
+                        b.full = wm1->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm1->active_time);
                         a.full = dfixed_const(16);
                         b.full = dfixed_div(b, a);
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         priority_mark12.full = a.full + b.full;
                 } else {
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark12.full = dfixed_div(a, b);
                 }
-                if (wm0.priority_mark.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark.full;
+                if (wm0->priority_mark.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark.full;
                 if (dfixed_trunc(priority_mark02) < 0)
                         priority_mark02.full = 0;
-                if (wm0.priority_mark_max.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark_max.full;
-                if (wm1.priority_mark.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark.full;
+                if (wm0->priority_mark_max.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark_max.full;
+                if (wm1->priority_mark.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark.full;
                 if (dfixed_trunc(priority_mark12) < 0)
                         priority_mark12.full = 0;
-                if (wm1.priority_mark_max.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark_max.full;
-                d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
-                d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
+                if (wm1->priority_mark_max.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark_max.full;
+                *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
+                *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
                 if (rdev->disp_priority == 2) {
-                        d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
-                        d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
+                        *d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
+                        *d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
                 }
         } else if (mode0) {
-                if (dfixed_trunc(wm0.dbpp) > 64)
-                        a.full = dfixed_div(wm0.dbpp, wm0.num_line_pair);
+                if (dfixed_trunc(wm0->dbpp) > 64)
+                        a.full = dfixed_div(wm0->dbpp, wm0->num_line_pair);
                 else
-                        a.full = wm0.num_line_pair.full;
-                fill_rate.full = dfixed_div(wm0.sclk, a);
-                if (wm0.consumption_rate.full > fill_rate.full) {
-                        b.full = wm0.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm0.active_time);
+                        a.full = wm0->num_line_pair.full;
+                fill_rate.full = dfixed_div(wm0->sclk, a);
+                if (wm0->consumption_rate.full > fill_rate.full) {
+                        b.full = wm0->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm0->active_time);
                         a.full = dfixed_const(16);
                         b.full = dfixed_div(b, a);
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         priority_mark02.full = a.full + b.full;
                 } else {
-                        a.full = dfixed_mul(wm0.worst_case_latency,
-                                                wm0.consumption_rate);
+                        a.full = dfixed_mul(wm0->worst_case_latency,
+                                                wm0->consumption_rate);
                         b.full = dfixed_const(16);
                         priority_mark02.full = dfixed_div(a, b);
                 }
-                if (wm0.priority_mark.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark.full;
+                if (wm0->priority_mark.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark.full;
                 if (dfixed_trunc(priority_mark02) < 0)
                         priority_mark02.full = 0;
-                if (wm0.priority_mark_max.full > priority_mark02.full)
-                        priority_mark02.full = wm0.priority_mark_max.full;
-                d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
+                if (wm0->priority_mark_max.full > priority_mark02.full)
+                        priority_mark02.full = wm0->priority_mark_max.full;
+                *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
                 if (rdev->disp_priority == 2)
-                        d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
+                        *d1mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
         } else if (mode1) {
-                if (dfixed_trunc(wm1.dbpp) > 64)
-                        a.full = dfixed_div(wm1.dbpp, wm1.num_line_pair);
+                if (dfixed_trunc(wm1->dbpp) > 64)
+                        a.full = dfixed_div(wm1->dbpp, wm1->num_line_pair);
                 else
-                        a.full = wm1.num_line_pair.full;
-                fill_rate.full = dfixed_div(wm1.sclk, a);
-                if (wm1.consumption_rate.full > fill_rate.full) {
-                        b.full = wm1.consumption_rate.full - fill_rate.full;
-                        b.full = dfixed_mul(b, wm1.active_time);
+                        a.full = wm1->num_line_pair.full;
+                fill_rate.full = dfixed_div(wm1->sclk, a);
+                if (wm1->consumption_rate.full > fill_rate.full) {
+                        b.full = wm1->consumption_rate.full - fill_rate.full;
+                        b.full = dfixed_mul(b, wm1->active_time);
                         a.full = dfixed_const(16);
                         b.full = dfixed_div(b, a);
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         priority_mark12.full = a.full + b.full;
                 } else {
-                        a.full = dfixed_mul(wm1.worst_case_latency,
-                                                wm1.consumption_rate);
+                        a.full = dfixed_mul(wm1->worst_case_latency,
+                                                wm1->consumption_rate);
                         b.full = dfixed_const(16 * 1000);
                         priority_mark12.full = dfixed_div(a, b);
                 }
-                if (wm1.priority_mark.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark.full;
+                if (wm1->priority_mark.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark.full;
                 if (dfixed_trunc(priority_mark12) < 0)
                         priority_mark12.full = 0;
-                if (wm1.priority_mark_max.full > priority_mark12.full)
-                        priority_mark12.full = wm1.priority_mark_max.full;
-                d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
+                if (wm1->priority_mark_max.full > priority_mark12.full)
+                        priority_mark12.full = wm1->priority_mark_max.full;
+                *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
                 if (rdev->disp_priority == 2)
-                        d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
+                        *d2mode_priority_a_cnt |= MODE_PRIORITY_ALWAYS_ON;
         }
+}
+
+void rv515_bandwidth_avivo_update(struct radeon_device *rdev)
+{
+        struct drm_display_mode *mode0 = NULL;
+        struct drm_display_mode *mode1 = NULL;
+        struct rv515_watermark wm0_high, wm0_low;
+        struct rv515_watermark wm1_high, wm1_low;
+        u32 tmp;
+        u32 d1mode_priority_a_cnt, d1mode_priority_b_cnt;
+        u32 d2mode_priority_a_cnt, d2mode_priority_b_cnt;
+
+        if (rdev->mode_info.crtcs[0]->base.enabled)
+                mode0 = &rdev->mode_info.crtcs[0]->base.mode;
+        if (rdev->mode_info.crtcs[1]->base.enabled)
+                mode1 = &rdev->mode_info.crtcs[1]->base.mode;
+        rs690_line_buffer_adjust(rdev, mode0, mode1);
+
+        rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_high, false);
+        rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_high, false);
+
+        rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[0], &wm0_low, false);
+        rv515_crtc_bandwidth_compute(rdev, rdev->mode_info.crtcs[1], &wm1_low, false);
+
+        tmp = wm0_high.lb_request_fifo_depth;
+        tmp |= wm1_high.lb_request_fifo_depth << 16;
+        WREG32(LB_MAX_REQ_OUTSTANDING, tmp);
+
+        rv515_compute_mode_priority(rdev,
+                                    &wm0_high, &wm1_high,
+                                    mode0, mode1,
+                                    &d1mode_priority_a_cnt, &d2mode_priority_a_cnt);
+        rv515_compute_mode_priority(rdev,
+                                    &wm0_low, &wm1_low,
+                                    mode0, mode1,
+                                    &d1mode_priority_b_cnt, &d2mode_priority_b_cnt);
 
         WREG32(D1MODE_PRIORITY_A_CNT, d1mode_priority_a_cnt);
-        WREG32(D1MODE_PRIORITY_B_CNT, d1mode_priority_a_cnt);
+        WREG32(D1MODE_PRIORITY_B_CNT, d1mode_priority_b_cnt);
         WREG32(D2MODE_PRIORITY_A_CNT, d2mode_priority_a_cnt);
-        WREG32(D2MODE_PRIORITY_B_CNT, d2mode_priority_a_cnt);
+        WREG32(D2MODE_PRIORITY_B_CNT, d2mode_priority_b_cnt);
 }
 
 void rv515_bandwidth_update(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/radeon/rv6xx_dpm.c b/drivers/gpu/drm/radeon/rv6xx_dpm.c
new file mode 100644
index 000000000000..8303de267ee5
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv6xx_dpm.c
@@ -0,0 +1,2085 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "rv6xxd.h"
+#include "r600_dpm.h"
+#include "rv6xx_dpm.h"
+#include "atom.h"
+#include <linux/seq_file.h>
+
+static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
+                                        u32 unscaled_count, u32 unit);
+
+static struct rv6xx_ps *rv6xx_get_ps(struct radeon_ps *rps)
+{
+        struct rv6xx_ps *ps = rps->ps_priv;
+
+        return ps;
+}
+
+static struct rv6xx_power_info *rv6xx_get_pi(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+static void rv6xx_force_pcie_gen1(struct radeon_device *rdev)
+{
+        u32 tmp;
+        int i;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        tmp &= LC_GEN2_EN;
+        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        tmp |= LC_INITIATE_LINK_SPEED_CHANGE;
+        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (!(RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE))
+                        break;
+                udelay(1);
+        }
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        tmp &= ~LC_INITIATE_LINK_SPEED_CHANGE;
+        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+}
+
+static void rv6xx_enable_pcie_gen2_support(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+
+        if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
+                tmp |= LC_GEN2_EN;
+                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+        }
+}
+
+static void rv6xx_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                               bool enable)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+        if (enable)
+                tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
+        else
+                tmp |= LC_HW_VOLTAGE_IF_CONTROL(0);
+        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+}
+
+static void rv6xx_enable_l0s(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
+        tmp |= LC_L0S_INACTIVITY(3);
+        WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
+}
+
+static void rv6xx_enable_l1(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+        tmp &= ~LC_L1_INACTIVITY_MASK;
+        tmp |= LC_L1_INACTIVITY(4);
+        tmp &= ~LC_PMI_TO_L1_DIS;
+        tmp &= ~LC_ASPM_TO_L1_DIS;
+        WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
+}
+
+static void rv6xx_enable_pll_sleep_in_l1(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
+        tmp |= LC_L1_INACTIVITY(8);
+        WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
+
+        /* NOTE, this is a PCIE indirect reg, not PCIE PORT */
+        tmp = RREG32_PCIE(PCIE_P_CNTL);
+        tmp |= P_PLL_PWRDN_IN_L1L23;
+        tmp &= ~P_PLL_BUF_PDNB;
+        tmp &= ~P_PLL_PDNB;
+        tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
+        WREG32_PCIE(PCIE_P_CNTL, tmp);
+}
+
+static int rv6xx_convert_clock_to_stepping(struct radeon_device *rdev,
+                                           u32 clock, struct rv6xx_sclk_stepping *step)
+{
+        int ret;
+        struct atom_clock_dividers dividers;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        if (dividers.enable_post_div)
+                step->post_divider = 2 + (dividers.post_div & 0xF) + (dividers.post_div >> 4);
+        else
+                step->post_divider = 1;
+
+        step->vco_frequency = clock * step->post_divider;
+
+        return 0;
+}
+
+static void rv6xx_output_stepping(struct radeon_device *rdev,
+                                  u32 step_index, struct rv6xx_sclk_stepping *step)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        u32 ref_clk = rdev->clock.spll.reference_freq;
+        u32 fb_divider;
+        u32 spll_step_count = rv6xx_scale_count_given_unit(rdev,
+                                                           R600_SPLLSTEPTIME_DFLT *
+                                                           pi->spll_ref_div,
+                                                           R600_SPLLSTEPUNIT_DFLT);
+
+        r600_engine_clock_entry_enable(rdev, step_index, true);
+        r600_engine_clock_entry_enable_pulse_skipping(rdev, step_index, false);
+
+        if (step->post_divider == 1)
+                r600_engine_clock_entry_enable_post_divider(rdev, step_index, false);
+        else {
+                u32 lo_len = (step->post_divider - 2) / 2;
+                u32 hi_len = step->post_divider - 2 - lo_len;
+
+                r600_engine_clock_entry_enable_post_divider(rdev, step_index, true);
+                r600_engine_clock_entry_set_post_divider(rdev, step_index, (hi_len << 4) | lo_len);
+        }
+
+        fb_divider = ((step->vco_frequency * pi->spll_ref_div) / ref_clk) >>
+                pi->fb_div_scale;
+
+        r600_engine_clock_entry_set_reference_divider(rdev, step_index,
+                                                      pi->spll_ref_div - 1);
+        r600_engine_clock_entry_set_feedback_divider(rdev, step_index, fb_divider);
+        r600_engine_clock_entry_set_step_time(rdev, step_index, spll_step_count);
+
+}
+
+static struct rv6xx_sclk_stepping rv6xx_next_vco_step(struct radeon_device *rdev,
+                                                      struct rv6xx_sclk_stepping *cur,
+                                                      bool increasing_vco, u32 step_size)
+{
+        struct rv6xx_sclk_stepping next;
+
+        next.post_divider = cur->post_divider;
+
+        if (increasing_vco)
+                next.vco_frequency = (cur->vco_frequency * (100 + step_size)) / 100;
+        else
+                next.vco_frequency = (cur->vco_frequency * 100 + 99 + step_size) / (100 + step_size);
+
+        return next;
+}
+
+static bool rv6xx_can_step_post_div(struct radeon_device *rdev,
+                                    struct rv6xx_sclk_stepping *cur,
+                                    struct rv6xx_sclk_stepping *target)
+{
+        return (cur->post_divider > target->post_divider) &&
+                ((cur->vco_frequency * target->post_divider) <=
+                 (target->vco_frequency * (cur->post_divider - 1)));
+}
+
+static struct rv6xx_sclk_stepping rv6xx_next_post_div_step(struct radeon_device *rdev,
+                                                           struct rv6xx_sclk_stepping *cur,
+                                                           struct rv6xx_sclk_stepping *target)
+{
+        struct rv6xx_sclk_stepping next = *cur;
+
+        while (rv6xx_can_step_post_div(rdev, &next, target))
+                next.post_divider--;
+
+        return next;
+}
+
+static bool rv6xx_reached_stepping_target(struct radeon_device *rdev,
+                                          struct rv6xx_sclk_stepping *cur,
+                                          struct rv6xx_sclk_stepping *target,
+                                          bool increasing_vco)
+{
+        return (increasing_vco && (cur->vco_frequency >= target->vco_frequency)) ||
+                (!increasing_vco && (cur->vco_frequency <= target->vco_frequency));
+}
+
+static void rv6xx_generate_steps(struct radeon_device *rdev,
+                                 u32 low, u32 high,
+                                 u32 start_index, u8 *end_index)
+{
+        struct rv6xx_sclk_stepping cur;
+        struct rv6xx_sclk_stepping target;
+        bool increasing_vco;
+        u32 step_index = start_index;
+
+        rv6xx_convert_clock_to_stepping(rdev, low, &cur);
+        rv6xx_convert_clock_to_stepping(rdev, high, &target);
+
+        rv6xx_output_stepping(rdev, step_index++, &cur);
+
+        increasing_vco = (target.vco_frequency >= cur.vco_frequency);
+
+        if (target.post_divider > cur.post_divider)
+                cur.post_divider = target.post_divider;
+
+        while (1) {
+                struct rv6xx_sclk_stepping next;
+
+                if (rv6xx_can_step_post_div(rdev, &cur, &target))
+                        next = rv6xx_next_post_div_step(rdev, &cur, &target);
+                else
+                        next = rv6xx_next_vco_step(rdev, &cur, increasing_vco, R600_VCOSTEPPCT_DFLT);
+
+                if (rv6xx_reached_stepping_target(rdev, &next, &target, increasing_vco)) {
+                        struct rv6xx_sclk_stepping tiny =
+                                rv6xx_next_vco_step(rdev, &target, !increasing_vco, R600_ENDINGVCOSTEPPCT_DFLT);
+                        tiny.post_divider = next.post_divider;
+
+                        if (!rv6xx_reached_stepping_target(rdev, &tiny, &cur, !increasing_vco))
+                                rv6xx_output_stepping(rdev, step_index++, &tiny);
+
+                        if ((next.post_divider != target.post_divider) &&
+                            (next.vco_frequency != target.vco_frequency)) {
+                                struct rv6xx_sclk_stepping final_vco;
+
+                                final_vco.vco_frequency = target.vco_frequency;
+                                final_vco.post_divider = next.post_divider;
+
+                                rv6xx_output_stepping(rdev, step_index++, &final_vco);
+                        }
+
+                        rv6xx_output_stepping(rdev, step_index++, &target);
+                        break;
+                } else
+                        rv6xx_output_stepping(rdev, step_index++, &next);
+
+                cur = next;
+        }
+
+        *end_index = (u8)step_index - 1;
+
+}
+
+static void rv6xx_generate_single_step(struct radeon_device *rdev,
+                                       u32 clock, u32 index)
+{
+        struct rv6xx_sclk_stepping step;
+
+        rv6xx_convert_clock_to_stepping(rdev, clock, &step);
+        rv6xx_output_stepping(rdev, index, &step);
+}
+
+static void rv6xx_invalidate_intermediate_steps_range(struct radeon_device *rdev,
+                                                      u32 start_index, u32 end_index)
+{
+        u32 step_index;
+
+        for (step_index = start_index + 1; step_index < end_index; step_index++)
+                r600_engine_clock_entry_enable(rdev, step_index, false);
+}
+
+static void rv6xx_set_engine_spread_spectrum_clk_s(struct radeon_device *rdev,
+                                                   u32 index, u32 clk_s)
+{
+        WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
+                 CLKS(clk_s), ~CLKS_MASK);
+}
+
+static void rv6xx_set_engine_spread_spectrum_clk_v(struct radeon_device *rdev,
+                                                   u32 index, u32 clk_v)
+{
+        WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
+                 CLKV(clk_v), ~CLKV_MASK);
+}
+
+static void rv6xx_enable_engine_spread_spectrum(struct radeon_device *rdev,
+                                                u32 index, bool enable)
+{
+        if (enable)
+                WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
+                         SSEN, ~SSEN);
+        else
+                WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
+                         0, ~SSEN);
+}
+
+static void rv6xx_set_memory_spread_spectrum_clk_s(struct radeon_device *rdev,
+                                                   u32 clk_s)
+{
+        WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKS(clk_s), ~CLKS_MASK);
+}
+
+static void rv6xx_set_memory_spread_spectrum_clk_v(struct radeon_device *rdev,
+                                                   u32 clk_v)
+{
+        WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKV(clk_v), ~CLKV_MASK);
+}
+
+static void rv6xx_enable_memory_spread_spectrum(struct radeon_device *rdev,
+                                                bool enable)
+{
+        if (enable)
+                WREG32_P(CG_MPLL_SPREAD_SPECTRUM, SSEN, ~SSEN);
+        else
+                WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
+}
+
+static void rv6xx_enable_dynamic_spread_spectrum(struct radeon_device *rdev,
+                                                 bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
+}
+
+static void rv6xx_memory_clock_entry_enable_post_divider(struct radeon_device *rdev,
+                                                         u32 index, bool enable)
+{
+        if (enable)
+                WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
+                         LEVEL0_MPLL_DIV_EN, ~LEVEL0_MPLL_DIV_EN);
+        else
+                WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), 0, ~LEVEL0_MPLL_DIV_EN);
+}
+
+static void rv6xx_memory_clock_entry_set_post_divider(struct radeon_device *rdev,
+                                                      u32 index, u32 divider)
+{
+        WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
+                 LEVEL0_MPLL_POST_DIV(divider), ~LEVEL0_MPLL_POST_DIV_MASK);
+}
+
+static void rv6xx_memory_clock_entry_set_feedback_divider(struct radeon_device *rdev,
+                                                          u32 index, u32 divider)
+{
+        WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), LEVEL0_MPLL_FB_DIV(divider),
+                 ~LEVEL0_MPLL_FB_DIV_MASK);
+}
+
+static void rv6xx_memory_clock_entry_set_reference_divider(struct radeon_device *rdev,
+                                                           u32 index, u32 divider)
+{
+        WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
+                 LEVEL0_MPLL_REF_DIV(divider), ~LEVEL0_MPLL_REF_DIV_MASK);
+}
+
+static void rv6xx_vid_response_set_brt(struct radeon_device *rdev, u32 rt)
+{
+        WREG32_P(VID_RT, BRT(rt), ~BRT_MASK);
+}
+
+static void rv6xx_enable_engine_feedback_and_reference_sync(struct radeon_device *rdev)
+{
+        WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
+}
+
+static u64 rv6xx_clocks_per_unit(u32 unit)
+{
+        u64 tmp = 1 << (2 * unit);
+
+        return tmp;
+}
+
+static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
+                                        u32 unscaled_count, u32 unit)
+{
+        u32 count_per_unit = (u32)rv6xx_clocks_per_unit(unit);
+
+        return (unscaled_count + count_per_unit - 1) / count_per_unit;
+}
+
+static u32 rv6xx_compute_count_for_delay(struct radeon_device *rdev,
+                                         u32 delay_us, u32 unit)
+{
+        u32 ref_clk = rdev->clock.spll.reference_freq;
+
+        return rv6xx_scale_count_given_unit(rdev, delay_us * (ref_clk / 100), unit);
+}
+
+static void rv6xx_calculate_engine_speed_stepping_parameters(struct radeon_device *rdev,
+                                                             struct rv6xx_ps *state)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        pi->hw.sclks[R600_POWER_LEVEL_LOW] =
+                state->low.sclk;
+        pi->hw.sclks[R600_POWER_LEVEL_MEDIUM] =
+                state->medium.sclk;
+        pi->hw.sclks[R600_POWER_LEVEL_HIGH] =
+                state->high.sclk;
+
+        pi->hw.low_sclk_index = R600_POWER_LEVEL_LOW;
+        pi->hw.medium_sclk_index = R600_POWER_LEVEL_MEDIUM;
+        pi->hw.high_sclk_index = R600_POWER_LEVEL_HIGH;
+}
+
+static void rv6xx_calculate_memory_clock_stepping_parameters(struct radeon_device *rdev,
+                                                             struct rv6xx_ps *state)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        pi->hw.mclks[R600_POWER_LEVEL_CTXSW] =
+                state->high.mclk;
+        pi->hw.mclks[R600_POWER_LEVEL_HIGH] =
+                state->high.mclk;
+        pi->hw.mclks[R600_POWER_LEVEL_MEDIUM] =
+                state->medium.mclk;
+        pi->hw.mclks[R600_POWER_LEVEL_LOW] =
+                state->low.mclk;
+
+        pi->hw.high_mclk_index = R600_POWER_LEVEL_HIGH;
+
+        if (state->high.mclk == state->medium.mclk)
+                pi->hw.medium_mclk_index =
+                        pi->hw.high_mclk_index;
+        else
+                pi->hw.medium_mclk_index = R600_POWER_LEVEL_MEDIUM;
+
+
+        if (state->medium.mclk == state->low.mclk)
+                pi->hw.low_mclk_index =
+                        pi->hw.medium_mclk_index;
+        else
+                pi->hw.low_mclk_index = R600_POWER_LEVEL_LOW;
+}
+
+static void rv6xx_calculate_voltage_stepping_parameters(struct radeon_device *rdev,
+                                                        struct rv6xx_ps *state)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        pi->hw.vddc[R600_POWER_LEVEL_CTXSW] = state->high.vddc;
+        pi->hw.vddc[R600_POWER_LEVEL_HIGH] = state->high.vddc;
+        pi->hw.vddc[R600_POWER_LEVEL_MEDIUM] = state->medium.vddc;
+        pi->hw.vddc[R600_POWER_LEVEL_LOW] = state->low.vddc;
+
+        pi->hw.backbias[R600_POWER_LEVEL_CTXSW] =
+                (state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
+        pi->hw.backbias[R600_POWER_LEVEL_HIGH] =
+                (state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
+        pi->hw.backbias[R600_POWER_LEVEL_MEDIUM] =
+                (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
+        pi->hw.backbias[R600_POWER_LEVEL_LOW] =
+                (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
+
+        pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH] =
+                (state->high.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
+        pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM] =
+                (state->medium.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
+        pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW] =
+                (state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
+
+        pi->hw.high_vddc_index = R600_POWER_LEVEL_HIGH;
+
+        if ((state->high.vddc == state->medium.vddc) &&
+            ((state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ==
+             (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)))
+                pi->hw.medium_vddc_index =
+                        pi->hw.high_vddc_index;
+        else
+                pi->hw.medium_vddc_index = R600_POWER_LEVEL_MEDIUM;
+
+        if ((state->medium.vddc == state->low.vddc) &&
+            ((state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ==
+             (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)))
+                pi->hw.low_vddc_index =
+                        pi->hw.medium_vddc_index;
+        else
+                pi->hw.medium_vddc_index = R600_POWER_LEVEL_LOW;
+}
+
+static inline u32 rv6xx_calculate_vco_frequency(u32 ref_clock,
+                                                struct atom_clock_dividers *dividers,
+                                                u32 fb_divider_scale)
+{
+        return ref_clock * ((dividers->fb_div & ~1) << fb_divider_scale) /
+                (dividers->ref_div + 1);
+}
+
+static inline u32 rv6xx_calculate_spread_spectrum_clk_v(u32 vco_freq, u32 ref_freq,
+                                                        u32 ss_rate, u32 ss_percent,
+                                                        u32 fb_divider_scale)
+{
+        u32 fb_divider = vco_freq / ref_freq;
+
+        return (ss_percent * ss_rate * 4 * (fb_divider * fb_divider) /
+                (5375 * ((vco_freq * 10) / (4096 >> fb_divider_scale))));
+}
+
+static inline u32 rv6xx_calculate_spread_spectrum_clk_s(u32 ss_rate, u32 ref_freq)
+{
+        return (((ref_freq * 10) / (ss_rate * 2)) - 1) / 4;
+}
+
+static void rv6xx_program_engine_spread_spectrum(struct radeon_device *rdev,
+                                                 u32 clock, enum r600_power_level level)
+{
+        u32 ref_clk = rdev->clock.spll.reference_freq;
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        struct radeon_atom_ss ss;
+        u32 vco_freq, clk_v, clk_s;
+
+        rv6xx_enable_engine_spread_spectrum(rdev, level, false);
+
+        if (clock && pi->sclk_ss) {
+                if (radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, clock, false, &dividers) == 0) {
+                        vco_freq = rv6xx_calculate_vco_frequency(ref_clk, &dividers,
+                                                                 pi->fb_div_scale);
+
+                        if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                             ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+                                clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq,
+                                                                              (ref_clk / (dividers.ref_div + 1)),
+                                                                              ss.rate,
+                                                                              ss.percentage,
+                                                                              pi->fb_div_scale);
+
+                                clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate,
+                                                                              (ref_clk / (dividers.ref_div + 1)));
+
+                                rv6xx_set_engine_spread_spectrum_clk_v(rdev, level, clk_v);
+                                rv6xx_set_engine_spread_spectrum_clk_s(rdev, level, clk_s);
+                                rv6xx_enable_engine_spread_spectrum(rdev, level, true);
+                        }
+                }
+        }
+}
+
+static void rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_program_engine_spread_spectrum(rdev,
+                                             pi->hw.sclks[R600_POWER_LEVEL_HIGH],
+                                             R600_POWER_LEVEL_HIGH);
+
+        rv6xx_program_engine_spread_spectrum(rdev,
+                                             pi->hw.sclks[R600_POWER_LEVEL_MEDIUM],
+                                             R600_POWER_LEVEL_MEDIUM);
+
+}
+
+static int rv6xx_program_mclk_stepping_entry(struct radeon_device *rdev,
+                                             u32 entry, u32 clock)
+{
+        struct atom_clock_dividers dividers;
+
+        if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, clock, false, &dividers))
+            return -EINVAL;
+
+
+        rv6xx_memory_clock_entry_set_reference_divider(rdev, entry, dividers.ref_div);
+        rv6xx_memory_clock_entry_set_feedback_divider(rdev, entry, dividers.fb_div);
+        rv6xx_memory_clock_entry_set_post_divider(rdev, entry, dividers.post_div);
+
+        if (dividers.enable_post_div)
+                rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, true);
+        else
+                rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, false);
+
+        return 0;
+}
+
+static void rv6xx_program_mclk_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        int i;
+
+        for (i = 1; i < R600_PM_NUMBER_OF_MCLKS; i++) {
+                if (pi->hw.mclks[i])
+                        rv6xx_program_mclk_stepping_entry(rdev, i,
+                                                          pi->hw.mclks[i]);
+        }
+}
+
+static void rv6xx_find_memory_clock_with_highest_vco(struct radeon_device *rdev,
+                                                     u32 requested_memory_clock,
+                                                     u32 ref_clk,
+                                                     struct atom_clock_dividers *dividers,
+                                                     u32 *vco_freq)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        struct atom_clock_dividers req_dividers;
+        u32 vco_freq_temp;
+
+        if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                           requested_memory_clock, false, &req_dividers) == 0) {
+                vco_freq_temp = rv6xx_calculate_vco_frequency(ref_clk, &req_dividers,
+                                                              pi->fb_div_scale);
+
+                if (vco_freq_temp > *vco_freq) {
+                        *dividers = req_dividers;
+                        *vco_freq = vco_freq_temp;
+                }
+        }
+}
+
+static void rv6xx_program_mclk_spread_spectrum_parameters(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        u32 ref_clk = rdev->clock.mpll.reference_freq;
+        struct atom_clock_dividers dividers;
+        struct radeon_atom_ss ss;
+        u32 vco_freq = 0, clk_v, clk_s;
+
+        rv6xx_enable_memory_spread_spectrum(rdev, false);
+
+        if (pi->mclk_ss) {
+                rv6xx_find_memory_clock_with_highest_vco(rdev,
+                                                         pi->hw.mclks[pi->hw.high_mclk_index],
+                                                         ref_clk,
+                                                         &dividers,
+                                                         &vco_freq);
+
+                rv6xx_find_memory_clock_with_highest_vco(rdev,
+                                                         pi->hw.mclks[pi->hw.medium_mclk_index],
+                                                         ref_clk,
+                                                         &dividers,
+                                                         &vco_freq);
+
+                rv6xx_find_memory_clock_with_highest_vco(rdev,
+                                                         pi->hw.mclks[pi->hw.low_mclk_index],
+                                                         ref_clk,
+                                                         &dividers,
+                                                         &vco_freq);
+
+                if (vco_freq) {
+                        if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                             ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
+                                clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq,
+                                                                             (ref_clk / (dividers.ref_div + 1)),
+                                                                             ss.rate,
+                                                                             ss.percentage,
+                                                                             pi->fb_div_scale);
+
+                                clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate,
+                                                                             (ref_clk / (dividers.ref_div + 1)));
+
+                                rv6xx_set_memory_spread_spectrum_clk_v(rdev, clk_v);
+                                rv6xx_set_memory_spread_spectrum_clk_s(rdev, clk_s);
+                                rv6xx_enable_memory_spread_spectrum(rdev, true);
+                        }
+                }
+        }
+}
+
+static int rv6xx_program_voltage_stepping_entry(struct radeon_device *rdev,
+                                                u32 entry, u16 voltage)
+{
+        u32 mask, set_pins;
+        int ret;
+
+        ret = radeon_atom_get_voltage_gpio_settings(rdev, voltage,
+                                                    SET_VOLTAGE_TYPE_ASIC_VDDC,
+                                                    &set_pins, &mask);
+        if (ret)
+                return ret;
+
+        r600_voltage_control_program_voltages(rdev, entry, set_pins);
+
+        return 0;
+}
+
+static void rv6xx_program_voltage_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        int i;
+
+        for (i = 1; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++)
+                rv6xx_program_voltage_stepping_entry(rdev, i,
+                                                     pi->hw.vddc[i]);
+
+}
+
+static void rv6xx_program_backbias_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if (pi->hw.backbias[1])
+                WREG32_P(VID_UPPER_GPIO_CNTL, MEDIUM_BACKBIAS_VALUE, ~MEDIUM_BACKBIAS_VALUE);
+        else
+                WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~MEDIUM_BACKBIAS_VALUE);
+
+        if (pi->hw.backbias[2])
+                WREG32_P(VID_UPPER_GPIO_CNTL, HIGH_BACKBIAS_VALUE, ~HIGH_BACKBIAS_VALUE);
+        else
+                WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~HIGH_BACKBIAS_VALUE);
+}
+
+static void rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_program_engine_spread_spectrum(rdev,
+                                             pi->hw.sclks[R600_POWER_LEVEL_LOW],
+                                             R600_POWER_LEVEL_LOW);
+}
+
+static void rv6xx_program_mclk_stepping_parameters_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if (pi->hw.mclks[0])
+                rv6xx_program_mclk_stepping_entry(rdev, 0,
+                                                  pi->hw.mclks[0]);
+}
+
+static void rv6xx_program_voltage_stepping_parameters_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_program_voltage_stepping_entry(rdev, 0,
+                                             pi->hw.vddc[0]);
+
+}
+
+static void rv6xx_program_backbias_stepping_parameters_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if (pi->hw.backbias[0])
+                WREG32_P(VID_UPPER_GPIO_CNTL, LOW_BACKBIAS_VALUE, ~LOW_BACKBIAS_VALUE);
+        else
+                WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~LOW_BACKBIAS_VALUE);
+}
+
+static u32 calculate_memory_refresh_rate(struct radeon_device *rdev,
+                                         u32 engine_clock)
+{
+        u32 dram_rows, dram_refresh_rate;
+        u32 tmp;
+
+        tmp = (RREG32(RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
+        dram_rows = 1 << (tmp + 10);
+        dram_refresh_rate = 1 << ((RREG32(MC_SEQ_RESERVE_M) & 0x3) + 3);
+
+        return ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
+}
+
+static void rv6xx_program_memory_timing_parameters(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        u32 sqm_ratio;
+        u32 arb_refresh_rate;
+        u32 high_clock;
+
+        if (pi->hw.sclks[R600_POWER_LEVEL_HIGH] <
+            (pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40))
+                high_clock = pi->hw.sclks[R600_POWER_LEVEL_HIGH];
+        else
+                high_clock =
+                        pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40;
+
+        radeon_atom_set_engine_dram_timings(rdev, high_clock, 0);
+
+        sqm_ratio = (STATE0(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_LOW]) |
+                     STATE1(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_MEDIUM]) |
+                     STATE2(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]) |
+                     STATE3(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]));
+        WREG32(SQM_RATIO, sqm_ratio);
+
+        arb_refresh_rate =
+                (POWERMODE0(calculate_memory_refresh_rate(rdev,
+                                                          pi->hw.sclks[R600_POWER_LEVEL_LOW])) |
+                 POWERMODE1(calculate_memory_refresh_rate(rdev,
+                                                          pi->hw.sclks[R600_POWER_LEVEL_MEDIUM])) |
+                 POWERMODE2(calculate_memory_refresh_rate(rdev,
+                                                          pi->hw.sclks[R600_POWER_LEVEL_MEDIUM])) |
+                 POWERMODE3(calculate_memory_refresh_rate(rdev,
+                                                          pi->hw.sclks[R600_POWER_LEVEL_HIGH])));
+        WREG32(ARB_RFSH_RATE, arb_refresh_rate);
+}
+
+static void rv6xx_program_mpll_timing_parameters(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        r600_set_mpll_lock_time(rdev, R600_MPLLLOCKTIME_DFLT *
+                                pi->mpll_ref_div);
+        r600_set_mpll_reset_time(rdev, R600_MPLLRESETTIME_DFLT);
+}
+
+static void rv6xx_program_bsp(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        u32 ref_clk = rdev->clock.spll.reference_freq;
+
+        r600_calculate_u_and_p(R600_ASI_DFLT,
+                               ref_clk, 16,
+                               &pi->bsp,
+                               &pi->bsu);
+
+        r600_set_bsp(rdev, pi->bsu, pi->bsp);
+}
+
+static void rv6xx_program_at(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        r600_set_at(rdev,
+                    (pi->hw.rp[0] * pi->bsp) / 200,
+                    (pi->hw.rp[1] * pi->bsp) / 200,
+                    (pi->hw.lp[2] * pi->bsp) / 200,
+                    (pi->hw.lp[1] * pi->bsp) / 200);
+}
+
+static void rv6xx_program_git(struct radeon_device *rdev)
+{
+        r600_set_git(rdev, R600_GICST_DFLT);
+}
+
+static void rv6xx_program_tp(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
+                r600_set_tc(rdev, i, r600_utc[i], r600_dtc[i]);
+
+        r600_select_td(rdev, R600_TD_DFLT);
+}
+
+static void rv6xx_program_vc(struct radeon_device *rdev)
+{
+        r600_set_vrc(rdev, R600_VRC_DFLT);
+}
+
+static void rv6xx_clear_vc(struct radeon_device *rdev)
+{
+        r600_set_vrc(rdev, 0);
+}
+
+static void rv6xx_program_tpp(struct radeon_device *rdev)
+{
+        r600_set_tpu(rdev, R600_TPU_DFLT);
+        r600_set_tpc(rdev, R600_TPC_DFLT);
+}
+
+static void rv6xx_program_sstp(struct radeon_device *rdev)
+{
+        r600_set_sstu(rdev, R600_SSTU_DFLT);
+        r600_set_sst(rdev, R600_SST_DFLT);
+}
+
+static void rv6xx_program_fcp(struct radeon_device *rdev)
+{
+        r600_set_fctu(rdev, R600_FCTU_DFLT);
+        r600_set_fct(rdev, R600_FCT_DFLT);
+}
+
+static void rv6xx_program_vddc3d_parameters(struct radeon_device *rdev)
+{
+        r600_set_vddc3d_oorsu(rdev, R600_VDDC3DOORSU_DFLT);
+        r600_set_vddc3d_oorphc(rdev, R600_VDDC3DOORPHC_DFLT);
+        r600_set_vddc3d_oorsdc(rdev, R600_VDDC3DOORSDC_DFLT);
+        r600_set_ctxcgtt3d_rphc(rdev, R600_CTXCGTT3DRPHC_DFLT);
+        r600_set_ctxcgtt3d_rsdc(rdev, R600_CTXCGTT3DRSDC_DFLT);
+}
+
+static void rv6xx_program_voltage_timing_parameters(struct radeon_device *rdev)
+{
+        u32 rt;
+
+        r600_vid_rt_set_vru(rdev, R600_VRU_DFLT);
+
+        r600_vid_rt_set_vrt(rdev,
+                            rv6xx_compute_count_for_delay(rdev,
+                                                          rdev->pm.dpm.voltage_response_time,
+                                                          R600_VRU_DFLT));
+
+        rt = rv6xx_compute_count_for_delay(rdev,
+                                           rdev->pm.dpm.backbias_response_time,
+                                           R600_VRU_DFLT);
+
+        rv6xx_vid_response_set_brt(rdev, (rt + 0x1F) >> 5);
+}
+
+static void rv6xx_program_engine_speed_parameters(struct radeon_device *rdev)
+{
+        r600_vid_rt_set_ssu(rdev, R600_SPLLSTEPUNIT_DFLT);
+        rv6xx_enable_engine_feedback_and_reference_sync(rdev);
+}
+
+static u64 rv6xx_get_master_voltage_mask(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        u64 master_mask = 0;
+        int i;
+
+        for (i = 0; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++) {
+                u32 tmp_mask, tmp_set_pins;
+                int ret;
+
+                ret = radeon_atom_get_voltage_gpio_settings(rdev,
+                                                            pi->hw.vddc[i],
+                                                            SET_VOLTAGE_TYPE_ASIC_VDDC,
+                                                            &tmp_set_pins, &tmp_mask);
+
+                if (ret == 0)
+                        master_mask |= tmp_mask;
+        }
+
+        return master_mask;
+}
+
+static void rv6xx_program_voltage_gpio_pins(struct radeon_device *rdev)
+{
+        r600_voltage_control_enable_pins(rdev,
+                                         rv6xx_get_master_voltage_mask(rdev));
+}
+
+static void rv6xx_enable_static_voltage_control(struct radeon_device *rdev,
+                                                struct radeon_ps *new_ps,
+                                                bool enable)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+
+        if (enable)
+                radeon_atom_set_voltage(rdev,
+                                        new_state->low.vddc,
+                                        SET_VOLTAGE_TYPE_ASIC_VDDC);
+        else
+                r600_voltage_control_deactivate_static_control(rdev,
+                                                               rv6xx_get_master_voltage_mask(rdev));
+}
+
+static void rv6xx_enable_display_gap(struct radeon_device *rdev, bool enable)
+{
+        if (enable) {
+                u32 tmp = (DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) |
+                           DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) |
+                           DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
+                           DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
+                           VBI_TIMER_COUNT(0x3FFF) |
+                           VBI_TIMER_UNIT(7));
+                WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+
+                WREG32_P(MCLK_PWRMGT_CNTL, USE_DISPLAY_GAP, ~USE_DISPLAY_GAP);
+        } else
+                WREG32_P(MCLK_PWRMGT_CNTL, 0, ~USE_DISPLAY_GAP);
+}
+
+static void rv6xx_program_power_level_enter_state(struct radeon_device *rdev)
+{
+        r600_power_level_set_enter_index(rdev, R600_POWER_LEVEL_MEDIUM);
+}
+
+static void rv6xx_calculate_t(u32 l_f, u32 h_f, int h,
+                              int d_l, int d_r, u8 *l, u8 *r)
+{
+        int a_n, a_d, h_r, l_r;
+
+        h_r = d_l;
+        l_r = 100 - d_r;
+
+        a_n = (int)h_f * d_l + (int)l_f * (h - d_r);
+        a_d = (int)l_f * l_r + (int)h_f * h_r;
+
+        if (a_d != 0) {
+                *l = d_l - h_r * a_n / a_d;
+                *r = d_r + l_r * a_n / a_d;
+        }
+}
+
+static void rv6xx_calculate_ap(struct radeon_device *rdev,
+                               struct rv6xx_ps *state)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        pi->hw.lp[0] = 0;
+        pi->hw.rp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS - 1]
+                = 100;
+
+        rv6xx_calculate_t(state->low.sclk,
+                          state->medium.sclk,
+                          R600_AH_DFLT,
+                          R600_LMP_DFLT,
+                          R600_RLP_DFLT,
+                          &pi->hw.lp[1],
+                          &pi->hw.rp[0]);
+
+        rv6xx_calculate_t(state->medium.sclk,
+                          state->high.sclk,
+                          R600_AH_DFLT,
+                          R600_LHP_DFLT,
+                          R600_RMP_DFLT,
+                          &pi->hw.lp[2],
+                          &pi->hw.rp[1]);
+
+}
+
+static void rv6xx_calculate_stepping_parameters(struct radeon_device *rdev,
+                                                struct radeon_ps *new_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+
+        rv6xx_calculate_engine_speed_stepping_parameters(rdev, new_state);
+        rv6xx_calculate_memory_clock_stepping_parameters(rdev, new_state);
+        rv6xx_calculate_voltage_stepping_parameters(rdev, new_state);
+        rv6xx_calculate_ap(rdev, new_state);
+}
+
+static void rv6xx_program_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_program_mclk_stepping_parameters_except_lowest_entry(rdev);
+        if (pi->voltage_control)
+                rv6xx_program_voltage_stepping_parameters_except_lowest_entry(rdev);
+        rv6xx_program_backbias_stepping_parameters_except_lowest_entry(rdev);
+        rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(rdev);
+        rv6xx_program_mclk_spread_spectrum_parameters(rdev);
+        rv6xx_program_memory_timing_parameters(rdev);
+}
+
+static void rv6xx_program_stepping_parameters_lowest_entry(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_program_mclk_stepping_parameters_lowest_entry(rdev);
+        if (pi->voltage_control)
+                rv6xx_program_voltage_stepping_parameters_lowest_entry(rdev);
+        rv6xx_program_backbias_stepping_parameters_lowest_entry(rdev);
+        rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(rdev);
+}
+
+static void rv6xx_program_power_level_low(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW,
+                                           pi->hw.low_vddc_index);
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW,
+                                             pi->hw.low_mclk_index);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW,
+                                             pi->hw.low_sclk_index);
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,
+                                          R600_DISPLAY_WATERMARK_LOW);
+        r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW,
+                                       pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
+}
+
+static void rv6xx_program_power_level_low_to_lowest_state(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW, 0);
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW, 0);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW, 0);
+
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,
+                                          R600_DISPLAY_WATERMARK_LOW);
+
+        r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW,
+                                       pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
+
+}
+
+static void rv6xx_program_power_level_medium(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM,
+                                          pi->hw.medium_vddc_index);
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
+                                            pi->hw.medium_mclk_index);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
+                                            pi->hw.medium_sclk_index);
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM,
+                                         R600_DISPLAY_WATERMARK_LOW);
+        r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM,
+                                      pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM]);
+}
+
+static void rv6xx_program_power_level_medium_for_transition(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_program_mclk_stepping_entry(rdev,
+                                          R600_POWER_LEVEL_CTXSW,
+                                          pi->hw.mclks[pi->hw.low_mclk_index]);
+
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM, 1);
+
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
+                                             R600_POWER_LEVEL_CTXSW);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
+                                             pi->hw.medium_sclk_index);
+
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM,
+                                          R600_DISPLAY_WATERMARK_LOW);
+
+        rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false);
+
+        r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM,
+                                       pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
+}
+
+static void rv6xx_program_power_level_high(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_HIGH,
+                                           pi->hw.high_vddc_index);
+        r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_HIGH,
+                                             pi->hw.high_mclk_index);
+        r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_HIGH,
+                                             pi->hw.high_sclk_index);
+
+        r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_HIGH,
+                                          R600_DISPLAY_WATERMARK_HIGH);
+
+        r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_HIGH,
+                                       pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH]);
+}
+
+static void rv6xx_enable_backbias(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL,
+                         ~(BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL));
+        else
+                WREG32_P(GENERAL_PWRMGT, 0,
+                         ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL));
+}
+
+static void rv6xx_program_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
+
+        tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
+        if (RREG32(AVIVO_D1CRTC_CONTROL) & AVIVO_CRTC_EN) {
+                tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
+                tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+        } else if (RREG32(AVIVO_D2CRTC_CONTROL) & AVIVO_CRTC_EN) {
+                tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+                tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
+        } else {
+                tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+                tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+        }
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+}
+
+static void rv6xx_set_sw_voltage_to_safe(struct radeon_device *rdev,
+                                         struct radeon_ps *new_ps,
+                                         struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+        u16 safe_voltage;
+
+        safe_voltage = (new_state->low.vddc >= old_state->low.vddc) ?
+                new_state->low.vddc : old_state->low.vddc;
+
+        rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
+                                             safe_voltage);
+
+        WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW),
+                 ~SW_GPIO_INDEX_MASK);
+}
+
+static void rv6xx_set_sw_voltage_to_low(struct radeon_device *rdev,
+                                        struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+
+        rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
+                                             old_state->low.vddc);
+
+        WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW),
+                ~SW_GPIO_INDEX_MASK);
+}
+
+static void rv6xx_set_safe_backbias(struct radeon_device *rdev,
+                                    struct radeon_ps *new_ps,
+                                    struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+
+        if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) &&
+            (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE))
+                WREG32_P(GENERAL_PWRMGT, BACKBIAS_VALUE, ~BACKBIAS_VALUE);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_VALUE);
+}
+
+static void rv6xx_set_safe_pcie_gen2(struct radeon_device *rdev,
+                                     struct radeon_ps *new_ps,
+                                     struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+
+        if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) !=
+            (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2))
+                rv6xx_force_pcie_gen1(rdev);
+}
+
+static void rv6xx_enable_dynamic_voltage_control(struct radeon_device *rdev,
+                                                 bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
+}
+
+static void rv6xx_enable_dynamic_backbias_control(struct radeon_device *rdev,
+                                                  bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, BACKBIAS_DPM_CNTL, ~BACKBIAS_DPM_CNTL);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_DPM_CNTL);
+}
+
+static int rv6xx_step_sw_voltage(struct radeon_device *rdev,
+                                 u16 initial_voltage,
+                                 u16 target_voltage)
+{
+        u16 current_voltage;
+        u16 true_target_voltage;
+        u16 voltage_step;
+        int signed_voltage_step;
+
+        if ((radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+                                          &voltage_step)) ||
+            (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+                                               initial_voltage, &current_voltage)) ||
+            (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+                                               target_voltage, &true_target_voltage)))
+                return -EINVAL;
+
+        if (true_target_voltage < current_voltage)
+                signed_voltage_step = -(int)voltage_step;
+        else
+                signed_voltage_step = voltage_step;
+
+        while (current_voltage != true_target_voltage) {
+                current_voltage += signed_voltage_step;
+                rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
+                                                     current_voltage);
+                msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000);
+        }
+
+        return 0;
+}
+
+static int rv6xx_step_voltage_if_increasing(struct radeon_device *rdev,
+                                            struct radeon_ps *new_ps,
+                                            struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+
+        if (new_state->low.vddc > old_state->low.vddc)
+                return rv6xx_step_sw_voltage(rdev,
+                                             old_state->low.vddc,
+                                             new_state->low.vddc);
+
+        return 0;
+}
+
+static int rv6xx_step_voltage_if_decreasing(struct radeon_device *rdev,
+                                            struct radeon_ps *new_ps,
+                                            struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+
+        if (new_state->low.vddc < old_state->low.vddc)
+                return rv6xx_step_sw_voltage(rdev,
+                                             old_state->low.vddc,
+                                             new_state->low.vddc);
+        else
+                return 0;
+}
+
+static void rv6xx_enable_high(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if ((pi->restricted_levels < 1) ||
+            (pi->restricted_levels == 3))
+                r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true);
+}
+
+static void rv6xx_enable_medium(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if (pi->restricted_levels < 2)
+                r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
+}
+
+static void rv6xx_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        bool want_thermal_protection;
+        enum radeon_dpm_event_src dpm_event_src;
+
+        switch (sources) {
+        case 0:
+        default:
+                want_thermal_protection = false;
+                break;
+        case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
+                break;
+
+        case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
+                break;
+
+        case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
+              (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
+                break;
+        }
+
+        if (want_thermal_protection) {
+                WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
+                if (pi->thermal_protection)
+                        WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+        } else {
+                WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+        }
+}
+
+static void rv6xx_enable_auto_throttle_source(struct radeon_device *rdev,
+                                              enum radeon_dpm_auto_throttle_src source,
+                                              bool enable)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if (enable) {
+                if (!(pi->active_auto_throttle_sources & (1 << source))) {
+                        pi->active_auto_throttle_sources |= 1 << source;
+                        rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
+                }
+        } else {
+                if (pi->active_auto_throttle_sources & (1 << source)) {
+                        pi->active_auto_throttle_sources &= ~(1 << source);
+                        rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
+                }
+        }
+}
+
+
+static void rv6xx_enable_thermal_protection(struct radeon_device *rdev,
+                                            bool enable)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        if (pi->active_auto_throttle_sources)
+                r600_enable_thermal_protection(rdev, enable);
+}
+
+static void rv6xx_generate_transition_stepping(struct radeon_device *rdev,
+                                               struct radeon_ps *new_ps,
+                                               struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_generate_steps(rdev,
+                             old_state->low.sclk,
+                             new_state->low.sclk,
+                             0, &pi->hw.medium_sclk_index);
+}
+
+static void rv6xx_generate_low_step(struct radeon_device *rdev,
+                                    struct radeon_ps *new_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        pi->hw.low_sclk_index = 0;
+        rv6xx_generate_single_step(rdev,
+                                   new_state->low.sclk,
+                                   0);
+}
+
+static void rv6xx_invalidate_intermediate_steps(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        rv6xx_invalidate_intermediate_steps_range(rdev, 0,
+                                                  pi->hw.medium_sclk_index);
+}
+
+static void rv6xx_generate_stepping_table(struct radeon_device *rdev,
+                                          struct radeon_ps *new_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+
+        pi->hw.low_sclk_index = 0;
+
+        rv6xx_generate_steps(rdev,
+                             new_state->low.sclk,
+                             new_state->medium.sclk,
+                             0,
+                             &pi->hw.medium_sclk_index);
+        rv6xx_generate_steps(rdev,
+                             new_state->medium.sclk,
+                             new_state->high.sclk,
+                             pi->hw.medium_sclk_index,
+                             &pi->hw.high_sclk_index);
+}
+
+static void rv6xx_enable_spread_spectrum(struct radeon_device *rdev,
+                                         bool enable)
+{
+        if (enable)
+                rv6xx_enable_dynamic_spread_spectrum(rdev, true);
+        else {
+                rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_LOW, false);
+                rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false);
+                rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_HIGH, false);
+                rv6xx_enable_dynamic_spread_spectrum(rdev, false);
+                rv6xx_enable_memory_spread_spectrum(rdev, false);
+        }
+}
+
+static void rv6xx_reset_lvtm_data_sync(struct radeon_device *rdev)
+{
+        if (ASIC_IS_DCE3(rdev))
+                WREG32_P(DCE3_LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG);
+        else
+                WREG32_P(LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG);
+}
+
+static void rv6xx_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                           struct radeon_ps *new_ps,
+                                           bool enable)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+
+        if (enable) {
+                rv6xx_enable_bif_dynamic_pcie_gen2(rdev, true);
+                rv6xx_enable_pcie_gen2_support(rdev);
+                r600_enable_dynamic_pcie_gen2(rdev, true);
+        } else {
+                if (!(new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2))
+                        rv6xx_force_pcie_gen1(rdev);
+                rv6xx_enable_bif_dynamic_pcie_gen2(rdev, false);
+                r600_enable_dynamic_pcie_gen2(rdev, false);
+        }
+}
+
+static void rv6xx_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                                     struct radeon_ps *new_ps,
+                                                     struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->high.sclk >= current_state->high.sclk)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+static void rv6xx_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                                    struct radeon_ps *new_ps,
+                                                    struct radeon_ps *old_ps)
+{
+        struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
+        struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->high.sclk < current_state->high.sclk)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+int rv6xx_dpm_enable(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        if (r600_dynamicpm_enabled(rdev))
+                return -EINVAL;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv6xx_enable_backbias(rdev, true);
+
+        if (pi->dynamic_ss)
+                rv6xx_enable_spread_spectrum(rdev, true);
+
+        rv6xx_program_mpll_timing_parameters(rdev);
+        rv6xx_program_bsp(rdev);
+        rv6xx_program_git(rdev);
+        rv6xx_program_tp(rdev);
+        rv6xx_program_tpp(rdev);
+        rv6xx_program_sstp(rdev);
+        rv6xx_program_fcp(rdev);
+        rv6xx_program_vddc3d_parameters(rdev);
+        rv6xx_program_voltage_timing_parameters(rdev);
+        rv6xx_program_engine_speed_parameters(rdev);
+
+        rv6xx_enable_display_gap(rdev, true);
+        if (pi->display_gap == false)
+                rv6xx_enable_display_gap(rdev, false);
+
+        rv6xx_program_power_level_enter_state(rdev);
+
+        rv6xx_calculate_stepping_parameters(rdev, boot_ps);
+
+        if (pi->voltage_control)
+                rv6xx_program_voltage_gpio_pins(rdev);
+
+        rv6xx_generate_stepping_table(rdev, boot_ps);
+
+        rv6xx_program_stepping_parameters_except_lowest_entry(rdev);
+        rv6xx_program_stepping_parameters_lowest_entry(rdev);
+
+        rv6xx_program_power_level_low(rdev);
+        rv6xx_program_power_level_medium(rdev);
+        rv6xx_program_power_level_high(rdev);
+        rv6xx_program_vc(rdev);
+        rv6xx_program_at(rdev);
+
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                ret = r600_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+        }
+
+        rv6xx_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+
+        r600_start_dpm(rdev);
+
+        if (pi->voltage_control)
+                rv6xx_enable_static_voltage_control(rdev, boot_ps, false);
+
+        if (pi->dynamic_pcie_gen2)
+                rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, true);
+
+        if (pi->gfx_clock_gating)
+                r600_gfx_clockgating_enable(rdev, true);
+
+        return 0;
+}
+
+void rv6xx_dpm_disable(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+
+        if (!r600_dynamicpm_enabled(rdev))
+                return;
+
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
+        rv6xx_enable_display_gap(rdev, false);
+        rv6xx_clear_vc(rdev);
+        r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
+
+        if (pi->thermal_protection)
+                r600_enable_thermal_protection(rdev, false);
+
+        r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv6xx_enable_backbias(rdev, false);
+
+        rv6xx_enable_spread_spectrum(rdev, false);
+
+        if (pi->voltage_control)
+                rv6xx_enable_static_voltage_control(rdev, boot_ps, true);
+
+        if (pi->dynamic_pcie_gen2)
+                rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, false);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        if (pi->gfx_clock_gating)
+                r600_gfx_clockgating_enable(rdev, false);
+
+        r600_stop_dpm(rdev);
+}
+
+int rv6xx_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
+        struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
+        struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
+        int ret;
+
+        rv6xx_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+
+        rv6xx_clear_vc(rdev);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
+        r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
+
+        if (pi->thermal_protection)
+                r600_enable_thermal_protection(rdev, false);
+
+        r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
+
+        rv6xx_generate_transition_stepping(rdev, new_ps, old_ps);
+        rv6xx_program_power_level_medium_for_transition(rdev);
+
+        if (pi->voltage_control) {
+                rv6xx_set_sw_voltage_to_safe(rdev, new_ps, old_ps);
+                if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                        rv6xx_set_sw_voltage_to_low(rdev, old_ps);
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv6xx_set_safe_backbias(rdev, new_ps, old_ps);
+
+        if (pi->dynamic_pcie_gen2)
+                rv6xx_set_safe_pcie_gen2(rdev, new_ps, old_ps);
+
+        if (pi->voltage_control)
+                rv6xx_enable_dynamic_voltage_control(rdev, false);
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv6xx_enable_dynamic_backbias_control(rdev, false);
+
+        if (pi->voltage_control) {
+                if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                        rv6xx_step_voltage_if_increasing(rdev, new_ps, old_ps);
+                msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000);
+        }
+
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false);
+        r600_wait_for_power_level_unequal(rdev, R600_POWER_LEVEL_LOW);
+
+        rv6xx_generate_low_step(rdev, new_ps);
+        rv6xx_invalidate_intermediate_steps(rdev);
+        rv6xx_calculate_stepping_parameters(rdev, new_ps);
+        rv6xx_program_stepping_parameters_lowest_entry(rdev);
+        rv6xx_program_power_level_low_to_lowest_state(rdev);
+
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
+        r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
+        r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
+
+        if (pi->voltage_control) {
+                if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) {
+                        ret = rv6xx_step_voltage_if_decreasing(rdev, new_ps, old_ps);
+                        if (ret)
+                                return ret;
+                }
+                rv6xx_enable_dynamic_voltage_control(rdev, true);
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv6xx_enable_dynamic_backbias_control(rdev, true);
+
+        if (pi->dynamic_pcie_gen2)
+                rv6xx_enable_dynamic_pcie_gen2(rdev, new_ps, true);
+
+        rv6xx_reset_lvtm_data_sync(rdev);
+
+        rv6xx_generate_stepping_table(rdev, new_ps);
+        rv6xx_program_stepping_parameters_except_lowest_entry(rdev);
+        rv6xx_program_power_level_low(rdev);
+        rv6xx_program_power_level_medium(rdev);
+        rv6xx_program_power_level_high(rdev);
+        rv6xx_enable_medium(rdev);
+        rv6xx_enable_high(rdev);
+
+        if (pi->thermal_protection)
+                rv6xx_enable_thermal_protection(rdev, true);
+        rv6xx_program_vc(rdev);
+        rv6xx_program_at(rdev);
+
+        rv6xx_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+
+        return 0;
+}
+
+void rv6xx_setup_asic(struct radeon_device *rdev)
+{
+        r600_enable_acpi_pm(rdev);
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
+                rv6xx_enable_l0s(rdev);
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
+                rv6xx_enable_l1(rdev);
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
+                rv6xx_enable_pll_sleep_in_l1(rdev);
+}
+
+void rv6xx_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+        rv6xx_program_display_gap(rdev);
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void rv6xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                             struct radeon_ps *rps,
+                                             struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info)
+{
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (r600_is_uvd_state(rps->class, rps->class2)) {
+                rps->vclk = RV6XX_DEFAULT_VCLK_FREQ;
+                rps->dclk = RV6XX_DEFAULT_DCLK_FREQ;
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+                rdev->pm.dpm.boot_ps = rps;
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void rv6xx_parse_pplib_clock_info(struct radeon_device *rdev,
+                                         struct radeon_ps *rps, int index,
+                                         union pplib_clock_info *clock_info)
+{
+        struct rv6xx_ps *ps = rv6xx_get_ps(rps);
+        u32 sclk, mclk;
+        u16 vddc;
+        struct rv6xx_pl *pl;
+
+        switch (index) {
+        case 0:
+                pl = &ps->low;
+                break;
+        case 1:
+                pl = &ps->medium;
+                break;
+        case 2:
+        default:
+                pl = &ps->high;
+                break;
+        }
+
+        sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
+        sclk |= clock_info->r600.ucEngineClockHigh << 16;
+        mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
+        mclk |= clock_info->r600.ucMemoryClockHigh << 16;
+
+        pl->mclk = mclk;
+        pl->sclk = sclk;
+        pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
+        pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
+
+        /* patch up vddc if necessary */
+        if (pl->vddc == 0xff01) {
+                if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
+                        pl->vddc = vddc;
+        }
+
+        /* fix up pcie gen2 */
+        if (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) {
+                if ((rdev->family == CHIP_RV610) || (rdev->family == CHIP_RV630)) {
+                        if (pl->vddc < 1100)
+                                pl->flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
+                }
+        }
+
+        /* patch up boot state */
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                u16 vddc, vddci, mvdd;
+                radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
+                pl->mclk = rdev->clock.default_mclk;
+                pl->sclk = rdev->clock.default_sclk;
+                pl->vddc = vddc;
+        }
+}
+
+static int rv6xx_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j;
+        union pplib_clock_info *clock_info;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        struct rv6xx_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  power_info->pplib.ucNumStates, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+
+        for (i = 0; i < power_info->pplib.ucNumStates; i++) {
+                power_state = (union pplib_power_state *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usStateArrayOffset) +
+                         i * power_info->pplib.ucStateEntrySize);
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
+                         (power_state->v1.ucNonClockStateIndex *
+                          power_info->pplib.ucNonClockSize));
+                if (power_info->pplib.ucStateEntrySize - 1) {
+                        ps = kzalloc(sizeof(struct rv6xx_ps), GFP_KERNEL);
+                        if (ps == NULL) {
+                                kfree(rdev->pm.dpm.ps);
+                                return -ENOMEM;
+                        }
+                        rdev->pm.dpm.ps[i].ps_priv = ps;
+                        rv6xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                                         non_clock_info);
+                        for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
+                                clock_info = (union pplib_clock_info *)
+                                        (mode_info->atom_context->bios + data_offset +
+                                         le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
+                                         (power_state->v1.ucClockStateIndices[j] *
+                                          power_info->pplib.ucClockInfoSize));
+                                rv6xx_parse_pplib_clock_info(rdev,
+                                                             &rdev->pm.dpm.ps[i], j,
+                                                             clock_info);
+                        }
+                }
+        }
+        rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
+        return 0;
+}
+
+int rv6xx_dpm_init(struct radeon_device *rdev)
+{
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        uint16_t data_offset, size;
+        uint8_t frev, crev;
+        struct atom_clock_dividers dividers;
+        struct rv6xx_power_info *pi;
+        int ret;
+
+        pi = kzalloc(sizeof(struct rv6xx_power_info), GFP_KERNEL);
+        if (pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = pi;
+
+        ret = rv6xx_parse_power_table(rdev);
+        if (ret)
+                return ret;
+
+        if (rdev->pm.dpm.voltage_response_time == 0)
+                rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+        if (rdev->pm.dpm.backbias_response_time == 0)
+                rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->spll_ref_div = dividers.ref_div + 1;
+        else
+                pi->spll_ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->mpll_ref_div = dividers.ref_div + 1;
+        else
+                pi->mpll_ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        if (rdev->family >= CHIP_RV670)
+                pi->fb_div_scale = 1;
+        else
+                pi->fb_div_scale = 0;
+
+        pi->voltage_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
+
+        pi->gfx_clock_gating = true;
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                pi->sclk_ss = true;
+                pi->mclk_ss = true;
+                pi->dynamic_ss = true;
+        } else {
+                pi->sclk_ss = false;
+                pi->mclk_ss = false;
+                pi->dynamic_ss = false;
+        }
+
+        pi->dynamic_pcie_gen2 = true;
+
+        if (pi->gfx_clock_gating &&
+            (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
+                pi->thermal_protection = true;
+        else
+                pi->thermal_protection = false;
+
+        pi->display_gap = true;
+
+        return 0;
+}
+
+void rv6xx_dpm_print_power_state(struct radeon_device *rdev,
+                                 struct radeon_ps *rps)
+{
+        struct rv6xx_ps *ps = rv6xx_get_ps(rps);
+        struct rv6xx_pl *pl;
+
+        r600_dpm_print_class_info(rps->class, rps->class2);
+        r600_dpm_print_cap_info(rps->caps);
+        printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+        pl = &ps->low;
+        printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
+               pl->sclk, pl->mclk, pl->vddc);
+        pl = &ps->medium;
+        printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
+               pl->sclk, pl->mclk, pl->vddc);
+        pl = &ps->high;
+        printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
+               pl->sclk, pl->mclk, pl->vddc);
+        r600_dpm_print_ps_status(rdev, rps);
+}
+
+void rv6xx_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                       struct seq_file *m)
+{
+        struct radeon_ps *rps = rdev->pm.dpm.current_ps;
+        struct rv6xx_ps *ps = rv6xx_get_ps(rps);
+        struct rv6xx_pl *pl;
+        u32 current_index =
+                (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
+                CURRENT_PROFILE_INDEX_SHIFT;
+
+        if (current_index > 2) {
+                seq_printf(m, "invalid dpm profile %d\n", current_index);
+        } else {
+                if (current_index == 0)
+                        pl = &ps->low;
+                else if (current_index == 1)
+                        pl = &ps->medium;
+                else /* current_index == 2 */
+                        pl = &ps->high;
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
+                           current_index, pl->sclk, pl->mclk, pl->vddc);
+        }
+}
+
+void rv6xx_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+}
+
+u32 rv6xx_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps);
+
+        if (low)
+                return requested_state->low.sclk;
+        else
+                return requested_state->high.sclk;
+}
+
+u32 rv6xx_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps);
+
+        if (low)
+                return requested_state->low.mclk;
+        else
+                return requested_state->high.mclk;
+}
diff --git a/drivers/gpu/drm/radeon/rv6xx_dpm.h b/drivers/gpu/drm/radeon/rv6xx_dpm.h
new file mode 100644
index 000000000000..8035d53ebea6
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv6xx_dpm.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#ifndef __RV6XX_DPM_H__
+#define __RV6XX_DPM_H__
+
+#include "r600_dpm.h"
+
+/* Represents a single SCLK step. */
+struct rv6xx_sclk_stepping
+{
+    u32 vco_frequency;
+    u32 post_divider;
+};
+
+struct rv6xx_pm_hw_state {
+        u32 sclks[R600_PM_NUMBER_OF_ACTIVITY_LEVELS];
+        u32 mclks[R600_PM_NUMBER_OF_MCLKS];
+        u16 vddc[R600_PM_NUMBER_OF_VOLTAGE_LEVELS];
+        bool backbias[R600_PM_NUMBER_OF_VOLTAGE_LEVELS];
+        bool pcie_gen2[R600_PM_NUMBER_OF_ACTIVITY_LEVELS];
+        u8 high_sclk_index;
+        u8 medium_sclk_index;
+        u8 low_sclk_index;
+        u8 high_mclk_index;
+        u8 medium_mclk_index;
+        u8 low_mclk_index;
+        u8 high_vddc_index;
+        u8 medium_vddc_index;
+        u8 low_vddc_index;
+        u8 rp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS];
+        u8 lp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS];
+};
+
+struct rv6xx_power_info {
+        /* flags */
+        bool voltage_control;
+        bool sclk_ss;
+        bool mclk_ss;
+        bool dynamic_ss;
+        bool dynamic_pcie_gen2;
+        bool thermal_protection;
+        bool display_gap;
+        bool gfx_clock_gating;
+        /* clk values */
+        u32 fb_div_scale;
+        u32 spll_ref_div;
+        u32 mpll_ref_div;
+        u32 bsu;
+        u32 bsp;
+        /* */
+        u32 active_auto_throttle_sources;
+        /* current power state */
+        u32 restricted_levels;
+        struct rv6xx_pm_hw_state hw;
+};
+
+struct rv6xx_pl {
+        u32 sclk;
+        u32 mclk;
+        u16 vddc;
+        u32 flags;
+};
+
+struct rv6xx_ps {
+        struct rv6xx_pl high;
+        struct rv6xx_pl medium;
+        struct rv6xx_pl low;
+};
+
+#define RV6XX_DEFAULT_VCLK_FREQ  40000 /* 10 khz */
+#define RV6XX_DEFAULT_DCLK_FREQ  30000 /* 10 khz */
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv6xxd.h b/drivers/gpu/drm/radeon/rv6xxd.h
new file mode 100644
index 000000000000..34e86f90b431
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv6xxd.h
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef RV6XXD_H
+#define RV6XXD_H
+
+/* RV6xx power management */
+#define SPLL_CNTL_MODE                                    0x60c
+#       define SPLL_DIV_SYNC                              (1 << 5)
+
+#define GENERAL_PWRMGT                                    0x618
+#       define GLOBAL_PWRMGT_EN                           (1 << 0)
+#       define STATIC_PM_EN                               (1 << 1)
+#       define MOBILE_SU                                  (1 << 2)
+#       define THERMAL_PROTECTION_DIS                     (1 << 3)
+#       define THERMAL_PROTECTION_TYPE                    (1 << 4)
+#       define ENABLE_GEN2PCIE                            (1 << 5)
+#       define SW_GPIO_INDEX(x)                           ((x) << 6)
+#       define SW_GPIO_INDEX_MASK                         (3 << 6)
+#       define LOW_VOLT_D2_ACPI                           (1 << 8)
+#       define LOW_VOLT_D3_ACPI                           (1 << 9)
+#       define VOLT_PWRMGT_EN                             (1 << 10)
+#       define BACKBIAS_PAD_EN                            (1 << 16)
+#       define BACKBIAS_VALUE                             (1 << 17)
+#       define BACKBIAS_DPM_CNTL                          (1 << 18)
+#       define DYN_SPREAD_SPECTRUM_EN                     (1 << 21)
+
+#define MCLK_PWRMGT_CNTL                                  0x624
+#       define MPLL_PWRMGT_OFF                            (1 << 0)
+#       define YCLK_TURNOFF                               (1 << 1)
+#       define MPLL_TURNOFF                               (1 << 2)
+#       define SU_MCLK_USE_BCLK                           (1 << 3)
+#       define DLL_READY                                  (1 << 4)
+#       define MC_BUSY                                    (1 << 5)
+#       define MC_INT_CNTL                                (1 << 7)
+#       define MRDCKA_SLEEP                               (1 << 8)
+#       define MRDCKB_SLEEP                               (1 << 9)
+#       define MRDCKC_SLEEP                               (1 << 10)
+#       define MRDCKD_SLEEP                               (1 << 11)
+#       define MRDCKE_SLEEP                               (1 << 12)
+#       define MRDCKF_SLEEP                               (1 << 13)
+#       define MRDCKG_SLEEP                               (1 << 14)
+#       define MRDCKH_SLEEP                               (1 << 15)
+#       define MRDCKA_RESET                               (1 << 16)
+#       define MRDCKB_RESET                               (1 << 17)
+#       define MRDCKC_RESET                               (1 << 18)
+#       define MRDCKD_RESET                               (1 << 19)
+#       define MRDCKE_RESET                               (1 << 20)
+#       define MRDCKF_RESET                               (1 << 21)
+#       define MRDCKG_RESET                               (1 << 22)
+#       define MRDCKH_RESET                               (1 << 23)
+#       define DLL_READY_READ                             (1 << 24)
+#       define USE_DISPLAY_GAP                            (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                  (1 << 26)
+#       define USE_DISPLAY_GAP_CTXSW                      (1 << 27)
+#       define MPLL_TURNOFF_D2                            (1 << 28)
+#       define USE_DISPLAY_URGENT_CTXSW                   (1 << 29)
+
+#define MPLL_FREQ_LEVEL_0                                 0x6e8
+#       define LEVEL0_MPLL_POST_DIV(x)                    ((x) << 0)
+#       define LEVEL0_MPLL_POST_DIV_MASK                  (0xff << 0)
+#       define LEVEL0_MPLL_FB_DIV(x)                      ((x) << 8)
+#       define LEVEL0_MPLL_FB_DIV_MASK                    (0xfff << 8)
+#       define LEVEL0_MPLL_REF_DIV(x)                     ((x) << 20)
+#       define LEVEL0_MPLL_REF_DIV_MASK                   (0x3f << 20)
+#       define LEVEL0_MPLL_DIV_EN                         (1 << 28)
+#       define LEVEL0_DLL_BYPASS                          (1 << 29)
+#       define LEVEL0_DLL_RESET                           (1 << 30)
+
+#define VID_RT                                            0x6f8
+#       define VID_CRT(x)                                 ((x) << 0)
+#       define VID_CRT_MASK                               (0x1fff << 0)
+#       define VID_CRTU(x)                                ((x) << 13)
+#       define VID_CRTU_MASK                              (7 << 13)
+#       define SSTU(x)                                    ((x) << 16)
+#       define SSTU_MASK                                  (7 << 16)
+#       define VID_SWT(x)                                 ((x) << 19)
+#       define VID_SWT_MASK                               (0x1f << 19)
+#       define BRT(x)                                     ((x) << 24)
+#       define BRT_MASK                                   (0xff << 24)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                  0x70c
+#       define TARGET_PROFILE_INDEX_MASK                  (3 << 0)
+#       define TARGET_PROFILE_INDEX_SHIFT                 0
+#       define CURRENT_PROFILE_INDEX_MASK                 (3 << 2)
+#       define CURRENT_PROFILE_INDEX_SHIFT                2
+#       define DYN_PWR_ENTER_INDEX(x)                     ((x) << 4)
+#       define DYN_PWR_ENTER_INDEX_MASK                   (3 << 4)
+#       define DYN_PWR_ENTER_INDEX_SHIFT                  4
+#       define CURR_MCLK_INDEX_MASK                       (3 << 6)
+#       define CURR_MCLK_INDEX_SHIFT                      6
+#       define CURR_SCLK_INDEX_MASK                       (0x1f << 8)
+#       define CURR_SCLK_INDEX_SHIFT                      8
+#       define CURR_VID_INDEX_MASK                        (3 << 13)
+#       define CURR_VID_INDEX_SHIFT                       13
+
+#define VID_UPPER_GPIO_CNTL                               0x740
+#       define CTXSW_UPPER_GPIO_VALUES(x)                 ((x) << 0)
+#       define CTXSW_UPPER_GPIO_VALUES_MASK               (7 << 0)
+#       define HIGH_UPPER_GPIO_VALUES(x)                  ((x) << 3)
+#       define HIGH_UPPER_GPIO_VALUES_MASK                (7 << 3)
+#       define MEDIUM_UPPER_GPIO_VALUES(x)                ((x) << 6)
+#       define MEDIUM_UPPER_GPIO_VALUES_MASK              (7 << 6)
+#       define LOW_UPPER_GPIO_VALUES(x)                   ((x) << 9)
+#       define LOW_UPPER_GPIO_VALUES_MASK                 (7 << 9)
+#       define CTXSW_BACKBIAS_VALUE                       (1 << 12)
+#       define HIGH_BACKBIAS_VALUE                        (1 << 13)
+#       define MEDIUM_BACKBIAS_VALUE                      (1 << 14)
+#       define LOW_BACKBIAS_VALUE                         (1 << 15)
+
+#define CG_DISPLAY_GAP_CNTL                               0x7dc
+#       define DISP1_GAP(x)                               ((x) << 0)
+#       define DISP1_GAP_MASK                             (3 << 0)
+#       define DISP2_GAP(x)                               ((x) << 2)
+#       define DISP2_GAP_MASK                             (3 << 2)
+#       define VBI_TIMER_COUNT(x)                         ((x) << 4)
+#       define VBI_TIMER_COUNT_MASK                       (0x3fff << 4)
+#       define VBI_TIMER_UNIT(x)                          ((x) << 20)
+#       define VBI_TIMER_UNIT_MASK                        (7 << 20)
+#       define DISP1_GAP_MCHG(x)                          ((x) << 24)
+#       define DISP1_GAP_MCHG_MASK                        (3 << 24)
+#       define DISP2_GAP_MCHG(x)                          ((x) << 26)
+#       define DISP2_GAP_MCHG_MASK                        (3 << 26)
+
+#define CG_THERMAL_CTRL                                   0x7f0
+#       define DPM_EVENT_SRC(x)                           ((x) << 0)
+#       define DPM_EVENT_SRC_MASK                         (7 << 0)
+#       define THERM_INC_CLK                              (1 << 3)
+#       define TOFFSET(x)                                 ((x) << 4)
+#       define TOFFSET_MASK                               (0xff << 4)
+#       define DIG_THERM_DPM(x)                           ((x) << 12)
+#       define DIG_THERM_DPM_MASK                         (0xff << 12)
+#       define CTF_SEL(x)                                 ((x) << 20)
+#       define CTF_SEL_MASK                               (7 << 20)
+#       define CTF_PAD_POLARITY                           (1 << 23)
+#       define CTF_PAD_EN                                 (1 << 24)
+
+#define CG_SPLL_SPREAD_SPECTRUM_LOW                       0x820
+#       define SSEN                                       (1 << 0)
+#       define CLKS(x)                                    ((x) << 3)
+#       define CLKS_MASK                                  (0xff << 3)
+#       define CLKS_SHIFT                                 3
+#       define CLKV(x)                                    ((x) << 11)
+#       define CLKV_MASK                                  (0x7ff << 11)
+#       define CLKV_SHIFT                                 11
+#define CG_MPLL_SPREAD_SPECTRUM                           0x830
+
+#define CITF_CNTL                                       0x200c
+#       define BLACKOUT_RD                              (1 << 0)
+#       define BLACKOUT_WR                              (1 << 1)
+
+#define RAMCFG                                          0x2408
+#define         NOOFBANK_SHIFT                                  0
+#define         NOOFBANK_MASK                                   0x00000001
+#define         NOOFRANK_SHIFT                                  1
+#define         NOOFRANK_MASK                                   0x00000002
+#define         NOOFROWS_SHIFT                                  2
+#define         NOOFROWS_MASK                                   0x0000001C
+#define         NOOFCOLS_SHIFT                                  5
+#define         NOOFCOLS_MASK                                   0x00000060
+#define         CHANSIZE_SHIFT                                  7
+#define         CHANSIZE_MASK                                   0x00000080
+#define         BURSTLENGTH_SHIFT                               8
+#define         BURSTLENGTH_MASK                                0x00000100
+#define         CHANSIZE_OVERRIDE                               (1 << 10)
+
+#define SQM_RATIO                                       0x2424
+#       define STATE0(x)                                ((x) << 0)
+#       define STATE0_MASK                              (0xff << 0)
+#       define STATE1(x)                                ((x) << 8)
+#       define STATE1_MASK                              (0xff << 8)
+#       define STATE2(x)                                ((x) << 16)
+#       define STATE2_MASK                              (0xff << 16)
+#       define STATE3(x)                                ((x) << 24)
+#       define STATE3_MASK                              (0xff << 24)
+
+#define ARB_RFSH_CNTL                                   0x2460
+#       define ENABLE                                   (1 << 0)
+#define ARB_RFSH_RATE                                   0x2464
+#       define POWERMODE0(x)                            ((x) << 0)
+#       define POWERMODE0_MASK                          (0xff << 0)
+#       define POWERMODE1(x)                            ((x) << 8)
+#       define POWERMODE1_MASK                          (0xff << 8)
+#       define POWERMODE2(x)                            ((x) << 16)
+#       define POWERMODE2_MASK                          (0xff << 16)
+#       define POWERMODE3(x)                            ((x) << 24)
+#       define POWERMODE3_MASK                          (0xff << 24)
+
+#define MC_SEQ_DRAM                                     0x2608
+#       define CKE_DYN                                  (1 << 12)
+
+#define MC_SEQ_CMD                                      0x26c4
+
+#define MC_SEQ_RESERVE_S                                0x2890
+#define MC_SEQ_RESERVE_M                                0x2894
+
+#define LVTMA_DATA_SYNCHRONIZATION                      0x7adc
+#       define LVTMA_PFREQCHG                           (1 << 8)
+#define DCE3_LVTMA_DATA_SYNCHRONIZATION                 0x7f98
+
+/* PCIE indirect regs */
+#define PCIE_P_CNTL                                       0x40
+#       define P_PLL_PWRDN_IN_L1L23                       (1 << 3)
+#       define P_PLL_BUF_PDNB                             (1 << 4)
+#       define P_PLL_PDNB                                 (1 << 9)
+#       define P_ALLOW_PRX_FRONTEND_SHUTOFF               (1 << 12)
+/* PCIE PORT indirect regs */
+#define PCIE_LC_CNTL                                      0xa0
+#       define LC_L0S_INACTIVITY(x)                       ((x) << 8)
+#       define LC_L0S_INACTIVITY_MASK                     (0xf << 8)
+#       define LC_L0S_INACTIVITY_SHIFT                    8
+#       define LC_L1_INACTIVITY(x)                        ((x) << 12)
+#       define LC_L1_INACTIVITY_MASK                      (0xf << 12)
+#       define LC_L1_INACTIVITY_SHIFT                     12
+#       define LC_PMI_TO_L1_DIS                           (1 << 16)
+#       define LC_ASPM_TO_L1_DIS                          (1 << 24)
+#define PCIE_LC_SPEED_CNTL                                0xa4
+#       define LC_GEN2_EN                                 (1 << 0)
+#       define LC_INITIATE_LINK_SPEED_CHANGE              (1 << 7)
+#       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_HW_VOLTAGE_IF_CONTROL(x)                ((x) << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_MASK              (3 << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_SHIFT             12
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 24)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv730_dpm.c b/drivers/gpu/drm/radeon/rv730_dpm.c
new file mode 100644
index 000000000000..3f5e1cf138ba
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv730_dpm.c
@@ -0,0 +1,508 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "rv730d.h"
+#include "r600_dpm.h"
+#include "rv770_dpm.h"
+#include "atom.h"
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps);
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
+
+int rv730_populate_sclk_value(struct radeon_device *rdev,
+                              u32 engine_clock,
+                              RV770_SMC_SCLK_VALUE *sclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        u32 spll_func_cntl = pi->clk_regs.rv730.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 = pi->clk_regs.rv730.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 = pi->clk_regs.rv730.cg_spll_func_cntl_3;
+        u32 cg_spll_spread_spectrum = pi->clk_regs.rv730.cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2 = pi->clk_regs.rv730.cg_spll_spread_spectrum_2;
+        u64 tmp;
+        u32 reference_clock = rdev->clock.spll.reference_freq;
+        u32 reference_divider, post_divider;
+        u32 fbdiv;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             engine_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        reference_divider = 1 + dividers.ref_div;
+
+        if (dividers.enable_post_div)
+                post_divider = ((dividers.post_div >> 4) & 0xf) +
+                        (dividers.post_div & 0xf) + 2;
+        else
+                post_divider = 1;
+
+        tmp = (u64) engine_clock * reference_divider * post_divider * 16384;
+        do_div(tmp, reference_clock);
+        fbdiv = (u32) tmp;
+
+        /* set up registers */
+        if (dividers.enable_post_div)
+                spll_func_cntl |= SPLL_DIVEN;
+        else
+                spll_func_cntl &= ~SPLL_DIVEN;
+        spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK);
+        spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
+        spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf);
+        spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(2);
+
+        spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
+        spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
+        spll_func_cntl_3 |= SPLL_DITHEN;
+
+        if (pi->sclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = engine_clock * post_divider;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+                        u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+                        u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000);
+
+                        cg_spll_spread_spectrum &= ~CLK_S_MASK;
+                        cg_spll_spread_spectrum |= CLK_S(clk_s);
+                        cg_spll_spread_spectrum |= SSEN;
+
+                        cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
+                        cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
+                }
+        }
+
+        sclk->sclk_value = cpu_to_be32(engine_clock);
+        sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+        sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum);
+        sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2);
+
+        return 0;
+}
+
+int rv730_populate_mclk_value(struct radeon_device *rdev,
+                              u32 engine_clock, u32 memory_clock,
+                              LPRV7XX_SMC_MCLK_VALUE mclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 mclk_pwrmgt_cntl = pi->clk_regs.rv730.mclk_pwrmgt_cntl;
+        u32 dll_cntl = pi->clk_regs.rv730.dll_cntl;
+        u32 mpll_func_cntl = pi->clk_regs.rv730.mpll_func_cntl;
+        u32 mpll_func_cntl_2 = pi->clk_regs.rv730.mpll_func_cntl2;
+        u32 mpll_func_cntl_3 = pi->clk_regs.rv730.mpll_func_cntl3;
+        u32 mpll_ss = pi->clk_regs.rv730.mpll_ss;
+        u32 mpll_ss2 = pi->clk_regs.rv730.mpll_ss2;
+        struct atom_clock_dividers dividers;
+        u32 post_divider, reference_divider;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                             memory_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        reference_divider = dividers.ref_div + 1;
+
+        if (dividers.enable_post_div)
+                post_divider = ((dividers.post_div >> 4) & 0xf) +
+                        (dividers.post_div & 0xf) + 2;
+        else
+                post_divider = 1;
+
+        /* setup the registers */
+        if (dividers.enable_post_div)
+                mpll_func_cntl |= MPLL_DIVEN;
+        else
+                mpll_func_cntl &= ~MPLL_DIVEN;
+
+        mpll_func_cntl &= ~(MPLL_REF_DIV_MASK | MPLL_HILEN_MASK | MPLL_LOLEN_MASK);
+        mpll_func_cntl |= MPLL_REF_DIV(dividers.ref_div);
+        mpll_func_cntl |= MPLL_HILEN((dividers.post_div >> 4) & 0xf);
+        mpll_func_cntl |= MPLL_LOLEN(dividers.post_div & 0xf);
+
+        mpll_func_cntl_3 &= ~MPLL_FB_DIV_MASK;
+        mpll_func_cntl_3 |= MPLL_FB_DIV(dividers.fb_div);
+        if (dividers.enable_dithen)
+                mpll_func_cntl_3 |= MPLL_DITHEN;
+        else
+                mpll_func_cntl_3 &= ~MPLL_DITHEN;
+
+        if (pi->mclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = memory_clock * post_divider;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
+                        u32 reference_clock = rdev->clock.mpll.reference_freq;
+                        u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+                        u32 clk_v = ss.percentage * dividers.fb_div / (clk_s * 10000);
+
+                        mpll_ss &= ~CLK_S_MASK;
+                        mpll_ss |= CLK_S(clk_s);
+                        mpll_ss |= SSEN;
+
+                        mpll_ss2 &= ~CLK_V_MASK;
+                        mpll_ss |= CLK_V(clk_v);
+                }
+        }
+
+
+        mclk->mclk730.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        mclk->mclk730.vDLL_CNTL = cpu_to_be32(dll_cntl);
+        mclk->mclk730.mclk_value = cpu_to_be32(memory_clock);
+        mclk->mclk730.vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl);
+        mclk->mclk730.vMPLL_FUNC_CNTL2 = cpu_to_be32(mpll_func_cntl_2);
+        mclk->mclk730.vMPLL_FUNC_CNTL3 = cpu_to_be32(mpll_func_cntl_3);
+        mclk->mclk730.vMPLL_SS = cpu_to_be32(mpll_ss);
+        mclk->mclk730.vMPLL_SS2 = cpu_to_be32(mpll_ss2);
+
+        return 0;
+}
+
+void rv730_read_clock_registers(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        pi->clk_regs.rv730.cg_spll_func_cntl =
+                RREG32(CG_SPLL_FUNC_CNTL);
+        pi->clk_regs.rv730.cg_spll_func_cntl_2 =
+                RREG32(CG_SPLL_FUNC_CNTL_2);
+        pi->clk_regs.rv730.cg_spll_func_cntl_3 =
+                RREG32(CG_SPLL_FUNC_CNTL_3);
+        pi->clk_regs.rv730.cg_spll_spread_spectrum =
+                RREG32(CG_SPLL_SPREAD_SPECTRUM);
+        pi->clk_regs.rv730.cg_spll_spread_spectrum_2 =
+                RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
+
+        pi->clk_regs.rv730.mclk_pwrmgt_cntl =
+                RREG32(TCI_MCLK_PWRMGT_CNTL);
+        pi->clk_regs.rv730.dll_cntl =
+                RREG32(TCI_DLL_CNTL);
+        pi->clk_regs.rv730.mpll_func_cntl =
+                RREG32(CG_MPLL_FUNC_CNTL);
+        pi->clk_regs.rv730.mpll_func_cntl2 =
+                RREG32(CG_MPLL_FUNC_CNTL_2);
+        pi->clk_regs.rv730.mpll_func_cntl3 =
+                RREG32(CG_MPLL_FUNC_CNTL_3);
+        pi->clk_regs.rv730.mpll_ss =
+                RREG32(CG_TCI_MPLL_SPREAD_SPECTRUM);
+        pi->clk_regs.rv730.mpll_ss2 =
+                RREG32(CG_TCI_MPLL_SPREAD_SPECTRUM_2);
+}
+
+int rv730_populate_smc_acpi_state(struct radeon_device *rdev,
+                                  RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 mpll_func_cntl = 0;
+        u32 mpll_func_cntl_2 = 0 ;
+        u32 mpll_func_cntl_3 = 0;
+        u32 mclk_pwrmgt_cntl;
+        u32 dll_cntl;
+        u32 spll_func_cntl;
+        u32 spll_func_cntl_2;
+        u32 spll_func_cntl_3;
+
+        table->ACPIState = table->initialState;
+        table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+        if (pi->acpi_vddc) {
+                rv770_populate_vddc_value(rdev, pi->acpi_vddc,
+                                          &table->ACPIState.levels[0].vddc);
+                table->ACPIState.levels[0].gen2PCIE = pi->pcie_gen2 ?
+                        pi->acpi_pcie_gen2 : 0;
+                table->ACPIState.levels[0].gen2XSP =
+                        pi->acpi_pcie_gen2;
+        } else {
+                rv770_populate_vddc_value(rdev, pi->min_vddc_in_table,
+                                          &table->ACPIState.levels[0].vddc);
+                table->ACPIState.levels[0].gen2PCIE = 0;
+        }
+
+        mpll_func_cntl = pi->clk_regs.rv730.mpll_func_cntl;
+        mpll_func_cntl_2 = pi->clk_regs.rv730.mpll_func_cntl2;
+        mpll_func_cntl_3 = pi->clk_regs.rv730.mpll_func_cntl3;
+
+        mpll_func_cntl |= MPLL_RESET | MPLL_BYPASS_EN;
+        mpll_func_cntl &= ~MPLL_SLEEP;
+
+        mpll_func_cntl_2 &= ~MCLK_MUX_SEL_MASK;
+        mpll_func_cntl_2 |= MCLK_MUX_SEL(1);
+
+        mclk_pwrmgt_cntl = (MRDCKA_RESET |
+                            MRDCKB_RESET |
+                            MRDCKC_RESET |
+                            MRDCKD_RESET |
+                            MRDCKE_RESET |
+                            MRDCKF_RESET |
+                            MRDCKG_RESET |
+                            MRDCKH_RESET |
+                            MRDCKA_SLEEP |
+                            MRDCKB_SLEEP |
+                            MRDCKC_SLEEP |
+                            MRDCKD_SLEEP |
+                            MRDCKE_SLEEP |
+                            MRDCKF_SLEEP |
+                            MRDCKG_SLEEP |
+                            MRDCKH_SLEEP);
+
+        dll_cntl = 0xff000000;
+
+        spll_func_cntl = pi->clk_regs.rv730.cg_spll_func_cntl;
+        spll_func_cntl_2 = pi->clk_regs.rv730.cg_spll_func_cntl_2;
+        spll_func_cntl_3 = pi->clk_regs.rv730.cg_spll_func_cntl_3;
+
+        spll_func_cntl |= SPLL_RESET | SPLL_BYPASS_EN;
+        spll_func_cntl &= ~SPLL_SLEEP;
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+        table->ACPIState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL2 = cpu_to_be32(mpll_func_cntl_2);
+        table->ACPIState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL3 = cpu_to_be32(mpll_func_cntl_3);
+        table->ACPIState.levels[0].mclk.mclk730.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        table->ACPIState.levels[0].mclk.mclk730.vDLL_CNTL = cpu_to_be32(dll_cntl);
+
+        table->ACPIState.levels[0].mclk.mclk730.mclk_value = 0;
+
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+
+        table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+        rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
+
+        table->ACPIState.levels[1] = table->ACPIState.levels[0];
+        table->ACPIState.levels[2] = table->ACPIState.levels[0];
+
+        return 0;
+}
+
+int rv730_populate_smc_initial_state(struct radeon_device *rdev,
+                                     struct radeon_ps *radeon_state,
+                                     RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 a_t;
+
+        table->initialState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv730.mpll_func_cntl);
+        table->initialState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL2 =
+                cpu_to_be32(pi->clk_regs.rv730.mpll_func_cntl2);
+        table->initialState.levels[0].mclk.mclk730.vMPLL_FUNC_CNTL3 =
+                cpu_to_be32(pi->clk_regs.rv730.mpll_func_cntl3);
+        table->initialState.levels[0].mclk.mclk730.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(pi->clk_regs.rv730.mclk_pwrmgt_cntl);
+        table->initialState.levels[0].mclk.mclk730.vDLL_CNTL =
+                cpu_to_be32(pi->clk_regs.rv730.dll_cntl);
+        table->initialState.levels[0].mclk.mclk730.vMPLL_SS =
+                cpu_to_be32(pi->clk_regs.rv730.mpll_ss);
+        table->initialState.levels[0].mclk.mclk730.vMPLL_SS2 =
+                cpu_to_be32(pi->clk_regs.rv730.mpll_ss2);
+
+        table->initialState.levels[0].mclk.mclk730.mclk_value =
+                cpu_to_be32(initial_state->low.mclk);
+
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv730.cg_spll_func_cntl);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv730.cg_spll_func_cntl_2);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(pi->clk_regs.rv730.cg_spll_func_cntl_3);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
+                cpu_to_be32(pi->clk_regs.rv730.cg_spll_spread_spectrum);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
+                cpu_to_be32(pi->clk_regs.rv730.cg_spll_spread_spectrum_2);
+
+        table->initialState.levels[0].sclk.sclk_value =
+                cpu_to_be32(initial_state->low.sclk);
+
+        table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
+
+        table->initialState.levels[0].seqValue =
+                rv770_get_seq_value(rdev, &initial_state->low);
+
+        rv770_populate_vddc_value(rdev,
+                                  initial_state->low.vddc,
+                                  &table->initialState.levels[0].vddc);
+        rv770_populate_initial_mvdd_value(rdev,
+                                          &table->initialState.levels[0].mvdd);
+
+        a_t = CG_R(0xffff) | CG_L(0);
+
+        table->initialState.levels[0].aT = cpu_to_be32(a_t);
+
+        table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
+
+        if (pi->boot_in_gen2)
+                table->initialState.levels[0].gen2PCIE = 1;
+        else
+                table->initialState.levels[0].gen2PCIE = 0;
+        if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
+                table->initialState.levels[0].gen2XSP = 1;
+        else
+                table->initialState.levels[0].gen2XSP = 0;
+
+        table->initialState.levels[1] = table->initialState.levels[0];
+        table->initialState.levels[2] = table->initialState.levels[0];
+
+        table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        return 0;
+}
+
+void rv730_program_memory_timing_parameters(struct radeon_device *rdev,
+                                            struct radeon_ps *radeon_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+        u32 arb_refresh_rate = 0;
+        u32 dram_timing = 0;
+        u32 dram_timing2 = 0;
+        u32 old_dram_timing = 0;
+        u32 old_dram_timing2 = 0;
+
+        arb_refresh_rate = RREG32(MC_ARB_RFSH_RATE) &
+                ~(POWERMODE1_MASK | POWERMODE2_MASK | POWERMODE3_MASK);
+        arb_refresh_rate |=
+                (POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) |
+                 POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) |
+                 POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk)));
+        WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate);
+
+        /* save the boot dram timings */
+        old_dram_timing = RREG32(MC_ARB_DRAM_TIMING);
+        old_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+
+        radeon_atom_set_engine_dram_timings(rdev,
+                                            state->high.sclk,
+                                            state->high.mclk);
+
+        dram_timing = RREG32(MC_ARB_DRAM_TIMING);
+        dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+
+        WREG32(MC_ARB_DRAM_TIMING_3, dram_timing);
+        WREG32(MC_ARB_DRAM_TIMING2_3, dram_timing2);
+
+        radeon_atom_set_engine_dram_timings(rdev,
+                                            state->medium.sclk,
+                                            state->medium.mclk);
+
+        dram_timing = RREG32(MC_ARB_DRAM_TIMING);
+        dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+
+        WREG32(MC_ARB_DRAM_TIMING_2, dram_timing);
+        WREG32(MC_ARB_DRAM_TIMING2_2, dram_timing2);
+
+        radeon_atom_set_engine_dram_timings(rdev,
+                                            state->low.sclk,
+                                            state->low.mclk);
+
+        dram_timing = RREG32(MC_ARB_DRAM_TIMING);
+        dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+
+        WREG32(MC_ARB_DRAM_TIMING_1, dram_timing);
+        WREG32(MC_ARB_DRAM_TIMING2_1, dram_timing2);
+
+        /* restore the boot dram timings */
+        WREG32(MC_ARB_DRAM_TIMING, old_dram_timing);
+        WREG32(MC_ARB_DRAM_TIMING2, old_dram_timing2);
+
+}
+
+void rv730_start_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
+
+        WREG32_P(TCI_MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
+
+        WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+}
+
+void rv730_stop_dpm(struct radeon_device *rdev)
+{
+        PPSMC_Result result;
+
+        result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled);
+
+        if (result != PPSMC_Result_OK)
+                DRM_ERROR("Could not force DPM to low\n");
+
+        WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
+
+        WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
+
+        WREG32_P(TCI_MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
+}
+
+void rv730_program_dcodt(struct radeon_device *rdev, bool use_dcodt)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 i = use_dcodt ? 0 : 1;
+        u32 mc4_io_pad_cntl;
+
+        mc4_io_pad_cntl = RREG32(MC4_IO_DQ_PAD_CNTL_D0_I0);
+        mc4_io_pad_cntl &= 0xFFFFFF00;
+        mc4_io_pad_cntl |= pi->odt_value_0[i];
+        WREG32(MC4_IO_DQ_PAD_CNTL_D0_I0, mc4_io_pad_cntl);
+        WREG32(MC4_IO_DQ_PAD_CNTL_D0_I1, mc4_io_pad_cntl);
+
+        mc4_io_pad_cntl = RREG32(MC4_IO_QS_PAD_CNTL_D0_I0);
+        mc4_io_pad_cntl &= 0xFFFFFF00;
+        mc4_io_pad_cntl |= pi->odt_value_1[i];
+        WREG32(MC4_IO_QS_PAD_CNTL_D0_I0, mc4_io_pad_cntl);
+        WREG32(MC4_IO_QS_PAD_CNTL_D0_I1, mc4_io_pad_cntl);
+}
+
+void rv730_get_odt_values(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 mc4_io_pad_cntl;
+
+        pi->odt_value_0[0] = (u8)0;
+        pi->odt_value_1[0] = (u8)0x80;
+
+        mc4_io_pad_cntl = RREG32(MC4_IO_DQ_PAD_CNTL_D0_I0);
+        pi->odt_value_0[1] = (u8)(mc4_io_pad_cntl & 0xff);
+
+        mc4_io_pad_cntl = RREG32(MC4_IO_QS_PAD_CNTL_D0_I0);
+        pi->odt_value_1[1] = (u8)(mc4_io_pad_cntl & 0xff);
+}
diff --git a/drivers/gpu/drm/radeon/rv730d.h b/drivers/gpu/drm/radeon/rv730d.h
new file mode 100644
index 000000000000..f0a7954fb1cb
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv730d.h
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef RV730_H
+#define RV730_H
+
+#define CG_SPLL_FUNC_CNTL                               0x600
+#define         SPLL_RESET                              (1 << 0)
+#define         SPLL_SLEEP                              (1 << 1)
+#define         SPLL_DIVEN                              (1 << 2)
+#define         SPLL_BYPASS_EN                          (1 << 3)
+#define         SPLL_REF_DIV(x)                         ((x) << 4)
+#define         SPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         SPLL_HILEN(x)                           ((x) << 12)
+#define         SPLL_HILEN_MASK                         (0xf << 12)
+#define         SPLL_LOLEN(x)                           ((x) << 16)
+#define         SPLL_LOLEN_MASK                         (0xf << 16)
+#define CG_SPLL_FUNC_CNTL_2                             0x604
+#define         SCLK_MUX_SEL(x)                         ((x) << 0)
+#define         SCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_SPLL_FUNC_CNTL_3                             0x608
+#define         SPLL_FB_DIV(x)                          ((x) << 0)
+#define         SPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         SPLL_DITHEN                             (1 << 28)
+
+#define CG_MPLL_FUNC_CNTL                               0x624
+#define         MPLL_RESET                              (1 << 0)
+#define         MPLL_SLEEP                              (1 << 1)
+#define         MPLL_DIVEN                              (1 << 2)
+#define         MPLL_BYPASS_EN                          (1 << 3)
+#define         MPLL_REF_DIV(x)                         ((x) << 4)
+#define         MPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         MPLL_HILEN(x)                           ((x) << 12)
+#define         MPLL_HILEN_MASK                         (0xf << 12)
+#define         MPLL_LOLEN(x)                           ((x) << 16)
+#define         MPLL_LOLEN_MASK                         (0xf << 16)
+#define CG_MPLL_FUNC_CNTL_2                             0x628
+#define         MCLK_MUX_SEL(x)                         ((x) << 0)
+#define         MCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_MPLL_FUNC_CNTL_3                             0x62c
+#define         MPLL_FB_DIV(x)                          ((x) << 0)
+#define         MPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         MPLL_DITHEN                             (1 << 28)
+
+#define CG_TCI_MPLL_SPREAD_SPECTRUM                     0x634
+#define CG_TCI_MPLL_SPREAD_SPECTRUM_2                   0x638
+#define GENERAL_PWRMGT                                  0x63c
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define ENABLE_GEN2PCIE                          (1 << 4)
+#       define ENABLE_GEN2XSP                           (1 << 5)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (3 << 6)
+#       define LOW_VOLT_D2_ACPI                         (1 << 8)
+#       define LOW_VOLT_D3_ACPI                         (1 << 9)
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define BACKBIAS_PAD_EN                          (1 << 18)
+#       define BACKBIAS_VALUE                           (1 << 19)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#       define AC_DC_SW                                 (1 << 24)
+
+#define SCLK_PWRMGT_CNTL                                  0x644
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_LOW_D1                                (1 << 1)
+#       define FIR_RESET                                  (1 << 4)
+#       define FIR_FORCE_TREND_SEL                        (1 << 5)
+#       define FIR_TREND_MODE                             (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 7)
+#       define GFX_CLK_FORCE_ON                           (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                        (1 << 9)
+#       define GFX_CLK_FORCE_OFF                          (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                        (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                        (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                        (1 << 13)
+
+#define TCI_MCLK_PWRMGT_CNTL                            0x648
+#       define MPLL_PWRMGT_OFF                          (1 << 5)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCKA_SLEEP                             (1 << 8)
+#       define MRDCKB_SLEEP                             (1 << 9)
+#       define MRDCKC_SLEEP                             (1 << 10)
+#       define MRDCKD_SLEEP                             (1 << 11)
+#       define MRDCKE_SLEEP                             (1 << 12)
+#       define MRDCKF_SLEEP                             (1 << 13)
+#       define MRDCKG_SLEEP                             (1 << 14)
+#       define MRDCKH_SLEEP                             (1 << 15)
+#       define MRDCKA_RESET                             (1 << 16)
+#       define MRDCKB_RESET                             (1 << 17)
+#       define MRDCKC_RESET                             (1 << 18)
+#       define MRDCKD_RESET                             (1 << 19)
+#       define MRDCKE_RESET                             (1 << 20)
+#       define MRDCKF_RESET                             (1 << 21)
+#       define MRDCKG_RESET                             (1 << 22)
+#       define MRDCKH_RESET                             (1 << 23)
+#       define DLL_READY_READ                           (1 << 24)
+#       define USE_DISPLAY_GAP                          (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                (1 << 26)
+#       define MPLL_TURNOFF_D2                          (1 << 28)
+#define TCI_DLL_CNTL                                    0x64c
+
+#define CG_PG_CNTL                                      0x858
+#       define PWRGATE_ENABLE                           (1 << 0)
+
+#define CG_AT                                           0x6d4
+#define         CG_R(x)                                 ((x) << 0)
+#define         CG_R_MASK                               (0xffff << 0)
+#define         CG_L(x)                                 ((x) << 16)
+#define         CG_L_MASK                               (0xffff << 16)
+
+#define CG_SPLL_SPREAD_SPECTRUM                         0x790
+#define         SSEN                                    (1 << 0)
+#define         CLK_S(x)                                ((x) << 4)
+#define         CLK_S_MASK                              (0xfff << 4)
+#define CG_SPLL_SPREAD_SPECTRUM_2                       0x794
+#define         CLK_V(x)                                ((x) << 0)
+#define         CLK_V_MASK                              (0x3ffffff << 0)
+
+#define MC_ARB_DRAM_TIMING                              0x2774
+#define MC_ARB_DRAM_TIMING2                             0x2778
+
+#define MC_ARB_RFSH_RATE                                0x27b0
+#define         POWERMODE0(x)                           ((x) << 0)
+#define         POWERMODE0_MASK                         (0xff << 0)
+#define         POWERMODE1(x)                           ((x) << 8)
+#define         POWERMODE1_MASK                         (0xff << 8)
+#define         POWERMODE2(x)                           ((x) << 16)
+#define         POWERMODE2_MASK                         (0xff << 16)
+#define         POWERMODE3(x)                           ((x) << 24)
+#define         POWERMODE3_MASK                         (0xff << 24)
+
+#define MC_ARB_DRAM_TIMING_1                            0x27f0
+#define MC_ARB_DRAM_TIMING_2                            0x27f4
+#define MC_ARB_DRAM_TIMING_3                            0x27f8
+#define MC_ARB_DRAM_TIMING2_1                           0x27fc
+#define MC_ARB_DRAM_TIMING2_2                           0x2800
+#define MC_ARB_DRAM_TIMING2_3                           0x2804
+
+#define MC4_IO_DQ_PAD_CNTL_D0_I0                        0x2978
+#define MC4_IO_DQ_PAD_CNTL_D0_I1                        0x297c
+#define MC4_IO_QS_PAD_CNTL_D0_I0                        0x2980
+#define MC4_IO_QS_PAD_CNTL_D0_I1                        0x2984
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv740_dpm.c b/drivers/gpu/drm/radeon/rv740_dpm.c
new file mode 100644
index 000000000000..c4c8da501da8
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv740_dpm.c
@@ -0,0 +1,416 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "rv740d.h"
+#include "r600_dpm.h"
+#include "rv770_dpm.h"
+#include "atom.h"
+
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
+
+u32 rv740_get_decoded_reference_divider(u32 encoded_ref)
+{
+        u32 ref = 0;
+
+        switch (encoded_ref) {
+        case 0:
+                ref = 1;
+                break;
+        case 16:
+                ref = 2;
+                break;
+        case 17:
+                ref = 3;
+                break;
+        case 18:
+                ref = 2;
+                break;
+        case 19:
+                ref = 3;
+                break;
+        case 20:
+                ref = 4;
+                break;
+        case 21:
+                ref = 5;
+                break;
+        default:
+                DRM_ERROR("Invalid encoded Reference Divider\n");
+                ref = 0;
+                break;
+        }
+
+        return ref;
+}
+
+struct dll_speed_setting {
+        u16 min;
+        u16 max;
+        u32 dll_speed;
+};
+
+static struct dll_speed_setting dll_speed_table[16] =
+{
+        { 270, 320, 0x0f },
+        { 240, 270, 0x0e },
+        { 200, 240, 0x0d },
+        { 180, 200, 0x0c },
+        { 160, 180, 0x0b },
+        { 140, 160, 0x0a },
+        { 120, 140, 0x09 },
+        { 110, 120, 0x08 },
+        {  95, 110, 0x07 },
+        {  85,  95, 0x06 },
+        {  78,  85, 0x05 },
+        {  70,  78, 0x04 },
+        {  65,  70, 0x03 },
+        {  60,  65, 0x02 },
+        {  42,  60, 0x01 },
+        {  00,  42, 0x00 }
+};
+
+u32 rv740_get_dll_speed(bool is_gddr5, u32 memory_clock)
+{
+        int i;
+        u32 factor;
+        u16 data_rate;
+
+        if (is_gddr5)
+                factor = 4;
+        else
+                factor = 2;
+
+        data_rate = (u16)(memory_clock * factor / 1000);
+
+        if (data_rate < dll_speed_table[0].max) {
+                for (i = 0; i < 16; i++) {
+                        if (data_rate > dll_speed_table[i].min &&
+                            data_rate <= dll_speed_table[i].max)
+                                return dll_speed_table[i].dll_speed;
+                }
+        }
+
+        DRM_DEBUG_KMS("Target MCLK greater than largest MCLK in DLL speed table\n");
+
+        return 0x0f;
+}
+
+int rv740_populate_sclk_value(struct radeon_device *rdev, u32 engine_clock,
+                              RV770_SMC_SCLK_VALUE *sclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        u32 spll_func_cntl = pi->clk_regs.rv770.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 = pi->clk_regs.rv770.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 = pi->clk_regs.rv770.cg_spll_func_cntl_3;
+        u32 cg_spll_spread_spectrum = pi->clk_regs.rv770.cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2 = pi->clk_regs.rv770.cg_spll_spread_spectrum_2;
+        u64 tmp;
+        u32 reference_clock = rdev->clock.spll.reference_freq;
+        u32 reference_divider;
+        u32 fbdiv;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             engine_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        reference_divider = 1 + dividers.ref_div;
+
+        tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16384;
+        do_div(tmp, reference_clock);
+        fbdiv = (u32) tmp;
+
+        spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
+        spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
+        spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(2);
+
+        spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
+        spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
+        spll_func_cntl_3 |= SPLL_DITHEN;
+
+        if (pi->sclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = engine_clock * dividers.post_div;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+                        u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+                        u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
+
+                        cg_spll_spread_spectrum &= ~CLK_S_MASK;
+                        cg_spll_spread_spectrum |= CLK_S(clk_s);
+                        cg_spll_spread_spectrum |= SSEN;
+
+                        cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
+                        cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
+                }
+        }
+
+        sclk->sclk_value = cpu_to_be32(engine_clock);
+        sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+        sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum);
+        sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2);
+
+        return 0;
+}
+
+int rv740_populate_mclk_value(struct radeon_device *rdev,
+                              u32 engine_clock, u32 memory_clock,
+                              RV7XX_SMC_MCLK_VALUE *mclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 mpll_ad_func_cntl = pi->clk_regs.rv770.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 = pi->clk_regs.rv770.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl = pi->clk_regs.rv770.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 = pi->clk_regs.rv770.mpll_dq_func_cntl_2;
+        u32 mclk_pwrmgt_cntl = pi->clk_regs.rv770.mclk_pwrmgt_cntl;
+        u32 dll_cntl = pi->clk_regs.rv770.dll_cntl;
+        u32 mpll_ss1 = pi->clk_regs.rv770.mpll_ss1;
+        u32 mpll_ss2 = pi->clk_regs.rv770.mpll_ss2;
+        struct atom_clock_dividers dividers;
+        u32 ibias;
+        u32 dll_speed;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                             memory_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        ibias = rv770_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
+
+        mpll_ad_func_cntl &= ~(CLKR_MASK |
+                               YCLK_POST_DIV_MASK |
+                               CLKF_MASK |
+                               CLKFRAC_MASK |
+                               IBIAS_MASK);
+        mpll_ad_func_cntl |= CLKR(dividers.ref_div);
+        mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
+        mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
+        mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
+        mpll_ad_func_cntl |= IBIAS(ibias);
+
+        if (dividers.vco_mode)
+                mpll_ad_func_cntl_2 |= VCO_MODE;
+        else
+                mpll_ad_func_cntl_2 &= ~VCO_MODE;
+
+        if (pi->mem_gddr5) {
+                mpll_dq_func_cntl &= ~(CLKR_MASK |
+                                       YCLK_POST_DIV_MASK |
+                                       CLKF_MASK |
+                                       CLKFRAC_MASK |
+                                       IBIAS_MASK);
+                mpll_dq_func_cntl |= CLKR(dividers.ref_div);
+                mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
+                mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
+                mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
+                mpll_dq_func_cntl |= IBIAS(ibias);
+
+                if (dividers.vco_mode)
+                        mpll_dq_func_cntl_2 |= VCO_MODE;
+                else
+                        mpll_dq_func_cntl_2 &= ~VCO_MODE;
+        }
+
+        if (pi->mclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = memory_clock * dividers.post_div;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
+                        u32 reference_clock = rdev->clock.mpll.reference_freq;
+                        u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
+                        u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
+                        u32 clk_v = 0x40000 * ss.percentage *
+                                (dividers.whole_fb_div + (dividers.frac_fb_div / 8)) / (clk_s * 10000);
+
+                        mpll_ss1 &= ~CLKV_MASK;
+                        mpll_ss1 |= CLKV(clk_v);
+
+                        mpll_ss2 &= ~CLKS_MASK;
+                        mpll_ss2 |= CLKS(clk_s);
+                }
+        }
+
+        dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
+                                        memory_clock);
+
+        mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
+        mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
+
+        mclk->mclk770.mclk_value = cpu_to_be32(memory_clock);
+        mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+        mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
+        mclk->mclk770.vMPLL_SS = cpu_to_be32(mpll_ss1);
+        mclk->mclk770.vMPLL_SS2 = cpu_to_be32(mpll_ss2);
+
+        return 0;
+}
+
+void rv740_read_clock_registers(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        pi->clk_regs.rv770.cg_spll_func_cntl =
+                RREG32(CG_SPLL_FUNC_CNTL);
+        pi->clk_regs.rv770.cg_spll_func_cntl_2 =
+                RREG32(CG_SPLL_FUNC_CNTL_2);
+        pi->clk_regs.rv770.cg_spll_func_cntl_3 =
+                RREG32(CG_SPLL_FUNC_CNTL_3);
+        pi->clk_regs.rv770.cg_spll_spread_spectrum =
+                RREG32(CG_SPLL_SPREAD_SPECTRUM);
+        pi->clk_regs.rv770.cg_spll_spread_spectrum_2 =
+                RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
+
+        pi->clk_regs.rv770.mpll_ad_func_cntl =
+                RREG32(MPLL_AD_FUNC_CNTL);
+        pi->clk_regs.rv770.mpll_ad_func_cntl_2 =
+                RREG32(MPLL_AD_FUNC_CNTL_2);
+        pi->clk_regs.rv770.mpll_dq_func_cntl =
+                RREG32(MPLL_DQ_FUNC_CNTL);
+        pi->clk_regs.rv770.mpll_dq_func_cntl_2 =
+                RREG32(MPLL_DQ_FUNC_CNTL_2);
+        pi->clk_regs.rv770.mclk_pwrmgt_cntl =
+                RREG32(MCLK_PWRMGT_CNTL);
+        pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL);
+        pi->clk_regs.rv770.mpll_ss1 = RREG32(MPLL_SS1);
+        pi->clk_regs.rv770.mpll_ss2 = RREG32(MPLL_SS2);
+}
+
+int rv740_populate_smc_acpi_state(struct radeon_device *rdev,
+                                  RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 mpll_ad_func_cntl = pi->clk_regs.rv770.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 = pi->clk_regs.rv770.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl = pi->clk_regs.rv770.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 = pi->clk_regs.rv770.mpll_dq_func_cntl_2;
+        u32 spll_func_cntl = pi->clk_regs.rv770.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 = pi->clk_regs.rv770.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 = pi->clk_regs.rv770.cg_spll_func_cntl_3;
+        u32 mclk_pwrmgt_cntl = pi->clk_regs.rv770.mclk_pwrmgt_cntl;
+        u32 dll_cntl = pi->clk_regs.rv770.dll_cntl;
+
+        table->ACPIState = table->initialState;
+
+        table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+        if (pi->acpi_vddc) {
+                rv770_populate_vddc_value(rdev, pi->acpi_vddc,
+                                          &table->ACPIState.levels[0].vddc);
+                table->ACPIState.levels[0].gen2PCIE =
+                        pi->pcie_gen2 ?
+                        pi->acpi_pcie_gen2 : 0;
+                table->ACPIState.levels[0].gen2XSP =
+                        pi->acpi_pcie_gen2;
+        } else {
+                rv770_populate_vddc_value(rdev, pi->min_vddc_in_table,
+                                          &table->ACPIState.levels[0].vddc);
+                table->ACPIState.levels[0].gen2PCIE = 0;
+        }
+
+        mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
+
+        mpll_dq_func_cntl_2 |= BYPASS | BIAS_GEN_PDNB | RESET_EN;
+
+        mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
+                             MRDCKA1_RESET |
+                             MRDCKB0_RESET |
+                             MRDCKB1_RESET |
+                             MRDCKC0_RESET |
+                             MRDCKC1_RESET |
+                             MRDCKD0_RESET |
+                             MRDCKD1_RESET);
+
+        dll_cntl |= (MRDCKA0_BYPASS |
+                     MRDCKA1_BYPASS |
+                     MRDCKB0_BYPASS |
+                     MRDCKB1_BYPASS |
+                     MRDCKC0_BYPASS |
+                     MRDCKC1_BYPASS |
+                     MRDCKD0_BYPASS |
+                     MRDCKD1_BYPASS);
+
+        spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN;
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+        table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
+
+        table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0;
+
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+
+        table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+        table->ACPIState.levels[1] = table->ACPIState.levels[0];
+        table->ACPIState.levels[2] = table->ACPIState.levels[0];
+
+        rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
+
+        return 0;
+}
+
+void rv740_enable_mclk_spread_spectrum(struct radeon_device *rdev,
+                                       bool enable)
+{
+        if (enable)
+                WREG32_P(MPLL_CNTL_MODE, SS_SSEN, ~SS_SSEN);
+        else
+                WREG32_P(MPLL_CNTL_MODE, 0, ~SS_SSEN);
+}
+
+u8 rv740_get_mclk_frequency_ratio(u32 memory_clock)
+{
+        u8 mc_para_index;
+
+        if ((memory_clock < 10000) || (memory_clock > 47500))
+                mc_para_index = 0x00;
+        else
+                mc_para_index = (u8)((memory_clock - 10000) / 2500);
+
+        return mc_para_index;
+}
diff --git a/drivers/gpu/drm/radeon/rv740d.h b/drivers/gpu/drm/radeon/rv740d.h
new file mode 100644
index 000000000000..fe5ab075dc17
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv740d.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef RV740_H
+#define RV740_H
+
+#define CG_SPLL_FUNC_CNTL                               0x600
+#define         SPLL_RESET                              (1 << 0)
+#define         SPLL_SLEEP                              (1 << 1)
+#define         SPLL_BYPASS_EN                          (1 << 3)
+#define         SPLL_REF_DIV(x)                         ((x) << 4)
+#define         SPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         SPLL_PDIV_A(x)                          ((x) << 20)
+#define         SPLL_PDIV_A_MASK                        (0x7f << 20)
+#define CG_SPLL_FUNC_CNTL_2                             0x604
+#define         SCLK_MUX_SEL(x)                         ((x) << 0)
+#define         SCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_SPLL_FUNC_CNTL_3                             0x608
+#define         SPLL_FB_DIV(x)                          ((x) << 0)
+#define         SPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         SPLL_DITHEN                             (1 << 28)
+
+#define MPLL_CNTL_MODE                                  0x61c
+#define         SS_SSEN                                 (1 << 24)
+
+#define MPLL_AD_FUNC_CNTL                               0x624
+#define         CLKF(x)                                 ((x) << 0)
+#define         CLKF_MASK                               (0x7f << 0)
+#define         CLKR(x)                                 ((x) << 7)
+#define         CLKR_MASK                               (0x1f << 7)
+#define         CLKFRAC(x)                              ((x) << 12)
+#define         CLKFRAC_MASK                            (0x1f << 12)
+#define         YCLK_POST_DIV(x)                        ((x) << 17)
+#define         YCLK_POST_DIV_MASK                      (3 << 17)
+#define         IBIAS(x)                                ((x) << 20)
+#define         IBIAS_MASK                              (0x3ff << 20)
+#define         RESET                                   (1 << 30)
+#define         PDNB                                    (1 << 31)
+#define MPLL_AD_FUNC_CNTL_2                             0x628
+#define         BYPASS                                  (1 << 19)
+#define         BIAS_GEN_PDNB                           (1 << 24)
+#define         RESET_EN                                (1 << 25)
+#define         VCO_MODE                                (1 << 29)
+#define MPLL_DQ_FUNC_CNTL                               0x62c
+#define MPLL_DQ_FUNC_CNTL_2                             0x630
+
+#define MCLK_PWRMGT_CNTL                                0x648
+#define         DLL_SPEED(x)                            ((x) << 0)
+#define         DLL_SPEED_MASK                          (0x1f << 0)
+#       define MPLL_PWRMGT_OFF                          (1 << 5)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCKA0_SLEEP                            (1 << 8)
+#       define MRDCKA1_SLEEP                            (1 << 9)
+#       define MRDCKB0_SLEEP                            (1 << 10)
+#       define MRDCKB1_SLEEP                            (1 << 11)
+#       define MRDCKC0_SLEEP                            (1 << 12)
+#       define MRDCKC1_SLEEP                            (1 << 13)
+#       define MRDCKD0_SLEEP                            (1 << 14)
+#       define MRDCKD1_SLEEP                            (1 << 15)
+#       define MRDCKA0_RESET                            (1 << 16)
+#       define MRDCKA1_RESET                            (1 << 17)
+#       define MRDCKB0_RESET                            (1 << 18)
+#       define MRDCKB1_RESET                            (1 << 19)
+#       define MRDCKC0_RESET                            (1 << 20)
+#       define MRDCKC1_RESET                            (1 << 21)
+#       define MRDCKD0_RESET                            (1 << 22)
+#       define MRDCKD1_RESET                            (1 << 23)
+#       define DLL_READY_READ                           (1 << 24)
+#       define USE_DISPLAY_GAP                          (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                (1 << 26)
+#       define MPLL_TURNOFF_D2                          (1 << 28)
+#define DLL_CNTL                                        0x64c
+#       define MRDCKA0_BYPASS                           (1 << 24)
+#       define MRDCKA1_BYPASS                           (1 << 25)
+#       define MRDCKB0_BYPASS                           (1 << 26)
+#       define MRDCKB1_BYPASS                           (1 << 27)
+#       define MRDCKC0_BYPASS                           (1 << 28)
+#       define MRDCKC1_BYPASS                           (1 << 29)
+#       define MRDCKD0_BYPASS                           (1 << 30)
+#       define MRDCKD1_BYPASS                           (1 << 31)
+
+#define CG_SPLL_SPREAD_SPECTRUM                         0x790
+#define         SSEN                                    (1 << 0)
+#define         CLK_S(x)                                ((x) << 4)
+#define         CLK_S_MASK                              (0xfff << 4)
+#define CG_SPLL_SPREAD_SPECTRUM_2                       0x794
+#define         CLK_V(x)                                ((x) << 0)
+#define         CLK_V_MASK                              (0x3ffffff << 0)
+
+#define MPLL_SS1                                        0x85c
+#define         CLKV(x)                                 ((x) << 0)
+#define         CLKV_MASK                               (0x3ffffff << 0)
+#define MPLL_SS2                                        0x860
+#define         CLKS(x)                                 ((x) << 0)
+#define         CLKS_MASK                               (0xfff << 0)
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv770_dpm.c b/drivers/gpu/drm/radeon/rv770_dpm.c
new file mode 100644
index 000000000000..9af464d48eaa
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv770_dpm.c
@@ -0,0 +1,2493 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "rv770d.h"
+#include "r600_dpm.h"
+#include "rv770_dpm.h"
+#include "cypress_dpm.h"
+#include "atom.h"
+#include <linux/seq_file.h>
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define MC_CG_SEQ_DRAMCONF_S0       0x05
+#define MC_CG_SEQ_DRAMCONF_S1       0x06
+
+#define PCIE_BUS_CLK                10000
+#define TCLK                        (PCIE_BUS_CLK / 10)
+
+#define SMC_RAM_END 0xC000
+
+struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps)
+{
+        struct rv7xx_ps *ps = rps->ps_priv;
+
+        return ps;
+}
+
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+static void rv770_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                               bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        if (enable) {
+                tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
+                tmp |= LC_GEN2_EN_STRAP;
+        } else {
+                if (!pi->boot_in_gen2) {
+                        tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
+                        tmp &= ~LC_GEN2_EN_STRAP;
+                }
+        }
+        if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
+            (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
+                WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
+
+}
+
+static void rv770_enable_l0s(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
+        tmp |= LC_L0S_INACTIVITY(3);
+        WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
+}
+
+static void rv770_enable_l1(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+        tmp &= ~LC_L1_INACTIVITY_MASK;
+        tmp |= LC_L1_INACTIVITY(4);
+        tmp &= ~LC_PMI_TO_L1_DIS;
+        tmp &= ~LC_ASPM_TO_L1_DIS;
+        WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
+}
+
+static void rv770_enable_pll_sleep_in_l1(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
+        tmp |= LC_L1_INACTIVITY(8);
+        WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
+
+        /* NOTE, this is a PCIE indirect reg, not PCIE PORT */
+        tmp = RREG32_PCIE(PCIE_P_CNTL);
+        tmp |= P_PLL_PWRDN_IN_L1L23;
+        tmp &= ~P_PLL_BUF_PDNB;
+        tmp &= ~P_PLL_PDNB;
+        tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
+        WREG32_PCIE(PCIE_P_CNTL, tmp);
+}
+
+static void rv770_gfx_clock_gating_enable(struct radeon_device *rdev,
+                                          bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+        else {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+                RREG32(GB_TILING_CONFIG);
+        }
+}
+
+static void rv770_mg_clock_gating_enable(struct radeon_device *rdev,
+                                         bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (enable) {
+                u32 mgcg_cgtt_local0;
+
+                if (rdev->family == CHIP_RV770)
+                        mgcg_cgtt_local0 = RV770_MGCGTTLOCAL0_DFLT;
+                else
+                        mgcg_cgtt_local0 = RV7XX_MGCGTTLOCAL0_DFLT;
+
+                WREG32(CG_CGTT_LOCAL_0, mgcg_cgtt_local0);
+                WREG32(CG_CGTT_LOCAL_1, (RV770_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF));
+
+                if (pi->mgcgtssm)
+                        WREG32(CGTS_SM_CTRL_REG, RV770_MGCGCGTSSMCTRL_DFLT);
+        } else {
+                WREG32(CG_CGTT_LOCAL_0, 0xFFFFFFFF);
+                WREG32(CG_CGTT_LOCAL_1, 0xFFFFCFFF);
+        }
+}
+
+void rv770_restore_cgcg(struct radeon_device *rdev)
+{
+        bool dpm_en = false, cg_en = false;
+
+        if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
+                dpm_en = true;
+        if (RREG32(SCLK_PWRMGT_CNTL) & DYN_GFX_CLK_OFF_EN)
+                cg_en = true;
+
+        if (dpm_en && !cg_en)
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+}
+
+static void rv770_start_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
+
+        WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
+
+        WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+}
+
+void rv770_stop_dpm(struct radeon_device *rdev)
+{
+        PPSMC_Result result;
+
+        result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled);
+
+        if (result != PPSMC_Result_OK)
+                DRM_ERROR("Could not force DPM to low.\n");
+
+        WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
+
+        WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
+
+        WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
+}
+
+bool rv770_dpm_enabled(struct radeon_device *rdev)
+{
+        if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
+                return true;
+        else
+                return false;
+}
+
+void rv770_enable_thermal_protection(struct radeon_device *rdev,
+                                     bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+        else
+                WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+}
+
+void rv770_enable_acpi_pm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
+}
+
+u8 rv770_get_seq_value(struct radeon_device *rdev,
+                       struct rv7xx_pl *pl)
+{
+        return (pl->flags & ATOM_PPLIB_R600_FLAGS_LOWPOWER) ?
+                MC_CG_SEQ_DRAMCONF_S0 : MC_CG_SEQ_DRAMCONF_S1;
+}
+
+int rv770_read_smc_soft_register(struct radeon_device *rdev,
+                                 u16 reg_offset, u32 *value)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        return rv770_read_smc_sram_dword(rdev,
+                                         pi->soft_regs_start + reg_offset,
+                                         value, pi->sram_end);
+}
+
+int rv770_write_smc_soft_register(struct radeon_device *rdev,
+                                  u16 reg_offset, u32 value)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        return rv770_write_smc_sram_dword(rdev,
+                                          pi->soft_regs_start + reg_offset,
+                                          value, pi->sram_end);
+}
+
+int rv770_populate_smc_t(struct radeon_device *rdev,
+                         struct radeon_ps *radeon_state,
+                         RV770_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+        int a_n;
+        int a_d;
+        u8 l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
+        u8 r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
+        u32 a_t;
+
+        l[0] = 0;
+        r[2] = 100;
+
+        a_n = (int)state->medium.sclk * pi->lmp +
+                (int)state->low.sclk * (R600_AH_DFLT - pi->rlp);
+        a_d = (int)state->low.sclk * (100 - (int)pi->rlp) +
+                (int)state->medium.sclk * pi->lmp;
+
+        l[1] = (u8)(pi->lmp - (int)pi->lmp * a_n / a_d);
+        r[0] = (u8)(pi->rlp + (100 - (int)pi->rlp) * a_n / a_d);
+
+        a_n = (int)state->high.sclk * pi->lhp + (int)state->medium.sclk *
+                (R600_AH_DFLT - pi->rmp);
+        a_d = (int)state->medium.sclk * (100 - (int)pi->rmp) +
+                (int)state->high.sclk * pi->lhp;
+
+        l[2] = (u8)(pi->lhp - (int)pi->lhp * a_n / a_d);
+        r[1] = (u8)(pi->rmp + (100 - (int)pi->rmp) * a_n / a_d);
+
+        for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++) {
+                a_t = CG_R(r[i] * pi->bsp / 200) | CG_L(l[i] * pi->bsp / 200);
+                smc_state->levels[i].aT = cpu_to_be32(a_t);
+        }
+
+        a_t = CG_R(r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200) |
+                CG_L(l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200);
+
+        smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].aT =
+                cpu_to_be32(a_t);
+
+        return 0;
+}
+
+int rv770_populate_smc_sp(struct radeon_device *rdev,
+                          struct radeon_ps *radeon_state,
+                          RV770_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+
+        for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++)
+                smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
+
+        smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].bSP =
+                cpu_to_be32(pi->psp);
+
+        return 0;
+}
+
+static void rv770_calculate_fractional_mpll_feedback_divider(u32 memory_clock,
+                                                             u32 reference_clock,
+                                                             bool gddr5,
+                                                             struct atom_clock_dividers *dividers,
+                                                             u32 *clkf,
+                                                             u32 *clkfrac)
+{
+        u32 post_divider, reference_divider, feedback_divider8;
+        u32 fyclk;
+
+        if (gddr5)
+                fyclk = (memory_clock * 8) / 2;
+        else
+                fyclk = (memory_clock * 4) / 2;
+
+        post_divider = dividers->post_div;
+        reference_divider = dividers->ref_div;
+
+        feedback_divider8 =
+                (8 * fyclk * reference_divider * post_divider) / reference_clock;
+
+        *clkf = feedback_divider8 / 8;
+        *clkfrac = feedback_divider8 % 8;
+}
+
+static int rv770_encode_yclk_post_div(u32 postdiv, u32 *encoded_postdiv)
+{
+        int ret = 0;
+
+        switch (postdiv) {
+        case 1:
+                *encoded_postdiv = 0;
+                break;
+        case 2:
+                *encoded_postdiv = 1;
+                break;
+        case 4:
+                *encoded_postdiv = 2;
+                break;
+        case 8:
+                *encoded_postdiv = 3;
+                break;
+        case 16:
+                *encoded_postdiv = 4;
+                break;
+        default:
+                ret = -EINVAL;
+                break;
+        }
+
+    return ret;
+}
+
+u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf)
+{
+        if (clkf <= 0x10)
+                return 0x4B;
+        if (clkf <= 0x19)
+                return 0x5B;
+        if (clkf <= 0x21)
+                return 0x2B;
+        if (clkf <= 0x27)
+                return 0x6C;
+        if (clkf <= 0x31)
+                return 0x9D;
+        return 0xC6;
+}
+
+static int rv770_populate_mclk_value(struct radeon_device *rdev,
+                                     u32 engine_clock, u32 memory_clock,
+                                     RV7XX_SMC_MCLK_VALUE *mclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 encoded_reference_dividers[] = { 0, 16, 17, 20, 21 };
+        u32 mpll_ad_func_cntl =
+                pi->clk_regs.rv770.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl =
+                pi->clk_regs.rv770.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_dq_func_cntl_2;
+        u32 mclk_pwrmgt_cntl =
+                pi->clk_regs.rv770.mclk_pwrmgt_cntl;
+        u32 dll_cntl = pi->clk_regs.rv770.dll_cntl;
+        struct atom_clock_dividers dividers;
+        u32 reference_clock = rdev->clock.mpll.reference_freq;
+        u32 clkf, clkfrac;
+        u32 postdiv_yclk;
+        u32 ibias;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
+                                             memory_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        if ((dividers.ref_div < 1) || (dividers.ref_div > 5))
+                return -EINVAL;
+
+        rv770_calculate_fractional_mpll_feedback_divider(memory_clock, reference_clock,
+                                                         pi->mem_gddr5,
+                                                         &dividers, &clkf, &clkfrac);
+
+        ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
+        if (ret)
+                return ret;
+
+        ibias = rv770_map_clkf_to_ibias(rdev, clkf);
+
+        mpll_ad_func_cntl &= ~(CLKR_MASK |
+                               YCLK_POST_DIV_MASK |
+                               CLKF_MASK |
+                               CLKFRAC_MASK |
+                               IBIAS_MASK);
+        mpll_ad_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
+        mpll_ad_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
+        mpll_ad_func_cntl |= CLKF(clkf);
+        mpll_ad_func_cntl |= CLKFRAC(clkfrac);
+        mpll_ad_func_cntl |= IBIAS(ibias);
+
+        if (dividers.vco_mode)
+                mpll_ad_func_cntl_2 |= VCO_MODE;
+        else
+                mpll_ad_func_cntl_2 &= ~VCO_MODE;
+
+        if (pi->mem_gddr5) {
+                rv770_calculate_fractional_mpll_feedback_divider(memory_clock,
+                                                                 reference_clock,
+                                                                 pi->mem_gddr5,
+                                                                 &dividers, &clkf, &clkfrac);
+
+                ibias = rv770_map_clkf_to_ibias(rdev, clkf);
+
+                ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
+                if (ret)
+                        return ret;
+
+                mpll_dq_func_cntl &= ~(CLKR_MASK |
+                                       YCLK_POST_DIV_MASK |
+                                       CLKF_MASK |
+                                       CLKFRAC_MASK |
+                                       IBIAS_MASK);
+                mpll_dq_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
+                mpll_dq_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
+                mpll_dq_func_cntl |= CLKF(clkf);
+                mpll_dq_func_cntl |= CLKFRAC(clkfrac);
+                mpll_dq_func_cntl |= IBIAS(ibias);
+
+                if (dividers.vco_mode)
+                        mpll_dq_func_cntl_2 |= VCO_MODE;
+                else
+                        mpll_dq_func_cntl_2 &= ~VCO_MODE;
+        }
+
+        mclk->mclk770.mclk_value = cpu_to_be32(memory_clock);
+        mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+        mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
+
+        return 0;
+}
+
+static int rv770_populate_sclk_value(struct radeon_device *rdev,
+                                     u32 engine_clock,
+                                     RV770_SMC_SCLK_VALUE *sclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        u32 spll_func_cntl =
+                pi->clk_regs.rv770.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 =
+                pi->clk_regs.rv770.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 =
+                pi->clk_regs.rv770.cg_spll_func_cntl_3;
+        u32 cg_spll_spread_spectrum =
+                pi->clk_regs.rv770.cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2 =
+                pi->clk_regs.rv770.cg_spll_spread_spectrum_2;
+        u64 tmp;
+        u32 reference_clock = rdev->clock.spll.reference_freq;
+        u32 reference_divider, post_divider;
+        u32 fbdiv;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             engine_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        reference_divider = 1 + dividers.ref_div;
+
+        if (dividers.enable_post_div)
+                post_divider = (0x0f & (dividers.post_div >> 4)) + (0x0f & dividers.post_div) + 2;
+        else
+                post_divider = 1;
+
+        tmp = (u64) engine_clock * reference_divider * post_divider * 16384;
+        do_div(tmp, reference_clock);
+        fbdiv = (u32) tmp;
+
+        if (dividers.enable_post_div)
+                spll_func_cntl |= SPLL_DIVEN;
+        else
+                spll_func_cntl &= ~SPLL_DIVEN;
+        spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK);
+        spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
+        spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf);
+        spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(2);
+
+        spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
+        spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
+        spll_func_cntl_3 |= SPLL_DITHEN;
+
+        if (pi->sclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = engine_clock * post_divider;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+                        u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+                        u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000);
+
+                        cg_spll_spread_spectrum &= ~CLKS_MASK;
+                        cg_spll_spread_spectrum |= CLKS(clk_s);
+                        cg_spll_spread_spectrum |= SSEN;
+
+                        cg_spll_spread_spectrum_2 &= ~CLKV_MASK;
+                        cg_spll_spread_spectrum_2 |= CLKV(clk_v);
+                }
+        }
+
+        sclk->sclk_value = cpu_to_be32(engine_clock);
+        sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+        sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum);
+        sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2);
+
+        return 0;
+}
+
+int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc,
+                              RV770_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+
+        if (!pi->voltage_control) {
+                voltage->index = 0;
+                voltage->value = 0;
+                return 0;
+        }
+
+        for (i = 0; i < pi->valid_vddc_entries; i++) {
+                if (vddc <= pi->vddc_table[i].vddc) {
+                        voltage->index = pi->vddc_table[i].vddc_index;
+                        voltage->value = cpu_to_be16(vddc);
+                        break;
+                }
+        }
+
+        if (i == pi->valid_vddc_entries)
+                return -EINVAL;
+
+        return 0;
+}
+
+int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
+                              RV770_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (!pi->mvdd_control) {
+                voltage->index = MVDD_HIGH_INDEX;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+                return 0;
+        }
+
+        if (mclk <= pi->mvdd_split_frequency) {
+                voltage->index = MVDD_LOW_INDEX;
+                voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
+        } else {
+                voltage->index = MVDD_HIGH_INDEX;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+        }
+
+        return 0;
+}
+
+static int rv770_convert_power_level_to_smc(struct radeon_device *rdev,
+                                            struct rv7xx_pl *pl,
+                                            RV770_SMC_HW_PERFORMANCE_LEVEL *level,
+                                            u8 watermark_level)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int ret;
+
+        level->gen2PCIE = pi->pcie_gen2 ?
+                ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
+        level->gen2XSP  = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0;
+        level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0;
+        level->displayWatermark = watermark_level;
+
+        if (rdev->family == CHIP_RV740)
+                ret = rv740_populate_sclk_value(rdev, pl->sclk,
+                                                &level->sclk);
+        else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                ret = rv730_populate_sclk_value(rdev, pl->sclk,
+                                                &level->sclk);
+        else
+                ret = rv770_populate_sclk_value(rdev, pl->sclk,
+                                                &level->sclk);
+        if (ret)
+                return ret;
+
+        if (rdev->family == CHIP_RV740) {
+                if (pi->mem_gddr5) {
+                        if (pl->mclk <= pi->mclk_strobe_mode_threshold)
+                                level->strobeMode =
+                                        rv740_get_mclk_frequency_ratio(pl->mclk) | 0x10;
+                        else
+                                level->strobeMode = 0;
+
+                        if (pl->mclk > pi->mclk_edc_enable_threshold)
+                                level->mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
+                        else
+                                level->mcFlags =  0;
+                }
+                ret = rv740_populate_mclk_value(rdev, pl->sclk,
+                                                pl->mclk, &level->mclk);
+        } else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                ret = rv730_populate_mclk_value(rdev, pl->sclk,
+                                                pl->mclk, &level->mclk);
+        else
+                ret = rv770_populate_mclk_value(rdev, pl->sclk,
+                                                pl->mclk, &level->mclk);
+        if (ret)
+                return ret;
+
+        ret = rv770_populate_vddc_value(rdev, pl->vddc,
+                                        &level->vddc);
+        if (ret)
+                return ret;
+
+        ret = rv770_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
+
+        return ret;
+}
+
+static int rv770_convert_power_state_to_smc(struct radeon_device *rdev,
+                                            struct radeon_ps *radeon_state,
+                                            RV770_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+        int ret;
+
+        if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
+                smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        ret = rv770_convert_power_level_to_smc(rdev,
+                                               &state->low,
+                                               &smc_state->levels[0],
+                                               PPSMC_DISPLAY_WATERMARK_LOW);
+        if (ret)
+                return ret;
+
+        ret = rv770_convert_power_level_to_smc(rdev,
+                                               &state->medium,
+                                               &smc_state->levels[1],
+                                               PPSMC_DISPLAY_WATERMARK_LOW);
+        if (ret)
+                return ret;
+
+        ret = rv770_convert_power_level_to_smc(rdev,
+                                               &state->high,
+                                               &smc_state->levels[2],
+                                               PPSMC_DISPLAY_WATERMARK_HIGH);
+        if (ret)
+                return ret;
+
+        smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1;
+        smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2;
+        smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3;
+
+        smc_state->levels[0].seqValue = rv770_get_seq_value(rdev,
+                                                            &state->low);
+        smc_state->levels[1].seqValue = rv770_get_seq_value(rdev,
+                                                            &state->medium);
+        smc_state->levels[2].seqValue = rv770_get_seq_value(rdev,
+                                                            &state->high);
+
+        rv770_populate_smc_sp(rdev, radeon_state, smc_state);
+
+        return rv770_populate_smc_t(rdev, radeon_state, smc_state);
+
+}
+
+u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev,
+                                        u32 engine_clock)
+{
+        u32 dram_rows;
+        u32 dram_refresh_rate;
+        u32 mc_arb_rfsh_rate;
+        u32 tmp;
+
+        tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
+        dram_rows = 1 << (tmp + 10);
+        tmp = RREG32(MC_SEQ_MISC0) & 3;
+        dram_refresh_rate = 1 << (tmp + 3);
+        mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
+
+        return mc_arb_rfsh_rate;
+}
+
+static void rv770_program_memory_timing_parameters(struct radeon_device *rdev,
+                                                   struct radeon_ps *radeon_state)
+{
+        struct rv7xx_ps *state = rv770_get_ps(radeon_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 sqm_ratio;
+        u32 arb_refresh_rate;
+        u32 high_clock;
+
+        if (state->high.sclk < (state->low.sclk * 0xFF / 0x40))
+                high_clock = state->high.sclk;
+        else
+                high_clock = (state->low.sclk * 0xFF / 0x40);
+
+        radeon_atom_set_engine_dram_timings(rdev, high_clock,
+                                            state->high.mclk);
+
+        sqm_ratio =
+                STATE0(64 * high_clock / pi->boot_sclk) |
+                STATE1(64 * high_clock / state->low.sclk) |
+                STATE2(64 * high_clock / state->medium.sclk) |
+                STATE3(64 * high_clock / state->high.sclk);
+        WREG32(MC_ARB_SQM_RATIO, sqm_ratio);
+
+        arb_refresh_rate =
+                POWERMODE0(rv770_calculate_memory_refresh_rate(rdev, pi->boot_sclk)) |
+                POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) |
+                POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) |
+                POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk));
+        WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate);
+}
+
+void rv770_enable_backbias(struct radeon_device *rdev,
+                           bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN, ~BACKBIAS_PAD_EN);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN));
+}
+
+static void rv770_enable_spread_spectrum(struct radeon_device *rdev,
+                                         bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (enable) {
+                if (pi->sclk_ss)
+                        WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
+
+                if (pi->mclk_ss) {
+                        if (rdev->family == CHIP_RV740)
+                                rv740_enable_mclk_spread_spectrum(rdev, true);
+                }
+        } else {
+                WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
+
+                WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
+
+                WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
+
+                if (rdev->family == CHIP_RV740)
+                        rv740_enable_mclk_spread_spectrum(rdev, false);
+        }
+}
+
+static void rv770_program_mpll_timing_parameters(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if ((rdev->family == CHIP_RV770) && !pi->mem_gddr5) {
+                WREG32(MPLL_TIME,
+                       (MPLL_LOCK_TIME(R600_MPLLLOCKTIME_DFLT * pi->ref_div) |
+                        MPLL_RESET_TIME(R600_MPLLRESETTIME_DFLT)));
+        }
+}
+
+void rv770_setup_bsp(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 xclk = radeon_get_xclk(rdev);
+
+        r600_calculate_u_and_p(pi->asi,
+                               xclk,
+                               16,
+                               &pi->bsp,
+                               &pi->bsu);
+
+        r600_calculate_u_and_p(pi->pasi,
+                               xclk,
+                               16,
+                               &pi->pbsp,
+                               &pi->pbsu);
+
+        pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
+        pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
+
+        WREG32(CG_BSP, pi->dsp);
+
+}
+
+void rv770_program_git(struct radeon_device *rdev)
+{
+        WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK);
+}
+
+void rv770_program_tp(struct radeon_device *rdev)
+{
+        int i;
+        enum r600_td td = R600_TD_DFLT;
+
+        for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
+                WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i])));
+
+        if (td == R600_TD_AUTO)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
+        else
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
+        if (td == R600_TD_UP)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
+        if (td == R600_TD_DOWN)
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
+}
+
+void rv770_program_tpp(struct radeon_device *rdev)
+{
+        WREG32(CG_TPC, R600_TPC_DFLT);
+}
+
+void rv770_program_sstp(struct radeon_device *rdev)
+{
+        WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
+}
+
+void rv770_program_engine_speed_parameters(struct radeon_device *rdev)
+{
+        WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
+}
+
+static void rv770_enable_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
+
+        tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
+        tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
+                DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+}
+
+void rv770_program_vc(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        WREG32(CG_FTV, pi->vrc);
+}
+
+void rv770_clear_vc(struct radeon_device *rdev)
+{
+        WREG32(CG_FTV, 0);
+}
+
+int rv770_upload_firmware(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int ret;
+
+        rv770_reset_smc(rdev);
+        rv770_stop_smc_clock(rdev);
+
+        ret = rv770_load_smc_ucode(rdev, pi->sram_end);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+static int rv770_populate_smc_acpi_state(struct radeon_device *rdev,
+                                         RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        u32 mpll_ad_func_cntl =
+                pi->clk_regs.rv770.mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl =
+                pi->clk_regs.rv770.mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2 =
+                pi->clk_regs.rv770.mpll_dq_func_cntl_2;
+        u32 spll_func_cntl =
+                pi->clk_regs.rv770.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 =
+                pi->clk_regs.rv770.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 =
+                pi->clk_regs.rv770.cg_spll_func_cntl_3;
+        u32 mclk_pwrmgt_cntl;
+        u32 dll_cntl;
+
+        table->ACPIState = table->initialState;
+
+        table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+        if (pi->acpi_vddc) {
+                rv770_populate_vddc_value(rdev, pi->acpi_vddc,
+                                          &table->ACPIState.levels[0].vddc);
+                if (pi->pcie_gen2) {
+                        if (pi->acpi_pcie_gen2)
+                                table->ACPIState.levels[0].gen2PCIE = 1;
+                        else
+                                table->ACPIState.levels[0].gen2PCIE = 0;
+                } else
+                        table->ACPIState.levels[0].gen2PCIE = 0;
+                if (pi->acpi_pcie_gen2)
+                        table->ACPIState.levels[0].gen2XSP = 1;
+                else
+                        table->ACPIState.levels[0].gen2XSP = 0;
+        } else {
+                rv770_populate_vddc_value(rdev, pi->min_vddc_in_table,
+                                          &table->ACPIState.levels[0].vddc);
+                table->ACPIState.levels[0].gen2PCIE = 0;
+        }
+
+
+        mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
+
+        mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
+
+        mclk_pwrmgt_cntl = (MRDCKA0_RESET |
+                            MRDCKA1_RESET |
+                            MRDCKB0_RESET |
+                            MRDCKB1_RESET |
+                            MRDCKC0_RESET |
+                            MRDCKC1_RESET |
+                            MRDCKD0_RESET |
+                            MRDCKD1_RESET);
+
+        dll_cntl = 0xff000000;
+
+        spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN;
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
+
+        table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
+
+        table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0;
+
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
+
+        table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+        rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
+
+        table->ACPIState.levels[1] = table->ACPIState.levels[0];
+        table->ACPIState.levels[2] = table->ACPIState.levels[0];
+
+        return 0;
+}
+
+int rv770_populate_initial_mvdd_value(struct radeon_device *rdev,
+                                      RV770_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if ((pi->s0_vid_lower_smio_cntl & pi->mvdd_mask_low) ==
+             (pi->mvdd_low_smio[MVDD_LOW_INDEX] & pi->mvdd_mask_low) ) {
+                voltage->index = MVDD_LOW_INDEX;
+                voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
+        } else {
+                voltage->index = MVDD_HIGH_INDEX;
+                voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
+        }
+
+        return 0;
+}
+
+static int rv770_populate_smc_initial_state(struct radeon_device *rdev,
+                                            struct radeon_ps *radeon_state,
+                                            RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 a_t;
+
+        table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2);
+        table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl);
+        table->initialState.levels[0].mclk.mclk770.vDLL_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.dll_cntl);
+
+        table->initialState.levels[0].mclk.mclk770.vMPLL_SS =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ss1);
+        table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 =
+                cpu_to_be32(pi->clk_regs.rv770.mpll_ss2);
+
+        table->initialState.levels[0].mclk.mclk770.mclk_value =
+                cpu_to_be32(initial_state->low.mclk);
+
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
+                cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2);
+
+        table->initialState.levels[0].sclk.sclk_value =
+                cpu_to_be32(initial_state->low.sclk);
+
+        table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
+
+        table->initialState.levels[0].seqValue =
+                rv770_get_seq_value(rdev, &initial_state->low);
+
+        rv770_populate_vddc_value(rdev,
+                                  initial_state->low.vddc,
+                                  &table->initialState.levels[0].vddc);
+        rv770_populate_initial_mvdd_value(rdev,
+                                          &table->initialState.levels[0].mvdd);
+
+        a_t = CG_R(0xffff) | CG_L(0);
+        table->initialState.levels[0].aT = cpu_to_be32(a_t);
+
+        table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
+
+        if (pi->boot_in_gen2)
+                table->initialState.levels[0].gen2PCIE = 1;
+        else
+                table->initialState.levels[0].gen2PCIE = 0;
+        if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
+                table->initialState.levels[0].gen2XSP = 1;
+        else
+                table->initialState.levels[0].gen2XSP = 0;
+
+        if (rdev->family == CHIP_RV740) {
+                if (pi->mem_gddr5) {
+                        if (initial_state->low.mclk <= pi->mclk_strobe_mode_threshold)
+                                table->initialState.levels[0].strobeMode =
+                                        rv740_get_mclk_frequency_ratio(initial_state->low.mclk) | 0x10;
+                        else
+                                table->initialState.levels[0].strobeMode = 0;
+
+                        if (initial_state->low.mclk >= pi->mclk_edc_enable_threshold)
+                                table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
+                        else
+                                table->initialState.levels[0].mcFlags =  0;
+                }
+        }
+
+        table->initialState.levels[1] = table->initialState.levels[0];
+        table->initialState.levels[2] = table->initialState.levels[0];
+
+        table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        return 0;
+}
+
+static int rv770_populate_smc_vddc_table(struct radeon_device *rdev,
+                                         RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+
+        for (i = 0; i < pi->valid_vddc_entries; i++) {
+                table->highSMIO[pi->vddc_table[i].vddc_index] =
+                        pi->vddc_table[i].high_smio;
+                table->lowSMIO[pi->vddc_table[i].vddc_index] =
+                        cpu_to_be32(pi->vddc_table[i].low_smio);
+        }
+
+        table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0;
+        table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] =
+                cpu_to_be32(pi->vddc_mask_low);
+
+        for (i = 0;
+             ((i < pi->valid_vddc_entries) &&
+              (pi->max_vddc_in_table >
+               pi->vddc_table[i].vddc));
+             i++);
+
+        table->maxVDDCIndexInPPTable =
+                pi->vddc_table[i].vddc_index;
+
+        return 0;
+}
+
+static int rv770_populate_smc_mvdd_table(struct radeon_device *rdev,
+                                         RV770_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (pi->mvdd_control) {
+                table->lowSMIO[MVDD_HIGH_INDEX] |=
+                        cpu_to_be32(pi->mvdd_low_smio[MVDD_HIGH_INDEX]);
+                table->lowSMIO[MVDD_LOW_INDEX] |=
+                        cpu_to_be32(pi->mvdd_low_smio[MVDD_LOW_INDEX]);
+
+                table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_MVDD] = 0;
+                table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_MVDD] =
+                        cpu_to_be32(pi->mvdd_mask_low);
+        }
+
+        return 0;
+}
+
+static int rv770_init_smc_table(struct radeon_device *rdev,
+                                struct radeon_ps *radeon_boot_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
+        RV770_SMC_STATETABLE *table = &pi->smc_statetable;
+        int ret;
+
+        memset(table, 0, sizeof(RV770_SMC_STATETABLE));
+
+        pi->boot_sclk = boot_state->low.sclk;
+
+        rv770_populate_smc_vddc_table(rdev, table);
+        rv770_populate_smc_mvdd_table(rdev, table);
+
+        switch (rdev->pm.int_thermal_type) {
+        case THERMAL_TYPE_RV770:
+        case THERMAL_TYPE_ADT7473_WITH_INTERNAL:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
+                break;
+        case THERMAL_TYPE_NONE:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
+                break;
+        case THERMAL_TYPE_EXTERNAL_GPIO:
+        default:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
+                break;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) {
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+                if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT)
+                        table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK;
+
+                if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT)
+                        table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+        if (pi->mem_gddr5)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                ret = rv730_populate_smc_initial_state(rdev, radeon_boot_state, table);
+        else
+                ret = rv770_populate_smc_initial_state(rdev, radeon_boot_state, table);
+        if (ret)
+                return ret;
+
+        if (rdev->family == CHIP_RV740)
+                ret = rv740_populate_smc_acpi_state(rdev, table);
+        else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                ret = rv730_populate_smc_acpi_state(rdev, table);
+        else
+                ret = rv770_populate_smc_acpi_state(rdev, table);
+        if (ret)
+                return ret;
+
+        table->driverState = table->initialState;
+
+        return rv770_copy_bytes_to_smc(rdev,
+                                       pi->state_table_start,
+                                       (const u8 *)table,
+                                       sizeof(RV770_SMC_STATETABLE),
+                                       pi->sram_end);
+}
+
+static int rv770_construct_vddc_table(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u16 min, max, step;
+        u32 steps = 0;
+        u8 vddc_index = 0;
+        u32 i;
+
+        radeon_atom_get_min_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &min);
+        radeon_atom_get_max_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &max);
+        radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &step);
+
+        steps = (max - min) / step + 1;
+
+        if (steps > MAX_NO_VREG_STEPS)
+                return -EINVAL;
+
+        for (i = 0; i < steps; i++) {
+                u32 gpio_pins, gpio_mask;
+
+                pi->vddc_table[i].vddc = (u16)(min + i * step);
+                radeon_atom_get_voltage_gpio_settings(rdev,
+                                                      pi->vddc_table[i].vddc,
+                                                      SET_VOLTAGE_TYPE_ASIC_VDDC,
+                                                      &gpio_pins, &gpio_mask);
+                pi->vddc_table[i].low_smio = gpio_pins & gpio_mask;
+                pi->vddc_table[i].high_smio = 0;
+                pi->vddc_mask_low = gpio_mask;
+                if (i > 0) {
+                        if ((pi->vddc_table[i].low_smio !=
+                             pi->vddc_table[i - 1].low_smio ) ||
+                             (pi->vddc_table[i].high_smio !=
+                              pi->vddc_table[i - 1].high_smio))
+                                vddc_index++;
+                }
+                pi->vddc_table[i].vddc_index = vddc_index;
+        }
+
+        pi->valid_vddc_entries = (u8)steps;
+
+        return 0;
+}
+
+static u32 rv770_get_mclk_split_point(struct atom_memory_info *memory_info)
+{
+        if (memory_info->mem_type == MEM_TYPE_GDDR3)
+                return 30000;
+
+        return 0;
+}
+
+static int rv770_get_mvdd_pin_configuration(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 gpio_pins, gpio_mask;
+
+        radeon_atom_get_voltage_gpio_settings(rdev,
+                                              MVDD_HIGH_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
+                                              &gpio_pins, &gpio_mask);
+        pi->mvdd_mask_low = gpio_mask;
+        pi->mvdd_low_smio[MVDD_HIGH_INDEX] =
+                gpio_pins & gpio_mask;
+
+        radeon_atom_get_voltage_gpio_settings(rdev,
+                                              MVDD_LOW_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
+                                              &gpio_pins, &gpio_mask);
+        pi->mvdd_low_smio[MVDD_LOW_INDEX] =
+                gpio_pins & gpio_mask;
+
+        return 0;
+}
+
+u8 rv770_get_memory_module_index(struct radeon_device *rdev)
+{
+        return (u8) ((RREG32(BIOS_SCRATCH_4) >> 16) & 0xff);
+}
+
+static int rv770_get_mvdd_configuration(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 memory_module_index;
+        struct atom_memory_info memory_info;
+
+        memory_module_index = rv770_get_memory_module_index(rdev);
+
+        if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info)) {
+                pi->mvdd_control = false;
+                return 0;
+        }
+
+        pi->mvdd_split_frequency =
+                rv770_get_mclk_split_point(&memory_info);
+
+        if (pi->mvdd_split_frequency == 0) {
+                pi->mvdd_control = false;
+                return 0;
+        }
+
+        return rv770_get_mvdd_pin_configuration(rdev);
+}
+
+void rv770_enable_voltage_control(struct radeon_device *rdev,
+                                  bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
+}
+
+static void rv770_program_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
+
+        tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
+        if (RREG32(AVIVO_D1CRTC_CONTROL) & AVIVO_CRTC_EN) {
+                tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
+                tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+        } else if (RREG32(AVIVO_D2CRTC_CONTROL) & AVIVO_CRTC_EN) {
+                tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+                tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
+        } else {
+                tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+                tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
+        }
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+}
+
+static void rv770_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
+                                           bool enable)
+{
+        rv770_enable_bif_dynamic_pcie_gen2(rdev, enable);
+
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
+}
+
+static void r7xx_program_memory_timing_parameters(struct radeon_device *rdev,
+                                                  struct radeon_ps *radeon_new_state)
+{
+        if ((rdev->family == CHIP_RV730) ||
+            (rdev->family == CHIP_RV710) ||
+            (rdev->family == CHIP_RV740))
+                rv730_program_memory_timing_parameters(rdev, radeon_new_state);
+        else
+                rv770_program_memory_timing_parameters(rdev, radeon_new_state);
+}
+
+static int rv770_upload_sw_state(struct radeon_device *rdev,
+                                 struct radeon_ps *radeon_new_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u16 address = pi->state_table_start +
+                offsetof(RV770_SMC_STATETABLE, driverState);
+        RV770_SMC_SWSTATE state = { 0 };
+        int ret;
+
+        ret = rv770_convert_power_state_to_smc(rdev, radeon_new_state, &state);
+        if (ret)
+                return ret;
+
+        return rv770_copy_bytes_to_smc(rdev, address, (const u8 *)&state,
+                                       sizeof(RV770_SMC_SWSTATE),
+                                       pi->sram_end);
+}
+
+int rv770_halt_smc(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        if (rv770_wait_for_smc_inactive(rdev) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return 0;
+}
+
+int rv770_resume_smc(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Resume) != PPSMC_Result_OK)
+                return -EINVAL;
+        return 0;
+}
+
+int rv770_set_sw_state(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) != PPSMC_Result_OK)
+                return -EINVAL;
+        return 0;
+}
+
+int rv770_set_boot_state(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) != PPSMC_Result_OK)
+                return -EINVAL;
+        return 0;
+}
+
+void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                              struct radeon_ps *new_ps,
+                                              struct radeon_ps *old_ps)
+{
+        struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
+        struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->high.sclk >= current_state->high.sclk)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                             struct radeon_ps *new_ps,
+                                             struct radeon_ps *old_ps)
+{
+        struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
+        struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
+
+        if ((new_ps->vclk == old_ps->vclk) &&
+            (new_ps->dclk == old_ps->dclk))
+                return;
+
+        if (new_state->high.sclk < current_state->high.sclk)
+                return;
+
+        radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
+}
+
+int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_NoForcedLevel)) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled)) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return 0;
+}
+
+int rv770_unrestrict_performance_levels_after_switch(struct radeon_device *rdev)
+{
+        if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_NoForcedLevel)) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_ZeroLevelsDisabled)) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return 0;
+}
+
+void r7xx_start_smc(struct radeon_device *rdev)
+{
+        rv770_start_smc(rdev);
+        rv770_start_smc_clock(rdev);
+}
+
+
+void r7xx_stop_smc(struct radeon_device *rdev)
+{
+        rv770_reset_smc(rdev);
+        rv770_stop_smc_clock(rdev);
+}
+
+static void rv770_read_clock_registers(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        pi->clk_regs.rv770.cg_spll_func_cntl =
+                RREG32(CG_SPLL_FUNC_CNTL);
+        pi->clk_regs.rv770.cg_spll_func_cntl_2 =
+                RREG32(CG_SPLL_FUNC_CNTL_2);
+        pi->clk_regs.rv770.cg_spll_func_cntl_3 =
+                RREG32(CG_SPLL_FUNC_CNTL_3);
+        pi->clk_regs.rv770.cg_spll_spread_spectrum =
+                RREG32(CG_SPLL_SPREAD_SPECTRUM);
+        pi->clk_regs.rv770.cg_spll_spread_spectrum_2 =
+                RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
+        pi->clk_regs.rv770.mpll_ad_func_cntl =
+                RREG32(MPLL_AD_FUNC_CNTL);
+        pi->clk_regs.rv770.mpll_ad_func_cntl_2 =
+                RREG32(MPLL_AD_FUNC_CNTL_2);
+        pi->clk_regs.rv770.mpll_dq_func_cntl =
+                RREG32(MPLL_DQ_FUNC_CNTL);
+        pi->clk_regs.rv770.mpll_dq_func_cntl_2 =
+                RREG32(MPLL_DQ_FUNC_CNTL_2);
+        pi->clk_regs.rv770.mclk_pwrmgt_cntl =
+                RREG32(MCLK_PWRMGT_CNTL);
+        pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL);
+}
+
+static void r7xx_read_clock_registers(struct radeon_device *rdev)
+{
+        if (rdev->family == CHIP_RV740)
+                rv740_read_clock_registers(rdev);
+        else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                rv730_read_clock_registers(rdev);
+        else
+                rv770_read_clock_registers(rdev);
+}
+
+void rv770_read_voltage_smio_registers(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        pi->s0_vid_lower_smio_cntl =
+                RREG32(S0_VID_LOWER_SMIO_CNTL);
+}
+
+void rv770_reset_smio_status(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 sw_smio_index, vid_smio_cntl;
+
+        sw_smio_index =
+                (RREG32(GENERAL_PWRMGT) & SW_SMIO_INDEX_MASK) >> SW_SMIO_INDEX_SHIFT;
+        switch (sw_smio_index) {
+        case 3:
+                vid_smio_cntl = RREG32(S3_VID_LOWER_SMIO_CNTL);
+                break;
+        case 2:
+                vid_smio_cntl = RREG32(S2_VID_LOWER_SMIO_CNTL);
+                break;
+        case 1:
+                vid_smio_cntl = RREG32(S1_VID_LOWER_SMIO_CNTL);
+                break;
+        case 0:
+                return;
+        default:
+                vid_smio_cntl = pi->s0_vid_lower_smio_cntl;
+                break;
+        }
+
+        WREG32(S0_VID_LOWER_SMIO_CNTL, vid_smio_cntl);
+        WREG32_P(GENERAL_PWRMGT, SW_SMIO_INDEX(0), ~SW_SMIO_INDEX_MASK);
+}
+
+void rv770_get_memory_type(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp;
+
+        tmp = RREG32(MC_SEQ_MISC0);
+
+        if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) ==
+            MC_SEQ_MISC0_GDDR5_VALUE)
+                pi->mem_gddr5 = true;
+        else
+                pi->mem_gddr5 = false;
+
+}
+
+void rv770_get_pcie_gen2_status(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 tmp;
+
+        tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+
+        if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
+            (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
+                pi->pcie_gen2 = true;
+        else
+                pi->pcie_gen2 = false;
+
+        if (pi->pcie_gen2) {
+                if (tmp & LC_CURRENT_DATA_RATE)
+                        pi->boot_in_gen2 = true;
+                else
+                        pi->boot_in_gen2 = false;
+        } else
+                pi->boot_in_gen2 = false;
+}
+
+#if 0
+static int rv770_enter_ulp_state(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (pi->gfx_clock_gating) {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+                RREG32(GB_TILING_CONFIG);
+        }
+
+        WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
+                 ~HOST_SMC_MSG_MASK);
+
+        udelay(7000);
+
+        return 0;
+}
+
+static int rv770_exit_ulp_state(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+
+        WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_ResumeFromMinimumPower),
+                 ~HOST_SMC_MSG_MASK);
+
+        udelay(7000);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (((RREG32(SMC_MSG) & HOST_SMC_RESP_MASK) >> HOST_SMC_RESP_SHIFT) == 1)
+                        break;
+                udelay(1000);
+        }
+
+        if (pi->gfx_clock_gating)
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+
+        return 0;
+}
+#endif
+
+static void rv770_get_mclk_odt_threshold(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 memory_module_index;
+        struct atom_memory_info memory_info;
+
+        pi->mclk_odt_threshold = 0;
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) {
+                memory_module_index = rv770_get_memory_module_index(rdev);
+
+                if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info))
+                        return;
+
+                if (memory_info.mem_type == MEM_TYPE_DDR2 ||
+                    memory_info.mem_type == MEM_TYPE_DDR3)
+                        pi->mclk_odt_threshold = 30000;
+        }
+}
+
+void rv770_get_max_vddc(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u16 vddc;
+
+        if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc))
+                pi->max_vddc = 0;
+        else
+                pi->max_vddc = vddc;
+}
+
+void rv770_program_response_times(struct radeon_device *rdev)
+{
+        u32 voltage_response_time, backbias_response_time;
+        u32 acpi_delay_time, vbi_time_out;
+        u32 vddc_dly, bb_dly, acpi_dly, vbi_dly;
+        u32 reference_clock;
+
+        voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
+        backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
+
+        if (voltage_response_time == 0)
+                voltage_response_time = 1000;
+
+        if (backbias_response_time == 0)
+                backbias_response_time = 1000;
+
+        acpi_delay_time = 15000;
+        vbi_time_out = 100000;
+
+        reference_clock = radeon_get_xclk(rdev);
+
+        vddc_dly = (voltage_response_time  * reference_clock) / 1600;
+        bb_dly = (backbias_response_time * reference_clock) / 1600;
+        acpi_dly = (acpi_delay_time * reference_clock) / 1600;
+        vbi_dly = (vbi_time_out * reference_clock) / 1600;
+
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_delay_vreg, vddc_dly);
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_delay_acpi, acpi_dly);
+        rv770_write_smc_soft_register(rdev,
+                                      RV770_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
+#if 0
+        /* XXX look up hw revision */
+        if (WEKIVA_A21)
+                rv770_write_smc_soft_register(rdev,
+                                              RV770_SMC_SOFT_REGISTER_baby_step_timer,
+                                              0x10);
+#endif
+}
+
+static void rv770_program_dcodt_before_state_switch(struct radeon_device *rdev,
+                                                    struct radeon_ps *radeon_new_state,
+                                                    struct radeon_ps *radeon_current_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
+        struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
+        bool current_use_dc = false;
+        bool new_use_dc = false;
+
+        if (pi->mclk_odt_threshold == 0)
+                return;
+
+        if (current_state->high.mclk <= pi->mclk_odt_threshold)
+                current_use_dc = true;
+
+        if (new_state->high.mclk <= pi->mclk_odt_threshold)
+                new_use_dc = true;
+
+        if (current_use_dc == new_use_dc)
+                return;
+
+        if (!current_use_dc && new_use_dc)
+                return;
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                rv730_program_dcodt(rdev, new_use_dc);
+}
+
+static void rv770_program_dcodt_after_state_switch(struct radeon_device *rdev,
+                                                   struct radeon_ps *radeon_new_state,
+                                                   struct radeon_ps *radeon_current_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
+        struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
+        bool current_use_dc = false;
+        bool new_use_dc = false;
+
+        if (pi->mclk_odt_threshold == 0)
+                return;
+
+        if (current_state->high.mclk <= pi->mclk_odt_threshold)
+                current_use_dc = true;
+
+        if (new_state->high.mclk <= pi->mclk_odt_threshold)
+                new_use_dc = true;
+
+        if (current_use_dc == new_use_dc)
+                return;
+
+        if (current_use_dc && !new_use_dc)
+                return;
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                rv730_program_dcodt(rdev, new_use_dc);
+}
+
+static void rv770_retrieve_odt_values(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (pi->mclk_odt_threshold == 0)
+                return;
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                rv730_get_odt_values(rdev);
+}
+
+static void rv770_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        bool want_thermal_protection;
+        enum radeon_dpm_event_src dpm_event_src;
+
+        switch (sources) {
+        case 0:
+        default:
+                want_thermal_protection = false;
+                break;
+        case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
+                break;
+
+        case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
+                break;
+
+        case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
+              (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
+                break;
+        }
+
+        if (want_thermal_protection) {
+                WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
+                if (pi->thermal_protection)
+                        WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+        } else {
+                WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+        }
+}
+
+void rv770_enable_auto_throttle_source(struct radeon_device *rdev,
+                                       enum radeon_dpm_auto_throttle_src source,
+                                       bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (enable) {
+                if (!(pi->active_auto_throttle_sources & (1 << source))) {
+                        pi->active_auto_throttle_sources |= 1 << source;
+                        rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
+                }
+        } else {
+                if (pi->active_auto_throttle_sources & (1 << source)) {
+                        pi->active_auto_throttle_sources &= ~(1 << source);
+                        rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
+                }
+        }
+}
+
+int rv770_set_thermal_temperature_range(struct radeon_device *rdev,
+                                        int min_temp, int max_temp)
+{
+        int low_temp = 0 * 1000;
+        int high_temp = 255 * 1000;
+
+        if (low_temp < min_temp)
+                low_temp = min_temp;
+        if (high_temp > max_temp)
+                high_temp = max_temp;
+        if (high_temp < low_temp) {
+                DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
+                return -EINVAL;
+        }
+
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
+        WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
+
+        rdev->pm.dpm.thermal.min_temp = low_temp;
+        rdev->pm.dpm.thermal.max_temp = high_temp;
+
+        return 0;
+}
+
+int rv770_dpm_enable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        if (pi->gfx_clock_gating)
+                rv770_restore_cgcg(rdev);
+
+        if (rv770_dpm_enabled(rdev))
+                return -EINVAL;
+
+        if (pi->voltage_control) {
+                rv770_enable_voltage_control(rdev, true);
+                ret = rv770_construct_vddc_table(rdev);
+                if (ret) {
+                        DRM_ERROR("rv770_construct_vddc_table failed\n");
+                        return ret;
+                }
+        }
+
+        if (pi->dcodt)
+                rv770_retrieve_odt_values(rdev);
+
+        if (pi->mvdd_control) {
+                ret = rv770_get_mvdd_configuration(rdev);
+                if (ret) {
+                        DRM_ERROR("rv770_get_mvdd_configuration failed\n");
+                        return ret;
+                }
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
+                rv770_enable_backbias(rdev, true);
+
+        rv770_enable_spread_spectrum(rdev, true);
+
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, true);
+
+        rv770_program_mpll_timing_parameters(rdev);
+        rv770_setup_bsp(rdev);
+        rv770_program_git(rdev);
+        rv770_program_tp(rdev);
+        rv770_program_tpp(rdev);
+        rv770_program_sstp(rdev);
+        rv770_program_engine_speed_parameters(rdev);
+        rv770_enable_display_gap(rdev);
+        rv770_program_vc(rdev);
+
+        if (pi->dynamic_pcie_gen2)
+                rv770_enable_dynamic_pcie_gen2(rdev, true);
+
+        ret = rv770_upload_firmware(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_upload_firmware failed\n");
+                return ret;
+        }
+        ret = rv770_init_smc_table(rdev, boot_ps);
+        if (ret) {
+                DRM_ERROR("rv770_init_smc_table failed\n");
+                return ret;
+        }
+
+        rv770_program_response_times(rdev);
+        r7xx_start_smc(rdev);
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                rv730_start_dpm(rdev);
+        else
+                rv770_start_dpm(rdev);
+
+        if (pi->gfx_clock_gating)
+                rv770_gfx_clock_gating_enable(rdev, true);
+
+        if (pi->mg_clock_gating)
+                rv770_mg_clock_gating_enable(rdev, true);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                PPSMC_Result result;
+
+                ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+                result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
+
+                if (result != PPSMC_Result_OK)
+                        DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
+        }
+
+        rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+
+        return 0;
+}
+
+void rv770_dpm_disable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (!rv770_dpm_enabled(rdev))
+                return;
+
+        rv770_clear_vc(rdev);
+
+        if (pi->thermal_protection)
+                rv770_enable_thermal_protection(rdev, false);
+
+        rv770_enable_spread_spectrum(rdev, false);
+
+        if (pi->dynamic_pcie_gen2)
+                rv770_enable_dynamic_pcie_gen2(rdev, false);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        if (pi->gfx_clock_gating)
+                rv770_gfx_clock_gating_enable(rdev, false);
+
+        if (pi->mg_clock_gating)
+                rv770_mg_clock_gating_enable(rdev, false);
+
+        if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
+                rv730_stop_dpm(rdev);
+        else
+                rv770_stop_dpm(rdev);
+
+        r7xx_stop_smc(rdev);
+        rv770_reset_smio_status(rdev);
+}
+
+int rv770_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
+        struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
+        int ret;
+
+        ret = rv770_restrict_performance_levels_before_switch(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n");
+                return ret;
+        }
+        rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+        ret = rv770_halt_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_halt_smc failed\n");
+                return ret;
+        }
+        ret = rv770_upload_sw_state(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("rv770_upload_sw_state failed\n");
+                return ret;
+        }
+        r7xx_program_memory_timing_parameters(rdev, new_ps);
+        if (pi->dcodt)
+                rv770_program_dcodt_before_state_switch(rdev, new_ps, old_ps);
+        ret = rv770_resume_smc(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_resume_smc failed\n");
+                return ret;
+        }
+        ret = rv770_set_sw_state(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_set_sw_state failed\n");
+                return ret;
+        }
+        if (pi->dcodt)
+                rv770_program_dcodt_after_state_switch(rdev, new_ps, old_ps);
+        rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+        ret = rv770_unrestrict_performance_levels_after_switch(rdev);
+        if (ret) {
+                DRM_ERROR("rv770_unrestrict_performance_levels_after_switch failed\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+void rv770_dpm_reset_asic(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+
+        rv770_restrict_performance_levels_before_switch(rdev);
+        if (pi->dcodt)
+                rv770_program_dcodt_before_state_switch(rdev, boot_ps, boot_ps);
+        rv770_set_boot_state(rdev);
+        if (pi->dcodt)
+                rv770_program_dcodt_after_state_switch(rdev, boot_ps, boot_ps);
+}
+
+void rv770_dpm_setup_asic(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        r7xx_read_clock_registers(rdev);
+        rv770_read_voltage_smio_registers(rdev);
+        rv770_get_memory_type(rdev);
+        if (pi->dcodt)
+                rv770_get_mclk_odt_threshold(rdev);
+        rv770_get_pcie_gen2_status(rdev);
+
+        rv770_enable_acpi_pm(rdev);
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
+                rv770_enable_l0s(rdev);
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
+                rv770_enable_l1(rdev);
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
+                rv770_enable_pll_sleep_in_l1(rdev);
+}
+
+void rv770_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+        rv770_program_display_gap(rdev);
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void rv7xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                             struct radeon_ps *rps,
+                                             struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+                                             u8 table_rev)
+{
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+        } else if (r600_is_uvd_state(rps->class, rps->class2)) {
+                rps->vclk = RV770_DEFAULT_VCLK_FREQ;
+                rps->dclk = RV770_DEFAULT_DCLK_FREQ;
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+                rdev->pm.dpm.boot_ps = rps;
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void rv7xx_parse_pplib_clock_info(struct radeon_device *rdev,
+                                         struct radeon_ps *rps, int index,
+                                         union pplib_clock_info *clock_info)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct rv7xx_ps *ps = rv770_get_ps(rps);
+        u32 sclk, mclk;
+        u16 vddc;
+        struct rv7xx_pl *pl;
+
+        switch (index) {
+        case 0:
+                pl = &ps->low;
+                break;
+        case 1:
+                pl = &ps->medium;
+                break;
+        case 2:
+        default:
+                pl = &ps->high;
+                break;
+        }
+
+        if (rdev->family >= CHIP_CEDAR) {
+                sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
+                sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
+                mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
+                mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
+
+                pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
+                pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
+                pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
+        } else {
+                sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
+                sclk |= clock_info->r600.ucEngineClockHigh << 16;
+                mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
+                mclk |= clock_info->r600.ucMemoryClockHigh << 16;
+
+                pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
+                pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
+        }
+
+        pl->mclk = mclk;
+        pl->sclk = sclk;
+
+        /* patch up vddc if necessary */
+        if (pl->vddc == 0xff01) {
+                if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
+                        pl->vddc = vddc;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
+                pi->acpi_vddc = pl->vddc;
+                if (rdev->family >= CHIP_CEDAR)
+                        eg_pi->acpi_vddci = pl->vddci;
+                if (ps->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
+                        pi->acpi_pcie_gen2 = true;
+                else
+                        pi->acpi_pcie_gen2 = false;
+        }
+
+        if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
+                if (rdev->family >= CHIP_BARTS) {
+                        eg_pi->ulv.supported = true;
+                        eg_pi->ulv.pl = pl;
+                }
+        }
+
+        if (pi->min_vddc_in_table > pl->vddc)
+                pi->min_vddc_in_table = pl->vddc;
+
+        if (pi->max_vddc_in_table < pl->vddc)
+                pi->max_vddc_in_table = pl->vddc;
+
+        /* patch up boot state */
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                u16 vddc, vddci, mvdd;
+                radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
+                pl->mclk = rdev->clock.default_mclk;
+                pl->sclk = rdev->clock.default_sclk;
+                pl->vddc = vddc;
+                pl->vddci = vddci;
+        }
+
+        if (rdev->family >= CHIP_BARTS) {
+                if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
+                    ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
+                        rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
+                        rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
+                        rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
+                        rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
+                }
+        }
+}
+
+int rv7xx_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j;
+        union pplib_clock_info *clock_info;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        struct rv7xx_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  power_info->pplib.ucNumStates, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+
+        for (i = 0; i < power_info->pplib.ucNumStates; i++) {
+                power_state = (union pplib_power_state *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usStateArrayOffset) +
+                         i * power_info->pplib.ucStateEntrySize);
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        (mode_info->atom_context->bios + data_offset +
+                         le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
+                         (power_state->v1.ucNonClockStateIndex *
+                          power_info->pplib.ucNonClockSize));
+                if (power_info->pplib.ucStateEntrySize - 1) {
+                        ps = kzalloc(sizeof(struct rv7xx_ps), GFP_KERNEL);
+                        if (ps == NULL) {
+                                kfree(rdev->pm.dpm.ps);
+                                return -ENOMEM;
+                        }
+                        rdev->pm.dpm.ps[i].ps_priv = ps;
+                        rv7xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                                         non_clock_info,
+                                                         power_info->pplib.ucNonClockSize);
+                        for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
+                                clock_info = (union pplib_clock_info *)
+                                        (mode_info->atom_context->bios + data_offset +
+                                         le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
+                                         (power_state->v1.ucClockStateIndices[j] *
+                                          power_info->pplib.ucClockInfoSize));
+                                rv7xx_parse_pplib_clock_info(rdev,
+                                                             &rdev->pm.dpm.ps[i], j,
+                                                             clock_info);
+                        }
+                }
+        }
+        rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
+        return 0;
+}
+
+int rv770_dpm_init(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi;
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        uint16_t data_offset, size;
+        uint8_t frev, crev;
+        struct atom_clock_dividers dividers;
+        int ret;
+
+        pi = kzalloc(sizeof(struct rv7xx_power_info), GFP_KERNEL);
+        if (pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = pi;
+
+        rv770_get_max_vddc(rdev);
+
+        pi->acpi_vddc = 0;
+        pi->min_vddc_in_table = 0;
+        pi->max_vddc_in_table = 0;
+
+        ret = rv7xx_parse_power_table(rdev);
+        if (ret)
+                return ret;
+
+        if (rdev->pm.dpm.voltage_response_time == 0)
+                rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+        if (rdev->pm.dpm.backbias_response_time == 0)
+                rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->ref_div = dividers.ref_div + 1;
+        else
+                pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        pi->mclk_strobe_mode_threshold = 30000;
+        pi->mclk_edc_enable_threshold = 30000;
+
+        pi->rlp = RV770_RLP_DFLT;
+        pi->rmp = RV770_RMP_DFLT;
+        pi->lhp = RV770_LHP_DFLT;
+        pi->lmp = RV770_LMP_DFLT;
+
+        pi->voltage_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
+
+        pi->mvdd_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                pi->sclk_ss = true;
+                pi->mclk_ss = true;
+                pi->dynamic_ss = true;
+        } else {
+                pi->sclk_ss = false;
+                pi->mclk_ss = false;
+                pi->dynamic_ss = false;
+        }
+
+        pi->asi = RV770_ASI_DFLT;
+        pi->pasi = RV770_HASI_DFLT;
+        pi->vrc = RV770_VRC_DFLT;
+
+        pi->power_gating = false;
+
+        pi->gfx_clock_gating = true;
+
+        pi->mg_clock_gating = true;
+        pi->mgcgtssm = true;
+
+        pi->dynamic_pcie_gen2 = true;
+
+        if (pi->gfx_clock_gating &&
+            (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
+                pi->thermal_protection = true;
+        else
+                pi->thermal_protection = false;
+
+        pi->display_gap = true;
+
+        if (rdev->flags & RADEON_IS_MOBILITY)
+                pi->dcodt = true;
+        else
+                pi->dcodt = false;
+
+        pi->ulps = true;
+
+        pi->mclk_stutter_mode_threshold = 0;
+
+        pi->sram_end = SMC_RAM_END;
+        pi->state_table_start = RV770_SMC_TABLE_ADDRESS;
+        pi->soft_regs_start = RV770_SMC_SOFT_REGISTERS_START;
+
+        return 0;
+}
+
+void rv770_dpm_print_power_state(struct radeon_device *rdev,
+                                 struct radeon_ps *rps)
+{
+        struct rv7xx_ps *ps = rv770_get_ps(rps);
+        struct rv7xx_pl *pl;
+
+        r600_dpm_print_class_info(rps->class, rps->class2);
+        r600_dpm_print_cap_info(rps->caps);
+        printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+        if (rdev->family >= CHIP_CEDAR) {
+                pl = &ps->low;
+                printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+                       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+                pl = &ps->medium;
+                printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+                       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+                pl = &ps->high;
+                printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+                       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+        } else {
+                pl = &ps->low;
+                printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
+                       pl->sclk, pl->mclk, pl->vddc);
+                pl = &ps->medium;
+                printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
+                       pl->sclk, pl->mclk, pl->vddc);
+                pl = &ps->high;
+                printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
+                       pl->sclk, pl->mclk, pl->vddc);
+        }
+        r600_dpm_print_ps_status(rdev, rps);
+}
+
+void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                       struct seq_file *m)
+{
+        struct radeon_ps *rps = rdev->pm.dpm.current_ps;
+        struct rv7xx_ps *ps = rv770_get_ps(rps);
+        struct rv7xx_pl *pl;
+        u32 current_index =
+                (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
+                CURRENT_PROFILE_INDEX_SHIFT;
+
+        if (current_index > 2) {
+                seq_printf(m, "invalid dpm profile %d\n", current_index);
+        } else {
+                if (current_index == 0)
+                        pl = &ps->low;
+                else if (current_index == 1)
+                        pl = &ps->medium;
+                else /* current_index == 2 */
+                        pl = &ps->high;
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                if (rdev->family >= CHIP_CEDAR) {
+                        seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+                                   current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+                } else {
+                        seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
+                                   current_index, pl->sclk, pl->mclk, pl->vddc);
+                }
+        }
+}
+
+void rv770_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+}
+
+u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
+
+        if (low)
+                return requested_state->low.sclk;
+        else
+                return requested_state->high.sclk;
+}
+
+u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
+
+        if (low)
+                return requested_state->low.mclk;
+        else
+                return requested_state->high.mclk;
+}
diff --git a/drivers/gpu/drm/radeon/rv770_dpm.h b/drivers/gpu/drm/radeon/rv770_dpm.h
new file mode 100644
index 000000000000..f1e1fcf7f622
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv770_dpm.h
@@ -0,0 +1,288 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RV770_DPM_H__
+#define __RV770_DPM_H__
+
+#include "rv770_smc.h"
+
+struct rv770_clock_registers {
+        u32 cg_spll_func_cntl;
+        u32 cg_spll_func_cntl_2;
+        u32 cg_spll_func_cntl_3;
+        u32 cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2;
+        u32 mpll_ad_func_cntl;
+        u32 mpll_ad_func_cntl_2;
+        u32 mpll_dq_func_cntl;
+        u32 mpll_dq_func_cntl_2;
+        u32 mclk_pwrmgt_cntl;
+        u32 dll_cntl;
+        u32 mpll_ss1;
+        u32 mpll_ss2;
+};
+
+struct rv730_clock_registers {
+        u32 cg_spll_func_cntl;
+        u32 cg_spll_func_cntl_2;
+        u32 cg_spll_func_cntl_3;
+        u32 cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2;
+        u32 mclk_pwrmgt_cntl;
+        u32 dll_cntl;
+        u32 mpll_func_cntl;
+        u32 mpll_func_cntl2;
+        u32 mpll_func_cntl3;
+        u32 mpll_ss;
+        u32 mpll_ss2;
+};
+
+union r7xx_clock_registers {
+        struct rv770_clock_registers rv770;
+        struct rv730_clock_registers rv730;
+};
+
+struct vddc_table_entry {
+        u16 vddc;
+        u8 vddc_index;
+        u8 high_smio;
+        u32 low_smio;
+};
+
+#define MAX_NO_OF_MVDD_VALUES 2
+#define MAX_NO_VREG_STEPS 32
+
+struct rv7xx_power_info {
+        /* flags */
+        bool mem_gddr5;
+        bool pcie_gen2;
+        bool dynamic_pcie_gen2;
+        bool acpi_pcie_gen2;
+        bool boot_in_gen2;
+        bool voltage_control; /* vddc */
+        bool mvdd_control;
+        bool sclk_ss;
+        bool mclk_ss;
+        bool dynamic_ss;
+        bool gfx_clock_gating;
+        bool mg_clock_gating;
+        bool mgcgtssm;
+        bool power_gating;
+        bool thermal_protection;
+        bool display_gap;
+        bool dcodt;
+        bool ulps;
+        /* registers */
+        union r7xx_clock_registers clk_regs;
+        u32 s0_vid_lower_smio_cntl;
+        /* voltage */
+        u32 vddc_mask_low;
+        u32 mvdd_mask_low;
+        u32 mvdd_split_frequency;
+        u32 mvdd_low_smio[MAX_NO_OF_MVDD_VALUES];
+        u16 max_vddc;
+        u16 max_vddc_in_table;
+        u16 min_vddc_in_table;
+        struct vddc_table_entry vddc_table[MAX_NO_VREG_STEPS];
+        u8 valid_vddc_entries;
+        /* dc odt */
+        u32 mclk_odt_threshold;
+        u8 odt_value_0[2];
+        u8 odt_value_1[2];
+        /* stored values */
+        u32 boot_sclk;
+        u16 acpi_vddc;
+        u32 ref_div;
+        u32 active_auto_throttle_sources;
+        u32 mclk_stutter_mode_threshold;
+        u32 mclk_strobe_mode_threshold;
+        u32 mclk_edc_enable_threshold;
+        u32 bsp;
+        u32 bsu;
+        u32 pbsp;
+        u32 pbsu;
+        u32 dsp;
+        u32 psp;
+        u32 asi;
+        u32 pasi;
+        u32 vrc;
+        u32 restricted_levels;
+        u32 rlp;
+        u32 rmp;
+        u32 lhp;
+        u32 lmp;
+        /* smc offsets */
+        u16 state_table_start;
+        u16 soft_regs_start;
+        u16 sram_end;
+        /* scratch structs */
+        RV770_SMC_STATETABLE smc_statetable;
+};
+
+struct rv7xx_pl {
+        u32 sclk;
+        u32 mclk;
+        u16 vddc;
+        u16 vddci; /* eg+ only */
+        u32 flags;
+        enum radeon_pcie_gen pcie_gen; /* si+ only */
+};
+
+struct rv7xx_ps {
+        struct rv7xx_pl high;
+        struct rv7xx_pl medium;
+        struct rv7xx_pl low;
+        bool dc_compatible;
+};
+
+#define RV770_RLP_DFLT                                10
+#define RV770_RMP_DFLT                                25
+#define RV770_LHP_DFLT                                25
+#define RV770_LMP_DFLT                                10
+#define RV770_VRC_DFLT                                0x003f
+#define RV770_ASI_DFLT                                1000
+#define RV770_HASI_DFLT                               200000
+#define RV770_MGCGTTLOCAL0_DFLT                       0x00100000
+#define RV7XX_MGCGTTLOCAL0_DFLT                       0
+#define RV770_MGCGTTLOCAL1_DFLT                       0xFFFF0000
+#define RV770_MGCGCGTSSMCTRL_DFLT                     0x55940000
+
+#define MVDD_LOW_INDEX  0
+#define MVDD_HIGH_INDEX 1
+
+#define MVDD_LOW_VALUE  0
+#define MVDD_HIGH_VALUE 0xffff
+
+#define RV770_DEFAULT_VCLK_FREQ  53300 /* 10 khz */
+#define RV770_DEFAULT_DCLK_FREQ  40000 /* 10 khz */
+
+/* rv730/rv710 */
+int rv730_populate_sclk_value(struct radeon_device *rdev,
+                              u32 engine_clock,
+                              RV770_SMC_SCLK_VALUE *sclk);
+int rv730_populate_mclk_value(struct radeon_device *rdev,
+                              u32 engine_clock, u32 memory_clock,
+                              LPRV7XX_SMC_MCLK_VALUE mclk);
+void rv730_read_clock_registers(struct radeon_device *rdev);
+int rv730_populate_smc_acpi_state(struct radeon_device *rdev,
+                                  RV770_SMC_STATETABLE *table);
+int rv730_populate_smc_initial_state(struct radeon_device *rdev,
+                                     struct radeon_ps *radeon_initial_state,
+                                     RV770_SMC_STATETABLE *table);
+void rv730_program_memory_timing_parameters(struct radeon_device *rdev,
+                                            struct radeon_ps *radeon_state);
+void rv730_power_gating_enable(struct radeon_device *rdev,
+                               bool enable);
+void rv730_start_dpm(struct radeon_device *rdev);
+void rv730_stop_dpm(struct radeon_device *rdev);
+void rv730_program_dcodt(struct radeon_device *rdev, bool use_dcodt);
+void rv730_get_odt_values(struct radeon_device *rdev);
+
+/* rv740 */
+int rv740_populate_sclk_value(struct radeon_device *rdev, u32 engine_clock,
+                              RV770_SMC_SCLK_VALUE *sclk);
+int rv740_populate_mclk_value(struct radeon_device *rdev,
+                              u32 engine_clock, u32 memory_clock,
+                              RV7XX_SMC_MCLK_VALUE *mclk);
+void rv740_read_clock_registers(struct radeon_device *rdev);
+int rv740_populate_smc_acpi_state(struct radeon_device *rdev,
+                                  RV770_SMC_STATETABLE *table);
+void rv740_enable_mclk_spread_spectrum(struct radeon_device *rdev,
+                                       bool enable);
+u8 rv740_get_mclk_frequency_ratio(u32 memory_clock);
+u32 rv740_get_dll_speed(bool is_gddr5, u32 memory_clock);
+u32 rv740_get_decoded_reference_divider(u32 encoded_ref);
+
+/* rv770 */
+u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf);
+int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc,
+                              RV770_SMC_VOLTAGE_VALUE *voltage);
+int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
+                              RV770_SMC_VOLTAGE_VALUE *voltage);
+u8 rv770_get_seq_value(struct radeon_device *rdev,
+                       struct rv7xx_pl *pl);
+int rv770_populate_initial_mvdd_value(struct radeon_device *rdev,
+                                      RV770_SMC_VOLTAGE_VALUE *voltage);
+u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev,
+                                        u32 engine_clock);
+void rv770_program_response_times(struct radeon_device *rdev);
+int rv770_populate_smc_sp(struct radeon_device *rdev,
+                          struct radeon_ps *radeon_state,
+                          RV770_SMC_SWSTATE *smc_state);
+int rv770_populate_smc_t(struct radeon_device *rdev,
+                         struct radeon_ps *radeon_state,
+                         RV770_SMC_SWSTATE *smc_state);
+void rv770_read_voltage_smio_registers(struct radeon_device *rdev);
+void rv770_get_memory_type(struct radeon_device *rdev);
+void r7xx_start_smc(struct radeon_device *rdev);
+u8 rv770_get_memory_module_index(struct radeon_device *rdev);
+void rv770_get_max_vddc(struct radeon_device *rdev);
+void rv770_get_pcie_gen2_status(struct radeon_device *rdev);
+void rv770_enable_acpi_pm(struct radeon_device *rdev);
+void rv770_restore_cgcg(struct radeon_device *rdev);
+bool rv770_dpm_enabled(struct radeon_device *rdev);
+void rv770_enable_voltage_control(struct radeon_device *rdev,
+                                  bool enable);
+void rv770_enable_backbias(struct radeon_device *rdev,
+                           bool enable);
+void rv770_enable_thermal_protection(struct radeon_device *rdev,
+                                     bool enable);
+void rv770_enable_auto_throttle_source(struct radeon_device *rdev,
+                                       enum radeon_dpm_auto_throttle_src source,
+                                       bool enable);
+void rv770_setup_bsp(struct radeon_device *rdev);
+void rv770_program_git(struct radeon_device *rdev);
+void rv770_program_tp(struct radeon_device *rdev);
+void rv770_program_tpp(struct radeon_device *rdev);
+void rv770_program_sstp(struct radeon_device *rdev);
+void rv770_program_engine_speed_parameters(struct radeon_device *rdev);
+void rv770_program_vc(struct radeon_device *rdev);
+void rv770_clear_vc(struct radeon_device *rdev);
+int rv770_upload_firmware(struct radeon_device *rdev);
+void rv770_stop_dpm(struct radeon_device *rdev);
+void r7xx_stop_smc(struct radeon_device *rdev);
+void rv770_reset_smio_status(struct radeon_device *rdev);
+int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev);
+int rv770_unrestrict_performance_levels_after_switch(struct radeon_device *rdev);
+int rv770_halt_smc(struct radeon_device *rdev);
+int rv770_resume_smc(struct radeon_device *rdev);
+int rv770_set_sw_state(struct radeon_device *rdev);
+int rv770_set_boot_state(struct radeon_device *rdev);
+int rv7xx_parse_power_table(struct radeon_device *rdev);
+void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                              struct radeon_ps *new_ps,
+                                              struct radeon_ps *old_ps);
+void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                             struct radeon_ps *new_ps,
+                                             struct radeon_ps *old_ps);
+
+/* smc */
+int rv770_read_smc_soft_register(struct radeon_device *rdev,
+                                 u16 reg_offset, u32 *value);
+int rv770_write_smc_soft_register(struct radeon_device *rdev,
+                                  u16 reg_offset, u32 value);
+
+/* thermal */
+int rv770_set_thermal_temperature_range(struct radeon_device *rdev,
+                                        int min_temp, int max_temp);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv770_smc.c b/drivers/gpu/drm/radeon/rv770_smc.c
new file mode 100644
index 000000000000..ab95da570215
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv770_smc.c
@@ -0,0 +1,621 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "radeon.h"
+#include "rv770d.h"
+#include "rv770_dpm.h"
+#include "rv770_smc.h"
+#include "atom.h"
+#include "radeon_ucode.h"
+
+#define FIRST_SMC_INT_VECT_REG 0xFFD8
+#define FIRST_INT_VECT_S19     0xFFC0
+
+static const u8 rv770_smc_int_vectors[] =
+{
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x0C, 0xD7,
+        0x08, 0x2B, 0x08, 0x10,
+        0x03, 0x51, 0x03, 0x51,
+        0x03, 0x51, 0x03, 0x51
+};
+
+static const u8 rv730_smc_int_vectors[] =
+{
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x08, 0x15,
+        0x08, 0x15, 0x0C, 0xBB,
+        0x08, 0x30, 0x08, 0x15,
+        0x03, 0x56, 0x03, 0x56,
+        0x03, 0x56, 0x03, 0x56
+};
+
+static const u8 rv710_smc_int_vectors[] =
+{
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x08, 0x04,
+        0x08, 0x04, 0x0C, 0xCB,
+        0x08, 0x1F, 0x08, 0x04,
+        0x03, 0x51, 0x03, 0x51,
+        0x03, 0x51, 0x03, 0x51
+};
+
+static const u8 rv740_smc_int_vectors[] =
+{
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x08, 0x10,
+        0x08, 0x10, 0x0C, 0xD7,
+        0x08, 0x2B, 0x08, 0x10,
+        0x03, 0x51, 0x03, 0x51,
+        0x03, 0x51, 0x03, 0x51
+};
+
+static const u8 cedar_smc_int_vectors[] =
+{
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x11, 0x8B,
+        0x0B, 0x20, 0x0B, 0x05,
+        0x04, 0xF6, 0x04, 0xF6,
+        0x04, 0xF6, 0x04, 0xF6
+};
+
+static const u8 redwood_smc_int_vectors[] =
+{
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x11, 0x8B,
+        0x0B, 0x20, 0x0B, 0x05,
+        0x04, 0xF6, 0x04, 0xF6,
+        0x04, 0xF6, 0x04, 0xF6
+};
+
+static const u8 juniper_smc_int_vectors[] =
+{
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x11, 0x8B,
+        0x0B, 0x20, 0x0B, 0x05,
+        0x04, 0xF6, 0x04, 0xF6,
+        0x04, 0xF6, 0x04, 0xF6
+};
+
+static const u8 cypress_smc_int_vectors[] =
+{
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x0B, 0x05,
+        0x0B, 0x05, 0x11, 0x8B,
+        0x0B, 0x20, 0x0B, 0x05,
+        0x04, 0xF6, 0x04, 0xF6,
+        0x04, 0xF6, 0x04, 0xF6
+};
+
+static const u8 barts_smc_int_vectors[] =
+{
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x12, 0xAA,
+        0x0C, 0x2F, 0x15, 0xF6,
+        0x15, 0xF6, 0x05, 0x0A,
+        0x05, 0x0A, 0x05, 0x0A
+};
+
+static const u8 turks_smc_int_vectors[] =
+{
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x12, 0xAA,
+        0x0C, 0x2F, 0x15, 0xF6,
+        0x15, 0xF6, 0x05, 0x0A,
+        0x05, 0x0A, 0x05, 0x0A
+};
+
+static const u8 caicos_smc_int_vectors[] =
+{
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x0C, 0x14,
+        0x0C, 0x14, 0x12, 0xAA,
+        0x0C, 0x2F, 0x15, 0xF6,
+        0x15, 0xF6, 0x05, 0x0A,
+        0x05, 0x0A, 0x05, 0x0A
+};
+
+static const u8 cayman_smc_int_vectors[] =
+{
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x12, 0x05,
+        0x12, 0x05, 0x18, 0xEA,
+        0x12, 0x20, 0x1C, 0x34,
+        0x1C, 0x34, 0x08, 0x72,
+        0x08, 0x72, 0x08, 0x72
+};
+
+int rv770_set_smc_sram_address(struct radeon_device *rdev,
+                               u16 smc_address, u16 limit)
+{
+        u32 addr;
+
+        if (smc_address & 3)
+                return -EINVAL;
+        if ((smc_address + 3) > limit)
+                return -EINVAL;
+
+        addr = smc_address;
+        addr |= SMC_SRAM_AUTO_INC_DIS;
+
+        WREG32(SMC_SRAM_ADDR, addr);
+
+        return 0;
+}
+
+int rv770_copy_bytes_to_smc(struct radeon_device *rdev,
+                            u16 smc_start_address, const u8 *src,
+                            u16 byte_count, u16 limit)
+{
+        u32 data, original_data, extra_shift;
+        u16 addr;
+        int ret;
+
+        if (smc_start_address & 3)
+                return -EINVAL;
+        if ((smc_start_address + byte_count) > limit)
+                return -EINVAL;
+
+        addr = smc_start_address;
+
+        while (byte_count >= 4) {
+                /* SMC address space is BE */
+                data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3];
+
+                ret = rv770_set_smc_sram_address(rdev, addr, limit);
+                if (ret)
+                        return ret;
+
+                WREG32(SMC_SRAM_DATA, data);
+
+                src += 4;
+                byte_count -= 4;
+                addr += 4;
+        }
+
+        /* RMW for final bytes */
+        if (byte_count > 0) {
+                data = 0;
+
+                ret = rv770_set_smc_sram_address(rdev, addr, limit);
+                if (ret)
+                        return ret;
+
+                original_data = RREG32(SMC_SRAM_DATA);
+
+                extra_shift = 8 * (4 - byte_count);
+
+                while (byte_count > 0) {
+                        /* SMC address space is BE */
+                        data = (data << 8) + *src++;
+                        byte_count--;
+                }
+
+                data <<= extra_shift;
+
+                data |= (original_data & ~((~0UL) << extra_shift));
+
+                ret = rv770_set_smc_sram_address(rdev, addr, limit);
+                if (ret)
+                        return ret;
+
+                WREG32(SMC_SRAM_DATA, data);
+        }
+
+        return 0;
+}
+
+static int rv770_program_interrupt_vectors(struct radeon_device *rdev,
+                                           u32 smc_first_vector, const u8 *src,
+                                           u32 byte_count)
+{
+        u32 tmp, i;
+
+        if (byte_count % 4)
+                return -EINVAL;
+
+        if (smc_first_vector < FIRST_SMC_INT_VECT_REG) {
+                tmp = FIRST_SMC_INT_VECT_REG - smc_first_vector;
+
+                if (tmp > byte_count)
+                        return 0;
+
+                byte_count -= tmp;
+                src += tmp;
+                smc_first_vector = FIRST_SMC_INT_VECT_REG;
+        }
+
+        for (i = 0; i < byte_count; i += 4) {
+                /* SMC address space is BE */
+                tmp = (src[i] << 24) | (src[i + 1] << 16) | (src[i + 2] << 8) | src[i + 3];
+
+                WREG32(SMC_ISR_FFD8_FFDB + i, tmp);
+        }
+
+        return 0;
+}
+
+void rv770_start_smc(struct radeon_device *rdev)
+{
+        WREG32_P(SMC_IO, SMC_RST_N, ~SMC_RST_N);
+}
+
+void rv770_reset_smc(struct radeon_device *rdev)
+{
+        WREG32_P(SMC_IO, 0, ~SMC_RST_N);
+}
+
+void rv770_stop_smc_clock(struct radeon_device *rdev)
+{
+        WREG32_P(SMC_IO, 0, ~SMC_CLK_EN);
+}
+
+void rv770_start_smc_clock(struct radeon_device *rdev)
+{
+        WREG32_P(SMC_IO, SMC_CLK_EN, ~SMC_CLK_EN);
+}
+
+bool rv770_is_smc_running(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32(SMC_IO);
+
+        if ((tmp & SMC_RST_N) && (tmp & SMC_CLK_EN))
+                return true;
+        else
+                return false;
+}
+
+PPSMC_Result rv770_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg)
+{
+        u32 tmp;
+        int i;
+        PPSMC_Result result;
+
+        if (!rv770_is_smc_running(rdev))
+                return PPSMC_Result_Failed;
+
+        WREG32_P(SMC_MSG, HOST_SMC_MSG(msg), ~HOST_SMC_MSG_MASK);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = RREG32(SMC_MSG) & HOST_SMC_RESP_MASK;
+                tmp >>= HOST_SMC_RESP_SHIFT;
+                if (tmp != 0)
+                        break;
+                udelay(1);
+        }
+
+        tmp = RREG32(SMC_MSG) & HOST_SMC_RESP_MASK;
+        tmp >>= HOST_SMC_RESP_SHIFT;
+
+        result = (PPSMC_Result)tmp;
+        return result;
+}
+
+PPSMC_Result rv770_wait_for_smc_inactive(struct radeon_device *rdev)
+{
+        int i;
+        PPSMC_Result result = PPSMC_Result_OK;
+
+        if (!rv770_is_smc_running(rdev))
+                return result;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(SMC_IO) & SMC_STOP_MODE)
+                        break;
+                udelay(1);
+        }
+
+        return result;
+}
+
+static void rv770_clear_smc_sram(struct radeon_device *rdev, u16 limit)
+{
+        u16 i;
+
+        for (i = 0;  i < limit; i += 4) {
+                rv770_set_smc_sram_address(rdev, i, limit);
+                WREG32(SMC_SRAM_DATA, 0);
+        }
+}
+
+int rv770_load_smc_ucode(struct radeon_device *rdev,
+                         u16 limit)
+{
+        int ret;
+        const u8 *int_vect;
+        u16 int_vect_start_address;
+        u16 int_vect_size;
+        const u8 *ucode_data;
+        u16 ucode_start_address;
+        u16 ucode_size;
+
+        if (!rdev->smc_fw)
+                return -EINVAL;
+
+        rv770_clear_smc_sram(rdev, limit);
+
+        switch (rdev->family) {
+        case CHIP_RV770:
+                ucode_start_address = RV770_SMC_UCODE_START;
+                ucode_size = RV770_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&rv770_smc_int_vectors;
+                int_vect_start_address = RV770_SMC_INT_VECTOR_START;
+                int_vect_size = RV770_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_RV730:
+                ucode_start_address = RV730_SMC_UCODE_START;
+                ucode_size = RV730_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&rv730_smc_int_vectors;
+                int_vect_start_address = RV730_SMC_INT_VECTOR_START;
+                int_vect_size = RV730_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_RV710:
+                ucode_start_address = RV710_SMC_UCODE_START;
+                ucode_size = RV710_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&rv710_smc_int_vectors;
+                int_vect_start_address = RV710_SMC_INT_VECTOR_START;
+                int_vect_size = RV710_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_RV740:
+                ucode_start_address = RV740_SMC_UCODE_START;
+                ucode_size = RV740_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&rv740_smc_int_vectors;
+                int_vect_start_address = RV740_SMC_INT_VECTOR_START;
+                int_vect_size = RV740_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_CEDAR:
+                ucode_start_address = CEDAR_SMC_UCODE_START;
+                ucode_size = CEDAR_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&cedar_smc_int_vectors;
+                int_vect_start_address = CEDAR_SMC_INT_VECTOR_START;
+                int_vect_size = CEDAR_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_REDWOOD:
+                ucode_start_address = REDWOOD_SMC_UCODE_START;
+                ucode_size = REDWOOD_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&redwood_smc_int_vectors;
+                int_vect_start_address = REDWOOD_SMC_INT_VECTOR_START;
+                int_vect_size = REDWOOD_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_JUNIPER:
+                ucode_start_address = JUNIPER_SMC_UCODE_START;
+                ucode_size = JUNIPER_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&juniper_smc_int_vectors;
+                int_vect_start_address = JUNIPER_SMC_INT_VECTOR_START;
+                int_vect_size = JUNIPER_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_CYPRESS:
+        case CHIP_HEMLOCK:
+                ucode_start_address = CYPRESS_SMC_UCODE_START;
+                ucode_size = CYPRESS_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&cypress_smc_int_vectors;
+                int_vect_start_address = CYPRESS_SMC_INT_VECTOR_START;
+                int_vect_size = CYPRESS_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_BARTS:
+                ucode_start_address = BARTS_SMC_UCODE_START;
+                ucode_size = BARTS_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&barts_smc_int_vectors;
+                int_vect_start_address = BARTS_SMC_INT_VECTOR_START;
+                int_vect_size = BARTS_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_TURKS:
+                ucode_start_address = TURKS_SMC_UCODE_START;
+                ucode_size = TURKS_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&turks_smc_int_vectors;
+                int_vect_start_address = TURKS_SMC_INT_VECTOR_START;
+                int_vect_size = TURKS_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_CAICOS:
+                ucode_start_address = CAICOS_SMC_UCODE_START;
+                ucode_size = CAICOS_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&caicos_smc_int_vectors;
+                int_vect_start_address = CAICOS_SMC_INT_VECTOR_START;
+                int_vect_size = CAICOS_SMC_INT_VECTOR_SIZE;
+                break;
+        case CHIP_CAYMAN:
+                ucode_start_address = CAYMAN_SMC_UCODE_START;
+                ucode_size = CAYMAN_SMC_UCODE_SIZE;
+                int_vect = (const u8 *)&cayman_smc_int_vectors;
+                int_vect_start_address = CAYMAN_SMC_INT_VECTOR_START;
+                int_vect_size = CAYMAN_SMC_INT_VECTOR_SIZE;
+                break;
+        default:
+                DRM_ERROR("unknown asic in smc ucode loader\n");
+                BUG();
+        }
+
+        /* load the ucode */
+        ucode_data = (const u8 *)rdev->smc_fw->data;
+        ret = rv770_copy_bytes_to_smc(rdev, ucode_start_address,
+                                      ucode_data, ucode_size, limit);
+        if (ret)
+                return ret;
+
+        /* set up the int vectors */
+        ret = rv770_program_interrupt_vectors(rdev, int_vect_start_address,
+                                              int_vect, int_vect_size);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+int rv770_read_smc_sram_dword(struct radeon_device *rdev,
+                              u16 smc_address, u32 *value, u16 limit)
+{
+        int ret;
+
+        ret = rv770_set_smc_sram_address(rdev, smc_address, limit);
+        if (ret)
+                return ret;
+
+        *value = RREG32(SMC_SRAM_DATA);
+
+        return 0;
+}
+
+int rv770_write_smc_sram_dword(struct radeon_device *rdev,
+                               u16 smc_address, u32 value, u16 limit)
+{
+        int ret;
+
+        ret = rv770_set_smc_sram_address(rdev, smc_address, limit);
+        if (ret)
+                return ret;
+
+        WREG32(SMC_SRAM_DATA, value);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/radeon/rv770_smc.h b/drivers/gpu/drm/radeon/rv770_smc.h
new file mode 100644
index 000000000000..f78d92a4b325
--- /dev/null
+++ b/drivers/gpu/drm/radeon/rv770_smc.h
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RV770_SMC_H__
+#define __RV770_SMC_H__
+
+#include "ppsmc.h"
+
+#pragma pack(push, 1)
+
+#define RV770_SMC_TABLE_ADDRESS 0xB000
+
+#define RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE    3
+
+struct RV770_SMC_SCLK_VALUE
+{
+    uint32_t        vCG_SPLL_FUNC_CNTL;
+    uint32_t        vCG_SPLL_FUNC_CNTL_2;
+    uint32_t        vCG_SPLL_FUNC_CNTL_3;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM_2;
+    uint32_t        sclk_value;
+};
+
+typedef struct RV770_SMC_SCLK_VALUE RV770_SMC_SCLK_VALUE;
+
+struct RV770_SMC_MCLK_VALUE
+{
+    uint32_t        vMPLL_AD_FUNC_CNTL;
+    uint32_t        vMPLL_AD_FUNC_CNTL_2;
+    uint32_t        vMPLL_DQ_FUNC_CNTL;
+    uint32_t        vMPLL_DQ_FUNC_CNTL_2;
+    uint32_t        vMCLK_PWRMGT_CNTL;
+    uint32_t        vDLL_CNTL;
+    uint32_t        vMPLL_SS;
+    uint32_t        vMPLL_SS2;
+    uint32_t        mclk_value;
+};
+
+typedef struct RV770_SMC_MCLK_VALUE RV770_SMC_MCLK_VALUE;
+
+
+struct RV730_SMC_MCLK_VALUE
+{
+    uint32_t        vMCLK_PWRMGT_CNTL;
+    uint32_t        vDLL_CNTL;
+    uint32_t        vMPLL_FUNC_CNTL;
+    uint32_t        vMPLL_FUNC_CNTL2;
+    uint32_t        vMPLL_FUNC_CNTL3;
+    uint32_t        vMPLL_SS;
+    uint32_t        vMPLL_SS2;
+    uint32_t        mclk_value;
+};
+
+typedef struct RV730_SMC_MCLK_VALUE RV730_SMC_MCLK_VALUE;
+
+struct RV770_SMC_VOLTAGE_VALUE
+{
+    uint16_t             value;
+    uint8_t              index;
+    uint8_t              padding;
+};
+
+typedef struct RV770_SMC_VOLTAGE_VALUE RV770_SMC_VOLTAGE_VALUE;
+
+union RV7XX_SMC_MCLK_VALUE
+{
+    RV770_SMC_MCLK_VALUE    mclk770;
+    RV730_SMC_MCLK_VALUE    mclk730;
+};
+
+typedef union RV7XX_SMC_MCLK_VALUE RV7XX_SMC_MCLK_VALUE, *LPRV7XX_SMC_MCLK_VALUE;
+
+struct RV770_SMC_HW_PERFORMANCE_LEVEL
+{
+    uint8_t                 arbValue;
+    union{
+        uint8_t             seqValue;
+        uint8_t             ACIndex;
+    };
+    uint8_t                 displayWatermark;
+    uint8_t                 gen2PCIE;
+    uint8_t                 gen2XSP;
+    uint8_t                 backbias;
+    uint8_t                 strobeMode;
+    uint8_t                 mcFlags;
+    uint32_t                aT;
+    uint32_t                bSP;
+    RV770_SMC_SCLK_VALUE    sclk;
+    RV7XX_SMC_MCLK_VALUE    mclk;
+    RV770_SMC_VOLTAGE_VALUE vddc;
+    RV770_SMC_VOLTAGE_VALUE mvdd;
+    RV770_SMC_VOLTAGE_VALUE vddci;
+    uint8_t                 reserved1;
+    uint8_t                 reserved2;
+    uint8_t                 stateFlags;
+    uint8_t                 padding;
+};
+
+#define SMC_STROBE_RATIO    0x0F
+#define SMC_STROBE_ENABLE   0x10
+
+#define SMC_MC_EDC_RD_FLAG  0x01
+#define SMC_MC_EDC_WR_FLAG  0x02
+#define SMC_MC_RTT_ENABLE   0x04
+#define SMC_MC_STUTTER_EN   0x08
+
+typedef struct RV770_SMC_HW_PERFORMANCE_LEVEL RV770_SMC_HW_PERFORMANCE_LEVEL;
+
+struct RV770_SMC_SWSTATE
+{
+    uint8_t           flags;
+    uint8_t           padding1;
+    uint8_t           padding2;
+    uint8_t           padding3;
+    RV770_SMC_HW_PERFORMANCE_LEVEL levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
+};
+
+typedef struct RV770_SMC_SWSTATE RV770_SMC_SWSTATE;
+
+#define RV770_SMC_VOLTAGEMASK_VDDC 0
+#define RV770_SMC_VOLTAGEMASK_MVDD 1
+#define RV770_SMC_VOLTAGEMASK_VDDCI 2
+#define RV770_SMC_VOLTAGEMASK_MAX  4
+
+struct RV770_SMC_VOLTAGEMASKTABLE
+{
+    uint8_t  highMask[RV770_SMC_VOLTAGEMASK_MAX];
+    uint32_t lowMask[RV770_SMC_VOLTAGEMASK_MAX];
+};
+
+typedef struct RV770_SMC_VOLTAGEMASKTABLE RV770_SMC_VOLTAGEMASKTABLE;
+
+#define MAX_NO_VREG_STEPS 32
+
+struct RV770_SMC_STATETABLE
+{
+    uint8_t             thermalProtectType;
+    uint8_t             systemFlags;
+    uint8_t             maxVDDCIndexInPPTable;
+    uint8_t             extraFlags;
+    uint8_t             highSMIO[MAX_NO_VREG_STEPS];
+    uint32_t            lowSMIO[MAX_NO_VREG_STEPS];
+    RV770_SMC_VOLTAGEMASKTABLE voltageMaskTable;
+    RV770_SMC_SWSTATE   initialState;
+    RV770_SMC_SWSTATE   ACPIState;
+    RV770_SMC_SWSTATE   driverState;
+    RV770_SMC_SWSTATE   ULVState;
+};
+
+typedef struct RV770_SMC_STATETABLE RV770_SMC_STATETABLE;
+
+#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP 0x01
+
+#pragma pack(pop)
+
+#define RV770_SMC_SOFT_REGISTERS_START        0x104
+
+#define RV770_SMC_SOFT_REGISTER_mclk_chg_timeout        0x0
+#define RV770_SMC_SOFT_REGISTER_baby_step_timer         0x8
+#define RV770_SMC_SOFT_REGISTER_delay_bbias             0xC
+#define RV770_SMC_SOFT_REGISTER_delay_vreg              0x10
+#define RV770_SMC_SOFT_REGISTER_delay_acpi              0x2C
+#define RV770_SMC_SOFT_REGISTER_seq_index               0x64
+#define RV770_SMC_SOFT_REGISTER_mvdd_chg_time           0x68
+#define RV770_SMC_SOFT_REGISTER_mclk_switch_lim         0x78
+#define RV770_SMC_SOFT_REGISTER_mc_block_delay          0x90
+#define RV770_SMC_SOFT_REGISTER_uvd_enabled             0x9C
+#define RV770_SMC_SOFT_REGISTER_is_asic_lombok          0xA0
+
+int rv770_set_smc_sram_address(struct radeon_device *rdev,
+                               u16 smc_address, u16 limit);
+int rv770_copy_bytes_to_smc(struct radeon_device *rdev,
+                            u16 smc_start_address, const u8 *src,
+                            u16 byte_count, u16 limit);
+void rv770_start_smc(struct radeon_device *rdev);
+void rv770_reset_smc(struct radeon_device *rdev);
+void rv770_stop_smc_clock(struct radeon_device *rdev);
+void rv770_start_smc_clock(struct radeon_device *rdev);
+bool rv770_is_smc_running(struct radeon_device *rdev);
+PPSMC_Result rv770_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg);
+PPSMC_Result rv770_wait_for_smc_inactive(struct radeon_device *rdev);
+int rv770_read_smc_sram_dword(struct radeon_device *rdev,
+                              u16 smc_address, u32 *value, u16 limit);
+int rv770_write_smc_sram_dword(struct radeon_device *rdev,
+                               u16 smc_address, u32 value, u16 limit);
+int rv770_load_smc_ucode(struct radeon_device *rdev,
+                         u16 limit);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/rv770d.h b/drivers/gpu/drm/radeon/rv770d.h
index 85b16266f748..6bef2b7d601b 100644
--- a/drivers/gpu/drm/radeon/rv770d.h
+++ b/drivers/gpu/drm/radeon/rv770d.h
@@ -62,6 +62,246 @@
 #       define UPLL_FB_DIV(x)                           ((x) << 0)
 #       define UPLL_FB_DIV_MASK                         0x01FFFFFF
 
+/* pm registers */
+#define SMC_SRAM_ADDR                                   0x200
+#define         SMC_SRAM_AUTO_INC_DIS                           (1 << 16)
+#define SMC_SRAM_DATA                                   0x204
+#define SMC_IO                                          0x208
+#define         SMC_RST_N                                       (1 << 0)
+#define         SMC_STOP_MODE                                   (1 << 2)
+#define         SMC_CLK_EN                                      (1 << 11)
+#define SMC_MSG                                         0x20c
+#define         HOST_SMC_MSG(x)                                 ((x) << 0)
+#define         HOST_SMC_MSG_MASK                               (0xff << 0)
+#define         HOST_SMC_MSG_SHIFT                              0
+#define         HOST_SMC_RESP(x)                                ((x) << 8)
+#define         HOST_SMC_RESP_MASK                              (0xff << 8)
+#define         HOST_SMC_RESP_SHIFT                             8
+#define         SMC_HOST_MSG(x)                                 ((x) << 16)
+#define         SMC_HOST_MSG_MASK                               (0xff << 16)
+#define         SMC_HOST_MSG_SHIFT                              16
+#define         SMC_HOST_RESP(x)                                ((x) << 24)
+#define         SMC_HOST_RESP_MASK                              (0xff << 24)
+#define         SMC_HOST_RESP_SHIFT                             24
+
+#define SMC_ISR_FFD8_FFDB                               0x218
+
+#define CG_SPLL_FUNC_CNTL                               0x600
+#define         SPLL_RESET                              (1 << 0)
+#define         SPLL_SLEEP                              (1 << 1)
+#define         SPLL_DIVEN                              (1 << 2)
+#define         SPLL_BYPASS_EN                          (1 << 3)
+#define         SPLL_REF_DIV(x)                         ((x) << 4)
+#define         SPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         SPLL_HILEN(x)                           ((x) << 12)
+#define         SPLL_HILEN_MASK                         (0xf << 12)
+#define         SPLL_LOLEN(x)                           ((x) << 16)
+#define         SPLL_LOLEN_MASK                         (0xf << 16)
+#define CG_SPLL_FUNC_CNTL_2                             0x604
+#define         SCLK_MUX_SEL(x)                         ((x) << 0)
+#define         SCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_SPLL_FUNC_CNTL_3                             0x608
+#define         SPLL_FB_DIV(x)                          ((x) << 0)
+#define         SPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         SPLL_DITHEN                             (1 << 28)
+
+#define SPLL_CNTL_MODE                                  0x610
+#define         SPLL_DIV_SYNC                           (1 << 5)
+
+#define MPLL_AD_FUNC_CNTL                               0x624
+#define         CLKF(x)                                 ((x) << 0)
+#define         CLKF_MASK                               (0x7f << 0)
+#define         CLKR(x)                                 ((x) << 7)
+#define         CLKR_MASK                               (0x1f << 7)
+#define         CLKFRAC(x)                              ((x) << 12)
+#define         CLKFRAC_MASK                            (0x1f << 12)
+#define         YCLK_POST_DIV(x)                        ((x) << 17)
+#define         YCLK_POST_DIV_MASK                      (3 << 17)
+#define         IBIAS(x)                                ((x) << 20)
+#define         IBIAS_MASK                              (0x3ff << 20)
+#define         RESET                                   (1 << 30)
+#define         PDNB                                    (1 << 31)
+#define MPLL_AD_FUNC_CNTL_2                             0x628
+#define         BYPASS                                  (1 << 19)
+#define         BIAS_GEN_PDNB                           (1 << 24)
+#define         RESET_EN                                (1 << 25)
+#define         VCO_MODE                                (1 << 29)
+#define MPLL_DQ_FUNC_CNTL                               0x62c
+#define MPLL_DQ_FUNC_CNTL_2                             0x630
+
+#define GENERAL_PWRMGT                                  0x63c
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define ENABLE_GEN2PCIE                          (1 << 4)
+#       define ENABLE_GEN2XSP                           (1 << 5)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (3 << 6)
+#       define SW_SMIO_INDEX_SHIFT                      6
+#       define LOW_VOLT_D2_ACPI                         (1 << 8)
+#       define LOW_VOLT_D3_ACPI                         (1 << 9)
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define BACKBIAS_PAD_EN                          (1 << 18)
+#       define BACKBIAS_VALUE                           (1 << 19)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#       define AC_DC_SW                                 (1 << 24)
+
+#define CG_TPC                                            0x640
+#define SCLK_PWRMGT_CNTL                                  0x644
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_LOW_D1                                (1 << 1)
+#       define FIR_RESET                                  (1 << 4)
+#       define FIR_FORCE_TREND_SEL                        (1 << 5)
+#       define FIR_TREND_MODE                             (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 7)
+#       define GFX_CLK_FORCE_ON                           (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                        (1 << 9)
+#       define GFX_CLK_FORCE_OFF                          (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                        (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                        (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                        (1 << 13)
+#define MCLK_PWRMGT_CNTL                                0x648
+#       define DLL_SPEED(x)                             ((x) << 0)
+#       define DLL_SPEED_MASK                           (0x1f << 0)
+#       define MPLL_PWRMGT_OFF                          (1 << 5)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCKA0_SLEEP                            (1 << 8)
+#       define MRDCKA1_SLEEP                            (1 << 9)
+#       define MRDCKB0_SLEEP                            (1 << 10)
+#       define MRDCKB1_SLEEP                            (1 << 11)
+#       define MRDCKC0_SLEEP                            (1 << 12)
+#       define MRDCKC1_SLEEP                            (1 << 13)
+#       define MRDCKD0_SLEEP                            (1 << 14)
+#       define MRDCKD1_SLEEP                            (1 << 15)
+#       define MRDCKA0_RESET                            (1 << 16)
+#       define MRDCKA1_RESET                            (1 << 17)
+#       define MRDCKB0_RESET                            (1 << 18)
+#       define MRDCKB1_RESET                            (1 << 19)
+#       define MRDCKC0_RESET                            (1 << 20)
+#       define MRDCKC1_RESET                            (1 << 21)
+#       define MRDCKD0_RESET                            (1 << 22)
+#       define MRDCKD1_RESET                            (1 << 23)
+#       define DLL_READY_READ                           (1 << 24)
+#       define USE_DISPLAY_GAP                          (1 << 25)
+#       define USE_DISPLAY_URGENT_NORMAL                (1 << 26)
+#       define MPLL_TURNOFF_D2                          (1 << 28)
+#define DLL_CNTL                                        0x64c
+#       define MRDCKA0_BYPASS                           (1 << 24)
+#       define MRDCKA1_BYPASS                           (1 << 25)
+#       define MRDCKB0_BYPASS                           (1 << 26)
+#       define MRDCKB1_BYPASS                           (1 << 27)
+#       define MRDCKC0_BYPASS                           (1 << 28)
+#       define MRDCKC1_BYPASS                           (1 << 29)
+#       define MRDCKD0_BYPASS                           (1 << 30)
+#       define MRDCKD1_BYPASS                           (1 << 31)
+
+#define MPLL_TIME                                         0x654
+#       define MPLL_LOCK_TIME(x)                        ((x) << 0)
+#       define MPLL_LOCK_TIME_MASK                      (0xffff << 0)
+#       define MPLL_RESET_TIME(x)                       ((x) << 16)
+#       define MPLL_RESET_TIME_MASK                     (0xffff << 16)
+
+#define CG_CLKPIN_CNTL                                    0x660
+#       define MUX_TCLK_TO_XCLK                           (1 << 8)
+#       define XTALIN_DIVIDE                              (1 << 9)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                  0x66c
+#       define CURRENT_PROFILE_INDEX_MASK                 (0xf << 4)
+#       define CURRENT_PROFILE_INDEX_SHIFT                4
+
+#define S0_VID_LOWER_SMIO_CNTL                            0x678
+#define S1_VID_LOWER_SMIO_CNTL                            0x67c
+#define S2_VID_LOWER_SMIO_CNTL                            0x680
+#define S3_VID_LOWER_SMIO_CNTL                            0x684
+
+#define CG_FTV                                            0x690
+#define CG_FFCT_0                                         0x694
+#       define UTC_0(x)                                   ((x) << 0)
+#       define UTC_0_MASK                                 (0x3ff << 0)
+#       define DTC_0(x)                                   ((x) << 10)
+#       define DTC_0_MASK                                 (0x3ff << 10)
+
+#define CG_BSP                                          0x6d0
+#       define BSP(x)                                   ((x) << 0)
+#       define BSP_MASK                                 (0xffff << 0)
+#       define BSU(x)                                   ((x) << 16)
+#       define BSU_MASK                                 (0xf << 16)
+#define CG_AT                                           0x6d4
+#       define CG_R(x)                                  ((x) << 0)
+#       define CG_R_MASK                                (0xffff << 0)
+#       define CG_L(x)                                  ((x) << 16)
+#       define CG_L_MASK                                (0xffff << 16)
+#define CG_GIT                                          0x6d8
+#       define CG_GICST(x)                              ((x) << 0)
+#       define CG_GICST_MASK                            (0xffff << 0)
+#       define CG_GIPOT(x)                              ((x) << 16)
+#       define CG_GIPOT_MASK                            (0xffff << 16)
+
+#define CG_SSP                                            0x6e8
+#       define SST(x)                                     ((x) << 0)
+#       define SST_MASK                                   (0xffff << 0)
+#       define SSTU(x)                                    ((x) << 16)
+#       define SSTU_MASK                                  (0xf << 16)
+
+#define CG_DISPLAY_GAP_CNTL                               0x714
+#       define DISP1_GAP(x)                               ((x) << 0)
+#       define DISP1_GAP_MASK                             (3 << 0)
+#       define DISP2_GAP(x)                               ((x) << 2)
+#       define DISP2_GAP_MASK                             (3 << 2)
+#       define VBI_TIMER_COUNT(x)                         ((x) << 4)
+#       define VBI_TIMER_COUNT_MASK                       (0x3fff << 4)
+#       define VBI_TIMER_UNIT(x)                          ((x) << 20)
+#       define VBI_TIMER_UNIT_MASK                        (7 << 20)
+#       define DISP1_GAP_MCHG(x)                          ((x) << 24)
+#       define DISP1_GAP_MCHG_MASK                        (3 << 24)
+#       define DISP2_GAP_MCHG(x)                          ((x) << 26)
+#       define DISP2_GAP_MCHG_MASK                        (3 << 26)
+
+#define CG_SPLL_SPREAD_SPECTRUM                         0x790
+#define         SSEN                                    (1 << 0)
+#define         CLKS(x)                                 ((x) << 4)
+#define         CLKS_MASK                               (0xfff << 4)
+#define CG_SPLL_SPREAD_SPECTRUM_2                       0x794
+#define         CLKV(x)                                 ((x) << 0)
+#define         CLKV_MASK                               (0x3ffffff << 0)
+#define CG_MPLL_SPREAD_SPECTRUM                         0x798
+#define CG_UPLL_SPREAD_SPECTRUM                         0x79c
+#       define SSEN_MASK                                0x00000001
+
+#define CG_CGTT_LOCAL_0                                   0x7d0
+#define CG_CGTT_LOCAL_1                                   0x7d4
+
+#define BIOS_SCRATCH_4                                    0x1734
+
+#define MC_SEQ_MISC0                                      0x2a00
+#define         MC_SEQ_MISC0_GDDR5_SHIFT                  28
+#define         MC_SEQ_MISC0_GDDR5_MASK                   0xf0000000
+#define         MC_SEQ_MISC0_GDDR5_VALUE                  5
+
+#define MC_ARB_SQM_RATIO                                  0x2770
+#define         STATE0(x)                               ((x) << 0)
+#define         STATE0_MASK                             (0xff << 0)
+#define         STATE1(x)                               ((x) << 8)
+#define         STATE1_MASK                             (0xff << 8)
+#define         STATE2(x)                               ((x) << 16)
+#define         STATE2_MASK                             (0xff << 16)
+#define         STATE3(x)                               ((x) << 24)
+#define         STATE3_MASK                             (0xff << 24)
+
+#define MC_ARB_RFSH_RATE                                0x27b0
+#define         POWERMODE0(x)                           ((x) << 0)
+#define         POWERMODE0_MASK                         (0xff << 0)
+#define         POWERMODE1(x)                           ((x) << 8)
+#define         POWERMODE1_MASK                         (0xff << 8)
+#define         POWERMODE2(x)                           ((x) << 16)
+#define         POWERMODE2_MASK                         (0xff << 16)
+#define         POWERMODE3(x)                           ((x) << 24)
+#define         POWERMODE3_MASK                         (0xff << 24)
+
+#define CGTS_SM_CTRL_REG                                  0x9150
+
 /* Registers */
 #define CB_COLOR0_BASE                                  0x28040
 #define CB_COLOR1_BASE                                  0x28044
@@ -86,8 +326,8 @@
 #define CONFIG_MEMSIZE                                  0x5428
 
 #define CP_ME_CNTL                                      0x86D8
-#define         CP_ME_HALT                                      (1<<28)
-#define         CP_PFP_HALT                                     (1<<26)
+#define         CP_ME_HALT                                      (1 << 28)
+#define         CP_PFP_HALT                                     (1 << 26)
 #define CP_ME_RAM_DATA                                  0xC160
 #define CP_ME_RAM_RADDR                                 0xC158
 #define CP_ME_RAM_WADDR                                 0xC15C
@@ -157,9 +397,22 @@
 #define         GUI_ACTIVE                                      (1<<31)
 #define GRBM_STATUS2                                    0x8014
 
-#define CG_CLKPIN_CNTL                                    0x660
-#       define MUX_TCLK_TO_XCLK                           (1 << 8)
-#       define XTALIN_DIVIDE                              (1 << 9)
+#define CG_THERMAL_CTRL                                 0x72C
+#define         DPM_EVENT_SRC(x)                        ((x) << 0)
+#define         DPM_EVENT_SRC_MASK                      (7 << 0)
+#define         DIG_THERM_DPM(x)                        ((x) << 14)
+#define         DIG_THERM_DPM_MASK                      0x003FC000
+#define         DIG_THERM_DPM_SHIFT                     14
+
+#define CG_THERMAL_INT                                  0x734
+#define         DIG_THERM_INTH(x)                       ((x) << 8)
+#define         DIG_THERM_INTH_MASK                     0x0000FF00
+#define         DIG_THERM_INTH_SHIFT                    8
+#define         DIG_THERM_INTL(x)                       ((x) << 16)
+#define         DIG_THERM_INTL_MASK                     0x00FF0000
+#define         DIG_THERM_INTL_SHIFT                    16
+#define         THERM_INT_MASK_HIGH                     (1 << 24)
+#define         THERM_INT_MASK_LOW                      (1 << 25)
 
 #define CG_MULT_THERMAL_STATUS                          0x740
 #define         ASIC_T(x)                               ((x) << 16)
@@ -662,7 +915,22 @@
 #define D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH             0x691c
 #define D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH             0x611c
 
-/* PCIE link stuff */
+/* PCIE indirect regs */
+#define PCIE_P_CNTL                                       0x40
+#       define P_PLL_PWRDN_IN_L1L23                       (1 << 3)
+#       define P_PLL_BUF_PDNB                             (1 << 4)
+#       define P_PLL_PDNB                                 (1 << 9)
+#       define P_ALLOW_PRX_FRONTEND_SHUTOFF               (1 << 12)
+/* PCIE PORT regs */
+#define PCIE_LC_CNTL                                      0xa0
+#       define LC_L0S_INACTIVITY(x)                       ((x) << 8)
+#       define LC_L0S_INACTIVITY_MASK                     (0xf << 8)
+#       define LC_L0S_INACTIVITY_SHIFT                    8
+#       define LC_L1_INACTIVITY(x)                        ((x) << 12)
+#       define LC_L1_INACTIVITY_MASK                      (0xf << 12)
+#       define LC_L1_INACTIVITY_SHIFT                     12
+#       define LC_PMI_TO_L1_DIS                           (1 << 16)
+#       define LC_ASPM_TO_L1_DIS                          (1 << 24)
 #define PCIE_LC_TRAINING_CNTL                             0xa1 /* PCIE_P */
 #define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
 #       define LC_LINK_WIDTH_SHIFT                        0
@@ -690,6 +958,9 @@
 #       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 8)
 #       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     3
 #       define LC_CURRENT_DATA_RATE                       (1 << 11)
+#       define LC_HW_VOLTAGE_IF_CONTROL(x)                ((x) << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_MASK              (3 << 12)
+#       define LC_HW_VOLTAGE_IF_CONTROL_SHIFT             12
 #       define LC_VOLTAGE_TIMER_SEL_MASK                  (0xf << 14)
 #       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 21)
 #       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 23)
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index a1b0da6b5808..234906709067 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -32,40 +32,43 @@
 #include "sid.h"
 #include "atom.h"
 #include "si_blit_shaders.h"
+#include "clearstate_si.h"
+#include "radeon_ucode.h"
 
-#define SI_PFP_UCODE_SIZE 2144
-#define SI_PM4_UCODE_SIZE 2144
-#define SI_CE_UCODE_SIZE 2144
-#define SI_RLC_UCODE_SIZE 2048
-#define SI_MC_UCODE_SIZE 7769
-#define OLAND_MC_UCODE_SIZE 7863
 
 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
+MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
+MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
 MODULE_FIRMWARE("radeon/VERDE_me.bin");
 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
+MODULE_FIRMWARE("radeon/VERDE_smc.bin");
 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
 MODULE_FIRMWARE("radeon/OLAND_me.bin");
 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
+MODULE_FIRMWARE("radeon/OLAND_smc.bin");
 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
+MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 
+static void si_pcie_gen3_enable(struct radeon_device *rdev);
+static void si_program_aspm(struct radeon_device *rdev);
 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 extern void r600_ih_ring_fini(struct radeon_device *rdev);
 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
@@ -75,6 +78,228 @@ extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 
+static const u32 verde_rlc_save_restore_register_list[] =
+{
+        (0x8000 << 16) | (0x98f4 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x98f4 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0xe80 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0xe80 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x89bc >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x89bc >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x8c1c >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x8c1c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x98f0 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xe7c >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x9148 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x9148 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9150 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x897c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8d8c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xac54 >> 2),
+        0X00000000,
+        0x3,
+        (0x9c00 << 16) | (0x98f8 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9910 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9914 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9918 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x991c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9920 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9924 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9928 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x992c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9930 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9934 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9938 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x993c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9940 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9944 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9948 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x994c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9950 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9954 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9958 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x995c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9960 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9964 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9968 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x996c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9970 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9974 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9978 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x997c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9980 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9984 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9988 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x998c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8c00 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8c14 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8c04 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8c08 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x9b7c >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x9b7c >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0xe84 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0xe84 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x89c0 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x89c0 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x914c >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x914c >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x8c20 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x8c20 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x9354 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x9354 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9060 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9364 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9100 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x913c >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x90e0 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x90e4 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x90e8 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x90e0 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x90e4 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x90e8 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8bcc >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8b24 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x88c4 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8e50 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8c0c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8e58 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8e5c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9508 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x950c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9494 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xac0c >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xac10 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xac14 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xae00 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0xac08 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x88d4 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x88c8 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x88cc >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x89b0 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8b10 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x8a14 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9830 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9834 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9838 >> 2),
+        0x00000000,
+        (0x9c00 << 16) | (0x9a10 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x9870 >> 2),
+        0x00000000,
+        (0x8000 << 16) | (0x9874 >> 2),
+        0x00000000,
+        (0x8001 << 16) | (0x9870 >> 2),
+        0x00000000,
+        (0x8001 << 16) | (0x9874 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x9870 >> 2),
+        0x00000000,
+        (0x8040 << 16) | (0x9874 >> 2),
+        0x00000000,
+        (0x8041 << 16) | (0x9870 >> 2),
+        0x00000000,
+        (0x8041 << 16) | (0x9874 >> 2),
+        0x00000000,
+        0x00000000
+};
+
 static const u32 tahiti_golden_rlc_registers[] =
 {
         0xc424, 0xffffffff, 0x00601005,
@@ -1320,6 +1545,7 @@ static int si_init_microcode(struct radeon_device *rdev)
         const char *chip_name;
         const char *rlc_chip_name;
         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
+        size_t smc_req_size;
         char fw_name[30];
         int err;
 
@@ -1341,6 +1567,7 @@ static int si_init_microcode(struct radeon_device *rdev)
                 ce_req_size = SI_CE_UCODE_SIZE * 4;
                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
                 mc_req_size = SI_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_PITCAIRN:
                 chip_name = "PITCAIRN";
@@ -1350,6 +1577,7 @@ static int si_init_microcode(struct radeon_device *rdev)
                 ce_req_size = SI_CE_UCODE_SIZE * 4;
                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
                 mc_req_size = SI_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_VERDE:
                 chip_name = "VERDE";
@@ -1359,6 +1587,7 @@ static int si_init_microcode(struct radeon_device *rdev)
                 ce_req_size = SI_CE_UCODE_SIZE * 4;
                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
                 mc_req_size = SI_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_OLAND:
                 chip_name = "OLAND";
@@ -1368,6 +1597,7 @@ static int si_init_microcode(struct radeon_device *rdev)
                 ce_req_size = SI_CE_UCODE_SIZE * 4;
                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
                 break;
         case CHIP_HAINAN:
                 chip_name = "HAINAN";
@@ -1377,6 +1607,7 @@ static int si_init_microcode(struct radeon_device *rdev)
                 ce_req_size = SI_CE_UCODE_SIZE * 4;
                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
                 break;
         default: BUG();
         }
@@ -1439,6 +1670,17 @@ static int si_init_microcode(struct radeon_device *rdev)
                 err = -EINVAL;
         }
 
+        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
+        err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
+        if (err)
+                goto out;
+        if (rdev->smc_fw->size != smc_req_size) {
+                printk(KERN_ERR
+                       "si_smc: Bogus length %zu in firmware \"%s\"\n",
+                       rdev->smc_fw->size, fw_name);
+                err = -EINVAL;
+        }
+
 out:
         platform_device_unregister(pdev);
 
@@ -1457,6 +1699,8 @@ out:
                 rdev->rlc_fw = NULL;
                 release_firmware(rdev->mc_fw);
                 rdev->mc_fw = NULL;
+                release_firmware(rdev->smc_fw);
+                rdev->smc_fw = NULL;
         }
         return err;
 }
@@ -1792,7 +2036,8 @@ static void dce6_program_watermarks(struct radeon_device *rdev,
                                          u32 lb_size, u32 num_heads)
 {
         struct drm_display_mode *mode = &radeon_crtc->base.mode;
-        struct dce6_wm_params wm;
+        struct dce6_wm_params wm_low, wm_high;
+        u32 dram_channels;
         u32 pixel_period;
         u32 line_time = 0;
         u32 latency_watermark_a = 0, latency_watermark_b = 0;
@@ -1808,38 +2053,83 @@ static void dce6_program_watermarks(struct radeon_device *rdev,
                 priority_a_cnt = 0;
                 priority_b_cnt = 0;
 
-                wm.yclk = rdev->pm.current_mclk * 10;
-                wm.sclk = rdev->pm.current_sclk * 10;
-                wm.disp_clk = mode->clock;
-                wm.src_width = mode->crtc_hdisplay;
-                wm.active_time = mode->crtc_hdisplay * pixel_period;
-                wm.blank_time = line_time - wm.active_time;
-                wm.interlaced = false;
-                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-                        wm.interlaced = true;
-                wm.vsc = radeon_crtc->vsc;
-                wm.vtaps = 1;
-                if (radeon_crtc->rmx_type != RMX_OFF)
-                        wm.vtaps = 2;
-                wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
-                wm.lb_size = lb_size;
                 if (rdev->family == CHIP_ARUBA)
-                        wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
+                        dram_channels = evergreen_get_number_of_dram_channels(rdev);
                 else
-                        wm.dram_channels = si_get_number_of_dram_channels(rdev);
-                wm.num_heads = num_heads;
+                        dram_channels = si_get_number_of_dram_channels(rdev);
+
+                /* watermark for high clocks */
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        wm_high.yclk =
+                                radeon_dpm_get_mclk(rdev, false) * 10;
+                        wm_high.sclk =
+                                radeon_dpm_get_sclk(rdev, false) * 10;
+                } else {
+                        wm_high.yclk = rdev->pm.current_mclk * 10;
+                        wm_high.sclk = rdev->pm.current_sclk * 10;
+                }
+
+                wm_high.disp_clk = mode->clock;
+                wm_high.src_width = mode->crtc_hdisplay;
+                wm_high.active_time = mode->crtc_hdisplay * pixel_period;
+                wm_high.blank_time = line_time - wm_high.active_time;
+                wm_high.interlaced = false;
+                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                        wm_high.interlaced = true;
+                wm_high.vsc = radeon_crtc->vsc;
+                wm_high.vtaps = 1;
+                if (radeon_crtc->rmx_type != RMX_OFF)
+                        wm_high.vtaps = 2;
+                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
+                wm_high.lb_size = lb_size;
+                wm_high.dram_channels = dram_channels;
+                wm_high.num_heads = num_heads;
+
+                /* watermark for low clocks */
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        wm_low.yclk =
+                                radeon_dpm_get_mclk(rdev, true) * 10;
+                        wm_low.sclk =
+                                radeon_dpm_get_sclk(rdev, true) * 10;
+                } else {
+                        wm_low.yclk = rdev->pm.current_mclk * 10;
+                        wm_low.sclk = rdev->pm.current_sclk * 10;
+                }
+
+                wm_low.disp_clk = mode->clock;
+                wm_low.src_width = mode->crtc_hdisplay;
+                wm_low.active_time = mode->crtc_hdisplay * pixel_period;
+                wm_low.blank_time = line_time - wm_low.active_time;
+                wm_low.interlaced = false;
+                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                        wm_low.interlaced = true;
+                wm_low.vsc = radeon_crtc->vsc;
+                wm_low.vtaps = 1;
+                if (radeon_crtc->rmx_type != RMX_OFF)
+                        wm_low.vtaps = 2;
+                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
+                wm_low.lb_size = lb_size;
+                wm_low.dram_channels = dram_channels;
+                wm_low.num_heads = num_heads;
 
                 /* set for high clocks */
-                latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
+                latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
                 /* set for low clocks */
-                /* wm.yclk = low clk; wm.sclk = low clk */
-                latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
+                latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
 
                 /* possibly force display priority to high */
                 /* should really do this at mode validation time... */
-                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
-                    !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
-                    !dce6_check_latency_hiding(&wm) ||
+                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
+                    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
+                    !dce6_check_latency_hiding(&wm_high) ||
+                    (rdev->disp_priority == 2)) {
+                        DRM_DEBUG_KMS("force priority to high\n");
+                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
+                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
+                }
+                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
+                    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
+                    !dce6_check_latency_hiding(&wm_low) ||
                     (rdev->disp_priority == 2)) {
                         DRM_DEBUG_KMS("force priority to high\n");
                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
@@ -1895,6 +2185,10 @@ static void dce6_program_watermarks(struct radeon_device *rdev,
         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
 
+        /* save values for DPM */
+        radeon_crtc->line_time = line_time;
+        radeon_crtc->wm_high = latency_watermark_a;
+        radeon_crtc->wm_low = latency_watermark_b;
 }
 
 void dce6_bandwidth_update(struct radeon_device *rdev)
@@ -3535,8 +3829,8 @@ static void si_mc_program(struct radeon_device *rdev)
         }
 }
 
-static void si_vram_gtt_location(struct radeon_device *rdev,
-                                 struct radeon_mc *mc)
+void si_vram_gtt_location(struct radeon_device *rdev,
+                          struct radeon_mc *mc)
 {
         if (mc->mc_vram_size > 0xFFC0000000ULL) {
                 /* leave room for at least 1024M GTT */
@@ -4282,6 +4576,450 @@ void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 }
 
 /*
+ *  Power and clock gating
+ */
+static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
+                        break;
+                udelay(1);
+        }
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
+                        break;
+                udelay(1);
+        }
+}
+
+static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
+                                         bool enable)
+{
+        u32 tmp = RREG32(CP_INT_CNTL_RING0);
+        u32 mask;
+        int i;
+
+        if (enable)
+                tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+        else
+                tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+        WREG32(CP_INT_CNTL_RING0, tmp);
+
+        if (!enable) {
+                /* read a gfx register */
+                tmp = RREG32(DB_DEPTH_INFO);
+
+                mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
+                for (i = 0; i < rdev->usec_timeout; i++) {
+                        if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
+                                break;
+                        udelay(1);
+                }
+        }
+}
+
+static void si_set_uvd_dcm(struct radeon_device *rdev,
+                           bool sw_mode)
+{
+        u32 tmp, tmp2;
+
+        tmp = RREG32(UVD_CGC_CTRL);
+        tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
+        tmp |= DCM | CG_DT(1) | CLK_OD(4);
+
+        if (sw_mode) {
+                tmp &= ~0x7ffff800;
+                tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
+        } else {
+                tmp |= 0x7ffff800;
+                tmp2 = 0;
+        }
+
+        WREG32(UVD_CGC_CTRL, tmp);
+        WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
+}
+
+static void si_init_uvd_internal_cg(struct radeon_device *rdev)
+{
+        bool hw_mode = true;
+
+        if (hw_mode) {
+                si_set_uvd_dcm(rdev, false);
+        } else {
+                u32 tmp = RREG32(UVD_CGC_CTRL);
+                tmp &= ~DCM;
+                WREG32(UVD_CGC_CTRL, tmp);
+        }
+}
+
+static u32 si_halt_rlc(struct radeon_device *rdev)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(RLC_CNTL);
+
+        if (data & RLC_ENABLE) {
+                data &= ~RLC_ENABLE;
+                WREG32(RLC_CNTL, data);
+
+                si_wait_for_rlc_serdes(rdev);
+        }
+
+        return orig;
+}
+
+static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
+{
+        u32 tmp;
+
+        tmp = RREG32(RLC_CNTL);
+        if (tmp != rlc)
+                WREG32(RLC_CNTL, rlc);
+}
+
+static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(DMA_PG);
+        if (enable)
+                data |= PG_CNTL_ENABLE;
+        else
+                data &= ~PG_CNTL_ENABLE;
+        if (orig != data)
+                WREG32(DMA_PG, data);
+}
+
+static void si_init_dma_pg(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        WREG32(DMA_PGFSM_WRITE,  0x00002000);
+        WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
+
+        for (tmp = 0; tmp < 5; tmp++)
+                WREG32(DMA_PGFSM_WRITE, 0);
+}
+
+static void si_enable_gfx_cgpg(struct radeon_device *rdev,
+                               bool enable)
+{
+        u32 tmp;
+
+        if (enable) {
+                tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
+                WREG32(RLC_TTOP_D, tmp);
+
+                tmp = RREG32(RLC_PG_CNTL);
+                tmp |= GFX_PG_ENABLE;
+                WREG32(RLC_PG_CNTL, tmp);
+
+                tmp = RREG32(RLC_AUTO_PG_CTRL);
+                tmp |= AUTO_PG_EN;
+                WREG32(RLC_AUTO_PG_CTRL, tmp);
+        } else {
+                tmp = RREG32(RLC_AUTO_PG_CTRL);
+                tmp &= ~AUTO_PG_EN;
+                WREG32(RLC_AUTO_PG_CTRL, tmp);
+
+                tmp = RREG32(DB_RENDER_CONTROL);
+        }
+}
+
+static void si_init_gfx_cgpg(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
+
+        tmp = RREG32(RLC_PG_CNTL);
+        tmp |= GFX_PG_SRC;
+        WREG32(RLC_PG_CNTL, tmp);
+
+        WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
+
+        tmp = RREG32(RLC_AUTO_PG_CTRL);
+
+        tmp &= ~GRBM_REG_SGIT_MASK;
+        tmp |= GRBM_REG_SGIT(0x700);
+        tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
+        WREG32(RLC_AUTO_PG_CTRL, tmp);
+}
+
+static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
+{
+        u32 mask = 0, tmp, tmp1;
+        int i;
+
+        si_select_se_sh(rdev, se, sh);
+        tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
+        tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
+        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
+
+        tmp &= 0xffff0000;
+
+        tmp |= tmp1;
+        tmp >>= 16;
+
+        for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
+                mask <<= 1;
+                mask |= 1;
+        }
+
+        return (~tmp) & mask;
+}
+
+static void si_init_ao_cu_mask(struct radeon_device *rdev)
+{
+        u32 i, j, k, active_cu_number = 0;
+        u32 mask, counter, cu_bitmap;
+        u32 tmp = 0;
+
+        for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
+                for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
+                        mask = 1;
+                        cu_bitmap = 0;
+                        counter  = 0;
+                        for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
+                                if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
+                                        if (counter < 2)
+                                                cu_bitmap |= mask;
+                                        counter++;
+                                }
+                                mask <<= 1;
+                        }
+
+                        active_cu_number += counter;
+                        tmp |= (cu_bitmap << (i * 16 + j * 8));
+                }
+        }
+
+        WREG32(RLC_PG_AO_CU_MASK, tmp);
+
+        tmp = RREG32(RLC_MAX_PG_CU);
+        tmp &= ~MAX_PU_CU_MASK;
+        tmp |= MAX_PU_CU(active_cu_number);
+        WREG32(RLC_MAX_PG_CU, tmp);
+}
+
+static void si_enable_cgcg(struct radeon_device *rdev,
+                           bool enable)
+{
+        u32 data, orig, tmp;
+
+        orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
+
+        si_enable_gui_idle_interrupt(rdev, enable);
+
+        if (enable) {
+                WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
+
+                tmp = si_halt_rlc(rdev);
+
+                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+                WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
+
+                si_wait_for_rlc_serdes(rdev);
+
+                si_update_rlc(rdev, tmp);
+
+                WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
+
+                data |= CGCG_EN | CGLS_EN;
+        } else {
+                RREG32(CB_CGTT_SCLK_CTRL);
+                RREG32(CB_CGTT_SCLK_CTRL);
+                RREG32(CB_CGTT_SCLK_CTRL);
+                RREG32(CB_CGTT_SCLK_CTRL);
+
+                data &= ~(CGCG_EN | CGLS_EN);
+        }
+
+        if (orig != data)
+                WREG32(RLC_CGCG_CGLS_CTRL, data);
+}
+
+static void si_enable_mgcg(struct radeon_device *rdev,
+                           bool enable)
+{
+        u32 data, orig, tmp = 0;
+
+        if (enable) {
+                orig = data = RREG32(CGTS_SM_CTRL_REG);
+                data = 0x96940200;
+                if (orig != data)
+                        WREG32(CGTS_SM_CTRL_REG, data);
+
+                orig = data = RREG32(CP_MEM_SLP_CNTL);
+                data |= CP_MEM_LS_EN;
+                if (orig != data)
+                        WREG32(CP_MEM_SLP_CNTL, data);
+
+                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
+                data &= 0xffffffc0;
+                if (orig != data)
+                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
+
+                tmp = si_halt_rlc(rdev);
+
+                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+                WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
+
+                si_update_rlc(rdev, tmp);
+        } else {
+                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
+                data |= 0x00000003;
+                if (orig != data)
+                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
+
+                data = RREG32(CP_MEM_SLP_CNTL);
+                if (data & CP_MEM_LS_EN) {
+                        data &= ~CP_MEM_LS_EN;
+                        WREG32(CP_MEM_SLP_CNTL, data);
+                }
+                orig = data = RREG32(CGTS_SM_CTRL_REG);
+                data |= LS_OVERRIDE | OVERRIDE;
+                if (orig != data)
+                        WREG32(CGTS_SM_CTRL_REG, data);
+
+                tmp = si_halt_rlc(rdev);
+
+                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+                WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
+
+                si_update_rlc(rdev, tmp);
+        }
+}
+
+static void si_enable_uvd_mgcg(struct radeon_device *rdev,
+                               bool enable)
+{
+        u32 orig, data, tmp;
+
+        if (enable) {
+                tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
+                tmp |= 0x3fff;
+                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
+
+                orig = data = RREG32(UVD_CGC_CTRL);
+                data |= DCM;
+                if (orig != data)
+                        WREG32(UVD_CGC_CTRL, data);
+
+                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
+                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
+        } else {
+                tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
+                tmp &= ~0x3fff;
+                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
+
+                orig = data = RREG32(UVD_CGC_CTRL);
+                data &= ~DCM;
+                if (orig != data)
+                        WREG32(UVD_CGC_CTRL, data);
+
+                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
+                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
+        }
+}
+
+static const u32 mc_cg_registers[] =
+{
+        MC_HUB_MISC_HUB_CG,
+        MC_HUB_MISC_SIP_CG,
+        MC_HUB_MISC_VM_CG,
+        MC_XPB_CLK_GAT,
+        ATC_MISC_CG,
+        MC_CITF_MISC_WR_CG,
+        MC_CITF_MISC_RD_CG,
+        MC_CITF_MISC_VM_CG,
+        VM_L2_CG,
+};
+
+static void si_enable_mc_ls(struct radeon_device *rdev,
+                            bool enable)
+{
+        int i;
+        u32 orig, data;
+
+        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
+                orig = data = RREG32(mc_cg_registers[i]);
+                if (enable)
+                        data |= MC_LS_ENABLE;
+                else
+                        data &= ~MC_LS_ENABLE;
+                if (data != orig)
+                        WREG32(mc_cg_registers[i], data);
+        }
+}
+
+
+static void si_init_cg(struct radeon_device *rdev)
+{
+        bool has_uvd = true;
+
+        si_enable_mgcg(rdev, true);
+        si_enable_cgcg(rdev, true);
+        /* disable MC LS on Tahiti */
+        if (rdev->family == CHIP_TAHITI)
+                si_enable_mc_ls(rdev, false);
+        if (has_uvd) {
+                si_enable_uvd_mgcg(rdev, true);
+                si_init_uvd_internal_cg(rdev);
+        }
+}
+
+static void si_fini_cg(struct radeon_device *rdev)
+{
+        bool has_uvd = true;
+
+        if (has_uvd)
+                si_enable_uvd_mgcg(rdev, false);
+        si_enable_cgcg(rdev, false);
+        si_enable_mgcg(rdev, false);
+}
+
+static void si_init_pg(struct radeon_device *rdev)
+{
+        bool has_pg = false;
+
+        /* only cape verde supports PG */
+        if (rdev->family == CHIP_VERDE)
+                has_pg = true;
+
+        if (has_pg) {
+                si_init_ao_cu_mask(rdev);
+                si_init_dma_pg(rdev);
+                si_enable_dma_pg(rdev, true);
+                si_init_gfx_cgpg(rdev);
+                si_enable_gfx_cgpg(rdev, true);
+        } else {
+                WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
+                WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
+        }
+}
+
+static void si_fini_pg(struct radeon_device *rdev)
+{
+        bool has_pg = false;
+
+        /* only cape verde supports PG */
+        if (rdev->family == CHIP_VERDE)
+                has_pg = true;
+
+        if (has_pg) {
+                si_enable_dma_pg(rdev, false);
+                si_enable_gfx_cgpg(rdev, false);
+        }
+}
+
+/*
  * RLC
  */
 void si_rlc_fini(struct radeon_device *rdev)
@@ -4313,8 +5051,15 @@ void si_rlc_fini(struct radeon_device *rdev)
         }
 }
 
+#define RLC_CLEAR_STATE_END_MARKER          0x00000001
+
 int si_rlc_init(struct radeon_device *rdev)
 {
+        volatile u32 *dst_ptr;
+        u32 dws, data, i, j, k, reg_num;
+        u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
+        u64 reg_list_mc_addr;
+        const struct cs_section_def *cs_data = si_cs_data;
         int r;
 
         /* save restore block */
@@ -4335,18 +5080,44 @@ int si_rlc_init(struct radeon_device *rdev)
         }
         r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
                           &rdev->rlc.save_restore_gpu_addr);
-        radeon_bo_unreserve(rdev->rlc.save_restore_obj);
         if (r) {
+                radeon_bo_unreserve(rdev->rlc.save_restore_obj);
                 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
                 si_rlc_fini(rdev);
                 return r;
         }
 
+        if (rdev->family == CHIP_VERDE) {
+                r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
+                if (r) {
+                        dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
+                        si_rlc_fini(rdev);
+                return r;
+                }
+                /* write the sr buffer */
+                dst_ptr = rdev->rlc.sr_ptr;
+                for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) {
+                        dst_ptr[i] = verde_rlc_save_restore_register_list[i];
+                }
+                radeon_bo_kunmap(rdev->rlc.save_restore_obj);
+        }
+        radeon_bo_unreserve(rdev->rlc.save_restore_obj);
+
         /* clear state block */
+        reg_list_num = 0;
+        dws = 0;
+        for (i = 0; cs_data[i].section != NULL; i++) {
+                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
+                        reg_list_num++;
+                        dws += cs_data[i].section[j].reg_count;
+                }
+        }
+        reg_list_blk_index = (3 * reg_list_num + 2);
+        dws += reg_list_blk_index;
+
         if (rdev->rlc.clear_state_obj == NULL) {
-                r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
-                                     RADEON_GEM_DOMAIN_VRAM, NULL,
-                                     &rdev->rlc.clear_state_obj);
+                r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
+                                     RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
                 if (r) {
                         dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
                         si_rlc_fini(rdev);
@@ -4360,24 +5131,113 @@ int si_rlc_init(struct radeon_device *rdev)
         }
         r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
                           &rdev->rlc.clear_state_gpu_addr);
-        radeon_bo_unreserve(rdev->rlc.clear_state_obj);
         if (r) {
+
+                radeon_bo_unreserve(rdev->rlc.clear_state_obj);
                 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
                 si_rlc_fini(rdev);
                 return r;
         }
+        r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
+        if (r) {
+                dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
+                si_rlc_fini(rdev);
+                return r;
+        }
+        /* set up the cs buffer */
+        dst_ptr = rdev->rlc.cs_ptr;
+        reg_list_hdr_blk_index = 0;
+        reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
+        data = upper_32_bits(reg_list_mc_addr);
+        dst_ptr[reg_list_hdr_blk_index] = data;
+        reg_list_hdr_blk_index++;
+        for (i = 0; cs_data[i].section != NULL; i++) {
+                for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
+                        reg_num = cs_data[i].section[j].reg_count;
+                        data = reg_list_mc_addr & 0xffffffff;
+                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        reg_list_hdr_blk_index++;
+
+                        data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
+                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        reg_list_hdr_blk_index++;
+
+                        data = 0x08000000 | (reg_num * 4);
+                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        reg_list_hdr_blk_index++;
+
+                        for (k = 0; k < reg_num; k++) {
+                                data = cs_data[i].section[j].extent[k];
+                                dst_ptr[reg_list_blk_index + k] = data;
+                        }
+                        reg_list_mc_addr += reg_num * 4;
+                        reg_list_blk_index += reg_num;
+                }
+        }
+        dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
+
+        radeon_bo_kunmap(rdev->rlc.clear_state_obj);
+        radeon_bo_unreserve(rdev->rlc.clear_state_obj);
 
         return 0;
 }
 
+static void si_rlc_reset(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(GRBM_SOFT_RESET);
+
+        tmp |= SOFT_RESET_RLC;
+        WREG32(GRBM_SOFT_RESET, tmp);
+        udelay(50);
+        tmp &= ~SOFT_RESET_RLC;
+        WREG32(GRBM_SOFT_RESET, tmp);
+        udelay(50);
+}
+
 static void si_rlc_stop(struct radeon_device *rdev)
 {
         WREG32(RLC_CNTL, 0);
+
+        si_enable_gui_idle_interrupt(rdev, false);
+
+        si_wait_for_rlc_serdes(rdev);
 }
 
 static void si_rlc_start(struct radeon_device *rdev)
 {
         WREG32(RLC_CNTL, RLC_ENABLE);
+
+        si_enable_gui_idle_interrupt(rdev, true);
+
+        udelay(50);
+}
+
+static bool si_lbpw_supported(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        /* Enable LBPW only for DDR3 */
+        tmp = RREG32(MC_SEQ_MISC0);
+        if ((tmp & 0xF0000000) == 0xB0000000)
+                return true;
+        return false;
+}
+
+static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
+{
+        u32 tmp;
+
+        tmp = RREG32(RLC_LB_CNTL);
+        if (enable)
+                tmp |= LOAD_BALANCE_ENABLE;
+        else
+                tmp &= ~LOAD_BALANCE_ENABLE;
+        WREG32(RLC_LB_CNTL, tmp);
+
+        if (!enable) {
+                si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
+                WREG32(SPI_LB_CU_MASK, 0x00ff);
+        }
 }
 
 static int si_rlc_resume(struct radeon_device *rdev)
@@ -4390,14 +5250,18 @@ static int si_rlc_resume(struct radeon_device *rdev)
 
         si_rlc_stop(rdev);
 
+        si_rlc_reset(rdev);
+
+        si_init_pg(rdev);
+
+        si_init_cg(rdev);
+
         WREG32(RLC_RL_BASE, 0);
         WREG32(RLC_RL_SIZE, 0);
         WREG32(RLC_LB_CNTL, 0);
         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
         WREG32(RLC_LB_CNTR_INIT, 0);
-
-        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
-        WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
+        WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
 
         WREG32(RLC_MC_CNTL, 0);
         WREG32(RLC_UCODE_CNTL, 0);
@@ -4409,6 +5273,8 @@ static int si_rlc_resume(struct radeon_device *rdev)
         }
         WREG32(RLC_UCODE_ADDR, 0);
 
+        si_enable_lbpw(rdev, si_lbpw_supported(rdev));
+
         si_rlc_start(rdev);
 
         return 0;
@@ -4578,6 +5444,7 @@ int si_irq_set(struct radeon_device *rdev)
         u32 grbm_int_cntl = 0;
         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
         u32 dma_cntl, dma_cntl1;
+        u32 thermal_int = 0;
 
         if (!rdev->irq.installed) {
                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -4603,6 +5470,9 @@ int si_irq_set(struct radeon_device *rdev)
         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
 
+        thermal_int = RREG32(CG_THERMAL_INT) &
+                ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
+
         /* enable CP interrupts on all rings */
         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
                 DRM_DEBUG("si_irq_set: sw int gfx\n");
@@ -4689,6 +5559,11 @@ int si_irq_set(struct radeon_device *rdev)
 
         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
 
+        if (rdev->irq.dpm_thermal) {
+                DRM_DEBUG("dpm thermal\n");
+                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
+        }
+
         if (rdev->num_crtc >= 2) {
                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
@@ -4724,6 +5599,8 @@ int si_irq_set(struct radeon_device *rdev)
                 WREG32(DC_HPD6_INT_CONTROL, hpd6);
         }
 
+        WREG32(CG_THERMAL_INT, thermal_int);
+
         return 0;
 }
 
@@ -4888,6 +5765,7 @@ int si_irq_process(struct radeon_device *rdev)
         u32 src_id, src_data, ring_id;
         u32 ring_index;
         bool queue_hotplug = false;
+        bool queue_thermal = false;
 
         if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
@@ -5158,6 +6036,16 @@ restart_ih:
                         DRM_DEBUG("IH: DMA trap\n");
                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
                         break;
+                case 230: /* thermal low to high */
+                        DRM_DEBUG("IH: thermal low to high\n");
+                        rdev->pm.dpm.thermal.high_to_low = false;
+                        queue_thermal = true;
+                        break;
+                case 231: /* thermal high to low */
+                        DRM_DEBUG("IH: thermal high to low\n");
+                        rdev->pm.dpm.thermal.high_to_low = true;
+                        queue_thermal = true;
+                        break;
                 case 233: /* GUI IDLE */
                         DRM_DEBUG("IH: GUI idle\n");
                         break;
@@ -5176,6 +6064,8 @@ restart_ih:
         }
         if (queue_hotplug)
                 schedule_work(&rdev->hotplug_work);
+        if (queue_thermal && rdev->pm.dpm_enabled)
+                schedule_work(&rdev->pm.dpm.thermal.work);
         rdev->ih.rptr = rptr;
         WREG32(IH_RB_RPTR, rdev->ih.rptr);
         atomic_set(&rdev->ih.lock, 0);
@@ -5270,6 +6160,11 @@ static int si_startup(struct radeon_device *rdev)
         struct radeon_ring *ring;
         int r;
 
+        /* enable pcie gen2/3 link */
+        si_pcie_gen3_enable(rdev);
+        /* enable aspm */
+        si_program_aspm(rdev);
+
         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
             !rdev->rlc_fw || !rdev->mc_fw) {
                 r = si_init_microcode(rdev);
@@ -5609,6 +6504,8 @@ void si_fini(struct radeon_device *rdev)
         cayman_dma_fini(rdev);
         si_irq_fini(rdev);
         si_rlc_fini(rdev);
+        si_fini_cg(rdev);
+        si_fini_pg(rdev);
         radeon_wb_fini(rdev);
         radeon_vm_manager_fini(rdev);
         radeon_ib_pool_fini(rdev);
@@ -5735,3 +6632,361 @@ int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
 
         return 0;
 }
+
+static void si_pcie_gen3_enable(struct radeon_device *rdev)
+{
+        struct pci_dev *root = rdev->pdev->bus->self;
+        int bridge_pos, gpu_pos;
+        u32 speed_cntl, mask, current_data_rate;
+        int ret, i;
+        u16 tmp16;
+
+        if (radeon_pcie_gen2 == 0)
+                return;
+
+        if (rdev->flags & RADEON_IS_IGP)
+                return;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
+        if (ret != 0)
+                return;
+
+        if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+                return;
+
+        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
+                LC_CURRENT_DATA_RATE_SHIFT;
+        if (mask & DRM_PCIE_SPEED_80) {
+                if (current_data_rate == 2) {
+                        DRM_INFO("PCIE gen 3 link speeds already enabled\n");
+                        return;
+                }
+                DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
+        } else if (mask & DRM_PCIE_SPEED_50) {
+                if (current_data_rate == 1) {
+                        DRM_INFO("PCIE gen 2 link speeds already enabled\n");
+                        return;
+                }
+                DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
+        }
+
+        bridge_pos = pci_pcie_cap(root);
+        if (!bridge_pos)
+                return;
+
+        gpu_pos = pci_pcie_cap(rdev->pdev);
+        if (!gpu_pos)
+                return;
+
+        if (mask & DRM_PCIE_SPEED_80) {
+                /* re-try equalization if gen3 is not already enabled */
+                if (current_data_rate != 2) {
+                        u16 bridge_cfg, gpu_cfg;
+                        u16 bridge_cfg2, gpu_cfg2;
+                        u32 max_lw, current_lw, tmp;
+
+                        pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
+                        pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
+
+                        tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
+                        pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
+
+                        tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
+                        pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
+
+                        tmp = RREG32_PCIE(PCIE_LC_STATUS1);
+                        max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
+                        current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
+
+                        if (current_lw < max_lw) {
+                                tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
+                                if (tmp & LC_RENEGOTIATION_SUPPORT) {
+                                        tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
+                                        tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
+                                        tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
+                                        WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
+                                }
+                        }
+
+                        for (i = 0; i < 10; i++) {
+                                /* check status */
+                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
+                                if (tmp16 & PCI_EXP_DEVSTA_TRPND)
+                                        break;
+
+                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
+                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
+
+                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
+                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
+
+                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
+                                tmp |= LC_SET_QUIESCE;
+                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
+
+                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
+                                tmp |= LC_REDO_EQ;
+                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
+
+                                mdelay(100);
+
+                                /* linkctl */
+                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
+                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
+                                tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
+                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
+
+                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
+                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
+                                tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
+                                pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
+
+                                /* linkctl2 */
+                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
+                                tmp16 &= ~((1 << 4) | (7 << 9));
+                                tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
+                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
+
+                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
+                                tmp16 &= ~((1 << 4) | (7 << 9));
+                                tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
+                                pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
+
+                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
+                                tmp &= ~LC_SET_QUIESCE;
+                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
+                        }
+                }
+        }
+
+        /* set the link speed */
+        speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
+        speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
+        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
+        tmp16 &= ~0xf;
+        if (mask & DRM_PCIE_SPEED_80)
+                tmp16 |= 3; /* gen3 */
+        else if (mask & DRM_PCIE_SPEED_50)
+                tmp16 |= 2; /* gen2 */
+        else
+                tmp16 |= 1; /* gen1 */
+        pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
+
+        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+        speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
+        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+                if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
+                        break;
+                udelay(1);
+        }
+}
+
+static void si_program_aspm(struct radeon_device *rdev)
+{
+        u32 data, orig;
+        bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
+        bool disable_clkreq = false;
+
+        if (!(rdev->flags & RADEON_IS_PCIE))
+                return;
+
+        orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
+        data &= ~LC_XMIT_N_FTS_MASK;
+        data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
+        if (orig != data)
+                WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
+
+        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
+        data |= LC_GO_TO_RECOVERY;
+        if (orig != data)
+                WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
+
+        orig = data = RREG32_PCIE(PCIE_P_CNTL);
+        data |= P_IGNORE_EDB_ERR;
+        if (orig != data)
+                WREG32_PCIE(PCIE_P_CNTL, data);
+
+        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+        data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
+        data |= LC_PMI_TO_L1_DIS;
+        if (!disable_l0s)
+                data |= LC_L0S_INACTIVITY(7);
+
+        if (!disable_l1) {
+                data |= LC_L1_INACTIVITY(7);
+                data &= ~LC_PMI_TO_L1_DIS;
+                if (orig != data)
+                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
+
+                if (!disable_plloff_in_l1) {
+                        bool clk_req_support;
+
+                        orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
+                        if (orig != data)
+                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
+
+                        orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
+                        if (orig != data)
+                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
+
+                        orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
+                        if (orig != data)
+                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
+
+                        orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
+                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
+                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
+                        if (orig != data)
+                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
+
+                        if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
+                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
+                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
+
+                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
+                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
+
+                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
+                                data &= ~PLL_RAMP_UP_TIME_2_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
+
+                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
+                                data &= ~PLL_RAMP_UP_TIME_3_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
+
+                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
+                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
+
+                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
+                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
+
+                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
+                                data &= ~PLL_RAMP_UP_TIME_2_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
+
+                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
+                                data &= ~PLL_RAMP_UP_TIME_3_MASK;
+                                if (orig != data)
+                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
+                        }
+                        orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
+                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
+                        data |= LC_DYN_LANES_PWR_STATE(3);
+                        if (orig != data)
+                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
+
+                        orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
+                        data &= ~LS2_EXIT_TIME_MASK;
+                        if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
+                                data |= LS2_EXIT_TIME(5);
+                        if (orig != data)
+                                WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
+
+                        orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
+                        data &= ~LS2_EXIT_TIME_MASK;
+                        if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
+                                data |= LS2_EXIT_TIME(5);
+                        if (orig != data)
+                                WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
+
+                        if (!disable_clkreq) {
+                                struct pci_dev *root = rdev->pdev->bus->self;
+                                u32 lnkcap;
+
+                                clk_req_support = false;
+                                pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
+                                if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
+                                        clk_req_support = true;
+                        } else {
+                                clk_req_support = false;
+                        }
+
+                        if (clk_req_support) {
+                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
+                                data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
+                                if (orig != data)
+                                        WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
+
+                                orig = data = RREG32(THM_CLK_CNTL);
+                                data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
+                                data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
+                                if (orig != data)
+                                        WREG32(THM_CLK_CNTL, data);
+
+                                orig = data = RREG32(MISC_CLK_CNTL);
+                                data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
+                                data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
+                                if (orig != data)
+                                        WREG32(MISC_CLK_CNTL, data);
+
+                                orig = data = RREG32(CG_CLKPIN_CNTL);
+                                data &= ~BCLK_AS_XCLK;
+                                if (orig != data)
+                                        WREG32(CG_CLKPIN_CNTL, data);
+
+                                orig = data = RREG32(CG_CLKPIN_CNTL_2);
+                                data &= ~FORCE_BIF_REFCLK_EN;
+                                if (orig != data)
+                                        WREG32(CG_CLKPIN_CNTL_2, data);
+
+                                orig = data = RREG32(MPLL_BYPASSCLK_SEL);
+                                data &= ~MPLL_CLKOUT_SEL_MASK;
+                                data |= MPLL_CLKOUT_SEL(4);
+                                if (orig != data)
+                                        WREG32(MPLL_BYPASSCLK_SEL, data);
+
+                                orig = data = RREG32(SPLL_CNTL_MODE);
+                                data &= ~SPLL_REFCLK_SEL_MASK;
+                                if (orig != data)
+                                        WREG32(SPLL_CNTL_MODE, data);
+                        }
+                }
+        } else {
+                if (orig != data)
+                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
+        }
+
+        orig = data = RREG32_PCIE(PCIE_CNTL2);
+        data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
+        if (orig != data)
+                WREG32_PCIE(PCIE_CNTL2, data);
+
+        if (!disable_l0s) {
+                data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
+                if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
+                        data = RREG32_PCIE(PCIE_LC_STATUS1);
+                        if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
+                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+                                data &= ~LC_L0S_INACTIVITY_MASK;
+                                if (orig != data)
+                                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
+                        }
+                }
+        }
+}
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
new file mode 100644
index 000000000000..a7e97cd05e96
--- /dev/null
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -0,0 +1,6407 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "sid.h"
+#include "r600_dpm.h"
+#include "si_dpm.h"
+#include "atom.h"
+#include <linux/math64.h>
+#include <linux/seq_file.h>
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define SMC_RAM_END                 0x20000
+
+#define DDR3_DRAM_ROWS              0x2000
+
+#define SCLK_MIN_DEEPSLEEP_FREQ     1350
+
+static const struct si_cac_config_reg cac_weights_tahiti[] =
+{
+        { 0x0, 0x0000ffff, 0, 0xc, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x101, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0xc, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x8fc, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x95, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x34e, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x1a1, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0xda, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x46, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x208, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0xe7, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x948, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x167, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x31, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x18e, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_tahiti[] =
+{
+        { 0x143, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x146, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x149, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0x149, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14c, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0x14c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x9e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x9e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x101, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x101, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8f, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x92, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x92, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x95, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x95, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x152, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x155, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x155, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x158, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x158, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x113, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x116, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x116, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x119, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+        { 0x119, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x122, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x122, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x125, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x125, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x128, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x128, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x12b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x12b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+
+};
+
+static const struct si_cac_config_reg cac_override_tahiti[] =
+{
+        { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_tahiti =
+{
+        ((1 << 16) | 27027),
+        6,
+        0,
+        4,
+        95,
+        {
+                0UL,
+                0UL,
+                4521550UL,
+                309631529UL,
+                -1270850L,
+                4513710L,
+                40
+        },
+        595000000UL,
+        12,
+        {
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+        },
+        true
+};
+
+static const struct si_dte_data dte_data_tahiti =
+{
+        { 1159409, 0, 0, 0, 0 },
+        { 777, 0, 0, 0, 0 },
+        2,
+        54000,
+        127000,
+        25,
+        2,
+        10,
+        13,
+        { 27, 31, 35, 39, 43, 47, 54, 61, 67, 74, 81, 88, 95, 0, 0, 0 },
+        { 240888759, 221057860, 235370597, 162287531, 158510299, 131423027, 116673180, 103067515, 87941937, 76209048, 68209175, 64090048, 58301890, 0, 0, 0 },
+        { 12024, 11189, 11451, 8411, 7939, 6666, 5681, 4905, 4241, 3720, 3354, 3122, 2890, 0, 0, 0 },
+        85,
+        false
+};
+
+static const struct si_dte_data dte_data_tahiti_le =
+{
+        { 0x1E8480, 0x7A1200, 0x2160EC0, 0x3938700, 0 },
+        { 0x7D, 0x7D, 0x4E4, 0xB00, 0 },
+        0x5,
+        0xAFC8,
+        0x64,
+        0x32,
+        1,
+        0,
+        0x10,
+        { 0x78, 0x7C, 0x82, 0x88, 0x8E, 0x94, 0x9A, 0xA0, 0xA6, 0xAC, 0xB0, 0xB4, 0xB8, 0xBC, 0xC0, 0xC4 },
+        { 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700 },
+        { 0x2AF8, 0x2AF8, 0x29BB, 0x27F9, 0x2637, 0x2475, 0x22B3, 0x20F1, 0x1F2F, 0x1D6D, 0x1734, 0x1414, 0x10F4, 0xDD4, 0xAB4, 0x794 },
+        85,
+        true
+};
+
+static const struct si_dte_data dte_data_tahiti_pro =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        45000,
+        100,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0x7D0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_new_zealand =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0 },
+        { 0x29B, 0x3E9, 0x537, 0x7D2, 0 },
+        0x5,
+        0xAFC8,
+        0x69,
+        0x32,
+        1,
+        0,
+        0x10,
+        { 0x82, 0xA0, 0xB4, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0xDAC, 0x1388, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685 },
+        85,
+        true
+};
+
+static const struct si_dte_data dte_data_aruba_pro =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        45000,
+        100,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0x1000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_malta =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        45000,
+        100,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+struct si_cac_config_reg cac_weights_pitcairn[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x8a, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x24d, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x19, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0xc11, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x7f3, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x403, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x367, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x4c9, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x45d, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x36d, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x534, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x5da, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x880, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0x201, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x9f, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x1f, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5de, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x7b, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x13, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0xf9, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x66, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x13, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x186, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_pitcairn[] =
+{
+        { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x113, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x152, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x146, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x9e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x9e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x116, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x116, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x155, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x155, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x92, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x92, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x149, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x149, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x101, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x101, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x119, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x119, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x158, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x158, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x95, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x95, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x14c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x122, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x122, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x125, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x125, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x128, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x128, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x12b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x12b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_override_pitcairn[] =
+{
+    { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_pitcairn =
+{
+        ((1 << 16) | 27027),
+        5,
+        0,
+        6,
+        100,
+        {
+                51600000UL,
+                1800000UL,
+                7194395UL,
+                309631529UL,
+                -1270850L,
+                4513710L,
+                100
+        },
+        117830498UL,
+        12,
+        {
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+        },
+        true
+};
+
+static const struct si_dte_data dte_data_pitcairn =
+{
+        { 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0 },
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        0,
+        false
+};
+
+static const struct si_dte_data dte_data_curacao_xt =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        45000,
+        100,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0x1D17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_curacao_pro =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        45000,
+        100,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0x1D17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_neptune_xt =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        45000,
+        100,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0x3A2F, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_cac_config_reg cac_weights_chelsea_pro[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x2BD, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_chelsea_xt[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x30A, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_heathrow[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x362, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_cape_verde_pro[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x315, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_cape_verde[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x3BA, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_cape_verde[] =
+{
+        { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x113, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x152, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x146, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_override_cape_verde[] =
+{
+    { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_cape_verde =
+{
+        ((1 << 16) | 0x6993),
+        5,
+        0,
+        7,
+        105,
+        {
+                0UL,
+                0UL,
+                7194395UL,
+                309631529UL,
+                -1270850L,
+                4513710L,
+                100
+        },
+        117830498UL,
+        12,
+        {
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+        },
+        true
+};
+
+static const struct si_dte_data dte_data_cape_verde =
+{
+        { 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0 },
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        0,
+        false
+};
+
+static const struct si_dte_data dte_data_venus_xtx =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x71C, 0xAAB, 0xE39, 0x11C7, 0x0 },
+        5,
+        55000,
+        0x69,
+        0xA,
+        1,
+        0,
+        0x3,
+        { 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        { 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        { 0xD6D8, 0x88B8, 0x1555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_venus_xt =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0xBDA, 0x11C7, 0x17B4, 0x1DA1, 0x0 },
+        5,
+        55000,
+        0x69,
+        0xA,
+        1,
+        0,
+        0x3,
+        { 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        { 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        { 0xAFC8, 0x88B8, 0x238E, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_venus_pro =
+{
+        {  0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x11C7, 0x1AAB, 0x238E, 0x2C72, 0x0 },
+        5,
+        55000,
+        0x69,
+        0xA,
+        1,
+        0,
+        0x3,
+        { 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        { 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        { 0x88B8, 0x88B8, 0x3555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+struct si_cac_config_reg cac_weights_oland[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x3BA, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_mars_pro[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_mars_xt[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x60, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_oland_pro[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x90, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_oland_xt[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x120, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_oland[] =
+{
+        { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_mars_pro[] =
+{
+        { 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+        { 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+        { 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_override_oland[] =
+{
+        { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_oland =
+{
+        ((1 << 16) | 0x6993),
+        5,
+        0,
+        7,
+        105,
+        {
+                0UL,
+                0UL,
+                7194395UL,
+                309631529UL,
+                -1270850L,
+                4513710L,
+                100
+        },
+        117830498UL,
+        12,
+        {
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+        },
+        true
+};
+
+static const struct si_powertune_data powertune_data_mars_pro =
+{
+        ((1 << 16) | 0x6993),
+        5,
+        0,
+        7,
+        105,
+        {
+                0UL,
+                0UL,
+                7194395UL,
+                309631529UL,
+                -1270850L,
+                4513710L,
+                100
+        },
+        117830498UL,
+        12,
+        {
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+        },
+        true
+};
+
+static const struct si_dte_data dte_data_oland =
+{
+        { 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0 },
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+        0,
+        false
+};
+
+static const struct si_dte_data dte_data_mars_pro =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        55000,
+        105,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0xF627, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+static const struct si_dte_data dte_data_sun_xt =
+{
+        { 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+        { 0x0, 0x0, 0x0, 0x0, 0x0 },
+        5,
+        55000,
+        105,
+        0xA,
+        1,
+        0,
+        0x10,
+        { 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+        { 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+        { 0xD555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+        90,
+        true
+};
+
+
+static const struct si_cac_config_reg cac_weights_hainan[] =
+{
+        { 0x0, 0x0000ffff, 0, 0x2d9, SISLANDS_CACCONFIG_CGIND },
+        { 0x0, 0xffff0000, 16, 0x22b, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0x0000ffff, 0, 0x21c, SISLANDS_CACCONFIG_CGIND },
+        { 0x1, 0xffff0000, 16, 0x1dc, SISLANDS_CACCONFIG_CGIND },
+        { 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0x0000ffff, 0, 0x24e, SISLANDS_CACCONFIG_CGIND },
+        { 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0x0000ffff, 0, 0x35e, SISLANDS_CACCONFIG_CGIND },
+        { 0x5, 0xffff0000, 16, 0x1143, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0x0000ffff, 0, 0xe17, SISLANDS_CACCONFIG_CGIND },
+        { 0x6, 0xffff0000, 16, 0x441, SISLANDS_CACCONFIG_CGIND },
+        { 0x18f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0x0000ffff, 0, 0x28b, SISLANDS_CACCONFIG_CGIND },
+        { 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x8, 0xffff0000, 16, 0xabe, SISLANDS_CACCONFIG_CGIND },
+        { 0x9, 0x0000ffff, 0, 0xf11, SISLANDS_CACCONFIG_CGIND },
+        { 0xa, 0x0000ffff, 0, 0x907, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0x0000ffff, 0, 0xb45, SISLANDS_CACCONFIG_CGIND },
+        { 0xb, 0xffff0000, 16, 0xd1e, SISLANDS_CACCONFIG_CGIND },
+        { 0xc, 0x0000ffff, 0, 0xa2c, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0x0000ffff, 0, 0x62, SISLANDS_CACCONFIG_CGIND },
+        { 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0xe, 0x0000ffff, 0, 0x1f3, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0x0000ffff, 0, 0x42, SISLANDS_CACCONFIG_CGIND },
+        { 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0x0000ffff, 0, 0x709, SISLANDS_CACCONFIG_CGIND },
+        { 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x12, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x13, 0xffff0000, 16, 0x3a, SISLANDS_CACCONFIG_CGIND },
+        { 0x14, 0x0000ffff, 0, 0x357, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0x0000ffff, 0, 0x9f, SISLANDS_CACCONFIG_CGIND },
+        { 0x15, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0x0000ffff, 0, 0x314, SISLANDS_CACCONFIG_CGIND },
+        { 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x17, 0x0000ffff, 0, 0x6d, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+        { 0x6d, 0x0000ffff, 0, 0x1b9, SISLANDS_CACCONFIG_CGIND },
+        { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_hainan =
+{
+        ((1 << 16) | 0x6993),
+        5,
+        0,
+        9,
+        105,
+        {
+                0UL,
+                0UL,
+                7194395UL,
+                309631529UL,
+                -1270850L,
+                4513710L,
+                100
+        },
+        117830498UL,
+        12,
+        {
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+        },
+        true
+};
+
+struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
+struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
+struct ni_power_info *ni_get_pi(struct radeon_device *rdev);
+struct ni_ps *ni_get_ps(struct radeon_ps *rps);
+
+static int si_populate_voltage_value(struct radeon_device *rdev,
+                                     const struct atom_voltage_table *table,
+                                     u16 value, SISLANDS_SMC_VOLTAGE_VALUE *voltage);
+static int si_get_std_voltage_value(struct radeon_device *rdev,
+                                    SISLANDS_SMC_VOLTAGE_VALUE *voltage,
+                                    u16 *std_voltage);
+static int si_write_smc_soft_register(struct radeon_device *rdev,
+                                      u16 reg_offset, u32 value);
+static int si_convert_power_level_to_smc(struct radeon_device *rdev,
+                                         struct rv7xx_pl *pl,
+                                         SISLANDS_SMC_HW_PERFORMANCE_LEVEL *level);
+static int si_calculate_sclk_params(struct radeon_device *rdev,
+                                    u32 engine_clock,
+                                    SISLANDS_SMC_SCLK_VALUE *sclk);
+
+static struct si_power_info *si_get_pi(struct radeon_device *rdev)
+{
+        struct si_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+static void si_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
+                                                     u16 v, s32 t, u32 ileakage, u32 *leakage)
+{
+        s64 kt, kv, leakage_w, i_leakage, vddc;
+        s64 temperature, t_slope, t_intercept, av, bv, t_ref;
+
+        i_leakage = drm_int2fixp(ileakage / 100);
+        vddc = div64_s64(drm_int2fixp(v), 1000);
+        temperature = div64_s64(drm_int2fixp(t), 1000);
+
+        t_slope = div64_s64(drm_int2fixp(coeff->t_slope), 100000000);
+        t_intercept = div64_s64(drm_int2fixp(coeff->t_intercept), 100000000);
+        av = div64_s64(drm_int2fixp(coeff->av), 100000000);
+        bv = div64_s64(drm_int2fixp(coeff->bv), 100000000);
+        t_ref = drm_int2fixp(coeff->t_ref);
+
+        kt = drm_fixp_div(drm_fixp_exp(drm_fixp_mul(drm_fixp_mul(t_slope, vddc) + t_intercept, temperature)),
+                          drm_fixp_exp(drm_fixp_mul(drm_fixp_mul(t_slope, vddc) + t_intercept, t_ref)));
+        kv = drm_fixp_mul(av, drm_fixp_exp(drm_fixp_mul(bv, vddc)));
+
+        leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
+
+        *leakage = drm_fixp2int(leakage_w * 1000);
+}
+
+static void si_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
+                                             const struct ni_leakage_coeffients *coeff,
+                                             u16 v,
+                                             s32 t,
+                                             u32 i_leakage,
+                                             u32 *leakage)
+{
+        si_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
+}
+
+static void si_calculate_leakage_for_v_formula(const struct ni_leakage_coeffients *coeff,
+                                               const u32 fixed_kt, u16 v,
+                                               u32 ileakage, u32 *leakage)
+{
+        s64 kt, kv, leakage_w, i_leakage, vddc;
+
+        i_leakage = div64_s64(drm_int2fixp(ileakage), 100);
+        vddc = div64_s64(drm_int2fixp(v), 1000);
+
+        kt = div64_s64(drm_int2fixp(fixed_kt), 100000000);
+        kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 100000000),
+                          drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 100000000), vddc)));
+
+        leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
+
+        *leakage = drm_fixp2int(leakage_w * 1000);
+}
+
+static void si_calculate_leakage_for_v(struct radeon_device *rdev,
+                                       const struct ni_leakage_coeffients *coeff,
+                                       const u32 fixed_kt,
+                                       u16 v,
+                                       u32 i_leakage,
+                                       u32 *leakage)
+{
+        si_calculate_leakage_for_v_formula(coeff, fixed_kt, v, i_leakage, leakage);
+}
+
+
+static void si_update_dte_from_pl2(struct radeon_device *rdev,
+                                   struct si_dte_data *dte_data)
+{
+        u32 p_limit1 = rdev->pm.dpm.tdp_limit;
+        u32 p_limit2 = rdev->pm.dpm.near_tdp_limit;
+        u32 k = dte_data->k;
+        u32 t_max = dte_data->max_t;
+        u32 t_split[5] = { 10, 15, 20, 25, 30 };
+        u32 t_0 = dte_data->t0;
+        u32 i;
+
+        if (p_limit2 != 0 && p_limit2 <= p_limit1) {
+                dte_data->tdep_count = 3;
+
+                for (i = 0; i < k; i++) {
+                        dte_data->r[i] =
+                                (t_split[i] * (t_max - t_0/(u32)1000) * (1 << 14)) /
+                                (p_limit2  * (u32)100);
+                }
+
+                dte_data->tdep_r[1] = dte_data->r[4] * 2;
+
+                for (i = 2; i < SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE; i++) {
+                        dte_data->tdep_r[i] = dte_data->r[4];
+                }
+        } else {
+                DRM_ERROR("Invalid PL2! DTE will not be updated.\n");
+        }
+}
+
+static void si_initialize_powertune_defaults(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        bool update_dte_from_pl2 = false;
+
+        if (rdev->family == CHIP_TAHITI) {
+                si_pi->cac_weights = cac_weights_tahiti;
+                si_pi->lcac_config = lcac_tahiti;
+                si_pi->cac_override = cac_override_tahiti;
+                si_pi->powertune_data = &powertune_data_tahiti;
+                si_pi->dte_data = dte_data_tahiti;
+
+                switch (rdev->pdev->device) {
+                case 0x6798:
+                        si_pi->dte_data.enable_dte_by_default = true;
+                        break;
+                case 0x6799:
+                        si_pi->dte_data = dte_data_new_zealand;
+                        break;
+                case 0x6790:
+                case 0x6791:
+                case 0x6792:
+                case 0x679E:
+                        si_pi->dte_data = dte_data_aruba_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x679B:
+                        si_pi->dte_data = dte_data_malta;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x679A:
+                        si_pi->dte_data = dte_data_tahiti_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                default:
+                        if (si_pi->dte_data.enable_dte_by_default == true)
+                                DRM_ERROR("DTE is not enabled!\n");
+                        break;
+                }
+        } else if (rdev->family == CHIP_PITCAIRN) {
+                switch (rdev->pdev->device) {
+                case 0x6810:
+                case 0x6818:
+                        si_pi->cac_weights = cac_weights_pitcairn;
+                        si_pi->lcac_config = lcac_pitcairn;
+                        si_pi->cac_override = cac_override_pitcairn;
+                        si_pi->powertune_data = &powertune_data_pitcairn;
+                        si_pi->dte_data = dte_data_curacao_xt;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x6819:
+                case 0x6811:
+                        si_pi->cac_weights = cac_weights_pitcairn;
+                        si_pi->lcac_config = lcac_pitcairn;
+                        si_pi->cac_override = cac_override_pitcairn;
+                        si_pi->powertune_data = &powertune_data_pitcairn;
+                        si_pi->dte_data = dte_data_curacao_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x6800:
+                case 0x6806:
+                        si_pi->cac_weights = cac_weights_pitcairn;
+                        si_pi->lcac_config = lcac_pitcairn;
+                        si_pi->cac_override = cac_override_pitcairn;
+                        si_pi->powertune_data = &powertune_data_pitcairn;
+                        si_pi->dte_data = dte_data_neptune_xt;
+                        update_dte_from_pl2 = true;
+                        break;
+                default:
+                        si_pi->cac_weights = cac_weights_pitcairn;
+                        si_pi->lcac_config = lcac_pitcairn;
+                        si_pi->cac_override = cac_override_pitcairn;
+                        si_pi->powertune_data = &powertune_data_pitcairn;
+                        si_pi->dte_data = dte_data_pitcairn;
+                }
+        } else if (rdev->family == CHIP_VERDE) {
+                si_pi->lcac_config = lcac_cape_verde;
+                si_pi->cac_override = cac_override_cape_verde;
+                si_pi->powertune_data = &powertune_data_cape_verde;
+
+                switch (rdev->pdev->device) {
+                case 0x683B:
+                case 0x683F:
+                case 0x6829:
+                        si_pi->cac_weights = cac_weights_cape_verde_pro;
+                        si_pi->dte_data = dte_data_cape_verde;
+                        break;
+                case 0x6825:
+                case 0x6827:
+                        si_pi->cac_weights = cac_weights_heathrow;
+                        si_pi->dte_data = dte_data_cape_verde;
+                        break;
+                case 0x6824:
+                case 0x682D:
+                        si_pi->cac_weights = cac_weights_chelsea_xt;
+                        si_pi->dte_data = dte_data_cape_verde;
+                        break;
+                case 0x682F:
+                        si_pi->cac_weights = cac_weights_chelsea_pro;
+                        si_pi->dte_data = dte_data_cape_verde;
+                        break;
+                case 0x6820:
+                        si_pi->cac_weights = cac_weights_heathrow;
+                        si_pi->dte_data = dte_data_venus_xtx;
+                        break;
+                case 0x6821:
+                        si_pi->cac_weights = cac_weights_heathrow;
+                        si_pi->dte_data = dte_data_venus_xt;
+                        break;
+                case 0x6823:
+                        si_pi->cac_weights = cac_weights_chelsea_pro;
+                        si_pi->dte_data = dte_data_venus_pro;
+                        break;
+                case 0x682B:
+                        si_pi->cac_weights = cac_weights_chelsea_pro;
+                        si_pi->dte_data = dte_data_venus_pro;
+                        break;
+                default:
+                        si_pi->cac_weights = cac_weights_cape_verde;
+                        si_pi->dte_data = dte_data_cape_verde;
+                        break;
+                }
+        } else if (rdev->family == CHIP_OLAND) {
+                switch (rdev->pdev->device) {
+                case 0x6601:
+                case 0x6621:
+                case 0x6603:
+                        si_pi->cac_weights = cac_weights_mars_pro;
+                        si_pi->lcac_config = lcac_mars_pro;
+                        si_pi->cac_override = cac_override_oland;
+                        si_pi->powertune_data = &powertune_data_mars_pro;
+                        si_pi->dte_data = dte_data_mars_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x6600:
+                case 0x6606:
+                case 0x6620:
+                        si_pi->cac_weights = cac_weights_mars_xt;
+                        si_pi->lcac_config = lcac_mars_pro;
+                        si_pi->cac_override = cac_override_oland;
+                        si_pi->powertune_data = &powertune_data_mars_pro;
+                        si_pi->dte_data = dte_data_mars_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x6611:
+                        si_pi->cac_weights = cac_weights_oland_pro;
+                        si_pi->lcac_config = lcac_mars_pro;
+                        si_pi->cac_override = cac_override_oland;
+                        si_pi->powertune_data = &powertune_data_mars_pro;
+                        si_pi->dte_data = dte_data_mars_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                case 0x6610:
+                        si_pi->cac_weights = cac_weights_oland_xt;
+                        si_pi->lcac_config = lcac_mars_pro;
+                        si_pi->cac_override = cac_override_oland;
+                        si_pi->powertune_data = &powertune_data_mars_pro;
+                        si_pi->dte_data = dte_data_mars_pro;
+                        update_dte_from_pl2 = true;
+                        break;
+                default:
+                        si_pi->cac_weights = cac_weights_oland;
+                        si_pi->lcac_config = lcac_oland;
+                        si_pi->cac_override = cac_override_oland;
+                        si_pi->powertune_data = &powertune_data_oland;
+                        si_pi->dte_data = dte_data_oland;
+                        break;
+                }
+        } else if (rdev->family == CHIP_HAINAN) {
+                si_pi->cac_weights = cac_weights_hainan;
+                si_pi->lcac_config = lcac_oland;
+                si_pi->cac_override = cac_override_oland;
+                si_pi->powertune_data = &powertune_data_hainan;
+                si_pi->dte_data = dte_data_sun_xt;
+                update_dte_from_pl2 = true;
+        } else {
+                DRM_ERROR("Unknown SI asic revision, failed to initialize PowerTune!\n");
+                return;
+        }
+
+        ni_pi->enable_power_containment = false;
+        ni_pi->enable_cac = false;
+        ni_pi->enable_sq_ramping = false;
+        si_pi->enable_dte = false;
+
+        if (si_pi->powertune_data->enable_powertune_by_default) {
+                ni_pi->enable_power_containment= true;
+                ni_pi->enable_cac = true;
+                if (si_pi->dte_data.enable_dte_by_default) {
+                        si_pi->enable_dte = true;
+                        if (update_dte_from_pl2)
+                                si_update_dte_from_pl2(rdev, &si_pi->dte_data);
+
+                }
+                ni_pi->enable_sq_ramping = true;
+        }
+
+        ni_pi->driver_calculate_cac_leakage = true;
+        ni_pi->cac_configuration_required = true;
+
+        if (ni_pi->cac_configuration_required) {
+                ni_pi->support_cac_long_term_average = true;
+                si_pi->dyn_powertune_data.l2_lta_window_size =
+                        si_pi->powertune_data->l2_lta_window_size_default;
+                si_pi->dyn_powertune_data.lts_truncate =
+                        si_pi->powertune_data->lts_truncate_default;
+        } else {
+                ni_pi->support_cac_long_term_average = false;
+                si_pi->dyn_powertune_data.l2_lta_window_size = 0;
+                si_pi->dyn_powertune_data.lts_truncate = 0;
+        }
+
+        si_pi->dyn_powertune_data.disable_uvd_powertune = false;
+}
+
+static u32 si_get_smc_power_scaling_factor(struct radeon_device *rdev)
+{
+        return 1;
+}
+
+static u32 si_calculate_cac_wintime(struct radeon_device *rdev)
+{
+        u32 xclk;
+        u32 wintime;
+        u32 cac_window;
+        u32 cac_window_size;
+
+        xclk = radeon_get_xclk(rdev);
+
+        if (xclk == 0)
+                return 0;
+
+        cac_window = RREG32(CG_CAC_CTRL) & CAC_WINDOW_MASK;
+        cac_window_size = ((cac_window & 0xFFFF0000) >> 16) * (cac_window & 0x0000FFFF);
+
+        wintime = (cac_window_size * 100) / xclk;
+
+        return wintime;
+}
+
+static u32 si_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
+{
+        return power_in_watts;
+}
+
+static int si_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
+                                            bool adjust_polarity,
+                                            u32 tdp_adjustment,
+                                            u32 *tdp_limit,
+                                            u32 *near_tdp_limit)
+{
+        u32 adjustment_delta, max_tdp_limit;
+
+        if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
+                return -EINVAL;
+
+        max_tdp_limit = ((100 + 100) * rdev->pm.dpm.tdp_limit) / 100;
+
+        if (adjust_polarity) {
+                *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
+                *near_tdp_limit = rdev->pm.dpm.near_tdp_limit_adjusted + (*tdp_limit - rdev->pm.dpm.tdp_limit);
+        } else {
+                *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
+                adjustment_delta  = rdev->pm.dpm.tdp_limit - *tdp_limit;
+                if (adjustment_delta < rdev->pm.dpm.near_tdp_limit_adjusted)
+                        *near_tdp_limit = rdev->pm.dpm.near_tdp_limit_adjusted - adjustment_delta;
+                else
+                        *near_tdp_limit = 0;
+        }
+
+        if ((*tdp_limit <= 0) || (*tdp_limit > max_tdp_limit))
+                return -EINVAL;
+        if ((*near_tdp_limit <= 0) || (*near_tdp_limit > *tdp_limit))
+                return -EINVAL;
+
+        return 0;
+}
+
+static int si_populate_smc_tdp_limits(struct radeon_device *rdev,
+                                      struct radeon_ps *radeon_state)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        if (ni_pi->enable_power_containment) {
+                SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable;
+                PP_SIslands_PAPMParameters *papm_parm;
+                struct radeon_ppm_table *ppm = rdev->pm.dpm.dyn_state.ppm_table;
+                u32 scaling_factor = si_get_smc_power_scaling_factor(rdev);
+                u32 tdp_limit;
+                u32 near_tdp_limit;
+                int ret;
+
+                if (scaling_factor == 0)
+                        return -EINVAL;
+
+                memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE));
+
+                ret = si_calculate_adjusted_tdp_limits(rdev,
+                                                       false, /* ??? */
+                                                       rdev->pm.dpm.tdp_adjustment,
+                                                       &tdp_limit,
+                                                       &near_tdp_limit);
+                if (ret)
+                        return ret;
+
+                smc_table->dpm2Params.TDPLimit =
+                        cpu_to_be32(si_scale_power_for_smc(tdp_limit, scaling_factor) * 1000);
+                smc_table->dpm2Params.NearTDPLimit =
+                        cpu_to_be32(si_scale_power_for_smc(near_tdp_limit, scaling_factor) * 1000);
+                smc_table->dpm2Params.SafePowerLimit =
+                        cpu_to_be32(si_scale_power_for_smc((near_tdp_limit * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000);
+
+                ret = si_copy_bytes_to_smc(rdev,
+                                           (si_pi->state_table_start + offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) +
+                                                 offsetof(PP_SIslands_DPM2Parameters, TDPLimit)),
+                                           (u8 *)(&(smc_table->dpm2Params.TDPLimit)),
+                                           sizeof(u32) * 3,
+                                           si_pi->sram_end);
+                if (ret)
+                        return ret;
+
+                if (si_pi->enable_ppm) {
+                        papm_parm = &si_pi->papm_parm;
+                        memset(papm_parm, 0, sizeof(PP_SIslands_PAPMParameters));
+                        papm_parm->NearTDPLimitTherm = cpu_to_be32(ppm->dgpu_tdp);
+                        papm_parm->dGPU_T_Limit = cpu_to_be32(ppm->tj_max);
+                        papm_parm->dGPU_T_Warning = cpu_to_be32(95);
+                        papm_parm->dGPU_T_Hysteresis = cpu_to_be32(5);
+                        papm_parm->PlatformPowerLimit = 0xffffffff;
+                        papm_parm->NearTDPLimitPAPM = 0xffffffff;
+
+                        ret = si_copy_bytes_to_smc(rdev, si_pi->papm_cfg_table_start,
+                                                   (u8 *)papm_parm,
+                                                   sizeof(PP_SIslands_PAPMParameters),
+                                                   si_pi->sram_end);
+                        if (ret)
+                                return ret;
+                }
+        }
+        return 0;
+}
+
+static int si_populate_smc_tdp_limits_2(struct radeon_device *rdev,
+                                        struct radeon_ps *radeon_state)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        if (ni_pi->enable_power_containment) {
+                SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable;
+                u32 scaling_factor = si_get_smc_power_scaling_factor(rdev);
+                int ret;
+
+                memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE));
+
+                smc_table->dpm2Params.NearTDPLimit =
+                        cpu_to_be32(si_scale_power_for_smc(rdev->pm.dpm.near_tdp_limit_adjusted, scaling_factor) * 1000);
+                smc_table->dpm2Params.SafePowerLimit =
+                        cpu_to_be32(si_scale_power_for_smc((rdev->pm.dpm.near_tdp_limit_adjusted * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000);
+
+                ret = si_copy_bytes_to_smc(rdev,
+                                           (si_pi->state_table_start +
+                                            offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) +
+                                            offsetof(PP_SIslands_DPM2Parameters, NearTDPLimit)),
+                                           (u8 *)(&(smc_table->dpm2Params.NearTDPLimit)),
+                                           sizeof(u32) * 2,
+                                           si_pi->sram_end);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static u16 si_calculate_power_efficiency_ratio(struct radeon_device *rdev,
+                                               const u16 prev_std_vddc,
+                                               const u16 curr_std_vddc)
+{
+        u64 margin = (u64)SISLANDS_DPM2_PWREFFICIENCYRATIO_MARGIN;
+        u64 prev_vddc = (u64)prev_std_vddc;
+        u64 curr_vddc = (u64)curr_std_vddc;
+        u64 pwr_efficiency_ratio, n, d;
+
+        if ((prev_vddc == 0) || (curr_vddc == 0))
+                return 0;
+
+        n = div64_u64((u64)1024 * curr_vddc * curr_vddc * ((u64)1000 + margin), (u64)1000);
+        d = prev_vddc * prev_vddc;
+        pwr_efficiency_ratio = div64_u64(n, d);
+
+        if (pwr_efficiency_ratio > (u64)0xFFFF)
+                return 0;
+
+        return (u16)pwr_efficiency_ratio;
+}
+
+static bool si_should_disable_uvd_powertune(struct radeon_device *rdev,
+                                            struct radeon_ps *radeon_state)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        if (si_pi->dyn_powertune_data.disable_uvd_powertune &&
+            radeon_state->vclk && radeon_state->dclk)
+                return true;
+
+        return false;
+}
+
+static int si_populate_power_containment_values(struct radeon_device *rdev,
+                                                struct radeon_ps *radeon_state,
+                                                SISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        SISLANDS_SMC_VOLTAGE_VALUE vddc;
+        u32 prev_sclk;
+        u32 max_sclk;
+        u32 min_sclk;
+        u16 prev_std_vddc;
+        u16 curr_std_vddc;
+        int i;
+        u16 pwr_efficiency_ratio;
+        u8 max_ps_percent;
+        bool disable_uvd_power_tune;
+        int ret;
+
+        if (ni_pi->enable_power_containment == false)
+                return 0;
+
+        if (state->performance_level_count == 0)
+                return -EINVAL;
+
+        if (smc_state->levelCount != state->performance_level_count)
+                return -EINVAL;
+
+        disable_uvd_power_tune = si_should_disable_uvd_powertune(rdev, radeon_state);
+
+        smc_state->levels[0].dpm2.MaxPS = 0;
+        smc_state->levels[0].dpm2.NearTDPDec = 0;
+        smc_state->levels[0].dpm2.AboveSafeInc = 0;
+        smc_state->levels[0].dpm2.BelowSafeInc = 0;
+        smc_state->levels[0].dpm2.PwrEfficiencyRatio = 0;
+
+        for (i = 1; i < state->performance_level_count; i++) {
+                prev_sclk = state->performance_levels[i-1].sclk;
+                max_sclk  = state->performance_levels[i].sclk;
+                if (i == 1)
+                        max_ps_percent = SISLANDS_DPM2_MAXPS_PERCENT_M;
+                else
+                        max_ps_percent = SISLANDS_DPM2_MAXPS_PERCENT_H;
+
+                if (prev_sclk > max_sclk)
+                        return -EINVAL;
+
+                if ((max_ps_percent == 0) ||
+                    (prev_sclk == max_sclk) ||
+                    disable_uvd_power_tune) {
+                        min_sclk = max_sclk;
+                } else if (i == 1) {
+                        min_sclk = prev_sclk;
+                } else {
+                        min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
+                }
+
+                if (min_sclk < state->performance_levels[0].sclk)
+                        min_sclk = state->performance_levels[0].sclk;
+
+                if (min_sclk == 0)
+                        return -EINVAL;
+
+                ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                                state->performance_levels[i-1].vddc, &vddc);
+                if (ret)
+                        return ret;
+
+                ret = si_get_std_voltage_value(rdev, &vddc, &prev_std_vddc);
+                if (ret)
+                        return ret;
+
+                ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                                state->performance_levels[i].vddc, &vddc);
+                if (ret)
+                        return ret;
+
+                ret = si_get_std_voltage_value(rdev, &vddc, &curr_std_vddc);
+                if (ret)
+                        return ret;
+
+                pwr_efficiency_ratio = si_calculate_power_efficiency_ratio(rdev,
+                                                                           prev_std_vddc, curr_std_vddc);
+
+                smc_state->levels[i].dpm2.MaxPS = (u8)((SISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
+                smc_state->levels[i].dpm2.NearTDPDec = SISLANDS_DPM2_NEAR_TDP_DEC;
+                smc_state->levels[i].dpm2.AboveSafeInc = SISLANDS_DPM2_ABOVE_SAFE_INC;
+                smc_state->levels[i].dpm2.BelowSafeInc = SISLANDS_DPM2_BELOW_SAFE_INC;
+                smc_state->levels[i].dpm2.PwrEfficiencyRatio = cpu_to_be16(pwr_efficiency_ratio);
+        }
+
+        return 0;
+}
+
+static int si_populate_sq_ramping_values(struct radeon_device *rdev,
+                                         struct radeon_ps *radeon_state,
+                                         SISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        u32 sq_power_throttle, sq_power_throttle2;
+        bool enable_sq_ramping = ni_pi->enable_sq_ramping;
+        int i;
+
+        if (state->performance_level_count == 0)
+                return -EINVAL;
+
+        if (smc_state->levelCount != state->performance_level_count)
+                return -EINVAL;
+
+        if (rdev->pm.dpm.sq_ramping_threshold == 0)
+                return -EINVAL;
+
+        if (SISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
+                enable_sq_ramping = false;
+
+        if (SISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
+                enable_sq_ramping = false;
+
+        if (SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
+                enable_sq_ramping = false;
+
+        if (SISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
+                enable_sq_ramping = false;
+
+        if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO <= (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
+                enable_sq_ramping = false;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                sq_power_throttle = 0;
+                sq_power_throttle2 = 0;
+
+                if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
+                    enable_sq_ramping) {
+                        sq_power_throttle |= MAX_POWER(SISLANDS_DPM2_SQ_RAMP_MAX_POWER);
+                        sq_power_throttle |= MIN_POWER(SISLANDS_DPM2_SQ_RAMP_MIN_POWER);
+                        sq_power_throttle2 |= MAX_POWER_DELTA(SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
+                        sq_power_throttle2 |= STI_SIZE(SISLANDS_DPM2_SQ_RAMP_STI_SIZE);
+                        sq_power_throttle2 |= LTI_RATIO(SISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
+                } else {
+                        sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
+                        sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+                }
+
+                smc_state->levels[i].SQPowerThrottle = cpu_to_be32(sq_power_throttle);
+                smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
+        }
+
+        return 0;
+}
+
+static int si_enable_power_containment(struct radeon_device *rdev,
+                                       struct radeon_ps *radeon_new_state,
+                                       bool enable)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        PPSMC_Result smc_result;
+        int ret = 0;
+
+        if (ni_pi->enable_power_containment) {
+                if (enable) {
+                        if (!si_should_disable_uvd_powertune(rdev, radeon_new_state)) {
+                                smc_result = si_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
+                                if (smc_result != PPSMC_Result_OK) {
+                                        ret = -EINVAL;
+                                        ni_pi->pc_enabled = false;
+                                } else {
+                                        ni_pi->pc_enabled = true;
+                                }
+                        }
+                } else {
+                        smc_result = si_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
+                        if (smc_result != PPSMC_Result_OK)
+                                ret = -EINVAL;
+                        ni_pi->pc_enabled = false;
+                }
+        }
+
+        return ret;
+}
+
+static int si_initialize_smc_dte_tables(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        int ret = 0;
+        struct si_dte_data *dte_data = &si_pi->dte_data;
+        Smc_SIslands_DTE_Configuration *dte_tables = NULL;
+        u32 table_size;
+        u8 tdep_count;
+        u32 i;
+
+        if (dte_data == NULL)
+                si_pi->enable_dte = false;
+
+        if (si_pi->enable_dte == false)
+                return 0;
+
+        if (dte_data->k <= 0)
+                return -EINVAL;
+
+        dte_tables = kzalloc(sizeof(Smc_SIslands_DTE_Configuration), GFP_KERNEL);
+        if (dte_tables == NULL) {
+                si_pi->enable_dte = false;
+                return -ENOMEM;
+        }
+
+        table_size = dte_data->k;
+
+        if (table_size > SMC_SISLANDS_DTE_MAX_FILTER_STAGES)
+                table_size = SMC_SISLANDS_DTE_MAX_FILTER_STAGES;
+
+        tdep_count = dte_data->tdep_count;
+        if (tdep_count > SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE)
+                tdep_count = SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE;
+
+        dte_tables->K = cpu_to_be32(table_size);
+        dte_tables->T0 = cpu_to_be32(dte_data->t0);
+        dte_tables->MaxT = cpu_to_be32(dte_data->max_t);
+        dte_tables->WindowSize = dte_data->window_size;
+        dte_tables->temp_select = dte_data->temp_select;
+        dte_tables->DTE_mode = dte_data->dte_mode;
+        dte_tables->Tthreshold = cpu_to_be32(dte_data->t_threshold);
+
+        if (tdep_count > 0)
+                table_size--;
+
+        for (i = 0; i < table_size; i++) {
+                dte_tables->tau[i] = cpu_to_be32(dte_data->tau[i]);
+                dte_tables->R[i]   = cpu_to_be32(dte_data->r[i]);
+        }
+
+        dte_tables->Tdep_count = tdep_count;
+
+        for (i = 0; i < (u32)tdep_count; i++) {
+                dte_tables->T_limits[i] = dte_data->t_limits[i];
+                dte_tables->Tdep_tau[i] = cpu_to_be32(dte_data->tdep_tau[i]);
+                dte_tables->Tdep_R[i] = cpu_to_be32(dte_data->tdep_r[i]);
+        }
+
+        ret = si_copy_bytes_to_smc(rdev, si_pi->dte_table_start, (u8 *)dte_tables,
+                                   sizeof(Smc_SIslands_DTE_Configuration), si_pi->sram_end);
+        kfree(dte_tables);
+
+        return ret;
+}
+
+static int si_get_cac_std_voltage_max_min(struct radeon_device *rdev,
+                                          u16 *max, u16 *min)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct radeon_cac_leakage_table *table =
+                &rdev->pm.dpm.dyn_state.cac_leakage_table;
+        u32 i;
+        u32 v0_loadline;
+
+
+        if (table == NULL)
+                return -EINVAL;
+
+        *max = 0;
+        *min = 0xFFFF;
+
+        for (i = 0; i < table->count; i++) {
+                if (table->entries[i].vddc > *max)
+                        *max = table->entries[i].vddc;
+                if (table->entries[i].vddc < *min)
+                        *min = table->entries[i].vddc;
+        }
+
+        if (si_pi->powertune_data->lkge_lut_v0_percent > 100)
+                return -EINVAL;
+
+        v0_loadline = (*min) * (100 - si_pi->powertune_data->lkge_lut_v0_percent) / 100;
+
+        if (v0_loadline > 0xFFFFUL)
+                return -EINVAL;
+
+        *min = (u16)v0_loadline;
+
+        if ((*min > *max) || (*max == 0) || (*min == 0))
+                return -EINVAL;
+
+        return 0;
+}
+
+static u16 si_get_cac_std_voltage_step(u16 max, u16 min)
+{
+        return ((max - min) + (SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES - 1)) /
+                SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
+}
+
+static int si_init_dte_leakage_table(struct radeon_device *rdev,
+                                     PP_SIslands_CacConfig *cac_tables,
+                                     u16 vddc_max, u16 vddc_min, u16 vddc_step,
+                                     u16 t0, u16 t_step)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 leakage;
+        unsigned int i, j;
+        s32 t;
+        u32 smc_leakage;
+        u32 scaling_factor;
+        u16 voltage;
+
+        scaling_factor = si_get_smc_power_scaling_factor(rdev);
+
+        for (i = 0; i < SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES ; i++) {
+                t = (1000 * (i * t_step + t0));
+
+                for (j = 0; j < SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
+                        voltage = vddc_max - (vddc_step * j);
+
+                        si_calculate_leakage_for_v_and_t(rdev,
+                                                         &si_pi->powertune_data->leakage_coefficients,
+                                                         voltage,
+                                                         t,
+                                                         si_pi->dyn_powertune_data.cac_leakage,
+                                                         &leakage);
+
+                        smc_leakage = si_scale_power_for_smc(leakage, scaling_factor) / 4;
+
+                        if (smc_leakage > 0xFFFF)
+                                smc_leakage = 0xFFFF;
+
+                        cac_tables->cac_lkge_lut[i][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES-1-j] =
+                                cpu_to_be16((u16)smc_leakage);
+                }
+        }
+        return 0;
+}
+
+static int si_init_simplified_leakage_table(struct radeon_device *rdev,
+                                            PP_SIslands_CacConfig *cac_tables,
+                                            u16 vddc_max, u16 vddc_min, u16 vddc_step)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 leakage;
+        unsigned int i, j;
+        u32 smc_leakage;
+        u32 scaling_factor;
+        u16 voltage;
+
+        scaling_factor = si_get_smc_power_scaling_factor(rdev);
+
+        for (j = 0; j < SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
+                voltage = vddc_max - (vddc_step * j);
+
+                si_calculate_leakage_for_v(rdev,
+                                           &si_pi->powertune_data->leakage_coefficients,
+                                           si_pi->powertune_data->fixed_kt,
+                                           voltage,
+                                           si_pi->dyn_powertune_data.cac_leakage,
+                                           &leakage);
+
+                smc_leakage = si_scale_power_for_smc(leakage, scaling_factor) / 4;
+
+                if (smc_leakage > 0xFFFF)
+                        smc_leakage = 0xFFFF;
+
+                for (i = 0; i < SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES ; i++)
+                        cac_tables->cac_lkge_lut[i][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES-1-j] =
+                                cpu_to_be16((u16)smc_leakage);
+        }
+        return 0;
+}
+
+static int si_initialize_smc_cac_tables(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        PP_SIslands_CacConfig *cac_tables = NULL;
+        u16 vddc_max, vddc_min, vddc_step;
+        u16 t0, t_step;
+        u32 load_line_slope, reg;
+        int ret = 0;
+        u32 ticks_per_us = radeon_get_xclk(rdev) / 100;
+
+        if (ni_pi->enable_cac == false)
+                return 0;
+
+        cac_tables = kzalloc(sizeof(PP_SIslands_CacConfig), GFP_KERNEL);
+        if (!cac_tables)
+                return -ENOMEM;
+
+        reg = RREG32(CG_CAC_CTRL) & ~CAC_WINDOW_MASK;
+        reg |= CAC_WINDOW(si_pi->powertune_data->cac_window);
+        WREG32(CG_CAC_CTRL, reg);
+
+        si_pi->dyn_powertune_data.cac_leakage = rdev->pm.dpm.cac_leakage;
+        si_pi->dyn_powertune_data.dc_pwr_value =
+                si_pi->powertune_data->dc_cac[NISLANDS_DCCAC_LEVEL_0];
+        si_pi->dyn_powertune_data.wintime = si_calculate_cac_wintime(rdev);
+        si_pi->dyn_powertune_data.shift_n = si_pi->powertune_data->shift_n_default;
+
+        si_pi->dyn_powertune_data.leakage_minimum_temperature = 80 * 1000;
+
+        ret = si_get_cac_std_voltage_max_min(rdev, &vddc_max, &vddc_min);
+        if (ret)
+                goto done_free;
+
+        vddc_step = si_get_cac_std_voltage_step(vddc_max, vddc_min);
+        vddc_min = vddc_max - (vddc_step * (SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES - 1));
+        t_step = 4;
+        t0 = 60;
+
+        if (si_pi->enable_dte || ni_pi->driver_calculate_cac_leakage)
+                ret = si_init_dte_leakage_table(rdev, cac_tables,
+                                                vddc_max, vddc_min, vddc_step,
+                                                t0, t_step);
+        else
+                ret = si_init_simplified_leakage_table(rdev, cac_tables,
+                                                       vddc_max, vddc_min, vddc_step);
+        if (ret)
+                goto done_free;
+
+        load_line_slope = ((u32)rdev->pm.dpm.load_line_slope << SMC_SISLANDS_SCALE_R) / 100;
+
+        cac_tables->l2numWin_TDP = cpu_to_be32(si_pi->dyn_powertune_data.l2_lta_window_size);
+        cac_tables->lts_truncate_n = si_pi->dyn_powertune_data.lts_truncate;
+        cac_tables->SHIFT_N = si_pi->dyn_powertune_data.shift_n;
+        cac_tables->lkge_lut_V0 = cpu_to_be32((u32)vddc_min);
+        cac_tables->lkge_lut_Vstep = cpu_to_be32((u32)vddc_step);
+        cac_tables->R_LL = cpu_to_be32(load_line_slope);
+        cac_tables->WinTime = cpu_to_be32(si_pi->dyn_powertune_data.wintime);
+        cac_tables->calculation_repeats = cpu_to_be32(2);
+        cac_tables->dc_cac = cpu_to_be32(0);
+        cac_tables->log2_PG_LKG_SCALE = 12;
+        cac_tables->cac_temp = si_pi->powertune_data->operating_temp;
+        cac_tables->lkge_lut_T0 = cpu_to_be32((u32)t0);
+        cac_tables->lkge_lut_Tstep = cpu_to_be32((u32)t_step);
+
+        ret = si_copy_bytes_to_smc(rdev, si_pi->cac_table_start, (u8 *)cac_tables,
+                                   sizeof(PP_SIslands_CacConfig), si_pi->sram_end);
+
+        if (ret)
+                goto done_free;
+
+        ret = si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_ticks_per_us, ticks_per_us);
+
+done_free:
+        if (ret) {
+                ni_pi->enable_cac = false;
+                ni_pi->enable_power_containment = false;
+        }
+
+        kfree(cac_tables);
+
+        return 0;
+}
+
+static int si_program_cac_config_registers(struct radeon_device *rdev,
+                                           const struct si_cac_config_reg *cac_config_regs)
+{
+        const struct si_cac_config_reg *config_regs = cac_config_regs;
+        u32 data = 0, offset;
+
+        if (!config_regs)
+                return -EINVAL;
+
+        while (config_regs->offset != 0xFFFFFFFF) {
+                switch (config_regs->type) {
+                case SISLANDS_CACCONFIG_CGIND:
+                        offset = SMC_CG_IND_START + config_regs->offset;
+                        if (offset < SMC_CG_IND_END)
+                                data = RREG32_SMC(offset);
+                        break;
+                default:
+                        data = RREG32(config_regs->offset << 2);
+                        break;
+                }
+
+                data &= ~config_regs->mask;
+                data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+
+                switch (config_regs->type) {
+                case SISLANDS_CACCONFIG_CGIND:
+                        offset = SMC_CG_IND_START + config_regs->offset;
+                        if (offset < SMC_CG_IND_END)
+                                WREG32_SMC(offset, data);
+                        break;
+                default:
+                        WREG32(config_regs->offset << 2, data);
+                        break;
+                }
+                config_regs++;
+        }
+        return 0;
+}
+
+static int si_initialize_hardware_cac_manager(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        int ret;
+
+        if ((ni_pi->enable_cac == false) ||
+            (ni_pi->cac_configuration_required == false))
+                return 0;
+
+        ret = si_program_cac_config_registers(rdev, si_pi->lcac_config);
+        if (ret)
+                return ret;
+        ret = si_program_cac_config_registers(rdev, si_pi->cac_override);
+        if (ret)
+                return ret;
+        ret = si_program_cac_config_registers(rdev, si_pi->cac_weights);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+static int si_enable_smc_cac(struct radeon_device *rdev,
+                             struct radeon_ps *radeon_new_state,
+                             bool enable)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        PPSMC_Result smc_result;
+        int ret = 0;
+
+        if (ni_pi->enable_cac) {
+                if (enable) {
+                        if (!si_should_disable_uvd_powertune(rdev, radeon_new_state)) {
+                                if (ni_pi->support_cac_long_term_average) {
+                                        smc_result = si_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
+                                        if (smc_result != PPSMC_Result_OK)
+                                                ni_pi->support_cac_long_term_average = false;
+                                }
+
+                                smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
+                                if (smc_result != PPSMC_Result_OK) {
+                                        ret = -EINVAL;
+                                        ni_pi->cac_enabled = false;
+                                } else {
+                                        ni_pi->cac_enabled = true;
+                                }
+
+                                if (si_pi->enable_dte) {
+                                        smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableDTE);
+                                        if (smc_result != PPSMC_Result_OK)
+                                                ret = -EINVAL;
+                                }
+                        }
+                } else if (ni_pi->cac_enabled) {
+                        if (si_pi->enable_dte)
+                                smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_DisableDTE);
+
+                        smc_result = si_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
+
+                        ni_pi->cac_enabled = false;
+
+                        if (ni_pi->support_cac_long_term_average)
+                                smc_result = si_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
+                }
+        }
+        return ret;
+}
+
+static int si_init_smc_spll_table(struct radeon_device *rdev)
+{
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        SMC_SISLANDS_SPLL_DIV_TABLE *spll_table;
+        SISLANDS_SMC_SCLK_VALUE sclk_params;
+        u32 fb_div, p_div;
+        u32 clk_s, clk_v;
+        u32 sclk = 0;
+        int ret = 0;
+        u32 tmp;
+        int i;
+
+        if (si_pi->spll_table_start == 0)
+                return -EINVAL;
+
+        spll_table = kzalloc(sizeof(SMC_SISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
+        if (spll_table == NULL)
+                return -ENOMEM;
+
+        for (i = 0; i < 256; i++) {
+                ret = si_calculate_sclk_params(rdev, sclk, &sclk_params);
+                if (ret)
+                        break;
+
+                p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
+                fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
+                clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
+                clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
+
+                fb_div &= ~0x00001FFF;
+                fb_div >>= 1;
+                clk_v >>= 6;
+
+                if (p_div & ~(SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
+                        ret = -EINVAL;
+                if (fb_div & ~(SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
+                        ret = -EINVAL;
+                if (clk_s & ~(SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
+                        ret = -EINVAL;
+                if (clk_v & ~(SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
+                        ret = -EINVAL;
+
+                if (ret)
+                        break;
+
+                tmp = ((fb_div << SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
+                        ((p_div << SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
+                spll_table->freq[i] = cpu_to_be32(tmp);
+
+                tmp = ((clk_v << SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
+                        ((clk_s << SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
+                spll_table->ss[i] = cpu_to_be32(tmp);
+
+                sclk += 512;
+        }
+
+
+        if (!ret)
+                ret = si_copy_bytes_to_smc(rdev, si_pi->spll_table_start,
+                                           (u8 *)spll_table, sizeof(SMC_SISLANDS_SPLL_DIV_TABLE),
+                                           si_pi->sram_end);
+
+        if (ret)
+                ni_pi->enable_power_containment = false;
+
+        kfree(spll_table);
+
+        return ret;
+}
+
+static void si_apply_state_adjust_rules(struct radeon_device *rdev,
+                                        struct radeon_ps *rps)
+{
+        struct ni_ps *ps = ni_get_ps(rps);
+        struct radeon_clock_and_voltage_limits *max_limits;
+        bool disable_mclk_switching;
+        u32 mclk, sclk;
+        u16 vddc, vddci;
+        int i;
+
+        if (rdev->pm.dpm.new_active_crtc_count > 1)
+                disable_mclk_switching = true;
+        else
+                disable_mclk_switching = false;
+
+        if (rdev->pm.dpm.ac_power)
+                max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
+        else
+                max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
+
+        for (i = ps->performance_level_count - 2; i >= 0; i--) {
+                if (ps->performance_levels[i].vddc > ps->performance_levels[i+1].vddc)
+                        ps->performance_levels[i].vddc = ps->performance_levels[i+1].vddc;
+        }
+        if (rdev->pm.dpm.ac_power == false) {
+                for (i = 0; i < ps->performance_level_count; i++) {
+                        if (ps->performance_levels[i].mclk > max_limits->mclk)
+                                ps->performance_levels[i].mclk = max_limits->mclk;
+                        if (ps->performance_levels[i].sclk > max_limits->sclk)
+                                ps->performance_levels[i].sclk = max_limits->sclk;
+                        if (ps->performance_levels[i].vddc > max_limits->vddc)
+                                ps->performance_levels[i].vddc = max_limits->vddc;
+                        if (ps->performance_levels[i].vddci > max_limits->vddci)
+                                ps->performance_levels[i].vddci = max_limits->vddci;
+                }
+        }
+
+        /* XXX validate the min clocks required for display */
+
+        if (disable_mclk_switching) {
+                mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
+                sclk = ps->performance_levels[0].sclk;
+                vddc = ps->performance_levels[0].vddc;
+                vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
+        } else {
+                sclk = ps->performance_levels[0].sclk;
+                mclk = ps->performance_levels[0].mclk;
+                vddc = ps->performance_levels[0].vddc;
+                vddci = ps->performance_levels[0].vddci;
+        }
+
+        /* adjusted low state */
+        ps->performance_levels[0].sclk = sclk;
+        ps->performance_levels[0].mclk = mclk;
+        ps->performance_levels[0].vddc = vddc;
+        ps->performance_levels[0].vddci = vddci;
+
+        for (i = 1; i < ps->performance_level_count; i++) {
+                if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
+                        ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
+                if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
+                        ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
+        }
+
+        if (disable_mclk_switching) {
+                mclk = ps->performance_levels[0].mclk;
+                for (i = 1; i < ps->performance_level_count; i++) {
+                        if (mclk < ps->performance_levels[i].mclk)
+                                mclk = ps->performance_levels[i].mclk;
+                }
+                for (i = 0; i < ps->performance_level_count; i++) {
+                        ps->performance_levels[i].mclk = mclk;
+                        ps->performance_levels[i].vddci = vddci;
+                }
+        } else {
+                for (i = 1; i < ps->performance_level_count; i++) {
+                        if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
+                                ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
+                        if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
+                                ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
+                }
+        }
+
+        for (i = 0; i < ps->performance_level_count; i++)
+                btc_adjust_clock_combinations(rdev, max_limits,
+                                              &ps->performance_levels[i]);
+
+        for (i = 0; i < ps->performance_level_count; i++) {
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+                                                   ps->performance_levels[i].sclk,
+                                                   max_limits->vddc,  &ps->performance_levels[i].vddc);
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+                                                   ps->performance_levels[i].mclk,
+                                                   max_limits->vddci, &ps->performance_levels[i].vddci);
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+                                                   ps->performance_levels[i].mclk,
+                                                   max_limits->vddc,  &ps->performance_levels[i].vddc);
+                btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
+                                                   rdev->clock.current_dispclk,
+                                                   max_limits->vddc,  &ps->performance_levels[i].vddc);
+        }
+
+        for (i = 0; i < ps->performance_level_count; i++) {
+                btc_apply_voltage_delta_rules(rdev,
+                                              max_limits->vddc, max_limits->vddci,
+                                              &ps->performance_levels[i].vddc,
+                                              &ps->performance_levels[i].vddci);
+        }
+
+        ps->dc_compatible = true;
+        for (i = 0; i < ps->performance_level_count; i++) {
+                if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
+                        ps->dc_compatible = false;
+        }
+
+}
+
+#if 0
+static int si_read_smc_soft_register(struct radeon_device *rdev,
+                                     u16 reg_offset, u32 *value)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        return si_read_smc_sram_dword(rdev,
+                                      si_pi->soft_regs_start + reg_offset, value,
+                                      si_pi->sram_end);
+}
+#endif
+
+static int si_write_smc_soft_register(struct radeon_device *rdev,
+                                      u16 reg_offset, u32 value)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        return si_write_smc_sram_dword(rdev,
+                                       si_pi->soft_regs_start + reg_offset,
+                                       value, si_pi->sram_end);
+}
+
+static bool si_is_special_1gb_platform(struct radeon_device *rdev)
+{
+        bool ret = false;
+        u32 tmp, width, row, column, bank, density;
+        bool is_memory_gddr5, is_special;
+
+        tmp = RREG32(MC_SEQ_MISC0);
+        is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE == ((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT));
+        is_special = (MC_SEQ_MISC0_REV_ID_VALUE == ((tmp & MC_SEQ_MISC0_REV_ID_MASK) >> MC_SEQ_MISC0_REV_ID_SHIFT))
+                & (MC_SEQ_MISC0_VEN_ID_VALUE == ((tmp & MC_SEQ_MISC0_VEN_ID_MASK) >> MC_SEQ_MISC0_VEN_ID_SHIFT));
+
+        WREG32(MC_SEQ_IO_DEBUG_INDEX, 0xb);
+        width = ((RREG32(MC_SEQ_IO_DEBUG_DATA) >> 1) & 1) ? 16 : 32;
+
+        tmp = RREG32(MC_ARB_RAMCFG);
+        row = ((tmp & NOOFROWS_MASK) >> NOOFROWS_SHIFT) + 10;
+        column = ((tmp & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) + 8;
+        bank = ((tmp & NOOFBANK_MASK) >> NOOFBANK_SHIFT) + 2;
+
+        density = (1 << (row + column - 20 + bank)) * width;
+
+        if ((rdev->pdev->device == 0x6819) &&
+            is_memory_gddr5 && is_special && (density == 0x400))
+                ret = true;
+
+        return ret;
+}
+
+static void si_get_leakage_vddc(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u16 vddc, count = 0;
+        int i, ret;
+
+        for (i = 0; i < SISLANDS_MAX_LEAKAGE_COUNT; i++) {
+                ret = radeon_atom_get_leakage_vddc_based_on_leakage_idx(rdev, &vddc, SISLANDS_LEAKAGE_INDEX0 + i);
+
+                if (!ret && (vddc > 0) && (vddc != (SISLANDS_LEAKAGE_INDEX0 + i))) {
+                        si_pi->leakage_voltage.entries[count].voltage = vddc;
+                        si_pi->leakage_voltage.entries[count].leakage_index =
+                                SISLANDS_LEAKAGE_INDEX0 + i;
+                        count++;
+                }
+        }
+        si_pi->leakage_voltage.count = count;
+}
+
+static int si_get_leakage_voltage_from_leakage_index(struct radeon_device *rdev,
+                                                     u32 index, u16 *leakage_voltage)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        int i;
+
+        if (leakage_voltage == NULL)
+                return -EINVAL;
+
+        if ((index & 0xff00) != 0xff00)
+                return -EINVAL;
+
+        if ((index & 0xff) > SISLANDS_MAX_LEAKAGE_COUNT + 1)
+                return -EINVAL;
+
+        if (index < SISLANDS_LEAKAGE_INDEX0)
+                return -EINVAL;
+
+        for (i = 0; i < si_pi->leakage_voltage.count; i++) {
+                if (si_pi->leakage_voltage.entries[i].leakage_index == index) {
+                        *leakage_voltage = si_pi->leakage_voltage.entries[i].voltage;
+                        return 0;
+                }
+        }
+        return -EAGAIN;
+}
+
+static void si_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        bool want_thermal_protection;
+        enum radeon_dpm_event_src dpm_event_src;
+
+        switch (sources) {
+        case 0:
+        default:
+                want_thermal_protection = false;
+                break;
+        case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
+                break;
+        case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
+                break;
+        case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
+              (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
+                want_thermal_protection = true;
+                dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
+                break;
+        }
+
+        if (want_thermal_protection) {
+                WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
+                if (pi->thermal_protection)
+                        WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+        } else {
+                WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+        }
+}
+
+static void si_enable_auto_throttle_source(struct radeon_device *rdev,
+                                           enum radeon_dpm_auto_throttle_src source,
+                                           bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (enable) {
+                if (!(pi->active_auto_throttle_sources & (1 << source))) {
+                        pi->active_auto_throttle_sources |= 1 << source;
+                        si_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
+                }
+        } else {
+                if (pi->active_auto_throttle_sources & (1 << source)) {
+                        pi->active_auto_throttle_sources &= ~(1 << source);
+                        si_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
+                }
+        }
+}
+
+static void si_start_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+}
+
+static void si_stop_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
+}
+
+static void si_enable_sclk_control(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
+        else
+                WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
+
+}
+
+#if 0
+static int si_notify_hardware_of_thermal_state(struct radeon_device *rdev,
+                                               u32 thermal_level)
+{
+        PPSMC_Result ret;
+
+        if (thermal_level == 0) {
+                ret = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
+                if (ret == PPSMC_Result_OK)
+                        return 0;
+                else
+                        return -EINVAL;
+        }
+        return 0;
+}
+
+static void si_notify_hardware_vpu_recovery_event(struct radeon_device *rdev)
+{
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_tdr_is_about_to_happen, true);
+}
+#endif
+
+#if 0
+static int si_notify_hw_of_powersource(struct radeon_device *rdev, bool ac_power)
+{
+        if (ac_power)
+                return (si_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
+                        0 : -EINVAL;
+
+        return 0;
+}
+#endif
+
+static PPSMC_Result si_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
+                                                      PPSMC_Msg msg, u32 parameter)
+{
+        WREG32(SMC_SCRATCH0, parameter);
+        return si_send_msg_to_smc(rdev, msg);
+}
+
+static int si_restrict_performance_levels_before_switch(struct radeon_device *rdev)
+{
+        if (si_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+#if 0
+static int si_unrestrict_performance_levels_after_switch(struct radeon_device *rdev)
+{
+        if (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return (si_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+#endif
+
+static int si_set_boot_state(struct radeon_device *rdev)
+{
+        return (si_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static int si_set_sw_state(struct radeon_device *rdev)
+{
+        return (si_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static int si_halt_smc(struct radeon_device *rdev)
+{
+        if (si_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return (si_wait_for_smc_inactive(rdev) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static int si_resume_smc(struct radeon_device *rdev)
+{
+        if (si_send_msg_to_smc(rdev, PPSMC_FlushDataCache) != PPSMC_Result_OK)
+                return -EINVAL;
+
+        return (si_send_msg_to_smc(rdev, PPSMC_MSG_Resume) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static void si_dpm_start_smc(struct radeon_device *rdev)
+{
+        si_program_jump_on_start(rdev);
+        si_start_smc(rdev);
+        si_start_smc_clock(rdev);
+}
+
+static void si_dpm_stop_smc(struct radeon_device *rdev)
+{
+        si_reset_smc(rdev);
+        si_stop_smc_clock(rdev);
+}
+
+static int si_process_firmware_header(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_stateTable,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->state_table_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->soft_regs_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->mc_reg_table_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->arb_table_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_CacConfigTable,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->cac_table_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_DteConfiguration,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->dte_table_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_spllTable,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->spll_table_start = tmp;
+
+        ret = si_read_smc_sram_dword(rdev,
+                                     SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+                                     SISLANDS_SMC_FIRMWARE_HEADER_PAPMParameters,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        si_pi->papm_cfg_table_start = tmp;
+
+        return ret;
+}
+
+static void si_read_clock_registers(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        si_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
+        si_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
+        si_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
+        si_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
+        si_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
+        si_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
+        si_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
+        si_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
+        si_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
+        si_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
+        si_pi->clock_registers.mpll_func_cntl = RREG32(MPLL_FUNC_CNTL);
+        si_pi->clock_registers.mpll_func_cntl_1 = RREG32(MPLL_FUNC_CNTL_1);
+        si_pi->clock_registers.mpll_func_cntl_2 = RREG32(MPLL_FUNC_CNTL_2);
+        si_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
+        si_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
+}
+
+static void si_enable_thermal_protection(struct radeon_device *rdev,
+                                          bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+        else
+                WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+}
+
+static void si_enable_acpi_power_management(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
+}
+
+#if 0
+static int si_enter_ulp_state(struct radeon_device *rdev)
+{
+        WREG32(SMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower);
+
+        udelay(25000);
+
+        return 0;
+}
+
+static int si_exit_ulp_state(struct radeon_device *rdev)
+{
+        int i;
+
+        WREG32(SMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower);
+
+        udelay(7000);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(SMC_RESP_0) == 1)
+                        break;
+                udelay(1000);
+        }
+
+        return 0;
+}
+#endif
+
+static int si_notify_smc_display_change(struct radeon_device *rdev,
+                                     bool has_display)
+{
+        PPSMC_Msg msg = has_display ?
+                PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay;
+
+        return (si_send_msg_to_smc(rdev, msg) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static void si_program_response_times(struct radeon_device *rdev)
+{
+        u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
+        u32 vddc_dly, acpi_dly, vbi_dly;
+        u32 reference_clock;
+
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
+
+        voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
+        backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
+
+        if (voltage_response_time == 0)
+                voltage_response_time = 1000;
+
+        acpi_delay_time = 15000;
+        vbi_time_out = 100000;
+
+        reference_clock = radeon_get_xclk(rdev);
+
+        vddc_dly = (voltage_response_time  * reference_clock) / 100;
+        acpi_dly = (acpi_delay_time * reference_clock) / 100;
+        vbi_dly  = (vbi_time_out * reference_clock) / 100;
+
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
+}
+
+static void si_program_ds_registers(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        u32 tmp = 1; /* XXX: 0x10 on tahiti A0 */
+
+        if (eg_pi->sclk_deep_sleep) {
+                WREG32_P(MISC_CLK_CNTL, DEEP_SLEEP_CLK_SEL(tmp), ~DEEP_SLEEP_CLK_SEL_MASK);
+                WREG32_P(CG_SPLL_AUTOSCALE_CNTL, AUTOSCALE_ON_SS_CLEAR,
+                         ~AUTOSCALE_ON_SS_CLEAR);
+        }
+}
+
+static void si_program_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp, pipe;
+        int i;
+
+        tmp = RREG32(CG_DISPLAY_GAP_CNTL) & ~(DISP1_GAP_MASK | DISP2_GAP_MASK);
+        if (rdev->pm.dpm.new_active_crtc_count > 0)
+                tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
+        else
+                tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE);
+
+        if (rdev->pm.dpm.new_active_crtc_count > 1)
+                tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
+        else
+                tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE);
+
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+
+        tmp = RREG32(DCCG_DISP_SLOW_SELECT_REG);
+        pipe = (tmp & DCCG_DISP1_SLOW_SELECT_MASK) >> DCCG_DISP1_SLOW_SELECT_SHIFT;
+
+        if ((rdev->pm.dpm.new_active_crtc_count > 0) &&
+            (!(rdev->pm.dpm.new_active_crtcs & (1 << pipe)))) {
+                /* find the first active crtc */
+                for (i = 0; i < rdev->num_crtc; i++) {
+                        if (rdev->pm.dpm.new_active_crtcs & (1 << i))
+                                break;
+                }
+                if (i == rdev->num_crtc)
+                        pipe = 0;
+                else
+                        pipe = i;
+
+                tmp &= ~DCCG_DISP1_SLOW_SELECT_MASK;
+                tmp |= DCCG_DISP1_SLOW_SELECT(pipe);
+                WREG32(DCCG_DISP_SLOW_SELECT_REG, tmp);
+        }
+
+        si_notify_smc_display_change(rdev, rdev->pm.dpm.new_active_crtc_count > 0);
+}
+
+static void si_enable_spread_spectrum(struct radeon_device *rdev, bool enable)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        if (enable) {
+                if (pi->sclk_ss)
+                        WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
+        } else {
+                WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
+                WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
+        }
+}
+
+static void si_setup_bsp(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 xclk = radeon_get_xclk(rdev);
+
+        r600_calculate_u_and_p(pi->asi,
+                               xclk,
+                               16,
+                               &pi->bsp,
+                               &pi->bsu);
+
+        r600_calculate_u_and_p(pi->pasi,
+                               xclk,
+                               16,
+                               &pi->pbsp,
+                               &pi->pbsu);
+
+
+        pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
+        pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
+
+        WREG32(CG_BSP, pi->dsp);
+}
+
+static void si_program_git(struct radeon_device *rdev)
+{
+        WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK);
+}
+
+static void si_program_tp(struct radeon_device *rdev)
+{
+        int i;
+        enum r600_td td = R600_TD_DFLT;
+
+        for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
+                WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i])));
+
+        if (td == R600_TD_AUTO)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
+        else
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
+
+        if (td == R600_TD_UP)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
+
+        if (td == R600_TD_DOWN)
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
+}
+
+static void si_program_tpp(struct radeon_device *rdev)
+{
+        WREG32(CG_TPC, R600_TPC_DFLT);
+}
+
+static void si_program_sstp(struct radeon_device *rdev)
+{
+        WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
+}
+
+static void si_enable_display_gap(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
+
+        tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
+        tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
+                DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
+        WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+}
+
+static void si_program_vc(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        WREG32(CG_FTV, pi->vrc);
+}
+
+static void si_clear_vc(struct radeon_device *rdev)
+{
+        WREG32(CG_FTV, 0);
+}
+
+static u8 si_get_ddr3_mclk_frequency_ratio(u32 memory_clock)
+{
+        u8 mc_para_index;
+
+        if (memory_clock < 10000)
+                mc_para_index = 0;
+        else if (memory_clock >= 80000)
+                mc_para_index = 0x0f;
+        else
+                mc_para_index = (u8)((memory_clock - 10000) / 5000 + 1);
+        return mc_para_index;
+}
+
+static u8 si_get_mclk_frequency_ratio(u32 memory_clock, bool strobe_mode)
+{
+        u8 mc_para_index;
+
+        if (strobe_mode) {
+                if (memory_clock < 12500)
+                        mc_para_index = 0x00;
+                else if (memory_clock > 47500)
+                        mc_para_index = 0x0f;
+                else
+                        mc_para_index = (u8)((memory_clock - 10000) / 2500);
+        } else {
+                if (memory_clock < 65000)
+                        mc_para_index = 0x00;
+                else if (memory_clock > 135000)
+                        mc_para_index = 0x0f;
+                else
+                        mc_para_index = (u8)((memory_clock - 60000) / 5000);
+        }
+        return mc_para_index;
+}
+
+static u8 si_get_strobe_mode_settings(struct radeon_device *rdev, u32 mclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        bool strobe_mode = false;
+        u8 result = 0;
+
+        if (mclk <= pi->mclk_strobe_mode_threshold)
+                strobe_mode = true;
+
+        if (pi->mem_gddr5)
+                result = si_get_mclk_frequency_ratio(mclk, strobe_mode);
+        else
+                result = si_get_ddr3_mclk_frequency_ratio(mclk);
+
+        if (strobe_mode)
+                result |= SISLANDS_SMC_STROBE_ENABLE;
+
+        return result;
+}
+
+static int si_upload_firmware(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        int ret;
+
+        si_reset_smc(rdev);
+        si_stop_smc_clock(rdev);
+
+        ret = si_load_smc_ucode(rdev, si_pi->sram_end);
+
+        return ret;
+}
+
+static bool si_validate_phase_shedding_tables(struct radeon_device *rdev,
+                                              const struct atom_voltage_table *table,
+                                              const struct radeon_phase_shedding_limits_table *limits)
+{
+        u32 data, num_bits, num_levels;
+
+        if ((table == NULL) || (limits == NULL))
+                return false;
+
+        data = table->mask_low;
+
+        num_bits = hweight32(data);
+
+        if (num_bits == 0)
+                return false;
+
+        num_levels = (1 << num_bits);
+
+        if (table->count != num_levels)
+                return false;
+
+        if (limits->count != (num_levels - 1))
+                return false;
+
+        return true;
+}
+
+static void si_trim_voltage_table_to_fit_state_table(struct radeon_device *rdev,
+                                                     struct atom_voltage_table *voltage_table)
+{
+        unsigned int i, diff;
+
+        if (voltage_table->count <= SISLANDS_MAX_NO_VREG_STEPS)
+                return;
+
+        diff = voltage_table->count - SISLANDS_MAX_NO_VREG_STEPS;
+
+        for (i= 0; i < SISLANDS_MAX_NO_VREG_STEPS; i++)
+                voltage_table->entries[i] = voltage_table->entries[i + diff];
+
+        voltage_table->count = SISLANDS_MAX_NO_VREG_STEPS;
+}
+
+static int si_construct_voltage_tables(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        int ret;
+
+        ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC,
+                                            VOLTAGE_OBJ_GPIO_LUT, &eg_pi->vddc_voltage_table);
+        if (ret)
+                return ret;
+
+        if (eg_pi->vddc_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS)
+                si_trim_voltage_table_to_fit_state_table(rdev, &eg_pi->vddc_voltage_table);
+
+        if (eg_pi->vddci_control) {
+                ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDCI,
+                                                    VOLTAGE_OBJ_GPIO_LUT, &eg_pi->vddci_voltage_table);
+                if (ret)
+                        return ret;
+
+                if (eg_pi->vddci_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS)
+                        si_trim_voltage_table_to_fit_state_table(rdev, &eg_pi->vddci_voltage_table);
+        }
+
+        if (pi->mvdd_control) {
+                ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_MVDDC,
+                                                    VOLTAGE_OBJ_GPIO_LUT, &si_pi->mvdd_voltage_table);
+
+                if (ret) {
+                        pi->mvdd_control = false;
+                        return ret;
+                }
+
+                if (si_pi->mvdd_voltage_table.count == 0) {
+                        pi->mvdd_control = false;
+                        return -EINVAL;
+                }
+
+                if (si_pi->mvdd_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS)
+                        si_trim_voltage_table_to_fit_state_table(rdev, &si_pi->mvdd_voltage_table);
+        }
+
+        if (si_pi->vddc_phase_shed_control) {
+                ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC,
+                                                    VOLTAGE_OBJ_PHASE_LUT, &si_pi->vddc_phase_shed_table);
+                if (ret)
+                        si_pi->vddc_phase_shed_control = false;
+
+                if ((si_pi->vddc_phase_shed_table.count == 0) ||
+                    (si_pi->vddc_phase_shed_table.count > SISLANDS_MAX_NO_VREG_STEPS))
+                        si_pi->vddc_phase_shed_control = false;
+        }
+
+        return 0;
+}
+
+static void si_populate_smc_voltage_table(struct radeon_device *rdev,
+                                          const struct atom_voltage_table *voltage_table,
+                                          SISLANDS_SMC_STATETABLE *table)
+{
+        unsigned int i;
+
+        for (i = 0; i < voltage_table->count; i++)
+                table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
+}
+
+static int si_populate_smc_voltage_tables(struct radeon_device *rdev,
+                                          SISLANDS_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u8 i;
+
+        if (eg_pi->vddc_voltage_table.count) {
+                si_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
+                table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC] =
+                        cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
+
+                for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
+                        if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
+                                table->maxVDDCIndexInPPTable = i;
+                                break;
+                        }
+                }
+        }
+
+        if (eg_pi->vddci_voltage_table.count) {
+                si_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
+
+                table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDCI] =
+                        cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
+        }
+
+
+        if (si_pi->mvdd_voltage_table.count) {
+                si_populate_smc_voltage_table(rdev, &si_pi->mvdd_voltage_table, table);
+
+                table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_MVDD] =
+                        cpu_to_be32(si_pi->mvdd_voltage_table.mask_low);
+        }
+
+        if (si_pi->vddc_phase_shed_control) {
+                if (si_validate_phase_shedding_tables(rdev, &si_pi->vddc_phase_shed_table,
+                                                      &rdev->pm.dpm.dyn_state.phase_shedding_limits_table)) {
+                        si_populate_smc_voltage_table(rdev, &si_pi->vddc_phase_shed_table, table);
+
+                        table->phaseMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC] =
+                                cpu_to_be32(si_pi->vddc_phase_shed_table.mask_low);
+
+                        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_phase_shedding_delay,
+                                                   (u32)si_pi->vddc_phase_shed_table.phase_delay);
+                } else {
+                        si_pi->vddc_phase_shed_control = false;
+                }
+        }
+
+        return 0;
+}
+
+static int si_populate_voltage_value(struct radeon_device *rdev,
+                                     const struct atom_voltage_table *table,
+                                     u16 value, SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        unsigned int i;
+
+        for (i = 0; i < table->count; i++) {
+                if (value <= table->entries[i].value) {
+                        voltage->index = (u8)i;
+                        voltage->value = cpu_to_be16(table->entries[i].value);
+                        break;
+                }
+        }
+
+        if (i >= table->count)
+                return -EINVAL;
+
+        return 0;
+}
+
+static int si_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
+                                  SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        if (pi->mvdd_control) {
+                if (mclk <= pi->mvdd_split_frequency)
+                        voltage->index = 0;
+                else
+                        voltage->index = (u8)(si_pi->mvdd_voltage_table.count) - 1;
+
+                voltage->value = cpu_to_be16(si_pi->mvdd_voltage_table.entries[voltage->index].value);
+        }
+        return 0;
+}
+
+static int si_get_std_voltage_value(struct radeon_device *rdev,
+                                    SISLANDS_SMC_VOLTAGE_VALUE *voltage,
+                                    u16 *std_voltage)
+{
+        u16 v_index;
+        bool voltage_found = false;
+        *std_voltage = be16_to_cpu(voltage->value);
+
+        if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries) {
+                if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_NEW_CAC_VOLTAGE) {
+                        if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL)
+                                return -EINVAL;
+
+                        for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
+                                if (be16_to_cpu(voltage->value) ==
+                                    (u16)rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
+                                        voltage_found = true;
+                                        if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
+                                                *std_voltage =
+                                                        rdev->pm.dpm.dyn_state.cac_leakage_table.entries[v_index].vddc;
+                                        else
+                                                *std_voltage =
+                                                        rdev->pm.dpm.dyn_state.cac_leakage_table.entries[rdev->pm.dpm.dyn_state.cac_leakage_table.count-1].vddc;
+                                        break;
+                                }
+                        }
+
+                        if (!voltage_found) {
+                                for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
+                                        if (be16_to_cpu(voltage->value) <=
+                                            (u16)rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
+                                                voltage_found = true;
+                                                if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
+                                                        *std_voltage =
+                                                                rdev->pm.dpm.dyn_state.cac_leakage_table.entries[v_index].vddc;
+                                                else
+                                                        *std_voltage =
+                                                                rdev->pm.dpm.dyn_state.cac_leakage_table.entries[rdev->pm.dpm.dyn_state.cac_leakage_table.count-1].vddc;
+                                                break;
+                                        }
+                                }
+                        }
+                } else {
+                        if ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
+                                *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
+                }
+        }
+
+        return 0;
+}
+
+static int si_populate_std_voltage_value(struct radeon_device *rdev,
+                                         u16 value, u8 index,
+                                         SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        voltage->index = index;
+        voltage->value = cpu_to_be16(value);
+
+        return 0;
+}
+
+static int si_populate_phase_shedding_value(struct radeon_device *rdev,
+                                            const struct radeon_phase_shedding_limits_table *limits,
+                                            u16 voltage, u32 sclk, u32 mclk,
+                                            SISLANDS_SMC_VOLTAGE_VALUE *smc_voltage)
+{
+        unsigned int i;
+
+        for (i = 0; i < limits->count; i++) {
+                if ((voltage <= limits->entries[i].voltage) &&
+                    (sclk <= limits->entries[i].sclk) &&
+                    (mclk <= limits->entries[i].mclk))
+                        break;
+        }
+
+        smc_voltage->phase_settings = (u8)i;
+
+        return 0;
+}
+
+static int si_init_arb_table_index(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = si_read_smc_sram_dword(rdev, si_pi->arb_table_start, &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        tmp &= 0x00FFFFFF;
+        tmp |= MC_CG_ARB_FREQ_F1 << 24;
+
+        return si_write_smc_sram_dword(rdev, si_pi->arb_table_start,  tmp, si_pi->sram_end);
+}
+
+static int si_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
+{
+        return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
+}
+
+static int si_reset_to_default(struct radeon_device *rdev)
+{
+        return (si_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ?
+                0 : -EINVAL;
+}
+
+static int si_force_switch_to_arb_f0(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 tmp;
+        int ret;
+
+        ret = si_read_smc_sram_dword(rdev, si_pi->arb_table_start,
+                                     &tmp, si_pi->sram_end);
+        if (ret)
+                return ret;
+
+        tmp = (tmp >> 24) & 0xff;
+
+        if (tmp == MC_CG_ARB_FREQ_F0)
+                return 0;
+
+        return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
+}
+
+static u32 si_calculate_memory_refresh_rate(struct radeon_device *rdev,
+                                            u32 engine_clock)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u32 dram_rows;
+        u32 dram_refresh_rate;
+        u32 mc_arb_rfsh_rate;
+        u32 tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
+
+        if (pi->mem_gddr5)
+                dram_rows = 1 << (tmp + 10);
+        else
+                dram_rows = DDR3_DRAM_ROWS;
+
+        dram_refresh_rate = 1 << ((RREG32(MC_SEQ_MISC0) & 0x3) + 3);
+        mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
+
+        return mc_arb_rfsh_rate;
+}
+
+static int si_populate_memory_timing_parameters(struct radeon_device *rdev,
+                                                struct rv7xx_pl *pl,
+                                                SMC_SIslands_MCArbDramTimingRegisterSet *arb_regs)
+{
+        u32 dram_timing;
+        u32 dram_timing2;
+        u32 burst_time;
+
+        arb_regs->mc_arb_rfsh_rate =
+                (u8)si_calculate_memory_refresh_rate(rdev, pl->sclk);
+
+        radeon_atom_set_engine_dram_timings(rdev,
+                                            pl->sclk,
+                                            pl->mclk);
+
+        dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
+        dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+        burst_time = RREG32(MC_ARB_BURST_TIME) & STATE0_MASK;
+
+        arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
+        arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
+        arb_regs->mc_arb_burst_time = (u8)burst_time;
+
+        return 0;
+}
+
+static int si_do_program_memory_timing_parameters(struct radeon_device *rdev,
+                                                  struct radeon_ps *radeon_state,
+                                                  unsigned int first_arb_set)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        SMC_SIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
+        int i, ret = 0;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                ret = si_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
+                if (ret)
+                        break;
+                ret = si_copy_bytes_to_smc(rdev,
+                                           si_pi->arb_table_start +
+                                           offsetof(SMC_SIslands_MCArbDramTimingRegisters, data) +
+                                           sizeof(SMC_SIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i),
+                                           (u8 *)&arb_regs,
+                                           sizeof(SMC_SIslands_MCArbDramTimingRegisterSet),
+                                           si_pi->sram_end);
+                if (ret)
+                        break;
+        }
+
+        return ret;
+}
+
+static int si_program_memory_timing_parameters(struct radeon_device *rdev,
+                                               struct radeon_ps *radeon_new_state)
+{
+        return si_do_program_memory_timing_parameters(rdev, radeon_new_state,
+                                                      SISLANDS_DRIVER_STATE_ARB_INDEX);
+}
+
+static int si_populate_initial_mvdd_value(struct radeon_device *rdev,
+                                          struct SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        if (pi->mvdd_control)
+                return si_populate_voltage_value(rdev, &si_pi->mvdd_voltage_table,
+                                                 si_pi->mvdd_bootup_value, voltage);
+
+        return 0;
+}
+
+static int si_populate_smc_initial_state(struct radeon_device *rdev,
+                                         struct radeon_ps *radeon_initial_state,
+                                         SISLANDS_SMC_STATETABLE *table)
+{
+        struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 reg;
+        int ret;
+
+        table->initialState.levels[0].mclk.vDLL_CNTL =
+                cpu_to_be32(si_pi->clock_registers.dll_cntl);
+        table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(si_pi->clock_registers.mclk_pwrmgt_cntl);
+        table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
+                cpu_to_be32(si_pi->clock_registers.mpll_ad_func_cntl);
+        table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
+                cpu_to_be32(si_pi->clock_registers.mpll_dq_func_cntl);
+        table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL =
+                cpu_to_be32(si_pi->clock_registers.mpll_func_cntl);
+        table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL_1 =
+                cpu_to_be32(si_pi->clock_registers.mpll_func_cntl_1);
+        table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL_2 =
+                cpu_to_be32(si_pi->clock_registers.mpll_func_cntl_2);
+        table->initialState.levels[0].mclk.vMPLL_SS =
+                cpu_to_be32(si_pi->clock_registers.mpll_ss1);
+        table->initialState.levels[0].mclk.vMPLL_SS2 =
+                cpu_to_be32(si_pi->clock_registers.mpll_ss2);
+
+        table->initialState.levels[0].mclk.mclk_value =
+                cpu_to_be32(initial_state->performance_levels[0].mclk);
+
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_2);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_3);
+        table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
+                cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_4);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
+                cpu_to_be32(si_pi->clock_registers.cg_spll_spread_spectrum);
+        table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2  =
+                cpu_to_be32(si_pi->clock_registers.cg_spll_spread_spectrum_2);
+
+        table->initialState.levels[0].sclk.sclk_value =
+                cpu_to_be32(initial_state->performance_levels[0].sclk);
+
+        table->initialState.levels[0].arbRefreshState =
+                SISLANDS_INITIAL_STATE_ARB_INDEX;
+
+        table->initialState.levels[0].ACIndex = 0;
+
+        ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                        initial_state->performance_levels[0].vddc,
+                                        &table->initialState.levels[0].vddc);
+
+        if (!ret) {
+                u16 std_vddc;
+
+                ret = si_get_std_voltage_value(rdev,
+                                               &table->initialState.levels[0].vddc,
+                                               &std_vddc);
+                if (!ret)
+                        si_populate_std_voltage_value(rdev, std_vddc,
+                                                      table->initialState.levels[0].vddc.index,
+                                                      &table->initialState.levels[0].std_vddc);
+        }
+
+        if (eg_pi->vddci_control)
+                si_populate_voltage_value(rdev,
+                                          &eg_pi->vddci_voltage_table,
+                                          initial_state->performance_levels[0].vddci,
+                                          &table->initialState.levels[0].vddci);
+
+        if (si_pi->vddc_phase_shed_control)
+                si_populate_phase_shedding_value(rdev,
+                                                 &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
+                                                 initial_state->performance_levels[0].vddc,
+                                                 initial_state->performance_levels[0].sclk,
+                                                 initial_state->performance_levels[0].mclk,
+                                                 &table->initialState.levels[0].vddc);
+
+        si_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
+
+        reg = CG_R(0xffff) | CG_L(0);
+        table->initialState.levels[0].aT = cpu_to_be32(reg);
+
+        table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
+
+        table->initialState.levels[0].gen2PCIE = (u8)si_pi->boot_pcie_gen;
+
+        if (pi->mem_gddr5) {
+                table->initialState.levels[0].strobeMode =
+                        si_get_strobe_mode_settings(rdev,
+                                                    initial_state->performance_levels[0].mclk);
+
+                if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
+                        table->initialState.levels[0].mcFlags = SISLANDS_SMC_MC_EDC_RD_FLAG | SISLANDS_SMC_MC_EDC_WR_FLAG;
+                else
+                        table->initialState.levels[0].mcFlags =  0;
+        }
+
+        table->initialState.levelCount = 1;
+
+        table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        table->initialState.levels[0].dpm2.MaxPS = 0;
+        table->initialState.levels[0].dpm2.NearTDPDec = 0;
+        table->initialState.levels[0].dpm2.AboveSafeInc = 0;
+        table->initialState.levels[0].dpm2.BelowSafeInc = 0;
+        table->initialState.levels[0].dpm2.PwrEfficiencyRatio = 0;
+
+        reg = MIN_POWER_MASK | MAX_POWER_MASK;
+        table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
+
+        reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+        table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
+
+        return 0;
+}
+
+static int si_populate_smc_acpi_state(struct radeon_device *rdev,
+                                      SISLANDS_SMC_STATETABLE *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 spll_func_cntl = si_pi->clock_registers.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 = si_pi->clock_registers.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 = si_pi->clock_registers.cg_spll_func_cntl_3;
+        u32 spll_func_cntl_4 = si_pi->clock_registers.cg_spll_func_cntl_4;
+        u32 dll_cntl = si_pi->clock_registers.dll_cntl;
+        u32 mclk_pwrmgt_cntl = si_pi->clock_registers.mclk_pwrmgt_cntl;
+        u32 mpll_ad_func_cntl = si_pi->clock_registers.mpll_ad_func_cntl;
+        u32 mpll_dq_func_cntl = si_pi->clock_registers.mpll_dq_func_cntl;
+        u32 mpll_func_cntl = si_pi->clock_registers.mpll_func_cntl;
+        u32 mpll_func_cntl_1 = si_pi->clock_registers.mpll_func_cntl_1;
+        u32 mpll_func_cntl_2 = si_pi->clock_registers.mpll_func_cntl_2;
+        u32 reg;
+        int ret;
+
+        table->ACPIState = table->initialState;
+
+        table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+        if (pi->acpi_vddc) {
+                ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                                pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
+                if (!ret) {
+                        u16 std_vddc;
+
+                        ret = si_get_std_voltage_value(rdev,
+                                                       &table->ACPIState.levels[0].vddc, &std_vddc);
+                        if (!ret)
+                                si_populate_std_voltage_value(rdev, std_vddc,
+                                                              table->ACPIState.levels[0].vddc.index,
+                                                              &table->ACPIState.levels[0].std_vddc);
+                }
+                table->ACPIState.levels[0].gen2PCIE = si_pi->acpi_pcie_gen;
+
+                if (si_pi->vddc_phase_shed_control) {
+                        si_populate_phase_shedding_value(rdev,
+                                                         &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
+                                                         pi->acpi_vddc,
+                                                         0,
+                                                         0,
+                                                         &table->ACPIState.levels[0].vddc);
+                }
+        } else {
+                ret = si_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
+                                                pi->min_vddc_in_table, &table->ACPIState.levels[0].vddc);
+                if (!ret) {
+                        u16 std_vddc;
+
+                        ret = si_get_std_voltage_value(rdev,
+                                                       &table->ACPIState.levels[0].vddc, &std_vddc);
+
+                        if (!ret)
+                                si_populate_std_voltage_value(rdev, std_vddc,
+                                                              table->ACPIState.levels[0].vddc.index,
+                                                              &table->ACPIState.levels[0].std_vddc);
+                }
+                table->ACPIState.levels[0].gen2PCIE = (u8)r600_get_pcie_gen_support(rdev,
+                                                                                    si_pi->sys_pcie_mask,
+                                                                                    si_pi->boot_pcie_gen,
+                                                                                    RADEON_PCIE_GEN1);
+
+                if (si_pi->vddc_phase_shed_control)
+                        si_populate_phase_shedding_value(rdev,
+                                                         &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
+                                                         pi->min_vddc_in_table,
+                                                         0,
+                                                         0,
+                                                         &table->ACPIState.levels[0].vddc);
+        }
+
+        if (pi->acpi_vddc) {
+                if (eg_pi->acpi_vddci)
+                        si_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
+                                                  eg_pi->acpi_vddci,
+                                                  &table->ACPIState.levels[0].vddci);
+        }
+
+        mclk_pwrmgt_cntl |= MRDCK0_RESET | MRDCK1_RESET;
+        mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
+
+        dll_cntl &= ~(MRDCK0_BYPASS | MRDCK1_BYPASS);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+        table->ACPIState.levels[0].mclk.vDLL_CNTL =
+                cpu_to_be32(dll_cntl);
+        table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
+                cpu_to_be32(mclk_pwrmgt_cntl);
+        table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
+                cpu_to_be32(mpll_ad_func_cntl);
+        table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
+                cpu_to_be32(mpll_dq_func_cntl);
+        table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL =
+                cpu_to_be32(mpll_func_cntl);
+        table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL_1 =
+                cpu_to_be32(mpll_func_cntl_1);
+        table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL_2 =
+                cpu_to_be32(mpll_func_cntl_2);
+        table->ACPIState.levels[0].mclk.vMPLL_SS =
+                cpu_to_be32(si_pi->clock_registers.mpll_ss1);
+        table->ACPIState.levels[0].mclk.vMPLL_SS2 =
+                cpu_to_be32(si_pi->clock_registers.mpll_ss2);
+
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+                cpu_to_be32(spll_func_cntl);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+                cpu_to_be32(spll_func_cntl_2);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+                cpu_to_be32(spll_func_cntl_3);
+        table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
+                cpu_to_be32(spll_func_cntl_4);
+
+        table->ACPIState.levels[0].mclk.mclk_value = 0;
+        table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+        si_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
+
+        if (eg_pi->dynamic_ac_timing)
+                table->ACPIState.levels[0].ACIndex = 0;
+
+        table->ACPIState.levels[0].dpm2.MaxPS = 0;
+        table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
+        table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
+        table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
+        table->ACPIState.levels[0].dpm2.PwrEfficiencyRatio = 0;
+
+        reg = MIN_POWER_MASK | MAX_POWER_MASK;
+        table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
+
+        reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+        table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
+
+        return 0;
+}
+
+static int si_populate_ulv_state(struct radeon_device *rdev,
+                                 SISLANDS_SMC_SWSTATE *state)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct si_ulv_param *ulv = &si_pi->ulv;
+        u32 sclk_in_sr = 1350; /* ??? */
+        int ret;
+
+        ret = si_convert_power_level_to_smc(rdev, &ulv->pl,
+                                            &state->levels[0]);
+        if (!ret) {
+                if (eg_pi->sclk_deep_sleep) {
+                        if (sclk_in_sr <= SCLK_MIN_DEEPSLEEP_FREQ)
+                                state->levels[0].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_BYPASS;
+                        else
+                                state->levels[0].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE;
+                }
+                if (ulv->one_pcie_lane_in_ulv)
+                        state->flags |= PPSMC_SWSTATE_FLAG_PCIE_X1;
+                state->levels[0].arbRefreshState = (u8)(SISLANDS_ULV_STATE_ARB_INDEX);
+                state->levels[0].ACIndex = 1;
+                state->levels[0].std_vddc = state->levels[0].vddc;
+                state->levelCount = 1;
+
+                state->flags |= PPSMC_SWSTATE_FLAG_DC;
+        }
+
+        return ret;
+}
+
+static int si_program_ulv_memory_timing_parameters(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct si_ulv_param *ulv = &si_pi->ulv;
+        SMC_SIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
+        int ret;
+
+        ret = si_populate_memory_timing_parameters(rdev, &ulv->pl,
+                                                   &arb_regs);
+        if (ret)
+                return ret;
+
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_ulv_volt_change_delay,
+                                   ulv->volt_change_delay);
+
+        ret = si_copy_bytes_to_smc(rdev,
+                                   si_pi->arb_table_start +
+                                   offsetof(SMC_SIslands_MCArbDramTimingRegisters, data) +
+                                   sizeof(SMC_SIslands_MCArbDramTimingRegisterSet) * SISLANDS_ULV_STATE_ARB_INDEX,
+                                   (u8 *)&arb_regs,
+                                   sizeof(SMC_SIslands_MCArbDramTimingRegisterSet),
+                                   si_pi->sram_end);
+
+        return ret;
+}
+
+static void si_get_mvdd_configuration(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+
+        pi->mvdd_split_frequency = 30000;
+}
+
+static int si_init_smc_table(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
+        const struct si_ulv_param *ulv = &si_pi->ulv;
+        SISLANDS_SMC_STATETABLE  *table = &si_pi->smc_statetable;
+        int ret;
+        u32 lane_width;
+        u32 vr_hot_gpio;
+
+        si_populate_smc_voltage_tables(rdev, table);
+
+        switch (rdev->pm.int_thermal_type) {
+        case THERMAL_TYPE_SI:
+        case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
+                break;
+        case THERMAL_TYPE_NONE:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
+                break;
+        default:
+                table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
+                break;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) {
+                if ((rdev->pdev->device != 0x6818) && (rdev->pdev->device != 0x6819))
+                        table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
+        }
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+        if (pi->mem_gddr5)
+                table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY)
+                table->systemFlags |= PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH;
+
+        if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE) {
+                table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT_PROG_GPIO;
+                vr_hot_gpio = rdev->pm.dpm.backbias_response_time;
+                si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_vr_hot_gpio,
+                                           vr_hot_gpio);
+        }
+
+        ret = si_populate_smc_initial_state(rdev, radeon_boot_state, table);
+        if (ret)
+                return ret;
+
+        ret = si_populate_smc_acpi_state(rdev, table);
+        if (ret)
+                return ret;
+
+        table->driverState = table->initialState;
+
+        ret = si_do_program_memory_timing_parameters(rdev, radeon_boot_state,
+                                                     SISLANDS_INITIAL_STATE_ARB_INDEX);
+        if (ret)
+                return ret;
+
+        if (ulv->supported && ulv->pl.vddc) {
+                ret = si_populate_ulv_state(rdev, &table->ULVState);
+                if (ret)
+                        return ret;
+
+                ret = si_program_ulv_memory_timing_parameters(rdev);
+                if (ret)
+                        return ret;
+
+                WREG32(CG_ULV_CONTROL, ulv->cg_ulv_control);
+                WREG32(CG_ULV_PARAMETER, ulv->cg_ulv_parameter);
+
+                lane_width = radeon_get_pcie_lanes(rdev);
+                si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width, lane_width);
+        } else {
+                table->ULVState = table->initialState;
+        }
+
+        return si_copy_bytes_to_smc(rdev, si_pi->state_table_start,
+                                    (u8 *)table, sizeof(SISLANDS_SMC_STATETABLE),
+                                    si_pi->sram_end);
+}
+
+static int si_calculate_sclk_params(struct radeon_device *rdev,
+                                    u32 engine_clock,
+                                    SISLANDS_SMC_SCLK_VALUE *sclk)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        u32 spll_func_cntl = si_pi->clock_registers.cg_spll_func_cntl;
+        u32 spll_func_cntl_2 = si_pi->clock_registers.cg_spll_func_cntl_2;
+        u32 spll_func_cntl_3 = si_pi->clock_registers.cg_spll_func_cntl_3;
+        u32 spll_func_cntl_4 = si_pi->clock_registers.cg_spll_func_cntl_4;
+        u32 cg_spll_spread_spectrum = si_pi->clock_registers.cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2 = si_pi->clock_registers.cg_spll_spread_spectrum_2;
+        u64 tmp;
+        u32 reference_clock = rdev->clock.spll.reference_freq;
+        u32 reference_divider;
+        u32 fbdiv;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             engine_clock, false, &dividers);
+        if (ret)
+                return ret;
+
+        reference_divider = 1 + dividers.ref_div;
+
+        tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16384;
+        do_div(tmp, reference_clock);
+        fbdiv = (u32) tmp;
+
+        spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
+        spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
+        spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
+
+        spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+        spll_func_cntl_2 |= SCLK_MUX_SEL(2);
+
+        spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
+        spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
+        spll_func_cntl_3 |= SPLL_DITHEN;
+
+        if (pi->sclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 vco_freq = engine_clock * dividers.post_div;
+
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+                        u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+                        u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
+
+                        cg_spll_spread_spectrum &= ~CLK_S_MASK;
+                        cg_spll_spread_spectrum |= CLK_S(clk_s);
+                        cg_spll_spread_spectrum |= SSEN;
+
+                        cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
+                        cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
+                }
+        }
+
+        sclk->sclk_value = engine_clock;
+        sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
+        sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
+        sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
+        sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
+        sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
+        sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
+
+        return 0;
+}
+
+static int si_populate_sclk_value(struct radeon_device *rdev,
+                                  u32 engine_clock,
+                                  SISLANDS_SMC_SCLK_VALUE *sclk)
+{
+        SISLANDS_SMC_SCLK_VALUE sclk_tmp;
+        int ret;
+
+        ret = si_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
+        if (!ret) {
+                sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
+                sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
+                sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
+                sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
+                sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
+                sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
+                sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
+        }
+
+        return ret;
+}
+
+static int si_populate_mclk_value(struct radeon_device *rdev,
+                                  u32 engine_clock,
+                                  u32 memory_clock,
+                                  SISLANDS_SMC_MCLK_VALUE *mclk,
+                                  bool strobe_mode,
+                                  bool dll_state_on)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32  dll_cntl = si_pi->clock_registers.dll_cntl;
+        u32  mclk_pwrmgt_cntl = si_pi->clock_registers.mclk_pwrmgt_cntl;
+        u32  mpll_ad_func_cntl = si_pi->clock_registers.mpll_ad_func_cntl;
+        u32  mpll_dq_func_cntl = si_pi->clock_registers.mpll_dq_func_cntl;
+        u32  mpll_func_cntl = si_pi->clock_registers.mpll_func_cntl;
+        u32  mpll_func_cntl_1 = si_pi->clock_registers.mpll_func_cntl_1;
+        u32  mpll_func_cntl_2 = si_pi->clock_registers.mpll_func_cntl_2;
+        u32  mpll_ss1 = si_pi->clock_registers.mpll_ss1;
+        u32  mpll_ss2 = si_pi->clock_registers.mpll_ss2;
+        struct atom_mpll_param mpll_param;
+        int ret;
+
+        ret = radeon_atom_get_memory_pll_dividers(rdev, memory_clock, strobe_mode, &mpll_param);
+        if (ret)
+                return ret;
+
+        mpll_func_cntl &= ~BWCTRL_MASK;
+        mpll_func_cntl |= BWCTRL(mpll_param.bwcntl);
+
+        mpll_func_cntl_1 &= ~(CLKF_MASK | CLKFRAC_MASK | VCO_MODE_MASK);
+        mpll_func_cntl_1 |= CLKF(mpll_param.clkf) |
+                CLKFRAC(mpll_param.clkfrac) | VCO_MODE(mpll_param.vco_mode);
+
+        mpll_ad_func_cntl &= ~YCLK_POST_DIV_MASK;
+        mpll_ad_func_cntl |= YCLK_POST_DIV(mpll_param.post_div);
+
+        if (pi->mem_gddr5) {
+                mpll_dq_func_cntl &= ~(YCLK_SEL_MASK | YCLK_POST_DIV_MASK);
+                mpll_dq_func_cntl |= YCLK_SEL(mpll_param.yclk_sel) |
+                        YCLK_POST_DIV(mpll_param.post_div);
+        }
+
+        if (pi->mclk_ss) {
+                struct radeon_atom_ss ss;
+                u32 freq_nom;
+                u32 tmp;
+                u32 reference_clock = rdev->clock.mpll.reference_freq;
+
+                if (pi->mem_gddr5)
+                        freq_nom = memory_clock * 4;
+                else
+                        freq_nom = memory_clock * 2;
+
+                tmp = freq_nom / reference_clock;
+                tmp = tmp * tmp;
+                if (radeon_atombios_get_asic_ss_info(rdev, &ss,
+                                                     ASIC_INTERNAL_MEMORY_SS, freq_nom)) {
+                        u32 clks = reference_clock * 5 / ss.rate;
+                        u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom);
+
+                        mpll_ss1 &= ~CLKV_MASK;
+                        mpll_ss1 |= CLKV(clkv);
+
+                        mpll_ss2 &= ~CLKS_MASK;
+                        mpll_ss2 |= CLKS(clks);
+                }
+        }
+
+        mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
+        mclk_pwrmgt_cntl |= DLL_SPEED(mpll_param.dll_speed);
+
+        if (dll_state_on)
+                mclk_pwrmgt_cntl |= MRDCK0_PDNB | MRDCK1_PDNB;
+        else
+                mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
+
+        mclk->mclk_value = cpu_to_be32(memory_clock);
+        mclk->vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl);
+        mclk->vMPLL_FUNC_CNTL_1 = cpu_to_be32(mpll_func_cntl_1);
+        mclk->vMPLL_FUNC_CNTL_2 = cpu_to_be32(mpll_func_cntl_2);
+        mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+        mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+        mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+        mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
+        mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
+        mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
+
+        return 0;
+}
+
+static void si_populate_smc_sp(struct radeon_device *rdev,
+                               struct radeon_ps *radeon_state,
+                               SISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct ni_ps *ps = ni_get_ps(radeon_state);
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        int i;
+
+        for (i = 0; i < ps->performance_level_count - 1; i++)
+                smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
+
+        smc_state->levels[ps->performance_level_count - 1].bSP =
+                cpu_to_be32(pi->psp);
+}
+
+static int si_convert_power_level_to_smc(struct radeon_device *rdev,
+                                         struct rv7xx_pl *pl,
+                                         SISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        int ret;
+        bool dll_state_on;
+        u16 std_vddc;
+        bool gmc_pg = false;
+
+        if (eg_pi->pcie_performance_request &&
+            (si_pi->force_pcie_gen != RADEON_PCIE_GEN_INVALID))
+                level->gen2PCIE = (u8)si_pi->force_pcie_gen;
+        else
+                level->gen2PCIE = (u8)pl->pcie_gen;
+
+        ret = si_populate_sclk_value(rdev, pl->sclk, &level->sclk);
+        if (ret)
+                return ret;
+
+        level->mcFlags =  0;
+
+        if (pi->mclk_stutter_mode_threshold &&
+            (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
+            !eg_pi->uvd_enabled &&
+            (RREG32(DPG_PIPE_STUTTER_CONTROL) & STUTTER_ENABLE) &&
+            (rdev->pm.dpm.new_active_crtc_count <= 2)) {
+                level->mcFlags |= SISLANDS_SMC_MC_STUTTER_EN;
+
+                if (gmc_pg)
+                        level->mcFlags |= SISLANDS_SMC_MC_PG_EN;
+        }
+
+        if (pi->mem_gddr5) {
+                if (pl->mclk > pi->mclk_edc_enable_threshold)
+                        level->mcFlags |= SISLANDS_SMC_MC_EDC_RD_FLAG;
+
+                if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
+                        level->mcFlags |= SISLANDS_SMC_MC_EDC_WR_FLAG;
+
+                level->strobeMode = si_get_strobe_mode_settings(rdev, pl->mclk);
+
+                if (level->strobeMode & SISLANDS_SMC_STROBE_ENABLE) {
+                        if (si_get_mclk_frequency_ratio(pl->mclk, true) >=
+                            ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
+                                dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
+                        else
+                                dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
+                } else {
+                        dll_state_on = false;
+                }
+        } else {
+                level->strobeMode = si_get_strobe_mode_settings(rdev,
+                                                                pl->mclk);
+
+                dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
+        }
+
+        ret = si_populate_mclk_value(rdev,
+                                     pl->sclk,
+                                     pl->mclk,
+                                     &level->mclk,
+                                     (level->strobeMode & SISLANDS_SMC_STROBE_ENABLE) != 0, dll_state_on);
+        if (ret)
+                return ret;
+
+        ret = si_populate_voltage_value(rdev,
+                                        &eg_pi->vddc_voltage_table,
+                                        pl->vddc, &level->vddc);
+        if (ret)
+                return ret;
+
+
+        ret = si_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
+        if (ret)
+                return ret;
+
+        ret = si_populate_std_voltage_value(rdev, std_vddc,
+                                            level->vddc.index, &level->std_vddc);
+        if (ret)
+                return ret;
+
+        if (eg_pi->vddci_control) {
+                ret = si_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
+                                                pl->vddci, &level->vddci);
+                if (ret)
+                        return ret;
+        }
+
+        if (si_pi->vddc_phase_shed_control) {
+                ret = si_populate_phase_shedding_value(rdev,
+                                                       &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
+                                                       pl->vddc,
+                                                       pl->sclk,
+                                                       pl->mclk,
+                                                       &level->vddc);
+                if (ret)
+                        return ret;
+        }
+
+        level->MaxPoweredUpCU = si_pi->max_cu;
+
+        ret = si_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
+
+        return ret;
+}
+
+static int si_populate_smc_t(struct radeon_device *rdev,
+                             struct radeon_ps *radeon_state,
+                             SISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        u32 a_t;
+        u32 t_l, t_h;
+        u32 high_bsp;
+        int i, ret;
+
+        if (state->performance_level_count >= 9)
+                return -EINVAL;
+
+        if (state->performance_level_count < 2) {
+                a_t = CG_R(0xffff) | CG_L(0);
+                smc_state->levels[0].aT = cpu_to_be32(a_t);
+                return 0;
+        }
+
+        smc_state->levels[0].aT = cpu_to_be32(0);
+
+        for (i = 0; i <= state->performance_level_count - 2; i++) {
+                ret = r600_calculate_at(
+                        (50 / SISLANDS_MAX_HARDWARE_POWERLEVELS) * 100 * (i + 1),
+                        100 * R600_AH_DFLT,
+                        state->performance_levels[i + 1].sclk,
+                        state->performance_levels[i].sclk,
+                        &t_l,
+                        &t_h);
+
+                if (ret) {
+                        t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
+                        t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
+                }
+
+                a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
+                a_t |= CG_R(t_l * pi->bsp / 20000);
+                smc_state->levels[i].aT = cpu_to_be32(a_t);
+
+                high_bsp = (i == state->performance_level_count - 2) ?
+                        pi->pbsp : pi->bsp;
+                a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
+                smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
+        }
+
+        return 0;
+}
+
+static int si_disable_ulv(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct si_ulv_param *ulv = &si_pi->ulv;
+
+        if (ulv->supported)
+                return (si_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ?
+                        0 : -EINVAL;
+
+        return 0;
+}
+
+static bool si_is_state_ulv_compatible(struct radeon_device *rdev,
+                                       struct radeon_ps *radeon_state)
+{
+        const struct si_power_info *si_pi = si_get_pi(rdev);
+        const struct si_ulv_param *ulv = &si_pi->ulv;
+        const struct ni_ps *state = ni_get_ps(radeon_state);
+        int i;
+
+        if (state->performance_levels[0].mclk != ulv->pl.mclk)
+                return false;
+
+        /* XXX validate against display requirements! */
+
+        for (i = 0; i < rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count; i++) {
+                if (rdev->clock.current_dispclk <=
+                    rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[i].clk) {
+                        if (ulv->pl.vddc <
+                            rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[i].v)
+                                return false;
+                }
+        }
+
+        if ((radeon_state->vclk != 0) || (radeon_state->dclk != 0))
+                return false;
+
+        return true;
+}
+
+static int si_set_power_state_conditionally_enable_ulv(struct radeon_device *rdev,
+                                                       struct radeon_ps *radeon_new_state)
+{
+        const struct si_power_info *si_pi = si_get_pi(rdev);
+        const struct si_ulv_param *ulv = &si_pi->ulv;
+
+        if (ulv->supported) {
+                if (si_is_state_ulv_compatible(rdev, radeon_new_state))
+                        return (si_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ?
+                                0 : -EINVAL;
+        }
+        return 0;
+}
+
+static int si_convert_power_state_to_smc(struct radeon_device *rdev,
+                                         struct radeon_ps *radeon_state,
+                                         SISLANDS_SMC_SWSTATE *smc_state)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct ni_power_info *ni_pi = ni_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        int i, ret;
+        u32 threshold;
+        u32 sclk_in_sr = 1350; /* ??? */
+
+        if (state->performance_level_count > SISLANDS_MAX_HARDWARE_POWERLEVELS)
+                return -EINVAL;
+
+        threshold = state->performance_levels[state->performance_level_count-1].sclk * 100 / 100;
+
+        if (radeon_state->vclk && radeon_state->dclk) {
+                eg_pi->uvd_enabled = true;
+                if (eg_pi->smu_uvd_hs)
+                        smc_state->flags |= PPSMC_SWSTATE_FLAG_UVD;
+        } else {
+                eg_pi->uvd_enabled = false;
+        }
+
+        if (state->dc_compatible)
+                smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
+
+        smc_state->levelCount = 0;
+        for (i = 0; i < state->performance_level_count; i++) {
+                if (eg_pi->sclk_deep_sleep) {
+                        if ((i == 0) || si_pi->sclk_deep_sleep_above_low) {
+                                if (sclk_in_sr <= SCLK_MIN_DEEPSLEEP_FREQ)
+                                        smc_state->levels[i].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_BYPASS;
+                                else
+                                        smc_state->levels[i].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE;
+                        }
+                }
+
+                ret = si_convert_power_level_to_smc(rdev, &state->performance_levels[i],
+                                                    &smc_state->levels[i]);
+                smc_state->levels[i].arbRefreshState =
+                        (u8)(SISLANDS_DRIVER_STATE_ARB_INDEX + i);
+
+                if (ret)
+                        return ret;
+
+                if (ni_pi->enable_power_containment)
+                        smc_state->levels[i].displayWatermark =
+                                (state->performance_levels[i].sclk < threshold) ?
+                                PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
+                else
+                        smc_state->levels[i].displayWatermark = (i < 2) ?
+                                PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
+
+                if (eg_pi->dynamic_ac_timing)
+                        smc_state->levels[i].ACIndex = SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
+                else
+                        smc_state->levels[i].ACIndex = 0;
+
+                smc_state->levelCount++;
+        }
+
+        si_write_smc_soft_register(rdev,
+                                   SI_SMC_SOFT_REGISTER_watermark_threshold,
+                                   threshold / 512);
+
+        si_populate_smc_sp(rdev, radeon_state, smc_state);
+
+        ret = si_populate_power_containment_values(rdev, radeon_state, smc_state);
+        if (ret)
+                ni_pi->enable_power_containment = false;
+
+        ret = si_populate_sq_ramping_values(rdev, radeon_state, smc_state);
+        if (ret)
+                ni_pi->enable_sq_ramping = false;
+
+        return si_populate_smc_t(rdev, radeon_state, smc_state);
+}
+
+static int si_upload_sw_state(struct radeon_device *rdev,
+                              struct radeon_ps *radeon_new_state)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct ni_ps *new_state = ni_get_ps(radeon_new_state);
+        int ret;
+        u32 address = si_pi->state_table_start +
+                offsetof(SISLANDS_SMC_STATETABLE, driverState);
+        u32 state_size = sizeof(SISLANDS_SMC_SWSTATE) +
+                ((new_state->performance_level_count - 1) *
+                 sizeof(SISLANDS_SMC_HW_PERFORMANCE_LEVEL));
+        SISLANDS_SMC_SWSTATE *smc_state = &si_pi->smc_statetable.driverState;
+
+        memset(smc_state, 0, state_size);
+
+        ret = si_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
+        if (ret)
+                return ret;
+
+        ret = si_copy_bytes_to_smc(rdev, address, (u8 *)smc_state,
+                                   state_size, si_pi->sram_end);
+
+        return ret;
+}
+
+static int si_upload_ulv_state(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct si_ulv_param *ulv = &si_pi->ulv;
+        int ret = 0;
+
+        if (ulv->supported && ulv->pl.vddc) {
+                u32 address = si_pi->state_table_start +
+                        offsetof(SISLANDS_SMC_STATETABLE, ULVState);
+                SISLANDS_SMC_SWSTATE *smc_state = &si_pi->smc_statetable.ULVState;
+                u32 state_size = sizeof(SISLANDS_SMC_SWSTATE);
+
+                memset(smc_state, 0, state_size);
+
+                ret = si_populate_ulv_state(rdev, smc_state);
+                if (!ret)
+                        ret = si_copy_bytes_to_smc(rdev, address, (u8 *)smc_state,
+                                                   state_size, si_pi->sram_end);
+        }
+
+        return ret;
+}
+
+static int si_upload_smc_data(struct radeon_device *rdev)
+{
+        struct radeon_crtc *radeon_crtc = NULL;
+        int i;
+
+        if (rdev->pm.dpm.new_active_crtc_count == 0)
+                return 0;
+
+        for (i = 0; i < rdev->num_crtc; i++) {
+                if (rdev->pm.dpm.new_active_crtcs & (1 << i)) {
+                        radeon_crtc = rdev->mode_info.crtcs[i];
+                        break;
+                }
+        }
+
+        if (radeon_crtc == NULL)
+                return 0;
+
+        if (radeon_crtc->line_time <= 0)
+                return 0;
+
+        if (si_write_smc_soft_register(rdev,
+                                       SI_SMC_SOFT_REGISTER_crtc_index,
+                                       radeon_crtc->crtc_id) != PPSMC_Result_OK)
+                return 0;
+
+        if (si_write_smc_soft_register(rdev,
+                                       SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min,
+                                       radeon_crtc->wm_high / radeon_crtc->line_time) != PPSMC_Result_OK)
+                return 0;
+
+        if (si_write_smc_soft_register(rdev,
+                                       SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max,
+                                       radeon_crtc->wm_low / radeon_crtc->line_time) != PPSMC_Result_OK)
+                return 0;
+
+        return 0;
+}
+
+static int si_set_mc_special_registers(struct radeon_device *rdev,
+                                       struct si_mc_reg_table *table)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        u8 i, j, k;
+        u32 temp_reg;
+
+        for (i = 0, j = table->last; i < table->last; i++) {
+                if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+                        return -EINVAL;
+                switch (table->mc_reg_address[i].s1 << 2) {
+                case MC_SEQ_MISC1:
+                        temp_reg = RREG32(MC_PMG_CMD_EMRS);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++)
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        ((temp_reg & 0xffff0000)) |
+                                        ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+                        j++;
+                        if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+
+                        temp_reg = RREG32(MC_PMG_CMD_MRS);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+                        for (k = 0; k < table->num_entries; k++) {
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        (temp_reg & 0xffff0000) |
+                                        (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+                                if (!pi->mem_gddr5)
+                                        table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+                        }
+                        j++;
+                        if (j > SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+
+                        if (!pi->mem_gddr5) {
+                                table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD >> 2;
+                                table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD >> 2;
+                                for (k = 0; k < table->num_entries; k++)
+                                        table->mc_reg_table_entry[k].mc_data[j] =
+                                                (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
+                                j++;
+                                if (j > SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                        return -EINVAL;
+                        }
+                        break;
+                case MC_SEQ_RESERVE_M:
+                        temp_reg = RREG32(MC_PMG_CMD_MRS1);
+                        table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
+                        table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+                        for(k = 0; k < table->num_entries; k++)
+                                table->mc_reg_table_entry[k].mc_data[j] =
+                                        (temp_reg & 0xffff0000) |
+                                        (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+                        j++;
+                        if (j > SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                return -EINVAL;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        table->last = j;
+
+        return 0;
+}
+
+static bool si_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
+{
+        bool result = true;
+
+        switch (in_reg) {
+        case  MC_SEQ_RAS_TIMING >> 2:
+                *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_CAS_TIMING >> 2:
+                *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_MISC_TIMING >> 2:
+                *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
+                break;
+        case MC_SEQ_MISC_TIMING2 >> 2:
+                *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
+                break;
+        case MC_SEQ_RD_CTL_D0 >> 2:
+                *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
+                break;
+        case MC_SEQ_RD_CTL_D1 >> 2:
+                *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_D0 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_D1 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
+                break;
+        case MC_PMG_CMD_EMRS >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS1 >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
+                break;
+        case MC_SEQ_PMG_TIMING >> 2:
+                *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
+                break;
+        case MC_PMG_CMD_MRS2 >> 2:
+                *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
+                break;
+        case MC_SEQ_WR_CTL_2 >> 2:
+                *out_reg = MC_SEQ_WR_CTL_2_LP >> 2;
+                break;
+        default:
+                result = false;
+                break;
+        }
+
+        return result;
+}
+
+static void si_set_valid_flag(struct si_mc_reg_table *table)
+{
+        u8 i, j;
+
+        for (i = 0; i < table->last; i++) {
+                for (j = 1; j < table->num_entries; j++) {
+                        if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
+                                table->valid_flag |= 1 << i;
+                                break;
+                        }
+                }
+        }
+}
+
+static void si_set_s0_mc_reg_index(struct si_mc_reg_table *table)
+{
+        u32 i;
+        u16 address;
+
+        for (i = 0; i < table->last; i++)
+                table->mc_reg_address[i].s0 = si_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
+                        address : table->mc_reg_address[i].s1;
+
+}
+
+static int si_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
+                                      struct si_mc_reg_table *si_table)
+{
+        u8 i, j;
+
+        if (table->last > SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+                return -EINVAL;
+        if (table->num_entries > MAX_AC_TIMING_ENTRIES)
+                return -EINVAL;
+
+        for (i = 0; i < table->last; i++)
+                si_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+        si_table->last = table->last;
+
+        for (i = 0; i < table->num_entries; i++) {
+                si_table->mc_reg_table_entry[i].mclk_max =
+                        table->mc_reg_table_entry[i].mclk_max;
+                for (j = 0; j < table->last; j++) {
+                        si_table->mc_reg_table_entry[i].mc_data[j] =
+                                table->mc_reg_table_entry[i].mc_data[j];
+                }
+        }
+        si_table->num_entries = table->num_entries;
+
+        return 0;
+}
+
+static int si_initialize_mc_reg_table(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct atom_mc_reg_table *table;
+        struct si_mc_reg_table *si_table = &si_pi->mc_reg_table;
+        u8 module_index = rv770_get_memory_module_index(rdev);
+        int ret;
+
+        table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
+        if (!table)
+                return -ENOMEM;
+
+        WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
+        WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
+        WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
+        WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
+        WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
+        WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
+        WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
+        WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
+        WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
+        WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
+        WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
+        WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
+        WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
+        WREG32(MC_SEQ_WR_CTL_2_LP, RREG32(MC_SEQ_WR_CTL_2));
+
+        ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
+        if (ret)
+                goto init_mc_done;
+
+        ret = si_copy_vbios_mc_reg_table(table, si_table);
+        if (ret)
+                goto init_mc_done;
+
+        si_set_s0_mc_reg_index(si_table);
+
+        ret = si_set_mc_special_registers(rdev, si_table);
+        if (ret)
+                goto init_mc_done;
+
+        si_set_valid_flag(si_table);
+
+init_mc_done:
+        kfree(table);
+
+        return ret;
+
+}
+
+static void si_populate_mc_reg_addresses(struct radeon_device *rdev,
+                                         SMC_SIslands_MCRegisters *mc_reg_table)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 i, j;
+
+        for (i = 0, j = 0; j < si_pi->mc_reg_table.last; j++) {
+                if (si_pi->mc_reg_table.valid_flag & (1 << j)) {
+                        if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
+                                break;
+                        mc_reg_table->address[i].s0 =
+                                cpu_to_be16(si_pi->mc_reg_table.mc_reg_address[j].s0);
+                        mc_reg_table->address[i].s1 =
+                                cpu_to_be16(si_pi->mc_reg_table.mc_reg_address[j].s1);
+                        i++;
+                }
+        }
+        mc_reg_table->last = (u8)i;
+}
+
+static void si_convert_mc_registers(const struct si_mc_reg_entry *entry,
+                                    SMC_SIslands_MCRegisterSet *data,
+                                    u32 num_entries, u32 valid_flag)
+{
+        u32 i, j;
+
+        for(i = 0, j = 0; j < num_entries; j++) {
+                if (valid_flag & (1 << j)) {
+                        data->value[i] = cpu_to_be32(entry->mc_data[j]);
+                        i++;
+                }
+        }
+}
+
+static void si_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
+                                                 struct rv7xx_pl *pl,
+                                                 SMC_SIslands_MCRegisterSet *mc_reg_table_data)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 i = 0;
+
+        for (i = 0; i < si_pi->mc_reg_table.num_entries; i++) {
+                if (pl->mclk <= si_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
+                        break;
+        }
+
+        if ((i == si_pi->mc_reg_table.num_entries) && (i > 0))
+                --i;
+
+        si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[i],
+                                mc_reg_table_data, si_pi->mc_reg_table.last,
+                                si_pi->mc_reg_table.valid_flag);
+}
+
+static void si_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
+                                           struct radeon_ps *radeon_state,
+                                           SMC_SIslands_MCRegisters *mc_reg_table)
+{
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        int i;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                si_convert_mc_reg_table_entry_to_smc(rdev,
+                                                     &state->performance_levels[i],
+                                                     &mc_reg_table->data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
+        }
+}
+
+static int si_populate_mc_reg_table(struct radeon_device *rdev,
+                                    struct radeon_ps *radeon_boot_state)
+{
+        struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct si_ulv_param *ulv = &si_pi->ulv;
+        SMC_SIslands_MCRegisters *smc_mc_reg_table = &si_pi->smc_mc_reg_table;
+
+        memset(smc_mc_reg_table, 0, sizeof(SMC_SIslands_MCRegisters));
+
+        si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_seq_index, 1);
+
+        si_populate_mc_reg_addresses(rdev, smc_mc_reg_table);
+
+        si_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
+                                             &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_INITIAL_SLOT]);
+
+        si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[0],
+                                &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ACPI_SLOT],
+                                si_pi->mc_reg_table.last,
+                                si_pi->mc_reg_table.valid_flag);
+
+        if (ulv->supported && ulv->pl.vddc != 0)
+                si_convert_mc_reg_table_entry_to_smc(rdev, &ulv->pl,
+                                                     &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ULV_SLOT]);
+        else
+                si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[0],
+                                        &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ULV_SLOT],
+                                        si_pi->mc_reg_table.last,
+                                        si_pi->mc_reg_table.valid_flag);
+
+        si_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, smc_mc_reg_table);
+
+        return si_copy_bytes_to_smc(rdev, si_pi->mc_reg_table_start,
+                                    (u8 *)smc_mc_reg_table,
+                                    sizeof(SMC_SIslands_MCRegisters), si_pi->sram_end);
+}
+
+static int si_upload_mc_reg_table(struct radeon_device *rdev,
+                                  struct radeon_ps *radeon_new_state)
+{
+        struct ni_ps *new_state = ni_get_ps(radeon_new_state);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        u32 address = si_pi->mc_reg_table_start +
+                offsetof(SMC_SIslands_MCRegisters,
+                         data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
+        SMC_SIslands_MCRegisters *smc_mc_reg_table = &si_pi->smc_mc_reg_table;
+
+        memset(smc_mc_reg_table, 0, sizeof(SMC_SIslands_MCRegisters));
+
+        si_convert_mc_reg_table_to_smc(rdev, radeon_new_state, smc_mc_reg_table);
+
+
+        return si_copy_bytes_to_smc(rdev, address,
+                                    (u8 *)&smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
+                                    sizeof(SMC_SIslands_MCRegisterSet) * new_state->performance_level_count,
+                                    si_pi->sram_end);
+
+}
+
+static void si_enable_voltage_control(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
+        else
+                WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
+}
+
+static enum radeon_pcie_gen si_get_maximum_link_speed(struct radeon_device *rdev,
+                                                      struct radeon_ps *radeon_state)
+{
+        struct ni_ps *state = ni_get_ps(radeon_state);
+        int i;
+        u16 pcie_speed, max_speed = 0;
+
+        for (i = 0; i < state->performance_level_count; i++) {
+                pcie_speed = state->performance_levels[i].pcie_gen;
+                if (max_speed < pcie_speed)
+                        max_speed = pcie_speed;
+        }
+        return max_speed;
+}
+
+static u16 si_get_current_pcie_speed(struct radeon_device *rdev)
+{
+        u32 speed_cntl;
+
+        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE_MASK;
+        speed_cntl >>= LC_CURRENT_DATA_RATE_SHIFT;
+
+        return (u16)speed_cntl;
+}
+
+static void si_request_link_speed_change_before_state_change(struct radeon_device *rdev,
+                                                             struct radeon_ps *radeon_new_state,
+                                                             struct radeon_ps *radeon_current_state)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        enum radeon_pcie_gen target_link_speed = si_get_maximum_link_speed(rdev, radeon_new_state);
+        enum radeon_pcie_gen current_link_speed;
+
+        if (si_pi->force_pcie_gen == RADEON_PCIE_GEN_INVALID)
+                current_link_speed = si_get_maximum_link_speed(rdev, radeon_current_state);
+        else
+                current_link_speed = si_pi->force_pcie_gen;
+
+        si_pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
+        si_pi->pspp_notify_required = false;
+        if (target_link_speed > current_link_speed) {
+                switch (target_link_speed) {
+#if defined(CONFIG_ACPI)
+                case RADEON_PCIE_GEN3:
+                        if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN3, false) == 0)
+                                break;
+                        si_pi->force_pcie_gen = RADEON_PCIE_GEN2;
+                        if (current_link_speed == RADEON_PCIE_GEN2)
+                                break;
+                case RADEON_PCIE_GEN2:
+                        if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0)
+                                break;
+#endif
+                default:
+                        si_pi->force_pcie_gen = si_get_current_pcie_speed(rdev);
+                        break;
+                }
+        } else {
+                if (target_link_speed < current_link_speed)
+                        si_pi->pspp_notify_required = true;
+        }
+}
+
+static void si_notify_link_speed_change_after_state_change(struct radeon_device *rdev,
+                                                           struct radeon_ps *radeon_new_state,
+                                                           struct radeon_ps *radeon_current_state)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        enum radeon_pcie_gen target_link_speed = si_get_maximum_link_speed(rdev, radeon_new_state);
+        u8 request;
+
+        if (si_pi->pspp_notify_required) {
+                if (target_link_speed == RADEON_PCIE_GEN3)
+                        request = PCIE_PERF_REQ_PECI_GEN3;
+                else if (target_link_speed == RADEON_PCIE_GEN2)
+                        request = PCIE_PERF_REQ_PECI_GEN2;
+                else
+                        request = PCIE_PERF_REQ_PECI_GEN1;
+
+                if ((request == PCIE_PERF_REQ_PECI_GEN1) &&
+                    (si_get_current_pcie_speed(rdev) > 0))
+                        return;
+
+#if defined(CONFIG_ACPI)
+                radeon_acpi_pcie_performance_request(rdev, request, false);
+#endif
+        }
+}
+
+#if 0
+static int si_ds_request(struct radeon_device *rdev,
+                         bool ds_status_on, u32 count_write)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+
+        if (eg_pi->sclk_deep_sleep) {
+                if (ds_status_on)
+                        return (si_send_msg_to_smc(rdev, PPSMC_MSG_CancelThrottleOVRDSCLKDS) ==
+                                PPSMC_Result_OK) ?
+                                0 : -EINVAL;
+                else
+                        return (si_send_msg_to_smc(rdev, PPSMC_MSG_ThrottleOVRDSCLKDS) ==
+                                PPSMC_Result_OK) ? 0 : -EINVAL;
+        }
+        return 0;
+}
+#endif
+
+static void si_set_max_cu_value(struct radeon_device *rdev)
+{
+        struct si_power_info *si_pi = si_get_pi(rdev);
+
+        if (rdev->family == CHIP_VERDE) {
+                switch (rdev->pdev->device) {
+                case 0x6820:
+                case 0x6825:
+                case 0x6821:
+                case 0x6823:
+                case 0x6827:
+                        si_pi->max_cu = 10;
+                        break;
+                case 0x682D:
+                case 0x6824:
+                case 0x682F:
+                case 0x6826:
+                        si_pi->max_cu = 8;
+                        break;
+                case 0x6828:
+                case 0x6830:
+                case 0x6831:
+                case 0x6838:
+                case 0x6839:
+                case 0x683D:
+                        si_pi->max_cu = 10;
+                        break;
+                case 0x683B:
+                case 0x683F:
+                case 0x6829:
+                        si_pi->max_cu = 8;
+                        break;
+                default:
+                        si_pi->max_cu = 0;
+                        break;
+                }
+        } else {
+                si_pi->max_cu = 0;
+        }
+}
+
+static int si_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
+                                                             struct radeon_clock_voltage_dependency_table *table)
+{
+        u32 i;
+        int j;
+        u16 leakage_voltage;
+
+        if (table) {
+                for (i = 0; i < table->count; i++) {
+                        switch (si_get_leakage_voltage_from_leakage_index(rdev,
+                                                                          table->entries[i].v,
+                                                                          &leakage_voltage)) {
+                        case 0:
+                                table->entries[i].v = leakage_voltage;
+                                break;
+                        case -EAGAIN:
+                                return -EINVAL;
+                        case -EINVAL:
+                        default:
+                                break;
+                        }
+                }
+
+                for (j = (table->count - 2); j >= 0; j--) {
+                        table->entries[j].v = (table->entries[j].v <= table->entries[j + 1].v) ?
+                                table->entries[j].v : table->entries[j + 1].v;
+                }
+        }
+        return 0;
+}
+
+static int si_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
+{
+        int ret = 0;
+
+        ret = si_patch_single_dependency_table_based_on_leakage(rdev,
+                                                                &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
+        ret = si_patch_single_dependency_table_based_on_leakage(rdev,
+                                                                &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
+        ret = si_patch_single_dependency_table_based_on_leakage(rdev,
+                                                                &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk);
+        return ret;
+}
+
+static void si_set_pcie_lane_width_in_smc(struct radeon_device *rdev,
+                                          struct radeon_ps *radeon_new_state,
+                                          struct radeon_ps *radeon_current_state)
+{
+        u32 lane_width;
+        u32 new_lane_width =
+                (radeon_new_state->caps & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >> ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT;
+        u32 current_lane_width =
+                (radeon_current_state->caps & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >> ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT;
+
+        if (new_lane_width != current_lane_width) {
+                radeon_set_pcie_lanes(rdev, new_lane_width);
+                lane_width = radeon_get_pcie_lanes(rdev);
+                si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width, lane_width);
+        }
+}
+
+void si_dpm_setup_asic(struct radeon_device *rdev)
+{
+        rv770_get_memory_type(rdev);
+        si_read_clock_registers(rdev);
+        si_enable_acpi_power_management(rdev);
+}
+
+static int si_set_thermal_temperature_range(struct radeon_device *rdev,
+                                        int min_temp, int max_temp)
+{
+        int low_temp = 0 * 1000;
+        int high_temp = 255 * 1000;
+
+        if (low_temp < min_temp)
+                low_temp = min_temp;
+        if (high_temp > max_temp)
+                high_temp = max_temp;
+        if (high_temp < low_temp) {
+                DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
+                return -EINVAL;
+        }
+
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
+        WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
+
+        rdev->pm.dpm.thermal.min_temp = low_temp;
+        rdev->pm.dpm.thermal.max_temp = high_temp;
+
+        return 0;
+}
+
+int si_dpm_enable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+        int ret;
+
+        if (si_is_smc_running(rdev))
+                return -EINVAL;
+        if (pi->voltage_control)
+                si_enable_voltage_control(rdev, true);
+        if (pi->mvdd_control)
+                si_get_mvdd_configuration(rdev);
+        if (pi->voltage_control) {
+                ret = si_construct_voltage_tables(rdev);
+                if (ret) {
+                        DRM_ERROR("si_construct_voltage_tables failed\n");
+                        return ret;
+                }
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = si_initialize_mc_reg_table(rdev);
+                if (ret)
+                        eg_pi->dynamic_ac_timing = false;
+        }
+        if (pi->dynamic_ss)
+                si_enable_spread_spectrum(rdev, true);
+        if (pi->thermal_protection)
+                si_enable_thermal_protection(rdev, true);
+        si_setup_bsp(rdev);
+        si_program_git(rdev);
+        si_program_tp(rdev);
+        si_program_tpp(rdev);
+        si_program_sstp(rdev);
+        si_enable_display_gap(rdev);
+        si_program_vc(rdev);
+        ret = si_upload_firmware(rdev);
+        if (ret) {
+                DRM_ERROR("si_upload_firmware failed\n");
+                return ret;
+        }
+        ret = si_process_firmware_header(rdev);
+        if (ret) {
+                DRM_ERROR("si_process_firmware_header failed\n");
+                return ret;
+        }
+        ret = si_initial_switch_from_arb_f0_to_f1(rdev);
+        if (ret) {
+                DRM_ERROR("si_initial_switch_from_arb_f0_to_f1 failed\n");
+                return ret;
+        }
+        ret = si_init_smc_table(rdev);
+        if (ret) {
+                DRM_ERROR("si_init_smc_table failed\n");
+                return ret;
+        }
+        ret = si_init_smc_spll_table(rdev);
+        if (ret) {
+                DRM_ERROR("si_init_smc_spll_table failed\n");
+                return ret;
+        }
+        ret = si_init_arb_table_index(rdev);
+        if (ret) {
+                DRM_ERROR("si_init_arb_table_index failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = si_populate_mc_reg_table(rdev, boot_ps);
+                if (ret) {
+                        DRM_ERROR("si_populate_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+        ret = si_initialize_smc_cac_tables(rdev);
+        if (ret) {
+                DRM_ERROR("si_initialize_smc_cac_tables failed\n");
+                return ret;
+        }
+        ret = si_initialize_hardware_cac_manager(rdev);
+        if (ret) {
+                DRM_ERROR("si_initialize_hardware_cac_manager failed\n");
+                return ret;
+        }
+        ret = si_initialize_smc_dte_tables(rdev);
+        if (ret) {
+                DRM_ERROR("si_initialize_smc_dte_tables failed\n");
+                return ret;
+        }
+        ret = si_populate_smc_tdp_limits(rdev, boot_ps);
+        if (ret) {
+                DRM_ERROR("si_populate_smc_tdp_limits failed\n");
+                return ret;
+        }
+        ret = si_populate_smc_tdp_limits_2(rdev, boot_ps);
+        if (ret) {
+                DRM_ERROR("si_populate_smc_tdp_limits_2 failed\n");
+                return ret;
+        }
+        si_program_response_times(rdev);
+        si_program_ds_registers(rdev);
+        si_dpm_start_smc(rdev);
+        ret = si_notify_smc_display_change(rdev, false);
+        if (ret) {
+                DRM_ERROR("si_notify_smc_display_change failed\n");
+                return ret;
+        }
+        si_enable_sclk_control(rdev, true);
+        si_start_dpm(rdev);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                PPSMC_Result result;
+
+                ret = si_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+                result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
+
+                if (result != PPSMC_Result_OK)
+                        DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
+        }
+
+        si_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+
+        ni_update_current_ps(rdev, boot_ps);
+
+        return 0;
+}
+
+void si_dpm_disable(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
+
+        if (!si_is_smc_running(rdev))
+                return;
+        si_disable_ulv(rdev);
+        si_clear_vc(rdev);
+        if (pi->thermal_protection)
+                si_enable_thermal_protection(rdev, false);
+        si_enable_power_containment(rdev, boot_ps, false);
+        si_enable_smc_cac(rdev, boot_ps, false);
+        si_enable_spread_spectrum(rdev, false);
+        si_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
+        si_stop_dpm(rdev);
+        si_reset_to_default(rdev);
+        si_dpm_stop_smc(rdev);
+        si_force_switch_to_arb_f0(rdev);
+
+        ni_update_current_ps(rdev, boot_ps);
+}
+
+int si_dpm_pre_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
+        struct radeon_ps *new_ps = &requested_ps;
+
+        ni_update_requested_ps(rdev, new_ps);
+
+        si_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
+
+        return 0;
+}
+
+static int si_power_control_set_level(struct radeon_device *rdev)
+{
+        struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
+        int ret;
+
+        ret = si_restrict_performance_levels_before_switch(rdev);
+        if (ret)
+                return ret;
+        ret = si_halt_smc(rdev);
+        if (ret)
+                return ret;
+        ret = si_populate_smc_tdp_limits(rdev, new_ps);
+        if (ret)
+                return ret;
+        ret = si_populate_smc_tdp_limits_2(rdev, new_ps);
+        if (ret)
+                return ret;
+        ret = si_resume_smc(rdev);
+        if (ret)
+                return ret;
+        ret = si_set_sw_state(rdev);
+        if (ret)
+                return ret;
+        return 0;
+}
+
+int si_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = &eg_pi->requested_rps;
+        struct radeon_ps *old_ps = &eg_pi->current_rps;
+        int ret;
+
+        ret = si_disable_ulv(rdev);
+        if (ret) {
+                DRM_ERROR("si_disable_ulv failed\n");
+                return ret;
+        }
+        ret = si_restrict_performance_levels_before_switch(rdev);
+        if (ret) {
+                DRM_ERROR("si_restrict_performance_levels_before_switch failed\n");
+                return ret;
+        }
+        if (eg_pi->pcie_performance_request)
+                si_request_link_speed_change_before_state_change(rdev, new_ps, old_ps);
+        ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+        ret = si_enable_power_containment(rdev, new_ps, false);
+        if (ret) {
+                DRM_ERROR("si_enable_power_containment failed\n");
+                return ret;
+        }
+        ret = si_enable_smc_cac(rdev, new_ps, false);
+        if (ret) {
+                DRM_ERROR("si_enable_smc_cac failed\n");
+                return ret;
+        }
+        ret = si_halt_smc(rdev);
+        if (ret) {
+                DRM_ERROR("si_halt_smc failed\n");
+                return ret;
+        }
+        ret = si_upload_sw_state(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("si_upload_sw_state failed\n");
+                return ret;
+        }
+        ret = si_upload_smc_data(rdev);
+        if (ret) {
+                DRM_ERROR("si_upload_smc_data failed\n");
+                return ret;
+        }
+        ret = si_upload_ulv_state(rdev);
+        if (ret) {
+                DRM_ERROR("si_upload_ulv_state failed\n");
+                return ret;
+        }
+        if (eg_pi->dynamic_ac_timing) {
+                ret = si_upload_mc_reg_table(rdev, new_ps);
+                if (ret) {
+                        DRM_ERROR("si_upload_mc_reg_table failed\n");
+                        return ret;
+                }
+        }
+        ret = si_program_memory_timing_parameters(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("si_program_memory_timing_parameters failed\n");
+                return ret;
+        }
+        si_set_pcie_lane_width_in_smc(rdev, new_ps, old_ps);
+
+        ret = si_resume_smc(rdev);
+        if (ret) {
+                DRM_ERROR("si_resume_smc failed\n");
+                return ret;
+        }
+        ret = si_set_sw_state(rdev);
+        if (ret) {
+                DRM_ERROR("si_set_sw_state failed\n");
+                return ret;
+        }
+        ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+        if (eg_pi->pcie_performance_request)
+                si_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps);
+        ret = si_set_power_state_conditionally_enable_ulv(rdev, new_ps);
+        if (ret) {
+                DRM_ERROR("si_set_power_state_conditionally_enable_ulv failed\n");
+                return ret;
+        }
+        ret = si_enable_smc_cac(rdev, new_ps, true);
+        if (ret) {
+                DRM_ERROR("si_enable_smc_cac failed\n");
+                return ret;
+        }
+        ret = si_enable_power_containment(rdev, new_ps, true);
+        if (ret) {
+                DRM_ERROR("si_enable_power_containment failed\n");
+                return ret;
+        }
+
+        ret = si_power_control_set_level(rdev);
+        if (ret) {
+                DRM_ERROR("si_power_control_set_level failed\n");
+                return ret;
+        }
+
+#if 0
+        /* XXX */
+        ret = si_unrestrict_performance_levels_after_switch(rdev);
+        if (ret) {
+                DRM_ERROR("si_unrestrict_performance_levels_after_switch failed\n");
+                return ret;
+        }
+#endif
+
+        return 0;
+}
+
+void si_dpm_post_set_power_state(struct radeon_device *rdev)
+{
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct radeon_ps *new_ps = &eg_pi->requested_rps;
+
+        ni_update_current_ps(rdev, new_ps);
+}
+
+
+void si_dpm_reset_asic(struct radeon_device *rdev)
+{
+        si_restrict_performance_levels_before_switch(rdev);
+        si_disable_ulv(rdev);
+        si_set_boot_state(rdev);
+}
+
+void si_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+        si_program_display_gap(rdev);
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+        struct _ATOM_PPLIB_SI_CLOCK_INFO si;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void si_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                          struct radeon_ps *rps,
+                                          struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+                                          u8 table_rev)
+{
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+        } else if (r600_is_uvd_state(rps->class, rps->class2)) {
+                rps->vclk = RV770_DEFAULT_VCLK_FREQ;
+                rps->dclk = RV770_DEFAULT_DCLK_FREQ;
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+                rdev->pm.dpm.boot_ps = rps;
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void si_parse_pplib_clock_info(struct radeon_device *rdev,
+                                      struct radeon_ps *rps, int index,
+                                      union pplib_clock_info *clock_info)
+{
+        struct rv7xx_power_info *pi = rv770_get_pi(rdev);
+        struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
+        struct si_power_info *si_pi = si_get_pi(rdev);
+        struct ni_ps *ps = ni_get_ps(rps);
+        u16 leakage_voltage;
+        struct rv7xx_pl *pl = &ps->performance_levels[index];
+        int ret;
+
+        ps->performance_level_count = index + 1;
+
+        pl->sclk = le16_to_cpu(clock_info->si.usEngineClockLow);
+        pl->sclk |= clock_info->si.ucEngineClockHigh << 16;
+        pl->mclk = le16_to_cpu(clock_info->si.usMemoryClockLow);
+        pl->mclk |= clock_info->si.ucMemoryClockHigh << 16;
+
+        pl->vddc = le16_to_cpu(clock_info->si.usVDDC);
+        pl->vddci = le16_to_cpu(clock_info->si.usVDDCI);
+        pl->flags = le32_to_cpu(clock_info->si.ulFlags);
+        pl->pcie_gen = r600_get_pcie_gen_support(rdev,
+                                                 si_pi->sys_pcie_mask,
+                                                 si_pi->boot_pcie_gen,
+                                                 clock_info->si.ucPCIEGen);
+
+        /* patch up vddc if necessary */
+        ret = si_get_leakage_voltage_from_leakage_index(rdev, pl->vddc,
+                                                        &leakage_voltage);
+        if (ret == 0)
+                pl->vddc = leakage_voltage;
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
+                pi->acpi_vddc = pl->vddc;
+                eg_pi->acpi_vddci = pl->vddci;
+                si_pi->acpi_pcie_gen = pl->pcie_gen;
+        }
+
+        if ((rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) &&
+            index == 0) {
+                /* XXX disable for A0 tahiti */
+                si_pi->ulv.supported = true;
+                si_pi->ulv.pl = *pl;
+                si_pi->ulv.one_pcie_lane_in_ulv = false;
+                si_pi->ulv.volt_change_delay = SISLANDS_ULVVOLTAGECHANGEDELAY_DFLT;
+                si_pi->ulv.cg_ulv_parameter = SISLANDS_CGULVPARAMETER_DFLT;
+                si_pi->ulv.cg_ulv_control = SISLANDS_CGULVCONTROL_DFLT;
+        }
+
+        if (pi->min_vddc_in_table > pl->vddc)
+                pi->min_vddc_in_table = pl->vddc;
+
+        if (pi->max_vddc_in_table < pl->vddc)
+                pi->max_vddc_in_table = pl->vddc;
+
+        /* patch up boot state */
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                u16 vddc, vddci, mvdd;
+                radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
+                pl->mclk = rdev->clock.default_mclk;
+                pl->sclk = rdev->clock.default_sclk;
+                pl->vddc = vddc;
+                pl->vddci = vddci;
+                si_pi->mvdd_bootup_value = mvdd;
+        }
+
+        if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
+            ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
+                rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
+        }
+}
+
+static int si_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j, k, non_clock_array_index, clock_array_index;
+        union pplib_clock_info *clock_info;
+        struct _StateArray *state_array;
+        struct _ClockInfoArray *clock_info_array;
+        struct _NonClockInfoArray *non_clock_info_array;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        u8 *power_state_offset;
+        struct ni_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        state_array = (struct _StateArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usStateArrayOffset));
+        clock_info_array = (struct _ClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
+        non_clock_info_array = (struct _NonClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  state_array->ucNumEntries, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        power_state_offset = (u8 *)state_array->states;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+        for (i = 0; i < state_array->ucNumEntries; i++) {
+                power_state = (union pplib_power_state *)power_state_offset;
+                non_clock_array_index = power_state->v2.nonClockInfoIndex;
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        &non_clock_info_array->nonClockInfo[non_clock_array_index];
+                if (!rdev->pm.power_state[i].clock_info)
+                        return -EINVAL;
+                ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
+                if (ps == NULL) {
+                        kfree(rdev->pm.dpm.ps);
+                        return -ENOMEM;
+                }
+                rdev->pm.dpm.ps[i].ps_priv = ps;
+                si_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                              non_clock_info,
+                                              non_clock_info_array->ucEntrySize);
+                k = 0;
+                for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
+                        clock_array_index = power_state->v2.clockInfoIndex[j];
+                        if (clock_array_index >= clock_info_array->ucNumEntries)
+                                continue;
+                        if (k >= SISLANDS_MAX_HARDWARE_POWERLEVELS)
+                                break;
+                        clock_info = (union pplib_clock_info *)
+                                &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
+                        si_parse_pplib_clock_info(rdev,
+                                                  &rdev->pm.dpm.ps[i], k,
+                                                  clock_info);
+                        k++;
+                }
+                power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
+        }
+        rdev->pm.dpm.num_ps = state_array->ucNumEntries;
+        return 0;
+}
+
+int si_dpm_init(struct radeon_device *rdev)
+{
+        struct rv7xx_power_info *pi;
+        struct evergreen_power_info *eg_pi;
+        struct ni_power_info *ni_pi;
+        struct si_power_info *si_pi;
+        int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
+        u16 data_offset, size;
+        u8 frev, crev;
+        struct atom_clock_dividers dividers;
+        int ret;
+        u32 mask;
+
+        si_pi = kzalloc(sizeof(struct si_power_info), GFP_KERNEL);
+        if (si_pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = si_pi;
+        ni_pi = &si_pi->ni;
+        eg_pi = &ni_pi->eg;
+        pi = &eg_pi->rv7xx;
+
+        ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
+        if (ret)
+                si_pi->sys_pcie_mask = 0;
+        else
+                si_pi->sys_pcie_mask = mask;
+        si_pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
+        si_pi->boot_pcie_gen = si_get_current_pcie_speed(rdev);
+
+        si_set_max_cu_value(rdev);
+
+        rv770_get_max_vddc(rdev);
+        si_get_leakage_vddc(rdev);
+        si_patch_dependency_tables_based_on_leakage(rdev);
+
+        pi->acpi_vddc = 0;
+        eg_pi->acpi_vddci = 0;
+        pi->min_vddc_in_table = 0;
+        pi->max_vddc_in_table = 0;
+
+        ret = si_parse_power_table(rdev);
+        if (ret)
+                return ret;
+        ret = r600_parse_extended_power_table(rdev);
+        if (ret)
+                return ret;
+
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
+                kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
+        if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
+                r600_free_extended_power_table(rdev);
+                return -ENOMEM;
+        }
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
+        rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
+
+        if (rdev->pm.dpm.voltage_response_time == 0)
+                rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+        if (rdev->pm.dpm.backbias_response_time == 0)
+                rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             0, false, &dividers);
+        if (ret)
+                pi->ref_div = dividers.ref_div + 1;
+        else
+                pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+        eg_pi->smu_uvd_hs = false;
+
+        pi->mclk_strobe_mode_threshold = 40000;
+        if (si_is_special_1gb_platform(rdev))
+                pi->mclk_stutter_mode_threshold = 0;
+        else
+                pi->mclk_stutter_mode_threshold = pi->mclk_strobe_mode_threshold;
+        pi->mclk_edc_enable_threshold = 40000;
+        eg_pi->mclk_edc_wr_enable_threshold = 40000;
+
+        ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
+
+        pi->voltage_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, VOLTAGE_OBJ_GPIO_LUT);
+
+        pi->mvdd_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, VOLTAGE_OBJ_GPIO_LUT);
+
+        eg_pi->vddci_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, VOLTAGE_OBJ_GPIO_LUT);
+
+        si_pi->vddc_phase_shed_control =
+                radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, VOLTAGE_OBJ_PHASE_LUT);
+
+        if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
+                                   &frev, &crev, &data_offset)) {
+                pi->sclk_ss = true;
+                pi->mclk_ss = true;
+                pi->dynamic_ss = true;
+        } else {
+                pi->sclk_ss = false;
+                pi->mclk_ss = false;
+                pi->dynamic_ss = true;
+        }
+
+        pi->asi = RV770_ASI_DFLT;
+        pi->pasi = CYPRESS_HASI_DFLT;
+        pi->vrc = SISLANDS_VRC_DFLT;
+
+        pi->gfx_clock_gating = true;
+
+        eg_pi->sclk_deep_sleep = true;
+        si_pi->sclk_deep_sleep_above_low = false;
+
+        if (pi->gfx_clock_gating &&
+            (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
+                pi->thermal_protection = true;
+        else
+                pi->thermal_protection = false;
+
+        eg_pi->dynamic_ac_timing = true;
+
+        eg_pi->light_sleep = true;
+#if defined(CONFIG_ACPI)
+        eg_pi->pcie_performance_request =
+                radeon_acpi_is_pcie_performance_request_supported(rdev);
+#else
+        eg_pi->pcie_performance_request = false;
+#endif
+
+        si_pi->sram_end = SMC_RAM_END;
+
+        rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4;
+        rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000;
+        rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
+        rdev->pm.dpm.dyn_state.valid_sclk_values.count = 0;
+        rdev->pm.dpm.dyn_state.valid_sclk_values.values = NULL;
+        rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
+        rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
+
+        si_initialize_powertune_defaults(rdev);
+
+        return 0;
+}
+
+void si_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+        kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
+        r600_free_extended_power_table(rdev);
+}
+
+void si_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                    struct seq_file *m)
+{
+        struct radeon_ps *rps = rdev->pm.dpm.current_ps;
+        struct ni_ps *ps = ni_get_ps(rps);
+        struct rv7xx_pl *pl;
+        u32 current_index =
+                (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
+                CURRENT_STATE_INDEX_SHIFT;
+
+        if (current_index >= ps->performance_level_count) {
+                seq_printf(m, "invalid dpm profile %d\n", current_index);
+        } else {
+                pl = &ps->performance_levels[current_index];
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
+                           current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
+        }
+}
diff --git a/drivers/gpu/drm/radeon/si_dpm.h b/drivers/gpu/drm/radeon/si_dpm.h
new file mode 100644
index 000000000000..4ce5032cdf49
--- /dev/null
+++ b/drivers/gpu/drm/radeon/si_dpm.h
@@ -0,0 +1,227 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SI_DPM_H__
+#define __SI_DPM_H__
+
+#include "ni_dpm.h"
+#include "sislands_smc.h"
+
+enum si_cac_config_reg_type
+{
+        SISLANDS_CACCONFIG_MMR = 0,
+        SISLANDS_CACCONFIG_CGIND,
+        SISLANDS_CACCONFIG_MAX
+};
+
+struct si_cac_config_reg
+{
+        u32 offset;
+        u32 mask;
+        u32 shift;
+        u32 value;
+        enum si_cac_config_reg_type type;
+};
+
+struct si_powertune_data
+{
+        u32 cac_window;
+        u32 l2_lta_window_size_default;
+        u8 lts_truncate_default;
+        u8 shift_n_default;
+        u8 operating_temp;
+        struct ni_leakage_coeffients leakage_coefficients;
+        u32 fixed_kt;
+        u32 lkge_lut_v0_percent;
+        u8 dc_cac[NISLANDS_DCCAC_MAX_LEVELS];
+        bool enable_powertune_by_default;
+};
+
+struct si_dyn_powertune_data
+{
+        u32 cac_leakage;
+        s32 leakage_minimum_temperature;
+        u32 wintime;
+        u32 l2_lta_window_size;
+        u8 lts_truncate;
+        u8 shift_n;
+        u8 dc_pwr_value;
+        bool disable_uvd_powertune;
+};
+
+struct si_dte_data
+{
+        u32 tau[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+        u32 r[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+        u32 k;
+        u32 t0;
+        u32 max_t;
+        u8 window_size;
+        u8 temp_select;
+        u8 dte_mode;
+        u8 tdep_count;
+        u8 t_limits[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+        u32 tdep_tau[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+        u32 tdep_r[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+        u32 t_threshold;
+        bool enable_dte_by_default;
+};
+
+struct si_clock_registers {
+        u32 cg_spll_func_cntl;
+        u32 cg_spll_func_cntl_2;
+        u32 cg_spll_func_cntl_3;
+        u32 cg_spll_func_cntl_4;
+        u32 cg_spll_spread_spectrum;
+        u32 cg_spll_spread_spectrum_2;
+        u32 dll_cntl;
+        u32 mclk_pwrmgt_cntl;
+        u32 mpll_ad_func_cntl;
+        u32 mpll_dq_func_cntl;
+        u32 mpll_func_cntl;
+        u32 mpll_func_cntl_1;
+        u32 mpll_func_cntl_2;
+        u32 mpll_ss1;
+        u32 mpll_ss2;
+};
+
+struct si_mc_reg_entry {
+        u32 mclk_max;
+        u32 mc_data[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct si_mc_reg_table {
+        u8 last;
+        u8 num_entries;
+        u16 valid_flag;
+        struct si_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+        SMC_NIslands_MCRegisterAddress mc_reg_address[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+#define SISLANDS_MCREGISTERTABLE_INITIAL_SLOT               0
+#define SISLANDS_MCREGISTERTABLE_ACPI_SLOT                  1
+#define SISLANDS_MCREGISTERTABLE_ULV_SLOT                   2
+#define SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT     3
+
+struct si_leakage_voltage_entry
+{
+        u16 voltage;
+        u16 leakage_index;
+};
+
+#define SISLANDS_LEAKAGE_INDEX0     0xff01
+#define SISLANDS_MAX_LEAKAGE_COUNT  4
+
+struct si_leakage_voltage
+{
+        u16 count;
+        struct si_leakage_voltage_entry entries[SISLANDS_MAX_LEAKAGE_COUNT];
+};
+
+#define SISLANDS_MAX_HARDWARE_POWERLEVELS 5
+
+struct si_ulv_param {
+        bool supported;
+        u32 cg_ulv_control;
+        u32 cg_ulv_parameter;
+        u32 volt_change_delay;
+        struct rv7xx_pl pl;
+        bool one_pcie_lane_in_ulv;
+};
+
+struct si_power_info {
+        /* must be first! */
+        struct ni_power_info ni;
+        struct si_clock_registers clock_registers;
+        struct si_mc_reg_table mc_reg_table;
+        struct atom_voltage_table mvdd_voltage_table;
+        struct atom_voltage_table vddc_phase_shed_table;
+        struct si_leakage_voltage leakage_voltage;
+        u16 mvdd_bootup_value;
+        struct si_ulv_param ulv;
+        u32 max_cu;
+        /* pcie gen */
+        enum radeon_pcie_gen force_pcie_gen;
+        enum radeon_pcie_gen boot_pcie_gen;
+        enum radeon_pcie_gen acpi_pcie_gen;
+        u32 sys_pcie_mask;
+        /* flags */
+        bool enable_dte;
+        bool enable_ppm;
+        bool vddc_phase_shed_control;
+        bool pspp_notify_required;
+        bool sclk_deep_sleep_above_low;
+        /* smc offsets */
+        u32 sram_end;
+        u32 state_table_start;
+        u32 soft_regs_start;
+        u32 mc_reg_table_start;
+        u32 arb_table_start;
+        u32 cac_table_start;
+        u32 dte_table_start;
+        u32 spll_table_start;
+        u32 papm_cfg_table_start;
+        /* CAC stuff */
+        const struct si_cac_config_reg *cac_weights;
+        const struct si_cac_config_reg *lcac_config;
+        const struct si_cac_config_reg *cac_override;
+        const struct si_powertune_data *powertune_data;
+        struct si_dyn_powertune_data dyn_powertune_data;
+        /* DTE stuff */
+        struct si_dte_data dte_data;
+        /* scratch structs */
+        SMC_SIslands_MCRegisters smc_mc_reg_table;
+        SISLANDS_SMC_STATETABLE smc_statetable;
+        PP_SIslands_PAPMParameters papm_parm;
+};
+
+#define SISLANDS_INITIAL_STATE_ARB_INDEX    0
+#define SISLANDS_ACPI_STATE_ARB_INDEX       1
+#define SISLANDS_ULV_STATE_ARB_INDEX        2
+#define SISLANDS_DRIVER_STATE_ARB_INDEX     3
+
+#define SISLANDS_DPM2_MAX_PULSE_SKIP        256
+
+#define SISLANDS_DPM2_NEAR_TDP_DEC          10
+#define SISLANDS_DPM2_ABOVE_SAFE_INC        5
+#define SISLANDS_DPM2_BELOW_SAFE_INC        20
+
+#define SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT            80
+
+#define SISLANDS_DPM2_MAXPS_PERCENT_H                   99
+#define SISLANDS_DPM2_MAXPS_PERCENT_M                   99
+
+#define SISLANDS_DPM2_SQ_RAMP_MAX_POWER                 0x3FFF
+#define SISLANDS_DPM2_SQ_RAMP_MIN_POWER                 0x12
+#define SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA           0x15
+#define SISLANDS_DPM2_SQ_RAMP_STI_SIZE                  0x1E
+#define SISLANDS_DPM2_SQ_RAMP_LTI_RATIO                 0xF
+
+#define SISLANDS_DPM2_PWREFFICIENCYRATIO_MARGIN         10
+
+#define SISLANDS_VRC_DFLT                               0xC000B3
+#define SISLANDS_ULVVOLTAGECHANGEDELAY_DFLT             1687
+#define SISLANDS_CGULVPARAMETER_DFLT                    0x00040035
+#define SISLANDS_CGULVCONTROL_DFLT                      0x1f007550
+
+
+#endif
diff --git a/drivers/gpu/drm/radeon/si_smc.c b/drivers/gpu/drm/radeon/si_smc.c
new file mode 100644
index 000000000000..5f524c0a541e
--- /dev/null
+++ b/drivers/gpu/drm/radeon/si_smc.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "radeon.h"
+#include "sid.h"
+#include "ppsmc.h"
+#include "radeon_ucode.h"
+
+int si_set_smc_sram_address(struct radeon_device *rdev,
+                            u32 smc_address, u32 limit)
+{
+        if (smc_address & 3)
+                return -EINVAL;
+        if ((smc_address + 3) > limit)
+                return -EINVAL;
+
+        WREG32(SMC_IND_INDEX_0, smc_address);
+        WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0);
+
+        return 0;
+}
+
+int si_copy_bytes_to_smc(struct radeon_device *rdev,
+                         u32 smc_start_address,
+                         const u8 *src, u32 byte_count, u32 limit)
+{
+        int ret;
+        u32 data, original_data, addr, extra_shift;
+
+        if (smc_start_address & 3)
+                return -EINVAL;
+        if ((smc_start_address + byte_count) > limit)
+                return -EINVAL;
+
+        addr = smc_start_address;
+
+        while (byte_count >= 4) {
+                /* SMC address space is BE */
+                data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3];
+
+                ret = si_set_smc_sram_address(rdev, addr, limit);
+                if (ret)
+                        return ret;
+
+                WREG32(SMC_IND_DATA_0, data);
+
+                src += 4;
+                byte_count -= 4;
+                addr += 4;
+        }
+
+        /* RMW for the final bytes */
+        if (byte_count > 0) {
+                data = 0;
+
+                ret = si_set_smc_sram_address(rdev, addr, limit);
+                if (ret)
+                        return ret;
+
+                original_data = RREG32(SMC_IND_DATA_0);
+
+                extra_shift = 8 * (4 - byte_count);
+
+                while (byte_count > 0) {
+                        /* SMC address space is BE */
+                        data = (data << 8) + *src++;
+                        byte_count--;
+                }
+
+                data <<= extra_shift;
+
+                data |= (original_data & ~((~0UL) << extra_shift));
+
+                ret = si_set_smc_sram_address(rdev, addr, limit);
+                if (ret)
+                        return ret;
+
+                WREG32(SMC_IND_DATA_0, data);
+        }
+        return 0;
+}
+
+void si_start_smc(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL);
+
+        tmp &= ~RST_REG;
+
+        WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp);
+}
+
+void si_reset_smc(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        RREG32(CB_CGTT_SCLK_CTRL);
+        RREG32(CB_CGTT_SCLK_CTRL);
+        RREG32(CB_CGTT_SCLK_CTRL);
+        RREG32(CB_CGTT_SCLK_CTRL);
+
+        tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL);
+        tmp |= RST_REG;
+        WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp);
+}
+
+int si_program_jump_on_start(struct radeon_device *rdev)
+{
+        static u8 data[] = { 0x0E, 0x00, 0x40, 0x40 };
+
+        return si_copy_bytes_to_smc(rdev, 0x0, data, 4, sizeof(data)+1);
+}
+
+void si_stop_smc_clock(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+
+        tmp |= CK_DISABLE;
+
+        WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp);
+}
+
+void si_start_smc_clock(struct radeon_device *rdev)
+{
+        u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+
+        tmp &= ~CK_DISABLE;
+
+        WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp);
+}
+
+bool si_is_smc_running(struct radeon_device *rdev)
+{
+        u32 rst = RREG32_SMC(SMC_SYSCON_RESET_CNTL);
+        u32 clk = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+
+        if (!(rst & RST_REG) && !(clk & CK_DISABLE))
+                return true;
+
+        return false;
+}
+
+PPSMC_Result si_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg)
+{
+        u32 tmp;
+        int i;
+
+        if (!si_is_smc_running(rdev))
+                return PPSMC_Result_Failed;
+
+        WREG32(SMC_MESSAGE_0, msg);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = RREG32(SMC_RESP_0);
+                if (tmp != 0)
+                        break;
+                udelay(1);
+        }
+        tmp = RREG32(SMC_RESP_0);
+
+        return (PPSMC_Result)tmp;
+}
+
+PPSMC_Result si_wait_for_smc_inactive(struct radeon_device *rdev)
+{
+        u32 tmp;
+        int i;
+
+        if (!si_is_smc_running(rdev))
+                return PPSMC_Result_OK;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+                if ((tmp & CKEN) == 0)
+                        break;
+                udelay(1);
+        }
+
+        return PPSMC_Result_OK;
+}
+
+int si_load_smc_ucode(struct radeon_device *rdev, u32 limit)
+{
+        u32 ucode_start_address;
+        u32 ucode_size;
+        const u8 *src;
+        u32 data;
+
+        if (!rdev->smc_fw)
+                return -EINVAL;
+
+        switch (rdev->family) {
+        case CHIP_TAHITI:
+                ucode_start_address = TAHITI_SMC_UCODE_START;
+                ucode_size = TAHITI_SMC_UCODE_SIZE;
+                break;
+        case CHIP_PITCAIRN:
+                ucode_start_address = PITCAIRN_SMC_UCODE_START;
+                ucode_size = PITCAIRN_SMC_UCODE_SIZE;
+                break;
+        case CHIP_VERDE:
+                ucode_start_address = VERDE_SMC_UCODE_START;
+                ucode_size = VERDE_SMC_UCODE_SIZE;
+                break;
+        case CHIP_OLAND:
+                ucode_start_address = OLAND_SMC_UCODE_START;
+                ucode_size = OLAND_SMC_UCODE_SIZE;
+                break;
+        case CHIP_HAINAN:
+                ucode_start_address = HAINAN_SMC_UCODE_START;
+                ucode_size = HAINAN_SMC_UCODE_SIZE;
+                break;
+        default:
+                DRM_ERROR("unknown asic in smc ucode loader\n");
+                BUG();
+        }
+
+        if (ucode_size & 3)
+                return -EINVAL;
+
+        src = (const u8 *)rdev->smc_fw->data;
+        WREG32(SMC_IND_INDEX_0, ucode_start_address);
+        WREG32_P(SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, ~AUTO_INCREMENT_IND_0);
+        while (ucode_size >= 4) {
+                /* SMC address space is BE */
+                data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3];
+
+                WREG32(SMC_IND_DATA_0, data);
+
+                src += 4;
+                ucode_size -= 4;
+        }
+        WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0);
+
+        return 0;
+}
+
+int si_read_smc_sram_dword(struct radeon_device *rdev, u32 smc_address,
+                           u32 *value, u32 limit)
+{
+        int ret;
+
+        ret = si_set_smc_sram_address(rdev, smc_address, limit);
+        if (ret)
+                return ret;
+
+        *value = RREG32(SMC_IND_DATA_0);
+        return 0;
+}
+
+int si_write_smc_sram_dword(struct radeon_device *rdev, u32 smc_address,
+                            u32 value, u32 limit)
+{
+        int ret;
+
+        ret = si_set_smc_sram_address(rdev, smc_address, limit);
+        if (ret)
+                return ret;
+
+        WREG32(SMC_IND_DATA_0, value);
+        return 0;
+}
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index 8f2d7d4f9b28..12a20eb77d0c 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -30,6 +30,94 @@
 #define VERDE_GB_ADDR_CONFIG_GOLDEN         0x12010002
 #define HAINAN_GB_ADDR_CONFIG_GOLDEN        0x02010001
 
+#define SI_MAX_SH_GPRS           256
+#define SI_MAX_TEMP_GPRS         16
+#define SI_MAX_SH_THREADS        256
+#define SI_MAX_SH_STACK_ENTRIES  4096
+#define SI_MAX_FRC_EOV_CNT       16384
+#define SI_MAX_BACKENDS          8
+#define SI_MAX_BACKENDS_MASK     0xFF
+#define SI_MAX_BACKENDS_PER_SE_MASK     0x0F
+#define SI_MAX_SIMDS             12
+#define SI_MAX_SIMDS_MASK        0x0FFF
+#define SI_MAX_SIMDS_PER_SE_MASK        0x00FF
+#define SI_MAX_PIPES             8
+#define SI_MAX_PIPES_MASK        0xFF
+#define SI_MAX_PIPES_PER_SIMD_MASK      0x3F
+#define SI_MAX_LDS_NUM           0xFFFF
+#define SI_MAX_TCC               16
+#define SI_MAX_TCC_MASK          0xFFFF
+
+/* SMC IND accessor regs */
+#define SMC_IND_INDEX_0                              0x200
+#define SMC_IND_DATA_0                               0x204
+
+#define SMC_IND_ACCESS_CNTL                          0x228
+#       define AUTO_INCREMENT_IND_0                  (1 << 0)
+#define SMC_MESSAGE_0                                0x22c
+#define SMC_RESP_0                                   0x230
+
+/* CG IND registers are accessed via SMC indirect space + SMC_CG_IND_START */
+#define SMC_CG_IND_START                    0xc0030000
+#define SMC_CG_IND_END                      0xc0040000
+
+#define CG_CGTT_LOCAL_0                         0x400
+#define CG_CGTT_LOCAL_1                         0x401
+
+/* SMC IND registers */
+#define SMC_SYSCON_RESET_CNTL                           0x80000000
+#       define RST_REG                                  (1 << 0)
+#define SMC_SYSCON_CLOCK_CNTL_0                         0x80000004
+#       define CK_DISABLE                               (1 << 0)
+#       define CKEN                                     (1 << 24)
+
+#define VGA_HDP_CONTROL                                 0x328
+#define         VGA_MEMORY_DISABLE                              (1 << 4)
+
+#define DCCG_DISP_SLOW_SELECT_REG                       0x4fc
+#define         DCCG_DISP1_SLOW_SELECT(x)               ((x) << 0)
+#define         DCCG_DISP1_SLOW_SELECT_MASK             (7 << 0)
+#define         DCCG_DISP1_SLOW_SELECT_SHIFT            0
+#define         DCCG_DISP2_SLOW_SELECT(x)               ((x) << 4)
+#define         DCCG_DISP2_SLOW_SELECT_MASK             (7 << 4)
+#define         DCCG_DISP2_SLOW_SELECT_SHIFT            4
+
+#define CG_SPLL_FUNC_CNTL                               0x600
+#define         SPLL_RESET                              (1 << 0)
+#define         SPLL_SLEEP                              (1 << 1)
+#define         SPLL_BYPASS_EN                          (1 << 3)
+#define         SPLL_REF_DIV(x)                         ((x) << 4)
+#define         SPLL_REF_DIV_MASK                       (0x3f << 4)
+#define         SPLL_PDIV_A(x)                          ((x) << 20)
+#define         SPLL_PDIV_A_MASK                        (0x7f << 20)
+#define         SPLL_PDIV_A_SHIFT                       20
+#define CG_SPLL_FUNC_CNTL_2                             0x604
+#define         SCLK_MUX_SEL(x)                         ((x) << 0)
+#define         SCLK_MUX_SEL_MASK                       (0x1ff << 0)
+#define CG_SPLL_FUNC_CNTL_3                             0x608
+#define         SPLL_FB_DIV(x)                          ((x) << 0)
+#define         SPLL_FB_DIV_MASK                        (0x3ffffff << 0)
+#define         SPLL_FB_DIV_SHIFT                       0
+#define         SPLL_DITHEN                             (1 << 28)
+#define CG_SPLL_FUNC_CNTL_4                             0x60c
+
+#define SPLL_CNTL_MODE                                  0x618
+#       define SPLL_REFCLK_SEL(x)                       ((x) << 8)
+#       define SPLL_REFCLK_SEL_MASK                     0xFF00
+
+#define CG_SPLL_SPREAD_SPECTRUM                         0x620
+#define         SSEN                                    (1 << 0)
+#define         CLK_S(x)                                ((x) << 4)
+#define         CLK_S_MASK                              (0xfff << 4)
+#define         CLK_S_SHIFT                             4
+#define CG_SPLL_SPREAD_SPECTRUM_2                       0x624
+#define         CLK_V(x)                                ((x) << 0)
+#define         CLK_V_MASK                              (0x3ffffff << 0)
+#define         CLK_V_SHIFT                             0
+
+#define CG_SPLL_AUTOSCALE_CNTL                          0x62c
+#       define AUTOSCALE_ON_SS_CLEAR                    (1 << 9)
+
 /* discrete uvd clocks */
 #define CG_UPLL_FUNC_CNTL                               0x634
 #       define UPLL_RESET_MASK                          0x00000001
@@ -59,6 +147,45 @@
 #define CG_UPLL_SPREAD_SPECTRUM                         0x650
 #       define SSEN_MASK                                0x00000001
 
+#define MPLL_BYPASSCLK_SEL                              0x65c
+#       define MPLL_CLKOUT_SEL(x)                       ((x) << 8)
+#       define MPLL_CLKOUT_SEL_MASK                     0xFF00
+
+#define CG_CLKPIN_CNTL                                    0x660
+#       define XTALIN_DIVIDE                              (1 << 1)
+#       define BCLK_AS_XCLK                               (1 << 2)
+#define CG_CLKPIN_CNTL_2                                  0x664
+#       define FORCE_BIF_REFCLK_EN                        (1 << 3)
+#       define MUX_TCLK_TO_XCLK                           (1 << 8)
+
+#define THM_CLK_CNTL                                    0x66c
+#       define CMON_CLK_SEL(x)                          ((x) << 0)
+#       define CMON_CLK_SEL_MASK                        0xFF
+#       define TMON_CLK_SEL(x)                          ((x) << 8)
+#       define TMON_CLK_SEL_MASK                        0xFF00
+#define MISC_CLK_CNTL                                   0x670
+#       define DEEP_SLEEP_CLK_SEL(x)                    ((x) << 0)
+#       define DEEP_SLEEP_CLK_SEL_MASK                  0xFF
+#       define ZCLK_SEL(x)                              ((x) << 8)
+#       define ZCLK_SEL_MASK                            0xFF00
+
+#define CG_THERMAL_CTRL                                 0x700
+#define         DPM_EVENT_SRC(x)                        ((x) << 0)
+#define         DPM_EVENT_SRC_MASK                      (7 << 0)
+#define         DIG_THERM_DPM(x)                        ((x) << 14)
+#define         DIG_THERM_DPM_MASK                      0x003FC000
+#define         DIG_THERM_DPM_SHIFT                     14
+
+#define CG_THERMAL_INT                                  0x708
+#define         DIG_THERM_INTH(x)                       ((x) << 8)
+#define         DIG_THERM_INTH_MASK                     0x0000FF00
+#define         DIG_THERM_INTH_SHIFT                    8
+#define         DIG_THERM_INTL(x)                       ((x) << 16)
+#define         DIG_THERM_INTL_MASK                     0x00FF0000
+#define         DIG_THERM_INTL_SHIFT                    16
+#define         THERM_INT_MASK_HIGH                     (1 << 24)
+#define         THERM_INT_MASK_LOW                      (1 << 25)
+
 #define CG_MULT_THERMAL_STATUS                                  0x714
 #define         ASIC_MAX_TEMP(x)                                ((x) << 0)
 #define         ASIC_MAX_TEMP_MASK                              0x000001ff
@@ -67,31 +194,89 @@
 #define         CTF_TEMP_MASK                                   0x0003fe00
 #define         CTF_TEMP_SHIFT                                  9
 
-#define SI_MAX_SH_GPRS           256
-#define SI_MAX_TEMP_GPRS         16
-#define SI_MAX_SH_THREADS        256
-#define SI_MAX_SH_STACK_ENTRIES  4096
-#define SI_MAX_FRC_EOV_CNT       16384
-#define SI_MAX_BACKENDS          8
-#define SI_MAX_BACKENDS_MASK     0xFF
-#define SI_MAX_BACKENDS_PER_SE_MASK     0x0F
-#define SI_MAX_SIMDS             12
-#define SI_MAX_SIMDS_MASK        0x0FFF
-#define SI_MAX_SIMDS_PER_SE_MASK        0x00FF
-#define SI_MAX_PIPES             8
-#define SI_MAX_PIPES_MASK        0xFF
-#define SI_MAX_PIPES_PER_SIMD_MASK      0x3F
-#define SI_MAX_LDS_NUM           0xFFFF
-#define SI_MAX_TCC               16
-#define SI_MAX_TCC_MASK          0xFFFF
-
-#define VGA_HDP_CONTROL                                 0x328
-#define         VGA_MEMORY_DISABLE                              (1 << 4)
-
-#define CG_CLKPIN_CNTL                                    0x660
-#       define XTALIN_DIVIDE                              (1 << 1)
-#define CG_CLKPIN_CNTL_2                                  0x664
-#       define MUX_TCLK_TO_XCLK                           (1 << 8)
+#define GENERAL_PWRMGT                                  0x780
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (1 << 6)
+#       define SW_SMIO_INDEX_SHIFT                      6
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#define CG_TPC                                            0x784
+#define SCLK_PWRMGT_CNTL                                  0x788
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_LOW_D1                                (1 << 1)
+#       define FIR_RESET                                  (1 << 4)
+#       define FIR_FORCE_TREND_SEL                        (1 << 5)
+#       define FIR_TREND_MODE                             (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 7)
+#       define GFX_CLK_FORCE_ON                           (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                        (1 << 9)
+#       define GFX_CLK_FORCE_OFF                          (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                        (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                        (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                        (1 << 13)
+#       define DYN_LIGHT_SLEEP_EN                         (1 << 14)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                  0x798
+#       define CURRENT_STATE_INDEX_MASK                   (0xf << 4)
+#       define CURRENT_STATE_INDEX_SHIFT                  4
+
+#define CG_FTV                                            0x7bc
+
+#define CG_FFCT_0                                         0x7c0
+#       define UTC_0(x)                                   ((x) << 0)
+#       define UTC_0_MASK                                 (0x3ff << 0)
+#       define DTC_0(x)                                   ((x) << 10)
+#       define DTC_0_MASK                                 (0x3ff << 10)
+
+#define CG_BSP                                          0x7fc
+#       define BSP(x)                                   ((x) << 0)
+#       define BSP_MASK                                 (0xffff << 0)
+#       define BSU(x)                                   ((x) << 16)
+#       define BSU_MASK                                 (0xf << 16)
+#define CG_AT                                           0x800
+#       define CG_R(x)                                  ((x) << 0)
+#       define CG_R_MASK                                (0xffff << 0)
+#       define CG_L(x)                                  ((x) << 16)
+#       define CG_L_MASK                                (0xffff << 16)
+
+#define CG_GIT                                          0x804
+#       define CG_GICST(x)                              ((x) << 0)
+#       define CG_GICST_MASK                            (0xffff << 0)
+#       define CG_GIPOT(x)                              ((x) << 16)
+#       define CG_GIPOT_MASK                            (0xffff << 16)
+
+#define CG_SSP                                            0x80c
+#       define SST(x)                                     ((x) << 0)
+#       define SST_MASK                                   (0xffff << 0)
+#       define SSTU(x)                                    ((x) << 16)
+#       define SSTU_MASK                                  (0xf << 16)
+
+#define CG_DISPLAY_GAP_CNTL                               0x828
+#       define DISP1_GAP(x)                               ((x) << 0)
+#       define DISP1_GAP_MASK                             (3 << 0)
+#       define DISP2_GAP(x)                               ((x) << 2)
+#       define DISP2_GAP_MASK                             (3 << 2)
+#       define VBI_TIMER_COUNT(x)                         ((x) << 4)
+#       define VBI_TIMER_COUNT_MASK                       (0x3fff << 4)
+#       define VBI_TIMER_UNIT(x)                          ((x) << 20)
+#       define VBI_TIMER_UNIT_MASK                        (7 << 20)
+#       define DISP1_GAP_MCHG(x)                          ((x) << 24)
+#       define DISP1_GAP_MCHG_MASK                        (3 << 24)
+#       define DISP2_GAP_MCHG(x)                          ((x) << 26)
+#       define DISP2_GAP_MCHG_MASK                        (3 << 26)
+
+#define CG_ULV_CONTROL                                  0x878
+#define CG_ULV_PARAMETER                                0x87c
+
+#define SMC_SCRATCH0                                    0x884
+
+#define CG_CAC_CTRL                                     0x8b8
+#       define CAC_WINDOW(x)                            ((x) << 0)
+#       define CAC_WINDOW_MASK                          0x00ffffff
 
 #define DMIF_ADDR_CONFIG                                0xBD4
 
@@ -203,6 +388,10 @@
 #define VM_CONTEXT0_PAGE_TABLE_END_ADDR                 0x157C
 #define VM_CONTEXT1_PAGE_TABLE_END_ADDR                 0x1580
 
+#define VM_L2_CG                                        0x15c0
+#define         MC_CG_ENABLE                            (1 << 18)
+#define         MC_LS_ENABLE                            (1 << 19)
+
 #define MC_SHARED_CHMAP                                         0x2004
 #define         NOOFCHAN_SHIFT                                  12
 #define         NOOFCHAN_MASK                                   0x0000f000
@@ -228,6 +417,17 @@
 
 #define MC_SHARED_BLACKOUT_CNTL                         0x20ac
 
+#define MC_HUB_MISC_HUB_CG                              0x20b8
+#define MC_HUB_MISC_VM_CG                               0x20bc
+
+#define MC_HUB_MISC_SIP_CG                              0x20c0
+
+#define MC_XPB_CLK_GAT                                  0x2478
+
+#define MC_CITF_MISC_RD_CG                              0x2648
+#define MC_CITF_MISC_WR_CG                              0x264c
+#define MC_CITF_MISC_VM_CG                              0x2650
+
 #define MC_ARB_RAMCFG                                   0x2760
 #define         NOOFBANK_SHIFT                                  0
 #define         NOOFBANK_MASK                                   0x00000003
@@ -243,6 +443,23 @@
 #define         NOOFGROUPS_SHIFT                                12
 #define         NOOFGROUPS_MASK                                 0x00001000
 
+#define MC_ARB_DRAM_TIMING                              0x2774
+#define MC_ARB_DRAM_TIMING2                             0x2778
+
+#define MC_ARB_BURST_TIME                               0x2808
+#define         STATE0(x)                               ((x) << 0)
+#define         STATE0_MASK                             (0x1f << 0)
+#define         STATE0_SHIFT                            0
+#define         STATE1(x)                               ((x) << 5)
+#define         STATE1_MASK                             (0x1f << 5)
+#define         STATE1_SHIFT                            5
+#define         STATE2(x)                               ((x) << 10)
+#define         STATE2_MASK                             (0x1f << 10)
+#define         STATE2_SHIFT                            10
+#define         STATE3(x)                               ((x) << 15)
+#define         STATE3_MASK                             (0x1f << 15)
+#define         STATE3_SHIFT                            15
+
 #define MC_SEQ_TRAIN_WAKEUP_CNTL                        0x2808
 #define         TRAIN_DONE_D0                           (1 << 30)
 #define         TRAIN_DONE_D1                           (1 << 31)
@@ -250,13 +467,105 @@
 #define MC_SEQ_SUP_CNTL                                 0x28c8
 #define         RUN_MASK                                (1 << 0)
 #define MC_SEQ_SUP_PGM                                  0x28cc
+#define MC_PMG_AUTO_CMD                                 0x28d0
 
 #define MC_IO_PAD_CNTL_D0                               0x29d0
 #define         MEM_FALL_OUT_CMD                        (1 << 8)
 
+#define MC_SEQ_RAS_TIMING                               0x28a0
+#define MC_SEQ_CAS_TIMING                               0x28a4
+#define MC_SEQ_MISC_TIMING                              0x28a8
+#define MC_SEQ_MISC_TIMING2                             0x28ac
+#define MC_SEQ_PMG_TIMING                               0x28b0
+#define MC_SEQ_RD_CTL_D0                                0x28b4
+#define MC_SEQ_RD_CTL_D1                                0x28b8
+#define MC_SEQ_WR_CTL_D0                                0x28bc
+#define MC_SEQ_WR_CTL_D1                                0x28c0
+
+#define MC_SEQ_MISC0                                    0x2a00
+#define         MC_SEQ_MISC0_VEN_ID_SHIFT               8
+#define         MC_SEQ_MISC0_VEN_ID_MASK                0x00000f00
+#define         MC_SEQ_MISC0_VEN_ID_VALUE               3
+#define         MC_SEQ_MISC0_REV_ID_SHIFT               12
+#define         MC_SEQ_MISC0_REV_ID_MASK                0x0000f000
+#define         MC_SEQ_MISC0_REV_ID_VALUE               1
+#define         MC_SEQ_MISC0_GDDR5_SHIFT                28
+#define         MC_SEQ_MISC0_GDDR5_MASK                 0xf0000000
+#define         MC_SEQ_MISC0_GDDR5_VALUE                5
+#define MC_SEQ_MISC1                                    0x2a04
+#define MC_SEQ_RESERVE_M                                0x2a08
+#define MC_PMG_CMD_EMRS                                 0x2a0c
+
 #define MC_SEQ_IO_DEBUG_INDEX                           0x2a44
 #define MC_SEQ_IO_DEBUG_DATA                            0x2a48
 
+#define MC_SEQ_MISC5                                    0x2a54
+#define MC_SEQ_MISC6                                    0x2a58
+
+#define MC_SEQ_MISC7                                    0x2a64
+
+#define MC_SEQ_RAS_TIMING_LP                            0x2a6c
+#define MC_SEQ_CAS_TIMING_LP                            0x2a70
+#define MC_SEQ_MISC_TIMING_LP                           0x2a74
+#define MC_SEQ_MISC_TIMING2_LP                          0x2a78
+#define MC_SEQ_WR_CTL_D0_LP                             0x2a7c
+#define MC_SEQ_WR_CTL_D1_LP                             0x2a80
+#define MC_SEQ_PMG_CMD_EMRS_LP                          0x2a84
+#define MC_SEQ_PMG_CMD_MRS_LP                           0x2a88
+
+#define MC_PMG_CMD_MRS                                  0x2aac
+
+#define MC_SEQ_RD_CTL_D0_LP                             0x2b1c
+#define MC_SEQ_RD_CTL_D1_LP                             0x2b20
+
+#define MC_PMG_CMD_MRS1                                 0x2b44
+#define MC_SEQ_PMG_CMD_MRS1_LP                          0x2b48
+#define MC_SEQ_PMG_TIMING_LP                            0x2b4c
+
+#define MC_SEQ_WR_CTL_2                                 0x2b54
+#define MC_SEQ_WR_CTL_2_LP                              0x2b58
+#define MC_PMG_CMD_MRS2                                 0x2b5c
+#define MC_SEQ_PMG_CMD_MRS2_LP                          0x2b60
+
+#define MCLK_PWRMGT_CNTL                                0x2ba0
+#       define DLL_SPEED(x)                             ((x) << 0)
+#       define DLL_SPEED_MASK                           (0x1f << 0)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCK0_PDNB                              (1 << 8)
+#       define MRDCK1_PDNB                              (1 << 9)
+#       define MRDCK0_RESET                             (1 << 16)
+#       define MRDCK1_RESET                             (1 << 17)
+#       define DLL_READY_READ                           (1 << 24)
+#define DLL_CNTL                                        0x2ba4
+#       define MRDCK0_BYPASS                            (1 << 24)
+#       define MRDCK1_BYPASS                            (1 << 25)
+
+#define MPLL_FUNC_CNTL                                  0x2bb4
+#define         BWCTRL(x)                               ((x) << 20)
+#define         BWCTRL_MASK                             (0xff << 20)
+#define MPLL_FUNC_CNTL_1                                0x2bb8
+#define         VCO_MODE(x)                             ((x) << 0)
+#define         VCO_MODE_MASK                           (3 << 0)
+#define         CLKFRAC(x)                              ((x) << 4)
+#define         CLKFRAC_MASK                            (0xfff << 4)
+#define         CLKF(x)                                 ((x) << 16)
+#define         CLKF_MASK                               (0xfff << 16)
+#define MPLL_FUNC_CNTL_2                                0x2bbc
+#define MPLL_AD_FUNC_CNTL                               0x2bc0
+#define         YCLK_POST_DIV(x)                        ((x) << 0)
+#define         YCLK_POST_DIV_MASK                      (7 << 0)
+#define MPLL_DQ_FUNC_CNTL                               0x2bc4
+#define         YCLK_SEL(x)                             ((x) << 4)
+#define         YCLK_SEL_MASK                           (1 << 4)
+
+#define MPLL_SS1                                        0x2bcc
+#define         CLKV(x)                                 ((x) << 0)
+#define         CLKV_MASK                               (0x3ffffff << 0)
+#define MPLL_SS2                                        0x2bd0
+#define         CLKS(x)                                 ((x) << 0)
+#define         CLKS_MASK                               (0xfff << 0)
+
 #define HDP_HOST_PATH_CNTL                              0x2C00
 #define HDP_NONSURFACE_BASE                             0x2C04
 #define HDP_NONSURFACE_INFO                             0x2C08
@@ -266,6 +575,8 @@
 #define HDP_MISC_CNTL                                   0x2F4C
 #define         HDP_FLUSH_INVALIDATE_CACHE                      (1 << 0)
 
+#define ATC_MISC_CG                                     0x3350
+
 #define IH_RB_CNTL                                        0x3e00
 #       define IH_RB_ENABLE                               (1 << 0)
 #       define IH_IB_SIZE(x)                              ((x) << 1) /* log2 */
@@ -424,6 +735,9 @@
 #       define DC_HPDx_RX_INT_TIMER(x)                    ((x) << 16)
 #       define DC_HPDx_EN                                 (1 << 28)
 
+#define DPG_PIPE_STUTTER_CONTROL                          0x6cd4
+#       define STUTTER_ENABLE                             (1 << 0)
+
 /* 0x6e98, 0x7a98, 0x10698, 0x11298, 0x11e98, 0x12a98 */
 #define CRTC_STATUS_FRAME_COUNT                         0x6e98
 
@@ -599,6 +913,24 @@
 
 #define SQC_CACHES                                      0x8C08
 
+#define SQ_POWER_THROTTLE                               0x8e58
+#define         MIN_POWER(x)                            ((x) << 0)
+#define         MIN_POWER_MASK                          (0x3fff << 0)
+#define         MIN_POWER_SHIFT                         0
+#define         MAX_POWER(x)                            ((x) << 16)
+#define         MAX_POWER_MASK                          (0x3fff << 16)
+#define         MAX_POWER_SHIFT                         0
+#define SQ_POWER_THROTTLE2                              0x8e5c
+#define         MAX_POWER_DELTA(x)                      ((x) << 0)
+#define         MAX_POWER_DELTA_MASK                    (0x3fff << 0)
+#define         MAX_POWER_DELTA_SHIFT                   0
+#define         STI_SIZE(x)                             ((x) << 16)
+#define         STI_SIZE_MASK                           (0x3ff << 16)
+#define         STI_SIZE_SHIFT                          16
+#define         LTI_RATIO(x)                            ((x) << 27)
+#define         LTI_RATIO_MASK                          (0xf << 27)
+#define         LTI_RATIO_SHIFT                         27
+
 #define SX_DEBUG_1                                      0x9060
 
 #define SPI_STATIC_THREAD_MGMT_1                        0x90E0
@@ -616,6 +948,11 @@
 #define CGTS_USER_TCC_DISABLE                           0x914C
 #define         TCC_DISABLE_MASK                                0xFFFF0000
 #define         TCC_DISABLE_SHIFT                               16
+#define CGTS_SM_CTRL_REG                                0x9150
+#define         OVERRIDE                                (1 << 21)
+#define         LS_OVERRIDE                             (1 << 22)
+
+#define SPI_LB_CU_MASK                                  0x9354
 
 #define TA_CNTL_AUX                                     0x9508
 
@@ -705,6 +1042,8 @@
 #define CB_PERFCOUNTER3_SELECT0                         0x9a38
 #define CB_PERFCOUNTER3_SELECT1                         0x9a3c
 
+#define CB_CGTT_SCLK_CTRL                               0x9a60
+
 #define GC_USER_RB_BACKEND_DISABLE                      0x9B7C
 #define         BACKEND_DISABLE_MASK                    0x00FF0000
 #define         BACKEND_DISABLE_SHIFT                   16
@@ -762,6 +1101,9 @@
 #       define CP_RINGID1_INT_STAT                      (1 << 30)
 #       define CP_RINGID0_INT_STAT                      (1 << 31)
 
+#define CP_MEM_SLP_CNTL                                 0xC1E4
+#       define CP_MEM_LS_EN                             (1 << 0)
+
 #define CP_DEBUG                                        0xC1FC
 
 #define RLC_CNTL                                          0xC300
@@ -769,6 +1111,7 @@
 #define RLC_RL_BASE                                       0xC304
 #define RLC_RL_SIZE                                       0xC308
 #define RLC_LB_CNTL                                       0xC30C
+#       define LOAD_BALANCE_ENABLE                        (1 << 0)
 #define RLC_SAVE_AND_RESTORE_BASE                         0xC310
 #define RLC_LB_CNTR_MAX                                   0xC314
 #define RLC_LB_CNTR_INIT                                  0xC318
@@ -783,6 +1126,56 @@
 #define RLC_CAPTURE_GPU_CLOCK_COUNT                       0xC340
 #define RLC_MC_CNTL                                       0xC344
 #define RLC_UCODE_CNTL                                    0xC348
+#define RLC_STAT                                          0xC34C
+#       define RLC_BUSY_STATUS                            (1 << 0)
+#       define GFX_POWER_STATUS                           (1 << 1)
+#       define GFX_CLOCK_STATUS                           (1 << 2)
+#       define GFX_LS_STATUS                              (1 << 3)
+
+#define RLC_PG_CNTL                                     0xC35C
+#       define GFX_PG_ENABLE                            (1 << 0)
+#       define GFX_PG_SRC                               (1 << 1)
+
+#define RLC_CGTT_MGCG_OVERRIDE                          0xC400
+#define RLC_CGCG_CGLS_CTRL                              0xC404
+#       define CGCG_EN                                  (1 << 0)
+#       define CGLS_EN                                  (1 << 1)
+
+#define RLC_TTOP_D                                      0xC414
+#       define RLC_PUD(x)                               ((x) << 0)
+#       define RLC_PUD_MASK                             (0xff << 0)
+#       define RLC_PDD(x)                               ((x) << 8)
+#       define RLC_PDD_MASK                             (0xff << 8)
+#       define RLC_TTPD(x)                              ((x) << 16)
+#       define RLC_TTPD_MASK                            (0xff << 16)
+#       define RLC_MSD(x)                               ((x) << 24)
+#       define RLC_MSD_MASK                             (0xff << 24)
+
+#define RLC_LB_INIT_CU_MASK                               0xC41C
+
+#define RLC_PG_AO_CU_MASK                               0xC42C
+#define RLC_MAX_PG_CU                                   0xC430
+#       define MAX_PU_CU(x)                             ((x) << 0)
+#       define MAX_PU_CU_MASK                           (0xff << 0)
+#define RLC_AUTO_PG_CTRL                                0xC434
+#       define AUTO_PG_EN                               (1 << 0)
+#       define GRBM_REG_SGIT(x)                         ((x) << 3)
+#       define GRBM_REG_SGIT_MASK                       (0xffff << 3)
+#       define PG_AFTER_GRBM_REG_ST(x)                  ((x) << 19)
+#       define PG_AFTER_GRBM_REG_ST_MASK                (0x1fff << 19)
+
+#define RLC_SERDES_WR_MASTER_MASK_0                       0xC454
+#define RLC_SERDES_WR_MASTER_MASK_1                       0xC458
+#define RLC_SERDES_WR_CTRL                                0xC45C
+
+#define RLC_SERDES_MASTER_BUSY_0                          0xC464
+#define RLC_SERDES_MASTER_BUSY_1                          0xC468
+
+#define RLC_GCPM_GENERAL_3                                0xC478
+
+#define DB_RENDER_CONTROL                               0x28000
+
+#define DB_DEPTH_INFO                                   0x2803c
 
 #define PA_SC_RASTER_CONFIG                             0x28350
 #       define RASTER_CONFIG_RB_MAP_0                   0
@@ -829,6 +1222,146 @@
 #       define THREAD_TRACE_FLUSH                       (54 << 0)
 #       define THREAD_TRACE_FINISH                      (55 << 0)
 
+/* PIF PHY0 registers idx/data 0x8/0xc */
+#define PB0_PIF_CNTL                                      0x10
+#       define LS2_EXIT_TIME(x)                           ((x) << 17)
+#       define LS2_EXIT_TIME_MASK                         (0x7 << 17)
+#       define LS2_EXIT_TIME_SHIFT                        17
+#define PB0_PIF_PAIRING                                   0x11
+#       define MULTI_PIF                                  (1 << 25)
+#define PB0_PIF_PWRDOWN_0                                 0x12
+#       define PLL_POWER_STATE_IN_TXS2_0(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_0_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_0_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_0(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_0_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_0_SHIFT             10
+#       define PLL_RAMP_UP_TIME_0(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_0_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_0_SHIFT                   24
+#define PB0_PIF_PWRDOWN_1                                 0x13
+#       define PLL_POWER_STATE_IN_TXS2_1(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_1_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_1_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_1(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_1_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_1_SHIFT             10
+#       define PLL_RAMP_UP_TIME_1(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_1_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_1_SHIFT                   24
+
+#define PB0_PIF_PWRDOWN_2                                 0x17
+#       define PLL_POWER_STATE_IN_TXS2_2(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_2_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_2_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_2(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_2_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_2_SHIFT             10
+#       define PLL_RAMP_UP_TIME_2(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_2_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_2_SHIFT                   24
+#define PB0_PIF_PWRDOWN_3                                 0x18
+#       define PLL_POWER_STATE_IN_TXS2_3(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_3_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_3_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_3(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_3_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_3_SHIFT             10
+#       define PLL_RAMP_UP_TIME_3(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_3_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_3_SHIFT                   24
+/* PIF PHY1 registers idx/data 0x10/0x14 */
+#define PB1_PIF_CNTL                                      0x10
+#define PB1_PIF_PAIRING                                   0x11
+#define PB1_PIF_PWRDOWN_0                                 0x12
+#define PB1_PIF_PWRDOWN_1                                 0x13
+
+#define PB1_PIF_PWRDOWN_2                                 0x17
+#define PB1_PIF_PWRDOWN_3                                 0x18
+/* PCIE registers idx/data 0x30/0x34 */
+#define PCIE_CNTL2                                        0x1c /* PCIE */
+#       define SLV_MEM_LS_EN                              (1 << 16)
+#       define MST_MEM_LS_EN                              (1 << 18)
+#       define REPLAY_MEM_LS_EN                           (1 << 19)
+#define PCIE_LC_STATUS1                                   0x28 /* PCIE */
+#       define LC_REVERSE_RCVR                            (1 << 0)
+#       define LC_REVERSE_XMIT                            (1 << 1)
+#       define LC_OPERATING_LINK_WIDTH_MASK               (0x7 << 2)
+#       define LC_OPERATING_LINK_WIDTH_SHIFT              2
+#       define LC_DETECTED_LINK_WIDTH_MASK                (0x7 << 5)
+#       define LC_DETECTED_LINK_WIDTH_SHIFT               5
+
+#define PCIE_P_CNTL                                       0x40 /* PCIE */
+#       define P_IGNORE_EDB_ERR                           (1 << 6)
+
+/* PCIE PORT registers idx/data 0x38/0x3c */
+#define PCIE_LC_CNTL                                      0xa0
+#       define LC_L0S_INACTIVITY(x)                       ((x) << 8)
+#       define LC_L0S_INACTIVITY_MASK                     (0xf << 8)
+#       define LC_L0S_INACTIVITY_SHIFT                    8
+#       define LC_L1_INACTIVITY(x)                        ((x) << 12)
+#       define LC_L1_INACTIVITY_MASK                      (0xf << 12)
+#       define LC_L1_INACTIVITY_SHIFT                     12
+#       define LC_PMI_TO_L1_DIS                           (1 << 16)
+#       define LC_ASPM_TO_L1_DIS                          (1 << 24)
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#       define LC_DYN_LANES_PWR_STATE(x)                  ((x) << 21)
+#       define LC_DYN_LANES_PWR_STATE_MASK                (0x3 << 21)
+#       define LC_DYN_LANES_PWR_STATE_SHIFT               21
+#define PCIE_LC_N_FTS_CNTL                                0xa3 /* PCIE_P */
+#       define LC_XMIT_N_FTS(x)                           ((x) << 0)
+#       define LC_XMIT_N_FTS_MASK                         (0xff << 0)
+#       define LC_XMIT_N_FTS_SHIFT                        0
+#       define LC_XMIT_N_FTS_OVERRIDE_EN                  (1 << 8)
+#       define LC_N_FTS_MASK                              (0xff << 24)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_GEN3_EN_STRAP                           (1 << 1)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 2)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_MASK         (0x3 << 3)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_SHIFT        3
+#       define LC_FORCE_EN_SW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_SW_SPEED_CHANGE               (1 << 6)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 7)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 8)
+#       define LC_INITIATE_LINK_SPEED_CHANGE              (1 << 9)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 10)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     10
+#       define LC_CURRENT_DATA_RATE_MASK                  (0x3 << 13) /* 0/1/2 = gen1/2/3 */
+#       define LC_CURRENT_DATA_RATE_SHIFT                 13
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 16)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 18)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 19)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN3               (1 << 20)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN3                (1 << 21)
+
+#define PCIE_LC_CNTL2                                     0xb1
+#       define LC_ALLOW_PDWN_IN_L1                        (1 << 17)
+#       define LC_ALLOW_PDWN_IN_L23                       (1 << 18)
+
+#define PCIE_LC_CNTL3                                     0xb5 /* PCIE_P */
+#       define LC_GO_TO_RECOVERY                          (1 << 30)
+#define PCIE_LC_CNTL4                                     0xb6 /* PCIE_P */
+#       define LC_REDO_EQ                                 (1 << 5)
+#       define LC_SET_QUIESCE                             (1 << 13)
+
 /*
  * UVD
  */
@@ -838,6 +1371,21 @@
 #define UVD_RBC_RB_RPTR                                 0xF690
 #define UVD_RBC_RB_WPTR                                 0xF694
 
+#define UVD_CGC_CTRL                                    0xF4B0
+#       define DCM                                      (1 << 0)
+#       define CG_DT(x)                                 ((x) << 2)
+#       define CG_DT_MASK                               (0xf << 2)
+#       define CLK_OD(x)                                ((x) << 6)
+#       define CLK_OD_MASK                              (0x1f << 6)
+
+ /* UVD CTX indirect */
+#define UVD_CGC_MEM_CTRL                                0xC0
+#define UVD_CGC_CTRL2                                   0xC1
+#       define DYN_OR_EN                                (1 << 0)
+#       define DYN_RR_EN                                (1 << 1)
+#       define G_DIV_ID(x)                              ((x) << 2)
+#       define G_DIV_ID_MASK                            (0x7 << 2)
+
 /*
  * PM4
  */
@@ -1082,6 +1630,11 @@
 #       define DMA_IDLE                                   (1 << 0)
 #define DMA_TILING_CONFIG                                 0xd0b8
 
+#define DMA_PG                                          0xd0d4
+#       define PG_CNTL_ENABLE                           (1 << 0)
+#define DMA_PGFSM_CONFIG                                0xd0d8
+#define DMA_PGFSM_WRITE                                 0xd0dc
+
 #define DMA_PACKET(cmd, b, t, s, n)     ((((cmd) & 0xF) << 28) |        \
                                          (((b) & 0x1) << 26) |          \
                                          (((t) & 0x1) << 23) |          \
diff --git a/drivers/gpu/drm/radeon/sislands_smc.h b/drivers/gpu/drm/radeon/sislands_smc.h
new file mode 100644
index 000000000000..5578e9837026
--- /dev/null
+++ b/drivers/gpu/drm/radeon/sislands_smc.h
@@ -0,0 +1,397 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef PP_SISLANDS_SMC_H
+#define PP_SISLANDS_SMC_H
+
+#include "ppsmc.h"
+
+#pragma pack(push, 1)
+
+#define SISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16
+
+struct PP_SIslands_Dpm2PerfLevel
+{
+    uint8_t MaxPS;
+    uint8_t TgtAct;
+    uint8_t MaxPS_StepInc;
+    uint8_t MaxPS_StepDec;
+    uint8_t PSSamplingTime;
+    uint8_t NearTDPDec;
+    uint8_t AboveSafeInc;
+    uint8_t BelowSafeInc;
+    uint8_t PSDeltaLimit;
+    uint8_t PSDeltaWin;
+    uint16_t PwrEfficiencyRatio;
+    uint8_t Reserved[4];
+};
+
+typedef struct PP_SIslands_Dpm2PerfLevel PP_SIslands_Dpm2PerfLevel;
+
+struct PP_SIslands_DPM2Status
+{
+    uint32_t    dpm2Flags;
+    uint8_t     CurrPSkip;
+    uint8_t     CurrPSkipPowerShift;
+    uint8_t     CurrPSkipTDP;
+    uint8_t     CurrPSkipOCP;
+    uint8_t     MaxSPLLIndex;
+    uint8_t     MinSPLLIndex;
+    uint8_t     CurrSPLLIndex;
+    uint8_t     InfSweepMode;
+    uint8_t     InfSweepDir;
+    uint8_t     TDPexceeded;
+    uint8_t     reserved;
+    uint8_t     SwitchDownThreshold;
+    uint32_t    SwitchDownCounter;
+    uint32_t    SysScalingFactor;
+};
+
+typedef struct PP_SIslands_DPM2Status PP_SIslands_DPM2Status;
+
+struct PP_SIslands_DPM2Parameters
+{
+    uint32_t    TDPLimit;
+    uint32_t    NearTDPLimit;
+    uint32_t    SafePowerLimit;
+    uint32_t    PowerBoostLimit;
+    uint32_t    MinLimitDelta;
+};
+typedef struct PP_SIslands_DPM2Parameters PP_SIslands_DPM2Parameters;
+
+struct PP_SIslands_PAPMStatus
+{
+    uint32_t    EstimatedDGPU_T;
+    uint32_t    EstimatedDGPU_P;
+    uint32_t    EstimatedAPU_T;
+    uint32_t    EstimatedAPU_P;
+    uint8_t     dGPU_T_Limit_Exceeded;
+    uint8_t     reserved[3];
+};
+typedef struct PP_SIslands_PAPMStatus PP_SIslands_PAPMStatus;
+
+struct PP_SIslands_PAPMParameters
+{
+    uint32_t    NearTDPLimitTherm;
+    uint32_t    NearTDPLimitPAPM;
+    uint32_t    PlatformPowerLimit;
+    uint32_t    dGPU_T_Limit;
+    uint32_t    dGPU_T_Warning;
+    uint32_t    dGPU_T_Hysteresis;
+};
+typedef struct PP_SIslands_PAPMParameters PP_SIslands_PAPMParameters;
+
+struct SISLANDS_SMC_SCLK_VALUE
+{
+    uint32_t    vCG_SPLL_FUNC_CNTL;
+    uint32_t    vCG_SPLL_FUNC_CNTL_2;
+    uint32_t    vCG_SPLL_FUNC_CNTL_3;
+    uint32_t    vCG_SPLL_FUNC_CNTL_4;
+    uint32_t    vCG_SPLL_SPREAD_SPECTRUM;
+    uint32_t    vCG_SPLL_SPREAD_SPECTRUM_2;
+    uint32_t    sclk_value;
+};
+
+typedef struct SISLANDS_SMC_SCLK_VALUE SISLANDS_SMC_SCLK_VALUE;
+
+struct SISLANDS_SMC_MCLK_VALUE
+{
+    uint32_t    vMPLL_FUNC_CNTL;
+    uint32_t    vMPLL_FUNC_CNTL_1;
+    uint32_t    vMPLL_FUNC_CNTL_2;
+    uint32_t    vMPLL_AD_FUNC_CNTL;
+    uint32_t    vMPLL_DQ_FUNC_CNTL;
+    uint32_t    vMCLK_PWRMGT_CNTL;
+    uint32_t    vDLL_CNTL;
+    uint32_t    vMPLL_SS;
+    uint32_t    vMPLL_SS2;
+    uint32_t    mclk_value;
+};
+
+typedef struct SISLANDS_SMC_MCLK_VALUE SISLANDS_SMC_MCLK_VALUE;
+
+struct SISLANDS_SMC_VOLTAGE_VALUE
+{
+    uint16_t    value;
+    uint8_t     index;
+    uint8_t     phase_settings;
+};
+
+typedef struct SISLANDS_SMC_VOLTAGE_VALUE SISLANDS_SMC_VOLTAGE_VALUE;
+
+struct SISLANDS_SMC_HW_PERFORMANCE_LEVEL
+{
+    uint8_t                     ACIndex;
+    uint8_t                     displayWatermark;
+    uint8_t                     gen2PCIE;
+    uint8_t                     UVDWatermark;
+    uint8_t                     VCEWatermark;
+    uint8_t                     strobeMode;
+    uint8_t                     mcFlags;
+    uint8_t                     padding;
+    uint32_t                    aT;
+    uint32_t                    bSP;
+    SISLANDS_SMC_SCLK_VALUE     sclk;
+    SISLANDS_SMC_MCLK_VALUE     mclk;
+    SISLANDS_SMC_VOLTAGE_VALUE  vddc;
+    SISLANDS_SMC_VOLTAGE_VALUE  mvdd;
+    SISLANDS_SMC_VOLTAGE_VALUE  vddci;
+    SISLANDS_SMC_VOLTAGE_VALUE  std_vddc;
+    uint8_t                     hysteresisUp;
+    uint8_t                     hysteresisDown;
+    uint8_t                     stateFlags;
+    uint8_t                     arbRefreshState;
+    uint32_t                    SQPowerThrottle;
+    uint32_t                    SQPowerThrottle_2;
+    uint32_t                    MaxPoweredUpCU;
+    SISLANDS_SMC_VOLTAGE_VALUE  high_temp_vddc;
+    SISLANDS_SMC_VOLTAGE_VALUE  low_temp_vddc;
+    uint32_t                    reserved[2];
+    PP_SIslands_Dpm2PerfLevel   dpm2;
+};
+
+#define SISLANDS_SMC_STROBE_RATIO    0x0F
+#define SISLANDS_SMC_STROBE_ENABLE   0x10
+
+#define SISLANDS_SMC_MC_EDC_RD_FLAG  0x01
+#define SISLANDS_SMC_MC_EDC_WR_FLAG  0x02
+#define SISLANDS_SMC_MC_RTT_ENABLE   0x04
+#define SISLANDS_SMC_MC_STUTTER_EN   0x08
+#define SISLANDS_SMC_MC_PG_EN        0x10
+
+typedef struct SISLANDS_SMC_HW_PERFORMANCE_LEVEL SISLANDS_SMC_HW_PERFORMANCE_LEVEL;
+
+struct SISLANDS_SMC_SWSTATE
+{
+    uint8_t                             flags;
+    uint8_t                             levelCount;
+    uint8_t                             padding2;
+    uint8_t                             padding3;
+    SISLANDS_SMC_HW_PERFORMANCE_LEVEL   levels[1];
+};
+
+typedef struct SISLANDS_SMC_SWSTATE SISLANDS_SMC_SWSTATE;
+
+#define SISLANDS_SMC_VOLTAGEMASK_VDDC  0
+#define SISLANDS_SMC_VOLTAGEMASK_MVDD  1
+#define SISLANDS_SMC_VOLTAGEMASK_VDDCI 2
+#define SISLANDS_SMC_VOLTAGEMASK_MAX   4
+
+struct SISLANDS_SMC_VOLTAGEMASKTABLE
+{
+    uint32_t lowMask[SISLANDS_SMC_VOLTAGEMASK_MAX];
+};
+
+typedef struct SISLANDS_SMC_VOLTAGEMASKTABLE SISLANDS_SMC_VOLTAGEMASKTABLE;
+
+#define SISLANDS_MAX_NO_VREG_STEPS 32
+
+struct SISLANDS_SMC_STATETABLE
+{
+    uint8_t                             thermalProtectType;
+    uint8_t                             systemFlags;
+    uint8_t                             maxVDDCIndexInPPTable;
+    uint8_t                             extraFlags;
+    uint32_t                            lowSMIO[SISLANDS_MAX_NO_VREG_STEPS];
+    SISLANDS_SMC_VOLTAGEMASKTABLE       voltageMaskTable;
+    SISLANDS_SMC_VOLTAGEMASKTABLE       phaseMaskTable;
+    PP_SIslands_DPM2Parameters          dpm2Params;
+    SISLANDS_SMC_SWSTATE                initialState;
+    SISLANDS_SMC_SWSTATE                ACPIState;
+    SISLANDS_SMC_SWSTATE                ULVState;
+    SISLANDS_SMC_SWSTATE                driverState;
+    SISLANDS_SMC_HW_PERFORMANCE_LEVEL   dpmLevels[SISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1];
+};
+
+typedef struct SISLANDS_SMC_STATETABLE SISLANDS_SMC_STATETABLE;
+
+#define SI_SMC_SOFT_REGISTER_mclk_chg_timeout         0x0
+#define SI_SMC_SOFT_REGISTER_delay_vreg               0xC
+#define SI_SMC_SOFT_REGISTER_delay_acpi               0x28
+#define SI_SMC_SOFT_REGISTER_seq_index                0x5C
+#define SI_SMC_SOFT_REGISTER_mvdd_chg_time            0x60
+#define SI_SMC_SOFT_REGISTER_mclk_switch_lim          0x70
+#define SI_SMC_SOFT_REGISTER_watermark_threshold      0x78
+#define SI_SMC_SOFT_REGISTER_phase_shedding_delay     0x88
+#define SI_SMC_SOFT_REGISTER_ulv_volt_change_delay    0x8C
+#define SI_SMC_SOFT_REGISTER_mc_block_delay           0x98
+#define SI_SMC_SOFT_REGISTER_ticks_per_us             0xA8
+#define SI_SMC_SOFT_REGISTER_crtc_index               0xC4
+#define SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min 0xC8
+#define SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max 0xCC
+#define SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width  0xF4
+#define SI_SMC_SOFT_REGISTER_tdr_is_about_to_happen   0xFC
+#define SI_SMC_SOFT_REGISTER_vr_hot_gpio              0x100
+
+#define SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES 32
+
+#define SMC_SISLANDS_SCALE_I  7
+#define SMC_SISLANDS_SCALE_R 12
+
+struct PP_SIslands_CacConfig
+{
+    uint16_t   cac_lkge_lut[SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES];
+    uint32_t   lkge_lut_V0;
+    uint32_t   lkge_lut_Vstep;
+    uint32_t   WinTime;
+    uint32_t   R_LL;
+    uint32_t   calculation_repeats;
+    uint32_t   l2numWin_TDP;
+    uint32_t   dc_cac;
+    uint8_t    lts_truncate_n;
+    uint8_t    SHIFT_N;
+    uint8_t    log2_PG_LKG_SCALE;
+    uint8_t    cac_temp;
+    uint32_t   lkge_lut_T0;
+    uint32_t   lkge_lut_Tstep;
+};
+
+typedef struct PP_SIslands_CacConfig PP_SIslands_CacConfig;
+
+#define SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE 16
+#define SMC_SISLANDS_MC_REGISTER_ARRAY_SET_COUNT 20
+
+struct SMC_SIslands_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMC_SIslands_MCRegisterAddress SMC_SIslands_MCRegisterAddress;
+
+struct SMC_SIslands_MCRegisterSet
+{
+    uint32_t value[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMC_SIslands_MCRegisterSet SMC_SIslands_MCRegisterSet;
+
+struct SMC_SIslands_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMC_SIslands_MCRegisterAddress      address[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+    SMC_SIslands_MCRegisterSet          data[SMC_SISLANDS_MC_REGISTER_ARRAY_SET_COUNT];
+};
+
+typedef struct SMC_SIslands_MCRegisters SMC_SIslands_MCRegisters;
+
+struct SMC_SIslands_MCArbDramTimingRegisterSet
+{
+    uint32_t mc_arb_dram_timing;
+    uint32_t mc_arb_dram_timing2;
+    uint8_t  mc_arb_rfsh_rate;
+    uint8_t  mc_arb_burst_time;
+    uint8_t  padding[2];
+};
+
+typedef struct SMC_SIslands_MCArbDramTimingRegisterSet SMC_SIslands_MCArbDramTimingRegisterSet;
+
+struct SMC_SIslands_MCArbDramTimingRegisters
+{
+    uint8_t                                     arb_current;
+    uint8_t                                     reserved[3];
+    SMC_SIslands_MCArbDramTimingRegisterSet     data[16];
+};
+
+typedef struct SMC_SIslands_MCArbDramTimingRegisters SMC_SIslands_MCArbDramTimingRegisters;
+
+struct SMC_SISLANDS_SPLL_DIV_TABLE
+{
+    uint32_t    freq[256];
+    uint32_t    ss[256];
+};
+
+#define SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK  0x01ffffff
+#define SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT 0
+#define SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK   0xfe000000
+#define SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT  25
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK   0x000fffff
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT  0
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK   0xfff00000
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT  20
+
+typedef struct SMC_SISLANDS_SPLL_DIV_TABLE SMC_SISLANDS_SPLL_DIV_TABLE;
+
+#define SMC_SISLANDS_DTE_MAX_FILTER_STAGES 5
+
+#define SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE 16
+
+struct Smc_SIslands_DTE_Configuration
+{
+    uint32_t tau[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+    uint32_t R[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+    uint32_t K;
+    uint32_t T0;
+    uint32_t MaxT;
+    uint8_t  WindowSize;
+    uint8_t  Tdep_count;
+    uint8_t  temp_select;
+    uint8_t  DTE_mode;
+    uint8_t  T_limits[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+    uint32_t Tdep_tau[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+    uint32_t Tdep_R[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+    uint32_t Tthreshold;
+};
+
+typedef struct Smc_SIslands_DTE_Configuration Smc_SIslands_DTE_Configuration;
+
+#define SMC_SISLANDS_DTE_STATUS_FLAG_DTE_ON 1
+
+#define SISLANDS_SMC_FIRMWARE_HEADER_LOCATION 0x10000
+
+#define SISLANDS_SMC_FIRMWARE_HEADER_version                   0x0
+#define SISLANDS_SMC_FIRMWARE_HEADER_flags                     0x4
+#define SISLANDS_SMC_FIRMWARE_HEADER_softRegisters             0xC
+#define SISLANDS_SMC_FIRMWARE_HEADER_stateTable                0x10
+#define SISLANDS_SMC_FIRMWARE_HEADER_fanTable                  0x14
+#define SISLANDS_SMC_FIRMWARE_HEADER_CacConfigTable            0x18
+#define SISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable           0x24
+#define SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable 0x30
+#define SISLANDS_SMC_FIRMWARE_HEADER_spllTable                 0x38
+#define SISLANDS_SMC_FIRMWARE_HEADER_DteConfiguration          0x40
+#define SISLANDS_SMC_FIRMWARE_HEADER_PAPMParameters            0x48
+
+#pragma pack(pop)
+
+int si_set_smc_sram_address(struct radeon_device *rdev,
+                            u32 smc_address, u32 limit);
+int si_copy_bytes_to_smc(struct radeon_device *rdev,
+                         u32 smc_start_address,
+                         const u8 *src, u32 byte_count, u32 limit);
+void si_start_smc(struct radeon_device *rdev);
+void si_reset_smc(struct radeon_device *rdev);
+int si_program_jump_on_start(struct radeon_device *rdev);
+void si_stop_smc_clock(struct radeon_device *rdev);
+void si_start_smc_clock(struct radeon_device *rdev);
+bool si_is_smc_running(struct radeon_device *rdev);
+PPSMC_Result si_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg);
+PPSMC_Result si_wait_for_smc_inactive(struct radeon_device *rdev);
+int si_load_smc_ucode(struct radeon_device *rdev, u32 limit);
+int si_read_smc_sram_dword(struct radeon_device *rdev, u32 smc_address,
+                           u32 *value, u32 limit);
+int si_write_smc_sram_dword(struct radeon_device *rdev, u32 smc_address,
+                            u32 value, u32 limit);
+
+#endif
+
diff --git a/drivers/gpu/drm/radeon/sumo_dpm.c b/drivers/gpu/drm/radeon/sumo_dpm.c
new file mode 100644
index 000000000000..dc599060a9a4
--- /dev/null
+++ b/drivers/gpu/drm/radeon/sumo_dpm.c
@@ -0,0 +1,1832 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "sumod.h"
+#include "r600_dpm.h"
+#include "cypress_dpm.h"
+#include "sumo_dpm.h"
+#include <linux/seq_file.h>
+
+#define SUMO_MAX_DEEPSLEEP_DIVIDER_ID 5
+#define SUMO_MINIMUM_ENGINE_CLOCK 800
+#define BOOST_DPM_LEVEL 7
+
+static const u32 sumo_utc[SUMO_PM_NUMBER_OF_TC] =
+{
+        SUMO_UTC_DFLT_00,
+        SUMO_UTC_DFLT_01,
+        SUMO_UTC_DFLT_02,
+        SUMO_UTC_DFLT_03,
+        SUMO_UTC_DFLT_04,
+        SUMO_UTC_DFLT_05,
+        SUMO_UTC_DFLT_06,
+        SUMO_UTC_DFLT_07,
+        SUMO_UTC_DFLT_08,
+        SUMO_UTC_DFLT_09,
+        SUMO_UTC_DFLT_10,
+        SUMO_UTC_DFLT_11,
+        SUMO_UTC_DFLT_12,
+        SUMO_UTC_DFLT_13,
+        SUMO_UTC_DFLT_14,
+};
+
+static const u32 sumo_dtc[SUMO_PM_NUMBER_OF_TC] =
+{
+        SUMO_DTC_DFLT_00,
+        SUMO_DTC_DFLT_01,
+        SUMO_DTC_DFLT_02,
+        SUMO_DTC_DFLT_03,
+        SUMO_DTC_DFLT_04,
+        SUMO_DTC_DFLT_05,
+        SUMO_DTC_DFLT_06,
+        SUMO_DTC_DFLT_07,
+        SUMO_DTC_DFLT_08,
+        SUMO_DTC_DFLT_09,
+        SUMO_DTC_DFLT_10,
+        SUMO_DTC_DFLT_11,
+        SUMO_DTC_DFLT_12,
+        SUMO_DTC_DFLT_13,
+        SUMO_DTC_DFLT_14,
+};
+
+struct sumo_ps *sumo_get_ps(struct radeon_ps *rps)
+{
+        struct sumo_ps *ps = rps->ps_priv;
+
+        return ps;
+}
+
+struct sumo_power_info *sumo_get_pi(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+static void sumo_gfx_clockgating_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+        else {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+                RREG32(GB_ADDR_CONFIG);
+        }
+}
+
+#define CGCG_CGTT_LOCAL0_MASK 0xE5BFFFFF
+#define CGCG_CGTT_LOCAL1_MASK 0xEFFF07FF
+
+static void sumo_mg_clockgating_enable(struct radeon_device *rdev, bool enable)
+{
+        u32 local0;
+        u32 local1;
+
+        local0 = RREG32(CG_CGTT_LOCAL_0);
+        local1 = RREG32(CG_CGTT_LOCAL_1);
+
+        if (enable) {
+                WREG32(CG_CGTT_LOCAL_0, (0 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
+                WREG32(CG_CGTT_LOCAL_1, (0 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
+        } else {
+                WREG32(CG_CGTT_LOCAL_0, (0xFFFFFFFF & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
+                WREG32(CG_CGTT_LOCAL_1, (0xFFFFCFFF & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
+        }
+}
+
+static void sumo_program_git(struct radeon_device *rdev)
+{
+        u32 p, u;
+        u32 xclk = radeon_get_xclk(rdev);
+
+        r600_calculate_u_and_p(SUMO_GICST_DFLT,
+                               xclk, 16, &p, &u);
+
+        WREG32_P(CG_GIT, CG_GICST(p), ~CG_GICST_MASK);
+}
+
+static void sumo_program_grsd(struct radeon_device *rdev)
+{
+        u32 p, u;
+        u32 xclk = radeon_get_xclk(rdev);
+        u32 grs = 256 * 25 / 100;
+
+        r600_calculate_u_and_p(1, xclk, 14, &p, &u);
+
+        WREG32(CG_GCOOR, PHC(grs) | SDC(p) | SU(u));
+}
+
+void sumo_gfx_clockgating_initialize(struct radeon_device *rdev)
+{
+        sumo_program_git(rdev);
+        sumo_program_grsd(rdev);
+}
+
+static void sumo_gfx_powergating_initialize(struct radeon_device *rdev)
+{
+        u32 rcu_pwr_gating_cntl;
+        u32 p, u;
+        u32 p_c, p_p, d_p;
+        u32 r_t, i_t;
+        u32 xclk = radeon_get_xclk(rdev);
+
+        if (rdev->family == CHIP_PALM) {
+                p_c = 4;
+                d_p = 10;
+                r_t = 10;
+                i_t = 4;
+                p_p = 50 + 1000/200 + 6 * 32;
+        } else {
+                p_c = 16;
+                d_p = 50;
+                r_t = 50;
+                i_t  = 50;
+                p_p = 113;
+        }
+
+        WREG32(CG_SCRATCH2, 0x01B60A17);
+
+        r600_calculate_u_and_p(SUMO_GFXPOWERGATINGT_DFLT,
+                               xclk, 16, &p, &u);
+
+        WREG32_P(CG_PWR_GATING_CNTL, PGP(p) | PGU(u),
+                 ~(PGP_MASK | PGU_MASK));
+
+        r600_calculate_u_and_p(SUMO_VOLTAGEDROPT_DFLT,
+                               xclk, 16, &p, &u);
+
+        WREG32_P(CG_CG_VOLTAGE_CNTL, PGP(p) | PGU(u),
+                 ~(PGP_MASK | PGU_MASK));
+
+        if (rdev->family == CHIP_PALM) {
+                WREG32_RCU(RCU_PWR_GATING_SEQ0, 0x10103210);
+                WREG32_RCU(RCU_PWR_GATING_SEQ1, 0x10101010);
+        } else {
+                WREG32_RCU(RCU_PWR_GATING_SEQ0, 0x76543210);
+                WREG32_RCU(RCU_PWR_GATING_SEQ1, 0xFEDCBA98);
+        }
+
+        rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL);
+        rcu_pwr_gating_cntl &=
+                ~(RSVD_MASK | PCV_MASK | PGS_MASK);
+        rcu_pwr_gating_cntl |= PCV(p_c) | PGS(1) | PWR_GATING_EN;
+        if (rdev->family == CHIP_PALM) {
+                rcu_pwr_gating_cntl &= ~PCP_MASK;
+                rcu_pwr_gating_cntl |= PCP(0x77);
+        }
+        WREG32_RCU(RCU_PWR_GATING_CNTL, rcu_pwr_gating_cntl);
+
+        rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_2);
+        rcu_pwr_gating_cntl &= ~(MPPU_MASK | MPPD_MASK);
+        rcu_pwr_gating_cntl |= MPPU(p_p) | MPPD(50);
+        WREG32_RCU(RCU_PWR_GATING_CNTL_2, rcu_pwr_gating_cntl);
+
+        rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_3);
+        rcu_pwr_gating_cntl &= ~(DPPU_MASK | DPPD_MASK);
+        rcu_pwr_gating_cntl |= DPPU(d_p) | DPPD(50);
+        WREG32_RCU(RCU_PWR_GATING_CNTL_3, rcu_pwr_gating_cntl);
+
+        rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_4);
+        rcu_pwr_gating_cntl &= ~(RT_MASK | IT_MASK);
+        rcu_pwr_gating_cntl |= RT(r_t) | IT(i_t);
+        WREG32_RCU(RCU_PWR_GATING_CNTL_4, rcu_pwr_gating_cntl);
+
+        if (rdev->family == CHIP_PALM)
+                WREG32_RCU(RCU_PWR_GATING_CNTL_5, 0xA02);
+
+        sumo_smu_pg_init(rdev);
+
+        rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL);
+        rcu_pwr_gating_cntl &=
+                ~(RSVD_MASK | PCV_MASK | PGS_MASK);
+        rcu_pwr_gating_cntl |= PCV(p_c) | PGS(4) | PWR_GATING_EN;
+        if (rdev->family == CHIP_PALM) {
+                rcu_pwr_gating_cntl &= ~PCP_MASK;
+                rcu_pwr_gating_cntl |= PCP(0x77);
+        }
+        WREG32_RCU(RCU_PWR_GATING_CNTL, rcu_pwr_gating_cntl);
+
+        if (rdev->family == CHIP_PALM) {
+                rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_2);
+                rcu_pwr_gating_cntl &= ~(MPPU_MASK | MPPD_MASK);
+                rcu_pwr_gating_cntl |= MPPU(113) | MPPD(50);
+                WREG32_RCU(RCU_PWR_GATING_CNTL_2, rcu_pwr_gating_cntl);
+
+                rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_3);
+                rcu_pwr_gating_cntl &= ~(DPPU_MASK | DPPD_MASK);
+                rcu_pwr_gating_cntl |= DPPU(16) | DPPD(50);
+                WREG32_RCU(RCU_PWR_GATING_CNTL_3, rcu_pwr_gating_cntl);
+        }
+
+        sumo_smu_pg_init(rdev);
+
+        rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL);
+        rcu_pwr_gating_cntl &=
+                ~(RSVD_MASK | PCV_MASK | PGS_MASK);
+        rcu_pwr_gating_cntl |= PGS(5) | PWR_GATING_EN;
+
+        if (rdev->family == CHIP_PALM) {
+                rcu_pwr_gating_cntl |= PCV(4);
+                rcu_pwr_gating_cntl &= ~PCP_MASK;
+                rcu_pwr_gating_cntl |= PCP(0x77);
+        } else
+                rcu_pwr_gating_cntl |= PCV(11);
+        WREG32_RCU(RCU_PWR_GATING_CNTL, rcu_pwr_gating_cntl);
+
+        if (rdev->family == CHIP_PALM) {
+                rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_2);
+                rcu_pwr_gating_cntl &= ~(MPPU_MASK | MPPD_MASK);
+                rcu_pwr_gating_cntl |= MPPU(113) | MPPD(50);
+                WREG32_RCU(RCU_PWR_GATING_CNTL_2, rcu_pwr_gating_cntl);
+
+                rcu_pwr_gating_cntl = RREG32_RCU(RCU_PWR_GATING_CNTL_3);
+                rcu_pwr_gating_cntl &= ~(DPPU_MASK | DPPD_MASK);
+                rcu_pwr_gating_cntl |= DPPU(22) | DPPD(50);
+                WREG32_RCU(RCU_PWR_GATING_CNTL_3, rcu_pwr_gating_cntl);
+        }
+
+        sumo_smu_pg_init(rdev);
+}
+
+static void sumo_gfx_powergating_enable(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(CG_PWR_GATING_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
+        else {
+                WREG32_P(CG_PWR_GATING_CNTL, 0, ~DYN_PWR_DOWN_EN);
+                RREG32(GB_ADDR_CONFIG);
+        }
+}
+
+static int sumo_enable_clock_power_gating(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        if (pi->enable_gfx_clock_gating)
+                sumo_gfx_clockgating_initialize(rdev);
+        if (pi->enable_gfx_power_gating)
+                sumo_gfx_powergating_initialize(rdev);
+        if (pi->enable_mg_clock_gating)
+                sumo_mg_clockgating_enable(rdev, true);
+        if (pi->enable_gfx_clock_gating)
+                sumo_gfx_clockgating_enable(rdev, true);
+        if (pi->enable_gfx_power_gating)
+                sumo_gfx_powergating_enable(rdev, true);
+
+        return 0;
+}
+
+static void sumo_disable_clock_power_gating(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        if (pi->enable_gfx_clock_gating)
+                sumo_gfx_clockgating_enable(rdev, false);
+        if (pi->enable_gfx_power_gating)
+                sumo_gfx_powergating_enable(rdev, false);
+        if (pi->enable_mg_clock_gating)
+                sumo_mg_clockgating_enable(rdev, false);
+}
+
+static void sumo_calculate_bsp(struct radeon_device *rdev,
+                               u32 high_clk)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 xclk = radeon_get_xclk(rdev);
+
+        pi->pasi = 65535 * 100 / high_clk;
+        pi->asi = 65535 * 100 / high_clk;
+
+        r600_calculate_u_and_p(pi->asi,
+                               xclk, 16, &pi->bsp, &pi->bsu);
+
+        r600_calculate_u_and_p(pi->pasi,
+                               xclk, 16, &pi->pbsp, &pi->pbsu);
+
+        pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
+        pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
+}
+
+static void sumo_init_bsp(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        WREG32(CG_BSP_0, pi->psp);
+}
+
+
+static void sumo_program_bsp(struct radeon_device *rdev,
+                             struct radeon_ps *rps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *ps = sumo_get_ps(rps);
+        u32 i;
+        u32 highest_engine_clock = ps->levels[ps->num_levels - 1].sclk;
+
+        if (ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE)
+                highest_engine_clock = pi->boost_pl.sclk;
+
+        sumo_calculate_bsp(rdev, highest_engine_clock);
+
+        for (i = 0; i < ps->num_levels - 1; i++)
+                WREG32(CG_BSP_0 + (i * 4), pi->dsp);
+
+        WREG32(CG_BSP_0 + (i * 4), pi->psp);
+
+        if (ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE)
+                WREG32(CG_BSP_0 + (BOOST_DPM_LEVEL * 4), pi->psp);
+}
+
+static void sumo_write_at(struct radeon_device *rdev,
+                          u32 index, u32 value)
+{
+        if (index == 0)
+                WREG32(CG_AT_0, value);
+        else if (index == 1)
+                WREG32(CG_AT_1, value);
+        else if (index == 2)
+                WREG32(CG_AT_2, value);
+        else if (index == 3)
+                WREG32(CG_AT_3, value);
+        else if (index == 4)
+                WREG32(CG_AT_4, value);
+        else if (index == 5)
+                WREG32(CG_AT_5, value);
+        else if (index == 6)
+                WREG32(CG_AT_6, value);
+        else if (index == 7)
+                WREG32(CG_AT_7, value);
+}
+
+static void sumo_program_at(struct radeon_device *rdev,
+                            struct radeon_ps *rps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *ps = sumo_get_ps(rps);
+        u32 asi;
+        u32 i;
+        u32 m_a;
+        u32 a_t;
+        u32 r[SUMO_MAX_HARDWARE_POWERLEVELS];
+        u32 l[SUMO_MAX_HARDWARE_POWERLEVELS];
+
+        r[0] = SUMO_R_DFLT0;
+        r[1] = SUMO_R_DFLT1;
+        r[2] = SUMO_R_DFLT2;
+        r[3] = SUMO_R_DFLT3;
+        r[4] = SUMO_R_DFLT4;
+
+        l[0] = SUMO_L_DFLT0;
+        l[1] = SUMO_L_DFLT1;
+        l[2] = SUMO_L_DFLT2;
+        l[3] = SUMO_L_DFLT3;
+        l[4] = SUMO_L_DFLT4;
+
+        for (i = 0; i < ps->num_levels; i++) {
+                asi = (i == ps->num_levels - 1) ? pi->pasi : pi->asi;
+
+                m_a = asi * ps->levels[i].sclk / 100;
+
+                a_t = CG_R(m_a * r[i] / 100) | CG_L(m_a * l[i] / 100);
+
+                sumo_write_at(rdev, i, a_t);
+        }
+
+        if (ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) {
+                asi = pi->pasi;
+
+                m_a = asi * pi->boost_pl.sclk / 100;
+
+                a_t = CG_R(m_a * r[ps->num_levels - 1] / 100) |
+                        CG_L(m_a * l[ps->num_levels - 1] / 100);
+
+                sumo_write_at(rdev, BOOST_DPM_LEVEL, a_t);
+        }
+}
+
+static void sumo_program_tp(struct radeon_device *rdev)
+{
+        int i;
+        enum r600_td td = R600_TD_DFLT;
+
+        for (i = 0; i < SUMO_PM_NUMBER_OF_TC; i++) {
+                WREG32_P(CG_FFCT_0 + (i * 4), UTC_0(sumo_utc[i]), ~UTC_0_MASK);
+                WREG32_P(CG_FFCT_0 + (i * 4), DTC_0(sumo_dtc[i]), ~DTC_0_MASK);
+        }
+
+        if (td == R600_TD_AUTO)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
+        else
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
+
+        if (td == R600_TD_UP)
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
+
+        if (td == R600_TD_DOWN)
+                WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
+}
+
+void sumo_program_vc(struct radeon_device *rdev, u32 vrc)
+{
+        WREG32(CG_FTV, vrc);
+}
+
+void sumo_clear_vc(struct radeon_device *rdev)
+{
+        WREG32(CG_FTV, 0);
+}
+
+void sumo_program_sstp(struct radeon_device *rdev)
+{
+        u32 p, u;
+        u32 xclk = radeon_get_xclk(rdev);
+
+        r600_calculate_u_and_p(SUMO_SST_DFLT,
+                               xclk, 16, &p, &u);
+
+        WREG32(CG_SSP, SSTU(u) | SST(p));
+}
+
+static void sumo_set_divider_value(struct radeon_device *rdev,
+                                   u32 index, u32 divider)
+{
+        u32 reg_index = index / 4;
+        u32 field_index = index % 4;
+
+        if (field_index == 0)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         SCLK_FSTATE_0_DIV(divider), ~SCLK_FSTATE_0_DIV_MASK);
+        else if (field_index == 1)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         SCLK_FSTATE_1_DIV(divider), ~SCLK_FSTATE_1_DIV_MASK);
+        else if (field_index == 2)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         SCLK_FSTATE_2_DIV(divider), ~SCLK_FSTATE_2_DIV_MASK);
+        else if (field_index == 3)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         SCLK_FSTATE_3_DIV(divider), ~SCLK_FSTATE_3_DIV_MASK);
+}
+
+static void sumo_set_ds_dividers(struct radeon_device *rdev,
+                                 u32 index, u32 divider)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        if (pi->enable_sclk_ds) {
+                u32 dpm_ctrl = RREG32(CG_SCLK_DPM_CTRL_6);
+
+                dpm_ctrl &= ~(0x7 << (index * 3));
+                dpm_ctrl |= (divider << (index * 3));
+                WREG32(CG_SCLK_DPM_CTRL_6, dpm_ctrl);
+        }
+}
+
+static void sumo_set_ss_dividers(struct radeon_device *rdev,
+                                 u32 index, u32 divider)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        if (pi->enable_sclk_ds) {
+                u32 dpm_ctrl = RREG32(CG_SCLK_DPM_CTRL_11);
+
+                dpm_ctrl &= ~(0x7 << (index * 3));
+                dpm_ctrl |= (divider << (index * 3));
+                WREG32(CG_SCLK_DPM_CTRL_11, dpm_ctrl);
+        }
+}
+
+static void sumo_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
+{
+        u32 voltage_cntl = RREG32(CG_DPM_VOLTAGE_CNTL);
+
+        voltage_cntl &= ~(DPM_STATE0_LEVEL_MASK << (index * 2));
+        voltage_cntl |= (vid << (DPM_STATE0_LEVEL_SHIFT + index * 2));
+        WREG32(CG_DPM_VOLTAGE_CNTL, voltage_cntl);
+}
+
+static void sumo_set_allos_gnb_slow(struct radeon_device *rdev, u32 index, u32 gnb_slow)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 temp = gnb_slow;
+        u32 cg_sclk_dpm_ctrl_3;
+
+        if (pi->driver_nbps_policy_disable)
+                temp = 1;
+
+        cg_sclk_dpm_ctrl_3 = RREG32(CG_SCLK_DPM_CTRL_3);
+        cg_sclk_dpm_ctrl_3 &= ~(GNB_SLOW_FSTATE_0_MASK << index);
+        cg_sclk_dpm_ctrl_3 |= (temp << (GNB_SLOW_FSTATE_0_SHIFT + index));
+
+        WREG32(CG_SCLK_DPM_CTRL_3, cg_sclk_dpm_ctrl_3);
+}
+
+static void sumo_program_power_level(struct radeon_device *rdev,
+                                     struct sumo_pl *pl, u32 index)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        int ret;
+        struct atom_clock_dividers dividers;
+        u32 ds_en = RREG32(DEEP_SLEEP_CNTL) & ENABLE_DS;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             pl->sclk, false, &dividers);
+        if (ret)
+                return;
+
+        sumo_set_divider_value(rdev, index, dividers.post_div);
+
+        sumo_set_vid(rdev, index, pl->vddc_index);
+
+        if (pl->ss_divider_index == 0 || pl->ds_divider_index == 0) {
+                if (ds_en)
+                        WREG32_P(DEEP_SLEEP_CNTL, 0, ~ENABLE_DS);
+        } else {
+                sumo_set_ss_dividers(rdev, index, pl->ss_divider_index);
+                sumo_set_ds_dividers(rdev, index, pl->ds_divider_index);
+
+                if (!ds_en)
+                        WREG32_P(DEEP_SLEEP_CNTL, ENABLE_DS, ~ENABLE_DS);
+        }
+
+        sumo_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
+
+        if (pi->enable_boost)
+                sumo_set_tdp_limit(rdev, index, pl->sclk_dpm_tdp_limit);
+}
+
+static void sumo_power_level_enable(struct radeon_device *rdev, u32 index, bool enable)
+{
+        u32 reg_index = index / 4;
+        u32 field_index = index % 4;
+
+        if (field_index == 0)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         enable ? SCLK_FSTATE_0_VLD : 0, ~SCLK_FSTATE_0_VLD);
+        else if (field_index == 1)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         enable ? SCLK_FSTATE_1_VLD : 0, ~SCLK_FSTATE_1_VLD);
+        else if (field_index == 2)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         enable ? SCLK_FSTATE_2_VLD : 0, ~SCLK_FSTATE_2_VLD);
+        else if (field_index == 3)
+                WREG32_P(CG_SCLK_DPM_CTRL + (reg_index * 4),
+                         enable ? SCLK_FSTATE_3_VLD : 0, ~SCLK_FSTATE_3_VLD);
+}
+
+static bool sumo_dpm_enabled(struct radeon_device *rdev)
+{
+        if (RREG32(CG_SCLK_DPM_CTRL_3) & DPM_SCLK_ENABLE)
+                return true;
+        else
+                return false;
+}
+
+static void sumo_start_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(CG_SCLK_DPM_CTRL_3, DPM_SCLK_ENABLE, ~DPM_SCLK_ENABLE);
+}
+
+static void sumo_stop_dpm(struct radeon_device *rdev)
+{
+        WREG32_P(CG_SCLK_DPM_CTRL_3, 0, ~DPM_SCLK_ENABLE);
+}
+
+static void sumo_set_forced_mode(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_P(CG_SCLK_DPM_CTRL_3, FORCE_SCLK_STATE_EN, ~FORCE_SCLK_STATE_EN);
+        else
+                WREG32_P(CG_SCLK_DPM_CTRL_3, 0, ~FORCE_SCLK_STATE_EN);
+}
+
+static void sumo_set_forced_mode_enabled(struct radeon_device *rdev)
+{
+        int i;
+
+        sumo_set_forced_mode(rdev, true);
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(CG_SCLK_STATUS) & SCLK_OVERCLK_DETECT)
+                        break;
+                udelay(1);
+        }
+}
+
+static void sumo_wait_for_level_0(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_SCLK_INDEX_MASK) == 0)
+                        break;
+                udelay(1);
+        }
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_INDEX_MASK) == 0)
+                        break;
+                udelay(1);
+        }
+}
+
+static void sumo_set_forced_mode_disabled(struct radeon_device *rdev)
+{
+        sumo_set_forced_mode(rdev, false);
+}
+
+static void sumo_enable_power_level_0(struct radeon_device *rdev)
+{
+        sumo_power_level_enable(rdev, 0, true);
+}
+
+static void sumo_patch_boost_state(struct radeon_device *rdev,
+                                   struct radeon_ps *rps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *new_ps = sumo_get_ps(rps);
+
+        if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE) {
+                pi->boost_pl = new_ps->levels[new_ps->num_levels - 1];
+                pi->boost_pl.sclk = pi->sys_info.boost_sclk;
+                pi->boost_pl.vddc_index = pi->sys_info.boost_vid_2bit;
+                pi->boost_pl.sclk_dpm_tdp_limit = pi->sys_info.sclk_dpm_tdp_limit_boost;
+        }
+}
+
+static void sumo_pre_notify_alt_vddnb_change(struct radeon_device *rdev,
+                                             struct radeon_ps *new_rps,
+                                             struct radeon_ps *old_rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(new_rps);
+        struct sumo_ps *old_ps = sumo_get_ps(old_rps);
+        u32 nbps1_old = 0;
+        u32 nbps1_new = 0;
+
+        if (old_ps != NULL)
+                nbps1_old = (old_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE) ? 1 : 0;
+
+        nbps1_new = (new_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE) ? 1 : 0;
+
+        if (nbps1_old == 1 && nbps1_new == 0)
+                sumo_smu_notify_alt_vddnb_change(rdev, 0, 0);
+}
+
+static void sumo_post_notify_alt_vddnb_change(struct radeon_device *rdev,
+                                              struct radeon_ps *new_rps,
+                                              struct radeon_ps *old_rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(new_rps);
+        struct sumo_ps *old_ps = sumo_get_ps(old_rps);
+        u32 nbps1_old = 0;
+        u32 nbps1_new = 0;
+
+        if (old_ps != NULL)
+                nbps1_old = (old_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE)? 1 : 0;
+
+        nbps1_new = (new_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE)? 1 : 0;
+
+        if (nbps1_old == 0 && nbps1_new == 1)
+                sumo_smu_notify_alt_vddnb_change(rdev, 1, 1);
+}
+
+static void sumo_enable_boost(struct radeon_device *rdev,
+                              struct radeon_ps *rps,
+                              bool enable)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(rps);
+
+        if (enable) {
+                if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE)
+                        sumo_boost_state_enable(rdev, true);
+        } else
+                sumo_boost_state_enable(rdev, false);
+}
+
+static void sumo_set_forced_level(struct radeon_device *rdev, u32 index)
+{
+        WREG32_P(CG_SCLK_DPM_CTRL_3, FORCE_SCLK_STATE(index), ~FORCE_SCLK_STATE_MASK);
+}
+
+static void sumo_set_forced_level_0(struct radeon_device *rdev)
+{
+        sumo_set_forced_level(rdev, 0);
+}
+
+static void sumo_program_wl(struct radeon_device *rdev,
+                            struct radeon_ps *rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(rps);
+        u32 dpm_ctrl4 = RREG32(CG_SCLK_DPM_CTRL_4);
+
+        dpm_ctrl4 &= 0xFFFFFF00;
+        dpm_ctrl4 |= (1 << (new_ps->num_levels - 1));
+
+        if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE)
+                dpm_ctrl4 |= (1 << BOOST_DPM_LEVEL);
+
+        WREG32(CG_SCLK_DPM_CTRL_4, dpm_ctrl4);
+}
+
+static void sumo_program_power_levels_0_to_n(struct radeon_device *rdev,
+                                             struct radeon_ps *new_rps,
+                                             struct radeon_ps *old_rps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *new_ps = sumo_get_ps(new_rps);
+        struct sumo_ps *old_ps = sumo_get_ps(old_rps);
+        u32 i;
+        u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
+
+        for (i = 0; i < new_ps->num_levels; i++) {
+                sumo_program_power_level(rdev, &new_ps->levels[i], i);
+                sumo_power_level_enable(rdev, i, true);
+        }
+
+        for (i = new_ps->num_levels; i < n_current_state_levels; i++)
+                sumo_power_level_enable(rdev, i, false);
+
+        if (new_ps->flags & SUMO_POWERSTATE_FLAGS_BOOST_STATE)
+                sumo_program_power_level(rdev, &pi->boost_pl, BOOST_DPM_LEVEL);
+}
+
+static void sumo_enable_acpi_pm(struct radeon_device *rdev)
+{
+        WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
+}
+
+static void sumo_program_power_level_enter_state(struct radeon_device *rdev)
+{
+        WREG32_P(CG_SCLK_DPM_CTRL_5, SCLK_FSTATE_BOOTUP(0), ~SCLK_FSTATE_BOOTUP_MASK);
+}
+
+static void sumo_program_acpi_power_level(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct atom_clock_dividers dividers;
+        int ret;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             pi->acpi_pl.sclk,
+                                             false, &dividers);
+        if (ret)
+                return;
+
+        WREG32_P(CG_ACPI_CNTL, SCLK_ACPI_DIV(dividers.post_div), ~SCLK_ACPI_DIV_MASK);
+        WREG32_P(CG_ACPI_VOLTAGE_CNTL, 0, ~ACPI_VOLTAGE_EN);
+}
+
+static void sumo_program_bootup_state(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 dpm_ctrl4 = RREG32(CG_SCLK_DPM_CTRL_4);
+        u32 i;
+
+        sumo_program_power_level(rdev, &pi->boot_pl, 0);
+
+        dpm_ctrl4 &= 0xFFFFFF00;
+        WREG32(CG_SCLK_DPM_CTRL_4, dpm_ctrl4);
+
+        for (i = 1; i < 8; i++)
+                sumo_power_level_enable(rdev, i, false);
+}
+
+static void sumo_setup_uvd_clocks(struct radeon_device *rdev,
+                                  struct radeon_ps *new_rps,
+                                  struct radeon_ps *old_rps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        if (pi->enable_gfx_power_gating) {
+                sumo_gfx_powergating_enable(rdev, false);
+        }
+
+        radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
+
+        if (pi->enable_gfx_power_gating) {
+                if (!pi->disable_gfx_power_gating_in_uvd ||
+                    !r600_is_uvd_state(new_rps->class, new_rps->class2))
+                        sumo_gfx_powergating_enable(rdev, true);
+        }
+}
+
+static void sumo_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                                    struct radeon_ps *new_rps,
+                                                    struct radeon_ps *old_rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(new_rps);
+        struct sumo_ps *current_ps = sumo_get_ps(old_rps);
+
+        if ((new_rps->vclk == old_rps->vclk) &&
+            (new_rps->dclk == old_rps->dclk))
+                return;
+
+        if (new_ps->levels[new_ps->num_levels - 1].sclk >=
+            current_ps->levels[current_ps->num_levels - 1].sclk)
+                return;
+
+        sumo_setup_uvd_clocks(rdev, new_rps, old_rps);
+}
+
+static void sumo_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                                   struct radeon_ps *new_rps,
+                                                   struct radeon_ps *old_rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(new_rps);
+        struct sumo_ps *current_ps = sumo_get_ps(old_rps);
+
+        if ((new_rps->vclk == old_rps->vclk) &&
+            (new_rps->dclk == old_rps->dclk))
+                return;
+
+        if (new_ps->levels[new_ps->num_levels - 1].sclk <
+            current_ps->levels[current_ps->num_levels - 1].sclk)
+                return;
+
+        sumo_setup_uvd_clocks(rdev, new_rps, old_rps);
+}
+
+void sumo_take_smu_control(struct radeon_device *rdev, bool enable)
+{
+/* This bit selects who handles display phy powergating.
+ * Clear the bit to let atom handle it.
+ * Set it to let the driver handle it.
+ * For now we just let atom handle it.
+ */
+#if 0
+        u32 v = RREG32(DOUT_SCRATCH3);
+
+        if (enable)
+                v |= 0x4;
+        else
+                v &= 0xFFFFFFFB;
+
+        WREG32(DOUT_SCRATCH3, v);
+#endif
+}
+
+static void sumo_enable_sclk_ds(struct radeon_device *rdev, bool enable)
+{
+        if (enable) {
+                u32 deep_sleep_cntl = RREG32(DEEP_SLEEP_CNTL);
+                u32 deep_sleep_cntl2 = RREG32(DEEP_SLEEP_CNTL2);
+                u32 t = 1;
+
+                deep_sleep_cntl &= ~R_DIS;
+                deep_sleep_cntl &= ~HS_MASK;
+                deep_sleep_cntl |= HS(t > 4095 ? 4095 : t);
+
+                deep_sleep_cntl2 |= LB_UFP_EN;
+                deep_sleep_cntl2 &= INOUT_C_MASK;
+                deep_sleep_cntl2 |= INOUT_C(0xf);
+
+                WREG32(DEEP_SLEEP_CNTL2, deep_sleep_cntl2);
+                WREG32(DEEP_SLEEP_CNTL, deep_sleep_cntl);
+        } else
+                WREG32_P(DEEP_SLEEP_CNTL, 0, ~ENABLE_DS);
+}
+
+static void sumo_program_bootup_at(struct radeon_device *rdev)
+{
+        WREG32_P(CG_AT_0, CG_R(0xffff), ~CG_R_MASK);
+        WREG32_P(CG_AT_0, CG_L(0), ~CG_L_MASK);
+}
+
+static void sumo_reset_am(struct radeon_device *rdev)
+{
+        WREG32_P(SCLK_PWRMGT_CNTL, FIR_RESET, ~FIR_RESET);
+}
+
+static void sumo_start_am(struct radeon_device *rdev)
+{
+        WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_RESET);
+}
+
+static void sumo_program_ttp(struct radeon_device *rdev)
+{
+        u32 xclk = radeon_get_xclk(rdev);
+        u32 p, u;
+        u32 cg_sclk_dpm_ctrl_5 = RREG32(CG_SCLK_DPM_CTRL_5);
+
+        r600_calculate_u_and_p(1000,
+                               xclk, 16, &p, &u);
+
+        cg_sclk_dpm_ctrl_5 &= ~(TT_TP_MASK | TT_TU_MASK);
+        cg_sclk_dpm_ctrl_5 |= TT_TP(p) | TT_TU(u);
+
+        WREG32(CG_SCLK_DPM_CTRL_5, cg_sclk_dpm_ctrl_5);
+}
+
+static void sumo_program_ttt(struct radeon_device *rdev)
+{
+        u32 cg_sclk_dpm_ctrl_3 = RREG32(CG_SCLK_DPM_CTRL_3);
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        cg_sclk_dpm_ctrl_3 &= ~(GNB_TT_MASK | GNB_THERMTHRO_MASK);
+        cg_sclk_dpm_ctrl_3 |= GNB_TT(pi->thermal_auto_throttling + 49);
+
+        WREG32(CG_SCLK_DPM_CTRL_3, cg_sclk_dpm_ctrl_3);
+}
+
+
+static void sumo_enable_voltage_scaling(struct radeon_device *rdev, bool enable)
+{
+        if (enable) {
+                WREG32_P(CG_DPM_VOLTAGE_CNTL, DPM_VOLTAGE_EN, ~DPM_VOLTAGE_EN);
+                WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~CG_VOLTAGE_EN);
+        } else {
+                WREG32_P(CG_CG_VOLTAGE_CNTL, CG_VOLTAGE_EN, ~CG_VOLTAGE_EN);
+                WREG32_P(CG_DPM_VOLTAGE_CNTL, 0, ~DPM_VOLTAGE_EN);
+        }
+}
+
+static void sumo_override_cnb_thermal_events(struct radeon_device *rdev)
+{
+        WREG32_P(CG_SCLK_DPM_CTRL_3, CNB_THERMTHRO_MASK_SCLK,
+                 ~CNB_THERMTHRO_MASK_SCLK);
+}
+
+static void sumo_program_dc_hto(struct radeon_device *rdev)
+{
+        u32 cg_sclk_dpm_ctrl_4 = RREG32(CG_SCLK_DPM_CTRL_4);
+        u32 p, u;
+        u32 xclk = radeon_get_xclk(rdev);
+
+        r600_calculate_u_and_p(100000,
+                               xclk, 14, &p, &u);
+
+        cg_sclk_dpm_ctrl_4 &= ~(DC_HDC_MASK | DC_HU_MASK);
+        cg_sclk_dpm_ctrl_4 |= DC_HDC(p) | DC_HU(u);
+
+        WREG32(CG_SCLK_DPM_CTRL_4, cg_sclk_dpm_ctrl_4);
+}
+
+static void sumo_force_nbp_state(struct radeon_device *rdev,
+                                 struct radeon_ps *rps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *new_ps = sumo_get_ps(rps);
+
+        if (!pi->driver_nbps_policy_disable) {
+                if (new_ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE)
+                        WREG32_P(CG_SCLK_DPM_CTRL_3, FORCE_NB_PSTATE_1, ~FORCE_NB_PSTATE_1);
+                else
+                        WREG32_P(CG_SCLK_DPM_CTRL_3, 0, ~FORCE_NB_PSTATE_1);
+        }
+}
+
+u32 sumo_get_sleep_divider_from_id(u32 id)
+{
+        return 1 << id;
+}
+
+u32 sumo_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
+                                         u32 sclk,
+                                         u32 min_sclk_in_sr)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 i;
+        u32 temp;
+        u32 min = (min_sclk_in_sr > SUMO_MINIMUM_ENGINE_CLOCK) ?
+                min_sclk_in_sr : SUMO_MINIMUM_ENGINE_CLOCK;
+
+        if (sclk < min)
+                return 0;
+
+        if (!pi->enable_sclk_ds)
+                return 0;
+
+        for (i = SUMO_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
+                temp = sclk / sumo_get_sleep_divider_from_id(i);
+
+                if (temp >= min || i == 0)
+                        break;
+        }
+        return i;
+}
+
+static u32 sumo_get_valid_engine_clock(struct radeon_device *rdev,
+                                       u32 lower_limit)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 i;
+
+        for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
+                if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
+                        return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
+        }
+
+        return pi->sys_info.sclk_voltage_mapping_table.entries[pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries - 1].sclk_frequency;
+}
+
+static void sumo_patch_thermal_state(struct radeon_device *rdev,
+                                     struct sumo_ps *ps,
+                                     struct sumo_ps *current_ps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
+        u32 current_vddc;
+        u32 current_sclk;
+        u32 current_index = 0;
+
+        if (current_ps) {
+                current_vddc = current_ps->levels[current_index].vddc_index;
+                current_sclk = current_ps->levels[current_index].sclk;
+        } else {
+                current_vddc = pi->boot_pl.vddc_index;
+                current_sclk = pi->boot_pl.sclk;
+        }
+
+        ps->levels[0].vddc_index = current_vddc;
+
+        if (ps->levels[0].sclk > current_sclk)
+                ps->levels[0].sclk = current_sclk;
+
+        ps->levels[0].ss_divider_index =
+                sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
+
+        ps->levels[0].ds_divider_index =
+                sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, SUMO_MINIMUM_ENGINE_CLOCK);
+
+        if (ps->levels[0].ds_divider_index > ps->levels[0].ss_divider_index + 1)
+                ps->levels[0].ds_divider_index = ps->levels[0].ss_divider_index + 1;
+
+        if (ps->levels[0].ss_divider_index == ps->levels[0].ds_divider_index) {
+                if (ps->levels[0].ss_divider_index > 1)
+                        ps->levels[0].ss_divider_index = ps->levels[0].ss_divider_index - 1;
+        }
+
+        if (ps->levels[0].ss_divider_index == 0)
+                ps->levels[0].ds_divider_index = 0;
+
+        if (ps->levels[0].ds_divider_index == 0)
+                ps->levels[0].ss_divider_index = 0;
+}
+
+static void sumo_apply_state_adjust_rules(struct radeon_device *rdev,
+                                          struct radeon_ps *new_rps,
+                                          struct radeon_ps *old_rps)
+{
+        struct sumo_ps *ps = sumo_get_ps(new_rps);
+        struct sumo_ps *current_ps = sumo_get_ps(old_rps);
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 min_voltage = 0; /* ??? */
+        u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
+        u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
+        u32 i;
+
+        if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
+                return sumo_patch_thermal_state(rdev, ps, current_ps);
+
+        if (pi->enable_boost) {
+                if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE)
+                        ps->flags |= SUMO_POWERSTATE_FLAGS_BOOST_STATE;
+        }
+
+        if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) ||
+            (new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) ||
+            (new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE))
+                ps->flags |= SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE;
+
+        for (i = 0; i < ps->num_levels; i++) {
+                if (ps->levels[i].vddc_index < min_voltage)
+                        ps->levels[i].vddc_index = min_voltage;
+
+                if (ps->levels[i].sclk < min_sclk)
+                        ps->levels[i].sclk =
+                                sumo_get_valid_engine_clock(rdev, min_sclk);
+
+                ps->levels[i].ss_divider_index =
+                        sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
+
+                ps->levels[i].ds_divider_index =
+                        sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, SUMO_MINIMUM_ENGINE_CLOCK);
+
+                if (ps->levels[i].ds_divider_index > ps->levels[i].ss_divider_index + 1)
+                        ps->levels[i].ds_divider_index = ps->levels[i].ss_divider_index + 1;
+
+                if (ps->levels[i].ss_divider_index == ps->levels[i].ds_divider_index) {
+                        if (ps->levels[i].ss_divider_index > 1)
+                                ps->levels[i].ss_divider_index = ps->levels[i].ss_divider_index - 1;
+                }
+
+                if (ps->levels[i].ss_divider_index == 0)
+                        ps->levels[i].ds_divider_index = 0;
+
+                if (ps->levels[i].ds_divider_index == 0)
+                        ps->levels[i].ss_divider_index = 0;
+
+                if (ps->flags & SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE)
+                        ps->levels[i].allow_gnb_slow = 1;
+                else if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) ||
+                         (new_rps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC))
+                        ps->levels[i].allow_gnb_slow = 0;
+                else if (i == ps->num_levels - 1)
+                        ps->levels[i].allow_gnb_slow = 0;
+                else
+                        ps->levels[i].allow_gnb_slow = 1;
+        }
+}
+
+static void sumo_cleanup_asic(struct radeon_device *rdev)
+{
+        sumo_take_smu_control(rdev, false);
+}
+
+static int sumo_set_thermal_temperature_range(struct radeon_device *rdev,
+                                              int min_temp, int max_temp)
+{
+        int low_temp = 0 * 1000;
+        int high_temp = 255 * 1000;
+
+        if (low_temp < min_temp)
+                low_temp = min_temp;
+        if (high_temp > max_temp)
+                high_temp = max_temp;
+        if (high_temp < low_temp) {
+                DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
+                return -EINVAL;
+        }
+
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
+        WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
+
+        rdev->pm.dpm.thermal.min_temp = low_temp;
+        rdev->pm.dpm.thermal.max_temp = high_temp;
+
+        return 0;
+}
+
+static void sumo_update_current_ps(struct radeon_device *rdev,
+                                   struct radeon_ps *rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(rps);
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        pi->current_rps = *rps;
+        pi->current_ps = *new_ps;
+        pi->current_rps.ps_priv = &pi->current_ps;
+}
+
+static void sumo_update_requested_ps(struct radeon_device *rdev,
+                                     struct radeon_ps *rps)
+{
+        struct sumo_ps *new_ps = sumo_get_ps(rps);
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        pi->requested_rps = *rps;
+        pi->requested_ps = *new_ps;
+        pi->requested_rps.ps_priv = &pi->requested_ps;
+}
+
+int sumo_dpm_enable(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        int ret;
+
+        if (sumo_dpm_enabled(rdev))
+                return -EINVAL;
+
+        ret = sumo_enable_clock_power_gating(rdev);
+        if (ret)
+                return ret;
+        sumo_program_bootup_state(rdev);
+        sumo_init_bsp(rdev);
+        sumo_reset_am(rdev);
+        sumo_program_tp(rdev);
+        sumo_program_bootup_at(rdev);
+        sumo_start_am(rdev);
+        if (pi->enable_auto_thermal_throttling) {
+                sumo_program_ttp(rdev);
+                sumo_program_ttt(rdev);
+        }
+        sumo_program_dc_hto(rdev);
+        sumo_program_power_level_enter_state(rdev);
+        sumo_enable_voltage_scaling(rdev, true);
+        sumo_program_sstp(rdev);
+        sumo_program_vc(rdev, SUMO_VRC_DFLT);
+        sumo_override_cnb_thermal_events(rdev);
+        sumo_start_dpm(rdev);
+        sumo_wait_for_level_0(rdev);
+        if (pi->enable_sclk_ds)
+                sumo_enable_sclk_ds(rdev, true);
+        if (pi->enable_boost)
+                sumo_enable_boost_timer(rdev);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                ret = sumo_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret)
+                        return ret;
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+        }
+
+        sumo_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
+
+        return 0;
+}
+
+void sumo_dpm_disable(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        if (!sumo_dpm_enabled(rdev))
+                return;
+        sumo_disable_clock_power_gating(rdev);
+        if (pi->enable_sclk_ds)
+                sumo_enable_sclk_ds(rdev, false);
+        sumo_clear_vc(rdev);
+        sumo_wait_for_level_0(rdev);
+        sumo_stop_dpm(rdev);
+        sumo_enable_voltage_scaling(rdev, false);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        sumo_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
+}
+
+int sumo_dpm_pre_set_power_state(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
+        struct radeon_ps *new_ps = &requested_ps;
+
+        sumo_update_requested_ps(rdev, new_ps);
+
+        if (pi->enable_dynamic_patch_ps)
+                sumo_apply_state_adjust_rules(rdev,
+                                              &pi->requested_rps,
+                                              &pi->current_rps);
+
+        return 0;
+}
+
+int sumo_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct radeon_ps *new_ps = &pi->requested_rps;
+        struct radeon_ps *old_ps = &pi->current_rps;
+
+        if (pi->enable_dpm)
+                sumo_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+        if (pi->enable_boost) {
+                sumo_enable_boost(rdev, new_ps, false);
+                sumo_patch_boost_state(rdev, new_ps);
+        }
+        if (pi->enable_dpm) {
+                sumo_pre_notify_alt_vddnb_change(rdev, new_ps, old_ps);
+                sumo_enable_power_level_0(rdev);
+                sumo_set_forced_level_0(rdev);
+                sumo_set_forced_mode_enabled(rdev);
+                sumo_wait_for_level_0(rdev);
+                sumo_program_power_levels_0_to_n(rdev, new_ps, old_ps);
+                sumo_program_wl(rdev, new_ps);
+                sumo_program_bsp(rdev, new_ps);
+                sumo_program_at(rdev, new_ps);
+                sumo_force_nbp_state(rdev, new_ps);
+                sumo_set_forced_mode_disabled(rdev);
+                sumo_set_forced_mode_enabled(rdev);
+                sumo_set_forced_mode_disabled(rdev);
+                sumo_post_notify_alt_vddnb_change(rdev, new_ps, old_ps);
+        }
+        if (pi->enable_boost)
+                sumo_enable_boost(rdev, new_ps, true);
+        if (pi->enable_dpm)
+                sumo_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+
+        return 0;
+}
+
+void sumo_dpm_post_set_power_state(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct radeon_ps *new_ps = &pi->requested_rps;
+
+        sumo_update_current_ps(rdev, new_ps);
+}
+
+void sumo_dpm_reset_asic(struct radeon_device *rdev)
+{
+        sumo_program_bootup_state(rdev);
+        sumo_enable_power_level_0(rdev);
+        sumo_set_forced_level_0(rdev);
+        sumo_set_forced_mode_enabled(rdev);
+        sumo_wait_for_level_0(rdev);
+        sumo_set_forced_mode_disabled(rdev);
+        sumo_set_forced_mode_enabled(rdev);
+        sumo_set_forced_mode_disabled(rdev);
+}
+
+void sumo_dpm_setup_asic(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        sumo_initialize_m3_arb(rdev);
+        pi->fw_version = sumo_get_running_fw_version(rdev);
+        DRM_INFO("Found smc ucode version: 0x%08x\n", pi->fw_version);
+        sumo_program_acpi_power_level(rdev);
+        sumo_enable_acpi_pm(rdev);
+        sumo_take_smu_control(rdev, true);
+}
+
+void sumo_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void sumo_patch_boot_state(struct radeon_device *rdev,
+                                  struct sumo_ps *ps)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        ps->num_levels = 1;
+        ps->flags = 0;
+        ps->levels[0] = pi->boot_pl;
+}
+
+static void sumo_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                            struct radeon_ps *rps,
+                                            struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+                                            u8 table_rev)
+{
+        struct sumo_ps *ps = sumo_get_ps(rps);
+
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                rdev->pm.dpm.boot_ps = rps;
+                sumo_patch_boot_state(rdev, ps);
+        }
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void sumo_parse_pplib_clock_info(struct radeon_device *rdev,
+                                        struct radeon_ps *rps, int index,
+                                        union pplib_clock_info *clock_info)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *ps = sumo_get_ps(rps);
+        struct sumo_pl *pl = &ps->levels[index];
+        u32 sclk;
+
+        sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
+        sclk |= clock_info->sumo.ucEngineClockHigh << 16;
+        pl->sclk = sclk;
+        pl->vddc_index = clock_info->sumo.vddcIndex;
+        pl->sclk_dpm_tdp_limit = clock_info->sumo.tdpLimit;
+
+        ps->num_levels = index + 1;
+
+        if (pi->enable_sclk_ds) {
+                pl->ds_divider_index = 5;
+                pl->ss_divider_index = 4;
+        }
+}
+
+static int sumo_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j, k, non_clock_array_index, clock_array_index;
+        union pplib_clock_info *clock_info;
+        struct _StateArray *state_array;
+        struct _ClockInfoArray *clock_info_array;
+        struct _NonClockInfoArray *non_clock_info_array;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        u8 *power_state_offset;
+        struct sumo_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        state_array = (struct _StateArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usStateArrayOffset));
+        clock_info_array = (struct _ClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
+        non_clock_info_array = (struct _NonClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  state_array->ucNumEntries, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        power_state_offset = (u8 *)state_array->states;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+        for (i = 0; i < state_array->ucNumEntries; i++) {
+                power_state = (union pplib_power_state *)power_state_offset;
+                non_clock_array_index = power_state->v2.nonClockInfoIndex;
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        &non_clock_info_array->nonClockInfo[non_clock_array_index];
+                if (!rdev->pm.power_state[i].clock_info)
+                        return -EINVAL;
+                ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
+                if (ps == NULL) {
+                        kfree(rdev->pm.dpm.ps);
+                        return -ENOMEM;
+                }
+                rdev->pm.dpm.ps[i].ps_priv = ps;
+                k = 0;
+                for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
+                        clock_array_index = power_state->v2.clockInfoIndex[j];
+                        if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
+                                break;
+                        clock_info = (union pplib_clock_info *)
+                                &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
+                        sumo_parse_pplib_clock_info(rdev,
+                                                    &rdev->pm.dpm.ps[i], k,
+                                                    clock_info);
+                        k++;
+                }
+                sumo_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                                non_clock_info,
+                                                non_clock_info_array->ucEntrySize);
+                power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
+        }
+        rdev->pm.dpm.num_ps = state_array->ucNumEntries;
+        return 0;
+}
+
+u32 sumo_convert_vid2_to_vid7(struct radeon_device *rdev,
+                              struct sumo_vid_mapping_table *vid_mapping_table,
+                              u32 vid_2bit)
+{
+        u32 i;
+
+        for (i = 0; i < vid_mapping_table->num_entries; i++) {
+                if (vid_mapping_table->entries[i].vid_2bit == vid_2bit)
+                        return vid_mapping_table->entries[i].vid_7bit;
+        }
+
+        return vid_mapping_table->entries[vid_mapping_table->num_entries - 1].vid_7bit;
+}
+
+static u16 sumo_convert_voltage_index_to_value(struct radeon_device *rdev,
+                                               u32 vid_2bit)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
+
+        if (vid_7bit > 0x7C)
+                return 0;
+
+        return (15500 - vid_7bit * 125 + 5) / 10;
+}
+
+static void sumo_construct_display_voltage_mapping_table(struct radeon_device *rdev,
+                                                         struct sumo_disp_clock_voltage_mapping_table *disp_clk_voltage_mapping_table,
+                                                         ATOM_CLK_VOLT_CAPABILITY *table)
+{
+        u32 i;
+
+        for (i = 0; i < SUMO_MAX_NUMBER_VOLTAGES; i++) {
+                if (table[i].ulMaximumSupportedCLK == 0)
+                        break;
+
+                disp_clk_voltage_mapping_table->display_clock_frequency[i] =
+                        table[i].ulMaximumSupportedCLK;
+        }
+
+        disp_clk_voltage_mapping_table->num_max_voltage_levels = i;
+
+        if (disp_clk_voltage_mapping_table->num_max_voltage_levels == 0) {
+                disp_clk_voltage_mapping_table->display_clock_frequency[0] = 80000;
+                disp_clk_voltage_mapping_table->num_max_voltage_levels = 1;
+        }
+}
+
+void sumo_construct_sclk_voltage_mapping_table(struct radeon_device *rdev,
+                                               struct sumo_sclk_voltage_mapping_table *sclk_voltage_mapping_table,
+                                               ATOM_AVAILABLE_SCLK_LIST *table)
+{
+        u32 i;
+        u32 n = 0;
+        u32 prev_sclk = 0;
+
+        for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++) {
+                if (table[i].ulSupportedSCLK > prev_sclk) {
+                        sclk_voltage_mapping_table->entries[n].sclk_frequency =
+                                table[i].ulSupportedSCLK;
+                        sclk_voltage_mapping_table->entries[n].vid_2bit =
+                                table[i].usVoltageIndex;
+                        prev_sclk = table[i].ulSupportedSCLK;
+                        n++;
+                }
+        }
+
+        sclk_voltage_mapping_table->num_max_dpm_entries = n;
+}
+
+void sumo_construct_vid_mapping_table(struct radeon_device *rdev,
+                                      struct sumo_vid_mapping_table *vid_mapping_table,
+                                      ATOM_AVAILABLE_SCLK_LIST *table)
+{
+        u32 i, j;
+
+        for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++) {
+                if (table[i].ulSupportedSCLK != 0) {
+                        vid_mapping_table->entries[table[i].usVoltageIndex].vid_7bit =
+                                table[i].usVoltageID;
+                        vid_mapping_table->entries[table[i].usVoltageIndex].vid_2bit =
+                                table[i].usVoltageIndex;
+                }
+        }
+
+        for (i = 0; i < SUMO_MAX_NUMBER_VOLTAGES; i++) {
+                if (vid_mapping_table->entries[i].vid_7bit == 0) {
+                        for (j = i + 1; j < SUMO_MAX_NUMBER_VOLTAGES; j++) {
+                                if (vid_mapping_table->entries[j].vid_7bit != 0) {
+                                        vid_mapping_table->entries[i] =
+                                                vid_mapping_table->entries[j];
+                                        vid_mapping_table->entries[j].vid_7bit = 0;
+                                        break;
+                                }
+                        }
+
+                        if (j == SUMO_MAX_NUMBER_VOLTAGES)
+                                break;
+                }
+        }
+
+        vid_mapping_table->num_entries = i;
+}
+
+union igp_info {
+        struct _ATOM_INTEGRATED_SYSTEM_INFO info;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
+};
+
+static int sumo_parse_sys_info_table(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
+        union igp_info *igp_info;
+        u8 frev, crev;
+        u16 data_offset;
+        int i;
+
+        if (atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset)) {
+                igp_info = (union igp_info *)(mode_info->atom_context->bios +
+                                              data_offset);
+
+                if (crev != 6) {
+                        DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
+                        return -EINVAL;
+                }
+                pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_6.ulBootUpEngineClock);
+                pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_6.ulMinEngineClock);
+                pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_6.ulBootUpUMAClock);
+                pi->sys_info.bootup_nb_voltage_index =
+                        le16_to_cpu(igp_info->info_6.usBootUpNBVoltage);
+                if (igp_info->info_6.ucHtcTmpLmt == 0)
+                        pi->sys_info.htc_tmp_lmt = 203;
+                else
+                        pi->sys_info.htc_tmp_lmt = igp_info->info_6.ucHtcTmpLmt;
+                if (igp_info->info_6.ucHtcHystLmt == 0)
+                        pi->sys_info.htc_hyst_lmt = 5;
+                else
+                        pi->sys_info.htc_hyst_lmt = igp_info->info_6.ucHtcHystLmt;
+                if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
+                        DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
+                }
+                for (i = 0; i < NUMBER_OF_M3ARB_PARAM_SETS; i++) {
+                        pi->sys_info.csr_m3_arb_cntl_default[i] =
+                                le32_to_cpu(igp_info->info_6.ulCSR_M3_ARB_CNTL_DEFAULT[i]);
+                        pi->sys_info.csr_m3_arb_cntl_uvd[i] =
+                                le32_to_cpu(igp_info->info_6.ulCSR_M3_ARB_CNTL_UVD[i]);
+                        pi->sys_info.csr_m3_arb_cntl_fs3d[i] =
+                                le32_to_cpu(igp_info->info_6.ulCSR_M3_ARB_CNTL_FS3D[i]);
+                }
+                pi->sys_info.sclk_dpm_boost_margin =
+                        le32_to_cpu(igp_info->info_6.SclkDpmBoostMargin);
+                pi->sys_info.sclk_dpm_throttle_margin =
+                        le32_to_cpu(igp_info->info_6.SclkDpmThrottleMargin);
+                pi->sys_info.sclk_dpm_tdp_limit_pg =
+                        le16_to_cpu(igp_info->info_6.SclkDpmTdpLimitPG);
+                pi->sys_info.gnb_tdp_limit = le16_to_cpu(igp_info->info_6.GnbTdpLimit);
+                pi->sys_info.sclk_dpm_tdp_limit_boost =
+                        le16_to_cpu(igp_info->info_6.SclkDpmTdpLimitBoost);
+                pi->sys_info.boost_sclk = le32_to_cpu(igp_info->info_6.ulBoostEngineCLock);
+                pi->sys_info.boost_vid_2bit = igp_info->info_6.ulBoostVid_2bit;
+                if (igp_info->info_6.EnableBoost)
+                        pi->sys_info.enable_boost = true;
+                else
+                        pi->sys_info.enable_boost = false;
+                sumo_construct_display_voltage_mapping_table(rdev,
+                                                             &pi->sys_info.disp_clk_voltage_mapping_table,
+                                                             igp_info->info_6.sDISPCLK_Voltage);
+                sumo_construct_sclk_voltage_mapping_table(rdev,
+                                                          &pi->sys_info.sclk_voltage_mapping_table,
+                                                          igp_info->info_6.sAvail_SCLK);
+                sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
+                                                 igp_info->info_6.sAvail_SCLK);
+
+        }
+        return 0;
+}
+
+static void sumo_construct_boot_and_acpi_state(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
+        pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
+        pi->boot_pl.ds_divider_index = 0;
+        pi->boot_pl.ss_divider_index = 0;
+        pi->boot_pl.allow_gnb_slow = 1;
+        pi->acpi_pl = pi->boot_pl;
+        pi->current_ps.num_levels = 1;
+        pi->current_ps.levels[0] = pi->boot_pl;
+}
+
+int sumo_dpm_init(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi;
+        u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
+        int ret;
+
+        pi = kzalloc(sizeof(struct sumo_power_info), GFP_KERNEL);
+        if (pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = pi;
+
+        pi->driver_nbps_policy_disable = false;
+        if ((rdev->family == CHIP_PALM) && (hw_rev < 3))
+                pi->disable_gfx_power_gating_in_uvd = true;
+        else
+                pi->disable_gfx_power_gating_in_uvd = false;
+        pi->enable_alt_vddnb = true;
+        pi->enable_sclk_ds = true;
+        pi->enable_dynamic_m3_arbiter = false;
+        pi->enable_dynamic_patch_ps = true;
+        pi->enable_gfx_power_gating = true;
+        pi->enable_gfx_clock_gating = true;
+        pi->enable_mg_clock_gating = true;
+        pi->enable_auto_thermal_throttling = true;
+
+        ret = sumo_parse_sys_info_table(rdev);
+        if (ret)
+                return ret;
+
+        sumo_construct_boot_and_acpi_state(rdev);
+
+        ret = sumo_parse_power_table(rdev);
+        if (ret)
+                return ret;
+
+        pi->pasi = CYPRESS_HASI_DFLT;
+        pi->asi = RV770_ASI_DFLT;
+        pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
+        pi->enable_boost = pi->sys_info.enable_boost;
+        pi->enable_dpm = true;
+
+        return 0;
+}
+
+void sumo_dpm_print_power_state(struct radeon_device *rdev,
+                                struct radeon_ps *rps)
+{
+        int i;
+        struct sumo_ps *ps = sumo_get_ps(rps);
+
+        r600_dpm_print_class_info(rps->class, rps->class2);
+        r600_dpm_print_cap_info(rps->caps);
+        printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+        for (i = 0; i < ps->num_levels; i++) {
+                struct sumo_pl *pl = &ps->levels[i];
+                printk("\t\tpower level %d    sclk: %u vddc: %u\n",
+                       i, pl->sclk,
+                       sumo_convert_voltage_index_to_value(rdev, pl->vddc_index));
+        }
+        r600_dpm_print_ps_status(rdev, rps);
+}
+
+void sumo_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                      struct seq_file *m)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct radeon_ps *rps = rdev->pm.dpm.current_ps;
+        struct sumo_ps *ps = sumo_get_ps(rps);
+        struct sumo_pl *pl;
+        u32 current_index =
+                (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURR_INDEX_MASK) >>
+                CURR_INDEX_SHIFT;
+
+        if (current_index == BOOST_DPM_LEVEL) {
+                pl = &pi->boost_pl;
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
+                           current_index, pl->sclk,
+                           sumo_convert_voltage_index_to_value(rdev, pl->vddc_index));
+        } else if (current_index >= ps->num_levels) {
+                seq_printf(m, "invalid dpm profile %d\n", current_index);
+        } else {
+                pl = &ps->levels[current_index];
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
+                           current_index, pl->sclk,
+                           sumo_convert_voltage_index_to_value(rdev, pl->vddc_index));
+        }
+}
+
+void sumo_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        sumo_cleanup_asic(rdev); /* ??? */
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+}
+
+u32 sumo_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        struct sumo_ps *requested_state = sumo_get_ps(&pi->requested_rps);
+
+        if (low)
+                return requested_state->levels[0].sclk;
+        else
+                return requested_state->levels[requested_state->num_levels - 1].sclk;
+}
+
+u32 sumo_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+
+        return pi->sys_info.bootup_uma_clk;
+}
diff --git a/drivers/gpu/drm/radeon/sumo_dpm.h b/drivers/gpu/drm/radeon/sumo_dpm.h
new file mode 100644
index 000000000000..07dda299c784
--- /dev/null
+++ b/drivers/gpu/drm/radeon/sumo_dpm.h
@@ -0,0 +1,220 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SUMO_DPM_H__
+#define __SUMO_DPM_H__
+
+#include "atom.h"
+
+#define SUMO_MAX_HARDWARE_POWERLEVELS 5
+#define SUMO_PM_NUMBER_OF_TC 15
+
+struct sumo_pl {
+        u32 sclk;
+        u32 vddc_index;
+        u32 ds_divider_index;
+        u32 ss_divider_index;
+        u32 allow_gnb_slow;
+        u32 sclk_dpm_tdp_limit;
+};
+
+/* used for the flags field */
+#define SUMO_POWERSTATE_FLAGS_FORCE_NBPS1_STATE (1 << 0)
+#define SUMO_POWERSTATE_FLAGS_BOOST_STATE       (1 << 1)
+
+struct sumo_ps {
+        struct sumo_pl levels[SUMO_MAX_HARDWARE_POWERLEVELS];
+        u32 num_levels;
+        /* flags */
+        u32 flags;
+};
+
+#define NUMBER_OF_M3ARB_PARAM_SETS 10
+#define SUMO_MAX_NUMBER_VOLTAGES    4
+
+struct sumo_disp_clock_voltage_mapping_table {
+        u32 num_max_voltage_levels;
+        u32 display_clock_frequency[SUMO_MAX_NUMBER_VOLTAGES];
+};
+
+struct sumo_vid_mapping_entry {
+        u16 vid_2bit;
+        u16 vid_7bit;
+};
+
+struct sumo_vid_mapping_table {
+        u32 num_entries;
+        struct sumo_vid_mapping_entry entries[SUMO_MAX_NUMBER_VOLTAGES];
+};
+
+struct sumo_sclk_voltage_mapping_entry {
+        u32 sclk_frequency;
+        u16 vid_2bit;
+        u16 rsv;
+};
+
+struct sumo_sclk_voltage_mapping_table {
+        u32 num_max_dpm_entries;
+        struct sumo_sclk_voltage_mapping_entry entries[SUMO_MAX_HARDWARE_POWERLEVELS];
+};
+
+struct sumo_sys_info {
+        u32 bootup_sclk;
+        u32 min_sclk;
+        u32 bootup_uma_clk;
+        u16 bootup_nb_voltage_index;
+        u8 htc_tmp_lmt;
+        u8 htc_hyst_lmt;
+        struct sumo_sclk_voltage_mapping_table sclk_voltage_mapping_table;
+        struct sumo_disp_clock_voltage_mapping_table disp_clk_voltage_mapping_table;
+        struct sumo_vid_mapping_table vid_mapping_table;
+        u32 csr_m3_arb_cntl_default[NUMBER_OF_M3ARB_PARAM_SETS];
+        u32 csr_m3_arb_cntl_uvd[NUMBER_OF_M3ARB_PARAM_SETS];
+        u32 csr_m3_arb_cntl_fs3d[NUMBER_OF_M3ARB_PARAM_SETS];
+        u32 sclk_dpm_boost_margin;
+        u32 sclk_dpm_throttle_margin;
+        u32 sclk_dpm_tdp_limit_pg;
+        u32 gnb_tdp_limit;
+        u32 sclk_dpm_tdp_limit_boost;
+        u32 boost_sclk;
+        u32 boost_vid_2bit;
+        bool enable_boost;
+};
+
+struct sumo_power_info {
+        u32 asi;
+        u32 pasi;
+        u32 bsp;
+        u32 bsu;
+        u32 pbsp;
+        u32 pbsu;
+        u32 dsp;
+        u32 psp;
+        u32 thermal_auto_throttling;
+        u32 uvd_m3_arbiter;
+        u32 fw_version;
+        struct sumo_sys_info sys_info;
+        struct sumo_pl acpi_pl;
+        struct sumo_pl boot_pl;
+        struct sumo_pl boost_pl;
+        bool disable_gfx_power_gating_in_uvd;
+        bool driver_nbps_policy_disable;
+        bool enable_alt_vddnb;
+        bool enable_dynamic_m3_arbiter;
+        bool enable_gfx_clock_gating;
+        bool enable_gfx_power_gating;
+        bool enable_mg_clock_gating;
+        bool enable_sclk_ds;
+        bool enable_auto_thermal_throttling;
+        bool enable_dynamic_patch_ps;
+        bool enable_dpm;
+        bool enable_boost;
+        struct radeon_ps current_rps;
+        struct sumo_ps current_ps;
+        struct radeon_ps requested_rps;
+        struct sumo_ps requested_ps;
+};
+
+#define SUMO_UTC_DFLT_00                     0x48
+#define SUMO_UTC_DFLT_01                     0x44
+#define SUMO_UTC_DFLT_02                     0x44
+#define SUMO_UTC_DFLT_03                     0x44
+#define SUMO_UTC_DFLT_04                     0x44
+#define SUMO_UTC_DFLT_05                     0x44
+#define SUMO_UTC_DFLT_06                     0x44
+#define SUMO_UTC_DFLT_07                     0x44
+#define SUMO_UTC_DFLT_08                     0x44
+#define SUMO_UTC_DFLT_09                     0x44
+#define SUMO_UTC_DFLT_10                     0x44
+#define SUMO_UTC_DFLT_11                     0x44
+#define SUMO_UTC_DFLT_12                     0x44
+#define SUMO_UTC_DFLT_13                     0x44
+#define SUMO_UTC_DFLT_14                     0x44
+
+#define SUMO_DTC_DFLT_00                     0x48
+#define SUMO_DTC_DFLT_01                     0x44
+#define SUMO_DTC_DFLT_02                     0x44
+#define SUMO_DTC_DFLT_03                     0x44
+#define SUMO_DTC_DFLT_04                     0x44
+#define SUMO_DTC_DFLT_05                     0x44
+#define SUMO_DTC_DFLT_06                     0x44
+#define SUMO_DTC_DFLT_07                     0x44
+#define SUMO_DTC_DFLT_08                     0x44
+#define SUMO_DTC_DFLT_09                     0x44
+#define SUMO_DTC_DFLT_10                     0x44
+#define SUMO_DTC_DFLT_11                     0x44
+#define SUMO_DTC_DFLT_12                     0x44
+#define SUMO_DTC_DFLT_13                     0x44
+#define SUMO_DTC_DFLT_14                     0x44
+
+#define SUMO_AH_DFLT               5
+
+#define SUMO_R_DFLT0               70
+#define SUMO_R_DFLT1               70
+#define SUMO_R_DFLT2               70
+#define SUMO_R_DFLT3               70
+#define SUMO_R_DFLT4               100
+
+#define SUMO_L_DFLT0               0
+#define SUMO_L_DFLT1               20
+#define SUMO_L_DFLT2               20
+#define SUMO_L_DFLT3               20
+#define SUMO_L_DFLT4               20
+#define SUMO_VRC_DFLT              0x30033
+#define SUMO_MGCGTTLOCAL0_DFLT     0
+#define SUMO_MGCGTTLOCAL1_DFLT     0
+#define SUMO_GICST_DFLT            19
+#define SUMO_SST_DFLT              8
+#define SUMO_VOLTAGEDROPT_DFLT     1
+#define SUMO_GFXPOWERGATINGT_DFLT  100
+
+/* sumo_dpm.c */
+void sumo_gfx_clockgating_initialize(struct radeon_device *rdev);
+void sumo_program_vc(struct radeon_device *rdev, u32 vrc);
+void sumo_clear_vc(struct radeon_device *rdev);
+void sumo_program_sstp(struct radeon_device *rdev);
+void sumo_take_smu_control(struct radeon_device *rdev, bool enable);
+void sumo_construct_sclk_voltage_mapping_table(struct radeon_device *rdev,
+                                               struct sumo_sclk_voltage_mapping_table *sclk_voltage_mapping_table,
+                                               ATOM_AVAILABLE_SCLK_LIST *table);
+void sumo_construct_vid_mapping_table(struct radeon_device *rdev,
+                                      struct sumo_vid_mapping_table *vid_mapping_table,
+                                      ATOM_AVAILABLE_SCLK_LIST *table);
+u32 sumo_convert_vid2_to_vid7(struct radeon_device *rdev,
+                              struct sumo_vid_mapping_table *vid_mapping_table,
+                              u32 vid_2bit);
+u32 sumo_get_sleep_divider_from_id(u32 id);
+u32 sumo_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
+                                         u32 sclk,
+                                         u32 min_sclk_in_sr);
+
+/* sumo_smc.c */
+void sumo_initialize_m3_arb(struct radeon_device *rdev);
+void sumo_smu_pg_init(struct radeon_device *rdev);
+void sumo_set_tdp_limit(struct radeon_device *rdev, u32 index, u32 tdp_limit);
+void sumo_smu_notify_alt_vddnb_change(struct radeon_device *rdev,
+                                      bool powersaving, bool force_nbps1);
+void sumo_boost_state_enable(struct radeon_device *rdev, bool enable);
+void sumo_enable_boost_timer(struct radeon_device *rdev);
+u32 sumo_get_running_fw_version(struct radeon_device *rdev);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/sumo_smc.c b/drivers/gpu/drm/radeon/sumo_smc.c
new file mode 100644
index 000000000000..18abba5b5810
--- /dev/null
+++ b/drivers/gpu/drm/radeon/sumo_smc.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "sumod.h"
+#include "sumo_dpm.h"
+#include "ppsmc.h"
+
+#define SUMO_SMU_SERVICE_ROUTINE_PG_INIT        1
+#define SUMO_SMU_SERVICE_ROUTINE_ALTVDDNB_NOTIFY  27
+#define SUMO_SMU_SERVICE_ROUTINE_GFX_SRV_ID_20  20
+
+struct sumo_ps *sumo_get_ps(struct radeon_ps *rps);
+struct sumo_power_info *sumo_get_pi(struct radeon_device *rdev);
+
+static void sumo_send_msg_to_smu(struct radeon_device *rdev, u32 id)
+{
+        u32 gfx_int_req;
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(GFX_INT_STATUS) & INT_DONE)
+                        break;
+                udelay(1);
+        }
+
+        gfx_int_req = SERV_INDEX(id) | INT_REQ;
+        WREG32(GFX_INT_REQ, gfx_int_req);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(GFX_INT_REQ) & INT_REQ)
+                        break;
+                udelay(1);
+        }
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(GFX_INT_STATUS) & INT_ACK)
+                        break;
+                udelay(1);
+        }
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(GFX_INT_STATUS) & INT_DONE)
+                        break;
+                udelay(1);
+        }
+
+        gfx_int_req &= ~INT_REQ;
+        WREG32(GFX_INT_REQ, gfx_int_req);
+}
+
+void sumo_initialize_m3_arb(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 i;
+
+        if (!pi->enable_dynamic_m3_arbiter)
+                return;
+
+        for (i = 0; i < NUMBER_OF_M3ARB_PARAM_SETS; i++)
+                WREG32_RCU(MCU_M3ARB_PARAMS + (i * 4),
+                           pi->sys_info.csr_m3_arb_cntl_default[i]);
+
+        for (; i < NUMBER_OF_M3ARB_PARAM_SETS * 2; i++)
+                WREG32_RCU(MCU_M3ARB_PARAMS + (i * 4),
+                           pi->sys_info.csr_m3_arb_cntl_uvd[i % NUMBER_OF_M3ARB_PARAM_SETS]);
+
+        for (; i < NUMBER_OF_M3ARB_PARAM_SETS * 3; i++)
+                WREG32_RCU(MCU_M3ARB_PARAMS + (i * 4),
+                           pi->sys_info.csr_m3_arb_cntl_fs3d[i % NUMBER_OF_M3ARB_PARAM_SETS]);
+}
+
+static bool sumo_is_alt_vddnb_supported(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        bool return_code = false;
+
+        if (!pi->enable_alt_vddnb)
+                return return_code;
+
+        if ((rdev->family == CHIP_SUMO) || (rdev->family == CHIP_SUMO2)) {
+                if (pi->fw_version >= 0x00010C00)
+                        return_code = true;
+        }
+
+        return return_code;
+}
+
+void sumo_smu_notify_alt_vddnb_change(struct radeon_device *rdev,
+                                      bool powersaving, bool force_nbps1)
+{
+        u32 param = 0;
+
+        if (!sumo_is_alt_vddnb_supported(rdev))
+                return;
+
+        if (powersaving)
+                param |= 1;
+
+        if (force_nbps1)
+                param |= 2;
+
+        WREG32_RCU(RCU_ALTVDDNB_NOTIFY, param);
+
+        sumo_send_msg_to_smu(rdev, SUMO_SMU_SERVICE_ROUTINE_ALTVDDNB_NOTIFY);
+}
+
+void sumo_smu_pg_init(struct radeon_device *rdev)
+{
+        sumo_send_msg_to_smu(rdev, SUMO_SMU_SERVICE_ROUTINE_PG_INIT);
+}
+
+static u32 sumo_power_of_4(u32 unit)
+{
+        u32 ret = 1;
+        u32 i;
+
+        for (i = 0; i < unit; i++)
+                ret *= 4;
+
+        return ret;
+}
+
+void sumo_enable_boost_timer(struct radeon_device *rdev)
+{
+        struct sumo_power_info *pi = sumo_get_pi(rdev);
+        u32 period, unit, timer_value;
+        u32 xclk = radeon_get_xclk(rdev);
+
+        unit = (RREG32_RCU(RCU_LCLK_SCALING_CNTL) & LCLK_SCALING_TIMER_PRESCALER_MASK)
+                >> LCLK_SCALING_TIMER_PRESCALER_SHIFT;
+
+        period = 100 * (xclk / 100 / sumo_power_of_4(unit));
+
+        timer_value = (period << 16) | (unit << 4);
+
+        WREG32_RCU(RCU_GNB_PWR_REP_TIMER_CNTL, timer_value);
+        WREG32_RCU(RCU_BOOST_MARGIN, pi->sys_info.sclk_dpm_boost_margin);
+        WREG32_RCU(RCU_THROTTLE_MARGIN, pi->sys_info.sclk_dpm_throttle_margin);
+        WREG32_RCU(GNB_TDP_LIMIT, pi->sys_info.gnb_tdp_limit);
+        WREG32_RCU(RCU_SclkDpmTdpLimitPG, pi->sys_info.sclk_dpm_tdp_limit_pg);
+
+        sumo_send_msg_to_smu(rdev, SUMO_SMU_SERVICE_ROUTINE_GFX_SRV_ID_20);
+}
+
+void sumo_set_tdp_limit(struct radeon_device *rdev, u32 index, u32 tdp_limit)
+{
+        u32 regoffset = 0;
+        u32 shift = 0;
+        u32 mask = 0xFFF;
+        u32 sclk_dpm_tdp_limit;
+
+        switch (index) {
+        case 0:
+                regoffset = RCU_SclkDpmTdpLimit01;
+                shift = 16;
+                break;
+        case 1:
+                regoffset = RCU_SclkDpmTdpLimit01;
+                shift = 0;
+                break;
+        case 2:
+                regoffset = RCU_SclkDpmTdpLimit23;
+                shift = 16;
+                break;
+        case 3:
+                regoffset = RCU_SclkDpmTdpLimit23;
+                shift = 0;
+                break;
+        case 4:
+                regoffset = RCU_SclkDpmTdpLimit47;
+                shift = 16;
+                break;
+        case 7:
+                regoffset = RCU_SclkDpmTdpLimit47;
+                shift = 0;
+                break;
+        default:
+                break;
+        }
+
+        sclk_dpm_tdp_limit = RREG32_RCU(regoffset);
+        sclk_dpm_tdp_limit &= ~(mask << shift);
+        sclk_dpm_tdp_limit |= (tdp_limit << shift);
+        WREG32_RCU(regoffset, sclk_dpm_tdp_limit);
+}
+
+void sumo_boost_state_enable(struct radeon_device *rdev, bool enable)
+{
+        u32 boost_disable = RREG32_RCU(RCU_GPU_BOOST_DISABLE);
+
+        boost_disable &= 0xFFFFFFFE;
+        boost_disable |= (enable ? 0 : 1);
+        WREG32_RCU(RCU_GPU_BOOST_DISABLE, boost_disable);
+}
+
+u32 sumo_get_running_fw_version(struct radeon_device *rdev)
+{
+        return RREG32_RCU(RCU_FW_VERSION);
+}
+
diff --git a/drivers/gpu/drm/radeon/sumod.h b/drivers/gpu/drm/radeon/sumod.h
new file mode 100644
index 000000000000..7c9c2d4b86c0
--- /dev/null
+++ b/drivers/gpu/drm/radeon/sumod.h
@@ -0,0 +1,372 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef _SUMOD_H_
+#define _SUMOD_H_
+
+/* pm registers */
+
+/* rcu */
+#define RCU_FW_VERSION                                  0x30c
+
+#define RCU_PWR_GATING_SEQ0                             0x408
+#define RCU_PWR_GATING_SEQ1                             0x40c
+#define RCU_PWR_GATING_CNTL                             0x410
+#       define PWR_GATING_EN                            (1 << 0)
+#       define RSVD_MASK                                (0x3 << 1)
+#       define PCV(x)                                   ((x) << 3)
+#       define PCV_MASK                                 (0x1f << 3)
+#       define PCV_SHIFT                                3
+#       define PCP(x)                                   ((x) << 8)
+#       define PCP_MASK                                 (0xf << 8)
+#       define PCP_SHIFT                                8
+#       define RPW(x)                                   ((x) << 16)
+#       define RPW_MASK                                 (0xf << 16)
+#       define RPW_SHIFT                                16
+#       define ID(x)                                    ((x) << 24)
+#       define ID_MASK                                  (0xf << 24)
+#       define ID_SHIFT                                 24
+#       define PGS(x)                                   ((x) << 28)
+#       define PGS_MASK                                 (0xf << 28)
+#       define PGS_SHIFT                                28
+
+#define RCU_ALTVDDNB_NOTIFY                             0x430
+#define RCU_LCLK_SCALING_CNTL                           0x434
+#       define LCLK_SCALING_EN                          (1 << 0)
+#       define LCLK_SCALING_TYPE                        (1 << 1)
+#       define LCLK_SCALING_TIMER_PRESCALER(x)          ((x) << 4)
+#       define LCLK_SCALING_TIMER_PRESCALER_MASK        (0xf << 4)
+#       define LCLK_SCALING_TIMER_PRESCALER_SHIFT       4
+#       define LCLK_SCALING_TIMER_PERIOD(x)             ((x) << 16)
+#       define LCLK_SCALING_TIMER_PERIOD_MASK           (0xf << 16)
+#       define LCLK_SCALING_TIMER_PERIOD_SHIFT          16
+
+#define RCU_PWR_GATING_CNTL_2                           0x4a0
+#       define MPPU(x)                                  ((x) << 0)
+#       define MPPU_MASK                                (0xffff << 0)
+#       define MPPU_SHIFT                               0
+#       define MPPD(x)                                  ((x) << 16)
+#       define MPPD_MASK                                (0xffff << 16)
+#       define MPPD_SHIFT                               16
+#define RCU_PWR_GATING_CNTL_3                           0x4a4
+#       define DPPU(x)                                  ((x) << 0)
+#       define DPPU_MASK                                (0xffff << 0)
+#       define DPPU_SHIFT                               0
+#       define DPPD(x)                                  ((x) << 16)
+#       define DPPD_MASK                                (0xffff << 16)
+#       define DPPD_SHIFT                               16
+#define RCU_PWR_GATING_CNTL_4                           0x4a8
+#       define RT(x)                                    ((x) << 0)
+#       define RT_MASK                                  (0xffff << 0)
+#       define RT_SHIFT                                 0
+#       define IT(x)                                    ((x) << 16)
+#       define IT_MASK                                  (0xffff << 16)
+#       define IT_SHIFT                                 16
+
+/* yes these two have the same address */
+#define RCU_PWR_GATING_CNTL_5                           0x504
+#define RCU_GPU_BOOST_DISABLE                           0x508
+
+#define MCU_M3ARB_INDEX                                 0x504
+#define MCU_M3ARB_PARAMS                                0x508
+
+#define RCU_GNB_PWR_REP_TIMER_CNTL                      0x50C
+
+#define RCU_SclkDpmTdpLimit01                           0x514
+#define RCU_SclkDpmTdpLimit23                           0x518
+#define RCU_SclkDpmTdpLimit47                           0x51C
+#define RCU_SclkDpmTdpLimitPG                           0x520
+
+#define GNB_TDP_LIMIT                                   0x540
+#define RCU_BOOST_MARGIN                                0x544
+#define RCU_THROTTLE_MARGIN                             0x548
+
+#define SMU_PCIE_PG_ARGS                                0x58C
+#define SMU_PCIE_PG_ARGS_2                              0x598
+#define SMU_PCIE_PG_ARGS_3                              0x59C
+
+/* mmio */
+#define RCU_STATUS                                      0x11c
+#       define GMC_PWR_GATER_BUSY                       (1 << 8)
+#       define GFX_PWR_GATER_BUSY                       (1 << 9)
+#       define UVD_PWR_GATER_BUSY                       (1 << 10)
+#       define PCIE_PWR_GATER_BUSY                      (1 << 11)
+#       define GMC_PWR_GATER_STATE                      (1 << 12)
+#       define GFX_PWR_GATER_STATE                      (1 << 13)
+#       define UVD_PWR_GATER_STATE                      (1 << 14)
+#       define PCIE_PWR_GATER_STATE                     (1 << 15)
+#       define GFX1_PWR_GATER_BUSY                      (1 << 16)
+#       define GFX2_PWR_GATER_BUSY                      (1 << 17)
+#       define GFX1_PWR_GATER_STATE                     (1 << 18)
+#       define GFX2_PWR_GATER_STATE                     (1 << 19)
+
+#define GFX_INT_REQ                                     0x120
+#       define INT_REQ                                  (1 << 0)
+#       define SERV_INDEX(x)                            ((x) << 1)
+#       define SERV_INDEX_MASK                          (0xff << 1)
+#       define SERV_INDEX_SHIFT                         1
+#define GFX_INT_STATUS                                  0x124
+#       define INT_ACK                                  (1 << 0)
+#       define INT_DONE                                 (1 << 1)
+
+#define CG_SCLK_CNTL                                    0x600
+#       define SCLK_DIVIDER(x)                          ((x) << 0)
+#       define SCLK_DIVIDER_MASK                        (0x7f << 0)
+#       define SCLK_DIVIDER_SHIFT                       0
+#define CG_SCLK_STATUS                                  0x604
+#       define SCLK_OVERCLK_DETECT                      (1 << 2)
+
+#define CG_DCLK_CNTL                                    0x610
+#       define DCLK_DIVIDER_MASK                        0x7f
+#       define DCLK_DIR_CNTL_EN                         (1 << 8)
+#define CG_DCLK_STATUS                                  0x614
+#       define DCLK_STATUS                              (1 << 0)
+#define CG_VCLK_CNTL                                    0x618
+#       define VCLK_DIVIDER_MASK                        0x7f
+#       define VCLK_DIR_CNTL_EN                         (1 << 8)
+#define CG_VCLK_STATUS                                  0x61c
+
+#define GENERAL_PWRMGT                                  0x63c
+#       define STATIC_PM_EN                             (1 << 1)
+
+#define SCLK_PWRMGT_CNTL                                0x644
+#       define SCLK_PWRMGT_OFF                          (1 << 0)
+#       define SCLK_LOW_D1                              (1 << 1)
+#       define FIR_RESET                                (1 << 4)
+#       define FIR_FORCE_TREND_SEL                      (1 << 5)
+#       define FIR_TREND_MODE                           (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                       (1 << 7)
+#       define GFX_CLK_FORCE_ON                         (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                      (1 << 9)
+#       define GFX_CLK_FORCE_OFF                        (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                      (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                      (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                      (1 << 13)
+#       define GFX_VOLTAGE_CHANGE_EN                    (1 << 16)
+#       define GFX_VOLTAGE_CHANGE_MODE                  (1 << 17)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                0x66c
+#       define TARG_SCLK_INDEX(x)                       ((x) << 6)
+#       define TARG_SCLK_INDEX_MASK                     (0x7 << 6)
+#       define TARG_SCLK_INDEX_SHIFT                    6
+#       define CURR_SCLK_INDEX(x)                       ((x) << 9)
+#       define CURR_SCLK_INDEX_MASK                     (0x7 << 9)
+#       define CURR_SCLK_INDEX_SHIFT                    9
+#       define TARG_INDEX(x)                            ((x) << 12)
+#       define TARG_INDEX_MASK                          (0x7 << 12)
+#       define TARG_INDEX_SHIFT                         12
+#       define CURR_INDEX(x)                            ((x) << 15)
+#       define CURR_INDEX_MASK                          (0x7 << 15)
+#       define CURR_INDEX_SHIFT                         15
+
+#define CG_SCLK_DPM_CTRL                                0x684
+#       define SCLK_FSTATE_0_DIV(x)                     ((x) << 0)
+#       define SCLK_FSTATE_0_DIV_MASK                   (0x7f << 0)
+#       define SCLK_FSTATE_0_DIV_SHIFT                  0
+#       define SCLK_FSTATE_0_VLD                        (1 << 7)
+#       define SCLK_FSTATE_1_DIV(x)                     ((x) << 8)
+#       define SCLK_FSTATE_1_DIV_MASK                   (0x7f << 8)
+#       define SCLK_FSTATE_1_DIV_SHIFT                  8
+#       define SCLK_FSTATE_1_VLD                        (1 << 15)
+#       define SCLK_FSTATE_2_DIV(x)                     ((x) << 16)
+#       define SCLK_FSTATE_2_DIV_MASK                   (0x7f << 16)
+#       define SCLK_FSTATE_2_DIV_SHIFT                  16
+#       define SCLK_FSTATE_2_VLD                        (1 << 23)
+#       define SCLK_FSTATE_3_DIV(x)                     ((x) << 24)
+#       define SCLK_FSTATE_3_DIV_MASK                   (0x7f << 24)
+#       define SCLK_FSTATE_3_DIV_SHIFT                  24
+#       define SCLK_FSTATE_3_VLD                        (1 << 31)
+#define CG_SCLK_DPM_CTRL_2                              0x688
+#define CG_GCOOR                                        0x68c
+#       define PHC(x)                                   ((x) << 0)
+#       define PHC_MASK                                 (0x1f << 0)
+#       define PHC_SHIFT                                0
+#       define SDC(x)                                   ((x) << 9)
+#       define SDC_MASK                                 (0x3ff << 9)
+#       define SDC_SHIFT                                9
+#       define SU(x)                                    ((x) << 23)
+#       define SU_MASK                                  (0xf << 23)
+#       define SU_SHIFT                                 23
+#       define DIV_ID(x)                                ((x) << 28)
+#       define DIV_ID_MASK                              (0x7 << 28)
+#       define DIV_ID_SHIFT                             28
+
+#define CG_FTV                                          0x690
+#define CG_FFCT_0                                       0x694
+#       define UTC_0(x)                                 ((x) << 0)
+#       define UTC_0_MASK                               (0x3ff << 0)
+#       define UTC_0_SHIFT                              0
+#       define DTC_0(x)                                 ((x) << 10)
+#       define DTC_0_MASK                               (0x3ff << 10)
+#       define DTC_0_SHIFT                              10
+
+#define CG_GIT                                          0x6d8
+#       define CG_GICST(x)                              ((x) << 0)
+#       define CG_GICST_MASK                            (0xffff << 0)
+#       define CG_GICST_SHIFT                           0
+#       define CG_GIPOT(x)                              ((x) << 16)
+#       define CG_GIPOT_MASK                            (0xffff << 16)
+#       define CG_GIPOT_SHIFT                           16
+
+#define CG_SCLK_DPM_CTRL_3                              0x6e0
+#       define FORCE_SCLK_STATE(x)                      ((x) << 0)
+#       define FORCE_SCLK_STATE_MASK                    (0x7 << 0)
+#       define FORCE_SCLK_STATE_SHIFT                   0
+#       define FORCE_SCLK_STATE_EN                      (1 << 3)
+#       define GNB_TT(x)                                ((x) << 8)
+#       define GNB_TT_MASK                              (0xff << 8)
+#       define GNB_TT_SHIFT                             8
+#       define GNB_THERMTHRO_MASK                       (1 << 16)
+#       define CNB_THERMTHRO_MASK_SCLK                  (1 << 17)
+#       define DPM_SCLK_ENABLE                          (1 << 18)
+#       define GNB_SLOW_FSTATE_0_MASK                   (1 << 23)
+#       define GNB_SLOW_FSTATE_0_SHIFT                  23
+#       define FORCE_NB_PSTATE_1                        (1 << 31)
+
+#define CG_SSP                                          0x6e8
+#       define SST(x)                                   ((x) << 0)
+#       define SST_MASK                                 (0xffff << 0)
+#       define SST_SHIFT                                0
+#       define SSTU(x)                                  ((x) << 16)
+#       define SSTU_MASK                                (0xffff << 16)
+#       define SSTU_SHIFT                               16
+
+#define CG_ACPI_CNTL                                    0x70c
+#       define SCLK_ACPI_DIV(x)                         ((x) << 0)
+#       define SCLK_ACPI_DIV_MASK                       (0x7f << 0)
+#       define SCLK_ACPI_DIV_SHIFT                      0
+
+#define CG_SCLK_DPM_CTRL_4                              0x71c
+#       define DC_HDC(x)                                ((x) << 14)
+#       define DC_HDC_MASK                              (0x3fff << 14)
+#       define DC_HDC_SHIFT                             14
+#       define DC_HU(x)                                 ((x) << 28)
+#       define DC_HU_MASK                               (0xf << 28)
+#       define DC_HU_SHIFT                              28
+#define CG_SCLK_DPM_CTRL_5                              0x720
+#       define SCLK_FSTATE_BOOTUP(x)                    ((x) << 0)
+#       define SCLK_FSTATE_BOOTUP_MASK                  (0x7 << 0)
+#       define SCLK_FSTATE_BOOTUP_SHIFT                 0
+#       define TT_TP(x)                                 ((x) << 3)
+#       define TT_TP_MASK                               (0xffff << 3)
+#       define TT_TP_SHIFT                              3
+#       define TT_TU(x)                                 ((x) << 19)
+#       define TT_TU_MASK                               (0xff << 19)
+#       define TT_TU_SHIFT                              19
+#define CG_SCLK_DPM_CTRL_6                              0x724
+#define CG_AT_0                                         0x728
+#       define CG_R(x)                                  ((x) << 0)
+#       define CG_R_MASK                                (0xffff << 0)
+#       define CG_R_SHIFT                               0
+#       define CG_L(x)                                  ((x) << 16)
+#       define CG_L_MASK                                (0xffff << 16)
+#       define CG_L_SHIFT                               16
+#define CG_AT_1                                         0x72c
+#define CG_AT_2                                         0x730
+#define CG_THERMAL_INT                                  0x734
+#define         DIG_THERM_INTH(x)                       ((x) << 8)
+#define         DIG_THERM_INTH_MASK                     0x0000FF00
+#define         DIG_THERM_INTH_SHIFT                    8
+#define         DIG_THERM_INTL(x)                       ((x) << 16)
+#define         DIG_THERM_INTL_MASK                     0x00FF0000
+#define         DIG_THERM_INTL_SHIFT                    16
+#define         THERM_INT_MASK_HIGH                     (1 << 24)
+#define         THERM_INT_MASK_LOW                      (1 << 25)
+#define CG_AT_3                                         0x738
+#define CG_AT_4                                         0x73c
+#define CG_AT_5                                         0x740
+#define CG_AT_6                                         0x744
+#define CG_AT_7                                         0x748
+
+#define CG_BSP_0                                        0x750
+#       define BSP(x)                                   ((x) << 0)
+#       define BSP_MASK                                 (0xffff << 0)
+#       define BSP_SHIFT                                0
+#       define BSU(x)                                   ((x) << 16)
+#       define BSU_MASK                                 (0xf << 16)
+#       define BSU_SHIFT                                16
+
+#define CG_CG_VOLTAGE_CNTL                              0x770
+#       define REQ                                      (1 << 0)
+#       define LEVEL(x)                                 ((x) << 1)
+#       define LEVEL_MASK                               (0x3 << 1)
+#       define LEVEL_SHIFT                              1
+#       define CG_VOLTAGE_EN                            (1 << 3)
+#       define FORCE                                    (1 << 4)
+#       define PERIOD(x)                                ((x) << 8)
+#       define PERIOD_MASK                              (0xffff << 8)
+#       define PERIOD_SHIFT                             8
+#       define UNIT(x)                                  ((x) << 24)
+#       define UNIT_MASK                                (0xf << 24)
+#       define UNIT_SHIFT                               24
+
+#define CG_ACPI_VOLTAGE_CNTL                            0x780
+#       define ACPI_VOLTAGE_EN                          (1 << 8)
+
+#define CG_DPM_VOLTAGE_CNTL                             0x788
+#       define DPM_STATE0_LEVEL_MASK                    (0x3 << 0)
+#       define DPM_STATE0_LEVEL_SHIFT                   0
+#       define DPM_VOLTAGE_EN                           (1 << 16)
+
+#define CG_PWR_GATING_CNTL                              0x7ac
+#       define DYN_PWR_DOWN_EN                          (1 << 0)
+#       define ACPI_PWR_DOWN_EN                         (1 << 1)
+#       define GFX_CLK_OFF_PWR_DOWN_EN                  (1 << 2)
+#       define IOC_DISGPU_PWR_DOWN_EN                   (1 << 3)
+#       define FORCE_POWR_ON                            (1 << 4)
+#       define PGP(x)                                   ((x) << 8)
+#       define PGP_MASK                                 (0xffff << 8)
+#       define PGP_SHIFT                                8
+#       define PGU(x)                                   ((x) << 24)
+#       define PGU_MASK                                 (0xf << 24)
+#       define PGU_SHIFT                                24
+
+#define CG_CGTT_LOCAL_0                                 0x7d0
+#define CG_CGTT_LOCAL_1                                 0x7d4
+
+#define DEEP_SLEEP_CNTL                                 0x818
+#       define R_DIS                                    (1 << 3)
+#       define HS(x)                                    ((x) << 4)
+#       define HS_MASK                                  (0xfff << 4)
+#       define HS_SHIFT                                 4
+#       define ENABLE_DS                                (1 << 31)
+#define DEEP_SLEEP_CNTL2                                0x81c
+#       define LB_UFP_EN                                (1 << 0)
+#       define INOUT_C(x)                               ((x) << 4)
+#       define INOUT_C_MASK                             (0xff << 4)
+#       define INOUT_C_SHIFT                            4
+
+#define CG_SCRATCH2                                     0x824
+
+#define CG_SCLK_DPM_CTRL_11                             0x830
+
+#define HW_REV                                          0x5564
+#       define ATI_REV_ID_MASK                          (0xf << 28)
+#       define ATI_REV_ID_SHIFT                         28
+/* 0 = A0, 1 = A1, 2 = B0, 3 = C0, etc. */
+
+#define DOUT_SCRATCH3                                   0x611c
+
+#define GB_ADDR_CONFIG                                  0x98f8
+
+#endif
diff --git a/drivers/gpu/drm/radeon/trinity_dpm.c b/drivers/gpu/drm/radeon/trinity_dpm.c
new file mode 100644
index 000000000000..8a32bcc6bbb5
--- /dev/null
+++ b/drivers/gpu/drm/radeon/trinity_dpm.c
@@ -0,0 +1,1917 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "trinityd.h"
+#include "r600_dpm.h"
+#include "trinity_dpm.h"
+#include <linux/seq_file.h>
+
+#define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
+#define TRINITY_MINIMUM_ENGINE_CLOCK 800
+#define SCLK_MIN_DIV_INTV_SHIFT     12
+#define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
+
+#ifndef TRINITY_MGCG_SEQUENCE
+#define TRINITY_MGCG_SEQUENCE  100
+
+static const u32 trinity_mgcg_shls_default[] =
+{
+        /* Register, Value, Mask */
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00003fc4, 0xc0000000, 0xffffffff,
+        0x00005448, 0x00000100, 0xffffffff,
+        0x000055e4, 0x00000100, 0xffffffff,
+        0x0000160c, 0x00000100, 0xffffffff,
+        0x00008984, 0x06000100, 0xffffffff,
+        0x0000c164, 0x00000100, 0xffffffff,
+        0x00008a18, 0x00000100, 0xffffffff,
+        0x0000897c, 0x06000100, 0xffffffff,
+        0x00008b28, 0x00000100, 0xffffffff,
+        0x00009144, 0x00800200, 0xffffffff,
+        0x00009a60, 0x00000100, 0xffffffff,
+        0x00009868, 0x00000100, 0xffffffff,
+        0x00008d58, 0x00000100, 0xffffffff,
+        0x00009510, 0x00000100, 0xffffffff,
+        0x0000949c, 0x00000100, 0xffffffff,
+        0x00009654, 0x00000100, 0xffffffff,
+        0x00009030, 0x00000100, 0xffffffff,
+        0x00009034, 0x00000100, 0xffffffff,
+        0x00009038, 0x00000100, 0xffffffff,
+        0x0000903c, 0x00000100, 0xffffffff,
+        0x00009040, 0x00000100, 0xffffffff,
+        0x0000a200, 0x00000100, 0xffffffff,
+        0x0000a204, 0x00000100, 0xffffffff,
+        0x0000a208, 0x00000100, 0xffffffff,
+        0x0000a20c, 0x00000100, 0xffffffff,
+        0x00009744, 0x00000100, 0xffffffff,
+        0x00003f80, 0x00000100, 0xffffffff,
+        0x0000a210, 0x00000100, 0xffffffff,
+        0x0000a214, 0x00000100, 0xffffffff,
+        0x000004d8, 0x00000100, 0xffffffff,
+        0x00009664, 0x00000100, 0xffffffff,
+        0x00009698, 0x00000100, 0xffffffff,
+        0x000004d4, 0x00000200, 0xffffffff,
+        0x000004d0, 0x00000000, 0xffffffff,
+        0x000030cc, 0x00000104, 0xffffffff,
+        0x0000d0c0, 0x00000100, 0xffffffff,
+        0x0000d8c0, 0x00000100, 0xffffffff,
+        0x0000951c, 0x00010000, 0xffffffff,
+        0x00009160, 0x00030002, 0xffffffff,
+        0x00009164, 0x00050004, 0xffffffff,
+        0x00009168, 0x00070006, 0xffffffff,
+        0x00009178, 0x00070000, 0xffffffff,
+        0x0000917c, 0x00030002, 0xffffffff,
+        0x00009180, 0x00050004, 0xffffffff,
+        0x0000918c, 0x00010006, 0xffffffff,
+        0x00009190, 0x00090008, 0xffffffff,
+        0x00009194, 0x00070000, 0xffffffff,
+        0x00009198, 0x00030002, 0xffffffff,
+        0x0000919c, 0x00050004, 0xffffffff,
+        0x000091a8, 0x00010006, 0xffffffff,
+        0x000091ac, 0x00090008, 0xffffffff,
+        0x000091b0, 0x00070000, 0xffffffff,
+        0x000091b4, 0x00030002, 0xffffffff,
+        0x000091b8, 0x00050004, 0xffffffff,
+        0x000091c4, 0x00010006, 0xffffffff,
+        0x000091c8, 0x00090008, 0xffffffff,
+        0x000091cc, 0x00070000, 0xffffffff,
+        0x000091d0, 0x00030002, 0xffffffff,
+        0x000091d4, 0x00050004, 0xffffffff,
+        0x000091e0, 0x00010006, 0xffffffff,
+        0x000091e4, 0x00090008, 0xffffffff,
+        0x000091e8, 0x00000000, 0xffffffff,
+        0x000091ec, 0x00070000, 0xffffffff,
+        0x000091f0, 0x00030002, 0xffffffff,
+        0x000091f4, 0x00050004, 0xffffffff,
+        0x00009200, 0x00010006, 0xffffffff,
+        0x00009204, 0x00090008, 0xffffffff,
+        0x00009208, 0x00070000, 0xffffffff,
+        0x0000920c, 0x00030002, 0xffffffff,
+        0x00009210, 0x00050004, 0xffffffff,
+        0x0000921c, 0x00010006, 0xffffffff,
+        0x00009220, 0x00090008, 0xffffffff,
+        0x00009294, 0x00000000, 0xffffffff
+};
+
+static const u32 trinity_mgcg_shls_enable[] =
+{
+        /* Register, Value, Mask */
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0x00000000, 0x000133FF,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0x00000000, 0xE00B03FC,
+        0x00009150, 0x96944200, 0xffffffff
+};
+
+static const u32 trinity_mgcg_shls_disable[] =
+{
+        /* Register, Value, Mask */
+        0x0000802c, 0xc0000000, 0xffffffff,
+        0x00009150, 0x00600000, 0xffffffff,
+        0x000008f8, 0x00000000, 0xffffffff,
+        0x000008fc, 0xffffffff, 0x000133FF,
+        0x000008f8, 0x00000001, 0xffffffff,
+        0x000008fc, 0xffffffff, 0xE00B03FC
+};
+#endif
+
+#ifndef TRINITY_SYSLS_SEQUENCE
+#define TRINITY_SYSLS_SEQUENCE  100
+
+static const u32 trinity_sysls_default[] =
+{
+        /* Register, Value, Mask */
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x0000d8bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x00002f50, 0x00000404, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x0000641c, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+
+static const u32 trinity_sysls_disable[] =
+{
+        /* Register, Value, Mask */
+        0x0000d0c0, 0x00000000, 0xffffffff,
+        0x0000d8c0, 0x00000000, 0xffffffff,
+        0x000055e8, 0x00000000, 0xffffffff,
+        0x0000d0bc, 0x00000000, 0xffffffff,
+        0x0000d8bc, 0x00000000, 0xffffffff,
+        0x000015c0, 0x00041401, 0xffffffff,
+        0x0000264c, 0x00040400, 0xffffffff,
+        0x00002648, 0x00040400, 0xffffffff,
+        0x00002650, 0x00040400, 0xffffffff,
+        0x000020b8, 0x00040400, 0xffffffff,
+        0x000020bc, 0x00040400, 0xffffffff,
+        0x000020c0, 0x00040c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680000, 0xffffffff,
+        0x00002f50, 0x00000404, 0xffffffff,
+        0x000004c8, 0x00000001, 0xffffffff,
+        0x0000641c, 0x00007ffd, 0xffffffff,
+        0x00000c7c, 0x0000ff00, 0xffffffff,
+        0x00006dfc, 0x0000007f, 0xffffffff
+};
+
+static const u32 trinity_sysls_enable[] =
+{
+        /* Register, Value, Mask */
+        0x000055e8, 0x00000001, 0xffffffff,
+        0x0000d0bc, 0x00000100, 0xffffffff,
+        0x0000d8bc, 0x00000100, 0xffffffff,
+        0x000015c0, 0x000c1401, 0xffffffff,
+        0x0000264c, 0x000c0400, 0xffffffff,
+        0x00002648, 0x000c0400, 0xffffffff,
+        0x00002650, 0x000c0400, 0xffffffff,
+        0x000020b8, 0x000c0400, 0xffffffff,
+        0x000020bc, 0x000c0400, 0xffffffff,
+        0x000020c0, 0x000c0c80, 0xffffffff,
+        0x0000f4a0, 0x000000c0, 0xffffffff,
+        0x0000f4a4, 0x00680fff, 0xffffffff,
+        0x00002f50, 0x00000903, 0xffffffff,
+        0x000004c8, 0x00000000, 0xffffffff,
+        0x0000641c, 0x00000000, 0xffffffff,
+        0x00000c7c, 0x00000000, 0xffffffff,
+        0x00006dfc, 0x00000000, 0xffffffff
+};
+#endif
+
+static const u32 trinity_override_mgpg_sequences[] =
+{
+        /* Register, Value */
+        0x00000200, 0xE030032C,
+        0x00000204, 0x00000FFF,
+        0x00000200, 0xE0300058,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE0300054,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE0300074,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE0300070,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE0300090,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE030008C,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE03000AC,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE03000A8,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE03000C8,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE03000C4,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE03000E4,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE03000E0,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE0300100,
+        0x00000204, 0x00030301,
+        0x00000200, 0xE03000FC,
+        0x00000204, 0x500010FF,
+        0x00000200, 0xE0300058,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE0300054,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE0300074,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE0300070,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE0300090,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE030008C,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE03000AC,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000A8,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE03000C8,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000C4,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE03000E4,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000E0,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE0300100,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000FC,
+        0x00000204, 0x600010FF,
+        0x00000200, 0xE0300058,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE0300054,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE0300074,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE0300070,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE0300090,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE030008C,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE03000AC,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000A8,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE03000C8,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000C4,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE03000E4,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000E0,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE0300100,
+        0x00000204, 0x00030303,
+        0x00000200, 0xE03000FC,
+        0x00000204, 0x700010FF,
+        0x00000200, 0xE0300058,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE0300054,
+        0x00000204, 0x800010FF,
+        0x00000200, 0xE0300074,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE0300070,
+        0x00000204, 0x800010FF,
+        0x00000200, 0xE0300090,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE030008C,
+        0x00000204, 0x800010FF,
+        0x00000200, 0xE03000AC,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE03000A8,
+        0x00000204, 0x800010FF,
+        0x00000200, 0xE03000C4,
+        0x00000204, 0x800010FF,
+        0x00000200, 0xE03000C8,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE03000E4,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE03000E0,
+        0x00000204, 0x800010FF,
+        0x00000200, 0xE0300100,
+        0x00000204, 0x00010303,
+        0x00000200, 0xE03000FC,
+        0x00000204, 0x800010FF,
+        0x00000200, 0x0001f198,
+        0x00000204, 0x0003ffff,
+        0x00000200, 0x0001f19C,
+        0x00000204, 0x3fffffff,
+        0x00000200, 0xE030032C,
+        0x00000204, 0x00000000,
+};
+
+static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
+                                                   const u32 *seq, u32 count);
+static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
+static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
+                                             struct radeon_ps *new_rps,
+                                             struct radeon_ps *old_rps);
+
+struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
+{
+        struct trinity_ps *ps = rps->ps_priv;
+
+        return ps;
+}
+
+struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = rdev->pm.dpm.priv;
+
+        return pi;
+}
+
+static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 p, u;
+        u32 value;
+        struct atom_clock_dividers dividers;
+        u32 xclk = radeon_get_xclk(rdev);
+        u32 sssd = 1;
+        int ret;
+        u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             25000, false, &dividers);
+        if (ret)
+                return;
+
+        value = RREG32_SMC(GFX_POWER_GATING_CNTL);
+        value &= ~(SSSD_MASK | PDS_DIV_MASK);
+        if (sssd)
+                value |= SSSD(1);
+        value |= PDS_DIV(dividers.post_div);
+        WREG32_SMC(GFX_POWER_GATING_CNTL, value);
+
+        r600_calculate_u_and_p(500, xclk, 16, &p, &u);
+
+        WREG32(CG_PG_CTRL, SP(p) | SU(u));
+
+        WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
+
+        /* XXX double check hw_rev */
+        if (pi->override_dynamic_mgpg && (hw_rev == 0))
+                trinity_override_dynamic_mg_powergating(rdev);
+
+}
+
+#define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
+#define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
+#define CGTS_SM_CTRL_REG_DISABLE    0x00600000
+#define CGTS_SM_CTRL_REG_ENABLE     0x96944200
+
+static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
+                                          bool enable)
+{
+        u32 local0;
+        u32 local1;
+
+        if (enable) {
+                local0 = RREG32_CG(CG_CGTT_LOCAL_0);
+                local1 = RREG32_CG(CG_CGTT_LOCAL_1);
+
+                WREG32_CG(CG_CGTT_LOCAL_0,
+                          (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
+                WREG32_CG(CG_CGTT_LOCAL_1,
+                          (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
+
+                WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
+        } else {
+                WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
+
+                local0 = RREG32_CG(CG_CGTT_LOCAL_0);
+                local1 = RREG32_CG(CG_CGTT_LOCAL_1);
+
+                WREG32_CG(CG_CGTT_LOCAL_0,
+                          CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
+                WREG32_CG(CG_CGTT_LOCAL_1,
+                          CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
+        }
+}
+
+static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *seq = NULL;
+
+        seq = &trinity_mgcg_shls_default[0];
+        count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
+
+        trinity_program_clk_gating_hw_sequence(rdev, seq, count);
+}
+
+static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
+                                           bool enable)
+{
+        if (enable) {
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
+        } else {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
+                WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
+                RREG32(GB_ADDR_CONFIG);
+        }
+}
+
+static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
+                                                   const u32 *seq, u32 count)
+{
+        u32 i, length = count * 3;
+
+        for (i = 0; i < length; i += 3)
+                WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
+}
+
+static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
+                                                    const u32 *seq, u32 count)
+{
+        u32  i, length = count * 2;
+
+        for (i = 0; i < length; i += 2)
+                WREG32(seq[i], seq[i+1]);
+
+}
+
+static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
+{
+        u32 count;
+        const u32 *seq = NULL;
+
+        seq = &trinity_override_mgpg_sequences[0];
+        count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
+
+        trinity_program_override_mgpg_sequences(rdev, seq, count);
+}
+
+static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
+                                          bool enable)
+{
+        u32 count;
+        const u32 *seq = NULL;
+
+        if (enable) {
+                seq = &trinity_sysls_enable[0];
+                count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
+        } else {
+                seq = &trinity_sysls_disable[0];
+                count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
+        }
+
+        trinity_program_clk_gating_hw_sequence(rdev, seq, count);
+}
+
+static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
+                                           bool enable)
+{
+        if (enable) {
+                if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
+                        WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
+
+                WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
+        } else {
+                WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
+                RREG32(GB_ADDR_CONFIG);
+        }
+}
+
+static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
+                                            bool enable)
+{
+        u32 value;
+
+        if (enable) {
+                value = RREG32_SMC(PM_I_CNTL_1);
+                value &= ~DS_PG_CNTL_MASK;
+                value |= DS_PG_CNTL(1);
+                WREG32_SMC(PM_I_CNTL_1, value);
+
+                value = RREG32_SMC(SMU_S_PG_CNTL);
+                value &= ~DS_PG_EN_MASK;
+                value |= DS_PG_EN(1);
+                WREG32_SMC(SMU_S_PG_CNTL, value);
+        } else {
+                value = RREG32_SMC(SMU_S_PG_CNTL);
+                value &= ~DS_PG_EN_MASK;
+                WREG32_SMC(SMU_S_PG_CNTL, value);
+
+                value = RREG32_SMC(PM_I_CNTL_1);
+                value &= ~DS_PG_CNTL_MASK;
+                WREG32_SMC(PM_I_CNTL_1, value);
+        }
+
+        trinity_gfx_dynamic_mgpg_config(rdev);
+
+}
+
+static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        if (pi->enable_gfx_clock_gating)
+                sumo_gfx_clockgating_initialize(rdev);
+        if (pi->enable_mg_clock_gating)
+                trinity_mg_clockgating_initialize(rdev);
+        if (pi->enable_gfx_power_gating)
+                trinity_gfx_powergating_initialize(rdev);
+        if (pi->enable_mg_clock_gating) {
+                trinity_ls_clockgating_enable(rdev, true);
+                trinity_mg_clockgating_enable(rdev, true);
+        }
+        if (pi->enable_gfx_clock_gating)
+                trinity_gfx_clockgating_enable(rdev, true);
+        if (pi->enable_gfx_dynamic_mgpg)
+                trinity_gfx_dynamic_mgpg_enable(rdev, true);
+        if (pi->enable_gfx_power_gating)
+                trinity_gfx_powergating_enable(rdev, true);
+}
+
+static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        if (pi->enable_gfx_power_gating)
+                trinity_gfx_powergating_enable(rdev, false);
+        if (pi->enable_gfx_dynamic_mgpg)
+                trinity_gfx_dynamic_mgpg_enable(rdev, false);
+        if (pi->enable_gfx_clock_gating)
+                trinity_gfx_clockgating_enable(rdev, false);
+        if (pi->enable_mg_clock_gating) {
+                trinity_mg_clockgating_enable(rdev, false);
+                trinity_ls_clockgating_enable(rdev, false);
+        }
+}
+
+static void trinity_set_divider_value(struct radeon_device *rdev,
+                                      u32 index, u32 sclk)
+{
+        struct atom_clock_dividers  dividers;
+        int ret;
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             sclk, false, &dividers);
+        if (ret)
+                return;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
+        value &= ~CLK_DIVIDER_MASK;
+        value |= CLK_DIVIDER(dividers.post_div);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
+
+        ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
+                                             sclk/2, false, &dividers);
+        if (ret)
+                return;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
+        value &= ~PD_SCLK_DIVIDER_MASK;
+        value |= PD_SCLK_DIVIDER(dividers.post_div);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
+}
+
+static void trinity_set_ds_dividers(struct radeon_device *rdev,
+                                    u32 index, u32 divider)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
+        value &= ~DS_DIV_MASK;
+        value |= DS_DIV(divider);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
+}
+
+static void trinity_set_ss_dividers(struct radeon_device *rdev,
+                                    u32 index, u32 divider)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
+        value &= ~DS_SH_DIV_MASK;
+        value |= DS_SH_DIV(divider);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
+}
+
+static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
+        value &= ~VID_MASK;
+        value |= VID(vid_7bit);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
+        value &= ~LVRT_MASK;
+        value |= LVRT(0);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
+}
+
+static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
+                                       u32 index, u32 gnb_slow)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
+        value &= ~GNB_SLOW_MASK;
+        value |= GNB_SLOW(gnb_slow);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
+}
+
+static void trinity_set_force_nbp_state(struct radeon_device *rdev,
+                                        u32 index, u32 force_nbp_state)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
+        value &= ~FORCE_NBPS1_MASK;
+        value |= FORCE_NBPS1(force_nbp_state);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
+}
+
+static void trinity_set_display_wm(struct radeon_device *rdev,
+                                   u32 index, u32 wm)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
+        value &= ~DISPLAY_WM_MASK;
+        value |= DISPLAY_WM(wm);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
+}
+
+static void trinity_set_vce_wm(struct radeon_device *rdev,
+                               u32 index, u32 wm)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
+        value &= ~VCE_WM_MASK;
+        value |= VCE_WM(wm);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
+}
+
+static void trinity_set_at(struct radeon_device *rdev,
+                           u32 index, u32 at)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
+        value &= ~AT_MASK;
+        value |= AT(at);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
+}
+
+static void trinity_program_power_level(struct radeon_device *rdev,
+                                        struct trinity_pl *pl, u32 index)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
+                return;
+
+        trinity_set_divider_value(rdev, index, pl->sclk);
+        trinity_set_vid(rdev, index, pl->vddc_index);
+        trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
+        trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
+        trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
+        trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
+        trinity_set_display_wm(rdev, index, pl->display_wm);
+        trinity_set_vce_wm(rdev, index, pl->vce_wm);
+        trinity_set_at(rdev, index, pi->at[index]);
+}
+
+static void trinity_power_level_enable_disable(struct radeon_device *rdev,
+                                               u32 index, bool enable)
+{
+        u32 value;
+        u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
+
+        value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
+        value &= ~STATE_VALID_MASK;
+        if (enable)
+                value |= STATE_VALID(1);
+        WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
+}
+
+static bool trinity_dpm_enabled(struct radeon_device *rdev)
+{
+        if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
+                return true;
+        else
+                return false;
+}
+
+static void trinity_start_dpm(struct radeon_device *rdev)
+{
+        u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
+
+        value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
+        value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
+        WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
+
+        WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+        WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
+
+        trinity_dpm_config(rdev, true);
+}
+
+static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
+                        break;
+                udelay(1);
+        }
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
+                        break;
+                udelay(1);
+        }
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
+                        break;
+                udelay(1);
+        }
+}
+
+static void trinity_stop_dpm(struct radeon_device *rdev)
+{
+        u32 sclk_dpm_cntl;
+
+        WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
+
+        sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
+        sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
+        WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
+
+        trinity_dpm_config(rdev, false);
+}
+
+static void trinity_start_am(struct radeon_device *rdev)
+{
+        WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
+}
+
+static void trinity_reset_am(struct radeon_device *rdev)
+{
+        WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
+                 ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
+}
+
+static void trinity_wait_for_level_0(struct radeon_device *rdev)
+{
+        int i;
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
+                        break;
+                udelay(1);
+        }
+}
+
+static void trinity_enable_power_level_0(struct radeon_device *rdev)
+{
+        trinity_power_level_enable_disable(rdev, 0, true);
+}
+
+static void trinity_force_level_0(struct radeon_device *rdev)
+{
+        trinity_dpm_force_state(rdev, 0);
+}
+
+static void trinity_unforce_levels(struct radeon_device *rdev)
+{
+        trinity_dpm_no_forced_level(rdev);
+}
+
+static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
+                                                struct radeon_ps *new_rps,
+                                                struct radeon_ps *old_rps)
+{
+        struct trinity_ps *new_ps = trinity_get_ps(new_rps);
+        struct trinity_ps *old_ps = trinity_get_ps(old_rps);
+        u32 i;
+        u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
+
+        for (i = 0; i < new_ps->num_levels; i++) {
+                trinity_program_power_level(rdev, &new_ps->levels[i], i);
+                trinity_power_level_enable_disable(rdev, i, true);
+        }
+
+        for (i = new_ps->num_levels; i < n_current_state_levels; i++)
+                trinity_power_level_enable_disable(rdev, i, false);
+}
+
+static void trinity_program_bootup_state(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 i;
+
+        trinity_program_power_level(rdev, &pi->boot_pl, 0);
+        trinity_power_level_enable_disable(rdev, 0, true);
+
+        for (i = 1; i < 8; i++)
+                trinity_power_level_enable_disable(rdev, i, false);
+}
+
+static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
+                                          struct radeon_ps *rps)
+{
+        struct trinity_ps *ps = trinity_get_ps(rps);
+        u32 uvdstates = (ps->vclk_low_divider |
+                         ps->vclk_high_divider << 8 |
+                         ps->dclk_low_divider << 16 |
+                         ps->dclk_high_divider << 24);
+
+        WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
+}
+
+static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
+                                           u32 interval)
+{
+        u32 p, u;
+        u32 tp = RREG32_SMC(PM_TP);
+        u32 val;
+        u32 xclk = radeon_get_xclk(rdev);
+
+        r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
+
+        val = (p + tp - 1) / tp;
+
+        WREG32_SMC(SMU_UVD_DPM_CNTL, val);
+}
+
+static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
+{
+        if ((rps->vclk == 0) && (rps->dclk == 0))
+                return true;
+        else
+                return false;
+}
+
+static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
+                                     struct radeon_ps *rps2)
+{
+        struct trinity_ps *ps1 = trinity_get_ps(rps1);
+        struct trinity_ps *ps2 = trinity_get_ps(rps2);
+
+        if ((rps1->vclk == rps2->vclk) &&
+            (rps1->dclk == rps2->dclk) &&
+            (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
+            (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
+            (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
+            (ps1->dclk_high_divider == ps2->dclk_high_divider))
+                return true;
+        else
+                return false;
+}
+
+static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
+                                     struct radeon_ps *new_rps,
+                                     struct radeon_ps *old_rps)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        if (pi->enable_gfx_power_gating) {
+                trinity_gfx_powergating_enable(rdev, false);
+        }
+
+        if (pi->uvd_dpm) {
+                if (trinity_uvd_clocks_zero(new_rps) &&
+                    !trinity_uvd_clocks_zero(old_rps)) {
+                        trinity_setup_uvd_dpm_interval(rdev, 0);
+                } else if (!trinity_uvd_clocks_zero(new_rps)) {
+                        trinity_setup_uvd_clock_table(rdev, new_rps);
+
+                        if (trinity_uvd_clocks_zero(old_rps)) {
+                                u32 tmp = RREG32(CG_MISC_REG);
+                                tmp &= 0xfffffffd;
+                                WREG32(CG_MISC_REG, tmp);
+
+                                radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
+
+                                trinity_setup_uvd_dpm_interval(rdev, 3000);
+                        }
+                }
+                trinity_uvd_dpm_config(rdev);
+        } else {
+                if (trinity_uvd_clocks_zero(new_rps) ||
+                    trinity_uvd_clocks_equal(new_rps, old_rps))
+                        return;
+
+                radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
+        }
+
+        if (pi->enable_gfx_power_gating) {
+                trinity_gfx_powergating_enable(rdev, true);
+        }
+}
+
+static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
+                                                       struct radeon_ps *new_rps,
+                                                       struct radeon_ps *old_rps)
+{
+        struct trinity_ps *new_ps = trinity_get_ps(new_rps);
+        struct trinity_ps *current_ps = trinity_get_ps(new_rps);
+
+        if (new_ps->levels[new_ps->num_levels - 1].sclk >=
+            current_ps->levels[current_ps->num_levels - 1].sclk)
+                return;
+
+        trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
+}
+
+static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
+                                                      struct radeon_ps *new_rps,
+                                                      struct radeon_ps *old_rps)
+{
+        struct trinity_ps *new_ps = trinity_get_ps(new_rps);
+        struct trinity_ps *current_ps = trinity_get_ps(old_rps);
+
+        if (new_ps->levels[new_ps->num_levels - 1].sclk <
+            current_ps->levels[current_ps->num_levels - 1].sclk)
+                return;
+
+        trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
+}
+
+static void trinity_program_ttt(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
+
+        value &= ~(HT_MASK | LT_MASK);
+        value |= HT((pi->thermal_auto_throttling + 49) * 8);
+        value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
+        WREG32_SMC(SMU_SCLK_DPM_TTT, value);
+}
+
+static void trinity_enable_att(struct radeon_device *rdev)
+{
+        u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
+
+        value &= ~SCLK_TT_EN_MASK;
+        value |= SCLK_TT_EN(1);
+        WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
+}
+
+static void trinity_program_sclk_dpm(struct radeon_device *rdev)
+{
+        u32 p, u;
+        u32 tp = RREG32_SMC(PM_TP);
+        u32 ni;
+        u32 xclk = radeon_get_xclk(rdev);
+        u32 value;
+
+        r600_calculate_u_and_p(400, xclk, 16, &p, &u);
+
+        ni = (p + tp - 1) / tp;
+
+        value = RREG32_SMC(PM_I_CNTL_1);
+        value &= ~SCLK_DPM_MASK;
+        value |= SCLK_DPM(ni);
+        WREG32_SMC(PM_I_CNTL_1, value);
+}
+
+static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
+                                                 int min_temp, int max_temp)
+{
+        int low_temp = 0 * 1000;
+        int high_temp = 255 * 1000;
+
+        if (low_temp < min_temp)
+                low_temp = min_temp;
+        if (high_temp > max_temp)
+                high_temp = max_temp;
+        if (high_temp < low_temp) {
+                DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
+                return -EINVAL;
+        }
+
+        WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
+        WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
+
+        rdev->pm.dpm.thermal.min_temp = low_temp;
+        rdev->pm.dpm.thermal.max_temp = high_temp;
+
+        return 0;
+}
+
+static void trinity_update_current_ps(struct radeon_device *rdev,
+                                      struct radeon_ps *rps)
+{
+        struct trinity_ps *new_ps = trinity_get_ps(rps);
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        pi->current_rps = *rps;
+        pi->current_ps = *new_ps;
+        pi->current_rps.ps_priv = &pi->current_ps;
+}
+
+static void trinity_update_requested_ps(struct radeon_device *rdev,
+                                        struct radeon_ps *rps)
+{
+        struct trinity_ps *new_ps = trinity_get_ps(rps);
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        pi->requested_rps = *rps;
+        pi->requested_ps = *new_ps;
+        pi->requested_rps.ps_priv = &pi->requested_ps;
+}
+
+int trinity_dpm_enable(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        int ret;
+
+        trinity_acquire_mutex(rdev);
+
+        if (trinity_dpm_enabled(rdev)) {
+                trinity_release_mutex(rdev);
+                return -EINVAL;
+        }
+
+        trinity_enable_clock_power_gating(rdev);
+        trinity_program_bootup_state(rdev);
+        sumo_program_vc(rdev, 0x00C00033);
+        trinity_start_am(rdev);
+        if (pi->enable_auto_thermal_throttling) {
+                trinity_program_ttt(rdev);
+                trinity_enable_att(rdev);
+        }
+        trinity_program_sclk_dpm(rdev);
+        trinity_start_dpm(rdev);
+        trinity_wait_for_dpm_enabled(rdev);
+        trinity_release_mutex(rdev);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+                if (ret) {
+                        trinity_release_mutex(rdev);
+                        return ret;
+                }
+                rdev->irq.dpm_thermal = true;
+                radeon_irq_set(rdev);
+        }
+
+        trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
+
+        return 0;
+}
+
+void trinity_dpm_disable(struct radeon_device *rdev)
+{
+        trinity_acquire_mutex(rdev);
+        if (!trinity_dpm_enabled(rdev)) {
+                trinity_release_mutex(rdev);
+                return;
+        }
+        trinity_disable_clock_power_gating(rdev);
+        sumo_clear_vc(rdev);
+        trinity_wait_for_level_0(rdev);
+        trinity_stop_dpm(rdev);
+        trinity_reset_am(rdev);
+        trinity_release_mutex(rdev);
+
+        if (rdev->irq.installed &&
+            r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
+                rdev->irq.dpm_thermal = false;
+                radeon_irq_set(rdev);
+        }
+
+        trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
+}
+
+static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        pi->min_sclk_did =
+                (RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
+}
+
+static void trinity_setup_nbp_sim(struct radeon_device *rdev,
+                                  struct radeon_ps *rps)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct trinity_ps *new_ps = trinity_get_ps(rps);
+        u32 nbpsconfig;
+
+        if (pi->sys_info.nb_dpm_enable) {
+                nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
+                nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
+                nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
+                               Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
+                               DpmXNbPsLo(new_ps->DpmXNbPsLo) |
+                               DpmXNbPsHi(new_ps->DpmXNbPsHi));
+                WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
+        }
+}
+
+int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
+        struct radeon_ps *new_ps = &requested_ps;
+
+        trinity_update_requested_ps(rdev, new_ps);
+
+        trinity_apply_state_adjust_rules(rdev,
+                                         &pi->requested_rps,
+                                         &pi->current_rps);
+
+        return 0;
+}
+
+int trinity_dpm_set_power_state(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct radeon_ps *new_ps = &pi->requested_rps;
+        struct radeon_ps *old_ps = &pi->current_rps;
+
+        trinity_acquire_mutex(rdev);
+        if (pi->enable_dpm) {
+                trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
+                trinity_enable_power_level_0(rdev);
+                trinity_force_level_0(rdev);
+                trinity_wait_for_level_0(rdev);
+                trinity_setup_nbp_sim(rdev, new_ps);
+                trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
+                trinity_force_level_0(rdev);
+                trinity_unforce_levels(rdev);
+                trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
+        }
+        trinity_release_mutex(rdev);
+
+        return 0;
+}
+
+void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct radeon_ps *new_ps = &pi->requested_rps;
+
+        trinity_update_current_ps(rdev, new_ps);
+}
+
+void trinity_dpm_setup_asic(struct radeon_device *rdev)
+{
+        trinity_acquire_mutex(rdev);
+        sumo_program_sstp(rdev);
+        sumo_take_smu_control(rdev, true);
+        trinity_get_min_sclk_divider(rdev);
+        trinity_release_mutex(rdev);
+}
+
+void trinity_dpm_reset_asic(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        trinity_acquire_mutex(rdev);
+        if (pi->enable_dpm) {
+                trinity_enable_power_level_0(rdev);
+                trinity_force_level_0(rdev);
+                trinity_wait_for_level_0(rdev);
+                trinity_program_bootup_state(rdev);
+                trinity_force_level_0(rdev);
+                trinity_unforce_levels(rdev);
+        }
+        trinity_release_mutex(rdev);
+}
+
+static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
+                                                  u32 vid_2bit)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
+        u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
+        u32 step = (svi_mode == 0) ? 1250 : 625;
+        u32 delta = vid_7bit * step + 50;
+
+        if (delta > 155000)
+                return 0;
+
+        return (155000 - delta) / 100;
+}
+
+static void trinity_patch_boot_state(struct radeon_device *rdev,
+                                     struct trinity_ps *ps)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        ps->num_levels = 1;
+        ps->nbps_flags = 0;
+        ps->bapm_flags = 0;
+        ps->levels[0] = pi->boot_pl;
+}
+
+static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
+{
+        if (sclk < 20000)
+                return 1;
+        return 0;
+}
+
+static void trinity_construct_boot_state(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
+        pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
+        pi->boot_pl.ds_divider_index = 0;
+        pi->boot_pl.ss_divider_index = 0;
+        pi->boot_pl.allow_gnb_slow = 1;
+        pi->boot_pl.force_nbp_state = 0;
+        pi->boot_pl.display_wm = 0;
+        pi->boot_pl.vce_wm = 0;
+        pi->current_ps.num_levels = 1;
+        pi->current_ps.levels[0] = pi->boot_pl;
+}
+
+static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
+                                                  u32 sclk, u32 min_sclk_in_sr)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 i;
+        u32 temp;
+        u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
+                min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
+
+        if (sclk < min)
+                return 0;
+
+        if (!pi->enable_sclk_ds)
+                return 0;
+
+        for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
+                temp = sclk / sumo_get_sleep_divider_from_id(i);
+                if (temp >= min || i == 0)
+                        break;
+        }
+
+        return (u8)i;
+}
+
+static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
+                                          u32 lower_limit)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 i;
+
+        for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
+                if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
+                        return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
+        }
+
+        if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
+                DRM_ERROR("engine clock out of range!");
+
+        return 0;
+}
+
+static void trinity_patch_thermal_state(struct radeon_device *rdev,
+                                        struct trinity_ps *ps,
+                                        struct trinity_ps *current_ps)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
+        u32 current_vddc;
+        u32 current_sclk;
+        u32 current_index = 0;
+
+        if (current_ps) {
+                current_vddc = current_ps->levels[current_index].vddc_index;
+                current_sclk = current_ps->levels[current_index].sclk;
+        } else {
+                current_vddc = pi->boot_pl.vddc_index;
+                current_sclk = pi->boot_pl.sclk;
+        }
+
+        ps->levels[0].vddc_index = current_vddc;
+
+        if (ps->levels[0].sclk > current_sclk)
+                ps->levels[0].sclk = current_sclk;
+
+        ps->levels[0].ds_divider_index =
+                trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
+        ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
+        ps->levels[0].allow_gnb_slow = 1;
+        ps->levels[0].force_nbp_state = 0;
+        ps->levels[0].display_wm = 0;
+        ps->levels[0].vce_wm =
+                trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
+}
+
+static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
+                                       struct trinity_ps *ps, u32 index)
+{
+        if (ps == NULL || ps->num_levels <= 1)
+                return 0;
+        else if (ps->num_levels == 2) {
+                if (index == 0)
+                        return 0;
+                else
+                        return 1;
+        } else {
+                if (index == 0)
+                        return 0;
+                else if (ps->levels[index].sclk < 30000)
+                        return 0;
+                else
+                        return 1;
+        }
+}
+
+static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
+                                       struct radeon_ps *rps)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 i = 0;
+
+        for (i = 0; i < 4; i++) {
+                if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
+                    (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
+                    break;
+        }
+
+        if (i >= 4) {
+                DRM_ERROR("UVD clock index not found!\n");
+                i = 3;
+        }
+        return i;
+}
+
+static void trinity_adjust_uvd_state(struct radeon_device *rdev,
+                                     struct radeon_ps *rps)
+{
+        struct trinity_ps *ps = trinity_get_ps(rps);
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 high_index = 0;
+        u32 low_index = 0;
+
+        if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
+                high_index = trinity_get_uvd_clock_index(rdev, rps);
+
+                switch(high_index) {
+                case 3:
+                case 2:
+                        low_index = 1;
+                        break;
+                case 1:
+                case 0:
+                default:
+                        low_index = 0;
+                        break;
+                }
+
+                ps->vclk_low_divider =
+                        pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
+                ps->dclk_low_divider =
+                        pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
+                ps->vclk_high_divider =
+                        pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
+                ps->dclk_high_divider =
+                        pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
+        }
+}
+
+
+
+static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
+                                             struct radeon_ps *new_rps,
+                                             struct radeon_ps *old_rps)
+{
+        struct trinity_ps *ps = trinity_get_ps(new_rps);
+        struct trinity_ps *current_ps = trinity_get_ps(old_rps);
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 min_voltage = 0; /* ??? */
+        u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
+        u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
+        u32 i;
+        bool force_high;
+        u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
+
+        if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
+                return trinity_patch_thermal_state(rdev, ps, current_ps);
+
+        trinity_adjust_uvd_state(rdev, new_rps);
+
+        for (i = 0; i < ps->num_levels; i++) {
+                if (ps->levels[i].vddc_index < min_voltage)
+                        ps->levels[i].vddc_index = min_voltage;
+
+                if (ps->levels[i].sclk < min_sclk)
+                        ps->levels[i].sclk =
+                                trinity_get_valid_engine_clock(rdev, min_sclk);
+
+                ps->levels[i].ds_divider_index =
+                        sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
+
+                ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
+
+                ps->levels[i].allow_gnb_slow = 1;
+                ps->levels[i].force_nbp_state = 0;
+                ps->levels[i].display_wm =
+                        trinity_calculate_display_wm(rdev, ps, i);
+                ps->levels[i].vce_wm =
+                        trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
+        }
+
+        if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
+            ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
+                ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
+
+        if (pi->sys_info.nb_dpm_enable) {
+                ps->Dpm0PgNbPsLo = 0x1;
+                ps->Dpm0PgNbPsHi = 0x0;
+                ps->DpmXNbPsLo = 0x2;
+                ps->DpmXNbPsHi = 0x1;
+
+                if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
+                    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
+                        force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
+                                      ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
+                                       (pi->sys_info.uma_channel_number == 1)));
+                        force_high = (num_active_displays >= 3) || force_high;
+                        ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
+                        ps->Dpm0PgNbPsHi = 0x1;
+                        ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
+                        ps->DpmXNbPsHi = 0x2;
+                        ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
+                }
+        }
+}
+
+static void trinity_cleanup_asic(struct radeon_device *rdev)
+{
+        sumo_take_smu_control(rdev, false);
+}
+
+#if 0
+static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        if (pi->voltage_drop_in_dce)
+                trinity_dce_enable_voltage_adjustment(rdev, false);
+}
+#endif
+
+static void trinity_add_dccac_value(struct radeon_device *rdev)
+{
+        u32 gpu_cac_avrg_cntl_window_size;
+        u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
+        u64 disp_clk = rdev->clock.default_dispclk / 100;
+        u32 dc_cac_value;
+
+        gpu_cac_avrg_cntl_window_size =
+                (RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
+
+        dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
+                             (32 - gpu_cac_avrg_cntl_window_size));
+
+        WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
+}
+
+void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        if (pi->voltage_drop_in_dce)
+                trinity_dce_enable_voltage_adjustment(rdev, true);
+        trinity_add_dccac_value(rdev);
+}
+
+union power_info {
+        struct _ATOM_POWERPLAY_INFO info;
+        struct _ATOM_POWERPLAY_INFO_V2 info_2;
+        struct _ATOM_POWERPLAY_INFO_V3 info_3;
+        struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+        struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+        struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+};
+
+union pplib_clock_info {
+        struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+        struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+        struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+        struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+};
+
+union pplib_power_state {
+        struct _ATOM_PPLIB_STATE v1;
+        struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
+                                               struct radeon_ps *rps,
+                                               struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+                                               u8 table_rev)
+{
+        struct trinity_ps *ps = trinity_get_ps(rps);
+
+        rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+        rps->class = le16_to_cpu(non_clock_info->usClassification);
+        rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+        if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+                rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+                rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+        } else {
+                rps->vclk = 0;
+                rps->dclk = 0;
+        }
+
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+                rdev->pm.dpm.boot_ps = rps;
+                trinity_patch_boot_state(rdev, ps);
+        }
+        if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+                rdev->pm.dpm.uvd_ps = rps;
+}
+
+static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
+                                           struct radeon_ps *rps, int index,
+                                           union pplib_clock_info *clock_info)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct trinity_ps *ps = trinity_get_ps(rps);
+        struct trinity_pl *pl = &ps->levels[index];
+        u32 sclk;
+
+        sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
+        sclk |= clock_info->sumo.ucEngineClockHigh << 16;
+        pl->sclk = sclk;
+        pl->vddc_index = clock_info->sumo.vddcIndex;
+
+        ps->num_levels = index + 1;
+
+        if (pi->enable_sclk_ds) {
+                pl->ds_divider_index = 5;
+                pl->ss_divider_index = 5;
+        }
+}
+
+static int trinity_parse_power_table(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+        union pplib_power_state *power_state;
+        int i, j, k, non_clock_array_index, clock_array_index;
+        union pplib_clock_info *clock_info;
+        struct _StateArray *state_array;
+        struct _ClockInfoArray *clock_info_array;
+        struct _NonClockInfoArray *non_clock_info_array;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+        u8 *power_state_offset;
+        struct sumo_ps *ps;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        state_array = (struct _StateArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usStateArrayOffset));
+        clock_info_array = (struct _ClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
+        non_clock_info_array = (struct _NonClockInfoArray *)
+                (mode_info->atom_context->bios + data_offset +
+                 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
+
+        rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
+                                  state_array->ucNumEntries, GFP_KERNEL);
+        if (!rdev->pm.dpm.ps)
+                return -ENOMEM;
+        power_state_offset = (u8 *)state_array->states;
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+        for (i = 0; i < state_array->ucNumEntries; i++) {
+                power_state = (union pplib_power_state *)power_state_offset;
+                non_clock_array_index = power_state->v2.nonClockInfoIndex;
+                non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+                        &non_clock_info_array->nonClockInfo[non_clock_array_index];
+                if (!rdev->pm.power_state[i].clock_info)
+                        return -EINVAL;
+                ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
+                if (ps == NULL) {
+                        kfree(rdev->pm.dpm.ps);
+                        return -ENOMEM;
+                }
+                rdev->pm.dpm.ps[i].ps_priv = ps;
+                k = 0;
+                for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
+                        clock_array_index = power_state->v2.clockInfoIndex[j];
+                        if (clock_array_index >= clock_info_array->ucNumEntries)
+                                continue;
+                        if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
+                                break;
+                        clock_info = (union pplib_clock_info *)
+                                &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
+                        trinity_parse_pplib_clock_info(rdev,
+                                                       &rdev->pm.dpm.ps[i], k,
+                                                       clock_info);
+                        k++;
+                }
+                trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
+                                                   non_clock_info,
+                                                   non_clock_info_array->ucEntrySize);
+                power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
+        }
+        rdev->pm.dpm.num_ps = state_array->ucNumEntries;
+        return 0;
+}
+
+union igp_info {
+        struct _ATOM_INTEGRATED_SYSTEM_INFO info;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
+        struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
+};
+
+static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        u32 divider;
+
+        if (did >= 8 && did <= 0x3f)
+                divider = did * 25;
+        else if (did > 0x3f && did <= 0x5f)
+                divider = (did - 64) * 50 + 1600;
+        else if (did > 0x5f && did <= 0x7e)
+                divider = (did - 96) * 100 + 3200;
+        else if (did == 0x7f)
+                divider = 128 * 100;
+        else
+                return 10000;
+
+        return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
+}
+
+static int trinity_parse_sys_info_table(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
+        union igp_info *igp_info;
+        u8 frev, crev;
+        u16 data_offset;
+        int i;
+
+        if (atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset)) {
+                igp_info = (union igp_info *)(mode_info->atom_context->bios +
+                                              data_offset);
+
+                if (crev != 7) {
+                        DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
+                        return -EINVAL;
+                }
+                pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
+                pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
+                pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
+                pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
+                pi->sys_info.bootup_nb_voltage_index =
+                        le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
+                if (igp_info->info_7.ucHtcTmpLmt == 0)
+                        pi->sys_info.htc_tmp_lmt = 203;
+                else
+                        pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
+                if (igp_info->info_7.ucHtcHystLmt == 0)
+                        pi->sys_info.htc_hyst_lmt = 5;
+                else
+                        pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
+                if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
+                        DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
+                }
+
+                if (pi->enable_nbps_policy)
+                        pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
+                else
+                        pi->sys_info.nb_dpm_enable = 0;
+
+                for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
+                        pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
+                        pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
+                }
+
+                pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
+                pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
+                pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
+                pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
+
+                if (!pi->sys_info.nb_dpm_enable) {
+                        for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
+                                pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
+                                pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
+                                pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
+                        }
+                }
+
+                pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
+
+                sumo_construct_sclk_voltage_mapping_table(rdev,
+                                                          &pi->sys_info.sclk_voltage_mapping_table,
+                                                          igp_info->info_7.sAvail_SCLK);
+                sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
+                                                 igp_info->info_7.sAvail_SCLK);
+
+                pi->sys_info.uvd_clock_table_entries[0].vclk_did =
+                        igp_info->info_7.ucDPMState0VclkFid;
+                pi->sys_info.uvd_clock_table_entries[1].vclk_did =
+                        igp_info->info_7.ucDPMState1VclkFid;
+                pi->sys_info.uvd_clock_table_entries[2].vclk_did =
+                        igp_info->info_7.ucDPMState2VclkFid;
+                pi->sys_info.uvd_clock_table_entries[3].vclk_did =
+                        igp_info->info_7.ucDPMState3VclkFid;
+
+                pi->sys_info.uvd_clock_table_entries[0].dclk_did =
+                        igp_info->info_7.ucDPMState0DclkFid;
+                pi->sys_info.uvd_clock_table_entries[1].dclk_did =
+                        igp_info->info_7.ucDPMState1DclkFid;
+                pi->sys_info.uvd_clock_table_entries[2].dclk_did =
+                        igp_info->info_7.ucDPMState2DclkFid;
+                pi->sys_info.uvd_clock_table_entries[3].dclk_did =
+                        igp_info->info_7.ucDPMState3DclkFid;
+
+                for (i = 0; i < 4; i++) {
+                        pi->sys_info.uvd_clock_table_entries[i].vclk =
+                                trinity_convert_did_to_freq(rdev,
+                                                            pi->sys_info.uvd_clock_table_entries[i].vclk_did);
+                        pi->sys_info.uvd_clock_table_entries[i].dclk =
+                                trinity_convert_did_to_freq(rdev,
+                                                            pi->sys_info.uvd_clock_table_entries[i].dclk_did);
+                }
+
+
+
+        }
+        return 0;
+}
+
+int trinity_dpm_init(struct radeon_device *rdev)
+{
+        struct trinity_power_info *pi;
+        int ret, i;
+
+        pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
+        if (pi == NULL)
+                return -ENOMEM;
+        rdev->pm.dpm.priv = pi;
+
+        for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
+                pi->at[i] = TRINITY_AT_DFLT;
+
+        pi->enable_nbps_policy = true;
+        pi->enable_sclk_ds = true;
+        pi->enable_gfx_power_gating = true;
+        pi->enable_gfx_clock_gating = true;
+        pi->enable_mg_clock_gating = true;
+        pi->enable_gfx_dynamic_mgpg = true; /* ??? */
+        pi->override_dynamic_mgpg = true;
+        pi->enable_auto_thermal_throttling = true;
+        pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
+        pi->uvd_dpm = true; /* ??? */
+
+        ret = trinity_parse_sys_info_table(rdev);
+        if (ret)
+                return ret;
+
+        trinity_construct_boot_state(rdev);
+
+        ret = trinity_parse_power_table(rdev);
+        if (ret)
+                return ret;
+
+        pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
+        pi->enable_dpm = true;
+
+        return 0;
+}
+
+void trinity_dpm_print_power_state(struct radeon_device *rdev,
+                                   struct radeon_ps *rps)
+{
+        int i;
+        struct trinity_ps *ps = trinity_get_ps(rps);
+
+        r600_dpm_print_class_info(rps->class, rps->class2);
+        r600_dpm_print_cap_info(rps->caps);
+        printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+        for (i = 0; i < ps->num_levels; i++) {
+                struct trinity_pl *pl = &ps->levels[i];
+                printk("\t\tpower level %d    sclk: %u vddc: %u\n",
+                       i, pl->sclk,
+                       trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
+        }
+        r600_dpm_print_ps_status(rdev, rps);
+}
+
+void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
+                                                         struct seq_file *m)
+{
+        struct radeon_ps *rps = rdev->pm.dpm.current_ps;
+        struct trinity_ps *ps = trinity_get_ps(rps);
+        struct trinity_pl *pl;
+        u32 current_index =
+                (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
+                CURRENT_STATE_SHIFT;
+
+        if (current_index >= ps->num_levels) {
+                seq_printf(m, "invalid dpm profile %d\n", current_index);
+        } else {
+                pl = &ps->levels[current_index];
+                seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+                seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
+                           current_index, pl->sclk,
+                           trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
+        }
+}
+
+void trinity_dpm_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        trinity_cleanup_asic(rdev); /* ??? */
+
+        for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
+                kfree(rdev->pm.dpm.ps[i].ps_priv);
+        }
+        kfree(rdev->pm.dpm.ps);
+        kfree(rdev->pm.dpm.priv);
+}
+
+u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+        struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
+
+        if (low)
+                return requested_state->levels[0].sclk;
+        else
+                return requested_state->levels[requested_state->num_levels - 1].sclk;
+}
+
+u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
+{
+        struct trinity_power_info *pi = trinity_get_pi(rdev);
+
+        return pi->sys_info.bootup_uma_clk;
+}
diff --git a/drivers/gpu/drm/radeon/trinity_dpm.h b/drivers/gpu/drm/radeon/trinity_dpm.h
new file mode 100644
index 000000000000..c621b843aab5
--- /dev/null
+++ b/drivers/gpu/drm/radeon/trinity_dpm.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __TRINITY_DPM_H__
+#define __TRINITY_DPM_H__
+
+#include "sumo_dpm.h"
+
+#define TRINITY_SIZEOF_DPM_STATE_TABLE (SMU_SCLK_DPM_STATE_1_CNTL_0 - SMU_SCLK_DPM_STATE_0_CNTL_0)
+
+struct trinity_pl {
+        u32 sclk;
+        u8 vddc_index;
+        u8 ds_divider_index;
+        u8 ss_divider_index;
+        u8 allow_gnb_slow;
+        u8 force_nbp_state;
+        u8 display_wm;
+        u8 vce_wm;
+};
+
+#define TRINITY_POWERSTATE_FLAGS_NBPS_FORCEHIGH  (1 << 0)
+#define TRINITY_POWERSTATE_FLAGS_NBPS_LOCKTOHIGH (1 << 1)
+#define TRINITY_POWERSTATE_FLAGS_NBPS_LOCKTOLOW  (1 << 2)
+
+#define TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE    (1 << 0)
+
+struct trinity_ps {
+        u32 num_levels;
+        struct trinity_pl levels[SUMO_MAX_HARDWARE_POWERLEVELS];
+
+        u32 nbps_flags;
+        u32 bapm_flags;
+
+        u8 Dpm0PgNbPsLo;
+        u8 Dpm0PgNbPsHi;
+        u8 DpmXNbPsLo;
+        u8 DpmXNbPsHi;
+
+        u32 vclk_low_divider;
+        u32 vclk_high_divider;
+        u32 dclk_low_divider;
+        u32 dclk_high_divider;
+};
+
+#define TRINITY_NUM_NBPSTATES   4
+
+struct trinity_uvd_clock_table_entry
+{
+        u32 vclk;
+        u32 dclk;
+        u8 vclk_did;
+        u8 dclk_did;
+        u8 rsv[2];
+};
+
+struct trinity_sys_info {
+        u32 bootup_uma_clk;
+        u32 bootup_sclk;
+        u32 min_sclk;
+        u32 dentist_vco_freq;
+        u32 nb_dpm_enable;
+        u32 nbp_mclk[TRINITY_NUM_NBPSTATES];
+        u32 nbp_nclk[TRINITY_NUM_NBPSTATES];
+        u16 nbp_voltage_index[TRINITY_NUM_NBPSTATES];
+        u16 bootup_nb_voltage_index;
+        u8 htc_tmp_lmt;
+        u8 htc_hyst_lmt;
+        struct sumo_sclk_voltage_mapping_table sclk_voltage_mapping_table;
+        struct sumo_vid_mapping_table vid_mapping_table;
+        u32 uma_channel_number;
+        struct trinity_uvd_clock_table_entry uvd_clock_table_entries[4];
+};
+
+struct trinity_power_info {
+        u32 at[SUMO_MAX_HARDWARE_POWERLEVELS];
+        u32 dpm_interval;
+        u32 thermal_auto_throttling;
+        struct trinity_sys_info sys_info;
+        struct trinity_pl boot_pl;
+        u32 min_sclk_did;
+        bool enable_nbps_policy;
+        bool voltage_drop_in_dce;
+        bool override_dynamic_mgpg;
+        bool enable_gfx_clock_gating;
+        bool enable_gfx_power_gating;
+        bool enable_mg_clock_gating;
+        bool enable_gfx_dynamic_mgpg;
+        bool enable_auto_thermal_throttling;
+        bool enable_dpm;
+        bool enable_sclk_ds;
+        bool uvd_dpm;
+        struct radeon_ps current_rps;
+        struct trinity_ps current_ps;
+        struct radeon_ps requested_rps;
+        struct trinity_ps requested_ps;
+};
+
+#define TRINITY_AT_DFLT            30
+
+/* trinity_smc.c */
+int trinity_dpm_config(struct radeon_device *rdev, bool enable);
+int trinity_uvd_dpm_config(struct radeon_device *rdev);
+int trinity_dpm_force_state(struct radeon_device *rdev, u32 n);
+int trinity_dpm_no_forced_level(struct radeon_device *rdev);
+int trinity_dce_enable_voltage_adjustment(struct radeon_device *rdev,
+                                          bool enable);
+int trinity_gfx_dynamic_mgpg_config(struct radeon_device *rdev);
+void trinity_acquire_mutex(struct radeon_device *rdev);
+void trinity_release_mutex(struct radeon_device *rdev);
+
+#endif
diff --git a/drivers/gpu/drm/radeon/trinity_smc.c b/drivers/gpu/drm/radeon/trinity_smc.c
new file mode 100644
index 000000000000..85f86a29513c
--- /dev/null
+++ b/drivers/gpu/drm/radeon/trinity_smc.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "radeon.h"
+#include "trinityd.h"
+#include "trinity_dpm.h"
+#include "ppsmc.h"
+
+struct trinity_ps *trinity_get_ps(struct radeon_ps *rps);
+struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev);
+
+static int trinity_notify_message_to_smu(struct radeon_device *rdev, u32 id)
+{
+        int i;
+        u32 v = 0;
+
+        WREG32(SMC_MESSAGE_0, id);
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(SMC_RESP_0) != 0)
+                        break;
+                udelay(1);
+        }
+        v = RREG32(SMC_RESP_0);
+
+        if (v != 1) {
+                if (v == 0xFF) {
+                        DRM_ERROR("SMC failed to handle the message!\n");
+                        return -EINVAL;
+                } else if (v == 0xFE) {
+                        DRM_ERROR("Unknown SMC message!\n");
+                        return -EINVAL;
+                }
+        }
+
+        return 0;
+}
+
+int trinity_dpm_config(struct radeon_device *rdev, bool enable)
+{
+        if (enable)
+                WREG32_SMC(SMU_SCRATCH0, 1);
+        else
+                WREG32_SMC(SMU_SCRATCH0, 0);
+
+        return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DPM_Config);
+}
+
+int trinity_dpm_force_state(struct radeon_device *rdev, u32 n)
+{
+        WREG32_SMC(SMU_SCRATCH0, n);
+
+        return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DPM_ForceState);
+}
+
+int trinity_uvd_dpm_config(struct radeon_device *rdev)
+{
+        return trinity_notify_message_to_smu(rdev, PPSMC_MSG_UVD_DPM_Config);
+}
+
+int trinity_dpm_no_forced_level(struct radeon_device *rdev)
+{
+        return trinity_notify_message_to_smu(rdev, PPSMC_MSG_NoForcedLevel);
+}
+
+int trinity_dce_enable_voltage_adjustment(struct radeon_device *rdev,
+                                          bool enable)
+{
+        if (enable)
+                return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DCE_AllowVoltageAdjustment);
+        else
+                return trinity_notify_message_to_smu(rdev, PPSMC_MSG_DCE_RemoveVoltageAdjustment);
+}
+
+int trinity_gfx_dynamic_mgpg_config(struct radeon_device *rdev)
+{
+        return trinity_notify_message_to_smu(rdev, PPSMC_MSG_PG_SIMD_Config);
+}
+
+void trinity_acquire_mutex(struct radeon_device *rdev)
+{
+        int i;
+
+        WREG32(SMC_INT_REQ, 1);
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if ((RREG32(SMC_INT_REQ) & 0xffff) == 1)
+                        break;
+                udelay(1);
+        }
+}
+
+void trinity_release_mutex(struct radeon_device *rdev)
+{
+        WREG32(SMC_INT_REQ, 0);
+}
diff --git a/drivers/gpu/drm/radeon/trinityd.h b/drivers/gpu/drm/radeon/trinityd.h
new file mode 100644
index 000000000000..fd32e2771755
--- /dev/null
+++ b/drivers/gpu/drm/radeon/trinityd.h
@@ -0,0 +1,228 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef _TRINITYD_H_
+#define _TRINITYD_H_
+
+/* pm registers */
+
+/* cg */
+#define CG_CGTT_LOCAL_0                                 0x0
+#define CG_CGTT_LOCAL_1                                 0x1
+
+/* smc */
+#define SMU_SCLK_DPM_STATE_0_CNTL_0                     0x1f000
+#       define STATE_VALID(x)                           ((x) << 0)
+#       define STATE_VALID_MASK                         (0xff << 0)
+#       define STATE_VALID_SHIFT                        0
+#       define CLK_DIVIDER(x)                           ((x) << 8)
+#       define CLK_DIVIDER_MASK                         (0xff << 8)
+#       define CLK_DIVIDER_SHIFT                        8
+#       define VID(x)                                   ((x) << 16)
+#       define VID_MASK                                 (0xff << 16)
+#       define VID_SHIFT                                16
+#       define LVRT(x)                                  ((x) << 24)
+#       define LVRT_MASK                                (0xff << 24)
+#       define LVRT_SHIFT                               24
+#define SMU_SCLK_DPM_STATE_0_CNTL_1                     0x1f004
+#       define DS_DIV(x)                                ((x) << 0)
+#       define DS_DIV_MASK                              (0xff << 0)
+#       define DS_DIV_SHIFT                             0
+#       define DS_SH_DIV(x)                             ((x) << 8)
+#       define DS_SH_DIV_MASK                           (0xff << 8)
+#       define DS_SH_DIV_SHIFT                          8
+#       define DISPLAY_WM(x)                            ((x) << 16)
+#       define DISPLAY_WM_MASK                          (0xff << 16)
+#       define DISPLAY_WM_SHIFT                         16
+#       define VCE_WM(x)                                ((x) << 24)
+#       define VCE_WM_MASK                              (0xff << 24)
+#       define VCE_WM_SHIFT                             24
+
+#define SMU_SCLK_DPM_STATE_0_CNTL_3                     0x1f00c
+#       define GNB_SLOW(x)                              ((x) << 0)
+#       define GNB_SLOW_MASK                            (0xff << 0)
+#       define GNB_SLOW_SHIFT                           0
+#       define FORCE_NBPS1(x)                           ((x) << 8)
+#       define FORCE_NBPS1_MASK                         (0xff << 8)
+#       define FORCE_NBPS1_SHIFT                        8
+#define SMU_SCLK_DPM_STATE_0_AT                         0x1f010
+#       define AT(x)                                    ((x) << 0)
+#       define AT_MASK                                  (0xff << 0)
+#       define AT_SHIFT                                 0
+
+#define SMU_SCLK_DPM_STATE_0_PG_CNTL                    0x1f014
+#       define PD_SCLK_DIVIDER(x)                       ((x) << 16)
+#       define PD_SCLK_DIVIDER_MASK                     (0xff << 16)
+#       define PD_SCLK_DIVIDER_SHIFT                    16
+
+#define SMU_SCLK_DPM_STATE_1_CNTL_0                     0x1f020
+
+#define SMU_SCLK_DPM_CNTL                               0x1f100
+#       define SCLK_DPM_EN(x)                           ((x) << 0)
+#       define SCLK_DPM_EN_MASK                         (0xff << 0)
+#       define SCLK_DPM_EN_SHIFT                        0
+#       define SCLK_DPM_BOOT_STATE(x)                   ((x) << 16)
+#       define SCLK_DPM_BOOT_STATE_MASK                 (0xff << 16)
+#       define SCLK_DPM_BOOT_STATE_SHIFT                16
+#       define VOLTAGE_CHG_EN(x)                        ((x) << 24)
+#       define VOLTAGE_CHG_EN_MASK                      (0xff << 24)
+#       define VOLTAGE_CHG_EN_SHIFT                     24
+
+#define SMU_SCLK_DPM_TT_CNTL                            0x1f108
+#       define SCLK_TT_EN(x)                            ((x) << 0)
+#       define SCLK_TT_EN_MASK                          (0xff << 0)
+#       define SCLK_TT_EN_SHIFT                         0
+#define SMU_SCLK_DPM_TTT                                0x1f10c
+#       define LT(x)                                    ((x) << 0)
+#       define LT_MASK                                  (0xffff << 0)
+#       define LT_SHIFT                                 0
+#       define HT(x)                                    ((x) << 16)
+#       define HT_MASK                                  (0xffff << 16)
+#       define HT_SHIFT                                 16
+
+#define SMU_UVD_DPM_STATES                              0x1f1a0
+#define SMU_UVD_DPM_CNTL                                0x1f1a4
+
+#define SMU_S_PG_CNTL                                   0x1f118
+#       define DS_PG_EN(x)                              ((x) << 16)
+#       define DS_PG_EN_MASK                            (0xff << 16)
+#       define DS_PG_EN_SHIFT                           16
+
+#define GFX_POWER_GATING_CNTL                           0x1f38c
+#       define PDS_DIV(x)                               ((x) << 0)
+#       define PDS_DIV_MASK                             (0xff << 0)
+#       define PDS_DIV_SHIFT                            0
+#       define SSSD(x)                                  ((x) << 8)
+#       define SSSD_MASK                                (0xff << 8)
+#       define SSSD_SHIFT                               8
+
+#define PM_CONFIG                                       0x1f428
+#       define SVI_Mode                                 (1 << 29)
+
+#define PM_I_CNTL_1                                     0x1f464
+#       define SCLK_DPM(x)                              ((x) << 0)
+#       define SCLK_DPM_MASK                            (0xff << 0)
+#       define SCLK_DPM_SHIFT                           0
+#       define DS_PG_CNTL(x)                            ((x) << 16)
+#       define DS_PG_CNTL_MASK                          (0xff << 16)
+#       define DS_PG_CNTL_SHIFT                         16
+#define PM_TP                                           0x1f468
+
+#define NB_PSTATE_CONFIG                                0x1f5f8
+#       define Dpm0PgNbPsLo(x)                          ((x) << 0)
+#       define Dpm0PgNbPsLo_MASK                        (3 << 0)
+#       define Dpm0PgNbPsLo_SHIFT                       0
+#       define Dpm0PgNbPsHi(x)                          ((x) << 2)
+#       define Dpm0PgNbPsHi_MASK                        (3 << 2)
+#       define Dpm0PgNbPsHi_SHIFT                       2
+#       define DpmXNbPsLo(x)                            ((x) << 4)
+#       define DpmXNbPsLo_MASK                          (3 << 4)
+#       define DpmXNbPsLo_SHIFT                         4
+#       define DpmXNbPsHi(x)                            ((x) << 6)
+#       define DpmXNbPsHi_MASK                          (3 << 6)
+#       define DpmXNbPsHi_SHIFT                         6
+
+#define DC_CAC_VALUE                                    0x1f908
+
+#define GPU_CAC_AVRG_CNTL                               0x1f920
+#       define WINDOW_SIZE(x)                           ((x) << 0)
+#       define WINDOW_SIZE_MASK                         (0xff << 0)
+#       define WINDOW_SIZE_SHIFT                        0
+
+#define CC_SMU_MISC_FUSES                               0xe0001004
+#       define MinSClkDid(x)                   ((x) << 2)
+#       define MinSClkDid_MASK                 (0x7f << 2)
+#       define MinSClkDid_SHIFT                2
+
+#define CC_SMU_TST_EFUSE1_MISC                          0xe000101c
+#       define RB_BACKEND_DISABLE(x)                    ((x) << 16)
+#       define RB_BACKEND_DISABLE_MASK                  (3 << 16)
+#       define RB_BACKEND_DISABLE_SHIFT                 16
+
+#define SMU_SCRATCH_A                                   0xe0003024
+
+#define SMU_SCRATCH0                                    0xe0003040
+
+/* mmio */
+#define SMC_INT_REQ                                     0x220
+
+#define SMC_MESSAGE_0                                   0x22c
+#define SMC_RESP_0                                      0x230
+
+#define GENERAL_PWRMGT                                  0x670
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+
+#define SCLK_PWRMGT_CNTL                                0x678
+#       define DYN_PWR_DOWN_EN                          (1 << 2)
+#       define RESET_BUSY_CNT                           (1 << 4)
+#       define RESET_SCLK_CNT                           (1 << 5)
+#       define DYN_GFX_CLK_OFF_EN                       (1 << 7)
+#       define GFX_CLK_FORCE_ON                         (1 << 8)
+#       define DYNAMIC_PM_EN                            (1 << 21)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                0x684
+#       define TARGET_STATE(x)                          ((x) << 0)
+#       define TARGET_STATE_MASK                        (0xf << 0)
+#       define TARGET_STATE_SHIFT                       0
+#       define CURRENT_STATE(x)                         ((x) << 4)
+#       define CURRENT_STATE_MASK                       (0xf << 4)
+#       define CURRENT_STATE_SHIFT                      4
+
+#define CG_GIPOTS                                       0x6d8
+#       define CG_GIPOT(x)                              ((x) << 16)
+#       define CG_GIPOT_MASK                            (0xffff << 16)
+#       define CG_GIPOT_SHIFT                           16
+
+#define CG_PG_CTRL                                      0x6e0
+#       define SP(x)                                    ((x) << 0)
+#       define SP_MASK                                  (0xffff << 0)
+#       define SP_SHIFT                                 0
+#       define SU(x)                                    ((x) << 16)
+#       define SU_MASK                                  (0xffff << 16)
+#       define SU_SHIFT                                 16
+
+#define CG_MISC_REG                                     0x708
+
+#define CG_THERMAL_INT_CTRL                             0x738
+#       define DIG_THERM_INTH(x)                        ((x) << 0)
+#       define DIG_THERM_INTH_MASK                      (0xff << 0)
+#       define DIG_THERM_INTH_SHIFT                     0
+#       define DIG_THERM_INTL(x)                        ((x) << 8)
+#       define DIG_THERM_INTL_MASK                      (0xff << 8)
+#       define DIG_THERM_INTL_SHIFT                     8
+#       define THERM_INTH_MASK                          (1 << 24)
+#       define THERM_INTL_MASK                          (1 << 25)
+
+#define CG_CG_VOLTAGE_CNTL                              0x770
+#       define EN                                       (1 << 9)
+
+#define HW_REV                                          0x5564
+#       define ATI_REV_ID_MASK                          (0xf << 28)
+#       define ATI_REV_ID_SHIFT                         28
+/* 0 = A0, 1 = A1, 2 = B0, 3 = C0, etc. */
+
+#define CGTS_SM_CTRL_REG                                0x9150
+
+#define GB_ADDR_CONFIG                                  0x98f8
+
+#endif
diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig
new file mode 100644
index 000000000000..72887df8dd76
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/Kconfig
@@ -0,0 +1,9 @@
+config DRM_RCAR_DU
+        tristate "DRM Support for R-Car Display Unit"
+        depends on DRM && ARM
+        select DRM_KMS_HELPER
+        select DRM_KMS_CMA_HELPER
+        select DRM_GEM_CMA_HELPER
+        help
+          Choose this option if you have an R-Car chipset.
+          If M is selected the module will be called rcar-du-drm.
diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile
new file mode 100644
index 000000000000..7333c0094015
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/Makefile
@@ -0,0 +1,8 @@
+rcar-du-drm-y := rcar_du_crtc.o \
+                 rcar_du_drv.o \
+                 rcar_du_kms.o \
+                 rcar_du_lvds.o \
+                 rcar_du_plane.o \
+                 rcar_du_vga.o
+
+obj-$(CONFIG_DRM_RCAR_DU)       += rcar-du-drm.o
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
new file mode 100644
index 000000000000..24183fb93592
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -0,0 +1,595 @@
+/*
+ * rcar_du_crtc.c  --  R-Car Display Unit CRTCs
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/mutex.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include "rcar_du_crtc.h"
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+#include "rcar_du_lvds.h"
+#include "rcar_du_plane.h"
+#include "rcar_du_regs.h"
+#include "rcar_du_vga.h"
+
+#define to_rcar_crtc(c) container_of(c, struct rcar_du_crtc, crtc)
+
+static u32 rcar_du_crtc_read(struct rcar_du_crtc *rcrtc, u32 reg)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+
+        return rcar_du_read(rcdu, rcrtc->mmio_offset + reg);
+}
+
+static void rcar_du_crtc_write(struct rcar_du_crtc *rcrtc, u32 reg, u32 data)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+
+        rcar_du_write(rcdu, rcrtc->mmio_offset + reg, data);
+}
+
+static void rcar_du_crtc_clr(struct rcar_du_crtc *rcrtc, u32 reg, u32 clr)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+
+        rcar_du_write(rcdu, rcrtc->mmio_offset + reg,
+                      rcar_du_read(rcdu, rcrtc->mmio_offset + reg) & ~clr);
+}
+
+static void rcar_du_crtc_set(struct rcar_du_crtc *rcrtc, u32 reg, u32 set)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+
+        rcar_du_write(rcdu, rcrtc->mmio_offset + reg,
+                      rcar_du_read(rcdu, rcrtc->mmio_offset + reg) | set);
+}
+
+static void rcar_du_crtc_clr_set(struct rcar_du_crtc *rcrtc, u32 reg,
+                                 u32 clr, u32 set)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+        u32 value = rcar_du_read(rcdu, rcrtc->mmio_offset + reg);
+
+        rcar_du_write(rcdu, rcrtc->mmio_offset + reg, (value & ~clr) | set);
+}
+
+static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
+{
+        struct drm_crtc *crtc = &rcrtc->crtc;
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+        const struct drm_display_mode *mode = &crtc->mode;
+        unsigned long clk;
+        u32 value;
+        u32 div;
+
+        /* Dot clock */
+        clk = clk_get_rate(rcdu->clock);
+        div = DIV_ROUND_CLOSEST(clk, mode->clock * 1000);
+        div = clamp(div, 1U, 64U) - 1;
+
+        rcar_du_write(rcdu, rcrtc->index ? ESCR2 : ESCR,
+                      ESCR_DCLKSEL_CLKS | div);
+        rcar_du_write(rcdu, rcrtc->index ? OTAR2 : OTAR, 0);
+
+        /* Signal polarities */
+        value = ((mode->flags & DRM_MODE_FLAG_PVSYNC) ? 0 : DSMR_VSL)
+              | ((mode->flags & DRM_MODE_FLAG_PHSYNC) ? 0 : DSMR_HSL)
+              | DSMR_DIPM_DE;
+        rcar_du_crtc_write(rcrtc, DSMR, value);
+
+        /* Display timings */
+        rcar_du_crtc_write(rcrtc, HDSR, mode->htotal - mode->hsync_start - 19);
+        rcar_du_crtc_write(rcrtc, HDER, mode->htotal - mode->hsync_start +
+                                        mode->hdisplay - 19);
+        rcar_du_crtc_write(rcrtc, HSWR, mode->hsync_end -
+                                        mode->hsync_start - 1);
+        rcar_du_crtc_write(rcrtc, HCR,  mode->htotal - 1);
+
+        rcar_du_crtc_write(rcrtc, VDSR, mode->vtotal - mode->vsync_end - 2);
+        rcar_du_crtc_write(rcrtc, VDER, mode->vtotal - mode->vsync_end +
+                                        mode->vdisplay - 2);
+        rcar_du_crtc_write(rcrtc, VSPR, mode->vtotal - mode->vsync_end +
+                                        mode->vsync_start - 1);
+        rcar_du_crtc_write(rcrtc, VCR,  mode->vtotal - 1);
+
+        rcar_du_crtc_write(rcrtc, DESR,  mode->htotal - mode->hsync_start);
+        rcar_du_crtc_write(rcrtc, DEWR,  mode->hdisplay);
+}
+
+static void rcar_du_crtc_set_routing(struct rcar_du_crtc *rcrtc)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+        u32 dorcr = rcar_du_read(rcdu, DORCR);
+
+        dorcr &= ~(DORCR_PG2T | DORCR_DK2S | DORCR_PG2D_MASK);
+
+        /* Set the DU1 pins sources. Select CRTC 0 if explicitly requested and
+         * CRTC 1 in all other cases to avoid cloning CRTC 0 to DU0 and DU1 by
+         * default.
+         */
+        if (rcrtc->outputs & (1 << 1) && rcrtc->index == 0)
+                dorcr |= DORCR_PG2D_DS1;
+        else
+                dorcr |= DORCR_PG2T | DORCR_DK2S | DORCR_PG2D_DS2;
+
+        rcar_du_write(rcdu, DORCR, dorcr);
+}
+
+static void __rcar_du_start_stop(struct rcar_du_device *rcdu, bool start)
+{
+        rcar_du_write(rcdu, DSYSR,
+                      (rcar_du_read(rcdu, DSYSR) & ~(DSYSR_DRES | DSYSR_DEN)) |
+                      (start ? DSYSR_DEN : DSYSR_DRES));
+}
+
+static void rcar_du_start_stop(struct rcar_du_device *rcdu, bool start)
+{
+        /* Many of the configuration bits are only updated when the display
+         * reset (DRES) bit in DSYSR is set to 1, disabling *both* CRTCs. Some
+         * of those bits could be pre-configured, but others (especially the
+         * bits related to plane assignment to display timing controllers) need
+         * to be modified at runtime.
+         *
+         * Restart the display controller if a start is requested. Sorry for the
+         * flicker. It should be possible to move most of the "DRES-update" bits
+         * setup to driver initialization time and minimize the number of cases
+         * when the display controller will have to be restarted.
+         */
+        if (start) {
+                if (rcdu->used_crtcs++ != 0)
+                        __rcar_du_start_stop(rcdu, false);
+                __rcar_du_start_stop(rcdu, true);
+        } else {
+                if (--rcdu->used_crtcs == 0)
+                        __rcar_du_start_stop(rcdu, false);
+        }
+}
+
+void rcar_du_crtc_route_output(struct drm_crtc *crtc, unsigned int output)
+{
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+
+        /* Store the route from the CRTC output to the DU output. The DU will be
+         * configured when starting the CRTC.
+         */
+        rcrtc->outputs |= 1 << output;
+}
+
+void rcar_du_crtc_update_planes(struct drm_crtc *crtc)
+{
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+        struct rcar_du_plane *planes[RCAR_DU_NUM_HW_PLANES];
+        unsigned int num_planes = 0;
+        unsigned int prio = 0;
+        unsigned int i;
+        u32 dptsr = 0;
+        u32 dspr = 0;
+
+        for (i = 0; i < ARRAY_SIZE(rcdu->planes.planes); ++i) {
+                struct rcar_du_plane *plane = &rcdu->planes.planes[i];
+                unsigned int j;
+
+                if (plane->crtc != &rcrtc->crtc || !plane->enabled)
+                        continue;
+
+                /* Insert the plane in the sorted planes array. */
+                for (j = num_planes++; j > 0; --j) {
+                        if (planes[j-1]->zpos <= plane->zpos)
+                                break;
+                        planes[j] = planes[j-1];
+                }
+
+                planes[j] = plane;
+                prio += plane->format->planes * 4;
+        }
+
+        for (i = 0; i < num_planes; ++i) {
+                struct rcar_du_plane *plane = planes[i];
+                unsigned int index = plane->hwindex;
+
+                prio -= 4;
+                dspr |= (index + 1) << prio;
+                dptsr |= DPTSR_PnDK(index) |  DPTSR_PnTS(index);
+
+                if (plane->format->planes == 2) {
+                        index = (index + 1) % 8;
+
+                        prio -= 4;
+                        dspr |= (index + 1) << prio;
+                        dptsr |= DPTSR_PnDK(index) |  DPTSR_PnTS(index);
+                }
+        }
+
+        /* Select display timing and dot clock generator 2 for planes associated
+         * with superposition controller 2.
+         */
+        if (rcrtc->index) {
+                u32 value = rcar_du_read(rcdu, DPTSR);
+
+                /* The DPTSR register is updated when the display controller is
+                 * stopped. We thus need to restart the DU. Once again, sorry
+                 * for the flicker. One way to mitigate the issue would be to
+                 * pre-associate planes with CRTCs (either with a fixed 4/4
+                 * split, or through a module parameter). Flicker would then
+                 * occur only if we need to break the pre-association.
+                 */
+                if (value != dptsr) {
+                        rcar_du_write(rcdu, DPTSR, dptsr);
+                        if (rcdu->used_crtcs) {
+                                __rcar_du_start_stop(rcdu, false);
+                                __rcar_du_start_stop(rcdu, true);
+                        }
+                }
+        }
+
+        rcar_du_write(rcdu, rcrtc->index ? DS2PR : DS1PR, dspr);
+}
+
+static void rcar_du_crtc_start(struct rcar_du_crtc *rcrtc)
+{
+        struct drm_crtc *crtc = &rcrtc->crtc;
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+        unsigned int i;
+
+        if (rcrtc->started)
+                return;
+
+        if (WARN_ON(rcrtc->plane->format == NULL))
+                return;
+
+        /* Set display off and background to black */
+        rcar_du_crtc_write(rcrtc, DOOR, DOOR_RGB(0, 0, 0));
+        rcar_du_crtc_write(rcrtc, BPOR, BPOR_RGB(0, 0, 0));
+
+        /* Configure display timings and output routing */
+        rcar_du_crtc_set_display_timing(rcrtc);
+        rcar_du_crtc_set_routing(rcrtc);
+
+        mutex_lock(&rcdu->planes.lock);
+        rcrtc->plane->enabled = true;
+        rcar_du_crtc_update_planes(crtc);
+        mutex_unlock(&rcdu->planes.lock);
+
+        /* Setup planes. */
+        for (i = 0; i < ARRAY_SIZE(rcdu->planes.planes); ++i) {
+                struct rcar_du_plane *plane = &rcdu->planes.planes[i];
+
+                if (plane->crtc != crtc || !plane->enabled)
+                        continue;
+
+                rcar_du_plane_setup(plane);
+        }
+
+        /* Select master sync mode. This enables display operation in master
+         * sync mode (with the HSYNC and VSYNC signals configured as outputs and
+         * actively driven).
+         */
+        rcar_du_crtc_clr_set(rcrtc, DSYSR, DSYSR_TVM_MASK, DSYSR_TVM_MASTER);
+
+        rcar_du_start_stop(rcdu, true);
+
+        rcrtc->started = true;
+}
+
+static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
+{
+        struct drm_crtc *crtc = &rcrtc->crtc;
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+
+        if (!rcrtc->started)
+                return;
+
+        mutex_lock(&rcdu->planes.lock);
+        rcrtc->plane->enabled = false;
+        rcar_du_crtc_update_planes(crtc);
+        mutex_unlock(&rcdu->planes.lock);
+
+        /* Select switch sync mode. This stops display operation and configures
+         * the HSYNC and VSYNC signals as inputs.
+         */
+        rcar_du_crtc_clr_set(rcrtc, DSYSR, DSYSR_TVM_MASK, DSYSR_TVM_SWITCH);
+
+        rcar_du_start_stop(rcdu, false);
+
+        rcrtc->started = false;
+}
+
+void rcar_du_crtc_suspend(struct rcar_du_crtc *rcrtc)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+
+        rcar_du_crtc_stop(rcrtc);
+        rcar_du_put(rcdu);
+}
+
+void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc)
+{
+        struct rcar_du_device *rcdu = rcrtc->crtc.dev->dev_private;
+
+        if (rcrtc->dpms != DRM_MODE_DPMS_ON)
+                return;
+
+        rcar_du_get(rcdu);
+        rcar_du_crtc_start(rcrtc);
+}
+
+static void rcar_du_crtc_update_base(struct rcar_du_crtc *rcrtc)
+{
+        struct drm_crtc *crtc = &rcrtc->crtc;
+
+        rcar_du_plane_compute_base(rcrtc->plane, crtc->fb);
+        rcar_du_plane_update_base(rcrtc->plane);
+}
+
+static void rcar_du_crtc_dpms(struct drm_crtc *crtc, int mode)
+{
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+
+        if (rcrtc->dpms == mode)
+                return;
+
+        if (mode == DRM_MODE_DPMS_ON) {
+                rcar_du_get(rcdu);
+                rcar_du_crtc_start(rcrtc);
+        } else {
+                rcar_du_crtc_stop(rcrtc);
+                rcar_du_put(rcdu);
+        }
+
+        rcrtc->dpms = mode;
+}
+
+static bool rcar_du_crtc_mode_fixup(struct drm_crtc *crtc,
+                                    const struct drm_display_mode *mode,
+                                    struct drm_display_mode *adjusted_mode)
+{
+        /* TODO Fixup modes */
+        return true;
+}
+
+static void rcar_du_crtc_mode_prepare(struct drm_crtc *crtc)
+{
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+
+        /* We need to access the hardware during mode set, acquire a reference
+         * to the DU.
+         */
+        rcar_du_get(rcdu);
+
+        /* Stop the CRTC and release the plane. Force the DPMS mode to off as a
+         * result.
+         */
+        rcar_du_crtc_stop(rcrtc);
+        rcar_du_plane_release(rcrtc->plane);
+
+        rcrtc->dpms = DRM_MODE_DPMS_OFF;
+}
+
+static int rcar_du_crtc_mode_set(struct drm_crtc *crtc,
+                                 struct drm_display_mode *mode,
+                                 struct drm_display_mode *adjusted_mode,
+                                 int x, int y,
+                                 struct drm_framebuffer *old_fb)
+{
+        struct rcar_du_device *rcdu = crtc->dev->dev_private;
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+        const struct rcar_du_format_info *format;
+        int ret;
+
+        format = rcar_du_format_info(crtc->fb->pixel_format);
+        if (format == NULL) {
+                dev_dbg(rcdu->dev, "mode_set: unsupported format %08x\n",
+                        crtc->fb->pixel_format);
+                ret = -EINVAL;
+                goto error;
+        }
+
+        ret = rcar_du_plane_reserve(rcrtc->plane, format);
+        if (ret < 0)
+                goto error;
+
+        rcrtc->plane->format = format;
+        rcrtc->plane->pitch = crtc->fb->pitches[0];
+
+        rcrtc->plane->src_x = x;
+        rcrtc->plane->src_y = y;
+        rcrtc->plane->width = mode->hdisplay;
+        rcrtc->plane->height = mode->vdisplay;
+
+        rcar_du_plane_compute_base(rcrtc->plane, crtc->fb);
+
+        rcrtc->outputs = 0;
+
+        return 0;
+
+error:
+        /* There's no rollback/abort operation to clean up in case of error. We
+         * thus need to release the reference to the DU acquired in prepare()
+         * here.
+         */
+        rcar_du_put(rcdu);
+        return ret;
+}
+
+static void rcar_du_crtc_mode_commit(struct drm_crtc *crtc)
+{
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+
+        /* We're done, restart the CRTC and set the DPMS mode to on. The
+         * reference to the DU acquired at prepare() time will thus be released
+         * by the DPMS handler (possibly called by the disable() handler).
+         */
+        rcar_du_crtc_start(rcrtc);
+        rcrtc->dpms = DRM_MODE_DPMS_ON;
+}
+
+static int rcar_du_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
+                                      struct drm_framebuffer *old_fb)
+{
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+
+        rcrtc->plane->src_x = x;
+        rcrtc->plane->src_y = y;
+
+        rcar_du_crtc_update_base(to_rcar_crtc(crtc));
+
+        return 0;
+}
+
+static void rcar_du_crtc_disable(struct drm_crtc *crtc)
+{
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+
+        rcar_du_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+        rcar_du_plane_release(rcrtc->plane);
+}
+
+static const struct drm_crtc_helper_funcs crtc_helper_funcs = {
+        .dpms = rcar_du_crtc_dpms,
+        .mode_fixup = rcar_du_crtc_mode_fixup,
+        .prepare = rcar_du_crtc_mode_prepare,
+        .commit = rcar_du_crtc_mode_commit,
+        .mode_set = rcar_du_crtc_mode_set,
+        .mode_set_base = rcar_du_crtc_mode_set_base,
+        .disable = rcar_du_crtc_disable,
+};
+
+void rcar_du_crtc_cancel_page_flip(struct rcar_du_crtc *rcrtc,
+                                   struct drm_file *file)
+{
+        struct drm_pending_vblank_event *event;
+        struct drm_device *dev = rcrtc->crtc.dev;
+        unsigned long flags;
+
+        /* Destroy the pending vertical blanking event associated with the
+         * pending page flip, if any, and disable vertical blanking interrupts.
+         */
+        spin_lock_irqsave(&dev->event_lock, flags);
+        event = rcrtc->event;
+        if (event && event->base.file_priv == file) {
+                rcrtc->event = NULL;
+                event->base.destroy(&event->base);
+                drm_vblank_put(dev, rcrtc->index);
+        }
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+static void rcar_du_crtc_finish_page_flip(struct rcar_du_crtc *rcrtc)
+{
+        struct drm_pending_vblank_event *event;
+        struct drm_device *dev = rcrtc->crtc.dev;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        event = rcrtc->event;
+        rcrtc->event = NULL;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        if (event == NULL)
+                return;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        drm_send_vblank_event(dev, rcrtc->index, event);
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        drm_vblank_put(dev, rcrtc->index);
+}
+
+static int rcar_du_crtc_page_flip(struct drm_crtc *crtc,
+                                  struct drm_framebuffer *fb,
+                                  struct drm_pending_vblank_event *event)
+{
+        struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
+        struct drm_device *dev = rcrtc->crtc.dev;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        if (rcrtc->event != NULL) {
+                spin_unlock_irqrestore(&dev->event_lock, flags);
+                return -EBUSY;
+        }
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        crtc->fb = fb;
+        rcar_du_crtc_update_base(rcrtc);
+
+        if (event) {
+                event->pipe = rcrtc->index;
+                drm_vblank_get(dev, rcrtc->index);
+                spin_lock_irqsave(&dev->event_lock, flags);
+                rcrtc->event = event;
+                spin_unlock_irqrestore(&dev->event_lock, flags);
+        }
+
+        return 0;
+}
+
+static const struct drm_crtc_funcs crtc_funcs = {
+        .destroy = drm_crtc_cleanup,
+        .set_config = drm_crtc_helper_set_config,
+        .page_flip = rcar_du_crtc_page_flip,
+};
+
+int rcar_du_crtc_create(struct rcar_du_device *rcdu, unsigned int index)
+{
+        struct rcar_du_crtc *rcrtc = &rcdu->crtcs[index];
+        struct drm_crtc *crtc = &rcrtc->crtc;
+        int ret;
+
+        rcrtc->mmio_offset = index ? DISP2_REG_OFFSET : 0;
+        rcrtc->index = index;
+        rcrtc->dpms = DRM_MODE_DPMS_OFF;
+        rcrtc->plane = &rcdu->planes.planes[index];
+
+        rcrtc->plane->crtc = crtc;
+
+        ret = drm_crtc_init(rcdu->ddev, crtc, &crtc_funcs);
+        if (ret < 0)
+                return ret;
+
+        drm_crtc_helper_add(crtc, &crtc_helper_funcs);
+
+        return 0;
+}
+
+void rcar_du_crtc_enable_vblank(struct rcar_du_crtc *rcrtc, bool enable)
+{
+        if (enable) {
+                rcar_du_crtc_write(rcrtc, DSRCR, DSRCR_VBCL);
+                rcar_du_crtc_set(rcrtc, DIER, DIER_VBE);
+        } else {
+                rcar_du_crtc_clr(rcrtc, DIER, DIER_VBE);
+        }
+}
+
+void rcar_du_crtc_irq(struct rcar_du_crtc *rcrtc)
+{
+        u32 status;
+
+        status = rcar_du_crtc_read(rcrtc, DSSR);
+        rcar_du_crtc_write(rcrtc, DSRCR, status & DSRCR_MASK);
+
+        if (status & DSSR_VBK) {
+                drm_handle_vblank(rcrtc->crtc.dev, rcrtc->index);
+                rcar_du_crtc_finish_page_flip(rcrtc);
+        }
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
new file mode 100644
index 000000000000..2a0365bcbd14
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
@@ -0,0 +1,50 @@
+/*
+ * rcar_du_crtc.h  --  R-Car Display Unit CRTCs
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_CRTC_H__
+#define __RCAR_DU_CRTC_H__
+
+#include <linux/mutex.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+
+struct rcar_du_device;
+struct rcar_du_plane;
+
+struct rcar_du_crtc {
+        struct drm_crtc crtc;
+
+        unsigned int mmio_offset;
+        unsigned int index;
+        bool started;
+
+        struct drm_pending_vblank_event *event;
+        unsigned int outputs;
+        int dpms;
+
+        struct rcar_du_plane *plane;
+};
+
+int rcar_du_crtc_create(struct rcar_du_device *rcdu, unsigned int index);
+void rcar_du_crtc_enable_vblank(struct rcar_du_crtc *rcrtc, bool enable);
+void rcar_du_crtc_irq(struct rcar_du_crtc *rcrtc);
+void rcar_du_crtc_cancel_page_flip(struct rcar_du_crtc *rcrtc,
+                                   struct drm_file *file);
+void rcar_du_crtc_suspend(struct rcar_du_crtc *rcrtc);
+void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc);
+
+void rcar_du_crtc_route_output(struct drm_crtc *crtc, unsigned int output);
+void rcar_du_crtc_update_planes(struct drm_crtc *crtc);
+
+#endif /* __RCAR_DU_CRTC_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
new file mode 100644
index 000000000000..003b34ee38e3
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -0,0 +1,325 @@
+/*
+ * rcar_du_drv.c  --  R-Car Display Unit DRM driver
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/slab.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include "rcar_du_crtc.h"
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+#include "rcar_du_regs.h"
+
+/* -----------------------------------------------------------------------------
+ * Core device operations
+ */
+
+/*
+ * rcar_du_get - Acquire a reference to the DU
+ *
+ * Acquiring a reference enables the device clock and setup core registers. A
+ * reference must be held before accessing any hardware registers.
+ *
+ * This function must be called with the DRM mode_config lock held.
+ *
+ * Return 0 in case of success or a negative error code otherwise.
+ */
+int rcar_du_get(struct rcar_du_device *rcdu)
+{
+        int ret;
+
+        if (rcdu->use_count)
+                goto done;
+
+        /* Enable clocks before accessing the hardware. */
+        ret = clk_prepare_enable(rcdu->clock);
+        if (ret < 0)
+                return ret;
+
+        /* Enable extended features */
+        rcar_du_write(rcdu, DEFR, DEFR_CODE | DEFR_DEFE);
+        rcar_du_write(rcdu, DEFR2, DEFR2_CODE | DEFR2_DEFE2G);
+        rcar_du_write(rcdu, DEFR3, DEFR3_CODE | DEFR3_DEFE3);
+        rcar_du_write(rcdu, DEFR4, DEFR4_CODE);
+        rcar_du_write(rcdu, DEFR5, DEFR5_CODE | DEFR5_DEFE5);
+
+        /* Use DS1PR and DS2PR to configure planes priorities and connects the
+         * superposition 0 to DU0 pins. DU1 pins will be configured dynamically.
+         */
+        rcar_du_write(rcdu, DORCR, DORCR_PG1D_DS1 | DORCR_DPRS);
+
+done:
+        rcdu->use_count++;
+        return 0;
+}
+
+/*
+ * rcar_du_put - Release a reference to the DU
+ *
+ * Releasing the last reference disables the device clock.
+ *
+ * This function must be called with the DRM mode_config lock held.
+ */
+void rcar_du_put(struct rcar_du_device *rcdu)
+{
+        if (--rcdu->use_count)
+                return;
+
+        clk_disable_unprepare(rcdu->clock);
+}
+
+/* -----------------------------------------------------------------------------
+ * DRM operations
+ */
+
+static int rcar_du_unload(struct drm_device *dev)
+{
+        drm_kms_helper_poll_fini(dev);
+        drm_mode_config_cleanup(dev);
+        drm_vblank_cleanup(dev);
+        drm_irq_uninstall(dev);
+
+        dev->dev_private = NULL;
+
+        return 0;
+}
+
+static int rcar_du_load(struct drm_device *dev, unsigned long flags)
+{
+        struct platform_device *pdev = dev->platformdev;
+        struct rcar_du_platform_data *pdata = pdev->dev.platform_data;
+        struct rcar_du_device *rcdu;
+        struct resource *ioarea;
+        struct resource *mem;
+        int ret;
+
+        if (pdata == NULL) {
+                dev_err(dev->dev, "no platform data\n");
+                return -ENODEV;
+        }
+
+        rcdu = devm_kzalloc(&pdev->dev, sizeof(*rcdu), GFP_KERNEL);
+        if (rcdu == NULL) {
+                dev_err(dev->dev, "failed to allocate private data\n");
+                return -ENOMEM;
+        }
+
+        rcdu->dev = &pdev->dev;
+        rcdu->pdata = pdata;
+        rcdu->ddev = dev;
+        dev->dev_private = rcdu;
+
+        /* I/O resources and clocks */
+        mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        if (mem == NULL) {
+                dev_err(&pdev->dev, "failed to get memory resource\n");
+                return -EINVAL;
+        }
+
+        ioarea = devm_request_mem_region(&pdev->dev, mem->start,
+                                         resource_size(mem), pdev->name);
+        if (ioarea == NULL) {
+                dev_err(&pdev->dev, "failed to request memory region\n");
+                return -EBUSY;
+        }
+
+        rcdu->mmio = devm_ioremap_nocache(&pdev->dev, ioarea->start,
+                                          resource_size(ioarea));
+        if (rcdu->mmio == NULL) {
+                dev_err(&pdev->dev, "failed to remap memory resource\n");
+                return -ENOMEM;
+        }
+
+        rcdu->clock = devm_clk_get(&pdev->dev, NULL);
+        if (IS_ERR(rcdu->clock)) {
+                dev_err(&pdev->dev, "failed to get clock\n");
+                return -ENOENT;
+        }
+
+        /* DRM/KMS objects */
+        ret = rcar_du_modeset_init(rcdu);
+        if (ret < 0) {
+                dev_err(&pdev->dev, "failed to initialize DRM/KMS\n");
+                goto done;
+        }
+
+        /* IRQ and vblank handling */
+        ret = drm_vblank_init(dev, (1 << rcdu->num_crtcs) - 1);
+        if (ret < 0) {
+                dev_err(&pdev->dev, "failed to initialize vblank\n");
+                goto done;
+        }
+
+        ret = drm_irq_install(dev);
+        if (ret < 0) {
+                dev_err(&pdev->dev, "failed to install IRQ handler\n");
+                goto done;
+        }
+
+        platform_set_drvdata(pdev, rcdu);
+
+done:
+        if (ret)
+                rcar_du_unload(dev);
+
+        return ret;
+}
+
+static void rcar_du_preclose(struct drm_device *dev, struct drm_file *file)
+{
+        struct rcar_du_device *rcdu = dev->dev_private;
+        unsigned int i;
+
+        for (i = 0; i < ARRAY_SIZE(rcdu->crtcs); ++i)
+                rcar_du_crtc_cancel_page_flip(&rcdu->crtcs[i], file);
+}
+
+static irqreturn_t rcar_du_irq(int irq, void *arg)
+{
+        struct drm_device *dev = arg;
+        struct rcar_du_device *rcdu = dev->dev_private;
+        unsigned int i;
+
+        for (i = 0; i < ARRAY_SIZE(rcdu->crtcs); ++i)
+                rcar_du_crtc_irq(&rcdu->crtcs[i]);
+
+        return IRQ_HANDLED;
+}
+
+static int rcar_du_enable_vblank(struct drm_device *dev, int crtc)
+{
+        struct rcar_du_device *rcdu = dev->dev_private;
+
+        rcar_du_crtc_enable_vblank(&rcdu->crtcs[crtc], true);
+
+        return 0;
+}
+
+static void rcar_du_disable_vblank(struct drm_device *dev, int crtc)
+{
+        struct rcar_du_device *rcdu = dev->dev_private;
+
+        rcar_du_crtc_enable_vblank(&rcdu->crtcs[crtc], false);
+}
+
+static const struct file_operations rcar_du_fops = {
+        .owner          = THIS_MODULE,
+        .open           = drm_open,
+        .release        = drm_release,
+        .unlocked_ioctl = drm_ioctl,
+#ifdef CONFIG_COMPAT
+        .compat_ioctl   = drm_compat_ioctl,
+#endif
+        .poll           = drm_poll,
+        .read           = drm_read,
+        .fasync         = drm_fasync,
+        .llseek         = no_llseek,
+        .mmap           = drm_gem_cma_mmap,
+};
+
+static struct drm_driver rcar_du_driver = {
+        .driver_features        = DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET
+                                | DRIVER_PRIME,
+        .load                   = rcar_du_load,
+        .unload                 = rcar_du_unload,
+        .preclose               = rcar_du_preclose,
+        .irq_handler            = rcar_du_irq,
+        .get_vblank_counter     = drm_vblank_count,
+        .enable_vblank          = rcar_du_enable_vblank,
+        .disable_vblank         = rcar_du_disable_vblank,
+        .gem_free_object        = drm_gem_cma_free_object,
+        .gem_vm_ops             = &drm_gem_cma_vm_ops,
+        .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
+        .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
+        .gem_prime_import       = drm_gem_cma_dmabuf_import,
+        .gem_prime_export       = drm_gem_cma_dmabuf_export,
+        .dumb_create            = drm_gem_cma_dumb_create,
+        .dumb_map_offset        = drm_gem_cma_dumb_map_offset,
+        .dumb_destroy           = drm_gem_cma_dumb_destroy,
+        .fops                   = &rcar_du_fops,
+        .name                   = "rcar-du",
+        .desc                   = "Renesas R-Car Display Unit",
+        .date                   = "20130110",
+        .major                  = 1,
+        .minor                  = 0,
+};
+
+/* -----------------------------------------------------------------------------
+ * Power management
+ */
+
+#if CONFIG_PM_SLEEP
+static int rcar_du_pm_suspend(struct device *dev)
+{
+        struct rcar_du_device *rcdu = dev_get_drvdata(dev);
+
+        drm_kms_helper_poll_disable(rcdu->ddev);
+        /* TODO Suspend the CRTC */
+
+        return 0;
+}
+
+static int rcar_du_pm_resume(struct device *dev)
+{
+        struct rcar_du_device *rcdu = dev_get_drvdata(dev);
+
+        /* TODO Resume the CRTC */
+
+        drm_kms_helper_poll_enable(rcdu->ddev);
+        return 0;
+}
+#endif
+
+static const struct dev_pm_ops rcar_du_pm_ops = {
+        SET_SYSTEM_SLEEP_PM_OPS(rcar_du_pm_suspend, rcar_du_pm_resume)
+};
+
+/* -----------------------------------------------------------------------------
+ * Platform driver
+ */
+
+static int rcar_du_probe(struct platform_device *pdev)
+{
+        return drm_platform_init(&rcar_du_driver, pdev);
+}
+
+static int rcar_du_remove(struct platform_device *pdev)
+{
+        drm_platform_exit(&rcar_du_driver, pdev);
+
+        return 0;
+}
+
+static struct platform_driver rcar_du_platform_driver = {
+        .probe          = rcar_du_probe,
+        .remove         = rcar_du_remove,
+        .driver         = {
+                .owner  = THIS_MODULE,
+                .name   = "rcar-du",
+                .pm     = &rcar_du_pm_ops,
+        },
+};
+
+module_platform_driver(rcar_du_platform_driver);
+
+MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
+MODULE_DESCRIPTION("Renesas R-Car Display Unit DRM Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.h b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
new file mode 100644
index 000000000000..193cc59d495c
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
@@ -0,0 +1,66 @@
+/*
+ * rcar_du_drv.h  --  R-Car Display Unit DRM driver
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_DRV_H__
+#define __RCAR_DU_DRV_H__
+
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/platform_data/rcar-du.h>
+
+#include "rcar_du_crtc.h"
+#include "rcar_du_plane.h"
+
+struct clk;
+struct device;
+struct drm_device;
+
+struct rcar_du_device {
+        struct device *dev;
+        const struct rcar_du_platform_data *pdata;
+
+        void __iomem *mmio;
+        struct clk *clock;
+        unsigned int use_count;
+
+        struct drm_device *ddev;
+
+        struct rcar_du_crtc crtcs[2];
+        unsigned int used_crtcs;
+        unsigned int num_crtcs;
+
+        struct {
+                struct rcar_du_plane planes[RCAR_DU_NUM_SW_PLANES];
+                unsigned int free;
+                struct mutex lock;
+
+                struct drm_property *alpha;
+                struct drm_property *colorkey;
+                struct drm_property *zpos;
+        } planes;
+};
+
+int rcar_du_get(struct rcar_du_device *rcdu);
+void rcar_du_put(struct rcar_du_device *rcdu);
+
+static inline u32 rcar_du_read(struct rcar_du_device *rcdu, u32 reg)
+{
+        return ioread32(rcdu->mmio + reg);
+}
+
+static inline void rcar_du_write(struct rcar_du_device *rcdu, u32 reg, u32 data)
+{
+        iowrite32(data, rcdu->mmio + reg);
+}
+
+#endif /* __RCAR_DU_DRV_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
new file mode 100644
index 000000000000..9c63f39658de
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -0,0 +1,245 @@
+/*
+ * rcar_du_kms.c  --  R-Car Display Unit Mode Setting
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include "rcar_du_crtc.h"
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+#include "rcar_du_lvds.h"
+#include "rcar_du_regs.h"
+#include "rcar_du_vga.h"
+
+/* -----------------------------------------------------------------------------
+ * Format helpers
+ */
+
+static const struct rcar_du_format_info rcar_du_format_infos[] = {
+        {
+                .fourcc = DRM_FORMAT_RGB565,
+                .bpp = 16,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_TP | PnMR_DDDF_16BPP,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                .fourcc = DRM_FORMAT_ARGB1555,
+                .bpp = 16,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_ALP | PnMR_DDDF_ARGB,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                .fourcc = DRM_FORMAT_XRGB1555,
+                .bpp = 16,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_ALP | PnMR_DDDF_ARGB,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                .fourcc = DRM_FORMAT_XRGB8888,
+                .bpp = 32,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_TP | PnMR_DDDF_16BPP,
+                .edf = PnDDCR4_EDF_RGB888,
+        }, {
+                .fourcc = DRM_FORMAT_ARGB8888,
+                .bpp = 32,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_ALP | PnMR_DDDF_16BPP,
+                .edf = PnDDCR4_EDF_ARGB8888,
+        }, {
+                .fourcc = DRM_FORMAT_UYVY,
+                .bpp = 16,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                .fourcc = DRM_FORMAT_YUYV,
+                .bpp = 16,
+                .planes = 1,
+                .pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                .fourcc = DRM_FORMAT_NV12,
+                .bpp = 12,
+                .planes = 2,
+                .pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                .fourcc = DRM_FORMAT_NV21,
+                .bpp = 12,
+                .planes = 2,
+                .pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
+                .edf = PnDDCR4_EDF_NONE,
+        }, {
+                /* In YUV 4:2:2, only NV16 is supported (NV61 isn't) */
+                .fourcc = DRM_FORMAT_NV16,
+                .bpp = 16,
+                .planes = 2,
+                .pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
+                .edf = PnDDCR4_EDF_NONE,
+        },
+};
+
+const struct rcar_du_format_info *rcar_du_format_info(u32 fourcc)
+{
+        unsigned int i;
+
+        for (i = 0; i < ARRAY_SIZE(rcar_du_format_infos); ++i) {
+                if (rcar_du_format_infos[i].fourcc == fourcc)
+                        return &rcar_du_format_infos[i];
+        }
+
+        return NULL;
+}
+
+/* -----------------------------------------------------------------------------
+ * Common connector and encoder functions
+ */
+
+struct drm_encoder *
+rcar_du_connector_best_encoder(struct drm_connector *connector)
+{
+        struct rcar_du_connector *rcon = to_rcar_connector(connector);
+
+        return &rcon->encoder->encoder;
+}
+
+void rcar_du_encoder_mode_prepare(struct drm_encoder *encoder)
+{
+}
+
+void rcar_du_encoder_mode_set(struct drm_encoder *encoder,
+                              struct drm_display_mode *mode,
+                              struct drm_display_mode *adjusted_mode)
+{
+        struct rcar_du_encoder *renc = to_rcar_encoder(encoder);
+
+        rcar_du_crtc_route_output(encoder->crtc, renc->output);
+}
+
+void rcar_du_encoder_mode_commit(struct drm_encoder *encoder)
+{
+}
+
+/* -----------------------------------------------------------------------------
+ * Frame buffer
+ */
+
+static struct drm_framebuffer *
+rcar_du_fb_create(struct drm_device *dev, struct drm_file *file_priv,
+                  struct drm_mode_fb_cmd2 *mode_cmd)
+{
+        const struct rcar_du_format_info *format;
+
+        format = rcar_du_format_info(mode_cmd->pixel_format);
+        if (format == NULL) {
+                dev_dbg(dev->dev, "unsupported pixel format %08x\n",
+                        mode_cmd->pixel_format);
+                return ERR_PTR(-EINVAL);
+        }
+
+        if (mode_cmd->pitches[0] & 15 || mode_cmd->pitches[0] >= 8192) {
+                dev_dbg(dev->dev, "invalid pitch value %u\n",
+                        mode_cmd->pitches[0]);
+                return ERR_PTR(-EINVAL);
+        }
+
+        if (format->planes == 2) {
+                if (mode_cmd->pitches[1] != mode_cmd->pitches[0]) {
+                        dev_dbg(dev->dev,
+                                "luma and chroma pitches do not match\n");
+                        return ERR_PTR(-EINVAL);
+                }
+        }
+
+        return drm_fb_cma_create(dev, file_priv, mode_cmd);
+}
+
+static const struct drm_mode_config_funcs rcar_du_mode_config_funcs = {
+        .fb_create = rcar_du_fb_create,
+};
+
+int rcar_du_modeset_init(struct rcar_du_device *rcdu)
+{
+        struct drm_device *dev = rcdu->ddev;
+        struct drm_encoder *encoder;
+        unsigned int i;
+        int ret;
+
+        drm_mode_config_init(rcdu->ddev);
+
+        rcdu->ddev->mode_config.min_width = 0;
+        rcdu->ddev->mode_config.min_height = 0;
+        rcdu->ddev->mode_config.max_width = 4095;
+        rcdu->ddev->mode_config.max_height = 2047;
+        rcdu->ddev->mode_config.funcs = &rcar_du_mode_config_funcs;
+
+        ret = rcar_du_plane_init(rcdu);
+        if (ret < 0)
+                return ret;
+
+        for (i = 0; i < ARRAY_SIZE(rcdu->crtcs); ++i)
+                rcar_du_crtc_create(rcdu, i);
+
+        rcdu->used_crtcs = 0;
+        rcdu->num_crtcs = i;
+
+        for (i = 0; i < rcdu->pdata->num_encoders; ++i) {
+                const struct rcar_du_encoder_data *pdata =
+                        &rcdu->pdata->encoders[i];
+
+                if (pdata->output >= ARRAY_SIZE(rcdu->crtcs)) {
+                        dev_warn(rcdu->dev,
+                                 "encoder %u references unexisting output %u, skipping\n",
+                                 i, pdata->output);
+                        continue;
+                }
+
+                switch (pdata->encoder) {
+                case RCAR_DU_ENCODER_VGA:
+                        rcar_du_vga_init(rcdu, &pdata->u.vga, pdata->output);
+                        break;
+
+                case RCAR_DU_ENCODER_LVDS:
+                        rcar_du_lvds_init(rcdu, &pdata->u.lvds, pdata->output);
+                        break;
+
+                default:
+                        break;
+                }
+        }
+
+        /* Set the possible CRTCs and possible clones. All encoders can be
+         * driven by the CRTC associated with the output they're connected to,
+         * as well as by CRTC 0.
+         */
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+                struct rcar_du_encoder *renc = to_rcar_encoder(encoder);
+
+                encoder->possible_crtcs = (1 << 0) | (1 << renc->output);
+                encoder->possible_clones = 1 << 0;
+        }
+
+        ret = rcar_du_plane_register(rcdu);
+        if (ret < 0)
+                return ret;
+
+        drm_kms_helper_poll_init(rcdu->ddev);
+
+        drm_helper_disable_unused_functions(rcdu->ddev);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.h b/drivers/gpu/drm/rcar-du/rcar_du_kms.h
new file mode 100644
index 000000000000..e4d8db069a06
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.h
@@ -0,0 +1,59 @@
+/*
+ * rcar_du_kms.h  --  R-Car Display Unit Mode Setting
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_KMS_H__
+#define __RCAR_DU_KMS_H__
+
+#include <linux/types.h>
+
+#include <drm/drm_crtc.h>
+
+struct rcar_du_device;
+
+struct rcar_du_format_info {
+        u32 fourcc;
+        unsigned int bpp;
+        unsigned int planes;
+        unsigned int pnmr;
+        unsigned int edf;
+};
+
+struct rcar_du_encoder {
+        struct drm_encoder encoder;
+        unsigned int output;
+};
+
+#define to_rcar_encoder(e) \
+        container_of(e, struct rcar_du_encoder, encoder)
+
+struct rcar_du_connector {
+        struct drm_connector connector;
+        struct rcar_du_encoder *encoder;
+};
+
+#define to_rcar_connector(c) \
+        container_of(c, struct rcar_du_connector, connector)
+
+const struct rcar_du_format_info *rcar_du_format_info(u32 fourcc);
+
+struct drm_encoder *
+rcar_du_connector_best_encoder(struct drm_connector *connector);
+void rcar_du_encoder_mode_prepare(struct drm_encoder *encoder);
+void rcar_du_encoder_mode_set(struct drm_encoder *encoder,
+                              struct drm_display_mode *mode,
+                              struct drm_display_mode *adjusted_mode);
+void rcar_du_encoder_mode_commit(struct drm_encoder *encoder);
+
+int rcar_du_modeset_init(struct rcar_du_device *rcdu);
+
+#endif /* __RCAR_DU_KMS_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvds.c b/drivers/gpu/drm/rcar-du/rcar_du_lvds.c
new file mode 100644
index 000000000000..7aefe7267e1d
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvds.c
@@ -0,0 +1,216 @@
+/*
+ * rcar_du_lvds.c  --  R-Car Display Unit LVDS Encoder and Connector
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+#include "rcar_du_lvds.h"
+
+struct rcar_du_lvds_connector {
+        struct rcar_du_connector connector;
+
+        const struct rcar_du_panel_data *panel;
+};
+
+#define to_rcar_lvds_connector(c) \
+        container_of(c, struct rcar_du_lvds_connector, connector.connector)
+
+/* -----------------------------------------------------------------------------
+ * Connector
+ */
+
+static int rcar_du_lvds_connector_get_modes(struct drm_connector *connector)
+{
+        struct rcar_du_lvds_connector *lvdscon = to_rcar_lvds_connector(connector);
+        struct drm_display_mode *mode;
+
+        mode = drm_mode_create(connector->dev);
+        if (mode == NULL)
+                return 0;
+
+        mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER;
+        mode->clock = lvdscon->panel->mode.clock;
+        mode->hdisplay = lvdscon->panel->mode.hdisplay;
+        mode->hsync_start = lvdscon->panel->mode.hsync_start;
+        mode->hsync_end = lvdscon->panel->mode.hsync_end;
+        mode->htotal = lvdscon->panel->mode.htotal;
+        mode->vdisplay = lvdscon->panel->mode.vdisplay;
+        mode->vsync_start = lvdscon->panel->mode.vsync_start;
+        mode->vsync_end = lvdscon->panel->mode.vsync_end;
+        mode->vtotal = lvdscon->panel->mode.vtotal;
+        mode->flags = lvdscon->panel->mode.flags;
+
+        drm_mode_set_name(mode);
+        drm_mode_probed_add(connector, mode);
+
+        return 1;
+}
+
+static int rcar_du_lvds_connector_mode_valid(struct drm_connector *connector,
+                                            struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs connector_helper_funcs = {
+        .get_modes = rcar_du_lvds_connector_get_modes,
+        .mode_valid = rcar_du_lvds_connector_mode_valid,
+        .best_encoder = rcar_du_connector_best_encoder,
+};
+
+static void rcar_du_lvds_connector_destroy(struct drm_connector *connector)
+{
+        drm_sysfs_connector_remove(connector);
+        drm_connector_cleanup(connector);
+}
+
+static enum drm_connector_status
+rcar_du_lvds_connector_detect(struct drm_connector *connector, bool force)
+{
+        return connector_status_connected;
+}
+
+static const struct drm_connector_funcs connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .detect = rcar_du_lvds_connector_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = rcar_du_lvds_connector_destroy,
+};
+
+static int rcar_du_lvds_connector_init(struct rcar_du_device *rcdu,
+                                       struct rcar_du_encoder *renc,
+                                       const struct rcar_du_panel_data *panel)
+{
+        struct rcar_du_lvds_connector *lvdscon;
+        struct drm_connector *connector;
+        int ret;
+
+        lvdscon = devm_kzalloc(rcdu->dev, sizeof(*lvdscon), GFP_KERNEL);
+        if (lvdscon == NULL)
+                return -ENOMEM;
+
+        lvdscon->panel = panel;
+
+        connector = &lvdscon->connector.connector;
+        connector->display_info.width_mm = panel->width_mm;
+        connector->display_info.height_mm = panel->height_mm;
+
+        ret = drm_connector_init(rcdu->ddev, connector, &connector_funcs,
+                                 DRM_MODE_CONNECTOR_LVDS);
+        if (ret < 0)
+                return ret;
+
+        drm_connector_helper_add(connector, &connector_helper_funcs);
+        ret = drm_sysfs_connector_add(connector);
+        if (ret < 0)
+                return ret;
+
+        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+        drm_object_property_set_value(&connector->base,
+                rcdu->ddev->mode_config.dpms_property, DRM_MODE_DPMS_OFF);
+
+        ret = drm_mode_connector_attach_encoder(connector, &renc->encoder);
+        if (ret < 0)
+                return ret;
+
+        connector->encoder = &renc->encoder;
+        lvdscon->connector.encoder = renc;
+
+        return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * Encoder
+ */
+
+static void rcar_du_lvds_encoder_dpms(struct drm_encoder *encoder, int mode)
+{
+}
+
+static bool rcar_du_lvds_encoder_mode_fixup(struct drm_encoder *encoder,
+                                           const struct drm_display_mode *mode,
+                                           struct drm_display_mode *adjusted_mode)
+{
+        const struct drm_display_mode *panel_mode;
+        struct drm_device *dev = encoder->dev;
+        struct drm_connector *connector;
+        bool found = false;
+
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                if (connector->encoder == encoder) {
+                        found = true;
+                        break;
+                }
+        }
+
+        if (!found) {
+                dev_dbg(dev->dev, "mode_fixup: no connector found\n");
+                return false;
+        }
+
+        if (list_empty(&connector->modes)) {
+                dev_dbg(dev->dev, "mode_fixup: empty modes list\n");
+                return false;
+        }
+
+        panel_mode = list_first_entry(&connector->modes,
+                                      struct drm_display_mode, head);
+
+        /* We're not allowed to modify the resolution. */
+        if (mode->hdisplay != panel_mode->hdisplay ||
+            mode->vdisplay != panel_mode->vdisplay)
+                return false;
+
+        /* The flat panel mode is fixed, just copy it to the adjusted mode. */
+        drm_mode_copy(adjusted_mode, panel_mode);
+
+        return true;
+}
+
+static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
+        .dpms = rcar_du_lvds_encoder_dpms,
+        .mode_fixup = rcar_du_lvds_encoder_mode_fixup,
+        .prepare = rcar_du_encoder_mode_prepare,
+        .commit = rcar_du_encoder_mode_commit,
+        .mode_set = rcar_du_encoder_mode_set,
+};
+
+static const struct drm_encoder_funcs encoder_funcs = {
+        .destroy = drm_encoder_cleanup,
+};
+
+int rcar_du_lvds_init(struct rcar_du_device *rcdu,
+                      const struct rcar_du_encoder_lvds_data *data,
+                      unsigned int output)
+{
+        struct rcar_du_encoder *renc;
+        int ret;
+
+        renc = devm_kzalloc(rcdu->dev, sizeof(*renc), GFP_KERNEL);
+        if (renc == NULL)
+                return -ENOMEM;
+
+        renc->output = output;
+
+        ret = drm_encoder_init(rcdu->ddev, &renc->encoder, &encoder_funcs,
+                               DRM_MODE_ENCODER_LVDS);
+        if (ret < 0)
+                return ret;
+
+        drm_encoder_helper_add(&renc->encoder, &encoder_helper_funcs);
+
+        return rcar_du_lvds_connector_init(rcdu, renc, &data->panel);
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvds.h b/drivers/gpu/drm/rcar-du/rcar_du_lvds.h
new file mode 100644
index 000000000000..b47f8328e103
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvds.h
@@ -0,0 +1,24 @@
+/*
+ * rcar_du_lvds.h  --  R-Car Display Unit LVDS Encoder and Connector
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_LVDS_H__
+#define __RCAR_DU_LVDS_H__
+
+struct rcar_du_device;
+struct rcar_du_encoder_lvds_data;
+
+int rcar_du_lvds_init(struct rcar_du_device *rcdu,
+                      const struct rcar_du_encoder_lvds_data *data,
+                      unsigned int output);
+
+#endif /* __RCAR_DU_LVDS_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
new file mode 100644
index 000000000000..a65f81ddf51d
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -0,0 +1,507 @@
+/*
+ * rcar_du_plane.c  --  R-Car Display Unit Planes
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+#include "rcar_du_plane.h"
+#include "rcar_du_regs.h"
+
+#define RCAR_DU_COLORKEY_NONE           (0 << 24)
+#define RCAR_DU_COLORKEY_SOURCE         (1 << 24)
+#define RCAR_DU_COLORKEY_MASK           (1 << 24)
+
+struct rcar_du_kms_plane {
+        struct drm_plane plane;
+        struct rcar_du_plane *hwplane;
+};
+
+static inline struct rcar_du_plane *to_rcar_plane(struct drm_plane *plane)
+{
+        return container_of(plane, struct rcar_du_kms_plane, plane)->hwplane;
+}
+
+static u32 rcar_du_plane_read(struct rcar_du_device *rcdu,
+                              unsigned int index, u32 reg)
+{
+        return rcar_du_read(rcdu, index * PLANE_OFF + reg);
+}
+
+static void rcar_du_plane_write(struct rcar_du_device *rcdu,
+                                unsigned int index, u32 reg, u32 data)
+{
+        rcar_du_write(rcdu, index * PLANE_OFF + reg, data);
+}
+
+int rcar_du_plane_reserve(struct rcar_du_plane *plane,
+                          const struct rcar_du_format_info *format)
+{
+        struct rcar_du_device *rcdu = plane->dev;
+        unsigned int i;
+        int ret = -EBUSY;
+
+        mutex_lock(&rcdu->planes.lock);
+
+        for (i = 0; i < ARRAY_SIZE(rcdu->planes.planes); ++i) {
+                if (!(rcdu->planes.free & (1 << i)))
+                        continue;
+
+                if (format->planes == 1 ||
+                    rcdu->planes.free & (1 << ((i + 1) % 8)))
+                        break;
+        }
+
+        if (i == ARRAY_SIZE(rcdu->planes.planes))
+                goto done;
+
+        rcdu->planes.free &= ~(1 << i);
+        if (format->planes == 2)
+                rcdu->planes.free &= ~(1 << ((i + 1) % 8));
+
+        plane->hwindex = i;
+
+        ret = 0;
+
+done:
+        mutex_unlock(&rcdu->planes.lock);
+        return ret;
+}
+
+void rcar_du_plane_release(struct rcar_du_plane *plane)
+{
+        struct rcar_du_device *rcdu = plane->dev;
+
+        if (plane->hwindex == -1)
+                return;
+
+        mutex_lock(&rcdu->planes.lock);
+        rcdu->planes.free |= 1 << plane->hwindex;
+        if (plane->format->planes == 2)
+                rcdu->planes.free |= 1 << ((plane->hwindex + 1) % 8);
+        mutex_unlock(&rcdu->planes.lock);
+
+        plane->hwindex = -1;
+}
+
+void rcar_du_plane_update_base(struct rcar_du_plane *plane)
+{
+        struct rcar_du_device *rcdu = plane->dev;
+        unsigned int index = plane->hwindex;
+
+        /* According to the datasheet the Y position is expressed in raster line
+         * units. However, 32bpp formats seem to require a doubled Y position
+         * value. Similarly, for the second plane, NV12 and NV21 formats seem to
+         * require a halved Y position value.
+         */
+        rcar_du_plane_write(rcdu, index, PnSPXR, plane->src_x);
+        rcar_du_plane_write(rcdu, index, PnSPYR, plane->src_y *
+                            (plane->format->bpp == 32 ? 2 : 1));
+        rcar_du_plane_write(rcdu, index, PnDSA0R, plane->dma[0]);
+
+        if (plane->format->planes == 2) {
+                index = (index + 1) % 8;
+
+                rcar_du_plane_write(rcdu, index, PnSPXR, plane->src_x);
+                rcar_du_plane_write(rcdu, index, PnSPYR, plane->src_y *
+                                    (plane->format->bpp == 16 ? 2 : 1) / 2);
+                rcar_du_plane_write(rcdu, index, PnDSA0R, plane->dma[1]);
+        }
+}
+
+void rcar_du_plane_compute_base(struct rcar_du_plane *plane,
+                                struct drm_framebuffer *fb)
+{
+        struct drm_gem_cma_object *gem;
+
+        gem = drm_fb_cma_get_gem_obj(fb, 0);
+        plane->dma[0] = gem->paddr + fb->offsets[0];
+
+        if (plane->format->planes == 2) {
+                gem = drm_fb_cma_get_gem_obj(fb, 1);
+                plane->dma[1] = gem->paddr + fb->offsets[1];
+        }
+}
+
+static void rcar_du_plane_setup_mode(struct rcar_du_plane *plane,
+                                     unsigned int index)
+{
+        struct rcar_du_device *rcdu = plane->dev;
+        u32 colorkey;
+        u32 pnmr;
+
+        /* The PnALPHAR register controls alpha-blending in 16bpp formats
+         * (ARGB1555 and XRGB1555).
+         *
+         * For ARGB, set the alpha value to 0, and enable alpha-blending when
+         * the A bit is 0. This maps A=0 to alpha=0 and A=1 to alpha=255.
+         *
+         * For XRGB, set the alpha value to the plane-wide alpha value and
+         * enable alpha-blending regardless of the X bit value.
+         */
+        if (plane->format->fourcc != DRM_FORMAT_XRGB1555)
+                rcar_du_plane_write(rcdu, index, PnALPHAR, PnALPHAR_ABIT_0);
+        else
+                rcar_du_plane_write(rcdu, index, PnALPHAR,
+                                    PnALPHAR_ABIT_X | plane->alpha);
+
+        pnmr = PnMR_BM_MD | plane->format->pnmr;
+
+        /* Disable color keying when requested. YUV formats have the
+         * PnMR_SPIM_TP_OFF bit set in their pnmr field, disabling color keying
+         * automatically.
+         */
+        if ((plane->colorkey & RCAR_DU_COLORKEY_MASK) == RCAR_DU_COLORKEY_NONE)
+                pnmr |= PnMR_SPIM_TP_OFF;
+
+        /* For packed YUV formats we need to select the U/V order. */
+        if (plane->format->fourcc == DRM_FORMAT_YUYV)
+                pnmr |= PnMR_YCDF_YUYV;
+
+        rcar_du_plane_write(rcdu, index, PnMR, pnmr);
+
+        switch (plane->format->fourcc) {
+        case DRM_FORMAT_RGB565:
+                colorkey = ((plane->colorkey & 0xf80000) >> 8)
+                         | ((plane->colorkey & 0x00fc00) >> 5)
+                         | ((plane->colorkey & 0x0000f8) >> 3);
+                rcar_du_plane_write(rcdu, index, PnTC2R, colorkey);
+                break;
+
+        case DRM_FORMAT_ARGB1555:
+        case DRM_FORMAT_XRGB1555:
+                colorkey = ((plane->colorkey & 0xf80000) >> 9)
+                         | ((plane->colorkey & 0x00f800) >> 6)
+                         | ((plane->colorkey & 0x0000f8) >> 3);
+                rcar_du_plane_write(rcdu, index, PnTC2R, colorkey);
+                break;
+
+        case DRM_FORMAT_XRGB8888:
+        case DRM_FORMAT_ARGB8888:
+                rcar_du_plane_write(rcdu, index, PnTC3R,
+                                    PnTC3R_CODE | (plane->colorkey & 0xffffff));
+                break;
+        }
+}
+
+static void __rcar_du_plane_setup(struct rcar_du_plane *plane,
+                                  unsigned int index)
+{
+        struct rcar_du_device *rcdu = plane->dev;
+        u32 ddcr2 = PnDDCR2_CODE;
+        u32 ddcr4;
+        u32 mwr;
+
+        /* Data format
+         *
+         * The data format is selected by the DDDF field in PnMR and the EDF
+         * field in DDCR4.
+         */
+        ddcr4 = rcar_du_plane_read(rcdu, index, PnDDCR4);
+        ddcr4 &= ~PnDDCR4_EDF_MASK;
+        ddcr4 |= plane->format->edf | PnDDCR4_CODE;
+
+        rcar_du_plane_setup_mode(plane, index);
+
+        if (plane->format->planes == 2) {
+                if (plane->hwindex != index) {
+                        if (plane->format->fourcc == DRM_FORMAT_NV12 ||
+                            plane->format->fourcc == DRM_FORMAT_NV21)
+                                ddcr2 |= PnDDCR2_Y420;
+
+                        if (plane->format->fourcc == DRM_FORMAT_NV21)
+                                ddcr2 |= PnDDCR2_NV21;
+
+                        ddcr2 |= PnDDCR2_DIVU;
+                } else {
+                        ddcr2 |= PnDDCR2_DIVY;
+                }
+        }
+
+        rcar_du_plane_write(rcdu, index, PnDDCR2, ddcr2);
+        rcar_du_plane_write(rcdu, index, PnDDCR4, ddcr4);
+
+        /* Memory pitch (expressed in pixels) */
+        if (plane->format->planes == 2)
+                mwr = plane->pitch;
+        else
+                mwr = plane->pitch * 8 / plane->format->bpp;
+
+        rcar_du_plane_write(rcdu, index, PnMWR, mwr);
+
+        /* Destination position and size */
+        rcar_du_plane_write(rcdu, index, PnDSXR, plane->width);
+        rcar_du_plane_write(rcdu, index, PnDSYR, plane->height);
+        rcar_du_plane_write(rcdu, index, PnDPXR, plane->dst_x);
+        rcar_du_plane_write(rcdu, index, PnDPYR, plane->dst_y);
+
+        /* Wrap-around and blinking, disabled */
+        rcar_du_plane_write(rcdu, index, PnWASPR, 0);
+        rcar_du_plane_write(rcdu, index, PnWAMWR, 4095);
+        rcar_du_plane_write(rcdu, index, PnBTR, 0);
+        rcar_du_plane_write(rcdu, index, PnMLR, 0);
+}
+
+void rcar_du_plane_setup(struct rcar_du_plane *plane)
+{
+        __rcar_du_plane_setup(plane, plane->hwindex);
+        if (plane->format->planes == 2)
+                __rcar_du_plane_setup(plane, (plane->hwindex + 1) % 8);
+
+        rcar_du_plane_update_base(plane);
+}
+
+static int
+rcar_du_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+                       struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+                       unsigned int crtc_w, unsigned int crtc_h,
+                       uint32_t src_x, uint32_t src_y,
+                       uint32_t src_w, uint32_t src_h)
+{
+        struct rcar_du_plane *rplane = to_rcar_plane(plane);
+        struct rcar_du_device *rcdu = plane->dev->dev_private;
+        const struct rcar_du_format_info *format;
+        unsigned int nplanes;
+        int ret;
+
+        format = rcar_du_format_info(fb->pixel_format);
+        if (format == NULL) {
+                dev_dbg(rcdu->dev, "%s: unsupported format %08x\n", __func__,
+                        fb->pixel_format);
+                return -EINVAL;
+        }
+
+        if (src_w >> 16 != crtc_w || src_h >> 16 != crtc_h) {
+                dev_dbg(rcdu->dev, "%s: scaling not supported\n", __func__);
+                return -EINVAL;
+        }
+
+        nplanes = rplane->format ? rplane->format->planes : 0;
+
+        /* Reallocate hardware planes if the number of required planes has
+         * changed.
+         */
+        if (format->planes != nplanes) {
+                rcar_du_plane_release(rplane);
+                ret = rcar_du_plane_reserve(rplane, format);
+                if (ret < 0)
+                        return ret;
+        }
+
+        rplane->crtc = crtc;
+        rplane->format = format;
+        rplane->pitch = fb->pitches[0];
+
+        rplane->src_x = src_x >> 16;
+        rplane->src_y = src_y >> 16;
+        rplane->dst_x = crtc_x;
+        rplane->dst_y = crtc_y;
+        rplane->width = crtc_w;
+        rplane->height = crtc_h;
+
+        rcar_du_plane_compute_base(rplane, fb);
+        rcar_du_plane_setup(rplane);
+
+        mutex_lock(&rcdu->planes.lock);
+        rplane->enabled = true;
+        rcar_du_crtc_update_planes(rplane->crtc);
+        mutex_unlock(&rcdu->planes.lock);
+
+        return 0;
+}
+
+static int rcar_du_plane_disable(struct drm_plane *plane)
+{
+        struct rcar_du_device *rcdu = plane->dev->dev_private;
+        struct rcar_du_plane *rplane = to_rcar_plane(plane);
+
+        if (!rplane->enabled)
+                return 0;
+
+        mutex_lock(&rcdu->planes.lock);
+        rplane->enabled = false;
+        rcar_du_crtc_update_planes(rplane->crtc);
+        mutex_unlock(&rcdu->planes.lock);
+
+        rcar_du_plane_release(rplane);
+
+        rplane->crtc = NULL;
+        rplane->format = NULL;
+
+        return 0;
+}
+
+/* Both the .set_property and the .update_plane operations are called with the
+ * mode_config lock held. There is this no need to explicitly protect access to
+ * the alpha and colorkey fields and the mode register.
+ */
+static void rcar_du_plane_set_alpha(struct rcar_du_plane *plane, u32 alpha)
+{
+        if (plane->alpha == alpha)
+                return;
+
+        plane->alpha = alpha;
+        if (!plane->enabled || plane->format->fourcc != DRM_FORMAT_XRGB1555)
+                return;
+
+        rcar_du_plane_setup_mode(plane, plane->hwindex);
+}
+
+static void rcar_du_plane_set_colorkey(struct rcar_du_plane *plane,
+                                       u32 colorkey)
+{
+        if (plane->colorkey == colorkey)
+                return;
+
+        plane->colorkey = colorkey;
+        if (!plane->enabled)
+                return;
+
+        rcar_du_plane_setup_mode(plane, plane->hwindex);
+}
+
+static void rcar_du_plane_set_zpos(struct rcar_du_plane *plane,
+                                   unsigned int zpos)
+{
+        struct rcar_du_device *rcdu = plane->dev;
+
+        mutex_lock(&rcdu->planes.lock);
+        if (plane->zpos == zpos)
+                goto done;
+
+        plane->zpos = zpos;
+        if (!plane->enabled)
+                goto done;
+
+        rcar_du_crtc_update_planes(plane->crtc);
+
+done:
+        mutex_unlock(&rcdu->planes.lock);
+}
+
+static int rcar_du_plane_set_property(struct drm_plane *plane,
+                                      struct drm_property *property,
+                                      uint64_t value)
+{
+        struct rcar_du_device *rcdu = plane->dev->dev_private;
+        struct rcar_du_plane *rplane = to_rcar_plane(plane);
+
+        if (property == rcdu->planes.alpha)
+                rcar_du_plane_set_alpha(rplane, value);
+        else if (property == rcdu->planes.colorkey)
+                rcar_du_plane_set_colorkey(rplane, value);
+        else if (property == rcdu->planes.zpos)
+                rcar_du_plane_set_zpos(rplane, value);
+        else
+                return -EINVAL;
+
+        return 0;
+}
+
+static const struct drm_plane_funcs rcar_du_plane_funcs = {
+        .update_plane = rcar_du_plane_update,
+        .disable_plane = rcar_du_plane_disable,
+        .set_property = rcar_du_plane_set_property,
+        .destroy = drm_plane_cleanup,
+};
+
+static const uint32_t formats[] = {
+        DRM_FORMAT_RGB565,
+        DRM_FORMAT_ARGB1555,
+        DRM_FORMAT_XRGB1555,
+        DRM_FORMAT_XRGB8888,
+        DRM_FORMAT_ARGB8888,
+        DRM_FORMAT_UYVY,
+        DRM_FORMAT_YUYV,
+        DRM_FORMAT_NV12,
+        DRM_FORMAT_NV21,
+        DRM_FORMAT_NV16,
+};
+
+int rcar_du_plane_init(struct rcar_du_device *rcdu)
+{
+        unsigned int i;
+
+        mutex_init(&rcdu->planes.lock);
+        rcdu->planes.free = 0xff;
+
+        rcdu->planes.alpha =
+                drm_property_create_range(rcdu->ddev, 0, "alpha", 0, 255);
+        if (rcdu->planes.alpha == NULL)
+                return -ENOMEM;
+
+        /* The color key is expressed as an RGB888 triplet stored in a 32-bit
+         * integer in XRGB8888 format. Bit 24 is used as a flag to disable (0)
+         * or enable source color keying (1).
+         */
+        rcdu->planes.colorkey =
+                drm_property_create_range(rcdu->ddev, 0, "colorkey",
+                                          0, 0x01ffffff);
+        if (rcdu->planes.colorkey == NULL)
+                return -ENOMEM;
+
+        rcdu->planes.zpos =
+                drm_property_create_range(rcdu->ddev, 0, "zpos", 1, 7);
+        if (rcdu->planes.zpos == NULL)
+                return -ENOMEM;
+
+        for (i = 0; i < ARRAY_SIZE(rcdu->planes.planes); ++i) {
+                struct rcar_du_plane *plane = &rcdu->planes.planes[i];
+
+                plane->dev = rcdu;
+                plane->hwindex = -1;
+                plane->alpha = 255;
+                plane->colorkey = RCAR_DU_COLORKEY_NONE;
+                plane->zpos = 0;
+        }
+
+        return 0;
+}
+
+int rcar_du_plane_register(struct rcar_du_device *rcdu)
+{
+        unsigned int i;
+        int ret;
+
+        for (i = 0; i < RCAR_DU_NUM_KMS_PLANES; ++i) {
+                struct rcar_du_kms_plane *plane;
+
+                plane = devm_kzalloc(rcdu->dev, sizeof(*plane), GFP_KERNEL);
+                if (plane == NULL)
+                        return -ENOMEM;
+
+                plane->hwplane = &rcdu->planes.planes[i + 2];
+                plane->hwplane->zpos = 1;
+
+                ret = drm_plane_init(rcdu->ddev, &plane->plane,
+                                     (1 << rcdu->num_crtcs) - 1,
+                                     &rcar_du_plane_funcs, formats,
+                                     ARRAY_SIZE(formats), false);
+                if (ret < 0)
+                        return ret;
+
+                drm_object_attach_property(&plane->plane.base,
+                                           rcdu->planes.alpha, 255);
+                drm_object_attach_property(&plane->plane.base,
+                                           rcdu->planes.colorkey,
+                                           RCAR_DU_COLORKEY_NONE);
+                drm_object_attach_property(&plane->plane.base,
+                                           rcdu->planes.zpos, 1);
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.h b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
new file mode 100644
index 000000000000..5397dba2fe57
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
@@ -0,0 +1,67 @@
+/*
+ * rcar_du_plane.h  --  R-Car Display Unit Planes
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_PLANE_H__
+#define __RCAR_DU_PLANE_H__
+
+struct drm_crtc;
+struct drm_framebuffer;
+struct rcar_du_device;
+struct rcar_du_format_info;
+
+/* The RCAR DU has 8 hardware planes, shared between KMS planes and CRTCs. As
+ * using KMS planes requires at least one of the CRTCs being enabled, no more
+ * than 7 KMS planes can be available. We thus create 7 KMS planes and
+ * 9 software planes (one for each KMS planes and one for each CRTC).
+ */
+
+#define RCAR_DU_NUM_KMS_PLANES          7
+#define RCAR_DU_NUM_HW_PLANES           8
+#define RCAR_DU_NUM_SW_PLANES           9
+
+struct rcar_du_plane {
+        struct rcar_du_device *dev;
+        struct drm_crtc *crtc;
+
+        bool enabled;
+
+        int hwindex;            /* 0-based, -1 means unused */
+        unsigned int alpha;
+        unsigned int colorkey;
+        unsigned int zpos;
+
+        const struct rcar_du_format_info *format;
+
+        unsigned long dma[2];
+        unsigned int pitch;
+
+        unsigned int width;
+        unsigned int height;
+
+        unsigned int src_x;
+        unsigned int src_y;
+        unsigned int dst_x;
+        unsigned int dst_y;
+};
+
+int rcar_du_plane_init(struct rcar_du_device *rcdu);
+int rcar_du_plane_register(struct rcar_du_device *rcdu);
+void rcar_du_plane_setup(struct rcar_du_plane *plane);
+void rcar_du_plane_update_base(struct rcar_du_plane *plane);
+void rcar_du_plane_compute_base(struct rcar_du_plane *plane,
+                                struct drm_framebuffer *fb);
+int rcar_du_plane_reserve(struct rcar_du_plane *plane,
+                          const struct rcar_du_format_info *format);
+void rcar_du_plane_release(struct rcar_du_plane *plane);
+
+#endif /* __RCAR_DU_PLANE_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_regs.h b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
new file mode 100644
index 000000000000..69f21f19b51c
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
@@ -0,0 +1,445 @@
+/*
+ * rcar_du_regs.h  --  R-Car Display Unit Registers Definitions
+ *
+ * Copyright (C) 2013 Renesas Electronics Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef __RCAR_DU_REGS_H__
+#define __RCAR_DU_REGS_H__
+
+#define DISP2_REG_OFFSET         0x30000
+
+/* -----------------------------------------------------------------------------
+ * Display Control Registers
+ */
+
+#define DSYSR                   0x00000 /* display 1 */
+#define D2SYSR                  0x30000 /* display 2 */
+#define DSYSR_ILTS              (1 << 29)
+#define DSYSR_DSEC              (1 << 20)
+#define DSYSR_IUPD              (1 << 16)
+#define DSYSR_DRES              (1 << 9)
+#define DSYSR_DEN               (1 << 8)
+#define DSYSR_TVM_MASTER        (0 << 6)
+#define DSYSR_TVM_SWITCH        (1 << 6)
+#define DSYSR_TVM_TVSYNC        (2 << 6)
+#define DSYSR_TVM_MASK          (3 << 6)
+#define DSYSR_SCM_INT_NONE      (0 << 4)
+#define DSYSR_SCM_INT_SYNC      (2 << 4)
+#define DSYSR_SCM_INT_VIDEO     (3 << 4)
+
+#define DSMR                    0x00004
+#define D2SMR                   0x30004
+#define DSMR_VSPM               (1 << 28)
+#define DSMR_ODPM               (1 << 27)
+#define DSMR_DIPM_DISP          (0 << 25)
+#define DSMR_DIPM_CSYNC         (1 << 25)
+#define DSMR_DIPM_DE            (3 << 25)
+#define DSMR_DIPM_MASK          (3 << 25)
+#define DSMR_CSPM               (1 << 24)
+#define DSMR_DIL                (1 << 19)
+#define DSMR_VSL                (1 << 18)
+#define DSMR_HSL                (1 << 17)
+#define DSMR_DDIS               (1 << 16)
+#define DSMR_CDEL               (1 << 15)
+#define DSMR_CDEM_CDE           (0 << 13)
+#define DSMR_CDEM_LOW           (2 << 13)
+#define DSMR_CDEM_HIGH          (3 << 13)
+#define DSMR_CDEM_MASK          (3 << 13)
+#define DSMR_CDED               (1 << 12)
+#define DSMR_ODEV               (1 << 8)
+#define DSMR_CSY_VH_OR          (0 << 6)
+#define DSMR_CSY_333            (2 << 6)
+#define DSMR_CSY_222            (3 << 6)
+#define DSMR_CSY_MASK           (3 << 6)
+
+#define DSSR                    0x00008
+#define D2SSR                   0x30008
+#define DSSR_VC1FB_DSA0         (0 << 30)
+#define DSSR_VC1FB_DSA1         (1 << 30)
+#define DSSR_VC1FB_DSA2         (2 << 30)
+#define DSSR_VC1FB_INIT         (3 << 30)
+#define DSSR_VC1FB_MASK         (3 << 30)
+#define DSSR_VC0FB_DSA0         (0 << 28)
+#define DSSR_VC0FB_DSA1         (1 << 28)
+#define DSSR_VC0FB_DSA2         (2 << 28)
+#define DSSR_VC0FB_INIT         (3 << 28)
+#define DSSR_VC0FB_MASK         (3 << 28)
+#define DSSR_DFB(n)             (1 << ((n)+15))
+#define DSSR_TVR                (1 << 15)
+#define DSSR_FRM                (1 << 14)
+#define DSSR_VBK                (1 << 11)
+#define DSSR_RINT               (1 << 9)
+#define DSSR_HBK                (1 << 8)
+#define DSSR_ADC(n)             (1 << ((n)-1))
+
+#define DSRCR                   0x0000c
+#define D2SRCR                  0x3000c
+#define DSRCR_TVCL              (1 << 15)
+#define DSRCR_FRCL              (1 << 14)
+#define DSRCR_VBCL              (1 << 11)
+#define DSRCR_RICL              (1 << 9)
+#define DSRCR_HBCL              (1 << 8)
+#define DSRCR_ADCL(n)           (1 << ((n)-1))
+#define DSRCR_MASK              0x0000cbff
+
+#define DIER                    0x00010
+#define D2IER                   0x30010
+#define DIER_TVE                (1 << 15)
+#define DIER_FRE                (1 << 14)
+#define DIER_VBE                (1 << 11)
+#define DIER_RIE                (1 << 9)
+#define DIER_HBE                (1 << 8)
+#define DIER_ADCE(n)            (1 << ((n)-1))
+
+#define CPCR                    0x00014
+#define CPCR_CP4CE              (1 << 19)
+#define CPCR_CP3CE              (1 << 18)
+#define CPCR_CP2CE              (1 << 17)
+#define CPCR_CP1CE              (1 << 16)
+
+#define DPPR                    0x00018
+#define DPPR_DPE(n)             (1 << ((n)*4-1))
+#define DPPR_DPS(n, p)          (((p)-1) << DPPR_DPS_SHIFT(n))
+#define DPPR_DPS_SHIFT(n)       (((n)-1)*4)
+#define DPPR_BPP16              (DPPR_DPE(8) | DPPR_DPS(8, 1))  /* plane1 */
+#define DPPR_BPP32_P1           (DPPR_DPE(7) | DPPR_DPS(7, 1))
+#define DPPR_BPP32_P2           (DPPR_DPE(8) | DPPR_DPS(8, 2))
+#define DPPR_BPP32              (DPPR_BPP32_P1 | DPPR_BPP32_P2) /* plane1 & 2 */
+
+#define DEFR                    0x00020
+#define D2EFR                   0x30020
+#define DEFR_CODE               (0x7773 << 16)
+#define DEFR_EXSL               (1 << 12)
+#define DEFR_EXVL               (1 << 11)
+#define DEFR_EXUP               (1 << 5)
+#define DEFR_VCUP               (1 << 4)
+#define DEFR_DEFE               (1 << 0)
+
+#define DAPCR                   0x00024
+#define DAPCR_CODE              (0x7773 << 16)
+#define DAPCR_AP2E              (1 << 4)
+#define DAPCR_AP1E              (1 << 0)
+
+#define DCPCR                   0x00028
+#define DCPCR_CODE              (0x7773 << 16)
+#define DCPCR_CA2B              (1 << 13)
+#define DCPCR_CD2F              (1 << 12)
+#define DCPCR_DC2E              (1 << 8)
+#define DCPCR_CAB               (1 << 5)
+#define DCPCR_CDF               (1 << 4)
+#define DCPCR_DCE               (1 << 0)
+
+#define DEFR2                   0x00034
+#define D2EFR2                  0x30034
+#define DEFR2_CODE              (0x7775 << 16)
+#define DEFR2_DEFE2G            (1 << 0)
+
+#define DEFR3                   0x00038
+#define D2EFR3                  0x30038
+#define DEFR3_CODE              (0x7776 << 16)
+#define DEFR3_EVDA              (1 << 14)
+#define DEFR3_EVDM_1            (1 << 12)
+#define DEFR3_EVDM_2            (2 << 12)
+#define DEFR3_EVDM_3            (3 << 12)
+#define DEFR3_VMSM2_EMA         (1 << 6)
+#define DEFR3_VMSM1_ENA         (1 << 4)
+#define DEFR3_DEFE3             (1 << 0)
+
+#define DEFR4                   0x0003c
+#define D2EFR4                  0x3003c
+#define DEFR4_CODE              (0x7777 << 16)
+#define DEFR4_LRUO              (1 << 5)
+#define DEFR4_SPCE              (1 << 4)
+
+#define DVCSR                   0x000d0
+#define DVCSR_VCnFB2_DSA0(n)    (0 << ((n)*2+16))
+#define DVCSR_VCnFB2_DSA1(n)    (1 << ((n)*2+16))
+#define DVCSR_VCnFB2_DSA2(n)    (2 << ((n)*2+16))
+#define DVCSR_VCnFB2_INIT(n)    (3 << ((n)*2+16))
+#define DVCSR_VCnFB2_MASK(n)    (3 << ((n)*2+16))
+#define DVCSR_VCnFB_DSA0(n)     (0 << ((n)*2))
+#define DVCSR_VCnFB_DSA1(n)     (1 << ((n)*2))
+#define DVCSR_VCnFB_DSA2(n)     (2 << ((n)*2))
+#define DVCSR_VCnFB_INIT(n)     (3 << ((n)*2))
+#define DVCSR_VCnFB_MASK(n)     (3 << ((n)*2))
+
+#define DEFR5                   0x000e0
+#define DEFR5_CODE              (0x66 << 24)
+#define DEFR5_YCRGB2_DIS        (0 << 14)
+#define DEFR5_YCRGB2_PRI1       (1 << 14)
+#define DEFR5_YCRGB2_PRI2       (2 << 14)
+#define DEFR5_YCRGB2_PRI3       (3 << 14)
+#define DEFR5_YCRGB2_MASK       (3 << 14)
+#define DEFR5_YCRGB1_DIS        (0 << 12)
+#define DEFR5_YCRGB1_PRI1       (1 << 12)
+#define DEFR5_YCRGB1_PRI2       (2 << 12)
+#define DEFR5_YCRGB1_PRI3       (3 << 12)
+#define DEFR5_YCRGB1_MASK       (3 << 12)
+#define DEFR5_DEFE5             (1 << 0)
+
+#define DDLTR                   0x000e4
+#define DDLTR_CODE              (0x7766 << 16)
+#define DDLTR_DLAR2             (1 << 6)
+#define DDLTR_DLAY2             (1 << 5)
+#define DDLTR_DLAY1             (1 << 1)
+
+#define DEFR6                   0x000e8
+#define DEFR6_CODE              (0x7778 << 16)
+#define DEFR6_ODPM22_D2SMR      (0 << 10)
+#define DEFR6_ODPM22_DISP       (2 << 10)
+#define DEFR6_ODPM22_CDE        (3 << 10)
+#define DEFR6_ODPM22_MASK       (3 << 10)
+#define DEFR6_ODPM12_DSMR       (0 << 8)
+#define DEFR6_ODPM12_DISP       (2 << 8)
+#define DEFR6_ODPM12_CDE        (3 << 8)
+#define DEFR6_ODPM12_MASK       (3 << 8)
+#define DEFR6_TCNE2             (1 << 6)
+#define DEFR6_MLOS1             (1 << 2)
+#define DEFR6_DEFAULT           (DEFR6_CODE | DEFR6_TCNE2)
+
+/* -----------------------------------------------------------------------------
+ * Display Timing Generation Registers
+ */
+
+#define HDSR                    0x00040
+#define HDER                    0x00044
+#define VDSR                    0x00048
+#define VDER                    0x0004c
+#define HCR                     0x00050
+#define HSWR                    0x00054
+#define VCR                     0x00058
+#define VSPR                    0x0005c
+#define EQWR                    0x00060
+#define SPWR                    0x00064
+#define CLAMPSR                 0x00070
+#define CLAMPWR                 0x00074
+#define DESR                    0x00078
+#define DEWR                    0x0007c
+
+/* -----------------------------------------------------------------------------
+ * Display Attribute Registers
+ */
+
+#define CP1TR                   0x00080
+#define CP2TR                   0x00084
+#define CP3TR                   0x00088
+#define CP4TR                   0x0008c
+
+#define DOOR                    0x00090
+#define DOOR_RGB(r, g, b)       (((r) << 18) | ((g) << 10) | ((b) << 2))
+#define CDER                    0x00094
+#define CDER_RGB(r, g, b)       (((r) << 18) | ((g) << 10) | ((b) << 2))
+#define BPOR                    0x00098
+#define BPOR_RGB(r, g, b)       (((r) << 18) | ((g) << 10) | ((b) << 2))
+
+#define RINTOFSR                0x0009c
+
+#define DSHPR                   0x000c8
+#define DSHPR_CODE              (0x7776 << 16)
+#define DSHPR_PRIH              (0xa << 4)
+#define DSHPR_PRIL_BPP16        (0x8 << 0)
+#define DSHPR_PRIL_BPP32        (0x9 << 0)
+
+/* -----------------------------------------------------------------------------
+ * Display Plane Registers
+ */
+
+#define PLANE_OFF               0x00100
+
+#define PnMR                    0x00100 /* plane 1 */
+#define PnMR_VISL_VIN0          (0 << 26)       /* use Video Input 0 */
+#define PnMR_VISL_VIN1          (1 << 26)       /* use Video Input 1 */
+#define PnMR_VISL_VIN2          (2 << 26)       /* use Video Input 2 */
+#define PnMR_VISL_VIN3          (3 << 26)       /* use Video Input 3 */
+#define PnMR_YCDF_YUYV          (1 << 20)       /* YUYV format */
+#define PnMR_TC_R               (0 << 17)       /* Tranparent color is PnTC1R */
+#define PnMR_TC_CP              (1 << 17)       /* Tranparent color is color palette */
+#define PnMR_WAE                (1 << 16)       /* Wrap around Enable */
+#define PnMR_SPIM_TP            (0 << 12)       /* Transparent Color */
+#define PnMR_SPIM_ALP           (1 << 12)       /* Alpha Blending */
+#define PnMR_SPIM_EOR           (2 << 12)       /* EOR */
+#define PnMR_SPIM_TP_OFF        (1 << 14)       /* No Transparent Color */
+#define PnMR_CPSL_CP1           (0 << 8)        /* Color Palette selected 1 */
+#define PnMR_CPSL_CP2           (1 << 8)        /* Color Palette selected 2 */
+#define PnMR_CPSL_CP3           (2 << 8)        /* Color Palette selected 3 */
+#define PnMR_CPSL_CP4           (3 << 8)        /* Color Palette selected 4 */
+#define PnMR_DC                 (1 << 7)        /* Display Area Change */
+#define PnMR_BM_MD              (0 << 4)        /* Manual Display Change Mode */
+#define PnMR_BM_AR              (1 << 4)        /* Auto Rendering Mode */
+#define PnMR_BM_AD              (2 << 4)        /* Auto Display Change Mode */
+#define PnMR_BM_VC              (3 << 4)        /* Video Capture Mode */
+#define PnMR_DDDF_8BPP          (0 << 0)        /* 8bit */
+#define PnMR_DDDF_16BPP         (1 << 0)        /* 16bit or 32bit */
+#define PnMR_DDDF_ARGB          (2 << 0)        /* ARGB */
+#define PnMR_DDDF_YC            (3 << 0)        /* YC */
+#define PnMR_DDDF_MASK          (3 << 0)
+
+#define PnMWR                   0x00104
+
+#define PnALPHAR                0x00108
+#define PnALPHAR_ABIT_1         (0 << 12)
+#define PnALPHAR_ABIT_0         (1 << 12)
+#define PnALPHAR_ABIT_X         (2 << 12)
+
+#define PnDSXR                  0x00110
+#define PnDSYR                  0x00114
+#define PnDPXR                  0x00118
+#define PnDPYR                  0x0011c
+
+#define PnDSA0R                 0x00120
+#define PnDSA1R                 0x00124
+#define PnDSA2R                 0x00128
+#define PnDSA_MASK              0xfffffff0
+
+#define PnSPXR                  0x00130
+#define PnSPYR                  0x00134
+#define PnWASPR                 0x00138
+#define PnWAMWR                 0x0013c
+
+#define PnBTR                   0x00140
+
+#define PnTC1R                  0x00144
+#define PnTC2R                  0x00148
+#define PnTC3R                  0x0014c
+#define PnTC3R_CODE             (0x66 << 24)
+
+#define PnMLR                   0x00150
+
+#define PnSWAPR                 0x00180
+#define PnSWAPR_DIGN            (1 << 4)
+#define PnSWAPR_SPQW            (1 << 3)
+#define PnSWAPR_SPLW            (1 << 2)
+#define PnSWAPR_SPWD            (1 << 1)
+#define PnSWAPR_SPBY            (1 << 0)
+
+#define PnDDCR                  0x00184
+#define PnDDCR_CODE             (0x7775 << 16)
+#define PnDDCR_LRGB1            (1 << 11)
+#define PnDDCR_LRGB0            (1 << 10)
+
+#define PnDDCR2                 0x00188
+#define PnDDCR2_CODE            (0x7776 << 16)
+#define PnDDCR2_NV21            (1 << 5)
+#define PnDDCR2_Y420            (1 << 4)
+#define PnDDCR2_DIVU            (1 << 1)
+#define PnDDCR2_DIVY            (1 << 0)
+
+#define PnDDCR4                 0x00190
+#define PnDDCR4_CODE            (0x7766 << 16)
+#define PnDDCR4_SDFS_RGB        (0 << 4)
+#define PnDDCR4_SDFS_YC         (5 << 4)
+#define PnDDCR4_SDFS_MASK       (7 << 4)
+#define PnDDCR4_EDF_NONE        (0 << 0)
+#define PnDDCR4_EDF_ARGB8888    (1 << 0)
+#define PnDDCR4_EDF_RGB888      (2 << 0)
+#define PnDDCR4_EDF_RGB666      (3 << 0)
+#define PnDDCR4_EDF_MASK        (7 << 0)
+
+#define APnMR                   0x0a100
+#define APnMR_WAE               (1 << 16)       /* Wrap around Enable */
+#define APnMR_DC                (1 << 7)        /* Display Area Change */
+#define APnMR_BM_MD             (0 << 4)        /* Manual Display Change Mode */
+#define APnMR_BM_AD             (2 << 4)        /* Auto Display Change Mode */
+
+#define APnMWR                  0x0a104
+#define APnDSA0R                0x0a120
+#define APnDSA1R                0x0a124
+#define APnDSA2R                0x0a128
+#define APnMLR                  0x0a150
+
+/* -----------------------------------------------------------------------------
+ * Display Capture Registers
+ */
+
+#define DCMWR                   0x0c104
+#define DC2MWR                  0x0c204
+#define DCSAR                   0x0c120
+#define DC2SAR                  0x0c220
+#define DCMLR                   0x0c150
+#define DC2MLR                  0x0c250
+
+/* -----------------------------------------------------------------------------
+ * Color Palette Registers
+ */
+
+#define CP1_000R                0x01000
+#define CP1_255R                0x013fc
+#define CP2_000R                0x02000
+#define CP2_255R                0x023fc
+#define CP3_000R                0x03000
+#define CP3_255R                0x033fc
+#define CP4_000R                0x04000
+#define CP4_255R                0x043fc
+
+/* -----------------------------------------------------------------------------
+ * External Synchronization Control Registers
+ */
+
+#define ESCR                    0x10000
+#define ESCR2                   0x31000
+#define ESCR_DCLKOINV           (1 << 25)
+#define ESCR_DCLKSEL_DCLKIN     (0 << 20)
+#define ESCR_DCLKSEL_CLKS       (1 << 20)
+#define ESCR_DCLKSEL_MASK       (1 << 20)
+#define ESCR_DCLKDIS            (1 << 16)
+#define ESCR_SYNCSEL_OFF        (0 << 8)
+#define ESCR_SYNCSEL_EXVSYNC    (2 << 8)
+#define ESCR_SYNCSEL_EXHSYNC    (3 << 8)
+#define ESCR_FRQSEL_MASK        (0x3f << 0)
+
+#define OTAR                    0x10004
+#define OTAR2                   0x31004
+
+/* -----------------------------------------------------------------------------
+ * Dual Display Output Control Registers
+ */
+
+#define DORCR                   0x11000
+#define DORCR_PG2T              (1 << 30)
+#define DORCR_DK2S              (1 << 28)
+#define DORCR_PG2D_DS1          (0 << 24)
+#define DORCR_PG2D_DS2          (1 << 24)
+#define DORCR_PG2D_FIX0         (2 << 24)
+#define DORCR_PG2D_DOOR         (3 << 24)
+#define DORCR_PG2D_MASK         (3 << 24)
+#define DORCR_DR1D              (1 << 21)
+#define DORCR_PG1D_DS1          (0 << 16)
+#define DORCR_PG1D_DS2          (1 << 16)
+#define DORCR_PG1D_FIX0         (2 << 16)
+#define DORCR_PG1D_DOOR         (3 << 16)
+#define DORCR_PG1D_MASK         (3 << 16)
+#define DORCR_RGPV              (1 << 4)
+#define DORCR_DPRS              (1 << 0)
+
+#define DPTSR                   0x11004
+#define DPTSR_PnDK(n)           (1 << ((n) + 16))
+#define DPTSR_PnTS(n)           (1 << (n))
+
+#define DAPTSR                  0x11008
+#define DAPTSR_APnDK(n)         (1 << ((n) + 16))
+#define DAPTSR_APnTS(n)         (1 << (n))
+
+#define DS1PR                   0x11020
+#define DS2PR                   0x11024
+
+/* -----------------------------------------------------------------------------
+ * YC-RGB Conversion Coefficient Registers
+ */
+
+#define YNCR                    0x11080
+#define YNOR                    0x11084
+#define CRNOR                   0x11088
+#define CBNOR                   0x1108c
+#define RCRCR                   0x11090
+#define GCRCR                   0x11094
+#define GCBCR                   0x11098
+#define BCBCR                   0x1109c
+
+#endif /* __RCAR_DU_REGS_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vga.c b/drivers/gpu/drm/rcar-du/rcar_du_vga.c
new file mode 100644
index 000000000000..327289ec380d
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vga.c
@@ -0,0 +1,149 @@
+/*
+ * rcar_du_vga.c  --  R-Car Display Unit VGA DAC and Connector
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+#include "rcar_du_vga.h"
+
+/* -----------------------------------------------------------------------------
+ * Connector
+ */
+
+static int rcar_du_vga_connector_get_modes(struct drm_connector *connector)
+{
+        return 0;
+}
+
+static int rcar_du_vga_connector_mode_valid(struct drm_connector *connector,
+                                            struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs connector_helper_funcs = {
+        .get_modes = rcar_du_vga_connector_get_modes,
+        .mode_valid = rcar_du_vga_connector_mode_valid,
+        .best_encoder = rcar_du_connector_best_encoder,
+};
+
+static void rcar_du_vga_connector_destroy(struct drm_connector *connector)
+{
+        drm_sysfs_connector_remove(connector);
+        drm_connector_cleanup(connector);
+}
+
+static enum drm_connector_status
+rcar_du_vga_connector_detect(struct drm_connector *connector, bool force)
+{
+        return connector_status_unknown;
+}
+
+static const struct drm_connector_funcs connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .detect = rcar_du_vga_connector_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = rcar_du_vga_connector_destroy,
+};
+
+static int rcar_du_vga_connector_init(struct rcar_du_device *rcdu,
+                                      struct rcar_du_encoder *renc)
+{
+        struct rcar_du_connector *rcon;
+        struct drm_connector *connector;
+        int ret;
+
+        rcon = devm_kzalloc(rcdu->dev, sizeof(*rcon), GFP_KERNEL);
+        if (rcon == NULL)
+                return -ENOMEM;
+
+        connector = &rcon->connector;
+        connector->display_info.width_mm = 0;
+        connector->display_info.height_mm = 0;
+
+        ret = drm_connector_init(rcdu->ddev, connector, &connector_funcs,
+                                 DRM_MODE_CONNECTOR_VGA);
+        if (ret < 0)
+                return ret;
+
+        drm_connector_helper_add(connector, &connector_helper_funcs);
+        ret = drm_sysfs_connector_add(connector);
+        if (ret < 0)
+                return ret;
+
+        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+        drm_object_property_set_value(&connector->base,
+                rcdu->ddev->mode_config.dpms_property, DRM_MODE_DPMS_OFF);
+
+        ret = drm_mode_connector_attach_encoder(connector, &renc->encoder);
+        if (ret < 0)
+                return ret;
+
+        connector->encoder = &renc->encoder;
+        rcon->encoder = renc;
+
+        return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * Encoder
+ */
+
+static void rcar_du_vga_encoder_dpms(struct drm_encoder *encoder, int mode)
+{
+}
+
+static bool rcar_du_vga_encoder_mode_fixup(struct drm_encoder *encoder,
+                                           const struct drm_display_mode *mode,
+                                           struct drm_display_mode *adjusted_mode)
+{
+        return true;
+}
+
+static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
+        .dpms = rcar_du_vga_encoder_dpms,
+        .mode_fixup = rcar_du_vga_encoder_mode_fixup,
+        .prepare = rcar_du_encoder_mode_prepare,
+        .commit = rcar_du_encoder_mode_commit,
+        .mode_set = rcar_du_encoder_mode_set,
+};
+
+static const struct drm_encoder_funcs encoder_funcs = {
+        .destroy = drm_encoder_cleanup,
+};
+
+int rcar_du_vga_init(struct rcar_du_device *rcdu,
+                     const struct rcar_du_encoder_vga_data *data,
+                     unsigned int output)
+{
+        struct rcar_du_encoder *renc;
+        int ret;
+
+        renc = devm_kzalloc(rcdu->dev, sizeof(*renc), GFP_KERNEL);
+        if (renc == NULL)
+                return -ENOMEM;
+
+        renc->output = output;
+
+        ret = drm_encoder_init(rcdu->ddev, &renc->encoder, &encoder_funcs,
+                               DRM_MODE_ENCODER_DAC);
+        if (ret < 0)
+                return ret;
+
+        drm_encoder_helper_add(&renc->encoder, &encoder_helper_funcs);
+
+        return rcar_du_vga_connector_init(rcdu, renc);
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vga.h b/drivers/gpu/drm/rcar-du/rcar_du_vga.h
new file mode 100644
index 000000000000..66b4d2d7190d
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vga.h
@@ -0,0 +1,24 @@
+/*
+ * rcar_du_vga.h  --  R-Car Display Unit VGA DAC and Connector
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_VGA_H__
+#define __RCAR_DU_VGA_H__
+
+struct rcar_du_device;
+struct rcar_du_encoder_vga_data;
+
+int rcar_du_vga_init(struct rcar_du_device *rcdu,
+                     const struct rcar_du_encoder_vga_data *data,
+                     unsigned int output);
+
+#endif /* __RCAR_DU_VGA_H__ */
diff --git a/drivers/gpu/drm/savage/savage_bci.c b/drivers/gpu/drm/savage/savage_bci.c
index b55c1d661147..bd6b2cf508d5 100644
--- a/drivers/gpu/drm/savage/savage_bci.c
+++ b/drivers/gpu/drm/savage/savage_bci.c
@@ -570,9 +570,6 @@ int savage_driver_firstopen(struct drm_device *dev)
         unsigned int fb_rsrc, aper_rsrc;
         int ret = 0;
 
-        dev_priv->mtrr[0].handle = -1;
-        dev_priv->mtrr[1].handle = -1;
-        dev_priv->mtrr[2].handle = -1;
         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
                 fb_rsrc = 0;
                 fb_base = pci_resource_start(dev->pdev, 0);
@@ -584,21 +581,14 @@ int savage_driver_firstopen(struct drm_device *dev)
                 if (pci_resource_len(dev->pdev, 0) == 0x08000000) {
                         /* Don't make MMIO write-cobining! We need 3
                          * MTRRs. */
-                        dev_priv->mtrr[0].base = fb_base;
-                        dev_priv->mtrr[0].size = 0x01000000;
-                        dev_priv->mtrr[0].handle =
-                            drm_mtrr_add(dev_priv->mtrr[0].base,
-                                         dev_priv->mtrr[0].size, DRM_MTRR_WC);
-                        dev_priv->mtrr[1].base = fb_base + 0x02000000;
-                        dev_priv->mtrr[1].size = 0x02000000;
-                        dev_priv->mtrr[1].handle =
-                            drm_mtrr_add(dev_priv->mtrr[1].base,
-                                         dev_priv->mtrr[1].size, DRM_MTRR_WC);
-                        dev_priv->mtrr[2].base = fb_base + 0x04000000;
-                        dev_priv->mtrr[2].size = 0x04000000;
-                        dev_priv->mtrr[2].handle =
-                            drm_mtrr_add(dev_priv->mtrr[2].base,
-                                         dev_priv->mtrr[2].size, DRM_MTRR_WC);
+                        dev_priv->mtrr_handles[0] =
+                                arch_phys_wc_add(fb_base, 0x01000000);
+                        dev_priv->mtrr_handles[1] =
+                                arch_phys_wc_add(fb_base + 0x02000000,
+                                                 0x02000000);
+                        dev_priv->mtrr_handles[2] =
+                                arch_phys_wc_add(fb_base + 0x04000000,
+                                                0x04000000);
                 } else {
                         DRM_ERROR("strange pci_resource_len %08llx\n",
                                   (unsigned long long)
@@ -616,11 +606,9 @@ int savage_driver_firstopen(struct drm_device *dev)
                 if (pci_resource_len(dev->pdev, 1) == 0x08000000) {
                         /* Can use one MTRR to cover both fb and
                          * aperture. */
-                        dev_priv->mtrr[0].base = fb_base;
-                        dev_priv->mtrr[0].size = 0x08000000;
-                        dev_priv->mtrr[0].handle =
-                            drm_mtrr_add(dev_priv->mtrr[0].base,
-                                         dev_priv->mtrr[0].size, DRM_MTRR_WC);
+                        dev_priv->mtrr_handles[0] =
+                                arch_phys_wc_add(fb_base,
+                                                 0x08000000);
                 } else {
                         DRM_ERROR("strange pci_resource_len %08llx\n",
                                   (unsigned long long)
@@ -660,11 +648,10 @@ void savage_driver_lastclose(struct drm_device *dev)
         drm_savage_private_t *dev_priv = dev->dev_private;
         int i;
 
-        for (i = 0; i < 3; ++i)
-                if (dev_priv->mtrr[i].handle >= 0)
-                        drm_mtrr_del(dev_priv->mtrr[i].handle,
-                                 dev_priv->mtrr[i].base,
-                                 dev_priv->mtrr[i].size, DRM_MTRR_WC);
+        for (i = 0; i < 3; ++i) {
+                arch_phys_wc_del(dev_priv->mtrr_handles[i]);
+                dev_priv->mtrr_handles[i] = 0;
+        }
 }
 
 int savage_driver_unload(struct drm_device *dev)
diff --git a/drivers/gpu/drm/savage/savage_drv.h b/drivers/gpu/drm/savage/savage_drv.h
index df2aac6636f7..c05082a59f6f 100644
--- a/drivers/gpu/drm/savage/savage_drv.h
+++ b/drivers/gpu/drm/savage/savage_drv.h
@@ -160,10 +160,7 @@ typedef struct drm_savage_private {
         drm_local_map_t *cmd_dma;
         drm_local_map_t fake_dma;
 
-        struct {
-                int handle;
-                unsigned long base, size;
-        } mtrr[3];
+        int mtrr_handles[3];
 
         /* BCI and status-related stuff */
         volatile uint32_t *status_ptr, *bci_ptr;
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
index 7e7d52b2a2fc..ca498d151a76 100644
--- a/drivers/gpu/drm/shmobile/Kconfig
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -1,6 +1,6 @@
 config DRM_SHMOBILE
         tristate "DRM Support for SH Mobile"
-        depends on DRM && (SUPERH || ARCH_SHMOBILE)
+        depends on DRM && (ARM || SUPERH)
         select DRM_KMS_HELPER
         select DRM_KMS_CMA_HELPER
         select DRM_GEM_CMA_HELPER
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.c b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
index f6e0b5395051..edc10181f551 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_drv.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
@@ -90,7 +90,7 @@ static int shmob_drm_setup_clocks(struct shmob_drm_device *sdev,
                 return -EINVAL;
         }
 
-        clk = clk_get(sdev->dev, clkname);
+        clk = devm_clk_get(sdev->dev, clkname);
         if (IS_ERR(clk)) {
                 dev_err(sdev->dev, "cannot get dot clock %s\n", clkname);
                 return PTR_ERR(clk);
@@ -106,21 +106,12 @@ static int shmob_drm_setup_clocks(struct shmob_drm_device *sdev,
 
 static int shmob_drm_unload(struct drm_device *dev)
 {
-        struct shmob_drm_device *sdev = dev->dev_private;
-
         drm_kms_helper_poll_fini(dev);
         drm_mode_config_cleanup(dev);
         drm_vblank_cleanup(dev);
         drm_irq_uninstall(dev);
 
-        if (sdev->clock)
-                clk_put(sdev->clock);
-
-        if (sdev->mmio)
-                iounmap(sdev->mmio);
-
         dev->dev_private = NULL;
-        kfree(sdev);
 
         return 0;
 }
@@ -139,7 +130,7 @@ static int shmob_drm_load(struct drm_device *dev, unsigned long flags)
                 return -EINVAL;
         }
 
-        sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
+        sdev = devm_kzalloc(&pdev->dev, sizeof(*sdev), GFP_KERNEL);
         if (sdev == NULL) {
                 dev_err(dev->dev, "failed to allocate private data\n");
                 return -ENOMEM;
@@ -156,29 +147,28 @@ static int shmob_drm_load(struct drm_device *dev, unsigned long flags)
         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         if (res == NULL) {
                 dev_err(&pdev->dev, "failed to get memory resource\n");
-                ret = -EINVAL;
-                goto done;
+                return -EINVAL;
         }
 
-        sdev->mmio = ioremap_nocache(res->start, resource_size(res));
+        sdev->mmio = devm_ioremap_nocache(&pdev->dev, res->start,
+                                          resource_size(res));
         if (sdev->mmio == NULL) {
                 dev_err(&pdev->dev, "failed to remap memory resource\n");
-                ret = -ENOMEM;
-                goto done;
+                return -ENOMEM;
         }
 
         ret = shmob_drm_setup_clocks(sdev, pdata->clk_source);
         if (ret < 0)
-                goto done;
+                return ret;
 
         ret = shmob_drm_init_interface(sdev);
         if (ret < 0)
-                goto done;
+                return ret;
 
         ret = shmob_drm_modeset_init(sdev);
         if (ret < 0) {
                 dev_err(&pdev->dev, "failed to initialize mode setting\n");
-                goto done;
+                return ret;
         }
 
         for (i = 0; i < 4; ++i) {
@@ -273,7 +263,8 @@ static const struct file_operations shmob_drm_fops = {
 };
 
 static struct drm_driver shmob_drm_driver = {
-        .driver_features        = DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET,
+        .driver_features        = DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET
+                                | DRIVER_PRIME,
         .load                   = shmob_drm_load,
         .unload                 = shmob_drm_unload,
         .preclose               = shmob_drm_preclose,
@@ -283,6 +274,10 @@ static struct drm_driver shmob_drm_driver = {
         .disable_vblank         = shmob_drm_disable_vblank,
         .gem_free_object        = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
+        .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
+        .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
+        .gem_prime_import       = drm_gem_cma_dmabuf_import,
+        .gem_prime_export       = drm_gem_cma_dmabuf_export,
         .dumb_create            = drm_gem_cma_dumb_create,
         .dumb_map_offset        = drm_gem_cma_dumb_map_offset,
         .dumb_destroy           = drm_gem_cma_dumb_destroy,
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.c b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
index c291ee385b4f..fc0ef0ca7d04 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_kms.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
@@ -116,7 +116,7 @@ shmob_drm_fb_create(struct drm_device *dev, struct drm_file *file_priv,
         }
 
         if (mode_cmd->pitches[0] & 7 || mode_cmd->pitches[0] >= 65536) {
-                dev_dbg(dev->dev, "valid pitch value %u\n",
+                dev_dbg(dev->dev, "invalid pitch value %u\n",
                         mode_cmd->pitches[0]);
                 return ERR_PTR(-EINVAL);
         }
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_plane.c b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
index e1eb899b0288..060ae03e5f9b 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_plane.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
@@ -166,7 +166,7 @@ void shmob_drm_plane_setup(struct drm_plane *plane)
 {
         struct shmob_drm_plane *splane = to_shmob_plane(plane);
 
-        if (plane->fb == NULL || !plane->enabled)
+        if (plane->fb == NULL)
                 return;
 
         __shmob_drm_plane_setup(splane, plane->fb);
@@ -221,11 +221,8 @@ static int shmob_drm_plane_disable(struct drm_plane *plane)
 
 static void shmob_drm_plane_destroy(struct drm_plane *plane)
 {
-        struct shmob_drm_plane *splane = to_shmob_plane(plane);
-
         shmob_drm_plane_disable(plane);
         drm_plane_cleanup(plane);
-        kfree(splane);
 }
 
 static const struct drm_plane_funcs shmob_drm_plane_funcs = {
@@ -251,7 +248,7 @@ int shmob_drm_plane_create(struct shmob_drm_device *sdev, unsigned int index)
         struct shmob_drm_plane *splane;
         int ret;
 
-        splane = kzalloc(sizeof(*splane), GFP_KERNEL);
+        splane = devm_kzalloc(sdev->dev, sizeof(*splane), GFP_KERNEL);
         if (splane == NULL)
                 return -ENOMEM;
 
@@ -261,8 +258,6 @@ int shmob_drm_plane_create(struct shmob_drm_device *sdev, unsigned int index)
         ret = drm_plane_init(sdev->ddev, &splane->plane, 1,
                              &shmob_drm_plane_funcs, formats,
                              ARRAY_SIZE(formats), false);
-        if (ret < 0)
-                kfree(splane);
 
         return ret;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 5dd3c7d031d5..7418dcd986d3 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -42,7 +42,8 @@ struct tilcdc_crtc {
 
 static void unref_worker(struct work_struct *work)
 {
-        struct tilcdc_crtc *tilcdc_crtc = container_of(work, struct tilcdc_crtc, work);
+        struct tilcdc_crtc *tilcdc_crtc =
+                container_of(work, struct tilcdc_crtc, work);
         struct drm_device *dev = tilcdc_crtc->base.dev;
         struct drm_framebuffer *fb;
 
@@ -55,10 +56,12 @@ static void unref_worker(struct work_struct *work)
 static void set_scanout(struct drm_crtc *crtc, int n)
 {
         static const uint32_t base_reg[] = {
-                        LCDC_DMA_FB_BASE_ADDR_0_REG, LCDC_DMA_FB_BASE_ADDR_1_REG,
+                        LCDC_DMA_FB_BASE_ADDR_0_REG,
+                        LCDC_DMA_FB_BASE_ADDR_1_REG,
         };
         static const uint32_t ceil_reg[] = {
-                        LCDC_DMA_FB_CEILING_ADDR_0_REG, LCDC_DMA_FB_CEILING_ADDR_1_REG,
+                        LCDC_DMA_FB_CEILING_ADDR_0_REG,
+                        LCDC_DMA_FB_CEILING_ADDR_1_REG,
         };
         static const uint32_t stat[] = {
                         LCDC_END_OF_FRAME0, LCDC_END_OF_FRAME1,
@@ -194,7 +197,8 @@ static void tilcdc_crtc_dpms(struct drm_crtc *crtc, int mode)
                 tilcdc_crtc->frame_done = false;
                 stop(crtc);
 
-                /* if necessary wait for framedone irq which will still come
+                /*
+                 * if necessary wait for framedone irq which will still come
                  * before putting things to sleep..
                  */
                 if (priv->rev == 2) {
@@ -289,17 +293,24 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
         reg = tilcdc_read(dev, LCDC_RASTER_TIMING_2_REG) & ~0x000fff00;
         reg |= LCDC_AC_BIAS_FREQUENCY(info->ac_bias) |
                 LCDC_AC_BIAS_TRANSITIONS_PER_INT(info->ac_bias_intrpt);
+
+        /*
+         * subtract one from hfp, hbp, hsw because the hardware uses
+         * a value of 0 as 1
+         */
         if (priv->rev == 2) {
-                reg |= (hfp & 0x300) >> 8;
-                reg |= (hbp & 0x300) >> 4;
-                reg |= (hsw & 0x3c0) << 21;
+                /* clear bits we're going to set */
+                reg &= ~0x78000033;
+                reg |= ((hfp-1) & 0x300) >> 8;
+                reg |= ((hbp-1) & 0x300) >> 4;
+                reg |= ((hsw-1) & 0x3c0) << 21;
         }
         tilcdc_write(dev, LCDC_RASTER_TIMING_2_REG, reg);
 
         reg = (((mode->hdisplay >> 4) - 1) << 4) |
-                ((hbp & 0xff) << 24) |
-                ((hfp & 0xff) << 16) |
-                ((hsw & 0x3f) << 10);
+                (((hbp-1) & 0xff) << 24) |
+                (((hfp-1) & 0xff) << 16) |
+                (((hsw-1) & 0x3f) << 10);
         if (priv->rev == 2)
                 reg |= (((mode->hdisplay >> 4) - 1) & 0x40) >> 3;
         tilcdc_write(dev, LCDC_RASTER_TIMING_0_REG, reg);
@@ -307,9 +318,24 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
         reg = ((mode->vdisplay - 1) & 0x3ff) |
                 ((vbp & 0xff) << 24) |
                 ((vfp & 0xff) << 16) |
-                ((vsw & 0x3f) << 10);
+                (((vsw-1) & 0x3f) << 10);
         tilcdc_write(dev, LCDC_RASTER_TIMING_1_REG, reg);
 
+        /*
+         * be sure to set Bit 10 for the V2 LCDC controller,
+         * otherwise limited to 1024 pixels width, stopping
+         * 1920x1080 being suppoted.
+         */
+        if (priv->rev == 2) {
+                if ((mode->vdisplay - 1) & 0x400) {
+                        tilcdc_set(dev, LCDC_RASTER_TIMING_2_REG,
+                                LCDC_LPP_B10);
+                } else {
+                        tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG,
+                                LCDC_LPP_B10);
+                }
+        }
+
         /* Configure display type: */
         reg = tilcdc_read(dev, LCDC_RASTER_CTRL_REG) &
                 ~(LCDC_TFT_MODE | LCDC_MONO_8BIT_MODE | LCDC_MONOCHROME_MODE |
@@ -384,10 +410,6 @@ static int tilcdc_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         return 0;
 }
 
-static void tilcdc_crtc_load_lut(struct drm_crtc *crtc)
-{
-}
-
 static const struct drm_crtc_funcs tilcdc_crtc_funcs = {
                 .destroy        = tilcdc_crtc_destroy,
                 .set_config     = drm_crtc_helper_set_config,
@@ -401,7 +423,6 @@ static const struct drm_crtc_helper_funcs tilcdc_crtc_helper_funcs = {
                 .commit         = tilcdc_crtc_commit,
                 .mode_set       = tilcdc_crtc_mode_set,
                 .mode_set_base  = tilcdc_crtc_mode_set_base,
-                .load_lut       = tilcdc_crtc_load_lut,
 };
 
 int tilcdc_crtc_max_width(struct drm_crtc *crtc)
@@ -422,7 +443,12 @@ int tilcdc_crtc_mode_valid(struct drm_crtc *crtc, struct drm_display_mode *mode)
 {
         struct tilcdc_drm_private *priv = crtc->dev->dev_private;
         unsigned int bandwidth;
+        uint32_t hbp, hfp, hsw, vbp, vfp, vsw;
 
+        /*
+         * check to see if the width is within the range that
+         * the LCD Controller physically supports
+         */
         if (mode->hdisplay > tilcdc_crtc_max_width(crtc))
                 return MODE_VIRTUAL_X;
 
@@ -433,10 +459,70 @@ int tilcdc_crtc_mode_valid(struct drm_crtc *crtc, struct drm_display_mode *mode)
         if (mode->vdisplay > 2048)
                 return MODE_VIRTUAL_Y;
 
+        DBG("Processing mode %dx%d@%d with pixel clock %d",
+                mode->hdisplay, mode->vdisplay,
+                drm_mode_vrefresh(mode), mode->clock);
+
+        hbp = mode->htotal - mode->hsync_end;
+        hfp = mode->hsync_start - mode->hdisplay;
+        hsw = mode->hsync_end - mode->hsync_start;
+        vbp = mode->vtotal - mode->vsync_end;
+        vfp = mode->vsync_start - mode->vdisplay;
+        vsw = mode->vsync_end - mode->vsync_start;
+
+        if ((hbp-1) & ~0x3ff) {
+                DBG("Pruning mode: Horizontal Back Porch out of range");
+                return MODE_HBLANK_WIDE;
+        }
+
+        if ((hfp-1) & ~0x3ff) {
+                DBG("Pruning mode: Horizontal Front Porch out of range");
+                return MODE_HBLANK_WIDE;
+        }
+
+        if ((hsw-1) & ~0x3ff) {
+                DBG("Pruning mode: Horizontal Sync Width out of range");
+                return MODE_HSYNC_WIDE;
+        }
+
+        if (vbp & ~0xff) {
+                DBG("Pruning mode: Vertical Back Porch out of range");
+                return MODE_VBLANK_WIDE;
+        }
+
+        if (vfp & ~0xff) {
+                DBG("Pruning mode: Vertical Front Porch out of range");
+                return MODE_VBLANK_WIDE;
+        }
+
+        if ((vsw-1) & ~0x3f) {
+                DBG("Pruning mode: Vertical Sync Width out of range");
+                return MODE_VSYNC_WIDE;
+        }
+
+        /*
+         * some devices have a maximum allowed pixel clock
+         * configured from the DT
+         */
+        if (mode->clock > priv->max_pixelclock) {
+                DBG("Pruning mode: pixel clock too high");
+                return MODE_CLOCK_HIGH;
+        }
+
+        /*
+         * some devices further limit the max horizontal resolution
+         * configured from the DT
+         */
+        if (mode->hdisplay > priv->max_width)
+                return MODE_BAD_WIDTH;
+
         /* filter out modes that would require too much memory bandwidth: */
-        bandwidth = mode->hdisplay * mode->vdisplay * drm_mode_vrefresh(mode);
-        if (bandwidth > priv->max_bandwidth)
+        bandwidth = mode->hdisplay * mode->vdisplay *
+                drm_mode_vrefresh(mode);
+        if (bandwidth > priv->max_bandwidth) {
+                DBG("Pruning mode: exceeds defined bandwidth limit");
                 return MODE_BAD;
+        }
 
         return MODE_OK;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 2b5461bcd9fb..40b71da5a214 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -26,6 +26,7 @@
 #include "drm_fb_helper.h"
 
 static LIST_HEAD(module_list);
+static bool slave_probing;
 
 void tilcdc_module_init(struct tilcdc_module *mod, const char *name,
                 const struct tilcdc_module_ops *funcs)
@@ -41,6 +42,11 @@ void tilcdc_module_cleanup(struct tilcdc_module *mod)
         list_del(&mod->list);
 }
 
+void tilcdc_slave_probedefer(bool defered)
+{
+        slave_probing = defered;
+}
+
 static struct of_device_id tilcdc_of_match[];
 
 static struct drm_framebuffer *tilcdc_fb_create(struct drm_device *dev,
@@ -157,7 +163,9 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
         struct platform_device *pdev = dev->platformdev;
         struct device_node *node = pdev->dev.of_node;
         struct tilcdc_drm_private *priv;
+        struct tilcdc_module *mod;
         struct resource *res;
+        u32 bpp = 0;
         int ret;
 
         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
@@ -210,7 +218,20 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 #endif
 
         if (of_property_read_u32(node, "max-bandwidth", &priv->max_bandwidth))
-                priv->max_bandwidth = 1280 * 1024 * 60;
+                priv->max_bandwidth = TILCDC_DEFAULT_MAX_BANDWIDTH;
+
+        DBG("Maximum Bandwidth Value %d", priv->max_bandwidth);
+
+        if (of_property_read_u32(node, "ti,max-width", &priv->max_width))
+                priv->max_width = TILCDC_DEFAULT_MAX_WIDTH;
+
+        DBG("Maximum Horizontal Pixel Width Value %dpixels", priv->max_width);
+
+        if (of_property_read_u32(node, "ti,max-pixelclock",
+                                        &priv->max_pixelclock))
+                priv->max_pixelclock = TILCDC_DEFAULT_MAX_PIXELCLOCK;
+
+        DBG("Maximum Pixel Clock Value %dKHz", priv->max_pixelclock);
 
         pm_runtime_enable(dev->dev);
 
@@ -256,7 +277,15 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 
         platform_set_drvdata(pdev, dev);
 
-        priv->fbdev = drm_fbdev_cma_init(dev, 16,
+
+        list_for_each_entry(mod, &module_list, list) {
+                DBG("%s: preferred_bpp: %d", mod->name, mod->preferred_bpp);
+                bpp = mod->preferred_bpp;
+                if (bpp > 0)
+                        break;
+        }
+
+        priv->fbdev = drm_fbdev_cma_init(dev, bpp,
                         dev->mode_config.num_crtc,
                         dev->mode_config.num_connector);
 
@@ -557,6 +586,10 @@ static int tilcdc_pdev_probe(struct platform_device *pdev)
                 return -ENXIO;
         }
 
+        /* defer probing if slave is in deferred probing */
+        if (slave_probing == true)
+                return -EPROBE_DEFER;
+
         return drm_platform_init(&tilcdc_driver, pdev);
 }
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.h b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
index 8242b5a4307b..093803683b25 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
@@ -34,6 +34,18 @@
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 
+/* Defaulting to pixel clock defined on AM335x */
+#define TILCDC_DEFAULT_MAX_PIXELCLOCK  126000
+/* Defaulting to max width as defined on AM335x */
+#define TILCDC_DEFAULT_MAX_WIDTH  2048
+/*
+ * This may need some tweaking, but want to allow at least 1280x1024@60
+ * with optimized DDR & EMIF settings tweaked 1920x1080@24 appears to
+ * be supportable
+ */
+#define TILCDC_DEFAULT_MAX_BANDWIDTH  (1280*1024*60)
+
+
 struct tilcdc_drm_private {
         void __iomem *mmio;
 
@@ -43,6 +55,16 @@ struct tilcdc_drm_private {
 
         /* don't attempt resolutions w/ higher W * H * Hz: */
         uint32_t max_bandwidth;
+        /*
+         * Pixel Clock will be restricted to some value as
+         * defined in the device datasheet measured in KHz
+         */
+        uint32_t max_pixelclock;
+        /*
+         * Max allowable width is limited on a per device basis
+         * measured in pixels
+         */
+        uint32_t max_width;
 
         /* register contents saved across suspend/resume: */
         u32 saved_register[12];
@@ -89,12 +111,13 @@ struct tilcdc_module {
         const char *name;
         struct list_head list;
         const struct tilcdc_module_ops *funcs;
+        unsigned int preferred_bpp;
 };
 
 void tilcdc_module_init(struct tilcdc_module *mod, const char *name,
                 const struct tilcdc_module_ops *funcs);
 void tilcdc_module_cleanup(struct tilcdc_module *mod);
-
+void tilcdc_slave_probedefer(bool defered);
 
 /* Panel config that needs to be set in the crtc, but is not coming from
  * the mode timings.  The display module is expected to call
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
index 09176654fddb..86c67329b605 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
@@ -393,6 +393,8 @@ static int panel_probe(struct platform_device *pdev)
                 goto fail;
         }
 
+        mod->preferred_bpp = panel_mod->info->bpp;
+
         panel_mod->backlight = of_find_backlight_by_node(node);
         if (panel_mod->backlight)
                 dev_info(&pdev->dev, "found backlight\n");
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_regs.h b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
index 17fd1b45428a..1bf5e2553acc 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_regs.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
@@ -80,6 +80,7 @@
 #define LCDC_INVERT_PIXEL_CLOCK                  BIT(22)
 #define LCDC_INVERT_HSYNC                        BIT(21)
 #define LCDC_INVERT_VSYNC                        BIT(20)
+#define LCDC_LPP_B10                             BIT(26)
 
 /* LCDC Block */
 #define LCDC_PID_REG                             0x0
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_slave.c b/drivers/gpu/drm/tilcdc/tilcdc_slave.c
index db1d2fc9dfb5..dfffaf014022 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_slave.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_slave.c
@@ -298,6 +298,7 @@ static int slave_probe(struct platform_device *pdev)
         struct tilcdc_module *mod;
         struct pinctrl *pinctrl;
         uint32_t i2c_phandle;
+        struct i2c_adapter *slavei2c;
         int ret = -EINVAL;
 
         /* bail out early if no DT data: */
@@ -306,42 +307,48 @@ static int slave_probe(struct platform_device *pdev)
                 return -ENXIO;
         }
 
-        slave_mod = kzalloc(sizeof(*slave_mod), GFP_KERNEL);
-        if (!slave_mod)
-                return -ENOMEM;
-
-        mod = &slave_mod->base;
-
-        tilcdc_module_init(mod, "slave", &slave_module_ops);
-
-        pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
-        if (IS_ERR(pinctrl))
-                dev_warn(&pdev->dev, "pins are not configured\n");
-
+        /* Bail out early if i2c not specified */
         if (of_property_read_u32(node, "i2c", &i2c_phandle)) {
                 dev_err(&pdev->dev, "could not get i2c bus phandle\n");
-                goto fail;
+                return ret;
         }
 
         i2c_node = of_find_node_by_phandle(i2c_phandle);
         if (!i2c_node) {
                 dev_err(&pdev->dev, "could not get i2c bus node\n");
-                goto fail;
+                return ret;
         }
 
-        slave_mod->i2c = of_find_i2c_adapter_by_node(i2c_node);
-        if (!slave_mod->i2c) {
+        /* but defer the probe if it can't be initialized it might come later */
+        slavei2c = of_find_i2c_adapter_by_node(i2c_node);
+        of_node_put(i2c_node);
+
+        if (!slavei2c) {
+                ret = -EPROBE_DEFER;
+                tilcdc_slave_probedefer(true);
                 dev_err(&pdev->dev, "could not get i2c\n");
-                goto fail;
+                return ret;
         }
 
-        of_node_put(i2c_node);
+        slave_mod = kzalloc(sizeof(*slave_mod), GFP_KERNEL);
+        if (!slave_mod)
+                return -ENOMEM;
 
-        return 0;
+        mod = &slave_mod->base;
 
-fail:
-        slave_destroy(mod);
-        return ret;
+        mod->preferred_bpp = slave_info.bpp;
+
+        slave_mod->i2c = slavei2c;
+
+        tilcdc_module_init(mod, "slave", &slave_module_ops);
+
+        pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
+        if (IS_ERR(pinctrl))
+                dev_warn(&pdev->dev, "pins are not configured\n");
+
+        tilcdc_slave_probedefer(false);
+
+        return 0;
 }
 
 static int slave_remove(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
index a36788fbcd98..925c7cddeff9 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
@@ -354,6 +354,8 @@ static int tfp410_probe(struct platform_device *pdev)
                 goto fail;
         }
 
+        mod->preferred_bpp = dvi_info.bpp;
+
         i2c_node = of_find_node_by_phandle(i2c_phandle);
         if (!i2c_node) {
                 dev_err(&pdev->dev, "could not get i2c bus node\n");
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 9b07b7d44a58..cb9dd674670c 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -150,6 +150,9 @@ static void ttm_bo_release_list(struct kref *list_kref)
         if (bo->ttm)
                 ttm_tt_destroy(bo->ttm);
         atomic_dec(&bo->glob->bo_count);
+        if (bo->resv == &bo->ttm_resv)
+                reservation_object_fini(&bo->ttm_resv);
+
         if (bo->destroy)
                 bo->destroy(bo);
         else {
@@ -158,24 +161,12 @@ static void ttm_bo_release_list(struct kref *list_kref)
         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
 }
 
-static int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo,
-                                  bool interruptible)
-{
-        if (interruptible) {
-                return wait_event_interruptible(bo->event_queue,
-                                               !ttm_bo_is_reserved(bo));
-        } else {
-                wait_event(bo->event_queue, !ttm_bo_is_reserved(bo));
-                return 0;
-        }
-}
-
 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 {
         struct ttm_bo_device *bdev = bo->bdev;
         struct ttm_mem_type_manager *man;
 
-        BUG_ON(!ttm_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->resv->lock.base);
 
         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 
@@ -191,6 +182,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
                 }
         }
 }
+EXPORT_SYMBOL(ttm_bo_add_to_lru);
 
 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 {
@@ -213,71 +205,6 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
         return put_count;
 }
 
-int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,
-                          bool interruptible,
-                          bool no_wait, bool use_sequence, uint32_t sequence)
-{
-        int ret;
-
-        while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {
-                /**
-                 * Deadlock avoidance for multi-bo reserving.
-                 */
-                if (use_sequence && bo->seq_valid) {
-                        /**
-                         * We've already reserved this one.
-                         */
-                        if (unlikely(sequence == bo->val_seq))
-                                return -EDEADLK;
-                        /**
-                         * Already reserved by a thread that will not back
-                         * off for us. We need to back off.
-                         */
-                        if (unlikely(sequence - bo->val_seq < (1 << 31)))
-                                return -EAGAIN;
-                }
-
-                if (no_wait)
-                        return -EBUSY;
-
-                ret = ttm_bo_wait_unreserved(bo, interruptible);
-
-                if (unlikely(ret))
-                        return ret;
-        }
-
-        if (use_sequence) {
-                bool wake_up = false;
-                /**
-                 * Wake up waiters that may need to recheck for deadlock,
-                 * if we decreased the sequence number.
-                 */
-                if (unlikely((bo->val_seq - sequence < (1 << 31))
-                             || !bo->seq_valid))
-                        wake_up = true;
-
-                /*
-                 * In the worst case with memory ordering these values can be
-                 * seen in the wrong order. However since we call wake_up_all
-                 * in that case, this will hopefully not pose a problem,
-                 * and the worst case would only cause someone to accidentally
-                 * hit -EAGAIN in ttm_bo_reserve when they see old value of
-                 * val_seq. However this would only happen if seq_valid was
-                 * written before val_seq was, and just means some slightly
-                 * increased cpu usage
-                 */
-                bo->val_seq = sequence;
-                bo->seq_valid = true;
-                if (wake_up)
-                        wake_up_all(&bo->event_queue);
-        } else {
-                bo->seq_valid = false;
-        }
-
-        return 0;
-}
-EXPORT_SYMBOL(ttm_bo_reserve);
-
 static void ttm_bo_ref_bug(struct kref *list_kref)
 {
         BUG();
@@ -290,89 +217,16 @@ void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
                  (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
 }
 
-int ttm_bo_reserve(struct ttm_buffer_object *bo,
-                   bool interruptible,
-                   bool no_wait, bool use_sequence, uint32_t sequence)
-{
-        struct ttm_bo_global *glob = bo->glob;
-        int put_count = 0;
-        int ret;
-
-        ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence,
-                                   sequence);
-        if (likely(ret == 0)) {
-                spin_lock(&glob->lru_lock);
-                put_count = ttm_bo_del_from_lru(bo);
-                spin_unlock(&glob->lru_lock);
-                ttm_bo_list_ref_sub(bo, put_count, true);
-        }
-
-        return ret;
-}
-
-int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,
-                                  bool interruptible, uint32_t sequence)
-{
-        bool wake_up = false;
-        int ret;
-
-        while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {
-                WARN_ON(bo->seq_valid && sequence == bo->val_seq);
-
-                ret = ttm_bo_wait_unreserved(bo, interruptible);
-
-                if (unlikely(ret))
-                        return ret;
-        }
-
-        if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid)
-                wake_up = true;
-
-        /**
-         * Wake up waiters that may need to recheck for deadlock,
-         * if we decreased the sequence number.
-         */
-        bo->val_seq = sequence;
-        bo->seq_valid = true;
-        if (wake_up)
-                wake_up_all(&bo->event_queue);
-
-        return 0;
-}
-
-int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,
-                            bool interruptible, uint32_t sequence)
-{
-        struct ttm_bo_global *glob = bo->glob;
-        int put_count, ret;
-
-        ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence);
-        if (likely(!ret)) {
-                spin_lock(&glob->lru_lock);
-                put_count = ttm_bo_del_from_lru(bo);
-                spin_unlock(&glob->lru_lock);
-                ttm_bo_list_ref_sub(bo, put_count, true);
-        }
-        return ret;
-}
-EXPORT_SYMBOL(ttm_bo_reserve_slowpath);
-
-void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
-{
-        ttm_bo_add_to_lru(bo);
-        atomic_set(&bo->reserved, 0);
-        wake_up_all(&bo->event_queue);
-}
-
-void ttm_bo_unreserve(struct ttm_buffer_object *bo)
+void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
 {
-        struct ttm_bo_global *glob = bo->glob;
+        int put_count;
 
-        spin_lock(&glob->lru_lock);
-        ttm_bo_unreserve_locked(bo);
-        spin_unlock(&glob->lru_lock);
+        spin_lock(&bo->glob->lru_lock);
+        put_count = ttm_bo_del_from_lru(bo);
+        spin_unlock(&bo->glob->lru_lock);
+        ttm_bo_list_ref_sub(bo, put_count, true);
 }
-EXPORT_SYMBOL(ttm_bo_unreserve);
+EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 
 /*
  * Call bo->mutex locked.
@@ -544,17 +398,7 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
         }
         ttm_bo_mem_put(bo, &bo->mem);
 
-        atomic_set(&bo->reserved, 0);
-        wake_up_all(&bo->event_queue);
-
-        /*
-         * Since the final reference to this bo may not be dropped by
-         * the current task we have to put a memory barrier here to make
-         * sure the changes done in this function are always visible.
-         *
-         * This function only needs protection against the final kref_put.
-         */
-        smp_mb__before_atomic_dec();
+        ww_mutex_unlock (&bo->resv->lock);
 }
 
 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
@@ -586,10 +430,8 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
         spin_unlock(&bdev->fence_lock);
 
-        if (!ret) {
-                atomic_set(&bo->reserved, 0);
-                wake_up_all(&bo->event_queue);
-        }
+        if (!ret)
+                ww_mutex_unlock(&bo->resv->lock);
 
         kref_get(&bo->list_kref);
         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
@@ -639,8 +481,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
                 spin_unlock(&bdev->fence_lock);
 
-                atomic_set(&bo->reserved, 0);
-                wake_up_all(&bo->event_queue);
+                ww_mutex_unlock(&bo->resv->lock);
                 spin_unlock(&glob->lru_lock);
 
                 ret = driver->sync_obj_wait(sync_obj, false, interruptible);
@@ -678,8 +519,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
                 spin_unlock(&bdev->fence_lock);
 
         if (ret || unlikely(list_empty(&bo->ddestroy))) {
-                atomic_set(&bo->reserved, 0);
-                wake_up_all(&bo->event_queue);
+                ww_mutex_unlock(&bo->resv->lock);
                 spin_unlock(&glob->lru_lock);
                 return ret;
         }
@@ -831,7 +671,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
                 goto out;
         }
 
-        BUG_ON(!ttm_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->resv->lock.base);
 
         evict_mem = bo->mem;
         evict_mem.mm_node = NULL;
@@ -1121,7 +961,7 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
         struct ttm_mem_reg mem;
         struct ttm_bo_device *bdev = bo->bdev;
 
-        BUG_ON(!ttm_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->resv->lock.base);
 
         /*
          * FIXME: It's possible to pipeline buffer moves.
@@ -1180,7 +1020,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
 {
         int ret;
 
-        BUG_ON(!ttm_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->resv->lock.base);
         /* Check that range is valid */
         if (placement->lpfn || placement->fpfn)
                 if (placement->fpfn > placement->lpfn ||
@@ -1239,6 +1079,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         int ret = 0;
         unsigned long num_pages;
         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
+        bool locked;
 
         ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
         if (ret) {
@@ -1265,8 +1106,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         kref_init(&bo->kref);
         kref_init(&bo->list_kref);
         atomic_set(&bo->cpu_writers, 0);
-        atomic_set(&bo->reserved, 1);
-        init_waitqueue_head(&bo->event_queue);
         INIT_LIST_HEAD(&bo->lru);
         INIT_LIST_HEAD(&bo->ddestroy);
         INIT_LIST_HEAD(&bo->swap);
@@ -1284,37 +1123,34 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         bo->mem.bus.io_reserved_count = 0;
         bo->priv_flags = 0;
         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
-        bo->seq_valid = false;
         bo->persistent_swap_storage = persistent_swap_storage;
         bo->acc_size = acc_size;
         bo->sg = sg;
+        bo->resv = &bo->ttm_resv;
+        reservation_object_init(bo->resv);
         atomic_inc(&bo->glob->bo_count);
 
         ret = ttm_bo_check_placement(bo, placement);
-        if (unlikely(ret != 0))
-                goto out_err;
 
         /*
          * For ttm_bo_type_device buffers, allocate
          * address space from the device.
          */
-        if (bo->type == ttm_bo_type_device ||
-            bo->type == ttm_bo_type_sg) {
+        if (likely(!ret) &&
+            (bo->type == ttm_bo_type_device ||
+             bo->type == ttm_bo_type_sg))
                 ret = ttm_bo_setup_vm(bo);
-                if (ret)
-                        goto out_err;
-        }
 
-        ret = ttm_bo_validate(bo, placement, interruptible, false);
-        if (ret)
-                goto out_err;
+        locked = ww_mutex_trylock(&bo->resv->lock);
+        WARN_ON(!locked);
 
-        ttm_bo_unreserve(bo);
-        return 0;
+        if (likely(!ret))
+                ret = ttm_bo_validate(bo, placement, interruptible, false);
 
-out_err:
         ttm_bo_unreserve(bo);
-        ttm_bo_unref(&bo);
+
+        if (unlikely(ret))
+                ttm_bo_unref(&bo);
 
         return ret;
 }
@@ -1619,9 +1455,7 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
                 goto out_no_sys;
 
         bdev->addr_space_rb = RB_ROOT;
-        ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
-        if (unlikely(ret != 0))
-                goto out_no_addr_mm;
+        drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
 
         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
         INIT_LIST_HEAD(&bdev->ddestroy);
@@ -1635,8 +1469,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
         mutex_unlock(&glob->device_list_mutex);
 
         return 0;
-out_no_addr_mm:
-        ttm_bo_clean_mm(bdev, 0);
 out_no_sys:
         return ret;
 }
@@ -1927,8 +1759,7 @@ out:
          * already swapped buffer.
          */
 
-        atomic_set(&bo->reserved, 0);
-        wake_up_all(&bo->event_queue);
+        ww_mutex_unlock(&bo->resv->lock);
         kref_put(&bo->list_kref, ttm_bo_release_list);
         return ret;
 }
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
index 9212494e9072..e4367f91472a 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_manager.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -103,18 +103,12 @@ static int ttm_bo_man_init(struct ttm_mem_type_manager *man,
                            unsigned long p_size)
 {
         struct ttm_range_manager *rman;
-        int ret;
 
         rman = kzalloc(sizeof(*rman), GFP_KERNEL);
         if (!rman)
                 return -ENOMEM;
 
-        ret = drm_mm_init(&rman->mm, 0, p_size);
-        if (ret) {
-                kfree(rman);
-                return ret;
-        }
-
+        drm_mm_init(&rman->mm, 0, p_size);
         spin_lock_init(&rman->lock);
         man->priv = rman;
         return 0;
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index af894584dd90..319cf4127c5b 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -433,6 +433,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
         struct ttm_buffer_object *fbo;
         struct ttm_bo_device *bdev = bo->bdev;
         struct ttm_bo_driver *driver = bdev->driver;
+        int ret;
 
         fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
         if (!fbo)
@@ -445,7 +446,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
          * TODO: Explicit member copy would probably be better here.
          */
 
-        init_waitqueue_head(&fbo->event_queue);
         INIT_LIST_HEAD(&fbo->ddestroy);
         INIT_LIST_HEAD(&fbo->lru);
         INIT_LIST_HEAD(&fbo->swap);
@@ -463,6 +463,10 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
         kref_init(&fbo->kref);
         fbo->destroy = &ttm_transfered_destroy;
         fbo->acc_size = 0;
+        fbo->resv = &fbo->ttm_resv;
+        reservation_object_init(fbo->resv);
+        ret = ww_mutex_trylock(&fbo->resv->lock);
+        WARN_ON(!ret);
 
         *new_obj = fbo;
         return 0;
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index 7b90def15674..6c911789ae5c 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -32,7 +32,8 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 
-static void ttm_eu_backoff_reservation_locked(struct list_head *list)
+static void ttm_eu_backoff_reservation_locked(struct list_head *list,
+                                              struct ww_acquire_ctx *ticket)
 {
         struct ttm_validate_buffer *entry;
 
@@ -41,14 +42,12 @@ static void ttm_eu_backoff_reservation_locked(struct list_head *list)
                 if (!entry->reserved)
                         continue;
 
+                entry->reserved = false;
                 if (entry->removed) {
                         ttm_bo_add_to_lru(bo);
                         entry->removed = false;
-
                 }
-                entry->reserved = false;
-                atomic_set(&bo->reserved, 0);
-                wake_up_all(&bo->event_queue);
+                ww_mutex_unlock(&bo->resv->lock);
         }
 }
 
@@ -82,7 +81,8 @@ static void ttm_eu_list_ref_sub(struct list_head *list)
         }
 }
 
-void ttm_eu_backoff_reservation(struct list_head *list)
+void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket,
+                                struct list_head *list)
 {
         struct ttm_validate_buffer *entry;
         struct ttm_bo_global *glob;
@@ -93,7 +93,8 @@ void ttm_eu_backoff_reservation(struct list_head *list)
         entry = list_first_entry(list, struct ttm_validate_buffer, head);
         glob = entry->bo->glob;
         spin_lock(&glob->lru_lock);
-        ttm_eu_backoff_reservation_locked(list);
+        ttm_eu_backoff_reservation_locked(list, ticket);
+        ww_acquire_fini(ticket);
         spin_unlock(&glob->lru_lock);
 }
 EXPORT_SYMBOL(ttm_eu_backoff_reservation);
@@ -110,12 +111,12 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation);
  * buffers in different orders.
  */
 
-int ttm_eu_reserve_buffers(struct list_head *list)
+int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket,
+                           struct list_head *list)
 {
         struct ttm_bo_global *glob;
         struct ttm_validate_buffer *entry;
         int ret;
-        uint32_t val_seq;
 
         if (list_empty(list))
                 return 0;
@@ -129,9 +130,7 @@ int ttm_eu_reserve_buffers(struct list_head *list)
         entry = list_first_entry(list, struct ttm_validate_buffer, head);
         glob = entry->bo->glob;
 
-        spin_lock(&glob->lru_lock);
-        val_seq = entry->bo->bdev->val_seq++;
-
+        ww_acquire_init(ticket, &reservation_ww_class);
 retry:
         list_for_each_entry(entry, list, head) {
                 struct ttm_buffer_object *bo = entry->bo;
@@ -140,49 +139,34 @@ retry:
                 if (entry->reserved)
                         continue;
 
-                ret = ttm_bo_reserve_nolru(bo, true, true, true, val_seq);
-                switch (ret) {
-                case 0:
-                        break;
-                case -EBUSY:
-                        ttm_eu_del_from_lru_locked(list);
-                        spin_unlock(&glob->lru_lock);
-                        ret = ttm_bo_reserve_nolru(bo, true, false,
-                                                   true, val_seq);
-                        spin_lock(&glob->lru_lock);
-                        if (!ret)
-                                break;
-
-                        if (unlikely(ret != -EAGAIN))
-                                goto err;
 
-                        /* fallthrough */
-                case -EAGAIN:
-                        ttm_eu_backoff_reservation_locked(list);
+                ret = ttm_bo_reserve_nolru(bo, true, false, true, ticket);
 
-                        /*
-                         * temporarily increase sequence number every retry,
-                         * to prevent us from seeing our old reservation
-                         * sequence when someone else reserved the buffer,
-                         * but hasn't updated the seq_valid/seqno members yet.
+                if (ret == -EDEADLK) {
+                        /* uh oh, we lost out, drop every reservation and try
+                         * to only reserve this buffer, then start over if
+                         * this succeeds.
                          */
-                        val_seq = entry->bo->bdev->val_seq++;
-
+                        spin_lock(&glob->lru_lock);
+                        ttm_eu_backoff_reservation_locked(list, ticket);
                         spin_unlock(&glob->lru_lock);
                         ttm_eu_list_ref_sub(list);
-                        ret = ttm_bo_reserve_slowpath_nolru(bo, true, val_seq);
-                        if (unlikely(ret != 0))
-                                return ret;
-                        spin_lock(&glob->lru_lock);
+                        ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
+                                                               ticket);
+                        if (unlikely(ret != 0)) {
+                                if (ret == -EINTR)
+                                        ret = -ERESTARTSYS;
+                                goto err_fini;
+                        }
+
                         entry->reserved = true;
                         if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
                                 ret = -EBUSY;
                                 goto err;
                         }
                         goto retry;
-                default:
+                } else if (ret)
                         goto err;
-                }
 
                 entry->reserved = true;
                 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
@@ -191,21 +175,27 @@ retry:
                 }
         }
 
+        ww_acquire_done(ticket);
+        spin_lock(&glob->lru_lock);
         ttm_eu_del_from_lru_locked(list);
         spin_unlock(&glob->lru_lock);
         ttm_eu_list_ref_sub(list);
-
         return 0;
 
 err:
-        ttm_eu_backoff_reservation_locked(list);
+        spin_lock(&glob->lru_lock);
+        ttm_eu_backoff_reservation_locked(list, ticket);
         spin_unlock(&glob->lru_lock);
         ttm_eu_list_ref_sub(list);
+err_fini:
+        ww_acquire_done(ticket);
+        ww_acquire_fini(ticket);
         return ret;
 }
 EXPORT_SYMBOL(ttm_eu_reserve_buffers);
 
-void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
+void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
+                                 struct list_head *list, void *sync_obj)
 {
         struct ttm_validate_buffer *entry;
         struct ttm_buffer_object *bo;
@@ -228,11 +218,13 @@ void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
                 bo = entry->bo;
                 entry->old_sync_obj = bo->sync_obj;
                 bo->sync_obj = driver->sync_obj_ref(sync_obj);
-                ttm_bo_unreserve_locked(bo);
+                ttm_bo_add_to_lru(bo);
+                ww_mutex_unlock(&bo->resv->lock);
                 entry->reserved = false;
         }
         spin_unlock(&bdev->fence_lock);
         spin_unlock(&glob->lru_lock);
+        ww_acquire_fini(ticket);
 
         list_for_each_entry(entry, list, head) {
                 if (entry->old_sync_obj)
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index dc0c065f8d39..97e9d614700f 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -393,19 +393,6 @@ static struct fb_ops udlfb_ops = {
         .fb_release = udl_fb_release,
 };
 
-static void udl_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-                           u16 blue, int regno)
-{
-}
-
-static void udl_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-                             u16 *blue, int regno)
-{
-        *red = 0;
-        *green = 0;
-        *blue = 0;
-}
-
 static int udl_user_framebuffer_dirty(struct drm_framebuffer *fb,
                                       struct drm_file *file,
                                       unsigned flags, unsigned color,
@@ -558,8 +545,6 @@ out:
 }
 
 static struct drm_fb_helper_funcs udl_fb_helper_funcs = {
-        .gamma_set = udl_crtc_fb_gamma_set,
-        .gamma_get = udl_crtc_fb_gamma_get,
         .fb_probe = udlfb_create,
 };
 
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index e96d2349bd54..2ae1eb7d1635 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -363,10 +363,6 @@ static void udl_crtc_destroy(struct drm_crtc *crtc)
         kfree(crtc);
 }
 
-static void udl_load_lut(struct drm_crtc *crtc)
-{
-}
-
 static void udl_crtc_prepare(struct drm_crtc *crtc)
 {
 }
@@ -383,7 +379,6 @@ static struct drm_crtc_helper_funcs udl_helper_funcs = {
         .prepare = udl_crtc_prepare,
         .commit = udl_crtc_commit,
         .disable = udl_crtc_disable,
-        .load_lut = udl_load_lut,
 };
 
 static const struct drm_crtc_funcs udl_crtc_funcs = {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
index 5fae06ad7e25..d4e54fcc0acd 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
@@ -302,7 +302,7 @@ void vmw_bo_pin(struct ttm_buffer_object *bo, bool pin)
         uint32_t old_mem_type = bo->mem.mem_type;
         int ret;
 
-        BUG_ON(!ttm_bo_is_reserved(bo));
+        lockdep_assert_held(&bo->resv->lock.base);
         BUG_ON(old_mem_type != TTM_PL_VRAM &&
                old_mem_type != VMW_PL_GMR);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 07dfd823cc30..78e21649d48a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -565,8 +565,8 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
                 dev_priv->has_gmr = false;
         }
 
-        dev_priv->mmio_mtrr = drm_mtrr_add(dev_priv->mmio_start,
-                                           dev_priv->mmio_size, DRM_MTRR_WC);
+        dev_priv->mmio_mtrr = arch_phys_wc_add(dev_priv->mmio_start,
+                                               dev_priv->mmio_size);
 
         dev_priv->mmio_virt = ioremap_wc(dev_priv->mmio_start,
                                          dev_priv->mmio_size);
@@ -664,8 +664,7 @@ out_no_device:
 out_err4:
         iounmap(dev_priv->mmio_virt);
 out_err3:
-        drm_mtrr_del(dev_priv->mmio_mtrr, dev_priv->mmio_start,
-                     dev_priv->mmio_size, DRM_MTRR_WC);
+        arch_phys_wc_del(dev_priv->mmio_mtrr);
         if (dev_priv->has_gmr)
                 (void) ttm_bo_clean_mm(&dev_priv->bdev, VMW_PL_GMR);
         (void)ttm_bo_clean_mm(&dev_priv->bdev, TTM_PL_VRAM);
@@ -709,8 +708,7 @@ static int vmw_driver_unload(struct drm_device *dev)
 
         ttm_object_device_release(&dev_priv->tdev);
         iounmap(dev_priv->mmio_virt);
-        drm_mtrr_del(dev_priv->mmio_mtrr, dev_priv->mmio_start,
-                     dev_priv->mmio_size, DRM_MTRR_WC);
+        arch_phys_wc_del(dev_priv->mmio_mtrr);
         if (dev_priv->has_gmr)
                 (void)ttm_bo_clean_mm(&dev_priv->bdev, VMW_PL_GMR);
         (void)ttm_bo_clean_mm(&dev_priv->bdev, TTM_PL_VRAM);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 394e6476105b..599f6469a1eb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -1432,6 +1432,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
         struct vmw_fence_obj *fence = NULL;
         struct vmw_resource *error_resource;
         struct list_head resource_list;
+        struct ww_acquire_ctx ticket;
         uint32_t handle;
         void *cmd;
         int ret;
@@ -1488,7 +1489,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
         if (unlikely(ret != 0))
                 goto out_err;
 
-        ret = ttm_eu_reserve_buffers(&sw_context->validate_nodes);
+        ret = ttm_eu_reserve_buffers(&ticket, &sw_context->validate_nodes);
         if (unlikely(ret != 0))
                 goto out_err;
 
@@ -1537,7 +1538,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
                 DRM_ERROR("Fence submission error. Syncing.\n");
 
         vmw_resource_list_unreserve(&sw_context->resource_list, false);
-        ttm_eu_fence_buffer_objects(&sw_context->validate_nodes,
+        ttm_eu_fence_buffer_objects(&ticket, &sw_context->validate_nodes,
                                     (void *) fence);
 
         if (unlikely(dev_priv->pinned_bo != NULL &&
@@ -1570,7 +1571,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 out_err:
         vmw_resource_relocations_free(&sw_context->res_relocations);
         vmw_free_relocations(sw_context);
-        ttm_eu_backoff_reservation(&sw_context->validate_nodes);
+        ttm_eu_backoff_reservation(&ticket, &sw_context->validate_nodes);
         vmw_resource_list_unreserve(&sw_context->resource_list, true);
         vmw_clear_validations(sw_context);
         if (unlikely(dev_priv->pinned_bo != NULL &&
@@ -1644,6 +1645,7 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
         struct list_head validate_list;
         struct ttm_validate_buffer pinned_val, query_val;
         struct vmw_fence_obj *lfence = NULL;
+        struct ww_acquire_ctx ticket;
 
         if (dev_priv->pinned_bo == NULL)
                 goto out_unlock;
@@ -1657,7 +1659,7 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
         list_add_tail(&query_val.head, &validate_list);
 
         do {
-                ret = ttm_eu_reserve_buffers(&validate_list);
+                ret = ttm_eu_reserve_buffers(&ticket, &validate_list);
         } while (ret == -ERESTARTSYS);
 
         if (unlikely(ret != 0)) {
@@ -1684,7 +1686,7 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
                                                   NULL);
                 fence = lfence;
         }
-        ttm_eu_fence_buffer_objects(&validate_list, (void *) fence);
+        ttm_eu_fence_buffer_objects(&ticket, &validate_list, (void *) fence);
         if (lfence != NULL)
                 vmw_fence_obj_unreference(&lfence);
 
@@ -1696,7 +1698,7 @@ out_unlock:
         return;
 
 out_no_emit:
-        ttm_eu_backoff_reservation(&validate_list);
+        ttm_eu_backoff_reservation(&ticket, &validate_list);
 out_no_reserve:
         ttm_bo_unref(&query_val.bo);
         ttm_bo_unref(&pinned_val.bo);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 3e3c7ab33ca2..d4607b2530d6 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -174,7 +174,6 @@ int vmw_du_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
                            uint32_t handle, uint32_t width, uint32_t height)
 {
         struct vmw_private *dev_priv = vmw_priv(crtc->dev);
-        struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
         struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
         struct vmw_surface *surface = NULL;
         struct vmw_dma_buffer *dmabuf = NULL;
@@ -197,6 +196,8 @@ int vmw_du_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
         }
 
         if (handle) {
+                struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+
                 ret = vmw_user_lookup_handle(dev_priv, tfile,
                                              handle, &surface, &dmabuf);
                 if (ret) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index bc784254e78e..7953d1f90b63 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -958,13 +958,13 @@ void vmw_resource_unreserve(struct vmw_resource *res,
         if (new_backup && new_backup != res->backup) {
 
                 if (res->backup) {
-                        BUG_ON(!ttm_bo_is_reserved(&res->backup->base));
+                        lockdep_assert_held(&res->backup->base.resv->lock.base);
                         list_del_init(&res->mob_head);
                         vmw_dmabuf_unreference(&res->backup);
                 }
 
                 res->backup = vmw_dmabuf_reference(new_backup);
-                BUG_ON(!ttm_bo_is_reserved(&new_backup->base));
+                lockdep_assert_held(&new_backup->base.resv->lock.base);
                 list_add_tail(&res->mob_head, &new_backup->res_list);
         }
         if (new_backup)
@@ -990,9 +990,11 @@ void vmw_resource_unreserve(struct vmw_resource *res,
  * @val_buf:        On successful return contains data about the
  *                  reserved and validated backup buffer.
  */
-int vmw_resource_check_buffer(struct vmw_resource *res,
-                              bool interruptible,
-                              struct ttm_validate_buffer *val_buf)
+static int
+vmw_resource_check_buffer(struct vmw_resource *res,
+                          struct ww_acquire_ctx *ticket,
+                          bool interruptible,
+                          struct ttm_validate_buffer *val_buf)
 {
         struct list_head val_list;
         bool backup_dirty = false;
@@ -1007,7 +1009,7 @@ int vmw_resource_check_buffer(struct vmw_resource *res,
         INIT_LIST_HEAD(&val_list);
         val_buf->bo = ttm_bo_reference(&res->backup->base);
         list_add_tail(&val_buf->head, &val_list);
-        ret = ttm_eu_reserve_buffers(&val_list);
+        ret = ttm_eu_reserve_buffers(ticket, &val_list);
         if (unlikely(ret != 0))
                 goto out_no_reserve;
 
@@ -1025,7 +1027,7 @@ int vmw_resource_check_buffer(struct vmw_resource *res,
         return 0;
 
 out_no_validate:
-        ttm_eu_backoff_reservation(&val_list);
+        ttm_eu_backoff_reservation(ticket, &val_list);
 out_no_reserve:
         ttm_bo_unref(&val_buf->bo);
         if (backup_dirty)
@@ -1069,7 +1071,9 @@ int vmw_resource_reserve(struct vmw_resource *res, bool no_backup)
  *.
  * @val_buf:        Backup buffer information.
  */
-void vmw_resource_backoff_reservation(struct ttm_validate_buffer *val_buf)
+static void
+vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
+                                 struct ttm_validate_buffer *val_buf)
 {
         struct list_head val_list;
 
@@ -1078,7 +1082,7 @@ void vmw_resource_backoff_reservation(struct ttm_validate_buffer *val_buf)
 
         INIT_LIST_HEAD(&val_list);
         list_add_tail(&val_buf->head, &val_list);
-        ttm_eu_backoff_reservation(&val_list);
+        ttm_eu_backoff_reservation(ticket, &val_list);
         ttm_bo_unref(&val_buf->bo);
 }
 
@@ -1092,12 +1096,13 @@ int vmw_resource_do_evict(struct vmw_resource *res)
 {
         struct ttm_validate_buffer val_buf;
         const struct vmw_res_func *func = res->func;
+        struct ww_acquire_ctx ticket;
         int ret;
 
         BUG_ON(!func->may_evict);
 
         val_buf.bo = NULL;
-        ret = vmw_resource_check_buffer(res, true, &val_buf);
+        ret = vmw_resource_check_buffer(res, &ticket, true, &val_buf);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -1112,7 +1117,7 @@ int vmw_resource_do_evict(struct vmw_resource *res)
         res->backup_dirty = true;
         res->res_dirty = false;
 out_no_unbind:
-        vmw_resource_backoff_reservation(&val_buf);
+        vmw_resource_backoff_reservation(&ticket, &val_buf);
 
         return ret;
 }
diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h
index a1607d6e135b..790ddf114e58 100644
--- a/drivers/gpu/host1x/dev.h
+++ b/drivers/gpu/host1x/dev.h
@@ -73,7 +73,7 @@ struct host1x_syncpt_ops {
         void (*restore_wait_base)(struct host1x_syncpt *syncpt);
         void (*load_wait_base)(struct host1x_syncpt *syncpt);
         u32 (*load)(struct host1x_syncpt *syncpt);
-        void (*cpu_incr)(struct host1x_syncpt *syncpt);
+        int (*cpu_incr)(struct host1x_syncpt *syncpt);
         int (*patch_wait)(struct host1x_syncpt *syncpt, void *patch_addr);
 };
 
@@ -157,10 +157,10 @@ static inline u32 host1x_hw_syncpt_load(struct host1x *host,
         return host->syncpt_op->load(sp);
 }
 
-static inline void host1x_hw_syncpt_cpu_incr(struct host1x *host,
-                                             struct host1x_syncpt *sp)
+static inline int host1x_hw_syncpt_cpu_incr(struct host1x *host,
+                                            struct host1x_syncpt *sp)
 {
-        host->syncpt_op->cpu_incr(sp);
+        return host->syncpt_op->cpu_incr(sp);
 }
 
 static inline int host1x_hw_syncpt_patch_wait(struct host1x *host,
diff --git a/drivers/gpu/host1x/drm/dc.c b/drivers/gpu/host1x/drm/dc.c
index 8c04943f82e3..5360e5a57ecc 100644
--- a/drivers/gpu/host1x/drm/dc.c
+++ b/drivers/gpu/host1x/drm/dc.c
@@ -79,6 +79,9 @@ static int tegra_plane_disable(struct drm_plane *plane)
         struct tegra_plane *p = to_tegra_plane(plane);
         unsigned long value;
 
+        if (!plane->crtc)
+                return 0;
+
         value = WINDOW_A_SELECT << p->index;
         tegra_dc_writel(dc, value, DC_CMD_DISPLAY_WINDOW_HEADER);
 
@@ -140,6 +143,7 @@ static int tegra_dc_add_planes(struct drm_device *drm, struct tegra_dc *dc)
 static int tegra_dc_set_base(struct tegra_dc *dc, int x, int y,
                              struct drm_framebuffer *fb)
 {
+        unsigned int format = tegra_dc_format(fb->pixel_format);
         struct tegra_bo *bo = tegra_fb_get_plane(fb, 0);
         unsigned long value;
 
@@ -150,6 +154,7 @@ static int tegra_dc_set_base(struct tegra_dc *dc, int x, int y,
 
         tegra_dc_writel(dc, bo->paddr + value, DC_WINBUF_START_ADDR);
         tegra_dc_writel(dc, fb->pitches[0], DC_WIN_LINE_STRIDE);
+        tegra_dc_writel(dc, format, DC_WIN_COLOR_DEPTH);
 
         value = GENERAL_UPDATE | WIN_A_UPDATE;
         tegra_dc_writel(dc, value, DC_CMD_STATE_CONTROL);
diff --git a/drivers/gpu/host1x/drm/drm.c b/drivers/gpu/host1x/drm/drm.c
index 2b561c9118c6..e184b00faacd 100644
--- a/drivers/gpu/host1x/drm/drm.c
+++ b/drivers/gpu/host1x/drm/drm.c
@@ -148,6 +148,7 @@ int host1x_drm_init(struct host1x_drm *host1x, struct drm_device *drm)
                                 dev_err(host1x->dev,
                                         "DRM setup failed for %s: %d\n",
                                         dev_name(client->dev), err);
+                                mutex_unlock(&host1x->clients_lock);
                                 return err;
                         }
                 }
@@ -175,6 +176,7 @@ int host1x_drm_exit(struct host1x_drm *host1x)
                                 dev_err(host1x->dev,
                                         "DRM cleanup failed for %s: %d\n",
                                         dev_name(client->dev), err);
+                                mutex_unlock(&host1x->clients_lock);
                                 return err;
                         }
                 }
@@ -257,6 +259,13 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
         if (err < 0)
                 return err;
 
+        /*
+         * We don't use the drm_irq_install() helpers provided by the DRM
+         * core, so we need to set this manually in order to allow the
+         * DRM_IOCTL_WAIT_VBLANK to operate correctly.
+         */
+        drm->irq_enabled = 1;
+
         err = drm_vblank_init(drm, drm->mode_config.num_crtc);
         if (err < 0)
                 return err;
@@ -378,8 +387,7 @@ static int tegra_syncpt_incr(struct drm_device *drm, void *data,
         if (!sp)
                 return -EINVAL;
 
-        host1x_syncpt_incr(sp);
-        return 0;
+        return host1x_syncpt_incr(sp);
 }
 
 static int tegra_syncpt_wait(struct drm_device *drm, void *data,
@@ -605,7 +613,7 @@ static void tegra_debugfs_cleanup(struct drm_minor *minor)
 #endif
 
 struct drm_driver tegra_drm_driver = {
-        .driver_features = DRIVER_BUS_PLATFORM | DRIVER_MODESET | DRIVER_GEM,
+        .driver_features = DRIVER_MODESET | DRIVER_GEM,
         .load = tegra_drm_load,
         .unload = tegra_drm_unload,
         .open = tegra_drm_open,
diff --git a/drivers/gpu/host1x/drm/gr2d.c b/drivers/gpu/host1x/drm/gr2d.c
index 6a45ae090ee7..27ffcf15a4b4 100644
--- a/drivers/gpu/host1x/drm/gr2d.c
+++ b/drivers/gpu/host1x/drm/gr2d.c
@@ -84,7 +84,7 @@ static struct host1x_bo *host1x_bo_lookup(struct drm_device *drm,
 
         gem = drm_gem_object_lookup(drm, file, handle);
         if (!gem)
-                return 0;
+                return NULL;
 
         mutex_lock(&drm->struct_mutex);
         drm_gem_object_unreference(gem);
@@ -135,8 +135,10 @@ static int gr2d_submit(struct host1x_drm_context *context,
                         goto fail;
 
                 bo = host1x_bo_lookup(drm, file, cmdbuf.handle);
-                if (!bo)
+                if (!bo) {
+                        err = -ENOENT;
                         goto fail;
+                }
 
                 host1x_job_add_gather(job, bo, cmdbuf.words, cmdbuf.offset);
                 num_cmdbufs--;
@@ -158,8 +160,10 @@ static int gr2d_submit(struct host1x_drm_context *context,
                 reloc->cmdbuf = cmdbuf;
                 reloc->target = target;
 
-                if (!reloc->target || !reloc->cmdbuf)
+                if (!reloc->target || !reloc->cmdbuf) {
+                        err = -ENOENT;
                         goto fail;
+                }
         }
 
         err = copy_from_user(job->waitchk, waitchks,
@@ -281,7 +285,7 @@ static int gr2d_probe(struct platform_device *pdev)
         if (!gr2d->channel)
                 return -ENOMEM;
 
-        *syncpts = host1x_syncpt_request(dev, 0);
+        *syncpts = host1x_syncpt_request(dev, false);
         if (!(*syncpts)) {
                 host1x_channel_free(gr2d->channel);
                 return -ENOMEM;
diff --git a/drivers/gpu/host1x/hw/cdma_hw.c b/drivers/gpu/host1x/hw/cdma_hw.c
index 590b69d91dab..2ee4ad55c4db 100644
--- a/drivers/gpu/host1x/hw/cdma_hw.c
+++ b/drivers/gpu/host1x/hw/cdma_hw.c
@@ -44,7 +44,7 @@ static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
         u32 i;
 
         for (i = 0; i < syncpt_incrs; i++)
-                host1x_syncpt_cpu_incr(cdma->timeout.syncpt);
+                host1x_syncpt_incr(cdma->timeout.syncpt);
 
         /* after CPU incr, ensure shadow is up to date */
         host1x_syncpt_load(cdma->timeout.syncpt);
diff --git a/drivers/gpu/host1x/hw/syncpt_hw.c b/drivers/gpu/host1x/hw/syncpt_hw.c
index 61174990102a..0cf6095d3367 100644
--- a/drivers/gpu/host1x/hw/syncpt_hw.c
+++ b/drivers/gpu/host1x/hw/syncpt_hw.c
@@ -77,21 +77,19 @@ static u32 syncpt_load(struct host1x_syncpt *sp)
  * Write a cpu syncpoint increment to the hardware, without touching
  * the cache.
  */
-static void syncpt_cpu_incr(struct host1x_syncpt *sp)
+static int syncpt_cpu_incr(struct host1x_syncpt *sp)
 {
         struct host1x *host = sp->host;
         u32 reg_offset = sp->id / 32;
 
         if (!host1x_syncpt_client_managed(sp) &&
-            host1x_syncpt_idle(sp)) {
-                dev_err(host->dev, "Trying to increment syncpoint id %d beyond max\n",
-                        sp->id);
-                host1x_debug_dump(sp->host);
-                return;
-        }
+            host1x_syncpt_idle(sp))
+                return -EINVAL;
         host1x_sync_writel(host, BIT_MASK(sp->id),
                            HOST1X_SYNC_SYNCPT_CPU_INCR(reg_offset));
         wmb();
+
+        return 0;
 }
 
 /* remove a wait pointed to by patch_addr */
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index f665d679031c..cc807667d8f1 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -228,17 +228,15 @@ static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
         void *cmdbuf_page_addr = NULL;
 
         /* pin & patch the relocs for one gather */
-        while (i < job->num_relocs) {
+        for (i = 0; i < job->num_relocs; i++) {
                 struct host1x_reloc *reloc = &job->relocarray[i];
                 u32 reloc_addr = (job->reloc_addr_phys[i] +
                         reloc->target_offset) >> reloc->shift;
                 u32 *target;
 
                 /* skip all other gathers */
-                if (!(reloc->cmdbuf && cmdbuf == reloc->cmdbuf)) {
-                        i++;
+                if (cmdbuf != reloc->cmdbuf)
                         continue;
-                }
 
                 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) {
                         if (cmdbuf_page_addr)
@@ -257,9 +255,6 @@ static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 
                 target = cmdbuf_page_addr + (reloc->cmdbuf_offset & ~PAGE_MASK);
                 *target = reloc_addr;
-
-                /* mark this gather as handled */
-                reloc->cmdbuf = 0;
         }
 
         if (cmdbuf_page_addr)
@@ -268,15 +263,15 @@ static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
         return 0;
 }
 
-static int check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
+static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
                        unsigned int offset)
 {
         offset *= sizeof(u32);
 
         if (reloc->cmdbuf != cmdbuf || reloc->cmdbuf_offset != offset)
-                return -EINVAL;
+                return false;
 
-        return 0;
+        return true;
 }
 
 struct host1x_firewall {
@@ -307,10 +302,10 @@ static int check_mask(struct host1x_firewall *fw)
 
                 if (mask & 1) {
                         if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
-                                bool bad_reloc = check_reloc(fw->reloc,
-                                                             fw->cmdbuf_id,
-                                                             fw->offset);
-                                if (!fw->num_relocs || bad_reloc)
+                                if (!fw->num_relocs)
+                                        return -EINVAL;
+                                if (!check_reloc(fw->reloc, fw->cmdbuf_id,
+                                                 fw->offset))
                                         return -EINVAL;
                                 fw->reloc++;
                                 fw->num_relocs--;
@@ -330,14 +325,14 @@ static int check_incr(struct host1x_firewall *fw)
         u32 count = fw->count;
         u32 reg = fw->reg;
 
-        while (fw) {
+        while (count) {
                 if (fw->words == 0)
                         return -EINVAL;
 
                 if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
-                        bool bad_reloc = check_reloc(fw->reloc, fw->cmdbuf_id,
-                                                     fw->offset);
-                        if (!fw->num_relocs || bad_reloc)
+                        if (!fw->num_relocs)
+                                return -EINVAL;
+                        if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset))
                                 return -EINVAL;
                         fw->reloc++;
                         fw->num_relocs--;
@@ -361,9 +356,9 @@ static int check_nonincr(struct host1x_firewall *fw)
                         return -EINVAL;
 
                 if (is_addr_reg) {
-                        bool bad_reloc = check_reloc(fw->reloc, fw->cmdbuf_id,
-                                                     fw->offset);
-                        if (!fw->num_relocs || bad_reloc)
+                        if (!fw->num_relocs)
+                                return -EINVAL;
+                        if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset))
                                 return -EINVAL;
                         fw->reloc++;
                         fw->num_relocs--;
@@ -376,69 +371,58 @@ static int check_nonincr(struct host1x_firewall *fw)
         return 0;
 }
 
-static int validate(struct host1x_job *job, struct device *dev,
-                    struct host1x_job_gather *g)
+static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 {
-        u32 *cmdbuf_base;
+        u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
+                (g->offset / sizeof(u32));
         int err = 0;
-        struct host1x_firewall fw;
 
-        fw.job = job;
-        fw.dev = dev;
-        fw.reloc = job->relocarray;
-        fw.num_relocs = job->num_relocs;
-        fw.cmdbuf_id = g->bo;
-
-        fw.offset = 0;
-        fw.class = 0;
-
-        if (!job->is_addr_reg)
+        if (!fw->job->is_addr_reg)
                 return 0;
 
-        cmdbuf_base = host1x_bo_mmap(g->bo);
-        if (!cmdbuf_base)
-                return -ENOMEM;
+        fw->words = g->words;
+        fw->cmdbuf_id = g->bo;
+        fw->offset = 0;
 
-        fw.words = g->words;
-        while (fw.words && !err) {
-                u32 word = cmdbuf_base[fw.offset];
+        while (fw->words && !err) {
+                u32 word = cmdbuf_base[fw->offset];
                 u32 opcode = (word & 0xf0000000) >> 28;
 
-                fw.mask = 0;
-                fw.reg = 0;
-                fw.count = 0;
-                fw.words--;
-                fw.offset++;
+                fw->mask = 0;
+                fw->reg = 0;
+                fw->count = 0;
+                fw->words--;
+                fw->offset++;
 
                 switch (opcode) {
                 case 0:
-                        fw.class = word >> 6 & 0x3ff;
-                        fw.mask = word & 0x3f;
-                        fw.reg = word >> 16 & 0xfff;
-                        err = check_mask(&fw);
+                        fw->class = word >> 6 & 0x3ff;
+                        fw->mask = word & 0x3f;
+                        fw->reg = word >> 16 & 0xfff;
+                        err = check_mask(fw);
                         if (err)
                                 goto out;
                         break;
                 case 1:
-                        fw.reg = word >> 16 & 0xfff;
-                        fw.count = word & 0xffff;
-                        err = check_incr(&fw);
+                        fw->reg = word >> 16 & 0xfff;
+                        fw->count = word & 0xffff;
+                        err = check_incr(fw);
                         if (err)
                                 goto out;
                         break;
 
                 case 2:
-                        fw.reg = word >> 16 & 0xfff;
-                        fw.count = word & 0xffff;
-                        err = check_nonincr(&fw);
+                        fw->reg = word >> 16 & 0xfff;
+                        fw->count = word & 0xffff;
+                        err = check_nonincr(fw);
                         if (err)
                                 goto out;
                         break;
 
                 case 3:
-                        fw.mask = word & 0xffff;
-                        fw.reg = word >> 16 & 0xfff;
-                        err = check_mask(&fw);
+                        fw->mask = word & 0xffff;
+                        fw->reg = word >> 16 & 0xfff;
+                        err = check_mask(fw);
                         if (err)
                                 goto out;
                         break;
@@ -453,21 +437,26 @@ static int validate(struct host1x_job *job, struct device *dev,
         }
 
         /* No relocs should remain at this point */
-        if (fw.num_relocs)
+        if (fw->num_relocs)
                 err = -EINVAL;
 
 out:
-        host1x_bo_munmap(g->bo, cmdbuf_base);
-
         return err;
 }
 
 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 {
+        struct host1x_firewall fw;
         size_t size = 0;
         size_t offset = 0;
         int i;
 
+        fw.job = job;
+        fw.dev = dev;
+        fw.reloc = job->relocarray;
+        fw.num_relocs = job->num_relocs;
+        fw.class = 0;
+
         for (i = 0; i < job->num_gathers; i++) {
                 struct host1x_job_gather *g = &job->gathers[i];
                 size += g->words * sizeof(u32);
@@ -488,14 +477,19 @@ static inline int copy_gathers(struct host1x_job *job, struct device *dev)
                 struct host1x_job_gather *g = &job->gathers[i];
                 void *gather;
 
+                /* Copy the gather */
                 gather = host1x_bo_mmap(g->bo);
                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
                        g->words * sizeof(u32));
                 host1x_bo_munmap(g->bo, gather);
 
+                /* Store the location in the buffer */
                 g->base = job->gather_copy;
                 g->offset = offset;
-                g->bo = NULL;
+
+                /* Validate the job */
+                if (validate(&fw, g))
+                        return -EINVAL;
 
                 offset += g->words * sizeof(u32);
         }
@@ -540,20 +534,11 @@ int host1x_job_pin(struct host1x_job *job, struct device *dev)
                         if (job->gathers[j].bo == g->bo)
                                 job->gathers[j].handled = true;
 
-                err = 0;
-
-                if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL))
-                        err = validate(job, dev, g);
-
+                err = do_relocs(job, g->bo);
                 if (err)
-                        dev_err(dev, "Job invalid (err=%d)\n", err);
-
-                if (!err)
-                        err = do_relocs(job, g->bo);
-
-                if (!err)
-                        err = do_waitchks(job, host, g->bo);
+                        break;
 
+                err = do_waitchks(job, host, g->bo);
                 if (err)
                         break;
         }
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index 4b493453e805..409745b949db 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -32,7 +32,7 @@
 
 static struct host1x_syncpt *_host1x_syncpt_alloc(struct host1x *host,
                                                   struct device *dev,
-                                                  int client_managed)
+                                                  bool client_managed)
 {
         int i;
         struct host1x_syncpt *sp = host->syncpt;
@@ -40,7 +40,8 @@ static struct host1x_syncpt *_host1x_syncpt_alloc(struct host1x *host,
 
         for (i = 0; i < host->info->nb_pts && sp->name; i++, sp++)
                 ;
-        if (sp->dev)
+
+        if (i >= host->info->nb_pts)
                 return NULL;
 
         name = kasprintf(GFP_KERNEL, "%02d-%s", sp->id,
@@ -128,22 +129,11 @@ u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp)
 }
 
 /*
- * Write a cpu syncpoint increment to the hardware, without touching
- * the cache. Caller is responsible for host being powered.
- */
-void host1x_syncpt_cpu_incr(struct host1x_syncpt *sp)
-{
-        host1x_hw_syncpt_cpu_incr(sp->host, sp);
-}
-
-/*
  * Increment syncpoint value from cpu, updating cache
  */
-void host1x_syncpt_incr(struct host1x_syncpt *sp)
+int host1x_syncpt_incr(struct host1x_syncpt *sp)
 {
-        if (host1x_syncpt_client_managed(sp))
-                host1x_syncpt_incr_max(sp, 1);
-        host1x_syncpt_cpu_incr(sp);
+        return host1x_hw_syncpt_cpu_incr(sp->host, sp);
 }
 
 /*
@@ -331,7 +321,7 @@ int host1x_syncpt_init(struct host1x *host)
         host1x_syncpt_restore(host);
 
         /* Allocate sync point to use for clearing waits for expired fences */
-        host->nop_sp = _host1x_syncpt_alloc(host, NULL, 0);
+        host->nop_sp = _host1x_syncpt_alloc(host, NULL, false);
         if (!host->nop_sp)
                 return -ENOMEM;
 
@@ -339,7 +329,7 @@ int host1x_syncpt_init(struct host1x *host)
 }
 
 struct host1x_syncpt *host1x_syncpt_request(struct device *dev,
-                                            int client_managed)
+                                            bool client_managed)
 {
         struct host1x *host = dev_get_drvdata(dev->parent);
         return _host1x_syncpt_alloc(host, dev, client_managed);
@@ -353,7 +343,7 @@ void host1x_syncpt_free(struct host1x_syncpt *sp)
         kfree(sp->name);
         sp->dev = NULL;
         sp->name = NULL;
-        sp->client_managed = 0;
+        sp->client_managed = false;
 }
 
 void host1x_syncpt_deinit(struct host1x *host)
diff --git a/drivers/gpu/host1x/syncpt.h b/drivers/gpu/host1x/syncpt.h
index c99806130f2e..267c0b9d3647 100644
--- a/drivers/gpu/host1x/syncpt.h
+++ b/drivers/gpu/host1x/syncpt.h
@@ -36,7 +36,7 @@ struct host1x_syncpt {
         atomic_t max_val;
         u32 base_val;
         const char *name;
-        int client_managed;
+        bool client_managed;
         struct host1x *host;
         struct device *dev;
 
@@ -94,7 +94,7 @@ static inline bool host1x_syncpt_check_max(struct host1x_syncpt *sp, u32 real)
 }
 
 /* Return true if sync point is client managed. */
-static inline int host1x_syncpt_client_managed(struct host1x_syncpt *sp)
+static inline bool host1x_syncpt_client_managed(struct host1x_syncpt *sp)
 {
         return sp->client_managed;
 }
@@ -115,9 +115,6 @@ static inline bool host1x_syncpt_idle(struct host1x_syncpt *sp)
 /* Return pointer to struct denoting sync point id. */
 struct host1x_syncpt *host1x_syncpt_get(struct host1x *host, u32 id);
 
-/* Request incrementing a sync point. */
-void host1x_syncpt_cpu_incr(struct host1x_syncpt *sp);
-
 /* Load current value from hardware to the shadow register. */
 u32 host1x_syncpt_load(struct host1x_syncpt *sp);
 
@@ -133,8 +130,8 @@ void host1x_syncpt_restore(struct host1x *host);
 /* Read current wait base value into shadow register and return it. */
 u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp);
 
-/* Increment sync point and its max. */
-void host1x_syncpt_incr(struct host1x_syncpt *sp);
+/* Request incrementing a sync point. */
+int host1x_syncpt_incr(struct host1x_syncpt *sp);
 
 /* Indicate future operations by incrementing the sync point max. */
 u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs);
@@ -157,7 +154,7 @@ u32 host1x_syncpt_id(struct host1x_syncpt *sp);
 
 /* Allocate a sync point for a device. */
 struct host1x_syncpt *host1x_syncpt_request(struct device *dev,
-                int client_managed);
+                                            bool client_managed);
 
 /* Free a sync point. */
 void host1x_syncpt_free(struct host1x_syncpt *sp);
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index d6cbfe9df218..fa061d46527f 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -137,7 +137,7 @@ static const struct xpad_device {
         { 0x0738, 0x4540, "Mad Catz Beat Pad", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX },
         { 0x0738, 0x4556, "Mad Catz Lynx Wireless Controller", 0, XTYPE_XBOX },
         { 0x0738, 0x4716, "Mad Catz Wired Xbox 360 Controller", 0, XTYPE_XBOX360 },
-        { 0x0738, 0x4728, "Mad Catz Street Fighter IV FightPad", XTYPE_XBOX360 },
+        { 0x0738, 0x4728, "Mad Catz Street Fighter IV FightPad", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
         { 0x0738, 0x4738, "Mad Catz Wired Xbox 360 Controller (SFIV)", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
         { 0x0738, 0x6040, "Mad Catz Beat Pad Pro", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX },
         { 0x0738, 0xbeef, "Mad Catz JOYTECH NEO SE Advanced GamePad", XTYPE_XBOX360 },
diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig
index 62a2c0e4cc99..7ac9c9818d55 100644
--- a/drivers/input/keyboard/Kconfig
+++ b/drivers/input/keyboard/Kconfig
@@ -431,6 +431,7 @@ config KEYBOARD_TEGRA
 
 config KEYBOARD_OPENCORES
         tristate "OpenCores Keyboard Controller"
+        depends on HAS_IOMEM
         help
           Say Y here if you want to use the OpenCores Keyboard Controller
           http://www.opencores.org/project,keyboardcontroller
diff --git a/drivers/input/serio/Kconfig b/drivers/input/serio/Kconfig
index aebfe3ecb945..1bda828f4b55 100644
--- a/drivers/input/serio/Kconfig
+++ b/drivers/input/serio/Kconfig
@@ -205,6 +205,7 @@ config SERIO_XILINX_XPS_PS2
 
 config SERIO_ALTERA_PS2
         tristate "Altera UP PS/2 controller"
+        depends on HAS_IOMEM
         help
           Say Y here if you have Altera University Program PS/2 ports.
 
diff --git a/drivers/input/tablet/wacom_wac.c b/drivers/input/tablet/wacom_wac.c
index 518282da6d85..384fbcd0cee0 100644
--- a/drivers/input/tablet/wacom_wac.c
+++ b/drivers/input/tablet/wacom_wac.c
@@ -363,6 +363,7 @@ static int wacom_intuos_inout(struct wacom_wac *wacom)
                 case 0x140802: /* Intuos4/5 13HD/24HD Classic Pen */
                 case 0x160802: /* Cintiq 13HD Pro Pen */
                 case 0x180802: /* DTH2242 Pen */
+                case 0x100802: /* Intuos4/5 13HD/24HD General Pen */
                         wacom->tool[idx] = BTN_TOOL_PEN;
                         break;
 
@@ -401,6 +402,7 @@ static int wacom_intuos_inout(struct wacom_wac *wacom)
                 case 0x10080c: /* Intuos4/5 13HD/24HD Art Pen Eraser */
                 case 0x16080a: /* Cintiq 13HD Pro Pen Eraser */
                 case 0x18080a: /* DTH2242 Eraser */
+                case 0x10080a: /* Intuos4/5 13HD/24HD General Pen Eraser */
                         wacom->tool[idx] = BTN_TOOL_RUBBER;
                         break;
 
diff --git a/drivers/input/touchscreen/cyttsp_core.c b/drivers/input/touchscreen/cyttsp_core.c
index 8e60437ac85b..ae89d2609ab0 100644
--- a/drivers/input/touchscreen/cyttsp_core.c
+++ b/drivers/input/touchscreen/cyttsp_core.c
@@ -116,6 +116,15 @@ static int ttsp_send_command(struct cyttsp *ts, u8 cmd)
         return ttsp_write_block_data(ts, CY_REG_BASE, sizeof(cmd), &cmd);
 }
 
+static int cyttsp_handshake(struct cyttsp *ts)
+{
+        if (ts->pdata->use_hndshk)
+                return ttsp_send_command(ts,
+                                ts->xy_data.hst_mode ^ CY_HNDSHK_BIT);
+
+        return 0;
+}
+
 static int cyttsp_load_bl_regs(struct cyttsp *ts)
 {
         memset(&ts->bl_data, 0, sizeof(ts->bl_data));
@@ -133,7 +142,7 @@ static int cyttsp_exit_bl_mode(struct cyttsp *ts)
         memcpy(bl_cmd, bl_command, sizeof(bl_command));
         if (ts->pdata->bl_keys)
                 memcpy(&bl_cmd[sizeof(bl_command) - CY_NUM_BL_KEYS],
-                        ts->pdata->bl_keys, sizeof(bl_command));
+                        ts->pdata->bl_keys, CY_NUM_BL_KEYS);
 
         error = ttsp_write_block_data(ts, CY_REG_BASE,
                                       sizeof(bl_cmd), bl_cmd);
@@ -167,6 +176,10 @@ static int cyttsp_set_operational_mode(struct cyttsp *ts)
         if (error)
                 return error;
 
+        error = cyttsp_handshake(ts);
+        if (error)
+                return error;
+
         return ts->xy_data.act_dist == CY_ACT_DIST_DFLT ? -EIO : 0;
 }
 
@@ -188,6 +201,10 @@ static int cyttsp_set_sysinfo_mode(struct cyttsp *ts)
         if (error)
                 return error;
 
+        error = cyttsp_handshake(ts);
+        if (error)
+                return error;
+
         if (!ts->sysinfo_data.tts_verh && !ts->sysinfo_data.tts_verl)
                 return -EIO;
 
@@ -344,12 +361,9 @@ static irqreturn_t cyttsp_irq(int irq, void *handle)
                 goto out;
 
         /* provide flow control handshake */
-        if (ts->pdata->use_hndshk) {
-                error = ttsp_send_command(ts,
-                                ts->xy_data.hst_mode ^ CY_HNDSHK_BIT);
-                if (error)
-                        goto out;
-        }
+        error = cyttsp_handshake(ts);
+        if (error)
+                goto out;
 
         if (unlikely(ts->state == CY_IDLE_STATE))
                 goto out;
diff --git a/drivers/input/touchscreen/cyttsp_core.h b/drivers/input/touchscreen/cyttsp_core.h
index 1aa3c6967e70..f1ebde369f86 100644
--- a/drivers/input/touchscreen/cyttsp_core.h
+++ b/drivers/input/touchscreen/cyttsp_core.h
@@ -67,8 +67,8 @@ struct cyttsp_xydata {
 /* TTSP System Information interface definition */
 struct cyttsp_sysinfo_data {
         u8 hst_mode;
-        u8 mfg_cmd;
         u8 mfg_stat;
+        u8 mfg_cmd;
         u8 cid[3];
         u8 tt_undef1;
         u8 uid[8];
diff --git a/drivers/spi/spi-pxa2xx-dma.c b/drivers/spi/spi-pxa2xx-dma.c
index c735c5a008a2..6427600b5bbe 100644
--- a/drivers/spi/spi-pxa2xx-dma.c
+++ b/drivers/spi/spi-pxa2xx-dma.c
@@ -59,7 +59,7 @@ static int pxa2xx_spi_map_dma_buffer(struct driver_data *drv_data,
                 int ret;
 
                 sg_free_table(sgt);
-                ret = sg_alloc_table(sgt, nents, GFP_KERNEL);
+                ret = sg_alloc_table(sgt, nents, GFP_ATOMIC);
                 if (ret)
                         return ret;
         }
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index f5d84d6f8222..48b396fced0a 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -1075,7 +1075,7 @@ pxa2xx_spi_acpi_get_pdata(struct platform_device *pdev)
             acpi_bus_get_device(ACPI_HANDLE(&pdev->dev), &adev))
                 return NULL;
 
-        pdata = devm_kzalloc(&pdev->dev, sizeof(*ssp), GFP_KERNEL);
+        pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
         if (!pdata) {
                 dev_err(&pdev->dev,
                         "failed to allocate memory for platform data\n");
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index 5000586cb98d..71cc3e6ef47c 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -444,7 +444,7 @@ static int s3c64xx_spi_prepare_transfer(struct spi_master *spi)
         }
 
         ret = pm_runtime_get_sync(&sdd->pdev->dev);
-        if (ret != 0) {
+        if (ret < 0) {
                 dev_err(dev, "Failed to enable device: %d\n", ret);
                 goto out_tx;
         }
diff --git a/drivers/staging/imx-drm/ipuv3-crtc.c b/drivers/staging/imx-drm/ipuv3-crtc.c
index ff5c63350932..b2730b1af5b4 100644
--- a/drivers/staging/imx-drm/ipuv3-crtc.c
+++ b/drivers/staging/imx-drm/ipuv3-crtc.c
@@ -364,17 +364,12 @@ static void ipu_crtc_commit(struct drm_crtc *crtc)
         ipu_fb_enable(ipu_crtc);
 }
 
-static void ipu_crtc_load_lut(struct drm_crtc *crtc)
-{
-}
-
 static struct drm_crtc_helper_funcs ipu_helper_funcs = {
         .dpms = ipu_crtc_dpms,
         .mode_fixup = ipu_crtc_mode_fixup,
         .mode_set = ipu_crtc_mode_set,
         .prepare = ipu_crtc_prepare,
         .commit = ipu_crtc_commit,
-        .load_lut = ipu_crtc_load_lut,
 };
 
 static int ipu_enable_vblank(struct drm_crtc *crtc)
diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c
index 5855d17d19ac..9d8feac67637 100644
--- a/drivers/video/console/vgacon.c
+++ b/drivers/video/console/vgacon.c
@@ -42,6 +42,7 @@
 #include <linux/kd.h>
 #include <linux/slab.h>
 #include <linux/vt_kern.h>
+#include <linux/sched.h>
 #include <linux/selection.h>
 #include <linux/spinlock.h>
 #include <linux/ioport.h>
@@ -1124,11 +1125,15 @@ static int vgacon_do_font_op(struct vgastate *state,char *arg,int set,int ch512)
 
         if (arg) {
                 if (set)
-                        for (i = 0; i < cmapsz; i++)
+                        for (i = 0; i < cmapsz; i++) {
                                 vga_writeb(arg[i], charmap + i);
+                                cond_resched();
+                        }
                 else
-                        for (i = 0; i < cmapsz; i++)
+                        for (i = 0; i < cmapsz; i++) {
                                 arg[i] = vga_readb(charmap + i);
+                                cond_resched();
+                        }
 
                 /*
                  * In 512-character mode, the character map is not contiguous if
@@ -1139,11 +1144,15 @@ static int vgacon_do_font_op(struct vgastate *state,char *arg,int set,int ch512)
                         charmap += 2 * cmapsz;
                         arg += cmapsz;
                         if (set)
-                                for (i = 0; i < cmapsz; i++)
+                                for (i = 0; i < cmapsz; i++) {
                                         vga_writeb(arg[i], charmap + i);
+                                        cond_resched();
+                                }
                         else
-                                for (i = 0; i < cmapsz; i++)
+                                for (i = 0; i < cmapsz; i++) {
                                         arg[i] = vga_readb(charmap + i);
+                                        cond_resched();
+                                }
                 }
         }
 
diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c
index 56009bc02b02..2894e0300a33 100644
--- a/drivers/video/of_display_timing.c
+++ b/drivers/video/of_display_timing.c
@@ -23,7 +23,7 @@
  * Every display_timing can be specified with either just the typical value or
  * a range consisting of min/typ/max. This function helps handling this
  **/
-static int parse_timing_property(struct device_node *np, const char *name,
+static int parse_timing_property(const struct device_node *np, const char *name,
                           struct timing_entry *result)
 {
         struct property *prop;
@@ -56,7 +56,8 @@ static int parse_timing_property(struct device_node *np, const char *name,
  * of_get_display_timing - parse display_timing entry from device_node
  * @np: device_node with the properties
  **/
-static struct display_timing *of_get_display_timing(struct device_node *np)
+static struct display_timing *of_get_display_timing(const struct device_node
+                                                    *np)
 {
         struct display_timing *dt;
         u32 val = 0;
@@ -97,6 +98,8 @@ static struct display_timing *of_get_display_timing(struct device_node *np)
                 dt->flags |= DISPLAY_FLAGS_INTERLACED;
         if (of_property_read_bool(np, "doublescan"))
                 dt->flags |= DISPLAY_FLAGS_DOUBLESCAN;
+        if (of_property_read_bool(np, "doubleclk"))
+                dt->flags |= DISPLAY_FLAGS_DOUBLECLK;
 
         if (ret) {
                 pr_err("%s: error reading timing properties\n",
diff --git a/drivers/video/uvesafb.c b/drivers/video/uvesafb.c
index e328a61b64ba..296279bc71d2 100644
--- a/drivers/video/uvesafb.c
+++ b/drivers/video/uvesafb.c
@@ -24,9 +24,6 @@
 #ifdef CONFIG_X86
 #include <video/vga.h>
 #endif
-#ifdef CONFIG_MTRR
-#include <asm/mtrr.h>
-#endif
 #include "edid.h"
 
 static struct cb_id uvesafb_cn_id = {
@@ -1540,67 +1537,30 @@ static void uvesafb_init_info(struct fb_info *info, struct vbe_mode_ib *mode)
 
 static void uvesafb_init_mtrr(struct fb_info *info)
 {
-#ifdef CONFIG_MTRR
+        struct uvesafb_par *par = info->par;
+
         if (mtrr && !(info->fix.smem_start & (PAGE_SIZE - 1))) {
                 int temp_size = info->fix.smem_len;
-                unsigned int type = 0;
 
-                switch (mtrr) {
-                case 1:
-                        type = MTRR_TYPE_UNCACHABLE;
-                        break;
-                case 2:
-                        type = MTRR_TYPE_WRBACK;
-                        break;
-                case 3:
-                        type = MTRR_TYPE_WRCOMB;
-                        break;
-                case 4:
-                        type = MTRR_TYPE_WRTHROUGH;
-                        break;
-                default:
-                        type = 0;
-                        break;
-                }
+                int rc;
 
-                if (type) {
-                        int rc;
+                /* Find the largest power-of-two */
+                temp_size = roundup_pow_of_two(temp_size);
 
-                        /* Find the largest power-of-two */
-                        temp_size = roundup_pow_of_two(temp_size);
+                /* Try and find a power of two to add */
+                do {
+                        rc = arch_phys_wc_add(info->fix.smem_start, temp_size);
+                        temp_size >>= 1;
+                } while (temp_size >= PAGE_SIZE && rc == -EINVAL);
 
-                        /* Try and find a power of two to add */
-                        do {
-                                rc = mtrr_add(info->fix.smem_start,
-                                              temp_size, type, 1);
-                                temp_size >>= 1;
-                        } while (temp_size >= PAGE_SIZE && rc == -EINVAL);
-                }
+                if (rc >= 0)
+                        par->mtrr_handle = rc;
         }
-#endif /* CONFIG_MTRR */
 }
 
 static void uvesafb_ioremap(struct fb_info *info)
 {
-#ifdef CONFIG_X86
-        switch (mtrr) {
-        case 1: /* uncachable */
-                info->screen_base = ioremap_nocache(info->fix.smem_start, info->fix.smem_len);
-                break;
-        case 2: /* write-back */
-                info->screen_base = ioremap_cache(info->fix.smem_start, info->fix.smem_len);
-                break;
-        case 3: /* write-combining */
-                info->screen_base = ioremap_wc(info->fix.smem_start, info->fix.smem_len);
-                break;
-        case 4: /* write-through */
-        default:
-                info->screen_base = ioremap(info->fix.smem_start, info->fix.smem_len);
-                break;
-        }
-#else
-        info->screen_base = ioremap(info->fix.smem_start, info->fix.smem_len);
-#endif /* CONFIG_X86 */
+        info->screen_base = ioremap_wc(info->fix.smem_start, info->fix.smem_len);
 }
 
 static ssize_t uvesafb_show_vbe_ver(struct device *dev,
@@ -1851,6 +1811,7 @@ static int uvesafb_remove(struct platform_device *dev)
                 unregister_framebuffer(info);
                 release_region(0x3c0, 32);
                 iounmap(info->screen_base);
+                arch_phys_wc_del(par->mtrr_handle);
                 release_mem_region(info->fix.smem_start, info->fix.smem_len);
                 fb_destroy_modedb(info->monspecs.modedb);
                 fb_dealloc_cmap(&info->cmap);
@@ -1930,6 +1891,9 @@ static int uvesafb_setup(char *options)
                 }
         }
 
+        if (mtrr != 3 && mtrr != 1)
+                pr_warn("uvesafb: mtrr should be set to 0 or 3; %d is unsupported", mtrr);
+
         return 0;
 }
 #endif /* !MODULE */
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index e570081f9f76..35f281033142 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -2470,13 +2470,16 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
                 .mode = mode
         };
         int err;
+        bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
+                           (mode & FALLOC_FL_PUNCH_HOLE);
 
         if (fc->no_fallocate)
                 return -EOPNOTSUPP;
 
-        if (mode & FALLOC_FL_PUNCH_HOLE) {
+        if (lock_inode) {
                 mutex_lock(&inode->i_mutex);
-                fuse_set_nowrite(inode);
+                if (mode & FALLOC_FL_PUNCH_HOLE)
+                        fuse_set_nowrite(inode);
         }
 
         req = fuse_get_req_nopages(fc);
@@ -2511,8 +2514,9 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
         fuse_invalidate_attr(inode);
 
 out:
-        if (mode & FALLOC_FL_PUNCH_HOLE) {
-                fuse_release_nowrite(inode);
+        if (lock_inode) {
+                if (mode & FALLOC_FL_PUNCH_HOLE)
+                        fuse_release_nowrite(inode);
                 mutex_unlock(&inode->i_mutex);
         }
 
diff --git a/fs/splice.c b/fs/splice.c
index 9eca476227d5..d37431dd60a1 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -1283,6 +1283,7 @@ static int direct_splice_actor(struct pipe_inode_info *pipe,
  * @in:         file to splice from
  * @ppos:       input file offset
  * @out:        file to splice to
+ * @opos:       output file offset
  * @len:        number of bytes to splice
  * @flags:      splice modifier flags
  *
diff --git a/include/asm-generic/mutex-dec.h b/include/asm-generic/mutex-dec.h
index f104af7cf437..d4f9fb4e53df 100644
--- a/include/asm-generic/mutex-dec.h
+++ b/include/asm-generic/mutex-dec.h
@@ -28,17 +28,15 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  *                                 from 1 to a 0 value
  *  @count: pointer of type atomic_t
- *  @fail_fn: function to call if the original value was not 1
  *
- * Change the count from 1 to a value lower than 1, and call <fail_fn> if
- * it wasn't 1 originally. This function returns 0 if the fastpath succeeds,
- * or anything the slow path function returns.
+ * Change the count from 1 to a value lower than 1. This function returns 0
+ * if the fastpath succeeds, or -1 otherwise.
  */
 static inline int
-__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
+__mutex_fastpath_lock_retval(atomic_t *count)
 {
         if (unlikely(atomic_dec_return(count) < 0))
-                return fail_fn(count);
+                return -1;
         return 0;
 }
 
diff --git a/include/asm-generic/mutex-null.h b/include/asm-generic/mutex-null.h
index e1bbbc72b6a2..61069ed334e2 100644
--- a/include/asm-generic/mutex-null.h
+++ b/include/asm-generic/mutex-null.h
@@ -11,7 +11,7 @@
 #define _ASM_GENERIC_MUTEX_NULL_H
 
 #define __mutex_fastpath_lock(count, fail_fn)           fail_fn(count)
-#define __mutex_fastpath_lock_retval(count, fail_fn)    fail_fn(count)
+#define __mutex_fastpath_lock_retval(count)             (-1)
 #define __mutex_fastpath_unlock(count, fail_fn)         fail_fn(count)
 #define __mutex_fastpath_trylock(count, fail_fn)        fail_fn(count)
 #define __mutex_slowpath_needs_to_unlock()              1
diff --git a/include/asm-generic/mutex-xchg.h b/include/asm-generic/mutex-xchg.h
index c04e0db8a2d6..f169ec064785 100644
--- a/include/asm-generic/mutex-xchg.h
+++ b/include/asm-generic/mutex-xchg.h
@@ -39,18 +39,16 @@ __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
  *  __mutex_fastpath_lock_retval - try to take the lock by moving the count
  *                                 from 1 to a 0 value
  *  @count: pointer of type atomic_t
- *  @fail_fn: function to call if the original value was not 1
  *
- * Change the count from 1 to a value lower than 1, and call <fail_fn> if it
- * wasn't 1 originally. This function returns 0 if the fastpath succeeds,
- * or anything the slow path function returns
+ * Change the count from 1 to a value lower than 1. This function returns 0
+ * if the fastpath succeeds, or -1 otherwise.
  */
 static inline int
-__mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
+__mutex_fastpath_lock_retval(atomic_t *count)
 {
         if (unlikely(atomic_xchg(count, 0) != 1))
                 if (likely(atomic_xchg(count, -1) != 1))
-                        return fail_fn(count);
+                        return -1;
         return 0;
 }
 
diff --git a/include/drm/drmP.h b/include/drm/drmP.h
index 63d17ee9eb48..82670ac0d774 100644
--- a/include/drm/drmP.h
+++ b/include/drm/drmP.h
@@ -55,16 +55,13 @@
 #include <linux/mm.h>
 #include <linux/cdev.h>
 #include <linux/mutex.h>
+#include <linux/io.h>
 #include <linux/slab.h>
 #if defined(__alpha__) || defined(__powerpc__)
 #include <asm/pgtable.h>        /* For pte_wrprotect */
 #endif
-#include <asm/io.h>
 #include <asm/mman.h>
 #include <asm/uaccess.h>
-#ifdef CONFIG_MTRR
-#include <asm/mtrr.h>
-#endif
 #if defined(CONFIG_AGP) || defined(CONFIG_AGP_MODULE)
 #include <linux/types.h>
 #include <linux/agp_backend.h>
@@ -933,6 +930,7 @@ struct drm_driver {
                                 struct dma_buf *dma_buf);
         /* low-level interface used by drm_gem_prime_{import,export} */
         int (*gem_prime_pin)(struct drm_gem_object *obj);
+        void (*gem_prime_unpin)(struct drm_gem_object *obj);
         struct sg_table *(*gem_prime_get_sg_table)(struct drm_gem_object *obj);
         struct drm_gem_object *(*gem_prime_import_sg_table)(
                                 struct drm_device *dev, size_t size,
@@ -1250,37 +1248,8 @@ static inline int drm_core_has_MTRR(struct drm_device *dev)
 {
         return drm_core_check_feature(dev, DRIVER_USE_MTRR);
 }
-
-#define DRM_MTRR_WC             MTRR_TYPE_WRCOMB
-
-static inline int drm_mtrr_add(unsigned long offset, unsigned long size,
-                               unsigned int flags)
-{
-        return mtrr_add(offset, size, flags, 1);
-}
-
-static inline int drm_mtrr_del(int handle, unsigned long offset,
-                               unsigned long size, unsigned int flags)
-{
-        return mtrr_del(handle, offset, size);
-}
-
 #else
 #define drm_core_has_MTRR(dev) (0)
-
-#define DRM_MTRR_WC             0
-
-static inline int drm_mtrr_add(unsigned long offset, unsigned long size,
-                               unsigned int flags)
-{
-        return 0;
-}
-
-static inline int drm_mtrr_del(int handle, unsigned long offset,
-                               unsigned long size, unsigned int flags)
-{
-        return 0;
-}
 #endif
 
 static inline void drm_device_set_unplugged(struct drm_device *dev)
@@ -1630,7 +1599,6 @@ extern void drm_sysfs_destroy(void);
 extern int drm_sysfs_device_add(struct drm_minor *minor);
 extern void drm_sysfs_hotplug_event(struct drm_device *dev);
 extern void drm_sysfs_device_remove(struct drm_minor *minor);
-extern char *drm_get_connector_status_name(enum drm_connector_status status);
 extern int drm_sysfs_connector_add(struct drm_connector *connector);
 extern void drm_sysfs_connector_remove(struct drm_connector *connector);
 
@@ -1648,6 +1616,8 @@ int drm_gem_private_object_init(struct drm_device *dev,
 void drm_gem_object_handle_free(struct drm_gem_object *obj);
 void drm_gem_vm_open(struct vm_area_struct *vma);
 void drm_gem_vm_close(struct vm_area_struct *vma);
+int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
+                     struct vm_area_struct *vma);
 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 
 #include <drm/drm_global.h>
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index adb3f9b625f6..fa12a2fa4293 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -339,6 +339,9 @@ struct drm_crtc_funcs {
         /* cursor controls */
         int (*cursor_set)(struct drm_crtc *crtc, struct drm_file *file_priv,
                           uint32_t handle, uint32_t width, uint32_t height);
+        int (*cursor_set2)(struct drm_crtc *crtc, struct drm_file *file_priv,
+                           uint32_t handle, uint32_t width, uint32_t height,
+                           int32_t hot_x, int32_t hot_y);
         int (*cursor_move)(struct drm_crtc *crtc, int x, int y);
 
         /* Set gamma on the CRTC */
@@ -409,6 +412,10 @@ struct drm_crtc {
         /* framebuffer the connector is currently bound to */
         struct drm_framebuffer *fb;
 
+        /* Temporary tracking of the old fb while a modeset is ongoing. Used
+         * by drm_mode_set_config_internal to implement correct refcounting. */
+        struct drm_framebuffer *old_fb;
+
         bool enabled;
 
         /* Requested mode from modesetting. */
@@ -654,11 +661,7 @@ struct drm_plane_funcs {
  * @format_count: number of formats supported
  * @crtc: currently bound CRTC
  * @fb: currently bound fb
- * @gamma_size: size of gamma table
- * @gamma_store: gamma correction table
- * @enabled: enabled flag
  * @funcs: helper functions
- * @helper_private: storage for drver layer
  * @properties: property tracking for this plane
  */
 struct drm_plane {
@@ -674,14 +677,7 @@ struct drm_plane {
         struct drm_crtc *crtc;
         struct drm_framebuffer *fb;
 
-        /* CRTC gamma size for reporting to userspace */
-        uint32_t gamma_size;
-        uint16_t *gamma_store;
-
-        bool enabled;
-
         const struct drm_plane_funcs *funcs;
-        void *helper_private;
 
         struct drm_object_properties properties;
 };
@@ -894,15 +890,17 @@ extern int drm_plane_init(struct drm_device *dev,
                           const uint32_t *formats, uint32_t format_count,
                           bool priv);
 extern void drm_plane_cleanup(struct drm_plane *plane);
+extern void drm_plane_force_disable(struct drm_plane *plane);
 
 extern void drm_encoder_cleanup(struct drm_encoder *encoder);
 
-extern char *drm_get_connector_name(struct drm_connector *connector);
-extern char *drm_get_dpms_name(int val);
-extern char *drm_get_dvi_i_subconnector_name(int val);
-extern char *drm_get_dvi_i_select_name(int val);
-extern char *drm_get_tv_subconnector_name(int val);
-extern char *drm_get_tv_select_name(int val);
+extern const char *drm_get_connector_name(const struct drm_connector *connector);
+extern const char *drm_get_connector_status_name(enum drm_connector_status status);
+extern const char *drm_get_dpms_name(int val);
+extern const char *drm_get_dvi_i_subconnector_name(int val);
+extern const char *drm_get_dvi_i_select_name(int val);
+extern const char *drm_get_tv_subconnector_name(int val);
+extern const char *drm_get_tv_select_name(int val);
 extern void drm_fb_release(struct drm_file *file_priv);
 extern int drm_mode_group_init_legacy_group(struct drm_device *dev, struct drm_mode_group *group);
 extern bool drm_probe_ddc(struct i2c_adapter *adapter);
@@ -994,7 +992,7 @@ extern int drm_mode_create_tv_properties(struct drm_device *dev, int num_formats
 extern int drm_mode_create_scaling_mode_property(struct drm_device *dev);
 extern int drm_mode_create_dithering_property(struct drm_device *dev);
 extern int drm_mode_create_dirty_info_property(struct drm_device *dev);
-extern char *drm_get_encoder_name(struct drm_encoder *encoder);
+extern const char *drm_get_encoder_name(const struct drm_encoder *encoder);
 
 extern int drm_mode_connector_attach_encoder(struct drm_connector *connector,
                                              struct drm_encoder *encoder);
@@ -1022,6 +1020,8 @@ extern int drm_mode_setplane(struct drm_device *dev,
                                void *data, struct drm_file *file_priv);
 extern int drm_mode_cursor_ioctl(struct drm_device *dev,
                                 void *data, struct drm_file *file_priv);
+extern int drm_mode_cursor2_ioctl(struct drm_device *dev,
+                                void *data, struct drm_file *file_priv);
 extern int drm_mode_addfb(struct drm_device *dev,
                           void *data, struct drm_file *file_priv);
 extern int drm_mode_addfb2(struct drm_device *dev,
@@ -1094,5 +1094,6 @@ extern int drm_format_num_planes(uint32_t format);
 extern int drm_format_plane_cpp(uint32_t format, int plane);
 extern int drm_format_horz_chroma_subsampling(uint32_t format);
 extern int drm_format_vert_chroma_subsampling(uint32_t format);
+extern const char *drm_get_format_name(uint32_t format);
 
 #endif /* __DRM_CRTC_H__ */
diff --git a/include/drm/drm_fixed.h b/include/drm/drm_fixed.h
index 0ead502e17d2..f5e1168c7647 100644
--- a/include/drm/drm_fixed.h
+++ b/include/drm/drm_fixed.h
@@ -20,10 +20,13 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  * Authors: Dave Airlie
+ *          Christian König
  */
 #ifndef DRM_FIXED_H
 #define DRM_FIXED_H
 
+#include <linux/math64.h>
+
 typedef union dfixed {
         u32 full;
 } fixed20_12;
@@ -65,4 +68,95 @@ static inline u32 dfixed_div(fixed20_12 A, fixed20_12 B)
         tmp /= 2;
         return lower_32_bits(tmp);
 }
+
+#define DRM_FIXED_POINT         32
+#define DRM_FIXED_ONE           (1ULL << DRM_FIXED_POINT)
+#define DRM_FIXED_DECIMAL_MASK  (DRM_FIXED_ONE - 1)
+#define DRM_FIXED_DIGITS_MASK   (~DRM_FIXED_DECIMAL_MASK)
+
+static inline s64 drm_int2fixp(int a)
+{
+        return ((s64)a) << DRM_FIXED_POINT;
+}
+
+static inline int drm_fixp2int(int64_t a)
+{
+        return ((s64)a) >> DRM_FIXED_POINT;
+}
+
+static inline s64 drm_fixp_msbset(int64_t a)
+{
+        unsigned shift, sign = (a >> 63) & 1;
+
+        for (shift = 62; shift > 0; --shift)
+                if ((a >> shift) != sign)
+                        return shift;
+
+        return 0;
+}
+
+static inline s64 drm_fixp_mul(s64 a, s64 b)
+{
+        unsigned shift = drm_fixp_msbset(a) + drm_fixp_msbset(b);
+        s64 result;
+
+        if (shift > 63) {
+                shift = shift - 63;
+                a >>= shift >> 1;
+                b >>= shift >> 1;
+        } else
+                shift = 0;
+
+        result = a * b;
+
+        if (shift > DRM_FIXED_POINT)
+                return result << (shift - DRM_FIXED_POINT);
+
+        if (shift < DRM_FIXED_POINT)
+                return result >> (DRM_FIXED_POINT - shift);
+
+        return result;
+}
+
+static inline s64 drm_fixp_div(s64 a, s64 b)
+{
+        unsigned shift = 63 - drm_fixp_msbset(a);
+        s64 result;
+
+        a <<= shift;
+
+        if (shift < DRM_FIXED_POINT)
+                b >>= (DRM_FIXED_POINT - shift);
+
+        result = div64_s64(a, b);
+
+        if (shift > DRM_FIXED_POINT)
+                return result >> (shift - DRM_FIXED_POINT);
+
+        return result;
+}
+
+static inline s64 drm_fixp_exp(s64 x)
+{
+        s64 tolerance = div64_s64(DRM_FIXED_ONE, 1000000);
+        s64 sum = DRM_FIXED_ONE, term, y = x;
+        u64 count = 1;
+
+        if (x < 0)
+                y = -1 * x;
+
+        term = y;
+
+        while (term >= tolerance) {
+                sum = sum + term;
+                count = count + 1;
+                term = drm_fixp_mul(term, div64_s64(y, count));
+        }
+
+        if (x < 0)
+                sum = drm_fixp_div(1, sum);
+
+        return sum;
+}
+
 #endif
diff --git a/include/drm/drm_gem_cma_helper.h b/include/drm/drm_gem_cma_helper.h
index 63397ced9254..6e17251e9b28 100644
--- a/include/drm/drm_gem_cma_helper.h
+++ b/include/drm/drm_gem_cma_helper.h
@@ -4,6 +4,9 @@
 struct drm_gem_cma_object {
         struct drm_gem_object base;
         dma_addr_t paddr;
+        struct sg_table *sgt;
+
+        /* For objects with DMA memory allocated by GEM CMA */
         void *vaddr;
 };
 
@@ -45,4 +48,10 @@ extern const struct vm_operations_struct drm_gem_cma_vm_ops;
 void drm_gem_cma_describe(struct drm_gem_cma_object *obj, struct seq_file *m);
 #endif
 
+struct dma_buf *drm_gem_cma_dmabuf_export(struct drm_device *drm_dev,
+                                          struct drm_gem_object *obj,
+                                          int flags);
+struct drm_gem_object *drm_gem_cma_dmabuf_import(struct drm_device *drm_dev,
+                                                 struct dma_buf *dma_buf);
+
 #endif /* __DRM_GEM_CMA_HELPER_H__ */
diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h
index 88591ef8fa24..4d06edb56d5f 100644
--- a/include/drm/drm_mm.h
+++ b/include/drm/drm_mm.h
@@ -177,17 +177,6 @@ static inline struct drm_mm_node *drm_mm_get_block_range(
         return drm_mm_get_block_range_generic(parent, size, alignment, 0,
                                               start, end, 0);
 }
-static inline struct drm_mm_node *drm_mm_get_color_block_range(
-                                                struct drm_mm_node *parent,
-                                                unsigned long size,
-                                                unsigned alignment,
-                                                unsigned long color,
-                                                unsigned long start,
-                                                unsigned long end)
-{
-        return drm_mm_get_block_range_generic(parent, size, alignment, color,
-                                              start, end, 0);
-}
 static inline struct drm_mm_node *drm_mm_get_block_atomic_range(
                                                 struct drm_mm_node *parent,
                                                 unsigned long size,
@@ -255,29 +244,10 @@ static inline  struct drm_mm_node *drm_mm_search_free_in_range(
         return drm_mm_search_free_in_range_generic(mm, size, alignment, 0,
                                                    start, end, best_match);
 }
-static inline struct drm_mm_node *drm_mm_search_free_color(const struct drm_mm *mm,
-                                                           unsigned long size,
-                                                           unsigned alignment,
-                                                           unsigned long color,
-                                                           bool best_match)
-{
-        return drm_mm_search_free_generic(mm,size, alignment, color, best_match);
-}
-static inline  struct drm_mm_node *drm_mm_search_free_in_range_color(
-                                                const struct drm_mm *mm,
-                                                unsigned long size,
-                                                unsigned alignment,
-                                                unsigned long color,
-                                                unsigned long start,
-                                                unsigned long end,
-                                                bool best_match)
-{
-        return drm_mm_search_free_in_range_generic(mm, size, alignment, color,
-                                                   start, end, best_match);
-}
-extern int drm_mm_init(struct drm_mm *mm,
-                       unsigned long start,
-                       unsigned long size);
+
+extern void drm_mm_init(struct drm_mm *mm,
+                        unsigned long start,
+                        unsigned long size);
 extern void drm_mm_takedown(struct drm_mm *mm);
 extern int drm_mm_clean(struct drm_mm *mm);
 extern int drm_mm_pre_get(struct drm_mm *mm);
diff --git a/include/drm/drm_os_linux.h b/include/drm/drm_os_linux.h
index 675ddf4b441f..815fafc6b4ad 100644
--- a/include/drm/drm_os_linux.h
+++ b/include/drm/drm_os_linux.h
@@ -65,22 +65,6 @@ struct no_agp_kern {
 #define DRM_AGP_KERN            struct no_agp_kern
 #endif
 
-#if !(__OS_HAS_MTRR)
-static __inline__ int mtrr_add(unsigned long base, unsigned long size,
-                               unsigned int type, char increment)
-{
-        return -ENODEV;
-}
-
-static __inline__ int mtrr_del(int reg, unsigned long base, unsigned long size)
-{
-        return -ENODEV;
-}
-
-#define MTRR_TYPE_WRCOMB     1
-
-#endif
-
 /** Other copying of data to kernel space */
 #define DRM_COPY_FROM_USER(arg1, arg2, arg3)            \
         copy_from_user(arg1, arg2, arg3)
diff --git a/include/drm/drm_pciids.h b/include/drm/drm_pciids.h
index bb1bc485390b..34efaf64cc87 100644
--- a/include/drm/drm_pciids.h
+++ b/include/drm/drm_pciids.h
@@ -152,6 +152,14 @@
         {0x1002, 0x6621, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
         {0x1002, 0x6623, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
         {0x1002, 0x6631, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x6640, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x6641, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x6649, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x6650, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x6651, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x6658, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x665c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \
+        {0x1002, 0x665d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|RADEON_NEW_MEMMAP}, \
         {0x1002, 0x6660, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
         {0x1002, 0x6663, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
         {0x1002, 0x6664, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
@@ -580,6 +588,22 @@
         {0x1002, 0x9808, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
         {0x1002, 0x9809, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
         {0x1002, 0x980A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9832, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9833, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9834, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9836, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x9839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x983a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x983b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x983c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x983d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x983e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
+        {0x1002, 0x983f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
         {0x1002, 0x9900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ARUBA|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
         {0x1002, 0x9901, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ARUBA|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
         {0x1002, 0x9903, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ARUBA|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
diff --git a/include/drm/drm_rect.h b/include/drm/drm_rect.h
new file mode 100644
index 000000000000..d1286297567b
--- /dev/null
+++ b/include/drm/drm_rect.h
@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2011-2013 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef DRM_RECT_H
+#define DRM_RECT_H
+
+/**
+ * DOC: rect utils
+ *
+ * Utility functions to help manage rectangular areas for
+ * clipping, scaling, etc. calculations.
+ */
+
+/**
+ * struct drm_rect - two dimensional rectangle
+ * @x1: horizontal starting coordinate (inclusive)
+ * @x2: horizontal ending coordinate (exclusive)
+ * @y1: vertical starting coordinate (inclusive)
+ * @y2: vertical ending coordinate (exclusive)
+ */
+struct drm_rect {
+        int x1, y1, x2, y2;
+};
+
+/**
+ * drm_rect_adjust_size - adjust the size of the rectangle
+ * @r: rectangle to be adjusted
+ * @dw: horizontal adjustment
+ * @dh: vertical adjustment
+ *
+ * Change the size of rectangle @r by @dw in the horizontal direction,
+ * and by @dh in the vertical direction, while keeping the center
+ * of @r stationary.
+ *
+ * Positive @dw and @dh increase the size, negative values decrease it.
+ */
+static inline void drm_rect_adjust_size(struct drm_rect *r, int dw, int dh)
+{
+        r->x1 -= dw >> 1;
+        r->y1 -= dh >> 1;
+        r->x2 += (dw + 1) >> 1;
+        r->y2 += (dh + 1) >> 1;
+}
+
+/**
+ * drm_rect_translate - translate the rectangle
+ * @r: rectangle to be tranlated
+ * @dx: horizontal translation
+ * @dy: vertical translation
+ *
+ * Move rectangle @r by @dx in the horizontal direction,
+ * and by @dy in the vertical direction.
+ */
+static inline void drm_rect_translate(struct drm_rect *r, int dx, int dy)
+{
+        r->x1 += dx;
+        r->y1 += dy;
+        r->x2 += dx;
+        r->y2 += dy;
+}
+
+/**
+ * drm_rect_downscale - downscale a rectangle
+ * @r: rectangle to be downscaled
+ * @horz: horizontal downscale factor
+ * @vert: vertical downscale factor
+ *
+ * Divide the coordinates of rectangle @r by @horz and @vert.
+ */
+static inline void drm_rect_downscale(struct drm_rect *r, int horz, int vert)
+{
+        r->x1 /= horz;
+        r->y1 /= vert;
+        r->x2 /= horz;
+        r->y2 /= vert;
+}
+
+/**
+ * drm_rect_width - determine the rectangle width
+ * @r: rectangle whose width is returned
+ *
+ * RETURNS:
+ * The width of the rectangle.
+ */
+static inline int drm_rect_width(const struct drm_rect *r)
+{
+        return r->x2 - r->x1;
+}
+
+/**
+ * drm_rect_height - determine the rectangle height
+ * @r: rectangle whose height is returned
+ *
+ * RETURNS:
+ * The height of the rectangle.
+ */
+static inline int drm_rect_height(const struct drm_rect *r)
+{
+        return r->y2 - r->y1;
+}
+
+/**
+ * drm_rect_visible - determine if the the rectangle is visible
+ * @r: rectangle whose visibility is returned
+ *
+ * RETURNS:
+ * %true if the rectangle is visible, %false otherwise.
+ */
+static inline bool drm_rect_visible(const struct drm_rect *r)
+{
+        return drm_rect_width(r) > 0 && drm_rect_height(r) > 0;
+}
+
+/**
+ * drm_rect_equals - determine if two rectangles are equal
+ * @r1: first rectangle
+ * @r2: second rectangle
+ *
+ * RETURNS:
+ * %true if the rectangles are equal, %false otherwise.
+ */
+static inline bool drm_rect_equals(const struct drm_rect *r1,
+                                   const struct drm_rect *r2)
+{
+        return r1->x1 == r2->x1 && r1->x2 == r2->x2 &&
+                r1->y1 == r2->y1 && r1->y2 == r2->y2;
+}
+
+bool drm_rect_intersect(struct drm_rect *r, const struct drm_rect *clip);
+bool drm_rect_clip_scaled(struct drm_rect *src, struct drm_rect *dst,
+                          const struct drm_rect *clip,
+                          int hscale, int vscale);
+int drm_rect_calc_hscale(const struct drm_rect *src,
+                         const struct drm_rect *dst,
+                         int min_hscale, int max_hscale);
+int drm_rect_calc_vscale(const struct drm_rect *src,
+                         const struct drm_rect *dst,
+                         int min_vscale, int max_vscale);
+int drm_rect_calc_hscale_relaxed(struct drm_rect *src,
+                                 struct drm_rect *dst,
+                                 int min_hscale, int max_hscale);
+int drm_rect_calc_vscale_relaxed(struct drm_rect *src,
+                                 struct drm_rect *dst,
+                                 int min_vscale, int max_vscale);
+void drm_rect_debug_print(const struct drm_rect *r, bool fixed_point);
+
+#endif
diff --git a/include/drm/i915_powerwell.h b/include/drm/i915_powerwell.h
new file mode 100644
index 000000000000..cfdc884405b7
--- /dev/null
+++ b/include/drm/i915_powerwell.h
@@ -0,0 +1,36 @@
+/**************************************************************************
+ *
+ * Copyright 2013 Intel Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ **************************************************************************/
+
+#ifndef _I915_POWERWELL_H_
+#define _I915_POWERWELL_H_
+
+/* For use by hda_i915 driver */
+extern void i915_request_power_well(void);
+extern void i915_release_power_well(void);
+
+#endif                          /* _I915_POWERWELL_H_ */
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index 3cb5d848fb66..8a6aa56ece52 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -39,6 +39,7 @@
 #include <linux/mm.h>
 #include <linux/rbtree.h>
 #include <linux/bitmap.h>
+#include <linux/reservation.h>
 
 struct ttm_bo_device;
 
@@ -153,7 +154,6 @@ struct ttm_tt;
  * Lru lists may keep one refcount, the delayed delete list, and kref != 0
  * keeps one refcount. When this refcount reaches zero,
  * the object is destroyed.
- * @event_queue: Queue for processes waiting on buffer object status change.
  * @mem: structure describing current placement.
  * @persistent_swap_storage: Usually the swap storage is deleted for buffers
  * pinned in physical memory. If this behaviour is not desired, this member
@@ -164,12 +164,6 @@ struct ttm_tt;
  * @lru: List head for the lru list.
  * @ddestroy: List head for the delayed destroy list.
  * @swap: List head for swap LRU list.
- * @val_seq: Sequence of the validation holding the @reserved lock.
- * Used to avoid starvation when many processes compete to validate the
- * buffer. This member is protected by the bo_device::lru_lock.
- * @seq_valid: The value of @val_seq is valid. This value is protected by
- * the bo_device::lru_lock.
- * @reserved: Deadlock-free lock used for synchronization state transitions.
  * @sync_obj: Pointer to a synchronization object.
  * @priv_flags: Flags describing buffer object internal state.
  * @vm_rb: Rb node for the vm rb tree.
@@ -209,10 +203,9 @@ struct ttm_buffer_object {
 
         struct kref kref;
         struct kref list_kref;
-        wait_queue_head_t event_queue;
 
         /**
-         * Members protected by the bo::reserved lock.
+         * Members protected by the bo::resv::reserved lock.
          */
 
         struct ttm_mem_reg mem;
@@ -234,15 +227,6 @@ struct ttm_buffer_object {
         struct list_head ddestroy;
         struct list_head swap;
         struct list_head io_reserve_lru;
-        uint32_t val_seq;
-        bool seq_valid;
-
-        /**
-         * Members protected by the bdev::lru_lock
-         * only when written to.
-         */
-
-        atomic_t reserved;
 
         /**
          * Members protected by struct buffer_object_device::fence_lock
@@ -272,6 +256,9 @@ struct ttm_buffer_object {
         uint32_t cur_placement;
 
         struct sg_table *sg;
+
+        struct reservation_object *resv;
+        struct reservation_object ttm_resv;
 };
 
 /**
@@ -725,18 +712,4 @@ extern ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
 
 extern void ttm_bo_swapout_all(struct ttm_bo_device *bdev);
 
-/**
- * ttm_bo_is_reserved - return an indication if a ttm buffer object is reserved
- *
- * @bo:     The buffer object to check.
- *
- * This function returns an indication if a bo is reserved or not, and should
- * only be used to print an error when it is not from incorrect api usage, since
- * there's no guarantee that it is the caller that is holding the reservation.
- */
-static inline bool ttm_bo_is_reserved(struct ttm_buffer_object *bo)
-{
-        return atomic_read(&bo->reserved);
-}
-
 #endif
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 9c8dca79808e..984fc2d571a1 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -33,11 +33,13 @@
 #include <ttm/ttm_bo_api.h>
 #include <ttm/ttm_memory.h>
 #include <ttm/ttm_module.h>
+#include <ttm/ttm_placement.h>
 #include <drm/drm_mm.h>
 #include <drm/drm_global.h>
 #include <linux/workqueue.h>
 #include <linux/fs.h>
 #include <linux/spinlock.h>
+#include <linux/reservation.h>
 
 struct ttm_backend_func {
         /**
@@ -771,6 +773,55 @@ extern int ttm_mem_io_lock(struct ttm_mem_type_manager *man,
                            bool interruptible);
 extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
 
+extern void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo);
+extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo);
+
+/**
+ * ttm_bo_reserve_nolru:
+ *
+ * @bo: A pointer to a struct ttm_buffer_object.
+ * @interruptible: Sleep interruptible if waiting.
+ * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY.
+ * @use_ticket: If @bo is already reserved, Only sleep waiting for
+ * it to become unreserved if @ticket->stamp is older.
+ *
+ * Will not remove reserved buffers from the lru lists.
+ * Otherwise identical to ttm_bo_reserve.
+ *
+ * Returns:
+ * -EDEADLK: The reservation may cause a deadlock.
+ * Release all buffer reservations, wait for @bo to become unreserved and
+ * try again. (only if use_sequence == 1).
+ * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
+ * a signal. Release all buffer reservations and return to user-space.
+ * -EBUSY: The function needed to sleep, but @no_wait was true
+ * -EALREADY: Bo already reserved using @ticket. This error code will only
+ * be returned if @use_ticket is set to true.
+ */
+static inline int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,
+                                       bool interruptible,
+                                       bool no_wait, bool use_ticket,
+                                       struct ww_acquire_ctx *ticket)
+{
+        int ret = 0;
+
+        if (no_wait) {
+                bool success;
+                if (WARN_ON(ticket))
+                        return -EBUSY;
+
+                success = ww_mutex_trylock(&bo->resv->lock);
+                return success ? 0 : -EBUSY;
+        }
+
+        if (interruptible)
+                ret = ww_mutex_lock_interruptible(&bo->resv->lock, ticket);
+        else
+                ret = ww_mutex_lock(&bo->resv->lock, ticket);
+        if (ret == -EINTR)
+                return -ERESTARTSYS;
+        return ret;
+}
 
 /**
  * ttm_bo_reserve:
@@ -778,8 +829,8 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
  * @bo: A pointer to a struct ttm_buffer_object.
  * @interruptible: Sleep interruptible if waiting.
  * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY.
- * @use_sequence: If @bo is already reserved, Only sleep waiting for
- * it to become unreserved if @sequence < (@bo)->sequence.
+ * @use_ticket: If @bo is already reserved, Only sleep waiting for
+ * it to become unreserved if @ticket->stamp is older.
  *
  * Locks a buffer object for validation. (Or prevents other processes from
  * locking it for validation) and removes it from lru lists, while taking
@@ -793,7 +844,7 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
  * Processes attempting to reserve multiple buffers other than for eviction,
  * (typically execbuf), should first obtain a unique 32-bit
  * validation sequence number,
- * and call this function with @use_sequence == 1 and @sequence == the unique
+ * and call this function with @use_ticket == 1 and @ticket->stamp == the unique
  * sequence number. If upon call of this function, the buffer object is already
  * reserved, the validation sequence is checked against the validation
  * sequence of the process currently reserving the buffer,
@@ -808,36 +859,31 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
  * will eventually succeed, preventing both deadlocks and starvation.
  *
  * Returns:
- * -EAGAIN: The reservation may cause a deadlock.
+ * -EDEADLK: The reservation may cause a deadlock.
  * Release all buffer reservations, wait for @bo to become unreserved and
  * try again. (only if use_sequence == 1).
  * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
  * a signal. Release all buffer reservations and return to user-space.
  * -EBUSY: The function needed to sleep, but @no_wait was true
- * -EDEADLK: Bo already reserved using @sequence. This error code will only
- * be returned if @use_sequence is set to true.
+ * -EALREADY: Bo already reserved using @ticket. This error code will only
+ * be returned if @use_ticket is set to true.
  */
-extern int ttm_bo_reserve(struct ttm_buffer_object *bo,
-                          bool interruptible,
-                          bool no_wait, bool use_sequence, uint32_t sequence);
+static inline int ttm_bo_reserve(struct ttm_buffer_object *bo,
+                                 bool interruptible,
+                                 bool no_wait, bool use_ticket,
+                                 struct ww_acquire_ctx *ticket)
+{
+        int ret;
 
-/**
- * ttm_bo_reserve_slowpath_nolru:
- * @bo: A pointer to a struct ttm_buffer_object.
- * @interruptible: Sleep interruptible if waiting.
- * @sequence: Set (@bo)->sequence to this value after lock
- *
- * This is called after ttm_bo_reserve returns -EAGAIN and we backed off
- * from all our other reservations. Because there are no other reservations
- * held by us, this function cannot deadlock any more.
- *
- * Will not remove reserved buffers from the lru lists.
- * Otherwise identical to ttm_bo_reserve_slowpath.
- */
-extern int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,
-                                         bool interruptible,
-                                         uint32_t sequence);
+        WARN_ON(!atomic_read(&bo->kref.refcount));
 
+        ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_ticket,
+                                    ticket);
+        if (likely(ret == 0))
+                ttm_bo_del_sub_from_lru(bo);
+
+        return ret;
+}
 
 /**
  * ttm_bo_reserve_slowpath:
@@ -849,54 +895,57 @@ extern int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,
  * from all our other reservations. Because there are no other reservations
  * held by us, this function cannot deadlock any more.
  */
-extern int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,
-                                   bool interruptible, uint32_t sequence);
+static inline int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,
+                                          bool interruptible,
+                                          struct ww_acquire_ctx *ticket)
+{
+        int ret = 0;
 
-/**
- * ttm_bo_reserve_nolru:
- *
- * @bo: A pointer to a struct ttm_buffer_object.
- * @interruptible: Sleep interruptible if waiting.
- * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY.
- * @use_sequence: If @bo is already reserved, Only sleep waiting for
- * it to become unreserved if @sequence < (@bo)->sequence.
- *
- * Will not remove reserved buffers from the lru lists.
- * Otherwise identical to ttm_bo_reserve.
- *
- * Returns:
- * -EAGAIN: The reservation may cause a deadlock.
- * Release all buffer reservations, wait for @bo to become unreserved and
- * try again. (only if use_sequence == 1).
- * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
- * a signal. Release all buffer reservations and return to user-space.
- * -EBUSY: The function needed to sleep, but @no_wait was true
- * -EDEADLK: Bo already reserved using @sequence. This error code will only
- * be returned if @use_sequence is set to true.
- */
-extern int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,
-                                 bool interruptible,
-                                 bool no_wait, bool use_sequence,
-                                 uint32_t sequence);
+        WARN_ON(!atomic_read(&bo->kref.refcount));
+
+        if (interruptible)
+                ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
+                                                       ticket);
+        else
+                ww_mutex_lock_slow(&bo->resv->lock, ticket);
+
+        if (likely(ret == 0))
+                ttm_bo_del_sub_from_lru(bo);
+        else if (ret == -EINTR)
+                ret = -ERESTARTSYS;
+
+        return ret;
+}
 
 /**
- * ttm_bo_unreserve
- *
+ * ttm_bo_unreserve_ticket
  * @bo: A pointer to a struct ttm_buffer_object.
+ * @ticket: ww_acquire_ctx used for reserving
  *
- * Unreserve a previous reservation of @bo.
+ * Unreserve a previous reservation of @bo made with @ticket.
  */
-extern void ttm_bo_unreserve(struct ttm_buffer_object *bo);
+static inline void ttm_bo_unreserve_ticket(struct ttm_buffer_object *bo,
+                                           struct ww_acquire_ctx *t)
+{
+        if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
+                spin_lock(&bo->glob->lru_lock);
+                ttm_bo_add_to_lru(bo);
+                spin_unlock(&bo->glob->lru_lock);
+        }
+        ww_mutex_unlock(&bo->resv->lock);
+}
 
 /**
- * ttm_bo_unreserve_locked
+ * ttm_bo_unreserve
  *
  * @bo: A pointer to a struct ttm_buffer_object.
  *
  * Unreserve a previous reservation of @bo.
- * Needs to be called with struct ttm_bo_global::lru_lock held.
  */
-extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo);
+static inline void ttm_bo_unreserve(struct ttm_buffer_object *bo)
+{
+        ttm_bo_unreserve_ticket(bo, NULL);
+}
 
 /*
  * ttm_bo_util.c
diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h
index 547e19f06e57..ec8a1d306510 100644
--- a/include/drm/ttm/ttm_execbuf_util.h
+++ b/include/drm/ttm/ttm_execbuf_util.h
@@ -57,17 +57,20 @@ struct ttm_validate_buffer {
 /**
  * function ttm_eu_backoff_reservation
  *
+ * @ticket:   ww_acquire_ctx from reserve call
  * @list:     thread private list of ttm_validate_buffer structs.
  *
  * Undoes all buffer validation reservations for bos pointed to by
  * the list entries.
  */
 
-extern void ttm_eu_backoff_reservation(struct list_head *list);
+extern void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket,
+                                       struct list_head *list);
 
 /**
  * function ttm_eu_reserve_buffers
  *
+ * @ticket:  [out] ww_acquire_ctx returned by call.
  * @list:    thread private list of ttm_validate_buffer structs.
  *
  * Tries to reserve bos pointed to by the list entries for validation.
@@ -90,11 +93,13 @@ extern void ttm_eu_backoff_reservation(struct list_head *list);
  * has failed.
  */
 
-extern int ttm_eu_reserve_buffers(struct list_head *list);
+extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket,
+                                  struct list_head *list);
 
 /**
  * function ttm_eu_fence_buffer_objects.
  *
+ * @ticket:      ww_acquire_ctx from reserve call
  * @list:        thread private list of ttm_validate_buffer structs.
  * @sync_obj:    The new sync object for the buffers.
  *
@@ -104,6 +109,7 @@ extern int ttm_eu_reserve_buffers(struct list_head *list);
  *
  */
 
-extern void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj);
+extern void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
+                                        struct list_head *list, void *sync_obj);
 
 #endif
diff --git a/include/linux/io.h b/include/linux/io.h
index 069e4075f872..f4f42faec686 100644
--- a/include/linux/io.h
+++ b/include/linux/io.h
@@ -76,4 +76,29 @@ void devm_ioremap_release(struct device *dev, void *res);
 #define arch_has_dev_port()     (1)
 #endif
 
+/*
+ * Some systems (x86 without PAT) have a somewhat reliable way to mark a
+ * physical address range such that uncached mappings will actually
+ * end up write-combining.  This facility should be used in conjunction
+ * with pgprot_writecombine, ioremap-wc, or set_memory_wc, since it has
+ * no effect if the per-page mechanisms are functional.
+ * (On x86 without PAT, these functions manipulate MTRRs.)
+ *
+ * arch_phys_del_wc(0) or arch_phys_del_wc(any error code) is guaranteed
+ * to have no effect.
+ */
+#ifndef arch_phys_wc_add
+static inline int __must_check arch_phys_wc_add(unsigned long base,
+                                                unsigned long size)
+{
+        return 0;  /* It worked (i.e. did nothing). */
+}
+
+static inline void arch_phys_wc_del(int handle)
+{
+}
+
+#define arch_phys_wc_add arch_phys_wc_add
+#endif
+
 #endif /* _LINUX_IO_H */
diff --git a/include/linux/mutex-debug.h b/include/linux/mutex-debug.h
index 731d77d6e155..4ac8b1977b73 100644
--- a/include/linux/mutex-debug.h
+++ b/include/linux/mutex-debug.h
@@ -3,6 +3,7 @@
 
 #include <linux/linkage.h>
 #include <linux/lockdep.h>
+#include <linux/debug_locks.h>
 
 /*
  * Mutexes - debugging helpers:
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index 433da8a1a426..3793ed7feeeb 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -10,6 +10,7 @@
 #ifndef __LINUX_MUTEX_H
 #define __LINUX_MUTEX_H
 
+#include <asm/current.h>
 #include <linux/list.h>
 #include <linux/spinlock_types.h>
 #include <linux/linkage.h>
@@ -77,6 +78,40 @@ struct mutex_waiter {
 #endif
 };
 
+struct ww_class {
+        atomic_long_t stamp;
+        struct lock_class_key acquire_key;
+        struct lock_class_key mutex_key;
+        const char *acquire_name;
+        const char *mutex_name;
+};
+
+struct ww_acquire_ctx {
+        struct task_struct *task;
+        unsigned long stamp;
+        unsigned acquired;
+#ifdef CONFIG_DEBUG_MUTEXES
+        unsigned done_acquire;
+        struct ww_class *ww_class;
+        struct ww_mutex *contending_lock;
+#endif
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+        struct lockdep_map dep_map;
+#endif
+#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
+        unsigned deadlock_inject_interval;
+        unsigned deadlock_inject_countdown;
+#endif
+};
+
+struct ww_mutex {
+        struct mutex base;
+        struct ww_acquire_ctx *ctx;
+#ifdef CONFIG_DEBUG_MUTEXES
+        struct ww_class *ww_class;
+#endif
+};
+
 #ifdef CONFIG_DEBUG_MUTEXES
 # include <linux/mutex-debug.h>
 #else
@@ -101,8 +136,11 @@ static inline void mutex_destroy(struct mutex *lock) {}
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 # define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
                 , .dep_map = { .name = #lockname }
+# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) \
+                , .ww_class = &ww_class
 #else
 # define __DEP_MAP_MUTEX_INITIALIZER(lockname)
+# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class)
 #endif
 
 #define __MUTEX_INITIALIZER(lockname) \
@@ -112,13 +150,49 @@ static inline void mutex_destroy(struct mutex *lock) {}
                 __DEBUG_MUTEX_INITIALIZER(lockname) \
                 __DEP_MAP_MUTEX_INITIALIZER(lockname) }
 
+#define __WW_CLASS_INITIALIZER(ww_class) \
+                { .stamp = ATOMIC_LONG_INIT(0) \
+                , .acquire_name = #ww_class "_acquire" \
+                , .mutex_name = #ww_class "_mutex" }
+
+#define __WW_MUTEX_INITIALIZER(lockname, class) \
+                { .base = { \__MUTEX_INITIALIZER(lockname) } \
+                __WW_CLASS_MUTEX_INITIALIZER(lockname, class) }
+
 #define DEFINE_MUTEX(mutexname) \
         struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
 
+#define DEFINE_WW_CLASS(classname) \
+        struct ww_class classname = __WW_CLASS_INITIALIZER(classname)
+
+#define DEFINE_WW_MUTEX(mutexname, ww_class) \
+        struct ww_mutex mutexname = __WW_MUTEX_INITIALIZER(mutexname, ww_class)
+
+
 extern void __mutex_init(struct mutex *lock, const char *name,
                          struct lock_class_key *key);
 
 /**
+ * ww_mutex_init - initialize the w/w mutex
+ * @lock: the mutex to be initialized
+ * @ww_class: the w/w class the mutex should belong to
+ *
+ * Initialize the w/w mutex to unlocked state and associate it with the given
+ * class.
+ *
+ * It is not allowed to initialize an already locked mutex.
+ */
+static inline void ww_mutex_init(struct ww_mutex *lock,
+                                 struct ww_class *ww_class)
+{
+        __mutex_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key);
+        lock->ctx = NULL;
+#ifdef CONFIG_DEBUG_MUTEXES
+        lock->ww_class = ww_class;
+#endif
+}
+
+/**
  * mutex_is_locked - is the mutex locked
  * @lock: the mutex to be queried
  *
@@ -136,6 +210,7 @@ static inline int mutex_is_locked(struct mutex *lock)
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass);
 extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
+
 extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock,
                                         unsigned int subclass);
 extern int __must_check mutex_lock_killable_nested(struct mutex *lock,
@@ -147,7 +222,7 @@ extern int __must_check mutex_lock_killable_nested(struct mutex *lock,
 
 #define mutex_lock_nest_lock(lock, nest_lock)                           \
 do {                                                                    \
-        typecheck(struct lockdep_map *, &(nest_lock)->dep_map);         \
+        typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
         _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map);             \
 } while (0)
 
@@ -170,6 +245,292 @@ extern int __must_check mutex_lock_killable(struct mutex *lock);
  */
 extern int mutex_trylock(struct mutex *lock);
 extern void mutex_unlock(struct mutex *lock);
+
+/**
+ * ww_acquire_init - initialize a w/w acquire context
+ * @ctx: w/w acquire context to initialize
+ * @ww_class: w/w class of the context
+ *
+ * Initializes an context to acquire multiple mutexes of the given w/w class.
+ *
+ * Context-based w/w mutex acquiring can be done in any order whatsoever within
+ * a given lock class. Deadlocks will be detected and handled with the
+ * wait/wound logic.
+ *
+ * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can
+ * result in undetected deadlocks and is so forbidden. Mixing different contexts
+ * for the same w/w class when acquiring mutexes can also result in undetected
+ * deadlocks, and is hence also forbidden. Both types of abuse will be caught by
+ * enabling CONFIG_PROVE_LOCKING.
+ *
+ * Nesting of acquire contexts for _different_ w/w classes is possible, subject
+ * to the usual locking rules between different lock classes.
+ *
+ * An acquire context must be released with ww_acquire_fini by the same task
+ * before the memory is freed. It is recommended to allocate the context itself
+ * on the stack.
+ */
+static inline void ww_acquire_init(struct ww_acquire_ctx *ctx,
+                                   struct ww_class *ww_class)
+{
+        ctx->task = current;
+        ctx->stamp = atomic_long_inc_return(&ww_class->stamp);
+        ctx->acquired = 0;
+#ifdef CONFIG_DEBUG_MUTEXES
+        ctx->ww_class = ww_class;
+        ctx->done_acquire = 0;
+        ctx->contending_lock = NULL;
+#endif
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+        debug_check_no_locks_freed((void *)ctx, sizeof(*ctx));
+        lockdep_init_map(&ctx->dep_map, ww_class->acquire_name,
+                         &ww_class->acquire_key, 0);
+        mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_);
+#endif
+#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
+        ctx->deadlock_inject_interval = 1;
+        ctx->deadlock_inject_countdown = ctx->stamp & 0xf;
+#endif
+}
+
+/**
+ * ww_acquire_done - marks the end of the acquire phase
+ * @ctx: the acquire context
+ *
+ * Marks the end of the acquire phase, any further w/w mutex lock calls using
+ * this context are forbidden.
+ *
+ * Calling this function is optional, it is just useful to document w/w mutex
+ * code and clearly designated the acquire phase from actually using the locked
+ * data structures.
+ */
+static inline void ww_acquire_done(struct ww_acquire_ctx *ctx)
+{
+#ifdef CONFIG_DEBUG_MUTEXES
+        lockdep_assert_held(ctx);
+
+        DEBUG_LOCKS_WARN_ON(ctx->done_acquire);
+        ctx->done_acquire = 1;
+#endif
+}
+
+/**
+ * ww_acquire_fini - releases a w/w acquire context
+ * @ctx: the acquire context to free
+ *
+ * Releases a w/w acquire context. This must be called _after_ all acquired w/w
+ * mutexes have been released with ww_mutex_unlock.
+ */
+static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx)
+{
+#ifdef CONFIG_DEBUG_MUTEXES
+        mutex_release(&ctx->dep_map, 0, _THIS_IP_);
+
+        DEBUG_LOCKS_WARN_ON(ctx->acquired);
+        if (!config_enabled(CONFIG_PROVE_LOCKING))
+                /*
+                 * lockdep will normally handle this,
+                 * but fail without anyway
+                 */
+                ctx->done_acquire = 1;
+
+        if (!config_enabled(CONFIG_DEBUG_LOCK_ALLOC))
+                /* ensure ww_acquire_fini will still fail if called twice */
+                ctx->acquired = ~0U;
+#endif
+}
+
+extern int __must_check __ww_mutex_lock(struct ww_mutex *lock,
+                                        struct ww_acquire_ctx *ctx);
+extern int __must_check __ww_mutex_lock_interruptible(struct ww_mutex *lock,
+                                                      struct ww_acquire_ctx *ctx);
+
+/**
+ * ww_mutex_lock - acquire the w/w mutex
+ * @lock: the mutex to be acquired
+ * @ctx: w/w acquire context, or NULL to acquire only a single lock.
+ *
+ * Lock the w/w mutex exclusively for this task.
+ *
+ * Deadlocks within a given w/w class of locks are detected and handled with the
+ * wait/wound algorithm. If the lock isn't immediately avaiable this function
+ * will either sleep until it is (wait case). Or it selects the current context
+ * for backing off by returning -EDEADLK (wound case). Trying to acquire the
+ * same lock with the same context twice is also detected and signalled by
+ * returning -EALREADY. Returns 0 if the mutex was successfully acquired.
+ *
+ * In the wound case the caller must release all currently held w/w mutexes for
+ * the given context and then wait for this contending lock to be available by
+ * calling ww_mutex_lock_slow. Alternatively callers can opt to not acquire this
+ * lock and proceed with trying to acquire further w/w mutexes (e.g. when
+ * scanning through lru lists trying to free resources).
+ *
+ * The mutex must later on be released by the same task that
+ * acquired it. The task may not exit without first unlocking the mutex. Also,
+ * kernel memory where the mutex resides must not be freed with the mutex still
+ * locked. The mutex must first be initialized (or statically defined) before it
+ * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be
+ * of the same w/w lock class as was used to initialize the acquire context.
+ *
+ * A mutex acquired with this function must be released with ww_mutex_unlock.
+ */
+static inline int ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        if (ctx)
+                return __ww_mutex_lock(lock, ctx);
+        else {
+                mutex_lock(&lock->base);
+                return 0;
+        }
+}
+
+/**
+ * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible
+ * @lock: the mutex to be acquired
+ * @ctx: w/w acquire context
+ *
+ * Lock the w/w mutex exclusively for this task.
+ *
+ * Deadlocks within a given w/w class of locks are detected and handled with the
+ * wait/wound algorithm. If the lock isn't immediately avaiable this function
+ * will either sleep until it is (wait case). Or it selects the current context
+ * for backing off by returning -EDEADLK (wound case). Trying to acquire the
+ * same lock with the same context twice is also detected and signalled by
+ * returning -EALREADY. Returns 0 if the mutex was successfully acquired. If a
+ * signal arrives while waiting for the lock then this function returns -EINTR.
+ *
+ * In the wound case the caller must release all currently held w/w mutexes for
+ * the given context and then wait for this contending lock to be available by
+ * calling ww_mutex_lock_slow_interruptible. Alternatively callers can opt to
+ * not acquire this lock and proceed with trying to acquire further w/w mutexes
+ * (e.g. when scanning through lru lists trying to free resources).
+ *
+ * The mutex must later on be released by the same task that
+ * acquired it. The task may not exit without first unlocking the mutex. Also,
+ * kernel memory where the mutex resides must not be freed with the mutex still
+ * locked. The mutex must first be initialized (or statically defined) before it
+ * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be
+ * of the same w/w lock class as was used to initialize the acquire context.
+ *
+ * A mutex acquired with this function must be released with ww_mutex_unlock.
+ */
+static inline int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock,
+                                                           struct ww_acquire_ctx *ctx)
+{
+        if (ctx)
+                return __ww_mutex_lock_interruptible(lock, ctx);
+        else
+                return mutex_lock_interruptible(&lock->base);
+}
+
+/**
+ * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex
+ * @lock: the mutex to be acquired
+ * @ctx: w/w acquire context
+ *
+ * Acquires a w/w mutex with the given context after a wound case. This function
+ * will sleep until the lock becomes available.
+ *
+ * The caller must have released all w/w mutexes already acquired with the
+ * context and then call this function on the contended lock.
+ *
+ * Afterwards the caller may continue to (re)acquire the other w/w mutexes it
+ * needs with ww_mutex_lock. Note that the -EALREADY return code from
+ * ww_mutex_lock can be used to avoid locking this contended mutex twice.
+ *
+ * It is forbidden to call this function with any other w/w mutexes associated
+ * with the context held. It is forbidden to call this on anything else than the
+ * contending mutex.
+ *
+ * Note that the slowpath lock acquiring can also be done by calling
+ * ww_mutex_lock directly. This function here is simply to help w/w mutex
+ * locking code readability by clearly denoting the slowpath.
+ */
+static inline void
+ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        int ret;
+#ifdef CONFIG_DEBUG_MUTEXES
+        DEBUG_LOCKS_WARN_ON(!ctx->contending_lock);
+#endif
+        ret = ww_mutex_lock(lock, ctx);
+        (void)ret;
+}
+
+/**
+ * ww_mutex_lock_slow_interruptible - slowpath acquiring of the w/w mutex,
+ *                                    interruptible
+ * @lock: the mutex to be acquired
+ * @ctx: w/w acquire context
+ *
+ * Acquires a w/w mutex with the given context after a wound case. This function
+ * will sleep until the lock becomes available and returns 0 when the lock has
+ * been acquired. If a signal arrives while waiting for the lock then this
+ * function returns -EINTR.
+ *
+ * The caller must have released all w/w mutexes already acquired with the
+ * context and then call this function on the contended lock.
+ *
+ * Afterwards the caller may continue to (re)acquire the other w/w mutexes it
+ * needs with ww_mutex_lock. Note that the -EALREADY return code from
+ * ww_mutex_lock can be used to avoid locking this contended mutex twice.
+ *
+ * It is forbidden to call this function with any other w/w mutexes associated
+ * with the given context held. It is forbidden to call this on anything else
+ * than the contending mutex.
+ *
+ * Note that the slowpath lock acquiring can also be done by calling
+ * ww_mutex_lock_interruptible directly. This function here is simply to help
+ * w/w mutex locking code readability by clearly denoting the slowpath.
+ */
+static inline int __must_check
+ww_mutex_lock_slow_interruptible(struct ww_mutex *lock,
+                                 struct ww_acquire_ctx *ctx)
+{
+#ifdef CONFIG_DEBUG_MUTEXES
+        DEBUG_LOCKS_WARN_ON(!ctx->contending_lock);
+#endif
+        return ww_mutex_lock_interruptible(lock, ctx);
+}
+
+extern void ww_mutex_unlock(struct ww_mutex *lock);
+
+/**
+ * ww_mutex_trylock - tries to acquire the w/w mutex without acquire context
+ * @lock: mutex to lock
+ *
+ * Trylocks a mutex without acquire context, so no deadlock detection is
+ * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise.
+ */
+static inline int __must_check ww_mutex_trylock(struct ww_mutex *lock)
+{
+        return mutex_trylock(&lock->base);
+}
+
+/***
+ * ww_mutex_destroy - mark a w/w mutex unusable
+ * @lock: the mutex to be destroyed
+ *
+ * This function marks the mutex uninitialized, and any subsequent
+ * use of the mutex is forbidden. The mutex must not be locked when
+ * this function is called.
+ */
+static inline void ww_mutex_destroy(struct ww_mutex *lock)
+{
+        mutex_destroy(&lock->base);
+}
+
+/**
+ * ww_mutex_is_locked - is the w/w mutex locked
+ * @lock: the mutex to be queried
+ *
+ * Returns 1 if the mutex is locked, 0 if unlocked.
+ */
+static inline bool ww_mutex_is_locked(struct ww_mutex *lock)
+{
+        return mutex_is_locked(&lock->base);
+}
+
 extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
 
 #ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX
diff --git a/include/linux/platform_data/rcar-du.h b/include/linux/platform_data/rcar-du.h
new file mode 100644
index 000000000000..80587fdbba3e
--- /dev/null
+++ b/include/linux/platform_data/rcar-du.h
@@ -0,0 +1,54 @@
+/*
+ * rcar_du.h  --  R-Car Display Unit DRM driver
+ *
+ * Copyright (C) 2013 Renesas Corporation
+ *
+ * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __RCAR_DU_H__
+#define __RCAR_DU_H__
+
+#include <drm/drm_mode.h>
+
+enum rcar_du_encoder_type {
+        RCAR_DU_ENCODER_UNUSED = 0,
+        RCAR_DU_ENCODER_VGA,
+        RCAR_DU_ENCODER_LVDS,
+};
+
+struct rcar_du_panel_data {
+        unsigned int width_mm;          /* Panel width in mm */
+        unsigned int height_mm;         /* Panel height in mm */
+        struct drm_mode_modeinfo mode;
+};
+
+struct rcar_du_encoder_lvds_data {
+        struct rcar_du_panel_data panel;
+};
+
+struct rcar_du_encoder_vga_data {
+        /* TODO: Add DDC information for EDID retrieval */
+};
+
+struct rcar_du_encoder_data {
+        enum rcar_du_encoder_type encoder;
+        unsigned int output;
+
+        union {
+                struct rcar_du_encoder_lvds_data lvds;
+                struct rcar_du_encoder_vga_data vga;
+        } u;
+};
+
+struct rcar_du_platform_data {
+        struct rcar_du_encoder_data *encoders;
+        unsigned int num_encoders;
+};
+
+#endif /* __RCAR_DU_H__ */
diff --git a/include/linux/reservation.h b/include/linux/reservation.h
new file mode 100644
index 000000000000..e9ee806a9d72
--- /dev/null
+++ b/include/linux/reservation.h
@@ -0,0 +1,62 @@
+/*
+ * Header file for reservations for dma-buf and ttm
+ *
+ * Copyright(C) 2011 Linaro Limited. All rights reserved.
+ * Copyright (C) 2012-2013 Canonical Ltd
+ * Copyright (C) 2012 Texas Instruments
+ *
+ * Authors:
+ * Rob Clark <rob.clark@linaro.org>
+ * Maarten Lankhorst <maarten.lankhorst@canonical.com>
+ * Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ *
+ * Based on bo.c which bears the following copyright notice,
+ * but is dual licensed:
+ *
+ * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _LINUX_RESERVATION_H
+#define _LINUX_RESERVATION_H
+
+#include <linux/mutex.h>
+
+extern struct ww_class reservation_ww_class;
+
+struct reservation_object {
+        struct ww_mutex lock;
+};
+
+static inline void
+reservation_object_init(struct reservation_object *obj)
+{
+        ww_mutex_init(&obj->lock, &reservation_ww_class);
+}
+
+static inline void
+reservation_object_fini(struct reservation_object *obj)
+{
+        ww_mutex_destroy(&obj->lock);
+}
+
+#endif /* _LINUX_RESERVATION_H */
diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h
index 5a57be68bab7..238a166b9fe6 100644
--- a/include/uapi/drm/drm.h
+++ b/include/uapi/drm/drm.h
@@ -732,6 +732,7 @@ struct drm_prime_handle {
 #define DRM_IOCTL_MODE_ADDFB2           DRM_IOWR(0xB8, struct drm_mode_fb_cmd2)
 #define DRM_IOCTL_MODE_OBJ_GETPROPERTIES        DRM_IOWR(0xB9, struct drm_mode_obj_get_properties)
 #define DRM_IOCTL_MODE_OBJ_SETPROPERTY  DRM_IOWR(0xBA, struct drm_mode_obj_set_property)
+#define DRM_IOCTL_MODE_CURSOR2          DRM_IOWR(0xBB, struct drm_mode_cursor2)
 
 /**
  * Device specific ioctls should only be in their respective headers
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index 090e5331ab7e..53db7cea373b 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -388,6 +388,19 @@ struct drm_mode_cursor {
         __u32 handle;
 };
 
+struct drm_mode_cursor2 {
+        __u32 flags;
+        __u32 crtc_id;
+        __s32 x;
+        __s32 y;
+        __u32 width;
+        __u32 height;
+        /* driver specific handle */
+        __u32 handle;
+        __s32 hot_x;
+        __s32 hot_y;
+};
+
 struct drm_mode_crtc_lut {
         __u32 crtc_id;
         __u32 gamma_size;
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index 07d59419fe6b..923ed7fe5775 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -305,7 +305,7 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_HAS_WAIT_TIMEOUT      19
 #define I915_PARAM_HAS_SEMAPHORES        20
 #define I915_PARAM_HAS_PRIME_VMAP_FLUSH  21
-#define I915_PARAM_RSVD_FOR_FUTURE_USE   22
+#define I915_PARAM_HAS_VEBOX             22
 #define I915_PARAM_HAS_SECURE_BATCHES    23
 #define I915_PARAM_HAS_PINNED_BATCHES    24
 #define I915_PARAM_HAS_EXEC_NO_RELOC     25
@@ -660,6 +660,7 @@ struct drm_i915_gem_execbuffer2 {
 #define I915_EXEC_RENDER                 (1<<0)
 #define I915_EXEC_BSD                    (2<<0)
 #define I915_EXEC_BLT                    (3<<0)
+#define I915_EXEC_VEBOX                  (4<<0)
 
 /* Used for switching the constants addressing mode on gen4+ RENDER ring.
  * Gen6+ only supports relative addressing to dynamic state (default) and
diff --git a/include/uapi/drm/tegra_drm.h b/include/uapi/drm/tegra_drm.h
index 6e132a2f7420..73bde4eaf16c 100644
--- a/include/uapi/drm/tegra_drm.h
+++ b/include/uapi/drm/tegra_drm.h
@@ -17,6 +17,8 @@
 #ifndef _UAPI_TEGRA_DRM_H_
 #define _UAPI_TEGRA_DRM_H_
 
+#include <drm/drm.h>
+
 struct drm_tegra_gem_create {
         __u64 size;
         __u32 flags;
diff --git a/include/video/display_timing.h b/include/video/display_timing.h
index 5d0259b08e01..28d9d0d566ca 100644
--- a/include/video/display_timing.h
+++ b/include/video/display_timing.h
@@ -27,6 +27,7 @@ enum display_flags {
         DISPLAY_FLAGS_PIXDATA_NEGEDGE   = BIT(7),
         DISPLAY_FLAGS_INTERLACED        = BIT(8),
         DISPLAY_FLAGS_DOUBLESCAN        = BIT(9),
+        DISPLAY_FLAGS_DOUBLECLK         = BIT(10),
 };
 
 /*
diff --git a/include/video/uvesafb.h b/include/video/uvesafb.h
index 1a91850cb961..30f53625415c 100644
--- a/include/video/uvesafb.h
+++ b/include/video/uvesafb.h
@@ -134,6 +134,7 @@ struct uvesafb_par {
 
         int mode_idx;
         struct vbe_crtc_ib crtc;
+        int mtrr_handle;
 };
 
 #endif /* _UVESAFB_H */
diff --git a/kernel/mutex.c b/kernel/mutex.c
index ad53a664f113..e581ada5faf4 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -254,16 +254,165 @@ void __sched mutex_unlock(struct mutex *lock)
 
 EXPORT_SYMBOL(mutex_unlock);
 
+/**
+ * ww_mutex_unlock - release the w/w mutex
+ * @lock: the mutex to be released
+ *
+ * Unlock a mutex that has been locked by this task previously with any of the
+ * ww_mutex_lock* functions (with or without an acquire context). It is
+ * forbidden to release the locks after releasing the acquire context.
+ *
+ * This function must not be used in interrupt context. Unlocking
+ * of a unlocked mutex is not allowed.
+ */
+void __sched ww_mutex_unlock(struct ww_mutex *lock)
+{
+        /*
+         * The unlocking fastpath is the 0->1 transition from 'locked'
+         * into 'unlocked' state:
+         */
+        if (lock->ctx) {
+#ifdef CONFIG_DEBUG_MUTEXES
+                DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
+#endif
+                if (lock->ctx->acquired > 0)
+                        lock->ctx->acquired--;
+                lock->ctx = NULL;
+        }
+
+#ifndef CONFIG_DEBUG_MUTEXES
+        /*
+         * When debugging is enabled we must not clear the owner before time,
+         * the slow path will always be taken, and that clears the owner field
+         * after verifying that it was indeed current.
+         */
+        mutex_clear_owner(&lock->base);
+#endif
+        __mutex_fastpath_unlock(&lock->base.count, __mutex_unlock_slowpath);
+}
+EXPORT_SYMBOL(ww_mutex_unlock);
+
+static inline int __sched
+__mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
+        struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
+
+        if (!hold_ctx)
+                return 0;
+
+        if (unlikely(ctx == hold_ctx))
+                return -EALREADY;
+
+        if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
+            (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
+#ifdef CONFIG_DEBUG_MUTEXES
+                DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
+                ctx->contending_lock = ww;
+#endif
+                return -EDEADLK;
+        }
+
+        return 0;
+}
+
+static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
+                                                   struct ww_acquire_ctx *ww_ctx)
+{
+#ifdef CONFIG_DEBUG_MUTEXES
+        /*
+         * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
+         * but released with a normal mutex_unlock in this call.
+         *
+         * This should never happen, always use ww_mutex_unlock.
+         */
+        DEBUG_LOCKS_WARN_ON(ww->ctx);
+
+        /*
+         * Not quite done after calling ww_acquire_done() ?
+         */
+        DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
+
+        if (ww_ctx->contending_lock) {
+                /*
+                 * After -EDEADLK you tried to
+                 * acquire a different ww_mutex? Bad!
+                 */
+                DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
+
+                /*
+                 * You called ww_mutex_lock after receiving -EDEADLK,
+                 * but 'forgot' to unlock everything else first?
+                 */
+                DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
+                ww_ctx->contending_lock = NULL;
+        }
+
+        /*
+         * Naughty, using a different class will lead to undefined behavior!
+         */
+        DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
+#endif
+        ww_ctx->acquired++;
+}
+
+/*
+ * after acquiring lock with fastpath or when we lost out in contested
+ * slowpath, set ctx and wake up any waiters so they can recheck.
+ *
+ * This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
+ * as the fastpath and opportunistic spinning are disabled in that case.
+ */
+static __always_inline void
+ww_mutex_set_context_fastpath(struct ww_mutex *lock,
+                               struct ww_acquire_ctx *ctx)
+{
+        unsigned long flags;
+        struct mutex_waiter *cur;
+
+        ww_mutex_lock_acquired(lock, ctx);
+
+        lock->ctx = ctx;
+
+        /*
+         * The lock->ctx update should be visible on all cores before
+         * the atomic read is done, otherwise contended waiters might be
+         * missed. The contended waiters will either see ww_ctx == NULL
+         * and keep spinning, or it will acquire wait_lock, add itself
+         * to waiter list and sleep.
+         */
+        smp_mb(); /* ^^^ */
+
+        /*
+         * Check if lock is contended, if not there is nobody to wake up
+         */
+        if (likely(atomic_read(&lock->base.count) == 0))
+                return;
+
+        /*
+         * Uh oh, we raced in fastpath, wake up everyone in this case,
+         * so they can see the new lock->ctx.
+         */
+        spin_lock_mutex(&lock->base.wait_lock, flags);
+        list_for_each_entry(cur, &lock->base.wait_list, list) {
+                debug_mutex_wake_waiter(&lock->base, cur);
+                wake_up_process(cur->task);
+        }
+        spin_unlock_mutex(&lock->base.wait_lock, flags);
+}
+
 /*
  * Lock a mutex (possibly interruptible), slowpath:
  */
-static inline int __sched
+static __always_inline int __sched
 __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
-                    struct lockdep_map *nest_lock, unsigned long ip)
+                    struct lockdep_map *nest_lock, unsigned long ip,
+                    struct ww_acquire_ctx *ww_ctx)
 {
         struct task_struct *task = current;
         struct mutex_waiter waiter;
         unsigned long flags;
+        int ret;
 
         preempt_disable();
         mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
@@ -298,6 +447,22 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 struct task_struct *owner;
                 struct mspin_node  node;
 
+                if (!__builtin_constant_p(ww_ctx == NULL) && ww_ctx->acquired > 0) {
+                        struct ww_mutex *ww;
+
+                        ww = container_of(lock, struct ww_mutex, base);
+                        /*
+                         * If ww->ctx is set the contents are undefined, only
+                         * by acquiring wait_lock there is a guarantee that
+                         * they are not invalid when reading.
+                         *
+                         * As such, when deadlock detection needs to be
+                         * performed the optimistic spinning cannot be done.
+                         */
+                        if (ACCESS_ONCE(ww->ctx))
+                                break;
+                }
+
                 /*
                  * If there's an owner, wait for it to either
                  * release the lock or go to sleep.
@@ -312,6 +477,13 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 if ((atomic_read(&lock->count) == 1) &&
                     (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
                         lock_acquired(&lock->dep_map, ip);
+                        if (!__builtin_constant_p(ww_ctx == NULL)) {
+                                struct ww_mutex *ww;
+                                ww = container_of(lock, struct ww_mutex, base);
+
+                                ww_mutex_set_context_fastpath(ww, ww_ctx);
+                        }
+
                         mutex_set_owner(lock);
                         mspin_unlock(MLOCK(lock), &node);
                         preempt_enable();
@@ -371,15 +543,16 @@ slowpath:
                  * TASK_UNINTERRUPTIBLE case.)
                  */
                 if (unlikely(signal_pending_state(state, task))) {
-                        mutex_remove_waiter(lock, &waiter,
-                                            task_thread_info(task));
-                        mutex_release(&lock->dep_map, 1, ip);
-                        spin_unlock_mutex(&lock->wait_lock, flags);
+                        ret = -EINTR;
+                        goto err;
+                }
 
-                        debug_mutex_free_waiter(&waiter);
-                        preempt_enable();
-                        return -EINTR;
+                if (!__builtin_constant_p(ww_ctx == NULL) && ww_ctx->acquired > 0) {
+                        ret = __mutex_lock_check_stamp(lock, ww_ctx);
+                        if (ret)
+                                goto err;
                 }
+
                 __set_task_state(task, state);
 
                 /* didn't get the lock, go to sleep: */
@@ -394,6 +567,30 @@ done:
         mutex_remove_waiter(lock, &waiter, current_thread_info());
         mutex_set_owner(lock);
 
+        if (!__builtin_constant_p(ww_ctx == NULL)) {
+                struct ww_mutex *ww = container_of(lock,
+                                                      struct ww_mutex,
+                                                      base);
+                struct mutex_waiter *cur;
+
+                /*
+                 * This branch gets optimized out for the common case,
+                 * and is only important for ww_mutex_lock.
+                 */
+
+                ww_mutex_lock_acquired(ww, ww_ctx);
+                ww->ctx = ww_ctx;
+
+                /*
+                 * Give any possible sleeping processes the chance to wake up,
+                 * so they can recheck if they have to back off.
+                 */
+                list_for_each_entry(cur, &lock->wait_list, list) {
+                        debug_mutex_wake_waiter(lock, cur);
+                        wake_up_process(cur->task);
+                }
+        }
+
         /* set it to 0 if there are no waiters left: */
         if (likely(list_empty(&lock->wait_list)))
                 atomic_set(&lock->count, 0);
@@ -404,6 +601,14 @@ done:
         preempt_enable();
 
         return 0;
+
+err:
+        mutex_remove_waiter(lock, &waiter, task_thread_info(task));
+        spin_unlock_mutex(&lock->wait_lock, flags);
+        debug_mutex_free_waiter(&waiter);
+        mutex_release(&lock->dep_map, 1, ip);
+        preempt_enable();
+        return ret;
 }
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -411,7 +616,8 @@ void __sched
 mutex_lock_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
-        __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
+        __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
+                            subclass, NULL, _RET_IP_, NULL);
 }
 
 EXPORT_SYMBOL_GPL(mutex_lock_nested);
@@ -420,7 +626,8 @@ void __sched
 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
 {
         might_sleep();
-        __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
+        __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
+                            0, nest, _RET_IP_, NULL);
 }
 
 EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
@@ -429,7 +636,8 @@ int __sched
 mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
-        return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);
+        return __mutex_lock_common(lock, TASK_KILLABLE,
+                                   subclass, NULL, _RET_IP_, NULL);
 }
 EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
 
@@ -438,10 +646,68 @@ mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
 {
         might_sleep();
         return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
-                                   subclass, NULL, _RET_IP_);
+                                   subclass, NULL, _RET_IP_, NULL);
 }
 
 EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
+
+static inline int
+ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
+        unsigned tmp;
+
+        if (ctx->deadlock_inject_countdown-- == 0) {
+                tmp = ctx->deadlock_inject_interval;
+                if (tmp > UINT_MAX/4)
+                        tmp = UINT_MAX;
+                else
+                        tmp = tmp*2 + tmp + tmp/2;
+
+                ctx->deadlock_inject_interval = tmp;
+                ctx->deadlock_inject_countdown = tmp;
+                ctx->contending_lock = lock;
+
+                ww_mutex_unlock(lock);
+
+                return -EDEADLK;
+        }
+#endif
+
+        return 0;
+}
+
+int __sched
+__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        int ret;
+
+        might_sleep();
+        ret =  __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
+                                   0, &ctx->dep_map, _RET_IP_, ctx);
+        if (!ret && ctx->acquired > 0)
+                return ww_mutex_deadlock_injection(lock, ctx);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(__ww_mutex_lock);
+
+int __sched
+__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        int ret;
+
+        might_sleep();
+        ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
+                                  0, &ctx->dep_map, _RET_IP_, ctx);
+
+        if (!ret && ctx->acquired > 0)
+                return ww_mutex_deadlock_injection(lock, ctx);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
+
 #endif
 
 /*
@@ -494,10 +760,10 @@ __mutex_unlock_slowpath(atomic_t *lock_count)
  * mutex_lock_interruptible() and mutex_trylock().
  */
 static noinline int __sched
-__mutex_lock_killable_slowpath(atomic_t *lock_count);
+__mutex_lock_killable_slowpath(struct mutex *lock);
 
 static noinline int __sched
-__mutex_lock_interruptible_slowpath(atomic_t *lock_count);
+__mutex_lock_interruptible_slowpath(struct mutex *lock);
 
 /**
  * mutex_lock_interruptible - acquire the mutex, interruptible
@@ -515,12 +781,12 @@ int __sched mutex_lock_interruptible(struct mutex *lock)
         int ret;
 
         might_sleep();
-        ret =  __mutex_fastpath_lock_retval
-                        (&lock->count, __mutex_lock_interruptible_slowpath);
-        if (!ret)
+        ret =  __mutex_fastpath_lock_retval(&lock->count);
+        if (likely(!ret)) {
                 mutex_set_owner(lock);
-
-        return ret;
+                return 0;
+        } else
+                return __mutex_lock_interruptible_slowpath(lock);
 }
 
 EXPORT_SYMBOL(mutex_lock_interruptible);
@@ -530,12 +796,12 @@ int __sched mutex_lock_killable(struct mutex *lock)
         int ret;
 
         might_sleep();
-        ret = __mutex_fastpath_lock_retval
-                        (&lock->count, __mutex_lock_killable_slowpath);
-        if (!ret)
+        ret = __mutex_fastpath_lock_retval(&lock->count);
+        if (likely(!ret)) {
                 mutex_set_owner(lock);
-
-        return ret;
+                return 0;
+        } else
+                return __mutex_lock_killable_slowpath(lock);
 }
 EXPORT_SYMBOL(mutex_lock_killable);
 
@@ -544,24 +810,39 @@ __mutex_lock_slowpath(atomic_t *lock_count)
 {
         struct mutex *lock = container_of(lock_count, struct mutex, count);
 
-        __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
+        __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0,
+                            NULL, _RET_IP_, NULL);
 }
 
 static noinline int __sched
-__mutex_lock_killable_slowpath(atomic_t *lock_count)
+__mutex_lock_killable_slowpath(struct mutex *lock)
 {
-        struct mutex *lock = container_of(lock_count, struct mutex, count);
+        return __mutex_lock_common(lock, TASK_KILLABLE, 0,
+                                   NULL, _RET_IP_, NULL);
+}
 
-        return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
+static noinline int __sched
+__mutex_lock_interruptible_slowpath(struct mutex *lock)
+{
+        return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0,
+                                   NULL, _RET_IP_, NULL);
 }
 
 static noinline int __sched
-__mutex_lock_interruptible_slowpath(atomic_t *lock_count)
+__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
 {
-        struct mutex *lock = container_of(lock_count, struct mutex, count);
+        return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0,
+                                   NULL, _RET_IP_, ctx);
+}
 
-        return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
+static noinline int __sched
+__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
+                                            struct ww_acquire_ctx *ctx)
+{
+        return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0,
+                                   NULL, _RET_IP_, ctx);
 }
+
 #endif
 
 /*
@@ -617,6 +898,45 @@ int __sched mutex_trylock(struct mutex *lock)
 }
 EXPORT_SYMBOL(mutex_trylock);
 
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
+int __sched
+__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        int ret;
+
+        might_sleep();
+
+        ret = __mutex_fastpath_lock_retval(&lock->base.count);
+
+        if (likely(!ret)) {
+                ww_mutex_set_context_fastpath(lock, ctx);
+                mutex_set_owner(&lock->base);
+        } else
+                ret = __ww_mutex_lock_slowpath(lock, ctx);
+        return ret;
+}
+EXPORT_SYMBOL(__ww_mutex_lock);
+
+int __sched
+__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
+{
+        int ret;
+
+        might_sleep();
+
+        ret = __mutex_fastpath_lock_retval(&lock->base.count);
+
+        if (likely(!ret)) {
+                ww_mutex_set_context_fastpath(lock, ctx);
+                mutex_set_owner(&lock->base);
+        } else
+                ret = __ww_mutex_lock_interruptible_slowpath(lock, ctx);
+        return ret;
+}
+EXPORT_SYMBOL(__ww_mutex_lock_interruptible);
+
+#endif
+
 /**
  * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
  * @cnt: the atomic which we are to dec
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 566cf2bc08ea..7154f799541a 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -547,6 +547,19 @@ config DEBUG_MUTEXES
          This feature allows mutex semantics violations to be detected and
          reported.
 
+config DEBUG_WW_MUTEX_SLOWPATH
+        bool "Wait/wound mutex debugging: Slowpath testing"
+        depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
+        select DEBUG_LOCK_ALLOC
+        select DEBUG_SPINLOCK
+        select DEBUG_MUTEXES
+        help
+         This feature enables slowpath testing for w/w mutex users by
+         injecting additional -EDEADLK wound/backoff cases. Together with
+         the full mutex checks enabled with (CONFIG_PROVE_LOCKING) this
+         will test all possible w/w mutex interface abuse with the
+         exception of simply not acquiring all the required locks.
+
 config DEBUG_LOCK_ALLOC
         bool "Lock debugging: detect incorrect freeing of live locks"
         depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
diff --git a/lib/debug_locks.c b/lib/debug_locks.c
index f2fa60c59343..96c4c633d95e 100644
--- a/lib/debug_locks.c
+++ b/lib/debug_locks.c
@@ -30,6 +30,7 @@ EXPORT_SYMBOL_GPL(debug_locks);
  * a locking bug is detected.
  */
 int debug_locks_silent;
+EXPORT_SYMBOL_GPL(debug_locks_silent);
 
 /*
  * Generic 'turn off all lock debugging' function:
@@ -44,3 +45,4 @@ int debug_locks_off(void)
         }
         return 0;
 }
+EXPORT_SYMBOL_GPL(debug_locks_off);
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index c3eb261a7df3..aad024dde3c4 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -26,6 +26,8 @@
  */
 static unsigned int debug_locks_verbose;
 
+static DEFINE_WW_CLASS(ww_lockdep);
+
 static int __init setup_debug_locks_verbose(char *str)
 {
         get_option(&str, &debug_locks_verbose);
@@ -42,6 +44,10 @@ __setup("debug_locks_verbose=", setup_debug_locks_verbose);
 #define LOCKTYPE_RWLOCK 0x2
 #define LOCKTYPE_MUTEX  0x4
 #define LOCKTYPE_RWSEM  0x8
+#define LOCKTYPE_WW     0x10
+
+static struct ww_acquire_ctx t, t2;
+static struct ww_mutex o, o2, o3;
 
 /*
  * Normal standalone locks, for the circular and irq-context
@@ -193,6 +199,20 @@ static void init_shared_classes(void)
 #define RSU(x)                  up_read(&rwsem_##x)
 #define RWSI(x)                 init_rwsem(&rwsem_##x)
 
+#ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
+#define WWAI(x)                 ww_acquire_init(x, &ww_lockdep)
+#else
+#define WWAI(x)                 do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0)
+#endif
+#define WWAD(x)                 ww_acquire_done(x)
+#define WWAF(x)                 ww_acquire_fini(x)
+
+#define WWL(x, c)               ww_mutex_lock(x, c)
+#define WWT(x)                  ww_mutex_trylock(x)
+#define WWL1(x)                 ww_mutex_lock(x, NULL)
+#define WWU(x)                  ww_mutex_unlock(x)
+
+
 #define LOCK_UNLOCK_2(x,y)      LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x)
 
 /*
@@ -894,11 +914,13 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
 # define I_RWLOCK(x)    lockdep_reset_lock(&rwlock_##x.dep_map)
 # define I_MUTEX(x)     lockdep_reset_lock(&mutex_##x.dep_map)
 # define I_RWSEM(x)     lockdep_reset_lock(&rwsem_##x.dep_map)
+# define I_WW(x)        lockdep_reset_lock(&x.dep_map)
 #else
 # define I_SPINLOCK(x)
 # define I_RWLOCK(x)
 # define I_MUTEX(x)
 # define I_RWSEM(x)
+# define I_WW(x)
 #endif
 
 #define I1(x)                                   \
@@ -920,11 +942,20 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
 static void reset_locks(void)
 {
         local_irq_disable();
+        lockdep_free_key_range(&ww_lockdep.acquire_key, 1);
+        lockdep_free_key_range(&ww_lockdep.mutex_key, 1);
+
         I1(A); I1(B); I1(C); I1(D);
         I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2);
+        I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base);
         lockdep_reset();
         I2(A); I2(B); I2(C); I2(D);
         init_shared_classes();
+
+        ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep);
+        memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2));
+        memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key));
+        memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key));
         local_irq_enable();
 }
 
@@ -938,7 +969,6 @@ static int unexpected_testcase_failures;
 static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
 {
         unsigned long saved_preempt_count = preempt_count();
-        int expected_failure = 0;
 
         WARN_ON(irqs_disabled());
 
@@ -947,25 +977,17 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
          * Filter out expected failures:
          */
 #ifndef CONFIG_PROVE_LOCKING
-        if ((lockclass_mask & LOCKTYPE_SPIN) && debug_locks != expected)
-                expected_failure = 1;
-        if ((lockclass_mask & LOCKTYPE_RWLOCK) && debug_locks != expected)
-                expected_failure = 1;
-        if ((lockclass_mask & LOCKTYPE_MUTEX) && debug_locks != expected)
-                expected_failure = 1;
-        if ((lockclass_mask & LOCKTYPE_RWSEM) && debug_locks != expected)
-                expected_failure = 1;
+        if (expected == FAILURE && debug_locks) {
+                expected_testcase_failures++;
+                printk("failed|");
+        }
+        else
 #endif
         if (debug_locks != expected) {
-                if (expected_failure) {
-                        expected_testcase_failures++;
-                        printk("failed|");
-                } else {
-                        unexpected_testcase_failures++;
-
-                        printk("FAILED|");
-                        dump_stack();
-                }
+                unexpected_testcase_failures++;
+                printk("FAILED|");
+
+                dump_stack();
         } else {
                 testcase_successes++;
                 printk("  ok  |");
@@ -1108,6 +1130,666 @@ static inline void print_testname(const char *testname)
         DO_TESTCASE_6IRW(desc, name, 312);                      \
         DO_TESTCASE_6IRW(desc, name, 321);
 
+static void ww_test_fail_acquire(void)
+{
+        int ret;
+
+        WWAI(&t);
+        t.stamp++;
+
+        ret = WWL(&o, &t);
+
+        if (WARN_ON(!o.ctx) ||
+            WARN_ON(ret))
+                return;
+
+        /* No lockdep test, pure API */
+        ret = WWL(&o, &t);
+        WARN_ON(ret != -EALREADY);
+
+        ret = WWT(&o);
+        WARN_ON(ret);
+
+        t2 = t;
+        t2.stamp++;
+        ret = WWL(&o, &t2);
+        WARN_ON(ret != -EDEADLK);
+        WWU(&o);
+
+        if (WWT(&o))
+                WWU(&o);
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+        else
+                DEBUG_LOCKS_WARN_ON(1);
+#endif
+}
+
+static void ww_test_normal(void)
+{
+        int ret;
+
+        WWAI(&t);
+
+        /*
+         * None of the ww_mutex codepaths should be taken in the 'normal'
+         * mutex calls. The easiest way to verify this is by using the
+         * normal mutex calls, and making sure o.ctx is unmodified.
+         */
+
+        /* mutex_lock (and indirectly, mutex_lock_nested) */
+        o.ctx = (void *)~0UL;
+        mutex_lock(&o.base);
+        mutex_unlock(&o.base);
+        WARN_ON(o.ctx != (void *)~0UL);
+
+        /* mutex_lock_interruptible (and *_nested) */
+        o.ctx = (void *)~0UL;
+        ret = mutex_lock_interruptible(&o.base);
+        if (!ret)
+                mutex_unlock(&o.base);
+        else
+                WARN_ON(1);
+        WARN_ON(o.ctx != (void *)~0UL);
+
+        /* mutex_lock_killable (and *_nested) */
+        o.ctx = (void *)~0UL;
+        ret = mutex_lock_killable(&o.base);
+        if (!ret)
+                mutex_unlock(&o.base);
+        else
+                WARN_ON(1);
+        WARN_ON(o.ctx != (void *)~0UL);
+
+        /* trylock, succeeding */
+        o.ctx = (void *)~0UL;
+        ret = mutex_trylock(&o.base);
+        WARN_ON(!ret);
+        if (ret)
+                mutex_unlock(&o.base);
+        else
+                WARN_ON(1);
+        WARN_ON(o.ctx != (void *)~0UL);
+
+        /* trylock, failing */
+        o.ctx = (void *)~0UL;
+        mutex_lock(&o.base);
+        ret = mutex_trylock(&o.base);
+        WARN_ON(ret);
+        mutex_unlock(&o.base);
+        WARN_ON(o.ctx != (void *)~0UL);
+
+        /* nest_lock */
+        o.ctx = (void *)~0UL;
+        mutex_lock_nest_lock(&o.base, &t);
+        mutex_unlock(&o.base);
+        WARN_ON(o.ctx != (void *)~0UL);
+}
+
+static void ww_test_two_contexts(void)
+{
+        WWAI(&t);
+        WWAI(&t2);
+}
+
+static void ww_test_diff_class(void)
+{
+        WWAI(&t);
+#ifdef CONFIG_DEBUG_MUTEXES
+        t.ww_class = NULL;
+#endif
+        WWL(&o, &t);
+}
+
+static void ww_test_context_done_twice(void)
+{
+        WWAI(&t);
+        WWAD(&t);
+        WWAD(&t);
+        WWAF(&t);
+}
+
+static void ww_test_context_unlock_twice(void)
+{
+        WWAI(&t);
+        WWAD(&t);
+        WWAF(&t);
+        WWAF(&t);
+}
+
+static void ww_test_context_fini_early(void)
+{
+        WWAI(&t);
+        WWL(&o, &t);
+        WWAD(&t);
+        WWAF(&t);
+}
+
+static void ww_test_context_lock_after_done(void)
+{
+        WWAI(&t);
+        WWAD(&t);
+        WWL(&o, &t);
+}
+
+static void ww_test_object_unlock_twice(void)
+{
+        WWL1(&o);
+        WWU(&o);
+        WWU(&o);
+}
+
+static void ww_test_object_lock_unbalanced(void)
+{
+        WWAI(&t);
+        WWL(&o, &t);
+        t.acquired = 0;
+        WWU(&o);
+        WWAF(&t);
+}
+
+static void ww_test_object_lock_stale_context(void)
+{
+        WWAI(&t);
+        o.ctx = &t2;
+        WWL(&o, &t);
+}
+
+static void ww_test_edeadlk_normal(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        o2.ctx = &t2;
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        o2.ctx = NULL;
+        mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
+        mutex_unlock(&o2.base);
+        WWU(&o);
+
+        WWL(&o2, &t);
+}
+
+static void ww_test_edeadlk_normal_slow(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        o2.ctx = NULL;
+        mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
+        mutex_unlock(&o2.base);
+        WWU(&o);
+
+        ww_mutex_lock_slow(&o2, &t);
+}
+
+static void ww_test_edeadlk_no_unlock(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        o2.ctx = &t2;
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        o2.ctx = NULL;
+        mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
+        mutex_unlock(&o2.base);
+
+        WWL(&o2, &t);
+}
+
+static void ww_test_edeadlk_no_unlock_slow(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        o2.ctx = NULL;
+        mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
+        mutex_unlock(&o2.base);
+
+        ww_mutex_lock_slow(&o2, &t);
+}
+
+static void ww_test_edeadlk_acquire_more(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        ret = WWL(&o3, &t);
+}
+
+static void ww_test_edeadlk_acquire_more_slow(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        ww_mutex_lock_slow(&o3, &t);
+}
+
+static void ww_test_edeadlk_acquire_more_edeadlk(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        mutex_lock(&o3.base);
+        mutex_release(&o3.base.dep_map, 1, _THIS_IP_);
+        o3.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        ret = WWL(&o3, &t);
+        WARN_ON(ret != -EDEADLK);
+}
+
+static void ww_test_edeadlk_acquire_more_edeadlk_slow(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        mutex_lock(&o3.base);
+        mutex_release(&o3.base.dep_map, 1, _THIS_IP_);
+        o3.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+
+        ww_mutex_lock_slow(&o3, &t);
+}
+
+static void ww_test_edeadlk_acquire_wrong(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+        if (!ret)
+                WWU(&o2);
+
+        WWU(&o);
+
+        ret = WWL(&o3, &t);
+}
+
+static void ww_test_edeadlk_acquire_wrong_slow(void)
+{
+        int ret;
+
+        mutex_lock(&o2.base);
+        mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
+        o2.ctx = &t2;
+
+        WWAI(&t);
+        t2 = t;
+        t2.stamp--;
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret != -EDEADLK);
+        if (!ret)
+                WWU(&o2);
+
+        WWU(&o);
+
+        ww_mutex_lock_slow(&o3, &t);
+}
+
+static void ww_test_spin_nest_unlocked(void)
+{
+        raw_spin_lock_nest_lock(&lock_A, &o.base);
+        U(A);
+}
+
+static void ww_test_unneeded_slow(void)
+{
+        WWAI(&t);
+
+        ww_mutex_lock_slow(&o, &t);
+}
+
+static void ww_test_context_block(void)
+{
+        int ret;
+
+        WWAI(&t);
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+        WWL1(&o2);
+}
+
+static void ww_test_context_try(void)
+{
+        int ret;
+
+        WWAI(&t);
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWT(&o2);
+        WARN_ON(!ret);
+        WWU(&o2);
+        WWU(&o);
+}
+
+static void ww_test_context_context(void)
+{
+        int ret;
+
+        WWAI(&t);
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret);
+
+        WWU(&o2);
+        WWU(&o);
+}
+
+static void ww_test_try_block(void)
+{
+        bool ret;
+
+        ret = WWT(&o);
+        WARN_ON(!ret);
+
+        WWL1(&o2);
+        WWU(&o2);
+        WWU(&o);
+}
+
+static void ww_test_try_try(void)
+{
+        bool ret;
+
+        ret = WWT(&o);
+        WARN_ON(!ret);
+        ret = WWT(&o2);
+        WARN_ON(!ret);
+        WWU(&o2);
+        WWU(&o);
+}
+
+static void ww_test_try_context(void)
+{
+        int ret;
+
+        ret = WWT(&o);
+        WARN_ON(!ret);
+
+        WWAI(&t);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret);
+}
+
+static void ww_test_block_block(void)
+{
+        WWL1(&o);
+        WWL1(&o2);
+}
+
+static void ww_test_block_try(void)
+{
+        bool ret;
+
+        WWL1(&o);
+        ret = WWT(&o2);
+        WARN_ON(!ret);
+}
+
+static void ww_test_block_context(void)
+{
+        int ret;
+
+        WWL1(&o);
+        WWAI(&t);
+
+        ret = WWL(&o2, &t);
+        WARN_ON(ret);
+}
+
+static void ww_test_spin_block(void)
+{
+        L(A);
+        U(A);
+
+        WWL1(&o);
+        L(A);
+        U(A);
+        WWU(&o);
+
+        L(A);
+        WWL1(&o);
+        WWU(&o);
+        U(A);
+}
+
+static void ww_test_spin_try(void)
+{
+        bool ret;
+
+        L(A);
+        U(A);
+
+        ret = WWT(&o);
+        WARN_ON(!ret);
+        L(A);
+        U(A);
+        WWU(&o);
+
+        L(A);
+        ret = WWT(&o);
+        WARN_ON(!ret);
+        WWU(&o);
+        U(A);
+}
+
+static void ww_test_spin_context(void)
+{
+        int ret;
+
+        L(A);
+        U(A);
+
+        WWAI(&t);
+
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+        L(A);
+        U(A);
+        WWU(&o);
+
+        L(A);
+        ret = WWL(&o, &t);
+        WARN_ON(ret);
+        WWU(&o);
+        U(A);
+}
+
+static void ww_tests(void)
+{
+        printk("  --------------------------------------------------------------------------\n");
+        printk("  | Wound/wait tests |\n");
+        printk("  ---------------------\n");
+
+        print_testname("ww api failures");
+        dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("ww contexts mixing");
+        dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("finishing ww context");
+        dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("locking mismatches");
+        dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("EDEADLK handling");
+        dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("spinlock nest unlocked");
+        dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        printk("  -----------------------------------------------------\n");
+        printk("                                 |block | try  |context|\n");
+        printk("  -----------------------------------------------------\n");
+
+        print_testname("context");
+        dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("try");
+        dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("block");
+        dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+
+        print_testname("spinlock");
+        dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW);
+        dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW);
+        dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW);
+        printk("\n");
+}
 
 void locking_selftest(void)
 {
@@ -1188,6 +1870,8 @@ void locking_selftest(void)
         DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion);
 //      DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2);
 
+        ww_tests();
+
         if (unexpected_testcase_failures) {
                 printk("-----------------------------------------------------------------\n");
                 debug_locks = 0;
diff --git a/sound/pci/hda/Kconfig b/sound/pci/hda/Kconfig
index 80a7d44bcf81..c5a872ca7703 100644
--- a/sound/pci/hda/Kconfig
+++ b/sound/pci/hda/Kconfig
@@ -152,6 +152,16 @@ config SND_HDA_CODEC_HDMI
           snd-hda-codec-hdmi.
           This module is automatically loaded at probing.
 
+config SND_HDA_I915
+        bool "Build Display HD-audio controller/codec power well support for i915 cards"
+        depends on DRM_I915
+        help
+          Say Y here to include full HDMI and DisplayPort HD-audio controller/codec
+          power-well support for Intel Haswell graphics cards based on the i915 driver.
+
+          Note that this option must be enabled for Intel Haswell C+ stepping machines, otherwise
+          the GPU audio controller/codecs will not be initialized or damaged when exit from S3 mode.
+
 config SND_HDA_CODEC_CIRRUS
         bool "Build Cirrus Logic codec support"
         default y
diff --git a/sound/pci/hda/Makefile b/sound/pci/hda/Makefile
index 24a251497a1f..c091438286a3 100644
--- a/sound/pci/hda/Makefile
+++ b/sound/pci/hda/Makefile
@@ -1,4 +1,6 @@
 snd-hda-intel-objs := hda_intel.o
+# for haswell power well
+snd-hda-intel-$(CONFIG_SND_HDA_I915) += hda_i915.o
 
 snd-hda-codec-y := hda_codec.o hda_jack.o hda_auto_parser.o
 snd-hda-codec-$(CONFIG_SND_HDA_GENERIC) += hda_generic.o
diff --git a/sound/pci/hda/hda_i915.c b/sound/pci/hda/hda_i915.c
new file mode 100644
index 000000000000..76c13d5b3ca0
--- /dev/null
+++ b/sound/pci/hda/hda_i915.c
@@ -0,0 +1,75 @@
+/*
+ *  hda_i915.c - routines for Haswell HDA controller power well support
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the Free
+ *  Software Foundation; either version 2 of the License, or (at your option)
+ *  any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ *  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software Foundation,
+ *  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <sound/core.h>
+#include <drm/i915_powerwell.h>
+#include "hda_i915.h"
+
+static void (*get_power)(void);
+static void (*put_power)(void);
+
+void hda_display_power(bool enable)
+{
+        if (!get_power || !put_power)
+                return;
+
+        snd_printdd("HDA display power %s \n",
+                        enable ? "Enable" : "Disable");
+        if (enable)
+                get_power();
+        else
+                put_power();
+}
+
+int hda_i915_init(void)
+{
+        int err = 0;
+
+        get_power = symbol_request(i915_request_power_well);
+        if (!get_power) {
+                snd_printk(KERN_WARNING "hda-i915: get_power symbol get fail\n");
+                return -ENODEV;
+        }
+
+        put_power = symbol_request(i915_release_power_well);
+        if (!put_power) {
+                symbol_put(i915_request_power_well);
+                get_power = NULL;
+                return -ENODEV;
+        }
+
+        snd_printd("HDA driver get symbol successfully from i915 module\n");
+
+        return err;
+}
+
+int hda_i915_exit(void)
+{
+        if (get_power) {
+                symbol_put(i915_request_power_well);
+                get_power = NULL;
+        }
+        if (put_power) {
+                symbol_put(i915_release_power_well);
+                put_power = NULL;
+        }
+
+        return 0;
+}
diff --git a/sound/pci/hda/hda_i915.h b/sound/pci/hda/hda_i915.h
new file mode 100644
index 000000000000..5a63da2c53e5
--- /dev/null
+++ b/sound/pci/hda/hda_i915.h
@@ -0,0 +1,35 @@
+/*
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the Free
+ *  Software Foundation; either version 2 of the License, or (at your option)
+ *  any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ *  more details.
+ *
+ *  You should have received a copy of the GNU General Public License along with
+ *  this program; if not, write to the Free Software Foundation, Inc., 59
+ *  Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+#ifndef __SOUND_HDA_I915_H
+#define __SOUND_HDA_I915_H
+
+#ifdef CONFIG_SND_HDA_I915
+void hda_display_power(bool enable);
+int hda_i915_init(void);
+int hda_i915_exit(void);
+#else
+static inline void hda_display_power(bool enable) {}
+static inline int hda_i915_init(void)
+{
+        return -ENODEV;
+}
+static inline int hda_i915_exit(void)
+{
+        return 0;
+}
+#endif
+
+#endif
diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c
index de18722c4873..35e9f8b010a7 100644
--- a/sound/pci/hda/hda_intel.c
+++ b/sound/pci/hda/hda_intel.c
@@ -62,6 +62,7 @@
 #include <linux/vga_switcheroo.h>
 #include <linux/firmware.h>
 #include "hda_codec.h"
+#include "hda_i915.h"
 
 
 static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX;
@@ -541,6 +542,10 @@ struct azx {
         /* for pending irqs */
         struct work_struct irq_pending_work;
 
+#ifdef CONFIG_SND_HDA_I915
+        struct work_struct probe_work;
+#endif
+
         /* reboot notifier (for mysterious hangup problem at power-down) */
         struct notifier_block reboot_notifier;
 
@@ -594,6 +599,7 @@ enum {
 #define AZX_DCAPS_4K_BDLE_BOUNDARY (1 << 23)    /* BDLE in 4k boundary */
 #define AZX_DCAPS_COUNT_LPIB_DELAY  (1 << 25)   /* Take LPIB as delay */
 #define AZX_DCAPS_PM_RUNTIME    (1 << 26)       /* runtime PM support */
+#define AZX_DCAPS_I915_POWERWELL (1 << 27)      /* HSW i915 power well support */
 
 /* quirks for Intel PCH */
 #define AZX_DCAPS_INTEL_PCH_NOPM \
@@ -2900,6 +2906,8 @@ static int azx_suspend(struct device *dev)
         pci_disable_device(pci);
         pci_save_state(pci);
         pci_set_power_state(pci, PCI_D3hot);
+        if (chip->driver_caps & AZX_DCAPS_I915_POWERWELL)
+                hda_display_power(false);
         return 0;
 }
 
@@ -2912,6 +2920,8 @@ static int azx_resume(struct device *dev)
         if (chip->disabled)
                 return 0;
 
+        if (chip->driver_caps & AZX_DCAPS_I915_POWERWELL)
+                hda_display_power(true);
         pci_set_power_state(pci, PCI_D0);
         pci_restore_state(pci);
         if (pci_enable_device(pci) < 0) {
@@ -2944,6 +2954,8 @@ static int azx_runtime_suspend(struct device *dev)
 
         azx_stop_chip(chip);
         azx_clear_irq_pending(chip);
+        if (chip->driver_caps & AZX_DCAPS_I915_POWERWELL)
+                hda_display_power(false);
         return 0;
 }
 
@@ -2952,6 +2964,8 @@ static int azx_runtime_resume(struct device *dev)
         struct snd_card *card = dev_get_drvdata(dev);
         struct azx *chip = card->private_data;
 
+        if (chip->driver_caps & AZX_DCAPS_I915_POWERWELL)
+                hda_display_power(true);
         azx_init_pci(chip);
         azx_init_chip(chip, 1);
         return 0;
@@ -3006,7 +3020,6 @@ static void azx_notifier_unregister(struct azx *chip)
                 unregister_reboot_notifier(&chip->reboot_notifier);
 }
 
-static int azx_first_init(struct azx *chip);
 static int azx_probe_continue(struct azx *chip);
 
 #ifdef SUPPORT_VGA_SWITCHEROO
@@ -3033,8 +3046,7 @@ static void azx_vs_set_state(struct pci_dev *pci,
                         snd_printk(KERN_INFO SFX
                                    "%s: Start delayed initialization\n",
                                    pci_name(chip->pci));
-                        if (azx_first_init(chip) < 0 ||
-                            azx_probe_continue(chip) < 0) {
+                        if (azx_probe_continue(chip) < 0) {
                                 snd_printk(KERN_ERR SFX
                                            "%s: initialization error\n",
                                            pci_name(chip->pci));
@@ -3120,8 +3132,13 @@ static int register_vga_switcheroo(struct azx *chip)
  */
 static int azx_free(struct azx *chip)
 {
+        struct pci_dev *pci = chip->pci;
         int i;
 
+        if ((chip->driver_caps & AZX_DCAPS_PM_RUNTIME)
+                        && chip->running)
+                pm_runtime_get_noresume(&pci->dev);
+
         azx_del_card_list(chip);
 
         azx_notifier_unregister(chip);
@@ -3173,6 +3190,10 @@ static int azx_free(struct azx *chip)
         if (chip->fw)
                 release_firmware(chip->fw);
 #endif
+        if (chip->driver_caps & AZX_DCAPS_I915_POWERWELL) {
+                hda_display_power(false);
+                hda_i915_exit();
+        }
         kfree(chip);
 
         return 0;
@@ -3398,6 +3419,13 @@ static void azx_check_snoop_available(struct azx *chip)
         }
 }
 
+#ifdef CONFIG_SND_HDA_I915
+static void azx_probe_work(struct work_struct *work)
+{
+        azx_probe_continue(container_of(work, struct azx, probe_work));
+}
+#endif
+
 /*
  * constructor
  */
@@ -3473,7 +3501,13 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
                 return err;
         }
 
+#ifdef CONFIG_SND_HDA_I915
+        /* continue probing in work context as may trigger request module */
+        INIT_WORK(&chip->probe_work, azx_probe_work);
+#endif
+
         *rchip = chip;
+
         return 0;
 }
 
@@ -3730,11 +3764,6 @@ static int azx_probe(struct pci_dev *pci,
         }
 
         probe_now = !chip->disabled;
-        if (probe_now) {
-                err = azx_first_init(chip);
-                if (err < 0)
-                        goto out_free;
-        }
 
 #ifdef CONFIG_SND_HDA_PATCH_LOADER
         if (patch[dev] && *patch[dev]) {
@@ -3749,15 +3778,22 @@ static int azx_probe(struct pci_dev *pci,
         }
 #endif /* CONFIG_SND_HDA_PATCH_LOADER */
 
+        /* continue probing in work context, avoid request_module deadlock */
+        if (probe_now && (chip->driver_caps & AZX_DCAPS_I915_POWERWELL)) {
+#ifdef CONFIG_SND_HDA_I915
+                probe_now = false;
+                schedule_work(&chip->probe_work);
+#else
+                snd_printk(KERN_ERR SFX "Haswell must build in CONFIG_SND_HDA_I915\n");
+#endif
+        }
+
         if (probe_now) {
                 err = azx_probe_continue(chip);
                 if (err < 0)
                         goto out_free;
         }
 
-        if (pci_dev_run_wake(pci))
-                pm_runtime_put_noidle(&pci->dev);
-
         dev++;
         complete_all(&chip->probe_wait);
         return 0;
@@ -3770,9 +3806,24 @@ out_free:
 
 static int azx_probe_continue(struct azx *chip)
 {
+        struct pci_dev *pci = chip->pci;
         int dev = chip->dev_index;
         int err;
 
+        /* Request power well for Haswell HDA controller and codec */
+        if (chip->driver_caps & AZX_DCAPS_I915_POWERWELL) {
+                err = hda_i915_init();
+                if (err < 0) {
+                        snd_printk(KERN_ERR SFX "Error request power-well from i915\n");
+                        goto out_free;
+                }
+                hda_display_power(true);
+        }
+
+        err = azx_first_init(chip);
+        if (err < 0)
+                goto out_free;
+
 #ifdef CONFIG_SND_HDA_INPUT_BEEP
         chip->beep_mode = beep_mode[dev];
 #endif
@@ -3817,6 +3868,8 @@ static int azx_probe_continue(struct azx *chip)
         power_down_all_codecs(chip);
         azx_notifier_register(chip);
         azx_add_card_list(chip);
+        if (chip->driver_caps & AZX_DCAPS_PM_RUNTIME)
+                pm_runtime_put_noidle(&pci->dev);
 
         return 0;
 
@@ -3829,9 +3882,6 @@ static void azx_remove(struct pci_dev *pci)
 {
         struct snd_card *card = pci_get_drvdata(pci);
 
-        if (pci_dev_run_wake(pci))
-                pm_runtime_get_noresume(&pci->dev);
-
         if (card)
                 snd_card_free(card);
         pci_set_drvdata(pci, NULL);
@@ -3864,11 +3914,14 @@ static DEFINE_PCI_DEVICE_TABLE(azx_ids) = {
           .driver_data = AZX_DRIVER_PCH | AZX_DCAPS_INTEL_PCH },
         /* Haswell */
         { PCI_DEVICE(0x8086, 0x0a0c),
-          .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH },
+          .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH |
+          AZX_DCAPS_I915_POWERWELL },
         { PCI_DEVICE(0x8086, 0x0c0c),
-          .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH },
+          .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH |
+          AZX_DCAPS_I915_POWERWELL },
         { PCI_DEVICE(0x8086, 0x0d0c),
-          .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH },
+          .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH |
+          AZX_DCAPS_I915_POWERWELL },
         /* 5 Series/3400 */
         { PCI_DEVICE(0x8086, 0x3b56),
           .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_NOPM },