aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_pm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_pm.c')
-rw-r--r--drivers/gpu/drm/i915/intel_pm.c1466
1 files changed, 899 insertions, 567 deletions
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 2863b92c9da6..f4f3fcc8b3be 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -26,6 +26,7 @@
26 */ 26 */
27 27
28#include <linux/cpufreq.h> 28#include <linux/cpufreq.h>
29#include <drm/drm_plane_helper.h>
29#include "i915_drv.h" 30#include "i915_drv.h"
30#include "intel_drv.h" 31#include "intel_drv.h"
31#include "../../../platform/x86/intel_ips.h" 32#include "../../../platform/x86/intel_ips.h"
@@ -82,7 +83,7 @@ static void gen9_init_clock_gating(struct drm_device *dev)
82 83
83static void bxt_init_clock_gating(struct drm_device *dev) 84static void bxt_init_clock_gating(struct drm_device *dev)
84{ 85{
85 struct drm_i915_private *dev_priv = dev->dev_private; 86 struct drm_i915_private *dev_priv = to_i915(dev);
86 87
87 gen9_init_clock_gating(dev); 88 gen9_init_clock_gating(dev);
88 89
@@ -108,7 +109,7 @@ static void bxt_init_clock_gating(struct drm_device *dev)
108 109
109static void i915_pineview_get_mem_freq(struct drm_device *dev) 110static void i915_pineview_get_mem_freq(struct drm_device *dev)
110{ 111{
111 struct drm_i915_private *dev_priv = dev->dev_private; 112 struct drm_i915_private *dev_priv = to_i915(dev);
112 u32 tmp; 113 u32 tmp;
113 114
114 tmp = I915_READ(CLKCFG); 115 tmp = I915_READ(CLKCFG);
@@ -147,7 +148,7 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev)
147 148
148static void i915_ironlake_get_mem_freq(struct drm_device *dev) 149static void i915_ironlake_get_mem_freq(struct drm_device *dev)
149{ 150{
150 struct drm_i915_private *dev_priv = dev->dev_private; 151 struct drm_i915_private *dev_priv = to_i915(dev);
151 u16 ddrpll, csipll; 152 u16 ddrpll, csipll;
152 153
153 ddrpll = I915_READ16(DDRMPLL1); 154 ddrpll = I915_READ16(DDRMPLL1);
@@ -318,7 +319,7 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
318 319
319void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 320void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
320{ 321{
321 struct drm_device *dev = dev_priv->dev; 322 struct drm_device *dev = &dev_priv->drm;
322 u32 val; 323 u32 val;
323 324
324 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { 325 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
@@ -374,7 +375,7 @@ static const int pessimal_latency_ns = 5000;
374static int vlv_get_fifo_size(struct drm_device *dev, 375static int vlv_get_fifo_size(struct drm_device *dev,
375 enum pipe pipe, int plane) 376 enum pipe pipe, int plane)
376{ 377{
377 struct drm_i915_private *dev_priv = dev->dev_private; 378 struct drm_i915_private *dev_priv = to_i915(dev);
378 int sprite0_start, sprite1_start, size; 379 int sprite0_start, sprite1_start, size;
379 380
380 switch (pipe) { 381 switch (pipe) {
@@ -425,7 +426,7 @@ static int vlv_get_fifo_size(struct drm_device *dev,
425 426
426static int i9xx_get_fifo_size(struct drm_device *dev, int plane) 427static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
427{ 428{
428 struct drm_i915_private *dev_priv = dev->dev_private; 429 struct drm_i915_private *dev_priv = to_i915(dev);
429 uint32_t dsparb = I915_READ(DSPARB); 430 uint32_t dsparb = I915_READ(DSPARB);
430 int size; 431 int size;
431 432
@@ -441,7 +442,7 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
441 442
442static int i830_get_fifo_size(struct drm_device *dev, int plane) 443static int i830_get_fifo_size(struct drm_device *dev, int plane)
443{ 444{
444 struct drm_i915_private *dev_priv = dev->dev_private; 445 struct drm_i915_private *dev_priv = to_i915(dev);
445 uint32_t dsparb = I915_READ(DSPARB); 446 uint32_t dsparb = I915_READ(DSPARB);
446 int size; 447 int size;
447 448
@@ -458,7 +459,7 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane)
458 459
459static int i845_get_fifo_size(struct drm_device *dev, int plane) 460static int i845_get_fifo_size(struct drm_device *dev, int plane)
460{ 461{
461 struct drm_i915_private *dev_priv = dev->dev_private; 462 struct drm_i915_private *dev_priv = to_i915(dev);
462 uint32_t dsparb = I915_READ(DSPARB); 463 uint32_t dsparb = I915_READ(DSPARB);
463 int size; 464 int size;
464 465
@@ -636,7 +637,7 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
636static void pineview_update_wm(struct drm_crtc *unused_crtc) 637static void pineview_update_wm(struct drm_crtc *unused_crtc)
637{ 638{
638 struct drm_device *dev = unused_crtc->dev; 639 struct drm_device *dev = unused_crtc->dev;
639 struct drm_i915_private *dev_priv = dev->dev_private; 640 struct drm_i915_private *dev_priv = to_i915(dev);
640 struct drm_crtc *crtc; 641 struct drm_crtc *crtc;
641 const struct cxsr_latency *latency; 642 const struct cxsr_latency *latency;
642 u32 reg; 643 u32 reg;
@@ -933,7 +934,7 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate,
933 934
934static void vlv_setup_wm_latency(struct drm_device *dev) 935static void vlv_setup_wm_latency(struct drm_device *dev)
935{ 936{
936 struct drm_i915_private *dev_priv = dev->dev_private; 937 struct drm_i915_private *dev_priv = to_i915(dev);
937 938
938 /* all latencies in usec */ 939 /* all latencies in usec */
939 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; 940 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
@@ -1324,7 +1325,7 @@ static void vlv_merge_wm(struct drm_device *dev,
1324static void vlv_update_wm(struct drm_crtc *crtc) 1325static void vlv_update_wm(struct drm_crtc *crtc)
1325{ 1326{
1326 struct drm_device *dev = crtc->dev; 1327 struct drm_device *dev = crtc->dev;
1327 struct drm_i915_private *dev_priv = dev->dev_private; 1328 struct drm_i915_private *dev_priv = to_i915(dev);
1328 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1329 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1329 enum pipe pipe = intel_crtc->pipe; 1330 enum pipe pipe = intel_crtc->pipe;
1330 struct vlv_wm_values wm = {}; 1331 struct vlv_wm_values wm = {};
@@ -1380,7 +1381,7 @@ static void g4x_update_wm(struct drm_crtc *crtc)
1380{ 1381{
1381 struct drm_device *dev = crtc->dev; 1382 struct drm_device *dev = crtc->dev;
1382 static const int sr_latency_ns = 12000; 1383 static const int sr_latency_ns = 12000;
1383 struct drm_i915_private *dev_priv = dev->dev_private; 1384 struct drm_i915_private *dev_priv = to_i915(dev);
1384 int planea_wm, planeb_wm, cursora_wm, cursorb_wm; 1385 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1385 int plane_sr, cursor_sr; 1386 int plane_sr, cursor_sr;
1386 unsigned int enabled = 0; 1387 unsigned int enabled = 0;
@@ -1437,7 +1438,7 @@ static void g4x_update_wm(struct drm_crtc *crtc)
1437static void i965_update_wm(struct drm_crtc *unused_crtc) 1438static void i965_update_wm(struct drm_crtc *unused_crtc)
1438{ 1439{
1439 struct drm_device *dev = unused_crtc->dev; 1440 struct drm_device *dev = unused_crtc->dev;
1440 struct drm_i915_private *dev_priv = dev->dev_private; 1441 struct drm_i915_private *dev_priv = to_i915(dev);
1441 struct drm_crtc *crtc; 1442 struct drm_crtc *crtc;
1442 int srwm = 1; 1443 int srwm = 1;
1443 int cursor_sr = 16; 1444 int cursor_sr = 16;
@@ -1511,7 +1512,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
1511static void i9xx_update_wm(struct drm_crtc *unused_crtc) 1512static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1512{ 1513{
1513 struct drm_device *dev = unused_crtc->dev; 1514 struct drm_device *dev = unused_crtc->dev;
1514 struct drm_i915_private *dev_priv = dev->dev_private; 1515 struct drm_i915_private *dev_priv = to_i915(dev);
1515 const struct intel_watermark_params *wm_info; 1516 const struct intel_watermark_params *wm_info;
1516 uint32_t fwater_lo; 1517 uint32_t fwater_lo;
1517 uint32_t fwater_hi; 1518 uint32_t fwater_hi;
@@ -1641,7 +1642,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1641static void i845_update_wm(struct drm_crtc *unused_crtc) 1642static void i845_update_wm(struct drm_crtc *unused_crtc)
1642{ 1643{
1643 struct drm_device *dev = unused_crtc->dev; 1644 struct drm_device *dev = unused_crtc->dev;
1644 struct drm_i915_private *dev_priv = dev->dev_private; 1645 struct drm_i915_private *dev_priv = to_i915(dev);
1645 struct drm_crtc *crtc; 1646 struct drm_crtc *crtc;
1646 const struct drm_display_mode *adjusted_mode; 1647 const struct drm_display_mode *adjusted_mode;
1647 uint32_t fwater_lo; 1648 uint32_t fwater_lo;
@@ -2040,10 +2041,10 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2040} 2041}
2041 2042
2042static uint32_t 2043static uint32_t
2043hsw_compute_linetime_wm(struct drm_device *dev, 2044hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2044 struct intel_crtc_state *cstate)
2045{ 2045{
2046 struct drm_i915_private *dev_priv = dev->dev_private; 2046 const struct intel_atomic_state *intel_state =
2047 to_intel_atomic_state(cstate->base.state);
2047 const struct drm_display_mode *adjusted_mode = 2048 const struct drm_display_mode *adjusted_mode =
2048 &cstate->base.adjusted_mode; 2049 &cstate->base.adjusted_mode;
2049 u32 linetime, ips_linetime; 2050 u32 linetime, ips_linetime;
@@ -2052,7 +2053,7 @@ hsw_compute_linetime_wm(struct drm_device *dev,
2052 return 0; 2053 return 0;
2053 if (WARN_ON(adjusted_mode->crtc_clock == 0)) 2054 if (WARN_ON(adjusted_mode->crtc_clock == 0))
2054 return 0; 2055 return 0;
2055 if (WARN_ON(dev_priv->cdclk_freq == 0)) 2056 if (WARN_ON(intel_state->cdclk == 0))
2056 return 0; 2057 return 0;
2057 2058
2058 /* The WM are computed with base on how long it takes to fill a single 2059 /* The WM are computed with base on how long it takes to fill a single
@@ -2061,7 +2062,7 @@ hsw_compute_linetime_wm(struct drm_device *dev,
2061 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, 2062 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2062 adjusted_mode->crtc_clock); 2063 adjusted_mode->crtc_clock);
2063 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, 2064 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2064 dev_priv->cdclk_freq); 2065 intel_state->cdclk);
2065 2066
2066 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) | 2067 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2067 PIPE_WM_LINETIME_TIME(linetime); 2068 PIPE_WM_LINETIME_TIME(linetime);
@@ -2069,7 +2070,7 @@ hsw_compute_linetime_wm(struct drm_device *dev,
2069 2070
2070static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) 2071static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
2071{ 2072{
2072 struct drm_i915_private *dev_priv = dev->dev_private; 2073 struct drm_i915_private *dev_priv = to_i915(dev);
2073 2074
2074 if (IS_GEN9(dev)) { 2075 if (IS_GEN9(dev)) {
2075 uint32_t val; 2076 uint32_t val;
@@ -2174,14 +2175,14 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
2174static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5]) 2175static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2175{ 2176{
2176 /* ILK sprite LP0 latency is 1300 ns */ 2177 /* ILK sprite LP0 latency is 1300 ns */
2177 if (INTEL_INFO(dev)->gen == 5) 2178 if (IS_GEN5(dev))
2178 wm[0] = 13; 2179 wm[0] = 13;
2179} 2180}
2180 2181
2181static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5]) 2182static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2182{ 2183{
2183 /* ILK cursor LP0 latency is 1300 ns */ 2184 /* ILK cursor LP0 latency is 1300 ns */
2184 if (INTEL_INFO(dev)->gen == 5) 2185 if (IS_GEN5(dev))
2185 wm[0] = 13; 2186 wm[0] = 13;
2186 2187
2187 /* WaDoubleCursorLP3Latency:ivb */ 2188 /* WaDoubleCursorLP3Latency:ivb */
@@ -2235,7 +2236,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
2235static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, 2236static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2236 uint16_t wm[5], uint16_t min) 2237 uint16_t wm[5], uint16_t min)
2237{ 2238{
2238 int level, max_level = ilk_wm_max_level(dev_priv->dev); 2239 int level, max_level = ilk_wm_max_level(&dev_priv->drm);
2239 2240
2240 if (wm[0] >= min) 2241 if (wm[0] >= min)
2241 return false; 2242 return false;
@@ -2249,7 +2250,7 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2249 2250
2250static void snb_wm_latency_quirk(struct drm_device *dev) 2251static void snb_wm_latency_quirk(struct drm_device *dev)
2251{ 2252{
2252 struct drm_i915_private *dev_priv = dev->dev_private; 2253 struct drm_i915_private *dev_priv = to_i915(dev);
2253 bool changed; 2254 bool changed;
2254 2255
2255 /* 2256 /*
@@ -2271,7 +2272,7 @@ static void snb_wm_latency_quirk(struct drm_device *dev)
2271 2272
2272static void ilk_setup_wm_latency(struct drm_device *dev) 2273static void ilk_setup_wm_latency(struct drm_device *dev)
2273{ 2274{
2274 struct drm_i915_private *dev_priv = dev->dev_private; 2275 struct drm_i915_private *dev_priv = to_i915(dev);
2275 2276
2276 intel_read_wm_latency(dev, dev_priv->wm.pri_latency); 2277 intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2277 2278
@@ -2293,7 +2294,7 @@ static void ilk_setup_wm_latency(struct drm_device *dev)
2293 2294
2294static void skl_setup_wm_latency(struct drm_device *dev) 2295static void skl_setup_wm_latency(struct drm_device *dev)
2295{ 2296{
2296 struct drm_i915_private *dev_priv = dev->dev_private; 2297 struct drm_i915_private *dev_priv = to_i915(dev);
2297 2298
2298 intel_read_wm_latency(dev, dev_priv->wm.skl_latency); 2299 intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
2299 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); 2300 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
@@ -2329,7 +2330,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
2329 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); 2330 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
2330 struct intel_pipe_wm *pipe_wm; 2331 struct intel_pipe_wm *pipe_wm;
2331 struct drm_device *dev = state->dev; 2332 struct drm_device *dev = state->dev;
2332 const struct drm_i915_private *dev_priv = dev->dev_private; 2333 const struct drm_i915_private *dev_priv = to_i915(dev);
2333 struct intel_plane *intel_plane; 2334 struct intel_plane *intel_plane;
2334 struct intel_plane_state *pristate = NULL; 2335 struct intel_plane_state *pristate = NULL;
2335 struct intel_plane_state *sprstate = NULL; 2336 struct intel_plane_state *sprstate = NULL;
@@ -2337,7 +2338,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
2337 int level, max_level = ilk_wm_max_level(dev), usable_level; 2338 int level, max_level = ilk_wm_max_level(dev), usable_level;
2338 struct ilk_wm_maximums max; 2339 struct ilk_wm_maximums max;
2339 2340
2340 pipe_wm = &cstate->wm.optimal.ilk; 2341 pipe_wm = &cstate->wm.ilk.optimal;
2341 2342
2342 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { 2343 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
2343 struct intel_plane_state *ps; 2344 struct intel_plane_state *ps;
@@ -2380,7 +2381,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
2380 pipe_wm->wm[0] = pipe_wm->raw_wm[0]; 2381 pipe_wm->wm[0] = pipe_wm->raw_wm[0];
2381 2382
2382 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2383 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2383 pipe_wm->linetime = hsw_compute_linetime_wm(dev, cstate); 2384 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
2384 2385
2385 if (!ilk_validate_pipe_wm(dev, pipe_wm)) 2386 if (!ilk_validate_pipe_wm(dev, pipe_wm))
2386 return -EINVAL; 2387 return -EINVAL;
@@ -2419,7 +2420,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
2419 struct intel_crtc *intel_crtc, 2420 struct intel_crtc *intel_crtc,
2420 struct intel_crtc_state *newstate) 2421 struct intel_crtc_state *newstate)
2421{ 2422{
2422 struct intel_pipe_wm *a = &newstate->wm.intermediate; 2423 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
2423 struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk; 2424 struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk;
2424 int level, max_level = ilk_wm_max_level(dev); 2425 int level, max_level = ilk_wm_max_level(dev);
2425 2426
@@ -2428,7 +2429,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
2428 * currently active watermarks to get values that are safe both before 2429 * currently active watermarks to get values that are safe both before
2429 * and after the vblank. 2430 * and after the vblank.
2430 */ 2431 */
2431 *a = newstate->wm.optimal.ilk; 2432 *a = newstate->wm.ilk.optimal;
2432 a->pipe_enabled |= b->pipe_enabled; 2433 a->pipe_enabled |= b->pipe_enabled;
2433 a->sprites_enabled |= b->sprites_enabled; 2434 a->sprites_enabled |= b->sprites_enabled;
2434 a->sprites_scaled |= b->sprites_scaled; 2435 a->sprites_scaled |= b->sprites_scaled;
@@ -2457,7 +2458,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
2457 * If our intermediate WM are identical to the final WM, then we can 2458 * If our intermediate WM are identical to the final WM, then we can
2458 * omit the post-vblank programming; only update if it's different. 2459 * omit the post-vblank programming; only update if it's different.
2459 */ 2460 */
2460 if (memcmp(a, &newstate->wm.optimal.ilk, sizeof(*a)) == 0) 2461 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) == 0)
2461 newstate->wm.need_postvbl_update = false; 2462 newstate->wm.need_postvbl_update = false;
2462 2463
2463 return 0; 2464 return 0;
@@ -2504,7 +2505,7 @@ static void ilk_wm_merge(struct drm_device *dev,
2504 const struct ilk_wm_maximums *max, 2505 const struct ilk_wm_maximums *max,
2505 struct intel_pipe_wm *merged) 2506 struct intel_pipe_wm *merged)
2506{ 2507{
2507 struct drm_i915_private *dev_priv = dev->dev_private; 2508 struct drm_i915_private *dev_priv = to_i915(dev);
2508 int level, max_level = ilk_wm_max_level(dev); 2509 int level, max_level = ilk_wm_max_level(dev);
2509 int last_enabled_level = max_level; 2510 int last_enabled_level = max_level;
2510 2511
@@ -2564,7 +2565,7 @@ static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2564/* The value we need to program into the WM_LPx latency field */ 2565/* The value we need to program into the WM_LPx latency field */
2565static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level) 2566static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2566{ 2567{
2567 struct drm_i915_private *dev_priv = dev->dev_private; 2568 struct drm_i915_private *dev_priv = to_i915(dev);
2568 2569
2569 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2570 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2570 return 2 * level; 2571 return 2 * level;
@@ -2764,7 +2765,7 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2764static void ilk_write_wm_values(struct drm_i915_private *dev_priv, 2765static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2765 struct ilk_wm_values *results) 2766 struct ilk_wm_values *results)
2766{ 2767{
2767 struct drm_device *dev = dev_priv->dev; 2768 struct drm_device *dev = &dev_priv->drm;
2768 struct ilk_wm_values *previous = &dev_priv->wm.hw; 2769 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2769 unsigned int dirty; 2770 unsigned int dirty;
2770 uint32_t val; 2771 uint32_t val;
@@ -2839,7 +2840,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2839 2840
2840bool ilk_disable_lp_wm(struct drm_device *dev) 2841bool ilk_disable_lp_wm(struct drm_device *dev)
2841{ 2842{
2842 struct drm_i915_private *dev_priv = dev->dev_private; 2843 struct drm_i915_private *dev_priv = to_i915(dev);
2843 2844
2844 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); 2845 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2845} 2846}
@@ -2877,20 +2878,29 @@ skl_wm_plane_id(const struct intel_plane *plane)
2877static void 2878static void
2878skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, 2879skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2879 const struct intel_crtc_state *cstate, 2880 const struct intel_crtc_state *cstate,
2880 const struct intel_wm_config *config, 2881 struct skl_ddb_entry *alloc, /* out */
2881 struct skl_ddb_entry *alloc /* out */) 2882 int *num_active /* out */)
2882{ 2883{
2884 struct drm_atomic_state *state = cstate->base.state;
2885 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
2886 struct drm_i915_private *dev_priv = to_i915(dev);
2883 struct drm_crtc *for_crtc = cstate->base.crtc; 2887 struct drm_crtc *for_crtc = cstate->base.crtc;
2884 struct drm_crtc *crtc;
2885 unsigned int pipe_size, ddb_size; 2888 unsigned int pipe_size, ddb_size;
2886 int nth_active_pipe; 2889 int nth_active_pipe;
2890 int pipe = to_intel_crtc(for_crtc)->pipe;
2887 2891
2888 if (!cstate->base.active) { 2892 if (WARN_ON(!state) || !cstate->base.active) {
2889 alloc->start = 0; 2893 alloc->start = 0;
2890 alloc->end = 0; 2894 alloc->end = 0;
2895 *num_active = hweight32(dev_priv->active_crtcs);
2891 return; 2896 return;
2892 } 2897 }
2893 2898
2899 if (intel_state->active_pipe_changes)
2900 *num_active = hweight32(intel_state->active_crtcs);
2901 else
2902 *num_active = hweight32(dev_priv->active_crtcs);
2903
2894 if (IS_BROXTON(dev)) 2904 if (IS_BROXTON(dev))
2895 ddb_size = BXT_DDB_SIZE; 2905 ddb_size = BXT_DDB_SIZE;
2896 else 2906 else
@@ -2898,25 +2908,29 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2898 2908
2899 ddb_size -= 4; /* 4 blocks for bypass path allocation */ 2909 ddb_size -= 4; /* 4 blocks for bypass path allocation */
2900 2910
2901 nth_active_pipe = 0; 2911 /*
2902 for_each_crtc(dev, crtc) { 2912 * If the state doesn't change the active CRTC's, then there's
2903 if (!to_intel_crtc(crtc)->active) 2913 * no need to recalculate; the existing pipe allocation limits
2904 continue; 2914 * should remain unchanged. Note that we're safe from racing
2905 2915 * commits since any racing commit that changes the active CRTC
2906 if (crtc == for_crtc) 2916 * list would need to grab _all_ crtc locks, including the one
2907 break; 2917 * we currently hold.
2908 2918 */
2909 nth_active_pipe++; 2919 if (!intel_state->active_pipe_changes) {
2920 *alloc = dev_priv->wm.skl_hw.ddb.pipe[pipe];
2921 return;
2910 } 2922 }
2911 2923
2912 pipe_size = ddb_size / config->num_pipes_active; 2924 nth_active_pipe = hweight32(intel_state->active_crtcs &
2913 alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active; 2925 (drm_crtc_mask(for_crtc) - 1));
2926 pipe_size = ddb_size / hweight32(intel_state->active_crtcs);
2927 alloc->start = nth_active_pipe * ddb_size / *num_active;
2914 alloc->end = alloc->start + pipe_size; 2928 alloc->end = alloc->start + pipe_size;
2915} 2929}
2916 2930
2917static unsigned int skl_cursor_allocation(const struct intel_wm_config *config) 2931static unsigned int skl_cursor_allocation(int num_active)
2918{ 2932{
2919 if (config->num_pipes_active == 1) 2933 if (num_active == 1)
2920 return 32; 2934 return 32;
2921 2935
2922 return 8; 2936 return 8;
@@ -2960,6 +2974,46 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
2960 } 2974 }
2961} 2975}
2962 2976
2977/*
2978 * Determines the downscale amount of a plane for the purposes of watermark calculations.
2979 * The bspec defines downscale amount as:
2980 *
2981 * """
2982 * Horizontal down scale amount = maximum[1, Horizontal source size /
2983 * Horizontal destination size]
2984 * Vertical down scale amount = maximum[1, Vertical source size /
2985 * Vertical destination size]
2986 * Total down scale amount = Horizontal down scale amount *
2987 * Vertical down scale amount
2988 * """
2989 *
2990 * Return value is provided in 16.16 fixed point form to retain fractional part.
2991 * Caller should take care of dividing & rounding off the value.
2992 */
2993static uint32_t
2994skl_plane_downscale_amount(const struct intel_plane_state *pstate)
2995{
2996 uint32_t downscale_h, downscale_w;
2997 uint32_t src_w, src_h, dst_w, dst_h;
2998
2999 if (WARN_ON(!pstate->visible))
3000 return DRM_PLANE_HELPER_NO_SCALING;
3001
3002 /* n.b., src is 16.16 fixed point, dst is whole integer */
3003 src_w = drm_rect_width(&pstate->src);
3004 src_h = drm_rect_height(&pstate->src);
3005 dst_w = drm_rect_width(&pstate->dst);
3006 dst_h = drm_rect_height(&pstate->dst);
3007 if (intel_rotation_90_or_270(pstate->base.rotation))
3008 swap(dst_w, dst_h);
3009
3010 downscale_h = max(src_h / dst_h, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
3011 downscale_w = max(src_w / dst_w, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
3012
3013 /* Provide result in 16.16 fixed point */
3014 return (uint64_t)downscale_w * downscale_h >> 16;
3015}
3016
2963static unsigned int 3017static unsigned int
2964skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, 3018skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
2965 const struct drm_plane_state *pstate, 3019 const struct drm_plane_state *pstate,
@@ -2967,7 +3021,16 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
2967{ 3021{
2968 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate); 3022 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
2969 struct drm_framebuffer *fb = pstate->fb; 3023 struct drm_framebuffer *fb = pstate->fb;
3024 uint32_t down_scale_amount, data_rate;
2970 uint32_t width = 0, height = 0; 3025 uint32_t width = 0, height = 0;
3026 unsigned format = fb ? fb->pixel_format : DRM_FORMAT_XRGB8888;
3027
3028 if (!intel_pstate->visible)
3029 return 0;
3030 if (pstate->plane->type == DRM_PLANE_TYPE_CURSOR)
3031 return 0;
3032 if (y && format != DRM_FORMAT_NV12)
3033 return 0;
2971 3034
2972 width = drm_rect_width(&intel_pstate->src) >> 16; 3035 width = drm_rect_width(&intel_pstate->src) >> 16;
2973 height = drm_rect_height(&intel_pstate->src) >> 16; 3036 height = drm_rect_height(&intel_pstate->src) >> 16;
@@ -2976,17 +3039,21 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
2976 swap(width, height); 3039 swap(width, height);
2977 3040
2978 /* for planar format */ 3041 /* for planar format */
2979 if (fb->pixel_format == DRM_FORMAT_NV12) { 3042 if (format == DRM_FORMAT_NV12) {
2980 if (y) /* y-plane data rate */ 3043 if (y) /* y-plane data rate */
2981 return width * height * 3044 data_rate = width * height *
2982 drm_format_plane_cpp(fb->pixel_format, 0); 3045 drm_format_plane_cpp(format, 0);
2983 else /* uv-plane data rate */ 3046 else /* uv-plane data rate */
2984 return (width / 2) * (height / 2) * 3047 data_rate = (width / 2) * (height / 2) *
2985 drm_format_plane_cpp(fb->pixel_format, 1); 3048 drm_format_plane_cpp(format, 1);
3049 } else {
3050 /* for packed formats */
3051 data_rate = width * height * drm_format_plane_cpp(format, 0);
2986 } 3052 }
2987 3053
2988 /* for packed formats */ 3054 down_scale_amount = skl_plane_downscale_amount(intel_pstate);
2989 return width * height * drm_format_plane_cpp(fb->pixel_format, 0); 3055
3056 return (uint64_t)data_rate * down_scale_amount >> 16;
2990} 3057}
2991 3058
2992/* 3059/*
@@ -2995,86 +3062,188 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
2995 * 3 * 4096 * 8192 * 4 < 2^32 3062 * 3 * 4096 * 8192 * 4 < 2^32
2996 */ 3063 */
2997static unsigned int 3064static unsigned int
2998skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate) 3065skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate)
2999{ 3066{
3000 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); 3067 struct drm_crtc_state *cstate = &intel_cstate->base;
3001 struct drm_device *dev = intel_crtc->base.dev; 3068 struct drm_atomic_state *state = cstate->state;
3069 struct drm_crtc *crtc = cstate->crtc;
3070 struct drm_device *dev = crtc->dev;
3071 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3072 const struct drm_plane *plane;
3002 const struct intel_plane *intel_plane; 3073 const struct intel_plane *intel_plane;
3003 unsigned int total_data_rate = 0; 3074 struct drm_plane_state *pstate;
3075 unsigned int rate, total_data_rate = 0;
3076 int id;
3077 int i;
3004 3078
3005 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { 3079 if (WARN_ON(!state))
3006 const struct drm_plane_state *pstate = intel_plane->base.state; 3080 return 0;
3007 3081
3008 if (pstate->fb == NULL) 3082 /* Calculate and cache data rate for each plane */
3009 continue; 3083 for_each_plane_in_state(state, plane, pstate, i) {
3084 id = skl_wm_plane_id(to_intel_plane(plane));
3085 intel_plane = to_intel_plane(plane);
3010 3086
3011 if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR) 3087 if (intel_plane->pipe != intel_crtc->pipe)
3012 continue; 3088 continue;
3013 3089
3014 /* packed/uv */ 3090 /* packed/uv */
3015 total_data_rate += skl_plane_relative_data_rate(cstate, 3091 rate = skl_plane_relative_data_rate(intel_cstate,
3016 pstate, 3092 pstate, 0);
3017 0); 3093 intel_cstate->wm.skl.plane_data_rate[id] = rate;
3094
3095 /* y-plane */
3096 rate = skl_plane_relative_data_rate(intel_cstate,
3097 pstate, 1);
3098 intel_cstate->wm.skl.plane_y_data_rate[id] = rate;
3099 }
3100
3101 /* Calculate CRTC's total data rate from cached values */
3102 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
3103 int id = skl_wm_plane_id(intel_plane);
3018 3104
3019 if (pstate->fb->pixel_format == DRM_FORMAT_NV12) 3105 /* packed/uv */
3020 /* y-plane */ 3106 total_data_rate += intel_cstate->wm.skl.plane_data_rate[id];
3021 total_data_rate += skl_plane_relative_data_rate(cstate, 3107 total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id];
3022 pstate,
3023 1);
3024 } 3108 }
3025 3109
3110 WARN_ON(cstate->plane_mask && total_data_rate == 0);
3111
3026 return total_data_rate; 3112 return total_data_rate;
3027} 3113}
3028 3114
3029static void 3115static uint16_t
3116skl_ddb_min_alloc(const struct drm_plane_state *pstate,
3117 const int y)
3118{
3119 struct drm_framebuffer *fb = pstate->fb;
3120 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
3121 uint32_t src_w, src_h;
3122 uint32_t min_scanlines = 8;
3123 uint8_t plane_bpp;
3124
3125 if (WARN_ON(!fb))
3126 return 0;
3127
3128 /* For packed formats, no y-plane, return 0 */
3129 if (y && fb->pixel_format != DRM_FORMAT_NV12)
3130 return 0;
3131
3132 /* For Non Y-tile return 8-blocks */
3133 if (fb->modifier[0] != I915_FORMAT_MOD_Y_TILED &&
3134 fb->modifier[0] != I915_FORMAT_MOD_Yf_TILED)
3135 return 8;
3136
3137 src_w = drm_rect_width(&intel_pstate->src) >> 16;
3138 src_h = drm_rect_height(&intel_pstate->src) >> 16;
3139
3140 if (intel_rotation_90_or_270(pstate->rotation))
3141 swap(src_w, src_h);
3142
3143 /* Halve UV plane width and height for NV12 */
3144 if (fb->pixel_format == DRM_FORMAT_NV12 && !y) {
3145 src_w /= 2;
3146 src_h /= 2;
3147 }
3148
3149 if (fb->pixel_format == DRM_FORMAT_NV12 && !y)
3150 plane_bpp = drm_format_plane_cpp(fb->pixel_format, 1);
3151 else
3152 plane_bpp = drm_format_plane_cpp(fb->pixel_format, 0);
3153
3154 if (intel_rotation_90_or_270(pstate->rotation)) {
3155 switch (plane_bpp) {
3156 case 1:
3157 min_scanlines = 32;
3158 break;
3159 case 2:
3160 min_scanlines = 16;
3161 break;
3162 case 4:
3163 min_scanlines = 8;
3164 break;
3165 case 8:
3166 min_scanlines = 4;
3167 break;
3168 default:
3169 WARN(1, "Unsupported pixel depth %u for rotation",
3170 plane_bpp);
3171 min_scanlines = 32;
3172 }
3173 }
3174
3175 return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
3176}
3177
3178static int
3030skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, 3179skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
3031 struct skl_ddb_allocation *ddb /* out */) 3180 struct skl_ddb_allocation *ddb /* out */)
3032{ 3181{
3182 struct drm_atomic_state *state = cstate->base.state;
3033 struct drm_crtc *crtc = cstate->base.crtc; 3183 struct drm_crtc *crtc = cstate->base.crtc;
3034 struct drm_device *dev = crtc->dev; 3184 struct drm_device *dev = crtc->dev;
3035 struct drm_i915_private *dev_priv = to_i915(dev);
3036 struct intel_wm_config *config = &dev_priv->wm.config;
3037 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3185 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3038 struct intel_plane *intel_plane; 3186 struct intel_plane *intel_plane;
3187 struct drm_plane *plane;
3188 struct drm_plane_state *pstate;
3039 enum pipe pipe = intel_crtc->pipe; 3189 enum pipe pipe = intel_crtc->pipe;
3040 struct skl_ddb_entry *alloc = &ddb->pipe[pipe]; 3190 struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
3041 uint16_t alloc_size, start, cursor_blocks; 3191 uint16_t alloc_size, start, cursor_blocks;
3042 uint16_t minimum[I915_MAX_PLANES]; 3192 uint16_t *minimum = cstate->wm.skl.minimum_blocks;
3043 uint16_t y_minimum[I915_MAX_PLANES]; 3193 uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks;
3044 unsigned int total_data_rate; 3194 unsigned int total_data_rate;
3195 int num_active;
3196 int id, i;
3045 3197
3046 skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc); 3198 if (WARN_ON(!state))
3199 return 0;
3200
3201 if (!cstate->base.active) {
3202 ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0;
3203 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
3204 memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
3205 return 0;
3206 }
3207
3208 skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active);
3047 alloc_size = skl_ddb_entry_size(alloc); 3209 alloc_size = skl_ddb_entry_size(alloc);
3048 if (alloc_size == 0) { 3210 if (alloc_size == 0) {
3049 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); 3211 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
3050 memset(&ddb->plane[pipe][PLANE_CURSOR], 0, 3212 return 0;
3051 sizeof(ddb->plane[pipe][PLANE_CURSOR]));
3052 return;
3053 } 3213 }
3054 3214
3055 cursor_blocks = skl_cursor_allocation(config); 3215 cursor_blocks = skl_cursor_allocation(num_active);
3056 ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks; 3216 ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
3057 ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; 3217 ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
3058 3218
3059 alloc_size -= cursor_blocks; 3219 alloc_size -= cursor_blocks;
3060 alloc->end -= cursor_blocks;
3061 3220
3062 /* 1. Allocate the mininum required blocks for each active plane */ 3221 /* 1. Allocate the mininum required blocks for each active plane */
3063 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { 3222 for_each_plane_in_state(state, plane, pstate, i) {
3064 struct drm_plane *plane = &intel_plane->base; 3223 intel_plane = to_intel_plane(plane);
3065 struct drm_framebuffer *fb = plane->state->fb; 3224 id = skl_wm_plane_id(intel_plane);
3066 int id = skl_wm_plane_id(intel_plane);
3067 3225
3068 if (!to_intel_plane_state(plane->state)->visible) 3226 if (intel_plane->pipe != pipe)
3069 continue; 3227 continue;
3070 3228
3071 if (plane->type == DRM_PLANE_TYPE_CURSOR) 3229 if (!to_intel_plane_state(pstate)->visible) {
3230 minimum[id] = 0;
3231 y_minimum[id] = 0;
3232 continue;
3233 }
3234 if (plane->type == DRM_PLANE_TYPE_CURSOR) {
3235 minimum[id] = 0;
3236 y_minimum[id] = 0;
3072 continue; 3237 continue;
3238 }
3239
3240 minimum[id] = skl_ddb_min_alloc(pstate, 0);
3241 y_minimum[id] = skl_ddb_min_alloc(pstate, 1);
3242 }
3073 3243
3074 minimum[id] = 8; 3244 for (i = 0; i < PLANE_CURSOR; i++) {
3075 alloc_size -= minimum[id]; 3245 alloc_size -= minimum[i];
3076 y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0; 3246 alloc_size -= y_minimum[i];
3077 alloc_size -= y_minimum[id];
3078 } 3247 }
3079 3248
3080 /* 3249 /*
@@ -3084,21 +3253,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
3084 * FIXME: we may not allocate every single block here. 3253 * FIXME: we may not allocate every single block here.
3085 */ 3254 */
3086 total_data_rate = skl_get_total_relative_data_rate(cstate); 3255 total_data_rate = skl_get_total_relative_data_rate(cstate);
3256 if (total_data_rate == 0)
3257 return 0;
3087 3258
3088 start = alloc->start; 3259 start = alloc->start;
3089 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { 3260 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
3090 struct drm_plane *plane = &intel_plane->base;
3091 struct drm_plane_state *pstate = intel_plane->base.state;
3092 unsigned int data_rate, y_data_rate; 3261 unsigned int data_rate, y_data_rate;
3093 uint16_t plane_blocks, y_plane_blocks = 0; 3262 uint16_t plane_blocks, y_plane_blocks = 0;
3094 int id = skl_wm_plane_id(intel_plane); 3263 int id = skl_wm_plane_id(intel_plane);
3095 3264
3096 if (!to_intel_plane_state(pstate)->visible) 3265 data_rate = cstate->wm.skl.plane_data_rate[id];
3097 continue;
3098 if (plane->type == DRM_PLANE_TYPE_CURSOR)
3099 continue;
3100
3101 data_rate = skl_plane_relative_data_rate(cstate, pstate, 0);
3102 3266
3103 /* 3267 /*
3104 * allocation for (packed formats) or (uv-plane part of planar format): 3268 * allocation for (packed formats) or (uv-plane part of planar format):
@@ -3109,30 +3273,32 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
3109 plane_blocks += div_u64((uint64_t)alloc_size * data_rate, 3273 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
3110 total_data_rate); 3274 total_data_rate);
3111 3275
3112 ddb->plane[pipe][id].start = start; 3276 /* Leave disabled planes at (0,0) */
3113 ddb->plane[pipe][id].end = start + plane_blocks; 3277 if (data_rate) {
3278 ddb->plane[pipe][id].start = start;
3279 ddb->plane[pipe][id].end = start + plane_blocks;
3280 }
3114 3281
3115 start += plane_blocks; 3282 start += plane_blocks;
3116 3283
3117 /* 3284 /*
3118 * allocation for y_plane part of planar format: 3285 * allocation for y_plane part of planar format:
3119 */ 3286 */
3120 if (pstate->fb->pixel_format == DRM_FORMAT_NV12) { 3287 y_data_rate = cstate->wm.skl.plane_y_data_rate[id];
3121 y_data_rate = skl_plane_relative_data_rate(cstate, 3288
3122 pstate, 3289 y_plane_blocks = y_minimum[id];
3123 1); 3290 y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
3124 y_plane_blocks = y_minimum[id]; 3291 total_data_rate);
3125 y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
3126 total_data_rate);
3127 3292
3293 if (y_data_rate) {
3128 ddb->y_plane[pipe][id].start = start; 3294 ddb->y_plane[pipe][id].start = start;
3129 ddb->y_plane[pipe][id].end = start + y_plane_blocks; 3295 ddb->y_plane[pipe][id].end = start + y_plane_blocks;
3130
3131 start += y_plane_blocks;
3132 } 3296 }
3133 3297
3298 start += y_plane_blocks;
3134 } 3299 }
3135 3300
3301 return 0;
3136} 3302}
3137 3303
3138static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config) 3304static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
@@ -3189,35 +3355,41 @@ static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
3189 return ret; 3355 return ret;
3190} 3356}
3191 3357
3192static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb, 3358static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
3193 const struct intel_crtc *intel_crtc) 3359 struct intel_plane_state *pstate)
3194{ 3360{
3195 struct drm_device *dev = intel_crtc->base.dev; 3361 uint64_t adjusted_pixel_rate;
3196 struct drm_i915_private *dev_priv = dev->dev_private; 3362 uint64_t downscale_amount;
3197 const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; 3363 uint64_t pixel_rate;
3364
3365 /* Shouldn't reach here on disabled planes... */
3366 if (WARN_ON(!pstate->visible))
3367 return 0;
3198 3368
3199 /* 3369 /*
3200 * If ddb allocation of pipes changed, it may require recalculation of 3370 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
3201 * watermarks 3371 * with additional adjustments for plane-specific scaling.
3202 */ 3372 */
3203 if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe))) 3373 adjusted_pixel_rate = skl_pipe_pixel_rate(cstate);
3204 return true; 3374 downscale_amount = skl_plane_downscale_amount(pstate);
3375
3376 pixel_rate = adjusted_pixel_rate * downscale_amount >> 16;
3377 WARN_ON(pixel_rate != clamp_t(uint32_t, pixel_rate, 0, ~0));
3205 3378
3206 return false; 3379 return pixel_rate;
3207} 3380}
3208 3381
3209static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, 3382static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3210 struct intel_crtc_state *cstate, 3383 struct intel_crtc_state *cstate,
3211 struct intel_plane *intel_plane, 3384 struct intel_plane_state *intel_pstate,
3212 uint16_t ddb_allocation, 3385 uint16_t ddb_allocation,
3213 int level, 3386 int level,
3214 uint16_t *out_blocks, /* out */ 3387 uint16_t *out_blocks, /* out */
3215 uint8_t *out_lines /* out */) 3388 uint8_t *out_lines, /* out */
3389 bool *enabled /* out */)
3216{ 3390{
3217 struct drm_plane *plane = &intel_plane->base; 3391 struct drm_plane_state *pstate = &intel_pstate->base;
3218 struct drm_framebuffer *fb = plane->state->fb; 3392 struct drm_framebuffer *fb = pstate->fb;
3219 struct intel_plane_state *intel_pstate =
3220 to_intel_plane_state(plane->state);
3221 uint32_t latency = dev_priv->wm.skl_latency[level]; 3393 uint32_t latency = dev_priv->wm.skl_latency[level];
3222 uint32_t method1, method2; 3394 uint32_t method1, method2;
3223 uint32_t plane_bytes_per_line, plane_blocks_per_line; 3395 uint32_t plane_bytes_per_line, plane_blocks_per_line;
@@ -3225,20 +3397,24 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3225 uint32_t selected_result; 3397 uint32_t selected_result;
3226 uint8_t cpp; 3398 uint8_t cpp;
3227 uint32_t width = 0, height = 0; 3399 uint32_t width = 0, height = 0;
3400 uint32_t plane_pixel_rate;
3228 3401
3229 if (latency == 0 || !cstate->base.active || !intel_pstate->visible) 3402 if (latency == 0 || !cstate->base.active || !intel_pstate->visible) {
3230 return false; 3403 *enabled = false;
3404 return 0;
3405 }
3231 3406
3232 width = drm_rect_width(&intel_pstate->src) >> 16; 3407 width = drm_rect_width(&intel_pstate->src) >> 16;
3233 height = drm_rect_height(&intel_pstate->src) >> 16; 3408 height = drm_rect_height(&intel_pstate->src) >> 16;
3234 3409
3235 if (intel_rotation_90_or_270(plane->state->rotation)) 3410 if (intel_rotation_90_or_270(pstate->rotation))
3236 swap(width, height); 3411 swap(width, height);
3237 3412
3238 cpp = drm_format_plane_cpp(fb->pixel_format, 0); 3413 cpp = drm_format_plane_cpp(fb->pixel_format, 0);
3239 method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate), 3414 plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
3240 cpp, latency); 3415
3241 method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate), 3416 method1 = skl_wm_method1(plane_pixel_rate, cpp, latency);
3417 method2 = skl_wm_method2(plane_pixel_rate,
3242 cstate->base.adjusted_mode.crtc_htotal, 3418 cstate->base.adjusted_mode.crtc_htotal,
3243 width, 3419 width,
3244 cpp, 3420 cpp,
@@ -3252,7 +3428,7 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3252 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { 3428 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
3253 uint32_t min_scanlines = 4; 3429 uint32_t min_scanlines = 4;
3254 uint32_t y_tile_minimum; 3430 uint32_t y_tile_minimum;
3255 if (intel_rotation_90_or_270(plane->state->rotation)) { 3431 if (intel_rotation_90_or_270(pstate->rotation)) {
3256 int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ? 3432 int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
3257 drm_format_plane_cpp(fb->pixel_format, 1) : 3433 drm_format_plane_cpp(fb->pixel_format, 1) :
3258 drm_format_plane_cpp(fb->pixel_format, 0); 3434 drm_format_plane_cpp(fb->pixel_format, 0);
@@ -3288,40 +3464,100 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3288 res_blocks++; 3464 res_blocks++;
3289 } 3465 }
3290 3466
3291 if (res_blocks >= ddb_allocation || res_lines > 31) 3467 if (res_blocks >= ddb_allocation || res_lines > 31) {
3292 return false; 3468 *enabled = false;
3469
3470 /*
3471 * If there are no valid level 0 watermarks, then we can't
3472 * support this display configuration.
3473 */
3474 if (level) {
3475 return 0;
3476 } else {
3477 DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
3478 DRM_DEBUG_KMS("Plane %d.%d: blocks required = %u/%u, lines required = %u/31\n",
3479 to_intel_crtc(cstate->base.crtc)->pipe,
3480 skl_wm_plane_id(to_intel_plane(pstate->plane)),
3481 res_blocks, ddb_allocation, res_lines);
3482
3483 return -EINVAL;
3484 }
3485 }
3293 3486
3294 *out_blocks = res_blocks; 3487 *out_blocks = res_blocks;
3295 *out_lines = res_lines; 3488 *out_lines = res_lines;
3489 *enabled = true;
3296 3490
3297 return true; 3491 return 0;
3298} 3492}
3299 3493
3300static void skl_compute_wm_level(const struct drm_i915_private *dev_priv, 3494static int
3301 struct skl_ddb_allocation *ddb, 3495skl_compute_wm_level(const struct drm_i915_private *dev_priv,
3302 struct intel_crtc_state *cstate, 3496 struct skl_ddb_allocation *ddb,
3303 int level, 3497 struct intel_crtc_state *cstate,
3304 struct skl_wm_level *result) 3498 int level,
3499 struct skl_wm_level *result)
3305{ 3500{
3306 struct drm_device *dev = dev_priv->dev; 3501 struct drm_atomic_state *state = cstate->base.state;
3307 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); 3502 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3503 struct drm_plane *plane;
3308 struct intel_plane *intel_plane; 3504 struct intel_plane *intel_plane;
3505 struct intel_plane_state *intel_pstate;
3309 uint16_t ddb_blocks; 3506 uint16_t ddb_blocks;
3310 enum pipe pipe = intel_crtc->pipe; 3507 enum pipe pipe = intel_crtc->pipe;
3508 int ret;
3311 3509
3312 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { 3510 /*
3511 * We'll only calculate watermarks for planes that are actually
3512 * enabled, so make sure all other planes are set as disabled.
3513 */
3514 memset(result, 0, sizeof(*result));
3515
3516 for_each_intel_plane_mask(&dev_priv->drm,
3517 intel_plane,
3518 cstate->base.plane_mask) {
3313 int i = skl_wm_plane_id(intel_plane); 3519 int i = skl_wm_plane_id(intel_plane);
3314 3520
3521 plane = &intel_plane->base;
3522 intel_pstate = NULL;
3523 if (state)
3524 intel_pstate =
3525 intel_atomic_get_existing_plane_state(state,
3526 intel_plane);
3527
3528 /*
3529 * Note: If we start supporting multiple pending atomic commits
3530 * against the same planes/CRTC's in the future, plane->state
3531 * will no longer be the correct pre-state to use for the
3532 * calculations here and we'll need to change where we get the
3533 * 'unchanged' plane data from.
3534 *
3535 * For now this is fine because we only allow one queued commit
3536 * against a CRTC. Even if the plane isn't modified by this
3537 * transaction and we don't have a plane lock, we still have
3538 * the CRTC's lock, so we know that no other transactions are
3539 * racing with us to update it.
3540 */
3541 if (!intel_pstate)
3542 intel_pstate = to_intel_plane_state(plane->state);
3543
3544 WARN_ON(!intel_pstate->base.fb);
3545
3315 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]); 3546 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
3316 3547
3317 result->plane_en[i] = skl_compute_plane_wm(dev_priv, 3548 ret = skl_compute_plane_wm(dev_priv,
3318 cstate, 3549 cstate,
3319 intel_plane, 3550 intel_pstate,
3320 ddb_blocks, 3551 ddb_blocks,
3321 level, 3552 level,
3322 &result->plane_res_b[i], 3553 &result->plane_res_b[i],
3323 &result->plane_res_l[i]); 3554 &result->plane_res_l[i],
3555 &result->plane_en[i]);
3556 if (ret)
3557 return ret;
3324 } 3558 }
3559
3560 return 0;
3325} 3561}
3326 3562
3327static uint32_t 3563static uint32_t
@@ -3355,21 +3591,26 @@ static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
3355 } 3591 }
3356} 3592}
3357 3593
3358static void skl_compute_pipe_wm(struct intel_crtc_state *cstate, 3594static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
3359 struct skl_ddb_allocation *ddb, 3595 struct skl_ddb_allocation *ddb,
3360 struct skl_pipe_wm *pipe_wm) 3596 struct skl_pipe_wm *pipe_wm)
3361{ 3597{
3362 struct drm_device *dev = cstate->base.crtc->dev; 3598 struct drm_device *dev = cstate->base.crtc->dev;
3363 const struct drm_i915_private *dev_priv = dev->dev_private; 3599 const struct drm_i915_private *dev_priv = to_i915(dev);
3364 int level, max_level = ilk_wm_max_level(dev); 3600 int level, max_level = ilk_wm_max_level(dev);
3601 int ret;
3365 3602
3366 for (level = 0; level <= max_level; level++) { 3603 for (level = 0; level <= max_level; level++) {
3367 skl_compute_wm_level(dev_priv, ddb, cstate, 3604 ret = skl_compute_wm_level(dev_priv, ddb, cstate,
3368 level, &pipe_wm->wm[level]); 3605 level, &pipe_wm->wm[level]);
3606 if (ret)
3607 return ret;
3369 } 3608 }
3370 pipe_wm->linetime = skl_compute_linetime_wm(cstate); 3609 pipe_wm->linetime = skl_compute_linetime_wm(cstate);
3371 3610
3372 skl_compute_transition_wm(cstate, &pipe_wm->trans_wm); 3611 skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
3612
3613 return 0;
3373} 3614}
3374 3615
3375static void skl_compute_wm_results(struct drm_device *dev, 3616static void skl_compute_wm_results(struct drm_device *dev,
@@ -3442,14 +3683,16 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
3442static void skl_write_wm_values(struct drm_i915_private *dev_priv, 3683static void skl_write_wm_values(struct drm_i915_private *dev_priv,
3443 const struct skl_wm_values *new) 3684 const struct skl_wm_values *new)
3444{ 3685{
3445 struct drm_device *dev = dev_priv->dev; 3686 struct drm_device *dev = &dev_priv->drm;
3446 struct intel_crtc *crtc; 3687 struct intel_crtc *crtc;
3447 3688
3448 for_each_intel_crtc(dev, crtc) { 3689 for_each_intel_crtc(dev, crtc) {
3449 int i, level, max_level = ilk_wm_max_level(dev); 3690 int i, level, max_level = ilk_wm_max_level(dev);
3450 enum pipe pipe = crtc->pipe; 3691 enum pipe pipe = crtc->pipe;
3451 3692
3452 if (!new->dirty[pipe]) 3693 if ((new->dirty_pipes & drm_crtc_mask(&crtc->base)) == 0)
3694 continue;
3695 if (!crtc->active)
3453 continue; 3696 continue;
3454 3697
3455 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]); 3698 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
@@ -3537,7 +3780,7 @@ skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
3537static void skl_flush_wm_values(struct drm_i915_private *dev_priv, 3780static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
3538 struct skl_wm_values *new_values) 3781 struct skl_wm_values *new_values)
3539{ 3782{
3540 struct drm_device *dev = dev_priv->dev; 3783 struct drm_device *dev = &dev_priv->drm;
3541 struct skl_ddb_allocation *cur_ddb, *new_ddb; 3784 struct skl_ddb_allocation *cur_ddb, *new_ddb;
3542 bool reallocated[I915_MAX_PIPES] = {}; 3785 bool reallocated[I915_MAX_PIPES] = {};
3543 struct intel_crtc *crtc; 3786 struct intel_crtc *crtc;
@@ -3616,116 +3859,182 @@ static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
3616 } 3859 }
3617} 3860}
3618 3861
3619static bool skl_update_pipe_wm(struct drm_crtc *crtc, 3862static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
3620 struct skl_ddb_allocation *ddb, /* out */ 3863 struct skl_ddb_allocation *ddb, /* out */
3621 struct skl_pipe_wm *pipe_wm /* out */) 3864 struct skl_pipe_wm *pipe_wm, /* out */
3865 bool *changed /* out */)
3622{ 3866{
3623 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3867 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->crtc);
3624 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); 3868 struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
3869 int ret;
3625 3870
3626 skl_allocate_pipe_ddb(cstate, ddb); 3871 ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
3627 skl_compute_pipe_wm(cstate, ddb, pipe_wm); 3872 if (ret)
3873 return ret;
3628 3874
3629 if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm))) 3875 if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
3630 return false; 3876 *changed = false;
3877 else
3878 *changed = true;
3631 3879
3632 intel_crtc->wm.active.skl = *pipe_wm; 3880 return 0;
3881}
3633 3882
3634 return true; 3883static uint32_t
3884pipes_modified(struct drm_atomic_state *state)
3885{
3886 struct drm_crtc *crtc;
3887 struct drm_crtc_state *cstate;
3888 uint32_t i, ret = 0;
3889
3890 for_each_crtc_in_state(state, crtc, cstate, i)
3891 ret |= drm_crtc_mask(crtc);
3892
3893 return ret;
3635} 3894}
3636 3895
3637static void skl_update_other_pipe_wm(struct drm_device *dev, 3896static int
3638 struct drm_crtc *crtc, 3897skl_compute_ddb(struct drm_atomic_state *state)
3639 struct skl_wm_values *r)
3640{ 3898{
3899 struct drm_device *dev = state->dev;
3900 struct drm_i915_private *dev_priv = to_i915(dev);
3901 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3641 struct intel_crtc *intel_crtc; 3902 struct intel_crtc *intel_crtc;
3642 struct intel_crtc *this_crtc = to_intel_crtc(crtc); 3903 struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
3904 uint32_t realloc_pipes = pipes_modified(state);
3905 int ret;
3643 3906
3644 /* 3907 /*
3645 * If the WM update hasn't changed the allocation for this_crtc (the 3908 * If this is our first atomic update following hardware readout,
3646 * crtc we are currently computing the new WM values for), other 3909 * we can't trust the DDB that the BIOS programmed for us. Let's
3647 * enabled crtcs will keep the same allocation and we don't need to 3910 * pretend that all pipes switched active status so that we'll
3648 * recompute anything for them. 3911 * ensure a full DDB recompute.
3649 */ 3912 */
3650 if (!skl_ddb_allocation_changed(&r->ddb, this_crtc)) 3913 if (dev_priv->wm.distrust_bios_wm)
3651 return; 3914 intel_state->active_pipe_changes = ~0;
3652 3915
3653 /* 3916 /*
3654 * Otherwise, because of this_crtc being freshly enabled/disabled, the 3917 * If the modeset changes which CRTC's are active, we need to
3655 * other active pipes need new DDB allocation and WM values. 3918 * recompute the DDB allocation for *all* active pipes, even
3919 * those that weren't otherwise being modified in any way by this
3920 * atomic commit. Due to the shrinking of the per-pipe allocations
3921 * when new active CRTC's are added, it's possible for a pipe that
3922 * we were already using and aren't changing at all here to suddenly
3923 * become invalid if its DDB needs exceeds its new allocation.
3924 *
3925 * Note that if we wind up doing a full DDB recompute, we can't let
3926 * any other display updates race with this transaction, so we need
3927 * to grab the lock on *all* CRTC's.
3656 */ 3928 */
3657 for_each_intel_crtc(dev, intel_crtc) { 3929 if (intel_state->active_pipe_changes) {
3658 struct skl_pipe_wm pipe_wm = {}; 3930 realloc_pipes = ~0;
3659 bool wm_changed; 3931 intel_state->wm_results.dirty_pipes = ~0;
3660 3932 }
3661 if (this_crtc->pipe == intel_crtc->pipe)
3662 continue;
3663 3933
3664 if (!intel_crtc->active) 3934 for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
3665 continue; 3935 struct intel_crtc_state *cstate;
3666 3936
3667 wm_changed = skl_update_pipe_wm(&intel_crtc->base, 3937 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
3668 &r->ddb, &pipe_wm); 3938 if (IS_ERR(cstate))
3939 return PTR_ERR(cstate);
3669 3940
3670 /* 3941 ret = skl_allocate_pipe_ddb(cstate, ddb);
3671 * If we end up re-computing the other pipe WM values, it's 3942 if (ret)
3672 * because it was really needed, so we expect the WM values to 3943 return ret;
3673 * be different.
3674 */
3675 WARN_ON(!wm_changed);
3676
3677 skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc);
3678 r->dirty[intel_crtc->pipe] = true;
3679 } 3944 }
3945
3946 return 0;
3680} 3947}
3681 3948
3682static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe) 3949static int
3950skl_compute_wm(struct drm_atomic_state *state)
3683{ 3951{
3684 watermarks->wm_linetime[pipe] = 0; 3952 struct drm_crtc *crtc;
3685 memset(watermarks->plane[pipe], 0, 3953 struct drm_crtc_state *cstate;
3686 sizeof(uint32_t) * 8 * I915_MAX_PLANES); 3954 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3687 memset(watermarks->plane_trans[pipe], 3955 struct skl_wm_values *results = &intel_state->wm_results;
3688 0, sizeof(uint32_t) * I915_MAX_PLANES); 3956 struct skl_pipe_wm *pipe_wm;
3689 watermarks->plane_trans[pipe][PLANE_CURSOR] = 0; 3957 bool changed = false;
3958 int ret, i;
3959
3960 /*
3961 * If this transaction isn't actually touching any CRTC's, don't
3962 * bother with watermark calculation. Note that if we pass this
3963 * test, we're guaranteed to hold at least one CRTC state mutex,
3964 * which means we can safely use values like dev_priv->active_crtcs
3965 * since any racing commits that want to update them would need to
3966 * hold _all_ CRTC state mutexes.
3967 */
3968 for_each_crtc_in_state(state, crtc, cstate, i)
3969 changed = true;
3970 if (!changed)
3971 return 0;
3972
3973 /* Clear all dirty flags */
3974 results->dirty_pipes = 0;
3975
3976 ret = skl_compute_ddb(state);
3977 if (ret)
3978 return ret;
3979
3980 /*
3981 * Calculate WM's for all pipes that are part of this transaction.
3982 * Note that the DDB allocation above may have added more CRTC's that
3983 * weren't otherwise being modified (and set bits in dirty_pipes) if
3984 * pipe allocations had to change.
3985 *
3986 * FIXME: Now that we're doing this in the atomic check phase, we
3987 * should allow skl_update_pipe_wm() to return failure in cases where
3988 * no suitable watermark values can be found.
3989 */
3990 for_each_crtc_in_state(state, crtc, cstate, i) {
3991 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3992 struct intel_crtc_state *intel_cstate =
3993 to_intel_crtc_state(cstate);
3994
3995 pipe_wm = &intel_cstate->wm.skl.optimal;
3996 ret = skl_update_pipe_wm(cstate, &results->ddb, pipe_wm,
3997 &changed);
3998 if (ret)
3999 return ret;
3690 4000
3691 /* Clear ddb entries for pipe */ 4001 if (changed)
3692 memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry)); 4002 results->dirty_pipes |= drm_crtc_mask(crtc);
3693 memset(&watermarks->ddb.plane[pipe], 0,
3694 sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
3695 memset(&watermarks->ddb.y_plane[pipe], 0,
3696 sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
3697 memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
3698 sizeof(struct skl_ddb_entry));
3699 4003
4004 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
4005 /* This pipe's WM's did not change */
4006 continue;
4007
4008 intel_cstate->update_wm_pre = true;
4009 skl_compute_wm_results(crtc->dev, pipe_wm, results, intel_crtc);
4010 }
4011
4012 return 0;
3700} 4013}
3701 4014
3702static void skl_update_wm(struct drm_crtc *crtc) 4015static void skl_update_wm(struct drm_crtc *crtc)
3703{ 4016{
3704 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 4017 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3705 struct drm_device *dev = crtc->dev; 4018 struct drm_device *dev = crtc->dev;
3706 struct drm_i915_private *dev_priv = dev->dev_private; 4019 struct drm_i915_private *dev_priv = to_i915(dev);
3707 struct skl_wm_values *results = &dev_priv->wm.skl_results; 4020 struct skl_wm_values *results = &dev_priv->wm.skl_results;
3708 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); 4021 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3709 struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl; 4022 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
3710 4023
3711 4024 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
3712 /* Clear all dirty flags */
3713 memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
3714
3715 skl_clear_wm(results, intel_crtc->pipe);
3716
3717 if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm))
3718 return; 4025 return;
3719 4026
3720 skl_compute_wm_results(dev, pipe_wm, results, intel_crtc); 4027 intel_crtc->wm.active.skl = *pipe_wm;
3721 results->dirty[intel_crtc->pipe] = true; 4028
4029 mutex_lock(&dev_priv->wm.wm_mutex);
3722 4030
3723 skl_update_other_pipe_wm(dev, crtc, results);
3724 skl_write_wm_values(dev_priv, results); 4031 skl_write_wm_values(dev_priv, results);
3725 skl_flush_wm_values(dev_priv, results); 4032 skl_flush_wm_values(dev_priv, results);
3726 4033
3727 /* store the new configuration */ 4034 /* store the new configuration */
3728 dev_priv->wm.skl_hw = *results; 4035 dev_priv->wm.skl_hw = *results;
4036
4037 mutex_unlock(&dev_priv->wm.wm_mutex);
3729} 4038}
3730 4039
3731static void ilk_compute_wm_config(struct drm_device *dev, 4040static void ilk_compute_wm_config(struct drm_device *dev,
@@ -3748,7 +4057,7 @@ static void ilk_compute_wm_config(struct drm_device *dev,
3748 4057
3749static void ilk_program_watermarks(struct drm_i915_private *dev_priv) 4058static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3750{ 4059{
3751 struct drm_device *dev = dev_priv->dev; 4060 struct drm_device *dev = &dev_priv->drm;
3752 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; 4061 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3753 struct ilk_wm_maximums max; 4062 struct ilk_wm_maximums max;
3754 struct intel_wm_config config = {}; 4063 struct intel_wm_config config = {};
@@ -3785,7 +4094,7 @@ static void ilk_initial_watermarks(struct intel_crtc_state *cstate)
3785 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); 4094 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3786 4095
3787 mutex_lock(&dev_priv->wm.wm_mutex); 4096 mutex_lock(&dev_priv->wm.wm_mutex);
3788 intel_crtc->wm.active.ilk = cstate->wm.intermediate; 4097 intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
3789 ilk_program_watermarks(dev_priv); 4098 ilk_program_watermarks(dev_priv);
3790 mutex_unlock(&dev_priv->wm.wm_mutex); 4099 mutex_unlock(&dev_priv->wm.wm_mutex);
3791} 4100}
@@ -3797,7 +4106,7 @@ static void ilk_optimize_watermarks(struct intel_crtc_state *cstate)
3797 4106
3798 mutex_lock(&dev_priv->wm.wm_mutex); 4107 mutex_lock(&dev_priv->wm.wm_mutex);
3799 if (cstate->wm.need_postvbl_update) { 4108 if (cstate->wm.need_postvbl_update) {
3800 intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk; 4109 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
3801 ilk_program_watermarks(dev_priv); 4110 ilk_program_watermarks(dev_priv);
3802 } 4111 }
3803 mutex_unlock(&dev_priv->wm.wm_mutex); 4112 mutex_unlock(&dev_priv->wm.wm_mutex);
@@ -3850,11 +4159,11 @@ static void skl_pipe_wm_active_state(uint32_t val,
3850static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) 4159static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3851{ 4160{
3852 struct drm_device *dev = crtc->dev; 4161 struct drm_device *dev = crtc->dev;
3853 struct drm_i915_private *dev_priv = dev->dev_private; 4162 struct drm_i915_private *dev_priv = to_i915(dev);
3854 struct skl_wm_values *hw = &dev_priv->wm.skl_hw; 4163 struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
3855 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 4164 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3856 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); 4165 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3857 struct skl_pipe_wm *active = &cstate->wm.optimal.skl; 4166 struct skl_pipe_wm *active = &cstate->wm.skl.optimal;
3858 enum pipe pipe = intel_crtc->pipe; 4167 enum pipe pipe = intel_crtc->pipe;
3859 int level, i, max_level; 4168 int level, i, max_level;
3860 uint32_t temp; 4169 uint32_t temp;
@@ -3877,7 +4186,7 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3877 if (!intel_crtc->active) 4186 if (!intel_crtc->active)
3878 return; 4187 return;
3879 4188
3880 hw->dirty[pipe] = true; 4189 hw->dirty_pipes |= drm_crtc_mask(crtc);
3881 4190
3882 active->linetime = hw->wm_linetime[pipe]; 4191 active->linetime = hw->wm_linetime[pipe];
3883 4192
@@ -3904,23 +4213,31 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3904 4213
3905void skl_wm_get_hw_state(struct drm_device *dev) 4214void skl_wm_get_hw_state(struct drm_device *dev)
3906{ 4215{
3907 struct drm_i915_private *dev_priv = dev->dev_private; 4216 struct drm_i915_private *dev_priv = to_i915(dev);
3908 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; 4217 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
3909 struct drm_crtc *crtc; 4218 struct drm_crtc *crtc;
3910 4219
3911 skl_ddb_get_hw_state(dev_priv, ddb); 4220 skl_ddb_get_hw_state(dev_priv, ddb);
3912 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) 4221 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3913 skl_pipe_wm_get_hw_state(crtc); 4222 skl_pipe_wm_get_hw_state(crtc);
4223
4224 if (dev_priv->active_crtcs) {
4225 /* Fully recompute DDB on first atomic commit */
4226 dev_priv->wm.distrust_bios_wm = true;
4227 } else {
4228 /* Easy/common case; just sanitize DDB now if everything off */
4229 memset(ddb, 0, sizeof(*ddb));
4230 }
3914} 4231}
3915 4232
3916static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) 4233static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3917{ 4234{
3918 struct drm_device *dev = crtc->dev; 4235 struct drm_device *dev = crtc->dev;
3919 struct drm_i915_private *dev_priv = dev->dev_private; 4236 struct drm_i915_private *dev_priv = to_i915(dev);
3920 struct ilk_wm_values *hw = &dev_priv->wm.hw; 4237 struct ilk_wm_values *hw = &dev_priv->wm.hw;
3921 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 4238 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3922 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); 4239 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3923 struct intel_pipe_wm *active = &cstate->wm.optimal.ilk; 4240 struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
3924 enum pipe pipe = intel_crtc->pipe; 4241 enum pipe pipe = intel_crtc->pipe;
3925 static const i915_reg_t wm0_pipe_reg[] = { 4242 static const i915_reg_t wm0_pipe_reg[] = {
3926 [PIPE_A] = WM0_PIPEA_ILK, 4243 [PIPE_A] = WM0_PIPEA_ILK,
@@ -4120,7 +4437,7 @@ void vlv_wm_get_hw_state(struct drm_device *dev)
4120 4437
4121void ilk_wm_get_hw_state(struct drm_device *dev) 4438void ilk_wm_get_hw_state(struct drm_device *dev)
4122{ 4439{
4123 struct drm_i915_private *dev_priv = dev->dev_private; 4440 struct drm_i915_private *dev_priv = to_i915(dev);
4124 struct ilk_wm_values *hw = &dev_priv->wm.hw; 4441 struct ilk_wm_values *hw = &dev_priv->wm.hw;
4125 struct drm_crtc *crtc; 4442 struct drm_crtc *crtc;
4126 4443
@@ -4182,7 +4499,7 @@ void ilk_wm_get_hw_state(struct drm_device *dev)
4182 */ 4499 */
4183void intel_update_watermarks(struct drm_crtc *crtc) 4500void intel_update_watermarks(struct drm_crtc *crtc)
4184{ 4501{
4185 struct drm_i915_private *dev_priv = crtc->dev->dev_private; 4502 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
4186 4503
4187 if (dev_priv->display.update_wm) 4504 if (dev_priv->display.update_wm)
4188 dev_priv->display.update_wm(crtc); 4505 dev_priv->display.update_wm(crtc);
@@ -4197,9 +4514,8 @@ DEFINE_SPINLOCK(mchdev_lock);
4197 * mchdev_lock. */ 4514 * mchdev_lock. */
4198static struct drm_i915_private *i915_mch_dev; 4515static struct drm_i915_private *i915_mch_dev;
4199 4516
4200bool ironlake_set_drps(struct drm_device *dev, u8 val) 4517bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
4201{ 4518{
4202 struct drm_i915_private *dev_priv = dev->dev_private;
4203 u16 rgvswctl; 4519 u16 rgvswctl;
4204 4520
4205 assert_spin_locked(&mchdev_lock); 4521 assert_spin_locked(&mchdev_lock);
@@ -4221,9 +4537,8 @@ bool ironlake_set_drps(struct drm_device *dev, u8 val)
4221 return true; 4537 return true;
4222} 4538}
4223 4539
4224static void ironlake_enable_drps(struct drm_device *dev) 4540static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
4225{ 4541{
4226 struct drm_i915_private *dev_priv = dev->dev_private;
4227 u32 rgvmodectl; 4542 u32 rgvmodectl;
4228 u8 fmax, fmin, fstart, vstart; 4543 u8 fmax, fmin, fstart, vstart;
4229 4544
@@ -4280,7 +4595,7 @@ static void ironlake_enable_drps(struct drm_device *dev)
4280 DRM_ERROR("stuck trying to change perf mode\n"); 4595 DRM_ERROR("stuck trying to change perf mode\n");
4281 mdelay(1); 4596 mdelay(1);
4282 4597
4283 ironlake_set_drps(dev, fstart); 4598 ironlake_set_drps(dev_priv, fstart);
4284 4599
4285 dev_priv->ips.last_count1 = I915_READ(DMIEC) + 4600 dev_priv->ips.last_count1 = I915_READ(DMIEC) +
4286 I915_READ(DDREC) + I915_READ(CSIEC); 4601 I915_READ(DDREC) + I915_READ(CSIEC);
@@ -4291,9 +4606,8 @@ static void ironlake_enable_drps(struct drm_device *dev)
4291 spin_unlock_irq(&mchdev_lock); 4606 spin_unlock_irq(&mchdev_lock);
4292} 4607}
4293 4608
4294static void ironlake_disable_drps(struct drm_device *dev) 4609static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
4295{ 4610{
4296 struct drm_i915_private *dev_priv = dev->dev_private;
4297 u16 rgvswctl; 4611 u16 rgvswctl;
4298 4612
4299 spin_lock_irq(&mchdev_lock); 4613 spin_lock_irq(&mchdev_lock);
@@ -4308,7 +4622,7 @@ static void ironlake_disable_drps(struct drm_device *dev)
4308 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT); 4622 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
4309 4623
4310 /* Go back to the starting frequency */ 4624 /* Go back to the starting frequency */
4311 ironlake_set_drps(dev, dev_priv->ips.fstart); 4625 ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
4312 mdelay(1); 4626 mdelay(1);
4313 rgvswctl |= MEMCTL_CMD_STS; 4627 rgvswctl |= MEMCTL_CMD_STS;
4314 I915_WRITE(MEMSWCTL, rgvswctl); 4628 I915_WRITE(MEMSWCTL, rgvswctl);
@@ -4354,19 +4668,23 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
4354 new_power = dev_priv->rps.power; 4668 new_power = dev_priv->rps.power;
4355 switch (dev_priv->rps.power) { 4669 switch (dev_priv->rps.power) {
4356 case LOW_POWER: 4670 case LOW_POWER:
4357 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) 4671 if (val > dev_priv->rps.efficient_freq + 1 &&
4672 val > dev_priv->rps.cur_freq)
4358 new_power = BETWEEN; 4673 new_power = BETWEEN;
4359 break; 4674 break;
4360 4675
4361 case BETWEEN: 4676 case BETWEEN:
4362 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) 4677 if (val <= dev_priv->rps.efficient_freq &&
4678 val < dev_priv->rps.cur_freq)
4363 new_power = LOW_POWER; 4679 new_power = LOW_POWER;
4364 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) 4680 else if (val >= dev_priv->rps.rp0_freq &&
4681 val > dev_priv->rps.cur_freq)
4365 new_power = HIGH_POWER; 4682 new_power = HIGH_POWER;
4366 break; 4683 break;
4367 4684
4368 case HIGH_POWER: 4685 case HIGH_POWER:
4369 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) 4686 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 &&
4687 val < dev_priv->rps.cur_freq)
4370 new_power = BETWEEN; 4688 new_power = BETWEEN;
4371 break; 4689 break;
4372 } 4690 }
@@ -4412,22 +4730,24 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
4412 } 4730 }
4413 4731
4414 I915_WRITE(GEN6_RP_UP_EI, 4732 I915_WRITE(GEN6_RP_UP_EI,
4415 GT_INTERVAL_FROM_US(dev_priv, ei_up)); 4733 GT_INTERVAL_FROM_US(dev_priv, ei_up));
4416 I915_WRITE(GEN6_RP_UP_THRESHOLD, 4734 I915_WRITE(GEN6_RP_UP_THRESHOLD,
4417 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100))); 4735 GT_INTERVAL_FROM_US(dev_priv,
4736 ei_up * threshold_up / 100));
4418 4737
4419 I915_WRITE(GEN6_RP_DOWN_EI, 4738 I915_WRITE(GEN6_RP_DOWN_EI,
4420 GT_INTERVAL_FROM_US(dev_priv, ei_down)); 4739 GT_INTERVAL_FROM_US(dev_priv, ei_down));
4421 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 4740 I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
4422 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100))); 4741 GT_INTERVAL_FROM_US(dev_priv,
4742 ei_down * threshold_down / 100));
4423 4743
4424 I915_WRITE(GEN6_RP_CONTROL, 4744 I915_WRITE(GEN6_RP_CONTROL,
4425 GEN6_RP_MEDIA_TURBO | 4745 GEN6_RP_MEDIA_TURBO |
4426 GEN6_RP_MEDIA_HW_NORMAL_MODE | 4746 GEN6_RP_MEDIA_HW_NORMAL_MODE |
4427 GEN6_RP_MEDIA_IS_GFX | 4747 GEN6_RP_MEDIA_IS_GFX |
4428 GEN6_RP_ENABLE | 4748 GEN6_RP_ENABLE |
4429 GEN6_RP_UP_BUSY_AVG | 4749 GEN6_RP_UP_BUSY_AVG |
4430 GEN6_RP_DOWN_IDLE_AVG); 4750 GEN6_RP_DOWN_IDLE_AVG);
4431 4751
4432 dev_priv->rps.power = new_power; 4752 dev_priv->rps.power = new_power;
4433 dev_priv->rps.up_threshold = threshold_up; 4753 dev_priv->rps.up_threshold = threshold_up;
@@ -4452,12 +4772,10 @@ static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
4452/* gen6_set_rps is called to update the frequency request, but should also be 4772/* gen6_set_rps is called to update the frequency request, but should also be
4453 * called when the range (min_delay and max_delay) is modified so that we can 4773 * called when the range (min_delay and max_delay) is modified so that we can
4454 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ 4774 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
4455static void gen6_set_rps(struct drm_device *dev, u8 val) 4775static void gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
4456{ 4776{
4457 struct drm_i915_private *dev_priv = dev->dev_private;
4458
4459 /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ 4777 /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
4460 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) 4778 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
4461 return; 4779 return;
4462 4780
4463 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4781 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -4470,10 +4788,10 @@ static void gen6_set_rps(struct drm_device *dev, u8 val)
4470 if (val != dev_priv->rps.cur_freq) { 4788 if (val != dev_priv->rps.cur_freq) {
4471 gen6_set_rps_thresholds(dev_priv, val); 4789 gen6_set_rps_thresholds(dev_priv, val);
4472 4790
4473 if (IS_GEN9(dev)) 4791 if (IS_GEN9(dev_priv))
4474 I915_WRITE(GEN6_RPNSWREQ, 4792 I915_WRITE(GEN6_RPNSWREQ,
4475 GEN9_FREQUENCY(val)); 4793 GEN9_FREQUENCY(val));
4476 else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 4794 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4477 I915_WRITE(GEN6_RPNSWREQ, 4795 I915_WRITE(GEN6_RPNSWREQ,
4478 HSW_FREQUENCY(val)); 4796 HSW_FREQUENCY(val));
4479 else 4797 else
@@ -4495,15 +4813,13 @@ static void gen6_set_rps(struct drm_device *dev, u8 val)
4495 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); 4813 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
4496} 4814}
4497 4815
4498static void valleyview_set_rps(struct drm_device *dev, u8 val) 4816static void valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
4499{ 4817{
4500 struct drm_i915_private *dev_priv = dev->dev_private;
4501
4502 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4818 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4503 WARN_ON(val > dev_priv->rps.max_freq); 4819 WARN_ON(val > dev_priv->rps.max_freq);
4504 WARN_ON(val < dev_priv->rps.min_freq); 4820 WARN_ON(val < dev_priv->rps.min_freq);
4505 4821
4506 if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1), 4822 if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
4507 "Odd GPU freq value\n")) 4823 "Odd GPU freq value\n"))
4508 val &= ~1; 4824 val &= ~1;
4509 4825
@@ -4536,7 +4852,7 @@ static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
4536 /* Wake up the media well, as that takes a lot less 4852 /* Wake up the media well, as that takes a lot less
4537 * power than the Render well. */ 4853 * power than the Render well. */
4538 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA); 4854 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
4539 valleyview_set_rps(dev_priv->dev, val); 4855 valleyview_set_rps(dev_priv, val);
4540 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA); 4856 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
4541} 4857}
4542 4858
@@ -4548,20 +4864,33 @@ void gen6_rps_busy(struct drm_i915_private *dev_priv)
4548 gen6_rps_reset_ei(dev_priv); 4864 gen6_rps_reset_ei(dev_priv);
4549 I915_WRITE(GEN6_PMINTRMSK, 4865 I915_WRITE(GEN6_PMINTRMSK,
4550 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); 4866 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4867
4868 gen6_enable_rps_interrupts(dev_priv);
4869
4870 /* Ensure we start at the user's desired frequency */
4871 intel_set_rps(dev_priv,
4872 clamp(dev_priv->rps.cur_freq,
4873 dev_priv->rps.min_freq_softlimit,
4874 dev_priv->rps.max_freq_softlimit));
4551 } 4875 }
4552 mutex_unlock(&dev_priv->rps.hw_lock); 4876 mutex_unlock(&dev_priv->rps.hw_lock);
4553} 4877}
4554 4878
4555void gen6_rps_idle(struct drm_i915_private *dev_priv) 4879void gen6_rps_idle(struct drm_i915_private *dev_priv)
4556{ 4880{
4557 struct drm_device *dev = dev_priv->dev; 4881 /* Flush our bottom-half so that it does not race with us
4882 * setting the idle frequency and so that it is bounded by
4883 * our rpm wakeref. And then disable the interrupts to stop any
4884 * futher RPS reclocking whilst we are asleep.
4885 */
4886 gen6_disable_rps_interrupts(dev_priv);
4558 4887
4559 mutex_lock(&dev_priv->rps.hw_lock); 4888 mutex_lock(&dev_priv->rps.hw_lock);
4560 if (dev_priv->rps.enabled) { 4889 if (dev_priv->rps.enabled) {
4561 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) 4890 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4562 vlv_set_rps_idle(dev_priv); 4891 vlv_set_rps_idle(dev_priv);
4563 else 4892 else
4564 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 4893 gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
4565 dev_priv->rps.last_adj = 0; 4894 dev_priv->rps.last_adj = 0;
4566 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); 4895 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4567 } 4896 }
@@ -4580,7 +4909,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
4580 /* This is intentionally racy! We peek at the state here, then 4909 /* This is intentionally racy! We peek at the state here, then
4581 * validate inside the RPS worker. 4910 * validate inside the RPS worker.
4582 */ 4911 */
4583 if (!(dev_priv->mm.busy && 4912 if (!(dev_priv->gt.awake &&
4584 dev_priv->rps.enabled && 4913 dev_priv->rps.enabled &&
4585 dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)) 4914 dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
4586 return; 4915 return;
@@ -4596,7 +4925,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
4596 spin_lock_irq(&dev_priv->irq_lock); 4925 spin_lock_irq(&dev_priv->irq_lock);
4597 if (dev_priv->rps.interrupts_enabled) { 4926 if (dev_priv->rps.interrupts_enabled) {
4598 dev_priv->rps.client_boost = true; 4927 dev_priv->rps.client_boost = true;
4599 queue_work(dev_priv->wq, &dev_priv->rps.work); 4928 schedule_work(&dev_priv->rps.work);
4600 } 4929 }
4601 spin_unlock_irq(&dev_priv->irq_lock); 4930 spin_unlock_irq(&dev_priv->irq_lock);
4602 4931
@@ -4609,49 +4938,39 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
4609 spin_unlock(&dev_priv->rps.client_lock); 4938 spin_unlock(&dev_priv->rps.client_lock);
4610} 4939}
4611 4940
4612void intel_set_rps(struct drm_device *dev, u8 val) 4941void intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
4613{ 4942{
4614 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) 4943 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4615 valleyview_set_rps(dev, val); 4944 valleyview_set_rps(dev_priv, val);
4616 else 4945 else
4617 gen6_set_rps(dev, val); 4946 gen6_set_rps(dev_priv, val);
4618} 4947}
4619 4948
4620static void gen9_disable_rc6(struct drm_device *dev) 4949static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
4621{ 4950{
4622 struct drm_i915_private *dev_priv = dev->dev_private;
4623
4624 I915_WRITE(GEN6_RC_CONTROL, 0); 4951 I915_WRITE(GEN6_RC_CONTROL, 0);
4625 I915_WRITE(GEN9_PG_ENABLE, 0); 4952 I915_WRITE(GEN9_PG_ENABLE, 0);
4626} 4953}
4627 4954
4628static void gen9_disable_rps(struct drm_device *dev) 4955static void gen9_disable_rps(struct drm_i915_private *dev_priv)
4629{ 4956{
4630 struct drm_i915_private *dev_priv = dev->dev_private;
4631
4632 I915_WRITE(GEN6_RP_CONTROL, 0); 4957 I915_WRITE(GEN6_RP_CONTROL, 0);
4633} 4958}
4634 4959
4635static void gen6_disable_rps(struct drm_device *dev) 4960static void gen6_disable_rps(struct drm_i915_private *dev_priv)
4636{ 4961{
4637 struct drm_i915_private *dev_priv = dev->dev_private;
4638
4639 I915_WRITE(GEN6_RC_CONTROL, 0); 4962 I915_WRITE(GEN6_RC_CONTROL, 0);
4640 I915_WRITE(GEN6_RPNSWREQ, 1 << 31); 4963 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
4641 I915_WRITE(GEN6_RP_CONTROL, 0); 4964 I915_WRITE(GEN6_RP_CONTROL, 0);
4642} 4965}
4643 4966
4644static void cherryview_disable_rps(struct drm_device *dev) 4967static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
4645{ 4968{
4646 struct drm_i915_private *dev_priv = dev->dev_private;
4647
4648 I915_WRITE(GEN6_RC_CONTROL, 0); 4969 I915_WRITE(GEN6_RC_CONTROL, 0);
4649} 4970}
4650 4971
4651static void valleyview_disable_rps(struct drm_device *dev) 4972static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
4652{ 4973{
4653 struct drm_i915_private *dev_priv = dev->dev_private;
4654
4655 /* we're doing forcewake before Disabling RC6, 4974 /* we're doing forcewake before Disabling RC6,
4656 * This what the BIOS expects when going into suspend */ 4975 * This what the BIOS expects when going into suspend */
4657 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4976 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
@@ -4661,34 +4980,45 @@ static void valleyview_disable_rps(struct drm_device *dev)
4661 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 4980 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4662} 4981}
4663 4982
4664static void intel_print_rc6_info(struct drm_device *dev, u32 mode) 4983static void intel_print_rc6_info(struct drm_i915_private *dev_priv, u32 mode)
4665{ 4984{
4666 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { 4985 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4667 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1))) 4986 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
4668 mode = GEN6_RC_CTL_RC6_ENABLE; 4987 mode = GEN6_RC_CTL_RC6_ENABLE;
4669 else 4988 else
4670 mode = 0; 4989 mode = 0;
4671 } 4990 }
4672 if (HAS_RC6p(dev)) 4991 if (HAS_RC6p(dev_priv))
4673 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n", 4992 DRM_DEBUG_DRIVER("Enabling RC6 states: "
4674 onoff(mode & GEN6_RC_CTL_RC6_ENABLE), 4993 "RC6 %s RC6p %s RC6pp %s\n",
4675 onoff(mode & GEN6_RC_CTL_RC6p_ENABLE), 4994 onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
4676 onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE)); 4995 onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
4996 onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
4677 4997
4678 else 4998 else
4679 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n", 4999 DRM_DEBUG_DRIVER("Enabling RC6 states: RC6 %s\n",
4680 onoff(mode & GEN6_RC_CTL_RC6_ENABLE)); 5000 onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
4681} 5001}
4682 5002
4683static bool bxt_check_bios_rc6_setup(const struct drm_device *dev) 5003static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
4684{ 5004{
4685 struct drm_i915_private *dev_priv = to_i915(dev);
4686 struct i915_ggtt *ggtt = &dev_priv->ggtt; 5005 struct i915_ggtt *ggtt = &dev_priv->ggtt;
4687 bool enable_rc6 = true; 5006 bool enable_rc6 = true;
4688 unsigned long rc6_ctx_base; 5007 unsigned long rc6_ctx_base;
5008 u32 rc_ctl;
5009 int rc_sw_target;
5010
5011 rc_ctl = I915_READ(GEN6_RC_CONTROL);
5012 rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
5013 RC_SW_TARGET_STATE_SHIFT;
5014 DRM_DEBUG_DRIVER("BIOS enabled RC states: "
5015 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
5016 onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
5017 onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
5018 rc_sw_target);
4689 5019
4690 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) { 5020 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
4691 DRM_DEBUG_KMS("RC6 Base location not set properly.\n"); 5021 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
4692 enable_rc6 = false; 5022 enable_rc6 = false;
4693 } 5023 }
4694 5024
@@ -4700,7 +5030,7 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
4700 if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) && 5030 if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) &&
4701 (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base + 5031 (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base +
4702 ggtt->stolen_reserved_size))) { 5032 ggtt->stolen_reserved_size))) {
4703 DRM_DEBUG_KMS("RC6 Base address not as expected.\n"); 5033 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
4704 enable_rc6 = false; 5034 enable_rc6 = false;
4705 } 5035 }
4706 5036
@@ -4708,31 +5038,40 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
4708 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) && 5038 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
4709 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) && 5039 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
4710 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) { 5040 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
4711 DRM_DEBUG_KMS("Engine Idle wait time not set properly.\n"); 5041 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
5042 enable_rc6 = false;
5043 }
5044
5045 if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
5046 !I915_READ(GEN8_PUSHBUS_ENABLE) ||
5047 !I915_READ(GEN8_PUSHBUS_SHIFT)) {
5048 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
5049 enable_rc6 = false;
5050 }
5051
5052 if (!I915_READ(GEN6_GFXPAUSE)) {
5053 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
4712 enable_rc6 = false; 5054 enable_rc6 = false;
4713 } 5055 }
4714 5056
4715 if (!(I915_READ(GEN6_RC_CONTROL) & (GEN6_RC_CTL_RC6_ENABLE | 5057 if (!I915_READ(GEN8_MISC_CTRL0)) {
4716 GEN6_RC_CTL_HW_ENABLE)) && 5058 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
4717 ((I915_READ(GEN6_RC_CONTROL) & GEN6_RC_CTL_HW_ENABLE) ||
4718 !(I915_READ(GEN6_RC_STATE) & RC6_STATE))) {
4719 DRM_DEBUG_KMS("HW/SW RC6 is not enabled by BIOS.\n");
4720 enable_rc6 = false; 5059 enable_rc6 = false;
4721 } 5060 }
4722 5061
4723 return enable_rc6; 5062 return enable_rc6;
4724} 5063}
4725 5064
4726int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) 5065int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6)
4727{ 5066{
4728 /* No RC6 before Ironlake and code is gone for ilk. */ 5067 /* No RC6 before Ironlake and code is gone for ilk. */
4729 if (INTEL_INFO(dev)->gen < 6) 5068 if (INTEL_INFO(dev_priv)->gen < 6)
4730 return 0; 5069 return 0;
4731 5070
4732 if (!enable_rc6) 5071 if (!enable_rc6)
4733 return 0; 5072 return 0;
4734 5073
4735 if (IS_BROXTON(dev) && !bxt_check_bios_rc6_setup(dev)) { 5074 if (IS_BROXTON(dev_priv) && !bxt_check_bios_rc6_setup(dev_priv)) {
4736 DRM_INFO("RC6 disabled by BIOS\n"); 5075 DRM_INFO("RC6 disabled by BIOS\n");
4737 return 0; 5076 return 0;
4738 } 5077 }
@@ -4741,33 +5080,28 @@ int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
4741 if (enable_rc6 >= 0) { 5080 if (enable_rc6 >= 0) {
4742 int mask; 5081 int mask;
4743 5082
4744 if (HAS_RC6p(dev)) 5083 if (HAS_RC6p(dev_priv))
4745 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE | 5084 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
4746 INTEL_RC6pp_ENABLE; 5085 INTEL_RC6pp_ENABLE;
4747 else 5086 else
4748 mask = INTEL_RC6_ENABLE; 5087 mask = INTEL_RC6_ENABLE;
4749 5088
4750 if ((enable_rc6 & mask) != enable_rc6) 5089 if ((enable_rc6 & mask) != enable_rc6)
4751 DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n", 5090 DRM_DEBUG_DRIVER("Adjusting RC6 mask to %d "
4752 enable_rc6 & mask, enable_rc6, mask); 5091 "(requested %d, valid %d)\n",
5092 enable_rc6 & mask, enable_rc6, mask);
4753 5093
4754 return enable_rc6 & mask; 5094 return enable_rc6 & mask;
4755 } 5095 }
4756 5096
4757 if (IS_IVYBRIDGE(dev)) 5097 if (IS_IVYBRIDGE(dev_priv))
4758 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); 5098 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
4759 5099
4760 return INTEL_RC6_ENABLE; 5100 return INTEL_RC6_ENABLE;
4761} 5101}
4762 5102
4763int intel_enable_rc6(const struct drm_device *dev) 5103static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
4764{ 5104{
4765 return i915.enable_rc6;
4766}
4767
4768static void gen6_init_rps_frequencies(struct drm_device *dev)
4769{
4770 struct drm_i915_private *dev_priv = dev->dev_private;
4771 uint32_t rp_state_cap; 5105 uint32_t rp_state_cap;
4772 u32 ddcc_status = 0; 5106 u32 ddcc_status = 0;
4773 int ret; 5107 int ret;
@@ -4775,7 +5109,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
4775 /* All of these values are in units of 50MHz */ 5109 /* All of these values are in units of 50MHz */
4776 dev_priv->rps.cur_freq = 0; 5110 dev_priv->rps.cur_freq = 0;
4777 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 5111 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
4778 if (IS_BROXTON(dev)) { 5112 if (IS_BROXTON(dev_priv)) {
4779 rp_state_cap = I915_READ(BXT_RP_STATE_CAP); 5113 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
4780 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff; 5114 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
4781 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; 5115 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
@@ -4791,8 +5125,8 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
4791 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; 5125 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4792 5126
4793 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; 5127 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
4794 if (IS_HASWELL(dev) || IS_BROADWELL(dev) || 5128 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
4795 IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { 5129 IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
4796 ret = sandybridge_pcode_read(dev_priv, 5130 ret = sandybridge_pcode_read(dev_priv,
4797 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 5131 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
4798 &ddcc_status); 5132 &ddcc_status);
@@ -4804,7 +5138,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
4804 dev_priv->rps.max_freq); 5138 dev_priv->rps.max_freq);
4805 } 5139 }
4806 5140
4807 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { 5141 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
4808 /* Store the frequency values in 16.66 MHZ units, which is 5142 /* Store the frequency values in 16.66 MHZ units, which is
4809 the natural hardware unit for SKL */ 5143 the natural hardware unit for SKL */
4810 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; 5144 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
@@ -4821,7 +5155,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
4821 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 5155 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4822 5156
4823 if (dev_priv->rps.min_freq_softlimit == 0) { 5157 if (dev_priv->rps.min_freq_softlimit == 0) {
4824 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 5158 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4825 dev_priv->rps.min_freq_softlimit = 5159 dev_priv->rps.min_freq_softlimit =
4826 max_t(int, dev_priv->rps.efficient_freq, 5160 max_t(int, dev_priv->rps.efficient_freq,
4827 intel_freq_opcode(dev_priv, 450)); 5161 intel_freq_opcode(dev_priv, 450));
@@ -4832,16 +5166,14 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
4832} 5166}
4833 5167
4834/* See the Gen9_GT_PM_Programming_Guide doc for the below */ 5168/* See the Gen9_GT_PM_Programming_Guide doc for the below */
4835static void gen9_enable_rps(struct drm_device *dev) 5169static void gen9_enable_rps(struct drm_i915_private *dev_priv)
4836{ 5170{
4837 struct drm_i915_private *dev_priv = dev->dev_private;
4838
4839 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5171 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4840 5172
4841 gen6_init_rps_frequencies(dev); 5173 gen6_init_rps_frequencies(dev_priv);
4842 5174
4843 /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ 5175 /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
4844 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { 5176 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
4845 /* 5177 /*
4846 * BIOS could leave the Hw Turbo enabled, so need to explicitly 5178 * BIOS could leave the Hw Turbo enabled, so need to explicitly
4847 * clear out the Control register just to avoid inconsitency 5179 * clear out the Control register just to avoid inconsitency
@@ -4851,7 +5183,7 @@ static void gen9_enable_rps(struct drm_device *dev)
4851 * if the Turbo is left enabled in the Control register, as the 5183 * if the Turbo is left enabled in the Control register, as the
4852 * Up/Down interrupts would remain masked. 5184 * Up/Down interrupts would remain masked.
4853 */ 5185 */
4854 gen9_disable_rps(dev); 5186 gen9_disable_rps(dev_priv);
4855 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5187 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4856 return; 5188 return;
4857 } 5189 }
@@ -4870,14 +5202,13 @@ static void gen9_enable_rps(struct drm_device *dev)
4870 * Up/Down EI & threshold registers, as well as the RP_CONTROL, 5202 * Up/Down EI & threshold registers, as well as the RP_CONTROL,
4871 * RP_INTERRUPT_LIMITS & RPNSWREQ registers */ 5203 * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
4872 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 5204 dev_priv->rps.power = HIGH_POWER; /* force a reset */
4873 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 5205 gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
4874 5206
4875 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5207 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4876} 5208}
4877 5209
4878static void gen9_enable_rc6(struct drm_device *dev) 5210static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
4879{ 5211{
4880 struct drm_i915_private *dev_priv = dev->dev_private;
4881 struct intel_engine_cs *engine; 5212 struct intel_engine_cs *engine;
4882 uint32_t rc6_mask = 0; 5213 uint32_t rc6_mask = 0;
4883 5214
@@ -4894,7 +5225,7 @@ static void gen9_enable_rc6(struct drm_device *dev)
4894 /* 2b: Program RC6 thresholds.*/ 5225 /* 2b: Program RC6 thresholds.*/
4895 5226
4896 /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */ 5227 /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
4897 if (IS_SKYLAKE(dev)) 5228 if (IS_SKYLAKE(dev_priv))
4898 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); 5229 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
4899 else 5230 else
4900 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); 5231 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
@@ -4903,7 +5234,7 @@ static void gen9_enable_rc6(struct drm_device *dev)
4903 for_each_engine(engine, dev_priv) 5234 for_each_engine(engine, dev_priv)
4904 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); 5235 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
4905 5236
4906 if (HAS_GUC_UCODE(dev)) 5237 if (HAS_GUC(dev_priv))
4907 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA); 5238 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
4908 5239
4909 I915_WRITE(GEN6_RC_SLEEP, 0); 5240 I915_WRITE(GEN6_RC_SLEEP, 0);
@@ -4913,12 +5244,12 @@ static void gen9_enable_rc6(struct drm_device *dev)
4913 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25); 5244 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
4914 5245
4915 /* 3a: Enable RC6 */ 5246 /* 3a: Enable RC6 */
4916 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 5247 if (intel_enable_rc6() & INTEL_RC6_ENABLE)
4917 rc6_mask = GEN6_RC_CTL_RC6_ENABLE; 5248 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4918 DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE)); 5249 DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
4919 /* WaRsUseTimeoutMode */ 5250 /* WaRsUseTimeoutMode */
4920 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || 5251 if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
4921 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { 5252 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
4922 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */ 5253 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
4923 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 5254 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4924 GEN7_RC_CTL_TO_MODE | 5255 GEN7_RC_CTL_TO_MODE |
@@ -4934,19 +5265,17 @@ static void gen9_enable_rc6(struct drm_device *dev)
4934 * 3b: Enable Coarse Power Gating only when RC6 is enabled. 5265 * 3b: Enable Coarse Power Gating only when RC6 is enabled.
4935 * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6. 5266 * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
4936 */ 5267 */
4937 if (NEEDS_WaRsDisableCoarsePowerGating(dev)) 5268 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
4938 I915_WRITE(GEN9_PG_ENABLE, 0); 5269 I915_WRITE(GEN9_PG_ENABLE, 0);
4939 else 5270 else
4940 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? 5271 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4941 (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0); 5272 (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
4942 5273
4943 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5274 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4944
4945} 5275}
4946 5276
4947static void gen8_enable_rps(struct drm_device *dev) 5277static void gen8_enable_rps(struct drm_i915_private *dev_priv)
4948{ 5278{
4949 struct drm_i915_private *dev_priv = dev->dev_private;
4950 struct intel_engine_cs *engine; 5279 struct intel_engine_cs *engine;
4951 uint32_t rc6_mask = 0; 5280 uint32_t rc6_mask = 0;
4952 5281
@@ -4961,7 +5290,7 @@ static void gen8_enable_rps(struct drm_device *dev)
4961 I915_WRITE(GEN6_RC_CONTROL, 0); 5290 I915_WRITE(GEN6_RC_CONTROL, 0);
4962 5291
4963 /* Initialize rps frequencies */ 5292 /* Initialize rps frequencies */
4964 gen6_init_rps_frequencies(dev); 5293 gen6_init_rps_frequencies(dev_priv);
4965 5294
4966 /* 2b: Program RC6 thresholds.*/ 5295 /* 2b: Program RC6 thresholds.*/
4967 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 5296 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
@@ -4970,16 +5299,16 @@ static void gen8_enable_rps(struct drm_device *dev)
4970 for_each_engine(engine, dev_priv) 5299 for_each_engine(engine, dev_priv)
4971 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); 5300 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
4972 I915_WRITE(GEN6_RC_SLEEP, 0); 5301 I915_WRITE(GEN6_RC_SLEEP, 0);
4973 if (IS_BROADWELL(dev)) 5302 if (IS_BROADWELL(dev_priv))
4974 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ 5303 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
4975 else 5304 else
4976 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ 5305 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4977 5306
4978 /* 3: Enable RC6 */ 5307 /* 3: Enable RC6 */
4979 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 5308 if (intel_enable_rc6() & INTEL_RC6_ENABLE)
4980 rc6_mask = GEN6_RC_CTL_RC6_ENABLE; 5309 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4981 intel_print_rc6_info(dev, rc6_mask); 5310 intel_print_rc6_info(dev_priv, rc6_mask);
4982 if (IS_BROADWELL(dev)) 5311 if (IS_BROADWELL(dev_priv))
4983 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 5312 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4984 GEN7_RC_CTL_TO_MODE | 5313 GEN7_RC_CTL_TO_MODE |
4985 rc6_mask); 5314 rc6_mask);
@@ -5020,14 +5349,13 @@ static void gen8_enable_rps(struct drm_device *dev)
5020 /* 6: Ring frequency + overclocking (our driver does this later */ 5349 /* 6: Ring frequency + overclocking (our driver does this later */
5021 5350
5022 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 5351 dev_priv->rps.power = HIGH_POWER; /* force a reset */
5023 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 5352 gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
5024 5353
5025 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5354 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5026} 5355}
5027 5356
5028static void gen6_enable_rps(struct drm_device *dev) 5357static void gen6_enable_rps(struct drm_i915_private *dev_priv)
5029{ 5358{
5030 struct drm_i915_private *dev_priv = dev->dev_private;
5031 struct intel_engine_cs *engine; 5359 struct intel_engine_cs *engine;
5032 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; 5360 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
5033 u32 gtfifodbg; 5361 u32 gtfifodbg;
@@ -5054,7 +5382,7 @@ static void gen6_enable_rps(struct drm_device *dev)
5054 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5382 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5055 5383
5056 /* Initialize rps frequencies */ 5384 /* Initialize rps frequencies */
5057 gen6_init_rps_frequencies(dev); 5385 gen6_init_rps_frequencies(dev_priv);
5058 5386
5059 /* disable the counters and set deterministic thresholds */ 5387 /* disable the counters and set deterministic thresholds */
5060 I915_WRITE(GEN6_RC_CONTROL, 0); 5388 I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -5070,7 +5398,7 @@ static void gen6_enable_rps(struct drm_device *dev)
5070 5398
5071 I915_WRITE(GEN6_RC_SLEEP, 0); 5399 I915_WRITE(GEN6_RC_SLEEP, 0);
5072 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); 5400 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
5073 if (IS_IVYBRIDGE(dev)) 5401 if (IS_IVYBRIDGE(dev_priv))
5074 I915_WRITE(GEN6_RC6_THRESHOLD, 125000); 5402 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
5075 else 5403 else
5076 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); 5404 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
@@ -5078,12 +5406,12 @@ static void gen6_enable_rps(struct drm_device *dev)
5078 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */ 5406 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
5079 5407
5080 /* Check if we are enabling RC6 */ 5408 /* Check if we are enabling RC6 */
5081 rc6_mode = intel_enable_rc6(dev_priv->dev); 5409 rc6_mode = intel_enable_rc6();
5082 if (rc6_mode & INTEL_RC6_ENABLE) 5410 if (rc6_mode & INTEL_RC6_ENABLE)
5083 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE; 5411 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
5084 5412
5085 /* We don't use those on Haswell */ 5413 /* We don't use those on Haswell */
5086 if (!IS_HASWELL(dev)) { 5414 if (!IS_HASWELL(dev_priv)) {
5087 if (rc6_mode & INTEL_RC6p_ENABLE) 5415 if (rc6_mode & INTEL_RC6p_ENABLE)
5088 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE; 5416 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
5089 5417
@@ -5091,7 +5419,7 @@ static void gen6_enable_rps(struct drm_device *dev)
5091 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE; 5419 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
5092 } 5420 }
5093 5421
5094 intel_print_rc6_info(dev, rc6_mask); 5422 intel_print_rc6_info(dev_priv, rc6_mask);
5095 5423
5096 I915_WRITE(GEN6_RC_CONTROL, 5424 I915_WRITE(GEN6_RC_CONTROL,
5097 rc6_mask | 5425 rc6_mask |
@@ -5115,13 +5443,13 @@ static void gen6_enable_rps(struct drm_device *dev)
5115 } 5443 }
5116 5444
5117 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 5445 dev_priv->rps.power = HIGH_POWER; /* force a reset */
5118 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 5446 gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
5119 5447
5120 rc6vids = 0; 5448 rc6vids = 0;
5121 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); 5449 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
5122 if (IS_GEN6(dev) && ret) { 5450 if (IS_GEN6(dev_priv) && ret) {
5123 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n"); 5451 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
5124 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { 5452 } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
5125 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", 5453 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
5126 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); 5454 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
5127 rc6vids &= 0xffff00; 5455 rc6vids &= 0xffff00;
@@ -5134,9 +5462,8 @@ static void gen6_enable_rps(struct drm_device *dev)
5134 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5462 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5135} 5463}
5136 5464
5137static void __gen6_update_ring_freq(struct drm_device *dev) 5465static void __gen6_update_ring_freq(struct drm_i915_private *dev_priv)
5138{ 5466{
5139 struct drm_i915_private *dev_priv = dev->dev_private;
5140 int min_freq = 15; 5467 int min_freq = 15;
5141 unsigned int gpu_freq; 5468 unsigned int gpu_freq;
5142 unsigned int max_ia_freq, min_ring_freq; 5469 unsigned int max_ia_freq, min_ring_freq;
@@ -5165,7 +5492,7 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
5165 /* convert DDR frequency from units of 266.6MHz to bandwidth */ 5492 /* convert DDR frequency from units of 266.6MHz to bandwidth */
5166 min_ring_freq = mult_frac(min_ring_freq, 8, 3); 5493 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
5167 5494
5168 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { 5495 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
5169 /* Convert GT frequency to 50 HZ units */ 5496 /* Convert GT frequency to 50 HZ units */
5170 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER; 5497 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
5171 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER; 5498 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
@@ -5183,16 +5510,16 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
5183 int diff = max_gpu_freq - gpu_freq; 5510 int diff = max_gpu_freq - gpu_freq;
5184 unsigned int ia_freq = 0, ring_freq = 0; 5511 unsigned int ia_freq = 0, ring_freq = 0;
5185 5512
5186 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { 5513 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
5187 /* 5514 /*
5188 * ring_freq = 2 * GT. ring_freq is in 100MHz units 5515 * ring_freq = 2 * GT. ring_freq is in 100MHz units
5189 * No floor required for ring frequency on SKL. 5516 * No floor required for ring frequency on SKL.
5190 */ 5517 */
5191 ring_freq = gpu_freq; 5518 ring_freq = gpu_freq;
5192 } else if (INTEL_INFO(dev)->gen >= 8) { 5519 } else if (INTEL_INFO(dev_priv)->gen >= 8) {
5193 /* max(2 * GT, DDR). NB: GT is 50MHz units */ 5520 /* max(2 * GT, DDR). NB: GT is 50MHz units */
5194 ring_freq = max(min_ring_freq, gpu_freq); 5521 ring_freq = max(min_ring_freq, gpu_freq);
5195 } else if (IS_HASWELL(dev)) { 5522 } else if (IS_HASWELL(dev_priv)) {
5196 ring_freq = mult_frac(gpu_freq, 5, 4); 5523 ring_freq = mult_frac(gpu_freq, 5, 4);
5197 ring_freq = max(min_ring_freq, ring_freq); 5524 ring_freq = max(min_ring_freq, ring_freq);
5198 /* leave ia_freq as the default, chosen by cpufreq */ 5525 /* leave ia_freq as the default, chosen by cpufreq */
@@ -5219,26 +5546,23 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
5219 } 5546 }
5220} 5547}
5221 5548
5222void gen6_update_ring_freq(struct drm_device *dev) 5549void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
5223{ 5550{
5224 struct drm_i915_private *dev_priv = dev->dev_private; 5551 if (!HAS_CORE_RING_FREQ(dev_priv))
5225
5226 if (!HAS_CORE_RING_FREQ(dev))
5227 return; 5552 return;
5228 5553
5229 mutex_lock(&dev_priv->rps.hw_lock); 5554 mutex_lock(&dev_priv->rps.hw_lock);
5230 __gen6_update_ring_freq(dev); 5555 __gen6_update_ring_freq(dev_priv);
5231 mutex_unlock(&dev_priv->rps.hw_lock); 5556 mutex_unlock(&dev_priv->rps.hw_lock);
5232} 5557}
5233 5558
5234static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) 5559static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
5235{ 5560{
5236 struct drm_device *dev = dev_priv->dev;
5237 u32 val, rp0; 5561 u32 val, rp0;
5238 5562
5239 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); 5563 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5240 5564
5241 switch (INTEL_INFO(dev)->eu_total) { 5565 switch (INTEL_INFO(dev_priv)->eu_total) {
5242 case 8: 5566 case 8:
5243 /* (2 * 4) config */ 5567 /* (2 * 4) config */
5244 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); 5568 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
@@ -5349,9 +5673,8 @@ static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
5349 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0); 5673 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
5350} 5674}
5351 5675
5352static void cherryview_setup_pctx(struct drm_device *dev) 5676static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
5353{ 5677{
5354 struct drm_i915_private *dev_priv = to_i915(dev);
5355 struct i915_ggtt *ggtt = &dev_priv->ggtt; 5678 struct i915_ggtt *ggtt = &dev_priv->ggtt;
5356 unsigned long pctx_paddr, paddr; 5679 unsigned long pctx_paddr, paddr;
5357 u32 pcbr; 5680 u32 pcbr;
@@ -5370,15 +5693,14 @@ static void cherryview_setup_pctx(struct drm_device *dev)
5370 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR)); 5693 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5371} 5694}
5372 5695
5373static void valleyview_setup_pctx(struct drm_device *dev) 5696static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
5374{ 5697{
5375 struct drm_i915_private *dev_priv = dev->dev_private;
5376 struct drm_i915_gem_object *pctx; 5698 struct drm_i915_gem_object *pctx;
5377 unsigned long pctx_paddr; 5699 unsigned long pctx_paddr;
5378 u32 pcbr; 5700 u32 pcbr;
5379 int pctx_size = 24*1024; 5701 int pctx_size = 24*1024;
5380 5702
5381 mutex_lock(&dev->struct_mutex); 5703 mutex_lock(&dev_priv->drm.struct_mutex);
5382 5704
5383 pcbr = I915_READ(VLV_PCBR); 5705 pcbr = I915_READ(VLV_PCBR);
5384 if (pcbr) { 5706 if (pcbr) {
@@ -5386,7 +5708,7 @@ static void valleyview_setup_pctx(struct drm_device *dev)
5386 int pcbr_offset; 5708 int pcbr_offset;
5387 5709
5388 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base; 5710 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
5389 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev, 5711 pctx = i915_gem_object_create_stolen_for_preallocated(&dev_priv->drm,
5390 pcbr_offset, 5712 pcbr_offset,
5391 I915_GTT_OFFSET_NONE, 5713 I915_GTT_OFFSET_NONE,
5392 pctx_size); 5714 pctx_size);
@@ -5403,7 +5725,7 @@ static void valleyview_setup_pctx(struct drm_device *dev)
5403 * overlap with other ranges, such as the frame buffer, protected 5725 * overlap with other ranges, such as the frame buffer, protected
5404 * memory, or any other relevant ranges. 5726 * memory, or any other relevant ranges.
5405 */ 5727 */
5406 pctx = i915_gem_object_create_stolen(dev, pctx_size); 5728 pctx = i915_gem_object_create_stolen(&dev_priv->drm, pctx_size);
5407 if (!pctx) { 5729 if (!pctx) {
5408 DRM_DEBUG("not enough stolen space for PCTX, disabling\n"); 5730 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
5409 goto out; 5731 goto out;
@@ -5415,13 +5737,11 @@ static void valleyview_setup_pctx(struct drm_device *dev)
5415out: 5737out:
5416 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR)); 5738 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5417 dev_priv->vlv_pctx = pctx; 5739 dev_priv->vlv_pctx = pctx;
5418 mutex_unlock(&dev->struct_mutex); 5740 mutex_unlock(&dev_priv->drm.struct_mutex);
5419} 5741}
5420 5742
5421static void valleyview_cleanup_pctx(struct drm_device *dev) 5743static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
5422{ 5744{
5423 struct drm_i915_private *dev_priv = dev->dev_private;
5424
5425 if (WARN_ON(!dev_priv->vlv_pctx)) 5745 if (WARN_ON(!dev_priv->vlv_pctx))
5426 return; 5746 return;
5427 5747
@@ -5440,12 +5760,11 @@ static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
5440 dev_priv->rps.gpll_ref_freq); 5760 dev_priv->rps.gpll_ref_freq);
5441} 5761}
5442 5762
5443static void valleyview_init_gt_powersave(struct drm_device *dev) 5763static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
5444{ 5764{
5445 struct drm_i915_private *dev_priv = dev->dev_private;
5446 u32 val; 5765 u32 val;
5447 5766
5448 valleyview_setup_pctx(dev); 5767 valleyview_setup_pctx(dev_priv);
5449 5768
5450 vlv_init_gpll_ref_freq(dev_priv); 5769 vlv_init_gpll_ref_freq(dev_priv);
5451 5770
@@ -5499,12 +5818,11 @@ static void valleyview_init_gt_powersave(struct drm_device *dev)
5499 mutex_unlock(&dev_priv->rps.hw_lock); 5818 mutex_unlock(&dev_priv->rps.hw_lock);
5500} 5819}
5501 5820
5502static void cherryview_init_gt_powersave(struct drm_device *dev) 5821static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
5503{ 5822{
5504 struct drm_i915_private *dev_priv = dev->dev_private;
5505 u32 val; 5823 u32 val;
5506 5824
5507 cherryview_setup_pctx(dev); 5825 cherryview_setup_pctx(dev_priv);
5508 5826
5509 vlv_init_gpll_ref_freq(dev_priv); 5827 vlv_init_gpll_ref_freq(dev_priv);
5510 5828
@@ -5564,14 +5882,13 @@ static void cherryview_init_gt_powersave(struct drm_device *dev)
5564 mutex_unlock(&dev_priv->rps.hw_lock); 5882 mutex_unlock(&dev_priv->rps.hw_lock);
5565} 5883}
5566 5884
5567static void valleyview_cleanup_gt_powersave(struct drm_device *dev) 5885static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
5568{ 5886{
5569 valleyview_cleanup_pctx(dev); 5887 valleyview_cleanup_pctx(dev_priv);
5570} 5888}
5571 5889
5572static void cherryview_enable_rps(struct drm_device *dev) 5890static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
5573{ 5891{
5574 struct drm_i915_private *dev_priv = dev->dev_private;
5575 struct intel_engine_cs *engine; 5892 struct intel_engine_cs *engine;
5576 u32 gtfifodbg, val, rc6_mode = 0, pcbr; 5893 u32 gtfifodbg, val, rc6_mode = 0, pcbr;
5577 5894
@@ -5616,8 +5933,8 @@ static void cherryview_enable_rps(struct drm_device *dev)
5616 pcbr = I915_READ(VLV_PCBR); 5933 pcbr = I915_READ(VLV_PCBR);
5617 5934
5618 /* 3: Enable RC6 */ 5935 /* 3: Enable RC6 */
5619 if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) && 5936 if ((intel_enable_rc6() & INTEL_RC6_ENABLE) &&
5620 (pcbr >> VLV_PCBR_ADDR_SHIFT)) 5937 (pcbr >> VLV_PCBR_ADDR_SHIFT))
5621 rc6_mode = GEN7_RC_CTL_TO_MODE; 5938 rc6_mode = GEN7_RC_CTL_TO_MODE;
5622 5939
5623 I915_WRITE(GEN6_RC_CONTROL, rc6_mode); 5940 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
@@ -5662,14 +5979,13 @@ static void cherryview_enable_rps(struct drm_device *dev)
5662 intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq), 5979 intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
5663 dev_priv->rps.idle_freq); 5980 dev_priv->rps.idle_freq);
5664 5981
5665 valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 5982 valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq);
5666 5983
5667 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5984 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5668} 5985}
5669 5986
5670static void valleyview_enable_rps(struct drm_device *dev) 5987static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
5671{ 5988{
5672 struct drm_i915_private *dev_priv = dev->dev_private;
5673 struct intel_engine_cs *engine; 5989 struct intel_engine_cs *engine;
5674 u32 gtfifodbg, val, rc6_mode = 0; 5990 u32 gtfifodbg, val, rc6_mode = 0;
5675 5991
@@ -5722,10 +6038,10 @@ static void valleyview_enable_rps(struct drm_device *dev)
5722 VLV_MEDIA_RC6_COUNT_EN | 6038 VLV_MEDIA_RC6_COUNT_EN |
5723 VLV_RENDER_RC6_COUNT_EN)); 6039 VLV_RENDER_RC6_COUNT_EN));
5724 6040
5725 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 6041 if (intel_enable_rc6() & INTEL_RC6_ENABLE)
5726 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; 6042 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
5727 6043
5728 intel_print_rc6_info(dev, rc6_mode); 6044 intel_print_rc6_info(dev_priv, rc6_mode);
5729 6045
5730 I915_WRITE(GEN6_RC_CONTROL, rc6_mode); 6046 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
5731 6047
@@ -5752,7 +6068,7 @@ static void valleyview_enable_rps(struct drm_device *dev)
5752 intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq), 6068 intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
5753 dev_priv->rps.idle_freq); 6069 dev_priv->rps.idle_freq);
5754 6070
5755 valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); 6071 valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq);
5756 6072
5757 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 6073 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5758} 6074}
@@ -5842,10 +6158,9 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
5842 6158
5843unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) 6159unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
5844{ 6160{
5845 struct drm_device *dev = dev_priv->dev;
5846 unsigned long val; 6161 unsigned long val;
5847 6162
5848 if (INTEL_INFO(dev)->gen != 5) 6163 if (INTEL_INFO(dev_priv)->gen != 5)
5849 return 0; 6164 return 0;
5850 6165
5851 spin_lock_irq(&mchdev_lock); 6166 spin_lock_irq(&mchdev_lock);
@@ -5885,11 +6200,10 @@ static int _pxvid_to_vd(u8 pxvid)
5885 6200
5886static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) 6201static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
5887{ 6202{
5888 struct drm_device *dev = dev_priv->dev;
5889 const int vd = _pxvid_to_vd(pxvid); 6203 const int vd = _pxvid_to_vd(pxvid);
5890 const int vm = vd - 1125; 6204 const int vm = vd - 1125;
5891 6205
5892 if (INTEL_INFO(dev)->is_mobile) 6206 if (INTEL_INFO(dev_priv)->is_mobile)
5893 return vm > 0 ? vm : 0; 6207 return vm > 0 ? vm : 0;
5894 6208
5895 return vd; 6209 return vd;
@@ -5930,9 +6244,7 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
5930 6244
5931void i915_update_gfx_val(struct drm_i915_private *dev_priv) 6245void i915_update_gfx_val(struct drm_i915_private *dev_priv)
5932{ 6246{
5933 struct drm_device *dev = dev_priv->dev; 6247 if (INTEL_INFO(dev_priv)->gen != 5)
5934
5935 if (INTEL_INFO(dev)->gen != 5)
5936 return; 6248 return;
5937 6249
5938 spin_lock_irq(&mchdev_lock); 6250 spin_lock_irq(&mchdev_lock);
@@ -5981,10 +6293,9 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
5981 6293
5982unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) 6294unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
5983{ 6295{
5984 struct drm_device *dev = dev_priv->dev;
5985 unsigned long val; 6296 unsigned long val;
5986 6297
5987 if (INTEL_INFO(dev)->gen != 5) 6298 if (INTEL_INFO(dev_priv)->gen != 5)
5988 return 0; 6299 return 0;
5989 6300
5990 spin_lock_irq(&mchdev_lock); 6301 spin_lock_irq(&mchdev_lock);
@@ -6125,7 +6436,7 @@ bool i915_gpu_turbo_disable(void)
6125 6436
6126 dev_priv->ips.max_delay = dev_priv->ips.fstart; 6437 dev_priv->ips.max_delay = dev_priv->ips.fstart;
6127 6438
6128 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart)) 6439 if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
6129 ret = false; 6440 ret = false;
6130 6441
6131out_unlock: 6442out_unlock:
@@ -6173,9 +6484,8 @@ void intel_gpu_ips_teardown(void)
6173 spin_unlock_irq(&mchdev_lock); 6484 spin_unlock_irq(&mchdev_lock);
6174} 6485}
6175 6486
6176static void intel_init_emon(struct drm_device *dev) 6487static void intel_init_emon(struct drm_i915_private *dev_priv)
6177{ 6488{
6178 struct drm_i915_private *dev_priv = dev->dev_private;
6179 u32 lcfuse; 6489 u32 lcfuse;
6180 u8 pxw[16]; 6490 u8 pxw[16];
6181 int i; 6491 int i;
@@ -6244,10 +6554,8 @@ static void intel_init_emon(struct drm_device *dev)
6244 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); 6554 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
6245} 6555}
6246 6556
6247void intel_init_gt_powersave(struct drm_device *dev) 6557void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
6248{ 6558{
6249 struct drm_i915_private *dev_priv = dev->dev_private;
6250
6251 /* 6559 /*
6252 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a 6560 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
6253 * requirement. 6561 * requirement.
@@ -6257,74 +6565,66 @@ void intel_init_gt_powersave(struct drm_device *dev)
6257 intel_runtime_pm_get(dev_priv); 6565 intel_runtime_pm_get(dev_priv);
6258 } 6566 }
6259 6567
6260 if (IS_CHERRYVIEW(dev)) 6568 if (IS_CHERRYVIEW(dev_priv))
6261 cherryview_init_gt_powersave(dev); 6569 cherryview_init_gt_powersave(dev_priv);
6262 else if (IS_VALLEYVIEW(dev)) 6570 else if (IS_VALLEYVIEW(dev_priv))
6263 valleyview_init_gt_powersave(dev); 6571 valleyview_init_gt_powersave(dev_priv);
6264} 6572}
6265 6573
6266void intel_cleanup_gt_powersave(struct drm_device *dev) 6574void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
6267{ 6575{
6268 struct drm_i915_private *dev_priv = dev->dev_private; 6576 if (IS_CHERRYVIEW(dev_priv))
6269
6270 if (IS_CHERRYVIEW(dev))
6271 return; 6577 return;
6272 else if (IS_VALLEYVIEW(dev)) 6578 else if (IS_VALLEYVIEW(dev_priv))
6273 valleyview_cleanup_gt_powersave(dev); 6579 valleyview_cleanup_gt_powersave(dev_priv);
6274 6580
6275 if (!i915.enable_rc6) 6581 if (!i915.enable_rc6)
6276 intel_runtime_pm_put(dev_priv); 6582 intel_runtime_pm_put(dev_priv);
6277} 6583}
6278 6584
6279static void gen6_suspend_rps(struct drm_device *dev) 6585static void gen6_suspend_rps(struct drm_i915_private *dev_priv)
6280{ 6586{
6281 struct drm_i915_private *dev_priv = dev->dev_private;
6282
6283 flush_delayed_work(&dev_priv->rps.delayed_resume_work); 6587 flush_delayed_work(&dev_priv->rps.delayed_resume_work);
6284 6588
6285 gen6_disable_rps_interrupts(dev); 6589 gen6_disable_rps_interrupts(dev_priv);
6286} 6590}
6287 6591
6288/** 6592/**
6289 * intel_suspend_gt_powersave - suspend PM work and helper threads 6593 * intel_suspend_gt_powersave - suspend PM work and helper threads
6290 * @dev: drm device 6594 * @dev_priv: i915 device
6291 * 6595 *
6292 * We don't want to disable RC6 or other features here, we just want 6596 * We don't want to disable RC6 or other features here, we just want
6293 * to make sure any work we've queued has finished and won't bother 6597 * to make sure any work we've queued has finished and won't bother
6294 * us while we're suspended. 6598 * us while we're suspended.
6295 */ 6599 */
6296void intel_suspend_gt_powersave(struct drm_device *dev) 6600void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
6297{ 6601{
6298 struct drm_i915_private *dev_priv = dev->dev_private; 6602 if (INTEL_GEN(dev_priv) < 6)
6299
6300 if (INTEL_INFO(dev)->gen < 6)
6301 return; 6603 return;
6302 6604
6303 gen6_suspend_rps(dev); 6605 gen6_suspend_rps(dev_priv);
6304 6606
6305 /* Force GPU to min freq during suspend */ 6607 /* Force GPU to min freq during suspend */
6306 gen6_rps_idle(dev_priv); 6608 gen6_rps_idle(dev_priv);
6307} 6609}
6308 6610
6309void intel_disable_gt_powersave(struct drm_device *dev) 6611void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
6310{ 6612{
6311 struct drm_i915_private *dev_priv = dev->dev_private; 6613 if (IS_IRONLAKE_M(dev_priv)) {
6312 6614 ironlake_disable_drps(dev_priv);
6313 if (IS_IRONLAKE_M(dev)) { 6615 } else if (INTEL_INFO(dev_priv)->gen >= 6) {
6314 ironlake_disable_drps(dev); 6616 intel_suspend_gt_powersave(dev_priv);
6315 } else if (INTEL_INFO(dev)->gen >= 6) {
6316 intel_suspend_gt_powersave(dev);
6317 6617
6318 mutex_lock(&dev_priv->rps.hw_lock); 6618 mutex_lock(&dev_priv->rps.hw_lock);
6319 if (INTEL_INFO(dev)->gen >= 9) { 6619 if (INTEL_INFO(dev_priv)->gen >= 9) {
6320 gen9_disable_rc6(dev); 6620 gen9_disable_rc6(dev_priv);
6321 gen9_disable_rps(dev); 6621 gen9_disable_rps(dev_priv);
6322 } else if (IS_CHERRYVIEW(dev)) 6622 } else if (IS_CHERRYVIEW(dev_priv))
6323 cherryview_disable_rps(dev); 6623 cherryview_disable_rps(dev_priv);
6324 else if (IS_VALLEYVIEW(dev)) 6624 else if (IS_VALLEYVIEW(dev_priv))
6325 valleyview_disable_rps(dev); 6625 valleyview_disable_rps(dev_priv);
6326 else 6626 else
6327 gen6_disable_rps(dev); 6627 gen6_disable_rps(dev_priv);
6328 6628
6329 dev_priv->rps.enabled = false; 6629 dev_priv->rps.enabled = false;
6330 mutex_unlock(&dev_priv->rps.hw_lock); 6630 mutex_unlock(&dev_priv->rps.hw_lock);
@@ -6336,27 +6636,26 @@ static void intel_gen6_powersave_work(struct work_struct *work)
6336 struct drm_i915_private *dev_priv = 6636 struct drm_i915_private *dev_priv =
6337 container_of(work, struct drm_i915_private, 6637 container_of(work, struct drm_i915_private,
6338 rps.delayed_resume_work.work); 6638 rps.delayed_resume_work.work);
6339 struct drm_device *dev = dev_priv->dev;
6340 6639
6341 mutex_lock(&dev_priv->rps.hw_lock); 6640 mutex_lock(&dev_priv->rps.hw_lock);
6342 6641
6343 gen6_reset_rps_interrupts(dev); 6642 gen6_reset_rps_interrupts(dev_priv);
6344 6643
6345 if (IS_CHERRYVIEW(dev)) { 6644 if (IS_CHERRYVIEW(dev_priv)) {
6346 cherryview_enable_rps(dev); 6645 cherryview_enable_rps(dev_priv);
6347 } else if (IS_VALLEYVIEW(dev)) { 6646 } else if (IS_VALLEYVIEW(dev_priv)) {
6348 valleyview_enable_rps(dev); 6647 valleyview_enable_rps(dev_priv);
6349 } else if (INTEL_INFO(dev)->gen >= 9) { 6648 } else if (INTEL_INFO(dev_priv)->gen >= 9) {
6350 gen9_enable_rc6(dev); 6649 gen9_enable_rc6(dev_priv);
6351 gen9_enable_rps(dev); 6650 gen9_enable_rps(dev_priv);
6352 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) 6651 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
6353 __gen6_update_ring_freq(dev); 6652 __gen6_update_ring_freq(dev_priv);
6354 } else if (IS_BROADWELL(dev)) { 6653 } else if (IS_BROADWELL(dev_priv)) {
6355 gen8_enable_rps(dev); 6654 gen8_enable_rps(dev_priv);
6356 __gen6_update_ring_freq(dev); 6655 __gen6_update_ring_freq(dev_priv);
6357 } else { 6656 } else {
6358 gen6_enable_rps(dev); 6657 gen6_enable_rps(dev_priv);
6359 __gen6_update_ring_freq(dev); 6658 __gen6_update_ring_freq(dev_priv);
6360 } 6659 }
6361 6660
6362 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq); 6661 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
@@ -6367,27 +6666,25 @@ static void intel_gen6_powersave_work(struct work_struct *work)
6367 6666
6368 dev_priv->rps.enabled = true; 6667 dev_priv->rps.enabled = true;
6369 6668
6370 gen6_enable_rps_interrupts(dev); 6669 gen6_enable_rps_interrupts(dev_priv);
6371 6670
6372 mutex_unlock(&dev_priv->rps.hw_lock); 6671 mutex_unlock(&dev_priv->rps.hw_lock);
6373 6672
6374 intel_runtime_pm_put(dev_priv); 6673 intel_runtime_pm_put(dev_priv);
6375} 6674}
6376 6675
6377void intel_enable_gt_powersave(struct drm_device *dev) 6676void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
6378{ 6677{
6379 struct drm_i915_private *dev_priv = dev->dev_private;
6380
6381 /* Powersaving is controlled by the host when inside a VM */ 6678 /* Powersaving is controlled by the host when inside a VM */
6382 if (intel_vgpu_active(dev)) 6679 if (intel_vgpu_active(dev_priv))
6383 return; 6680 return;
6384 6681
6385 if (IS_IRONLAKE_M(dev)) { 6682 if (IS_IRONLAKE_M(dev_priv)) {
6386 ironlake_enable_drps(dev); 6683 ironlake_enable_drps(dev_priv);
6387 mutex_lock(&dev->struct_mutex); 6684 mutex_lock(&dev_priv->drm.struct_mutex);
6388 intel_init_emon(dev); 6685 intel_init_emon(dev_priv);
6389 mutex_unlock(&dev->struct_mutex); 6686 mutex_unlock(&dev_priv->drm.struct_mutex);
6390 } else if (INTEL_INFO(dev)->gen >= 6) { 6687 } else if (INTEL_INFO(dev_priv)->gen >= 6) {
6391 /* 6688 /*
6392 * PCU communication is slow and this doesn't need to be 6689 * PCU communication is slow and this doesn't need to be
6393 * done at any specific time, so do this out of our fast path 6690 * done at any specific time, so do this out of our fast path
@@ -6406,20 +6703,18 @@ void intel_enable_gt_powersave(struct drm_device *dev)
6406 } 6703 }
6407} 6704}
6408 6705
6409void intel_reset_gt_powersave(struct drm_device *dev) 6706void intel_reset_gt_powersave(struct drm_i915_private *dev_priv)
6410{ 6707{
6411 struct drm_i915_private *dev_priv = dev->dev_private; 6708 if (INTEL_INFO(dev_priv)->gen < 6)
6412
6413 if (INTEL_INFO(dev)->gen < 6)
6414 return; 6709 return;
6415 6710
6416 gen6_suspend_rps(dev); 6711 gen6_suspend_rps(dev_priv);
6417 dev_priv->rps.enabled = false; 6712 dev_priv->rps.enabled = false;
6418} 6713}
6419 6714
6420static void ibx_init_clock_gating(struct drm_device *dev) 6715static void ibx_init_clock_gating(struct drm_device *dev)
6421{ 6716{
6422 struct drm_i915_private *dev_priv = dev->dev_private; 6717 struct drm_i915_private *dev_priv = to_i915(dev);
6423 6718
6424 /* 6719 /*
6425 * On Ibex Peak and Cougar Point, we need to disable clock 6720 * On Ibex Peak and Cougar Point, we need to disable clock
@@ -6431,7 +6726,7 @@ static void ibx_init_clock_gating(struct drm_device *dev)
6431 6726
6432static void g4x_disable_trickle_feed(struct drm_device *dev) 6727static void g4x_disable_trickle_feed(struct drm_device *dev)
6433{ 6728{
6434 struct drm_i915_private *dev_priv = dev->dev_private; 6729 struct drm_i915_private *dev_priv = to_i915(dev);
6435 enum pipe pipe; 6730 enum pipe pipe;
6436 6731
6437 for_each_pipe(dev_priv, pipe) { 6732 for_each_pipe(dev_priv, pipe) {
@@ -6446,7 +6741,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev)
6446 6741
6447static void ilk_init_lp_watermarks(struct drm_device *dev) 6742static void ilk_init_lp_watermarks(struct drm_device *dev)
6448{ 6743{
6449 struct drm_i915_private *dev_priv = dev->dev_private; 6744 struct drm_i915_private *dev_priv = to_i915(dev);
6450 6745
6451 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); 6746 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6452 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); 6747 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
@@ -6460,7 +6755,7 @@ static void ilk_init_lp_watermarks(struct drm_device *dev)
6460 6755
6461static void ironlake_init_clock_gating(struct drm_device *dev) 6756static void ironlake_init_clock_gating(struct drm_device *dev)
6462{ 6757{
6463 struct drm_i915_private *dev_priv = dev->dev_private; 6758 struct drm_i915_private *dev_priv = to_i915(dev);
6464 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 6759 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6465 6760
6466 /* 6761 /*
@@ -6534,7 +6829,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
6534 6829
6535static void cpt_init_clock_gating(struct drm_device *dev) 6830static void cpt_init_clock_gating(struct drm_device *dev)
6536{ 6831{
6537 struct drm_i915_private *dev_priv = dev->dev_private; 6832 struct drm_i915_private *dev_priv = to_i915(dev);
6538 int pipe; 6833 int pipe;
6539 uint32_t val; 6834 uint32_t val;
6540 6835
@@ -6571,7 +6866,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
6571 6866
6572static void gen6_check_mch_setup(struct drm_device *dev) 6867static void gen6_check_mch_setup(struct drm_device *dev)
6573{ 6868{
6574 struct drm_i915_private *dev_priv = dev->dev_private; 6869 struct drm_i915_private *dev_priv = to_i915(dev);
6575 uint32_t tmp; 6870 uint32_t tmp;
6576 6871
6577 tmp = I915_READ(MCH_SSKPD); 6872 tmp = I915_READ(MCH_SSKPD);
@@ -6582,7 +6877,7 @@ static void gen6_check_mch_setup(struct drm_device *dev)
6582 6877
6583static void gen6_init_clock_gating(struct drm_device *dev) 6878static void gen6_init_clock_gating(struct drm_device *dev)
6584{ 6879{
6585 struct drm_i915_private *dev_priv = dev->dev_private; 6880 struct drm_i915_private *dev_priv = to_i915(dev);
6586 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 6881 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6587 6882
6588 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 6883 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
@@ -6697,7 +6992,7 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6697 6992
6698static void lpt_init_clock_gating(struct drm_device *dev) 6993static void lpt_init_clock_gating(struct drm_device *dev)
6699{ 6994{
6700 struct drm_i915_private *dev_priv = dev->dev_private; 6995 struct drm_i915_private *dev_priv = to_i915(dev);
6701 6996
6702 /* 6997 /*
6703 * TODO: this bit should only be enabled when really needed, then 6998 * TODO: this bit should only be enabled when really needed, then
@@ -6716,7 +7011,7 @@ static void lpt_init_clock_gating(struct drm_device *dev)
6716 7011
6717static void lpt_suspend_hw(struct drm_device *dev) 7012static void lpt_suspend_hw(struct drm_device *dev)
6718{ 7013{
6719 struct drm_i915_private *dev_priv = dev->dev_private; 7014 struct drm_i915_private *dev_priv = to_i915(dev);
6720 7015
6721 if (HAS_PCH_LPT_LP(dev)) { 7016 if (HAS_PCH_LPT_LP(dev)) {
6722 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); 7017 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
@@ -6726,6 +7021,29 @@ static void lpt_suspend_hw(struct drm_device *dev)
6726 } 7021 }
6727} 7022}
6728 7023
7024static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7025 int general_prio_credits,
7026 int high_prio_credits)
7027{
7028 u32 misccpctl;
7029
7030 /* WaTempDisableDOPClkGating:bdw */
7031 misccpctl = I915_READ(GEN7_MISCCPCTL);
7032 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7033
7034 I915_WRITE(GEN8_L3SQCREG1,
7035 L3_GENERAL_PRIO_CREDITS(general_prio_credits) |
7036 L3_HIGH_PRIO_CREDITS(high_prio_credits));
7037
7038 /*
7039 * Wait at least 100 clocks before re-enabling clock gating.
7040 * See the definition of L3SQCREG1 in BSpec.
7041 */
7042 POSTING_READ(GEN8_L3SQCREG1);
7043 udelay(1);
7044 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7045}
7046
6729static void kabylake_init_clock_gating(struct drm_device *dev) 7047static void kabylake_init_clock_gating(struct drm_device *dev)
6730{ 7048{
6731 struct drm_i915_private *dev_priv = dev->dev_private; 7049 struct drm_i915_private *dev_priv = dev->dev_private;
@@ -6753,6 +7071,10 @@ static void skylake_init_clock_gating(struct drm_device *dev)
6753 7071
6754 gen9_init_clock_gating(dev); 7072 gen9_init_clock_gating(dev);
6755 7073
7074 /* WAC6entrylatency:skl */
7075 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7076 FBC_LLC_FULLY_OPEN);
7077
6756 /* WaFbcNukeOnHostModify:skl */ 7078 /* WaFbcNukeOnHostModify:skl */
6757 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | 7079 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6758 ILK_DPFC_NUKE_ON_ANY_MODIFICATION); 7080 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
@@ -6760,9 +7082,8 @@ static void skylake_init_clock_gating(struct drm_device *dev)
6760 7082
6761static void broadwell_init_clock_gating(struct drm_device *dev) 7083static void broadwell_init_clock_gating(struct drm_device *dev)
6762{ 7084{
6763 struct drm_i915_private *dev_priv = dev->dev_private; 7085 struct drm_i915_private *dev_priv = to_i915(dev);
6764 enum pipe pipe; 7086 enum pipe pipe;
6765 uint32_t misccpctl;
6766 7087
6767 ilk_init_lp_watermarks(dev); 7088 ilk_init_lp_watermarks(dev);
6768 7089
@@ -6793,20 +7114,8 @@ static void broadwell_init_clock_gating(struct drm_device *dev)
6793 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 7114 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6794 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 7115 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6795 7116
6796 /* 7117 /* WaProgramL3SqcReg1Default:bdw */
6797 * WaProgramL3SqcReg1Default:bdw 7118 gen8_set_l3sqc_credits(dev_priv, 30, 2);
6798 * WaTempDisableDOPClkGating:bdw
6799 */
6800 misccpctl = I915_READ(GEN7_MISCCPCTL);
6801 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
6802 I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6803 /*
6804 * Wait at least 100 clocks before re-enabling clock gating. See
6805 * the definition of L3SQCREG1 in BSpec.
6806 */
6807 POSTING_READ(GEN8_L3SQCREG1);
6808 udelay(1);
6809 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
6810 7119
6811 /* 7120 /*
6812 * WaGttCachingOffByDefault:bdw 7121 * WaGttCachingOffByDefault:bdw
@@ -6815,12 +7124,16 @@ static void broadwell_init_clock_gating(struct drm_device *dev)
6815 */ 7124 */
6816 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL); 7125 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
6817 7126
7127 /* WaKVMNotificationOnConfigChange:bdw */
7128 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7129 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7130
6818 lpt_init_clock_gating(dev); 7131 lpt_init_clock_gating(dev);
6819} 7132}
6820 7133
6821static void haswell_init_clock_gating(struct drm_device *dev) 7134static void haswell_init_clock_gating(struct drm_device *dev)
6822{ 7135{
6823 struct drm_i915_private *dev_priv = dev->dev_private; 7136 struct drm_i915_private *dev_priv = to_i915(dev);
6824 7137
6825 ilk_init_lp_watermarks(dev); 7138 ilk_init_lp_watermarks(dev);
6826 7139
@@ -6876,7 +7189,7 @@ static void haswell_init_clock_gating(struct drm_device *dev)
6876 7189
6877static void ivybridge_init_clock_gating(struct drm_device *dev) 7190static void ivybridge_init_clock_gating(struct drm_device *dev)
6878{ 7191{
6879 struct drm_i915_private *dev_priv = dev->dev_private; 7192 struct drm_i915_private *dev_priv = to_i915(dev);
6880 uint32_t snpcr; 7193 uint32_t snpcr;
6881 7194
6882 ilk_init_lp_watermarks(dev); 7195 ilk_init_lp_watermarks(dev);
@@ -6974,7 +7287,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
6974 7287
6975static void valleyview_init_clock_gating(struct drm_device *dev) 7288static void valleyview_init_clock_gating(struct drm_device *dev)
6976{ 7289{
6977 struct drm_i915_private *dev_priv = dev->dev_private; 7290 struct drm_i915_private *dev_priv = to_i915(dev);
6978 7291
6979 /* WaDisableEarlyCull:vlv */ 7292 /* WaDisableEarlyCull:vlv */
6980 I915_WRITE(_3D_CHICKEN3, 7293 I915_WRITE(_3D_CHICKEN3,
@@ -7056,7 +7369,7 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
7056 7369
7057static void cherryview_init_clock_gating(struct drm_device *dev) 7370static void cherryview_init_clock_gating(struct drm_device *dev)
7058{ 7371{
7059 struct drm_i915_private *dev_priv = dev->dev_private; 7372 struct drm_i915_private *dev_priv = to_i915(dev);
7060 7373
7061 /* WaVSRefCountFullforceMissDisable:chv */ 7374 /* WaVSRefCountFullforceMissDisable:chv */
7062 /* WaDSRefCountFullforceMissDisable:chv */ 7375 /* WaDSRefCountFullforceMissDisable:chv */
@@ -7077,6 +7390,13 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
7077 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 7390 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7078 7391
7079 /* 7392 /*
7393 * WaProgramL3SqcReg1Default:chv
7394 * See gfxspecs/Related Documents/Performance Guide/
7395 * LSQC Setting Recommendations.
7396 */
7397 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7398
7399 /*
7080 * GTT cache may not work with big pages, so if those 7400 * GTT cache may not work with big pages, so if those
7081 * are ever enabled GTT cache may need to be disabled. 7401 * are ever enabled GTT cache may need to be disabled.
7082 */ 7402 */
@@ -7085,7 +7405,7 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
7085 7405
7086static void g4x_init_clock_gating(struct drm_device *dev) 7406static void g4x_init_clock_gating(struct drm_device *dev)
7087{ 7407{
7088 struct drm_i915_private *dev_priv = dev->dev_private; 7408 struct drm_i915_private *dev_priv = to_i915(dev);
7089 uint32_t dspclk_gate; 7409 uint32_t dspclk_gate;
7090 7410
7091 I915_WRITE(RENCLK_GATE_D1, 0); 7411 I915_WRITE(RENCLK_GATE_D1, 0);
@@ -7112,7 +7432,7 @@ static void g4x_init_clock_gating(struct drm_device *dev)
7112 7432
7113static void crestline_init_clock_gating(struct drm_device *dev) 7433static void crestline_init_clock_gating(struct drm_device *dev)
7114{ 7434{
7115 struct drm_i915_private *dev_priv = dev->dev_private; 7435 struct drm_i915_private *dev_priv = to_i915(dev);
7116 7436
7117 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); 7437 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7118 I915_WRITE(RENCLK_GATE_D2, 0); 7438 I915_WRITE(RENCLK_GATE_D2, 0);
@@ -7128,7 +7448,7 @@ static void crestline_init_clock_gating(struct drm_device *dev)
7128 7448
7129static void broadwater_init_clock_gating(struct drm_device *dev) 7449static void broadwater_init_clock_gating(struct drm_device *dev)
7130{ 7450{
7131 struct drm_i915_private *dev_priv = dev->dev_private; 7451 struct drm_i915_private *dev_priv = to_i915(dev);
7132 7452
7133 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | 7453 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7134 I965_RCC_CLOCK_GATE_DISABLE | 7454 I965_RCC_CLOCK_GATE_DISABLE |
@@ -7145,7 +7465,7 @@ static void broadwater_init_clock_gating(struct drm_device *dev)
7145 7465
7146static void gen3_init_clock_gating(struct drm_device *dev) 7466static void gen3_init_clock_gating(struct drm_device *dev)
7147{ 7467{
7148 struct drm_i915_private *dev_priv = dev->dev_private; 7468 struct drm_i915_private *dev_priv = to_i915(dev);
7149 u32 dstate = I915_READ(D_STATE); 7469 u32 dstate = I915_READ(D_STATE);
7150 7470
7151 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | 7471 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
@@ -7170,7 +7490,7 @@ static void gen3_init_clock_gating(struct drm_device *dev)
7170 7490
7171static void i85x_init_clock_gating(struct drm_device *dev) 7491static void i85x_init_clock_gating(struct drm_device *dev)
7172{ 7492{
7173 struct drm_i915_private *dev_priv = dev->dev_private; 7493 struct drm_i915_private *dev_priv = to_i915(dev);
7174 7494
7175 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); 7495 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7176 7496
@@ -7184,7 +7504,7 @@ static void i85x_init_clock_gating(struct drm_device *dev)
7184 7504
7185static void i830_init_clock_gating(struct drm_device *dev) 7505static void i830_init_clock_gating(struct drm_device *dev)
7186{ 7506{
7187 struct drm_i915_private *dev_priv = dev->dev_private; 7507 struct drm_i915_private *dev_priv = to_i915(dev);
7188 7508
7189 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); 7509 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
7190 7510
@@ -7195,7 +7515,7 @@ static void i830_init_clock_gating(struct drm_device *dev)
7195 7515
7196void intel_init_clock_gating(struct drm_device *dev) 7516void intel_init_clock_gating(struct drm_device *dev)
7197{ 7517{
7198 struct drm_i915_private *dev_priv = dev->dev_private; 7518 struct drm_i915_private *dev_priv = to_i915(dev);
7199 7519
7200 dev_priv->display.init_clock_gating(dev); 7520 dev_priv->display.init_clock_gating(dev);
7201} 7521}
@@ -7263,7 +7583,7 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7263/* Set up chip specific power management-related functions */ 7583/* Set up chip specific power management-related functions */
7264void intel_init_pm(struct drm_device *dev) 7584void intel_init_pm(struct drm_device *dev)
7265{ 7585{
7266 struct drm_i915_private *dev_priv = dev->dev_private; 7586 struct drm_i915_private *dev_priv = to_i915(dev);
7267 7587
7268 intel_fbc_init(dev_priv); 7588 intel_fbc_init(dev_priv);
7269 7589
@@ -7277,6 +7597,7 @@ void intel_init_pm(struct drm_device *dev)
7277 if (INTEL_INFO(dev)->gen >= 9) { 7597 if (INTEL_INFO(dev)->gen >= 9) {
7278 skl_setup_wm_latency(dev); 7598 skl_setup_wm_latency(dev);
7279 dev_priv->display.update_wm = skl_update_wm; 7599 dev_priv->display.update_wm = skl_update_wm;
7600 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7280 } else if (HAS_PCH_SPLIT(dev)) { 7601 } else if (HAS_PCH_SPLIT(dev)) {
7281 ilk_setup_wm_latency(dev); 7602 ilk_setup_wm_latency(dev);
7282 7603
@@ -7340,46 +7661,59 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
7340{ 7661{
7341 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 7662 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
7342 7663
7343 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { 7664 /* GEN6_PCODE_* are outside of the forcewake domain, we can
7665 * use te fw I915_READ variants to reduce the amount of work
7666 * required when reading/writing.
7667 */
7668
7669 if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
7344 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n"); 7670 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
7345 return -EAGAIN; 7671 return -EAGAIN;
7346 } 7672 }
7347 7673
7348 I915_WRITE(GEN6_PCODE_DATA, *val); 7674 I915_WRITE_FW(GEN6_PCODE_DATA, *val);
7349 I915_WRITE(GEN6_PCODE_DATA1, 0); 7675 I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
7350 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); 7676 I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
7351 7677
7352 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 7678 if (intel_wait_for_register_fw(dev_priv,
7353 500)) { 7679 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
7680 500)) {
7354 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox); 7681 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
7355 return -ETIMEDOUT; 7682 return -ETIMEDOUT;
7356 } 7683 }
7357 7684
7358 *val = I915_READ(GEN6_PCODE_DATA); 7685 *val = I915_READ_FW(GEN6_PCODE_DATA);
7359 I915_WRITE(GEN6_PCODE_DATA, 0); 7686 I915_WRITE_FW(GEN6_PCODE_DATA, 0);
7360 7687
7361 return 0; 7688 return 0;
7362} 7689}
7363 7690
7364int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val) 7691int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
7692 u32 mbox, u32 val)
7365{ 7693{
7366 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 7694 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
7367 7695
7368 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { 7696 /* GEN6_PCODE_* are outside of the forcewake domain, we can
7697 * use te fw I915_READ variants to reduce the amount of work
7698 * required when reading/writing.
7699 */
7700
7701 if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
7369 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n"); 7702 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
7370 return -EAGAIN; 7703 return -EAGAIN;
7371 } 7704 }
7372 7705
7373 I915_WRITE(GEN6_PCODE_DATA, val); 7706 I915_WRITE_FW(GEN6_PCODE_DATA, val);
7374 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); 7707 I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
7375 7708
7376 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 7709 if (intel_wait_for_register_fw(dev_priv,
7377 500)) { 7710 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
7711 500)) {
7378 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox); 7712 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
7379 return -ETIMEDOUT; 7713 return -ETIMEDOUT;
7380 } 7714 }
7381 7715
7382 I915_WRITE(GEN6_PCODE_DATA, 0); 7716 I915_WRITE_FW(GEN6_PCODE_DATA, 0);
7383 7717
7384 return 0; 7718 return 0;
7385} 7719}
@@ -7449,23 +7783,21 @@ static void __intel_rps_boost_work(struct work_struct *work)
7449 struct request_boost *boost = container_of(work, struct request_boost, work); 7783 struct request_boost *boost = container_of(work, struct request_boost, work);
7450 struct drm_i915_gem_request *req = boost->req; 7784 struct drm_i915_gem_request *req = boost->req;
7451 7785
7452 if (!i915_gem_request_completed(req, true)) 7786 if (!i915_gem_request_completed(req))
7453 gen6_rps_boost(to_i915(req->engine->dev), NULL, 7787 gen6_rps_boost(req->i915, NULL, req->emitted_jiffies);
7454 req->emitted_jiffies);
7455 7788
7456 i915_gem_request_unreference__unlocked(req); 7789 i915_gem_request_unreference(req);
7457 kfree(boost); 7790 kfree(boost);
7458} 7791}
7459 7792
7460void intel_queue_rps_boost_for_request(struct drm_device *dev, 7793void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req)
7461 struct drm_i915_gem_request *req)
7462{ 7794{
7463 struct request_boost *boost; 7795 struct request_boost *boost;
7464 7796
7465 if (req == NULL || INTEL_INFO(dev)->gen < 6) 7797 if (req == NULL || INTEL_GEN(req->i915) < 6)
7466 return; 7798 return;
7467 7799
7468 if (i915_gem_request_completed(req, true)) 7800 if (i915_gem_request_completed(req))
7469 return; 7801 return;
7470 7802
7471 boost = kmalloc(sizeof(*boost), GFP_ATOMIC); 7803 boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
@@ -7476,12 +7808,12 @@ void intel_queue_rps_boost_for_request(struct drm_device *dev,
7476 boost->req = req; 7808 boost->req = req;
7477 7809
7478 INIT_WORK(&boost->work, __intel_rps_boost_work); 7810 INIT_WORK(&boost->work, __intel_rps_boost_work);
7479 queue_work(to_i915(dev)->wq, &boost->work); 7811 queue_work(req->i915->wq, &boost->work);
7480} 7812}
7481 7813
7482void intel_pm_setup(struct drm_device *dev) 7814void intel_pm_setup(struct drm_device *dev)
7483{ 7815{
7484 struct drm_i915_private *dev_priv = dev->dev_private; 7816 struct drm_i915_private *dev_priv = to_i915(dev);
7485 7817
7486 mutex_init(&dev_priv->rps.hw_lock); 7818 mutex_init(&dev_priv->rps.hw_lock);
7487 spin_lock_init(&dev_priv->rps.client_lock); 7819 spin_lock_init(&dev_priv->rps.client_lock);
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-22 03:38:42 -0500