aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/radeon/si.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/radeon/si.c')
-rw-r--r--drivers/gpu/drm/radeon/si.c1337
1 files changed, 1296 insertions, 41 deletions
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index a1b0da6b5808..234906709067 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -32,40 +32,43 @@
32#include "sid.h" 32#include "sid.h"
33#include "atom.h" 33#include "atom.h"
34#include "si_blit_shaders.h" 34#include "si_blit_shaders.h"
35#include "clearstate_si.h"
36#include "radeon_ucode.h"
35 37
36#define SI_PFP_UCODE_SIZE 2144
37#define SI_PM4_UCODE_SIZE 2144
38#define SI_CE_UCODE_SIZE 2144
39#define SI_RLC_UCODE_SIZE 2048
40#define SI_MC_UCODE_SIZE 7769
41#define OLAND_MC_UCODE_SIZE 7863
42 38
43MODULE_FIRMWARE("radeon/TAHITI_pfp.bin"); 39MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44MODULE_FIRMWARE("radeon/TAHITI_me.bin"); 40MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45MODULE_FIRMWARE("radeon/TAHITI_ce.bin"); 41MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46MODULE_FIRMWARE("radeon/TAHITI_mc.bin"); 42MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47MODULE_FIRMWARE("radeon/TAHITI_rlc.bin"); 43MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
44MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); 45MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); 46MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin"); 47MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin"); 48MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin"); 49MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
53MODULE_FIRMWARE("radeon/VERDE_pfp.bin"); 51MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54MODULE_FIRMWARE("radeon/VERDE_me.bin"); 52MODULE_FIRMWARE("radeon/VERDE_me.bin");
55MODULE_FIRMWARE("radeon/VERDE_ce.bin"); 53MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56MODULE_FIRMWARE("radeon/VERDE_mc.bin"); 54MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57MODULE_FIRMWARE("radeon/VERDE_rlc.bin"); 55MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
56MODULE_FIRMWARE("radeon/VERDE_smc.bin");
58MODULE_FIRMWARE("radeon/OLAND_pfp.bin"); 57MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59MODULE_FIRMWARE("radeon/OLAND_me.bin"); 58MODULE_FIRMWARE("radeon/OLAND_me.bin");
60MODULE_FIRMWARE("radeon/OLAND_ce.bin"); 59MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61MODULE_FIRMWARE("radeon/OLAND_mc.bin"); 60MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62MODULE_FIRMWARE("radeon/OLAND_rlc.bin"); 61MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
62MODULE_FIRMWARE("radeon/OLAND_smc.bin");
63MODULE_FIRMWARE("radeon/HAINAN_pfp.bin"); 63MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64MODULE_FIRMWARE("radeon/HAINAN_me.bin"); 64MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65MODULE_FIRMWARE("radeon/HAINAN_ce.bin"); 65MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66MODULE_FIRMWARE("radeon/HAINAN_mc.bin"); 66MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67MODULE_FIRMWARE("radeon/HAINAN_rlc.bin"); 67MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
68MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
68 69
70static void si_pcie_gen3_enable(struct radeon_device *rdev);
71static void si_program_aspm(struct radeon_device *rdev);
69extern int r600_ih_ring_alloc(struct radeon_device *rdev); 72extern int r600_ih_ring_alloc(struct radeon_device *rdev);
70extern void r600_ih_ring_fini(struct radeon_device *rdev); 73extern void r600_ih_ring_fini(struct radeon_device *rdev);
71extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); 74extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
@@ -75,6 +78,228 @@ extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
75extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); 78extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
76extern bool evergreen_is_display_hung(struct radeon_device *rdev); 79extern bool evergreen_is_display_hung(struct radeon_device *rdev);
77 80
81static const u32 verde_rlc_save_restore_register_list[] =
82{
83 (0x8000 << 16) | (0x98f4 >> 2),
84 0x00000000,
85 (0x8040 << 16) | (0x98f4 >> 2),
86 0x00000000,
87 (0x8000 << 16) | (0xe80 >> 2),
88 0x00000000,
89 (0x8040 << 16) | (0xe80 >> 2),
90 0x00000000,
91 (0x8000 << 16) | (0x89bc >> 2),
92 0x00000000,
93 (0x8040 << 16) | (0x89bc >> 2),
94 0x00000000,
95 (0x8000 << 16) | (0x8c1c >> 2),
96 0x00000000,
97 (0x8040 << 16) | (0x8c1c >> 2),
98 0x00000000,
99 (0x9c00 << 16) | (0x98f0 >> 2),
100 0x00000000,
101 (0x9c00 << 16) | (0xe7c >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0x9148 >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0x9148 >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0x9150 >> 2),
108 0x00000000,
109 (0x9c00 << 16) | (0x897c >> 2),
110 0x00000000,
111 (0x9c00 << 16) | (0x8d8c >> 2),
112 0x00000000,
113 (0x9c00 << 16) | (0xac54 >> 2),
114 0X00000000,
115 0x3,
116 (0x9c00 << 16) | (0x98f8 >> 2),
117 0x00000000,
118 (0x9c00 << 16) | (0x9910 >> 2),
119 0x00000000,
120 (0x9c00 << 16) | (0x9914 >> 2),
121 0x00000000,
122 (0x9c00 << 16) | (0x9918 >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0x991c >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9920 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9924 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x9928 >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x992c >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9930 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9934 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9938 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x993c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9940 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9944 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9948 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x994c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9950 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9954 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9958 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x995c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9960 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9964 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9968 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x996c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9970 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9974 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9978 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x997c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9980 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9984 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9988 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x998c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x8c00 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x8c14 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x8c04 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x8c08 >> 2),
189 0x00000000,
190 (0x8000 << 16) | (0x9b7c >> 2),
191 0x00000000,
192 (0x8040 << 16) | (0x9b7c >> 2),
193 0x00000000,
194 (0x8000 << 16) | (0xe84 >> 2),
195 0x00000000,
196 (0x8040 << 16) | (0xe84 >> 2),
197 0x00000000,
198 (0x8000 << 16) | (0x89c0 >> 2),
199 0x00000000,
200 (0x8040 << 16) | (0x89c0 >> 2),
201 0x00000000,
202 (0x8000 << 16) | (0x914c >> 2),
203 0x00000000,
204 (0x8040 << 16) | (0x914c >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x8c20 >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x8c20 >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0x9354 >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0x9354 >> 2),
213 0x00000000,
214 (0x9c00 << 16) | (0x9060 >> 2),
215 0x00000000,
216 (0x9c00 << 16) | (0x9364 >> 2),
217 0x00000000,
218 (0x9c00 << 16) | (0x9100 >> 2),
219 0x00000000,
220 (0x9c00 << 16) | (0x913c >> 2),
221 0x00000000,
222 (0x8000 << 16) | (0x90e0 >> 2),
223 0x00000000,
224 (0x8000 << 16) | (0x90e4 >> 2),
225 0x00000000,
226 (0x8000 << 16) | (0x90e8 >> 2),
227 0x00000000,
228 (0x8040 << 16) | (0x90e0 >> 2),
229 0x00000000,
230 (0x8040 << 16) | (0x90e4 >> 2),
231 0x00000000,
232 (0x8040 << 16) | (0x90e8 >> 2),
233 0x00000000,
234 (0x9c00 << 16) | (0x8bcc >> 2),
235 0x00000000,
236 (0x9c00 << 16) | (0x8b24 >> 2),
237 0x00000000,
238 (0x9c00 << 16) | (0x88c4 >> 2),
239 0x00000000,
240 (0x9c00 << 16) | (0x8e50 >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8c0c >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x8e58 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x8e5c >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x9508 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x950c >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x9494 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0xac0c >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0xac10 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0xac14 >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0xae00 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac08 >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0x88d4 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0x88c8 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0x88cc >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0x89b0 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x8b10 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x8a14 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x9830 >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x9834 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x9838 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x9a10 >> 2),
283 0x00000000,
284 (0x8000 << 16) | (0x9870 >> 2),
285 0x00000000,
286 (0x8000 << 16) | (0x9874 >> 2),
287 0x00000000,
288 (0x8001 << 16) | (0x9870 >> 2),
289 0x00000000,
290 (0x8001 << 16) | (0x9874 >> 2),
291 0x00000000,
292 (0x8040 << 16) | (0x9870 >> 2),
293 0x00000000,
294 (0x8040 << 16) | (0x9874 >> 2),
295 0x00000000,
296 (0x8041 << 16) | (0x9870 >> 2),
297 0x00000000,
298 (0x8041 << 16) | (0x9874 >> 2),
299 0x00000000,
300 0x00000000
301};
302
78static const u32 tahiti_golden_rlc_registers[] = 303static const u32 tahiti_golden_rlc_registers[] =
79{ 304{
80 0xc424, 0xffffffff, 0x00601005, 305 0xc424, 0xffffffff, 0x00601005,
@@ -1320,6 +1545,7 @@ static int si_init_microcode(struct radeon_device *rdev)
1320 const char *chip_name; 1545 const char *chip_name;
1321 const char *rlc_chip_name; 1546 const char *rlc_chip_name;
1322 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; 1547 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1548 size_t smc_req_size;
1323 char fw_name[30]; 1549 char fw_name[30];
1324 int err; 1550 int err;
1325 1551
@@ -1341,6 +1567,7 @@ static int si_init_microcode(struct radeon_device *rdev)
1341 ce_req_size = SI_CE_UCODE_SIZE * 4; 1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1342 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1343 mc_req_size = SI_MC_UCODE_SIZE * 4; 1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1344 break; 1571 break;
1345 case CHIP_PITCAIRN: 1572 case CHIP_PITCAIRN:
1346 chip_name = "PITCAIRN"; 1573 chip_name = "PITCAIRN";
@@ -1350,6 +1577,7 @@ static int si_init_microcode(struct radeon_device *rdev)
1350 ce_req_size = SI_CE_UCODE_SIZE * 4; 1577 ce_req_size = SI_CE_UCODE_SIZE * 4;
1351 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1352 mc_req_size = SI_MC_UCODE_SIZE * 4; 1579 mc_req_size = SI_MC_UCODE_SIZE * 4;
1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1353 break; 1581 break;
1354 case CHIP_VERDE: 1582 case CHIP_VERDE:
1355 chip_name = "VERDE"; 1583 chip_name = "VERDE";
@@ -1359,6 +1587,7 @@ static int si_init_microcode(struct radeon_device *rdev)
1359 ce_req_size = SI_CE_UCODE_SIZE * 4; 1587 ce_req_size = SI_CE_UCODE_SIZE * 4;
1360 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1361 mc_req_size = SI_MC_UCODE_SIZE * 4; 1589 mc_req_size = SI_MC_UCODE_SIZE * 4;
1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1362 break; 1591 break;
1363 case CHIP_OLAND: 1592 case CHIP_OLAND:
1364 chip_name = "OLAND"; 1593 chip_name = "OLAND";
@@ -1368,6 +1597,7 @@ static int si_init_microcode(struct radeon_device *rdev)
1368 ce_req_size = SI_CE_UCODE_SIZE * 4; 1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1369 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1370 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1371 break; 1601 break;
1372 case CHIP_HAINAN: 1602 case CHIP_HAINAN:
1373 chip_name = "HAINAN"; 1603 chip_name = "HAINAN";
@@ -1377,6 +1607,7 @@ static int si_init_microcode(struct radeon_device *rdev)
1377 ce_req_size = SI_CE_UCODE_SIZE * 4; 1607 ce_req_size = SI_CE_UCODE_SIZE * 4;
1378 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1379 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1380 break; 1611 break;
1381 default: BUG(); 1612 default: BUG();
1382 } 1613 }
@@ -1439,6 +1670,17 @@ static int si_init_microcode(struct radeon_device *rdev)
1439 err = -EINVAL; 1670 err = -EINVAL;
1440 } 1671 }
1441 1672
1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1674 err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
1675 if (err)
1676 goto out;
1677 if (rdev->smc_fw->size != smc_req_size) {
1678 printk(KERN_ERR
1679 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1680 rdev->smc_fw->size, fw_name);
1681 err = -EINVAL;
1682 }
1683
1442out: 1684out:
1443 platform_device_unregister(pdev); 1685 platform_device_unregister(pdev);
1444 1686
@@ -1457,6 +1699,8 @@ out:
1457 rdev->rlc_fw = NULL; 1699 rdev->rlc_fw = NULL;
1458 release_firmware(rdev->mc_fw); 1700 release_firmware(rdev->mc_fw);
1459 rdev->mc_fw = NULL; 1701 rdev->mc_fw = NULL;
1702 release_firmware(rdev->smc_fw);
1703 rdev->smc_fw = NULL;
1460 } 1704 }
1461 return err; 1705 return err;
1462} 1706}
@@ -1792,7 +2036,8 @@ static void dce6_program_watermarks(struct radeon_device *rdev,
1792 u32 lb_size, u32 num_heads) 2036 u32 lb_size, u32 num_heads)
1793{ 2037{
1794 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2038 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1795 struct dce6_wm_params wm; 2039 struct dce6_wm_params wm_low, wm_high;
2040 u32 dram_channels;
1796 u32 pixel_period; 2041 u32 pixel_period;
1797 u32 line_time = 0; 2042 u32 line_time = 0;
1798 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2043 u32 latency_watermark_a = 0, latency_watermark_b = 0;
@@ -1808,38 +2053,83 @@ static void dce6_program_watermarks(struct radeon_device *rdev,
1808 priority_a_cnt = 0; 2053 priority_a_cnt = 0;
1809 priority_b_cnt = 0; 2054 priority_b_cnt = 0;
1810 2055
1811 wm.yclk = rdev->pm.current_mclk * 10;
1812 wm.sclk = rdev->pm.current_sclk * 10;
1813 wm.disp_clk = mode->clock;
1814 wm.src_width = mode->crtc_hdisplay;
1815 wm.active_time = mode->crtc_hdisplay * pixel_period;
1816 wm.blank_time = line_time - wm.active_time;
1817 wm.interlaced = false;
1818 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1819 wm.interlaced = true;
1820 wm.vsc = radeon_crtc->vsc;
1821 wm.vtaps = 1;
1822 if (radeon_crtc->rmx_type != RMX_OFF)
1823 wm.vtaps = 2;
1824 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1825 wm.lb_size = lb_size;
1826 if (rdev->family == CHIP_ARUBA) 2056 if (rdev->family == CHIP_ARUBA)
1827 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); 2057 dram_channels = evergreen_get_number_of_dram_channels(rdev);
1828 else 2058 else
1829 wm.dram_channels = si_get_number_of_dram_channels(rdev); 2059 dram_channels = si_get_number_of_dram_channels(rdev);
1830 wm.num_heads = num_heads; 2060
2061 /* watermark for high clocks */
2062 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2063 wm_high.yclk =
2064 radeon_dpm_get_mclk(rdev, false) * 10;
2065 wm_high.sclk =
2066 radeon_dpm_get_sclk(rdev, false) * 10;
2067 } else {
2068 wm_high.yclk = rdev->pm.current_mclk * 10;
2069 wm_high.sclk = rdev->pm.current_sclk * 10;
2070 }
2071
2072 wm_high.disp_clk = mode->clock;
2073 wm_high.src_width = mode->crtc_hdisplay;
2074 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2075 wm_high.blank_time = line_time - wm_high.active_time;
2076 wm_high.interlaced = false;
2077 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2078 wm_high.interlaced = true;
2079 wm_high.vsc = radeon_crtc->vsc;
2080 wm_high.vtaps = 1;
2081 if (radeon_crtc->rmx_type != RMX_OFF)
2082 wm_high.vtaps = 2;
2083 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2084 wm_high.lb_size = lb_size;
2085 wm_high.dram_channels = dram_channels;
2086 wm_high.num_heads = num_heads;
2087
2088 /* watermark for low clocks */
2089 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2090 wm_low.yclk =
2091 radeon_dpm_get_mclk(rdev, true) * 10;
2092 wm_low.sclk =
2093 radeon_dpm_get_sclk(rdev, true) * 10;
2094 } else {
2095 wm_low.yclk = rdev->pm.current_mclk * 10;
2096 wm_low.sclk = rdev->pm.current_sclk * 10;
2097 }
2098
2099 wm_low.disp_clk = mode->clock;
2100 wm_low.src_width = mode->crtc_hdisplay;
2101 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2102 wm_low.blank_time = line_time - wm_low.active_time;
2103 wm_low.interlaced = false;
2104 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2105 wm_low.interlaced = true;
2106 wm_low.vsc = radeon_crtc->vsc;
2107 wm_low.vtaps = 1;
2108 if (radeon_crtc->rmx_type != RMX_OFF)
2109 wm_low.vtaps = 2;
2110 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2111 wm_low.lb_size = lb_size;
2112 wm_low.dram_channels = dram_channels;
2113 wm_low.num_heads = num_heads;
1831 2114
1832 /* set for high clocks */ 2115 /* set for high clocks */
1833 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535); 2116 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
1834 /* set for low clocks */ 2117 /* set for low clocks */
1835 /* wm.yclk = low clk; wm.sclk = low clk */ 2118 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
1836 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
1837 2119
1838 /* possibly force display priority to high */ 2120 /* possibly force display priority to high */
1839 /* should really do this at mode validation time... */ 2121 /* should really do this at mode validation time... */
1840 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || 2122 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
1841 !dce6_average_bandwidth_vs_available_bandwidth(&wm) || 2123 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
1842 !dce6_check_latency_hiding(&wm) || 2124 !dce6_check_latency_hiding(&wm_high) ||
2125 (rdev->disp_priority == 2)) {
2126 DRM_DEBUG_KMS("force priority to high\n");
2127 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2128 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2129 }
2130 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2131 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2132 !dce6_check_latency_hiding(&wm_low) ||
1843 (rdev->disp_priority == 2)) { 2133 (rdev->disp_priority == 2)) {
1844 DRM_DEBUG_KMS("force priority to high\n"); 2134 DRM_DEBUG_KMS("force priority to high\n");
1845 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2135 priority_a_cnt |= PRIORITY_ALWAYS_ON;
@@ -1895,6 +2185,10 @@ static void dce6_program_watermarks(struct radeon_device *rdev,
1895 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2185 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1896 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2186 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1897 2187
2188 /* save values for DPM */
2189 radeon_crtc->line_time = line_time;
2190 radeon_crtc->wm_high = latency_watermark_a;
2191 radeon_crtc->wm_low = latency_watermark_b;
1898} 2192}
1899 2193
1900void dce6_bandwidth_update(struct radeon_device *rdev) 2194void dce6_bandwidth_update(struct radeon_device *rdev)
@@ -3535,8 +3829,8 @@ static void si_mc_program(struct radeon_device *rdev)
3535 } 3829 }
3536} 3830}
3537 3831
3538static void si_vram_gtt_location(struct radeon_device *rdev, 3832void si_vram_gtt_location(struct radeon_device *rdev,
3539 struct radeon_mc *mc) 3833 struct radeon_mc *mc)
3540{ 3834{
3541 if (mc->mc_vram_size > 0xFFC0000000ULL) { 3835 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3542 /* leave room for at least 1024M GTT */ 3836 /* leave room for at least 1024M GTT */
@@ -4282,6 +4576,450 @@ void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4282} 4576}
4283 4577
4284/* 4578/*
4579 * Power and clock gating
4580 */
4581static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4582{
4583 int i;
4584
4585 for (i = 0; i < rdev->usec_timeout; i++) {
4586 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4587 break;
4588 udelay(1);
4589 }
4590
4591 for (i = 0; i < rdev->usec_timeout; i++) {
4592 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4593 break;
4594 udelay(1);
4595 }
4596}
4597
4598static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4599 bool enable)
4600{
4601 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4602 u32 mask;
4603 int i;
4604
4605 if (enable)
4606 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4607 else
4608 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4609 WREG32(CP_INT_CNTL_RING0, tmp);
4610
4611 if (!enable) {
4612 /* read a gfx register */
4613 tmp = RREG32(DB_DEPTH_INFO);
4614
4615 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4616 for (i = 0; i < rdev->usec_timeout; i++) {
4617 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4618 break;
4619 udelay(1);
4620 }
4621 }
4622}
4623
4624static void si_set_uvd_dcm(struct radeon_device *rdev,
4625 bool sw_mode)
4626{
4627 u32 tmp, tmp2;
4628
4629 tmp = RREG32(UVD_CGC_CTRL);
4630 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4631 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4632
4633 if (sw_mode) {
4634 tmp &= ~0x7ffff800;
4635 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4636 } else {
4637 tmp |= 0x7ffff800;
4638 tmp2 = 0;
4639 }
4640
4641 WREG32(UVD_CGC_CTRL, tmp);
4642 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4643}
4644
4645static void si_init_uvd_internal_cg(struct radeon_device *rdev)
4646{
4647 bool hw_mode = true;
4648
4649 if (hw_mode) {
4650 si_set_uvd_dcm(rdev, false);
4651 } else {
4652 u32 tmp = RREG32(UVD_CGC_CTRL);
4653 tmp &= ~DCM;
4654 WREG32(UVD_CGC_CTRL, tmp);
4655 }
4656}
4657
4658static u32 si_halt_rlc(struct radeon_device *rdev)
4659{
4660 u32 data, orig;
4661
4662 orig = data = RREG32(RLC_CNTL);
4663
4664 if (data & RLC_ENABLE) {
4665 data &= ~RLC_ENABLE;
4666 WREG32(RLC_CNTL, data);
4667
4668 si_wait_for_rlc_serdes(rdev);
4669 }
4670
4671 return orig;
4672}
4673
4674static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4675{
4676 u32 tmp;
4677
4678 tmp = RREG32(RLC_CNTL);
4679 if (tmp != rlc)
4680 WREG32(RLC_CNTL, rlc);
4681}
4682
4683static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4684{
4685 u32 data, orig;
4686
4687 orig = data = RREG32(DMA_PG);
4688 if (enable)
4689 data |= PG_CNTL_ENABLE;
4690 else
4691 data &= ~PG_CNTL_ENABLE;
4692 if (orig != data)
4693 WREG32(DMA_PG, data);
4694}
4695
4696static void si_init_dma_pg(struct radeon_device *rdev)
4697{
4698 u32 tmp;
4699
4700 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4701 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4702
4703 for (tmp = 0; tmp < 5; tmp++)
4704 WREG32(DMA_PGFSM_WRITE, 0);
4705}
4706
4707static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4708 bool enable)
4709{
4710 u32 tmp;
4711
4712 if (enable) {
4713 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4714 WREG32(RLC_TTOP_D, tmp);
4715
4716 tmp = RREG32(RLC_PG_CNTL);
4717 tmp |= GFX_PG_ENABLE;
4718 WREG32(RLC_PG_CNTL, tmp);
4719
4720 tmp = RREG32(RLC_AUTO_PG_CTRL);
4721 tmp |= AUTO_PG_EN;
4722 WREG32(RLC_AUTO_PG_CTRL, tmp);
4723 } else {
4724 tmp = RREG32(RLC_AUTO_PG_CTRL);
4725 tmp &= ~AUTO_PG_EN;
4726 WREG32(RLC_AUTO_PG_CTRL, tmp);
4727
4728 tmp = RREG32(DB_RENDER_CONTROL);
4729 }
4730}
4731
4732static void si_init_gfx_cgpg(struct radeon_device *rdev)
4733{
4734 u32 tmp;
4735
4736 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4737
4738 tmp = RREG32(RLC_PG_CNTL);
4739 tmp |= GFX_PG_SRC;
4740 WREG32(RLC_PG_CNTL, tmp);
4741
4742 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4743
4744 tmp = RREG32(RLC_AUTO_PG_CTRL);
4745
4746 tmp &= ~GRBM_REG_SGIT_MASK;
4747 tmp |= GRBM_REG_SGIT(0x700);
4748 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4749 WREG32(RLC_AUTO_PG_CTRL, tmp);
4750}
4751
4752static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4753{
4754 u32 mask = 0, tmp, tmp1;
4755 int i;
4756
4757 si_select_se_sh(rdev, se, sh);
4758 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4759 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4760 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4761
4762 tmp &= 0xffff0000;
4763
4764 tmp |= tmp1;
4765 tmp >>= 16;
4766
4767 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4768 mask <<= 1;
4769 mask |= 1;
4770 }
4771
4772 return (~tmp) & mask;
4773}
4774
4775static void si_init_ao_cu_mask(struct radeon_device *rdev)
4776{
4777 u32 i, j, k, active_cu_number = 0;
4778 u32 mask, counter, cu_bitmap;
4779 u32 tmp = 0;
4780
4781 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4782 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4783 mask = 1;
4784 cu_bitmap = 0;
4785 counter = 0;
4786 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4787 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4788 if (counter < 2)
4789 cu_bitmap |= mask;
4790 counter++;
4791 }
4792 mask <<= 1;
4793 }
4794
4795 active_cu_number += counter;
4796 tmp |= (cu_bitmap << (i * 16 + j * 8));
4797 }
4798 }
4799
4800 WREG32(RLC_PG_AO_CU_MASK, tmp);
4801
4802 tmp = RREG32(RLC_MAX_PG_CU);
4803 tmp &= ~MAX_PU_CU_MASK;
4804 tmp |= MAX_PU_CU(active_cu_number);
4805 WREG32(RLC_MAX_PG_CU, tmp);
4806}
4807
4808static void si_enable_cgcg(struct radeon_device *rdev,
4809 bool enable)
4810{
4811 u32 data, orig, tmp;
4812
4813 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4814
4815 si_enable_gui_idle_interrupt(rdev, enable);
4816
4817 if (enable) {
4818 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4819
4820 tmp = si_halt_rlc(rdev);
4821
4822 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4823 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4824 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4825
4826 si_wait_for_rlc_serdes(rdev);
4827
4828 si_update_rlc(rdev, tmp);
4829
4830 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4831
4832 data |= CGCG_EN | CGLS_EN;
4833 } else {
4834 RREG32(CB_CGTT_SCLK_CTRL);
4835 RREG32(CB_CGTT_SCLK_CTRL);
4836 RREG32(CB_CGTT_SCLK_CTRL);
4837 RREG32(CB_CGTT_SCLK_CTRL);
4838
4839 data &= ~(CGCG_EN | CGLS_EN);
4840 }
4841
4842 if (orig != data)
4843 WREG32(RLC_CGCG_CGLS_CTRL, data);
4844}
4845
4846static void si_enable_mgcg(struct radeon_device *rdev,
4847 bool enable)
4848{
4849 u32 data, orig, tmp = 0;
4850
4851 if (enable) {
4852 orig = data = RREG32(CGTS_SM_CTRL_REG);
4853 data = 0x96940200;
4854 if (orig != data)
4855 WREG32(CGTS_SM_CTRL_REG, data);
4856
4857 orig = data = RREG32(CP_MEM_SLP_CNTL);
4858 data |= CP_MEM_LS_EN;
4859 if (orig != data)
4860 WREG32(CP_MEM_SLP_CNTL, data);
4861
4862 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4863 data &= 0xffffffc0;
4864 if (orig != data)
4865 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4866
4867 tmp = si_halt_rlc(rdev);
4868
4869 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4870 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4871 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
4872
4873 si_update_rlc(rdev, tmp);
4874 } else {
4875 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4876 data |= 0x00000003;
4877 if (orig != data)
4878 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4879
4880 data = RREG32(CP_MEM_SLP_CNTL);
4881 if (data & CP_MEM_LS_EN) {
4882 data &= ~CP_MEM_LS_EN;
4883 WREG32(CP_MEM_SLP_CNTL, data);
4884 }
4885 orig = data = RREG32(CGTS_SM_CTRL_REG);
4886 data |= LS_OVERRIDE | OVERRIDE;
4887 if (orig != data)
4888 WREG32(CGTS_SM_CTRL_REG, data);
4889
4890 tmp = si_halt_rlc(rdev);
4891
4892 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4893 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4894 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
4895
4896 si_update_rlc(rdev, tmp);
4897 }
4898}
4899
4900static void si_enable_uvd_mgcg(struct radeon_device *rdev,
4901 bool enable)
4902{
4903 u32 orig, data, tmp;
4904
4905 if (enable) {
4906 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4907 tmp |= 0x3fff;
4908 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4909
4910 orig = data = RREG32(UVD_CGC_CTRL);
4911 data |= DCM;
4912 if (orig != data)
4913 WREG32(UVD_CGC_CTRL, data);
4914
4915 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
4916 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
4917 } else {
4918 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4919 tmp &= ~0x3fff;
4920 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4921
4922 orig = data = RREG32(UVD_CGC_CTRL);
4923 data &= ~DCM;
4924 if (orig != data)
4925 WREG32(UVD_CGC_CTRL, data);
4926
4927 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
4928 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
4929 }
4930}
4931
4932static const u32 mc_cg_registers[] =
4933{
4934 MC_HUB_MISC_HUB_CG,
4935 MC_HUB_MISC_SIP_CG,
4936 MC_HUB_MISC_VM_CG,
4937 MC_XPB_CLK_GAT,
4938 ATC_MISC_CG,
4939 MC_CITF_MISC_WR_CG,
4940 MC_CITF_MISC_RD_CG,
4941 MC_CITF_MISC_VM_CG,
4942 VM_L2_CG,
4943};
4944
4945static void si_enable_mc_ls(struct radeon_device *rdev,
4946 bool enable)
4947{
4948 int i;
4949 u32 orig, data;
4950
4951 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
4952 orig = data = RREG32(mc_cg_registers[i]);
4953 if (enable)
4954 data |= MC_LS_ENABLE;
4955 else
4956 data &= ~MC_LS_ENABLE;
4957 if (data != orig)
4958 WREG32(mc_cg_registers[i], data);
4959 }
4960}
4961
4962
4963static void si_init_cg(struct radeon_device *rdev)
4964{
4965 bool has_uvd = true;
4966
4967 si_enable_mgcg(rdev, true);
4968 si_enable_cgcg(rdev, true);
4969 /* disable MC LS on Tahiti */
4970 if (rdev->family == CHIP_TAHITI)
4971 si_enable_mc_ls(rdev, false);
4972 if (has_uvd) {
4973 si_enable_uvd_mgcg(rdev, true);
4974 si_init_uvd_internal_cg(rdev);
4975 }
4976}
4977
4978static void si_fini_cg(struct radeon_device *rdev)
4979{
4980 bool has_uvd = true;
4981
4982 if (has_uvd)
4983 si_enable_uvd_mgcg(rdev, false);
4984 si_enable_cgcg(rdev, false);
4985 si_enable_mgcg(rdev, false);
4986}
4987
4988static void si_init_pg(struct radeon_device *rdev)
4989{
4990 bool has_pg = false;
4991
4992 /* only cape verde supports PG */
4993 if (rdev->family == CHIP_VERDE)
4994 has_pg = true;
4995
4996 if (has_pg) {
4997 si_init_ao_cu_mask(rdev);
4998 si_init_dma_pg(rdev);
4999 si_enable_dma_pg(rdev, true);
5000 si_init_gfx_cgpg(rdev);
5001 si_enable_gfx_cgpg(rdev, true);
5002 } else {
5003 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5004 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5005 }
5006}
5007
5008static void si_fini_pg(struct radeon_device *rdev)
5009{
5010 bool has_pg = false;
5011
5012 /* only cape verde supports PG */
5013 if (rdev->family == CHIP_VERDE)
5014 has_pg = true;
5015
5016 if (has_pg) {
5017 si_enable_dma_pg(rdev, false);
5018 si_enable_gfx_cgpg(rdev, false);
5019 }
5020}
5021
5022/*
4285 * RLC 5023 * RLC
4286 */ 5024 */
4287void si_rlc_fini(struct radeon_device *rdev) 5025void si_rlc_fini(struct radeon_device *rdev)
@@ -4313,8 +5051,15 @@ void si_rlc_fini(struct radeon_device *rdev)
4313 } 5051 }
4314} 5052}
4315 5053
5054#define RLC_CLEAR_STATE_END_MARKER 0x00000001
5055
4316int si_rlc_init(struct radeon_device *rdev) 5056int si_rlc_init(struct radeon_device *rdev)
4317{ 5057{
5058 volatile u32 *dst_ptr;
5059 u32 dws, data, i, j, k, reg_num;
5060 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
5061 u64 reg_list_mc_addr;
5062 const struct cs_section_def *cs_data = si_cs_data;
4318 int r; 5063 int r;
4319 5064
4320 /* save restore block */ 5065 /* save restore block */
@@ -4335,18 +5080,44 @@ int si_rlc_init(struct radeon_device *rdev)
4335 } 5080 }
4336 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 5081 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4337 &rdev->rlc.save_restore_gpu_addr); 5082 &rdev->rlc.save_restore_gpu_addr);
4338 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4339 if (r) { 5083 if (r) {
5084 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4340 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 5085 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4341 si_rlc_fini(rdev); 5086 si_rlc_fini(rdev);
4342 return r; 5087 return r;
4343 } 5088 }
4344 5089
5090 if (rdev->family == CHIP_VERDE) {
5091 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
5092 if (r) {
5093 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
5094 si_rlc_fini(rdev);
5095 return r;
5096 }
5097 /* write the sr buffer */
5098 dst_ptr = rdev->rlc.sr_ptr;
5099 for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) {
5100 dst_ptr[i] = verde_rlc_save_restore_register_list[i];
5101 }
5102 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
5103 }
5104 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5105
4345 /* clear state block */ 5106 /* clear state block */
5107 reg_list_num = 0;
5108 dws = 0;
5109 for (i = 0; cs_data[i].section != NULL; i++) {
5110 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5111 reg_list_num++;
5112 dws += cs_data[i].section[j].reg_count;
5113 }
5114 }
5115 reg_list_blk_index = (3 * reg_list_num + 2);
5116 dws += reg_list_blk_index;
5117
4346 if (rdev->rlc.clear_state_obj == NULL) { 5118 if (rdev->rlc.clear_state_obj == NULL) {
4347 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, 5119 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4348 RADEON_GEM_DOMAIN_VRAM, NULL, 5120 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4349 &rdev->rlc.clear_state_obj);
4350 if (r) { 5121 if (r) {
4351 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 5122 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4352 si_rlc_fini(rdev); 5123 si_rlc_fini(rdev);
@@ -4360,24 +5131,113 @@ int si_rlc_init(struct radeon_device *rdev)
4360 } 5131 }
4361 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 5132 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4362 &rdev->rlc.clear_state_gpu_addr); 5133 &rdev->rlc.clear_state_gpu_addr);
4363 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4364 if (r) { 5134 if (r) {
5135
5136 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4365 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 5137 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4366 si_rlc_fini(rdev); 5138 si_rlc_fini(rdev);
4367 return r; 5139 return r;
4368 } 5140 }
5141 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
5142 if (r) {
5143 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
5144 si_rlc_fini(rdev);
5145 return r;
5146 }
5147 /* set up the cs buffer */
5148 dst_ptr = rdev->rlc.cs_ptr;
5149 reg_list_hdr_blk_index = 0;
5150 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
5151 data = upper_32_bits(reg_list_mc_addr);
5152 dst_ptr[reg_list_hdr_blk_index] = data;
5153 reg_list_hdr_blk_index++;
5154 for (i = 0; cs_data[i].section != NULL; i++) {
5155 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5156 reg_num = cs_data[i].section[j].reg_count;
5157 data = reg_list_mc_addr & 0xffffffff;
5158 dst_ptr[reg_list_hdr_blk_index] = data;
5159 reg_list_hdr_blk_index++;
5160
5161 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
5162 dst_ptr[reg_list_hdr_blk_index] = data;
5163 reg_list_hdr_blk_index++;
5164
5165 data = 0x08000000 | (reg_num * 4);
5166 dst_ptr[reg_list_hdr_blk_index] = data;
5167 reg_list_hdr_blk_index++;
5168
5169 for (k = 0; k < reg_num; k++) {
5170 data = cs_data[i].section[j].extent[k];
5171 dst_ptr[reg_list_blk_index + k] = data;
5172 }
5173 reg_list_mc_addr += reg_num * 4;
5174 reg_list_blk_index += reg_num;
5175 }
5176 }
5177 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
5178
5179 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
5180 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4369 5181
4370 return 0; 5182 return 0;
4371} 5183}
4372 5184
5185static void si_rlc_reset(struct radeon_device *rdev)
5186{
5187 u32 tmp = RREG32(GRBM_SOFT_RESET);
5188
5189 tmp |= SOFT_RESET_RLC;
5190 WREG32(GRBM_SOFT_RESET, tmp);
5191 udelay(50);
5192 tmp &= ~SOFT_RESET_RLC;
5193 WREG32(GRBM_SOFT_RESET, tmp);
5194 udelay(50);
5195}
5196
4373static void si_rlc_stop(struct radeon_device *rdev) 5197static void si_rlc_stop(struct radeon_device *rdev)
4374{ 5198{
4375 WREG32(RLC_CNTL, 0); 5199 WREG32(RLC_CNTL, 0);
5200
5201 si_enable_gui_idle_interrupt(rdev, false);
5202
5203 si_wait_for_rlc_serdes(rdev);
4376} 5204}
4377 5205
4378static void si_rlc_start(struct radeon_device *rdev) 5206static void si_rlc_start(struct radeon_device *rdev)
4379{ 5207{
4380 WREG32(RLC_CNTL, RLC_ENABLE); 5208 WREG32(RLC_CNTL, RLC_ENABLE);
5209
5210 si_enable_gui_idle_interrupt(rdev, true);
5211
5212 udelay(50);
5213}
5214
5215static bool si_lbpw_supported(struct radeon_device *rdev)
5216{
5217 u32 tmp;
5218
5219 /* Enable LBPW only for DDR3 */
5220 tmp = RREG32(MC_SEQ_MISC0);
5221 if ((tmp & 0xF0000000) == 0xB0000000)
5222 return true;
5223 return false;
5224}
5225
5226static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5227{
5228 u32 tmp;
5229
5230 tmp = RREG32(RLC_LB_CNTL);
5231 if (enable)
5232 tmp |= LOAD_BALANCE_ENABLE;
5233 else
5234 tmp &= ~LOAD_BALANCE_ENABLE;
5235 WREG32(RLC_LB_CNTL, tmp);
5236
5237 if (!enable) {
5238 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5239 WREG32(SPI_LB_CU_MASK, 0x00ff);
5240 }
4381} 5241}
4382 5242
4383static int si_rlc_resume(struct radeon_device *rdev) 5243static int si_rlc_resume(struct radeon_device *rdev)
@@ -4390,14 +5250,18 @@ static int si_rlc_resume(struct radeon_device *rdev)
4390 5250
4391 si_rlc_stop(rdev); 5251 si_rlc_stop(rdev);
4392 5252
5253 si_rlc_reset(rdev);
5254
5255 si_init_pg(rdev);
5256
5257 si_init_cg(rdev);
5258
4393 WREG32(RLC_RL_BASE, 0); 5259 WREG32(RLC_RL_BASE, 0);
4394 WREG32(RLC_RL_SIZE, 0); 5260 WREG32(RLC_RL_SIZE, 0);
4395 WREG32(RLC_LB_CNTL, 0); 5261 WREG32(RLC_LB_CNTL, 0);
4396 WREG32(RLC_LB_CNTR_MAX, 0xffffffff); 5262 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4397 WREG32(RLC_LB_CNTR_INIT, 0); 5263 WREG32(RLC_LB_CNTR_INIT, 0);
4398 5264 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
4399 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4400 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4401 5265
4402 WREG32(RLC_MC_CNTL, 0); 5266 WREG32(RLC_MC_CNTL, 0);
4403 WREG32(RLC_UCODE_CNTL, 0); 5267 WREG32(RLC_UCODE_CNTL, 0);
@@ -4409,6 +5273,8 @@ static int si_rlc_resume(struct radeon_device *rdev)
4409 } 5273 }
4410 WREG32(RLC_UCODE_ADDR, 0); 5274 WREG32(RLC_UCODE_ADDR, 0);
4411 5275
5276 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5277
4412 si_rlc_start(rdev); 5278 si_rlc_start(rdev);
4413 5279
4414 return 0; 5280 return 0;
@@ -4578,6 +5444,7 @@ int si_irq_set(struct radeon_device *rdev)
4578 u32 grbm_int_cntl = 0; 5444 u32 grbm_int_cntl = 0;
4579 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 5445 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4580 u32 dma_cntl, dma_cntl1; 5446 u32 dma_cntl, dma_cntl1;
5447 u32 thermal_int = 0;
4581 5448
4582 if (!rdev->irq.installed) { 5449 if (!rdev->irq.installed) {
4583 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 5450 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -4603,6 +5470,9 @@ int si_irq_set(struct radeon_device *rdev)
4603 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5470 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4604 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5471 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4605 5472
5473 thermal_int = RREG32(CG_THERMAL_INT) &
5474 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5475
4606 /* enable CP interrupts on all rings */ 5476 /* enable CP interrupts on all rings */
4607 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 5477 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4608 DRM_DEBUG("si_irq_set: sw int gfx\n"); 5478 DRM_DEBUG("si_irq_set: sw int gfx\n");
@@ -4689,6 +5559,11 @@ int si_irq_set(struct radeon_device *rdev)
4689 5559
4690 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 5560 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4691 5561
5562 if (rdev->irq.dpm_thermal) {
5563 DRM_DEBUG("dpm thermal\n");
5564 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5565 }
5566
4692 if (rdev->num_crtc >= 2) { 5567 if (rdev->num_crtc >= 2) {
4693 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 5568 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4694 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 5569 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
@@ -4724,6 +5599,8 @@ int si_irq_set(struct radeon_device *rdev)
4724 WREG32(DC_HPD6_INT_CONTROL, hpd6); 5599 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4725 } 5600 }
4726 5601
5602 WREG32(CG_THERMAL_INT, thermal_int);
5603
4727 return 0; 5604 return 0;
4728} 5605}
4729 5606
@@ -4888,6 +5765,7 @@ int si_irq_process(struct radeon_device *rdev)
4888 u32 src_id, src_data, ring_id; 5765 u32 src_id, src_data, ring_id;
4889 u32 ring_index; 5766 u32 ring_index;
4890 bool queue_hotplug = false; 5767 bool queue_hotplug = false;
5768 bool queue_thermal = false;
4891 5769
4892 if (!rdev->ih.enabled || rdev->shutdown) 5770 if (!rdev->ih.enabled || rdev->shutdown)
4893 return IRQ_NONE; 5771 return IRQ_NONE;
@@ -5158,6 +6036,16 @@ restart_ih:
5158 DRM_DEBUG("IH: DMA trap\n"); 6036 DRM_DEBUG("IH: DMA trap\n");
5159 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 6037 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5160 break; 6038 break;
6039 case 230: /* thermal low to high */
6040 DRM_DEBUG("IH: thermal low to high\n");
6041 rdev->pm.dpm.thermal.high_to_low = false;
6042 queue_thermal = true;
6043 break;
6044 case 231: /* thermal high to low */
6045 DRM_DEBUG("IH: thermal high to low\n");
6046 rdev->pm.dpm.thermal.high_to_low = true;
6047 queue_thermal = true;
6048 break;
5161 case 233: /* GUI IDLE */ 6049 case 233: /* GUI IDLE */
5162 DRM_DEBUG("IH: GUI idle\n"); 6050 DRM_DEBUG("IH: GUI idle\n");
5163 break; 6051 break;
@@ -5176,6 +6064,8 @@ restart_ih:
5176 } 6064 }
5177 if (queue_hotplug) 6065 if (queue_hotplug)
5178 schedule_work(&rdev->hotplug_work); 6066 schedule_work(&rdev->hotplug_work);
6067 if (queue_thermal && rdev->pm.dpm_enabled)
6068 schedule_work(&rdev->pm.dpm.thermal.work);
5179 rdev->ih.rptr = rptr; 6069 rdev->ih.rptr = rptr;
5180 WREG32(IH_RB_RPTR, rdev->ih.rptr); 6070 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5181 atomic_set(&rdev->ih.lock, 0); 6071 atomic_set(&rdev->ih.lock, 0);
@@ -5270,6 +6160,11 @@ static int si_startup(struct radeon_device *rdev)
5270 struct radeon_ring *ring; 6160 struct radeon_ring *ring;
5271 int r; 6161 int r;
5272 6162
6163 /* enable pcie gen2/3 link */
6164 si_pcie_gen3_enable(rdev);
6165 /* enable aspm */
6166 si_program_aspm(rdev);
6167
5273 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 6168 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5274 !rdev->rlc_fw || !rdev->mc_fw) { 6169 !rdev->rlc_fw || !rdev->mc_fw) {
5275 r = si_init_microcode(rdev); 6170 r = si_init_microcode(rdev);
@@ -5609,6 +6504,8 @@ void si_fini(struct radeon_device *rdev)
5609 cayman_dma_fini(rdev); 6504 cayman_dma_fini(rdev);
5610 si_irq_fini(rdev); 6505 si_irq_fini(rdev);
5611 si_rlc_fini(rdev); 6506 si_rlc_fini(rdev);
6507 si_fini_cg(rdev);
6508 si_fini_pg(rdev);
5612 radeon_wb_fini(rdev); 6509 radeon_wb_fini(rdev);
5613 radeon_vm_manager_fini(rdev); 6510 radeon_vm_manager_fini(rdev);
5614 radeon_ib_pool_fini(rdev); 6511 radeon_ib_pool_fini(rdev);
@@ -5735,3 +6632,361 @@ int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5735 6632
5736 return 0; 6633 return 0;
5737} 6634}
6635
6636static void si_pcie_gen3_enable(struct radeon_device *rdev)
6637{
6638 struct pci_dev *root = rdev->pdev->bus->self;
6639 int bridge_pos, gpu_pos;
6640 u32 speed_cntl, mask, current_data_rate;
6641 int ret, i;
6642 u16 tmp16;
6643
6644 if (radeon_pcie_gen2 == 0)
6645 return;
6646
6647 if (rdev->flags & RADEON_IS_IGP)
6648 return;
6649
6650 if (!(rdev->flags & RADEON_IS_PCIE))
6651 return;
6652
6653 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6654 if (ret != 0)
6655 return;
6656
6657 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6658 return;
6659
6660 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6661 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6662 LC_CURRENT_DATA_RATE_SHIFT;
6663 if (mask & DRM_PCIE_SPEED_80) {
6664 if (current_data_rate == 2) {
6665 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6666 return;
6667 }
6668 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6669 } else if (mask & DRM_PCIE_SPEED_50) {
6670 if (current_data_rate == 1) {
6671 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6672 return;
6673 }
6674 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6675 }
6676
6677 bridge_pos = pci_pcie_cap(root);
6678 if (!bridge_pos)
6679 return;
6680
6681 gpu_pos = pci_pcie_cap(rdev->pdev);
6682 if (!gpu_pos)
6683 return;
6684
6685 if (mask & DRM_PCIE_SPEED_80) {
6686 /* re-try equalization if gen3 is not already enabled */
6687 if (current_data_rate != 2) {
6688 u16 bridge_cfg, gpu_cfg;
6689 u16 bridge_cfg2, gpu_cfg2;
6690 u32 max_lw, current_lw, tmp;
6691
6692 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6693 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6694
6695 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6696 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6697
6698 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6699 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6700
6701 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6702 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6703 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6704
6705 if (current_lw < max_lw) {
6706 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6707 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6708 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6709 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6710 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6711 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6712 }
6713 }
6714
6715 for (i = 0; i < 10; i++) {
6716 /* check status */
6717 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6718 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6719 break;
6720
6721 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6722 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6723
6724 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6725 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6726
6727 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6728 tmp |= LC_SET_QUIESCE;
6729 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6730
6731 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6732 tmp |= LC_REDO_EQ;
6733 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6734
6735 mdelay(100);
6736
6737 /* linkctl */
6738 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6739 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6740 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6741 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6742
6743 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6744 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6745 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6746 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6747
6748 /* linkctl2 */
6749 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6750 tmp16 &= ~((1 << 4) | (7 << 9));
6751 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6752 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6753
6754 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6755 tmp16 &= ~((1 << 4) | (7 << 9));
6756 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6757 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6758
6759 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6760 tmp &= ~LC_SET_QUIESCE;
6761 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6762 }
6763 }
6764 }
6765
6766 /* set the link speed */
6767 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6768 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6769 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6770
6771 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6772 tmp16 &= ~0xf;
6773 if (mask & DRM_PCIE_SPEED_80)
6774 tmp16 |= 3; /* gen3 */
6775 else if (mask & DRM_PCIE_SPEED_50)
6776 tmp16 |= 2; /* gen2 */
6777 else
6778 tmp16 |= 1; /* gen1 */
6779 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6780
6781 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6782 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6783 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6784
6785 for (i = 0; i < rdev->usec_timeout; i++) {
6786 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6787 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6788 break;
6789 udelay(1);
6790 }
6791}
6792
6793static void si_program_aspm(struct radeon_device *rdev)
6794{
6795 u32 data, orig;
6796 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6797 bool disable_clkreq = false;
6798
6799 if (!(rdev->flags & RADEON_IS_PCIE))
6800 return;
6801
6802 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6803 data &= ~LC_XMIT_N_FTS_MASK;
6804 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6805 if (orig != data)
6806 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6807
6808 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6809 data |= LC_GO_TO_RECOVERY;
6810 if (orig != data)
6811 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6812
6813 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6814 data |= P_IGNORE_EDB_ERR;
6815 if (orig != data)
6816 WREG32_PCIE(PCIE_P_CNTL, data);
6817
6818 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6819 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6820 data |= LC_PMI_TO_L1_DIS;
6821 if (!disable_l0s)
6822 data |= LC_L0S_INACTIVITY(7);
6823
6824 if (!disable_l1) {
6825 data |= LC_L1_INACTIVITY(7);
6826 data &= ~LC_PMI_TO_L1_DIS;
6827 if (orig != data)
6828 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6829
6830 if (!disable_plloff_in_l1) {
6831 bool clk_req_support;
6832
6833 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6834 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6835 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6836 if (orig != data)
6837 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6838
6839 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6840 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6841 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6842 if (orig != data)
6843 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6844
6845 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6846 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6847 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6848 if (orig != data)
6849 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6850
6851 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6852 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6853 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6854 if (orig != data)
6855 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6856
6857 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
6858 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6859 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6860 if (orig != data)
6861 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6862
6863 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6864 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6865 if (orig != data)
6866 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6867
6868 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
6869 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6870 if (orig != data)
6871 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
6872
6873 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
6874 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6875 if (orig != data)
6876 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
6877
6878 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6879 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6880 if (orig != data)
6881 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6882
6883 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6884 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6885 if (orig != data)
6886 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6887
6888 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
6889 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6890 if (orig != data)
6891 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
6892
6893 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
6894 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6895 if (orig != data)
6896 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
6897 }
6898 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6899 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6900 data |= LC_DYN_LANES_PWR_STATE(3);
6901 if (orig != data)
6902 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6903
6904 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6905 data &= ~LS2_EXIT_TIME_MASK;
6906 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6907 data |= LS2_EXIT_TIME(5);
6908 if (orig != data)
6909 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6910
6911 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6912 data &= ~LS2_EXIT_TIME_MASK;
6913 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6914 data |= LS2_EXIT_TIME(5);
6915 if (orig != data)
6916 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6917
6918 if (!disable_clkreq) {
6919 struct pci_dev *root = rdev->pdev->bus->self;
6920 u32 lnkcap;
6921
6922 clk_req_support = false;
6923 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
6924 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
6925 clk_req_support = true;
6926 } else {
6927 clk_req_support = false;
6928 }
6929
6930 if (clk_req_support) {
6931 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
6932 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
6933 if (orig != data)
6934 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
6935
6936 orig = data = RREG32(THM_CLK_CNTL);
6937 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
6938 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
6939 if (orig != data)
6940 WREG32(THM_CLK_CNTL, data);
6941
6942 orig = data = RREG32(MISC_CLK_CNTL);
6943 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
6944 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
6945 if (orig != data)
6946 WREG32(MISC_CLK_CNTL, data);
6947
6948 orig = data = RREG32(CG_CLKPIN_CNTL);
6949 data &= ~BCLK_AS_XCLK;
6950 if (orig != data)
6951 WREG32(CG_CLKPIN_CNTL, data);
6952
6953 orig = data = RREG32(CG_CLKPIN_CNTL_2);
6954 data &= ~FORCE_BIF_REFCLK_EN;
6955 if (orig != data)
6956 WREG32(CG_CLKPIN_CNTL_2, data);
6957
6958 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
6959 data &= ~MPLL_CLKOUT_SEL_MASK;
6960 data |= MPLL_CLKOUT_SEL(4);
6961 if (orig != data)
6962 WREG32(MPLL_BYPASSCLK_SEL, data);
6963
6964 orig = data = RREG32(SPLL_CNTL_MODE);
6965 data &= ~SPLL_REFCLK_SEL_MASK;
6966 if (orig != data)
6967 WREG32(SPLL_CNTL_MODE, data);
6968 }
6969 }
6970 } else {
6971 if (orig != data)
6972 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6973 }
6974
6975 orig = data = RREG32_PCIE(PCIE_CNTL2);
6976 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
6977 if (orig != data)
6978 WREG32_PCIE(PCIE_CNTL2, data);
6979
6980 if (!disable_l0s) {
6981 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6982 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
6983 data = RREG32_PCIE(PCIE_LC_STATUS1);
6984 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
6985 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6986 data &= ~LC_L0S_INACTIVITY_MASK;
6987 if (orig != data)
6988 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6989 }
6990 }
6991 }
6992}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-06 17:37:35 -0500