aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm
diff options
context:
space:
mode:
authorJordan Crouse <jcrouse@codeaurora.org>2018-08-06 13:33:24 -0400
committerRob Clark <robdclark@gmail.com>2018-08-10 18:49:18 -0400
commit4b565ca5a2cbbbb6345e8789da89c193b6b00e5a (patch)
tree7e85c925f7286cf04b0b14619ff2c1038bd33a4f /drivers/gpu/drm
parent2d756322533322650f4476265ba413498e998b56 (diff)
drm/msm: Add A6XX device support
Add support for the A6XX family of Adreno GPUs. The biggest addition is the GMU (Graphics Management Unit) which takes over most of the power management of the GPU itself but in a ironic twist of fate needs a goodly amount of management itself. Add support for the A6XX core code, the GMU and the HFI (hardware firmware interface) queue that the CPU uses to communicate with the GMU. Signed-off-by: Jordan Crouse <jcrouse@codeaurora.org> Signed-off-by: Rob Clark <robdclark@gmail.com>
Diffstat (limited to 'drivers/gpu/drm')
-rw-r--r--drivers/gpu/drm/msm/Makefile3
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gmu.c1207
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gmu.h162
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gpu.c818
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gpu.h60
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_hfi.c435
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_hfi.h127
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_device.c12
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_gpu.h3
-rw-r--r--drivers/gpu/drm/msm/msm_gpu.c2
10 files changed, 2828 insertions, 1 deletions
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 7c773e003663..261fa79d456d 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -11,6 +11,9 @@ msm-y := \
11 adreno/a5xx_gpu.o \ 11 adreno/a5xx_gpu.o \
12 adreno/a5xx_power.o \ 12 adreno/a5xx_power.o \
13 adreno/a5xx_preempt.o \ 13 adreno/a5xx_preempt.o \
14 adreno/a6xx_gpu.o \
15 adreno/a6xx_gmu.o \
16 adreno/a6xx_hfi.o \
14 hdmi/hdmi.o \ 17 hdmi/hdmi.o \
15 hdmi/hdmi_audio.o \ 18 hdmi/hdmi_audio.o \
16 hdmi/hdmi_bridge.o \ 19 hdmi/hdmi_bridge.o \
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
new file mode 100644
index 000000000000..fbb501986720
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -0,0 +1,1207 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
3
4#include <linux/clk.h>
5#include <linux/iopoll.h>
6#include <linux/pm_opp.h>
7#include <soc/qcom/cmd-db.h>
8
9#include "a6xx_gpu.h"
10#include "a6xx_gmu.xml.h"
11
12static irqreturn_t a6xx_gmu_irq(int irq, void *data)
13{
14 struct a6xx_gmu *gmu = data;
15 u32 status;
16
17 status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS);
18 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status);
19
20 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
21 dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
22
23 /* Temporary until we can recover safely */
24 BUG();
25 }
26
27 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
28 dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n");
29
30 if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
31 dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n",
32 gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS));
33
34 return IRQ_HANDLED;
35}
36
37static irqreturn_t a6xx_hfi_irq(int irq, void *data)
38{
39 struct a6xx_gmu *gmu = data;
40 u32 status;
41
42 status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO);
43 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status);
44
45 if (status & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ)
46 tasklet_schedule(&gmu->hfi_tasklet);
47
48 if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
49 dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
50
51 /* Temporary until we can recover safely */
52 BUG();
53 }
54
55 return IRQ_HANDLED;
56}
57
58/* Check to see if the GX rail is still powered */
59static bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu)
60{
61 u32 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
62
63 return !(val &
64 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
65 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
66}
67
68static int a6xx_gmu_set_freq(struct a6xx_gmu *gmu, int index)
69{
70 gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0);
71
72 gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING,
73 ((index << 24) & 0xff) | (3 & 0xf));
74
75 /*
76 * Send an invalid index as a vote for the bus bandwidth and let the
77 * firmware decide on the right vote
78 */
79 gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff);
80
81 /* Set and clear the OOB for DCVS to trigger the GMU */
82 a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET);
83 a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET);
84
85 return gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
86}
87
88static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
89{
90 u32 val;
91 int local = gmu->idle_level;
92
93 /* SPTP and IFPC both report as IFPC */
94 if (gmu->idle_level == GMU_IDLE_STATE_SPTP)
95 local = GMU_IDLE_STATE_IFPC;
96
97 val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
98
99 if (val == local) {
100 if (gmu->idle_level != GMU_IDLE_STATE_IFPC ||
101 !a6xx_gmu_gx_is_on(gmu))
102 return true;
103 }
104
105 return false;
106}
107
108/* Wait for the GMU to get to its most idle state */
109int a6xx_gmu_wait_for_idle(struct a6xx_gpu *a6xx_gpu)
110{
111 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
112
113 return spin_until(a6xx_gmu_check_idle_level(gmu));
114}
115
116static int a6xx_gmu_start(struct a6xx_gmu *gmu)
117{
118 int ret;
119 u32 val;
120
121 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
122 gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
123
124 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
125 val == 0xbabeface, 100, 10000);
126
127 if (ret)
128 dev_err(gmu->dev, "GMU firmware initalization timed out\n");
129
130 return ret;
131}
132
133static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu)
134{
135 u32 val;
136 int ret;
137
138 gmu_rmw(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
139 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 0);
140
141 gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1);
142
143 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val,
144 val & 1, 100, 10000);
145 if (ret)
146 dev_err(gmu->dev, "Unable to start the HFI queues\n");
147
148 return ret;
149}
150
151/* Trigger a OOB (out of band) request to the GMU */
152int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
153{
154 int ret;
155 u32 val;
156 int request, ack;
157 const char *name;
158
159 switch (state) {
160 case GMU_OOB_GPU_SET:
161 request = GMU_OOB_GPU_SET_REQUEST;
162 ack = GMU_OOB_GPU_SET_ACK;
163 name = "GPU_SET";
164 break;
165 case GMU_OOB_BOOT_SLUMBER:
166 request = GMU_OOB_BOOT_SLUMBER_REQUEST;
167 ack = GMU_OOB_BOOT_SLUMBER_ACK;
168 name = "BOOT_SLUMBER";
169 break;
170 case GMU_OOB_DCVS_SET:
171 request = GMU_OOB_DCVS_REQUEST;
172 ack = GMU_OOB_DCVS_ACK;
173 name = "GPU_DCVS";
174 break;
175 default:
176 return -EINVAL;
177 }
178
179 /* Trigger the equested OOB operation */
180 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request);
181
182 /* Wait for the acknowledge interrupt */
183 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
184 val & (1 << ack), 100, 10000);
185
186 if (ret)
187 dev_err(gmu->dev,
188 "Timeout waiting for GMU OOB set %s: 0x%x\n",
189 name,
190 gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO));
191
192 /* Clear the acknowledge interrupt */
193 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack);
194
195 return ret;
196}
197
198/* Clear a pending OOB state in the GMU */
199void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
200{
201 switch (state) {
202 case GMU_OOB_GPU_SET:
203 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
204 1 << GMU_OOB_GPU_SET_CLEAR);
205 break;
206 case GMU_OOB_BOOT_SLUMBER:
207 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
208 1 << GMU_OOB_BOOT_SLUMBER_CLEAR);
209 break;
210 case GMU_OOB_DCVS_SET:
211 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
212 1 << GMU_OOB_DCVS_CLEAR);
213 break;
214 }
215}
216
217/* Enable CPU control of SPTP power power collapse */
218static int a6xx_sptprac_enable(struct a6xx_gmu *gmu)
219{
220 int ret;
221 u32 val;
222
223 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
224
225 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
226 (val & 0x38) == 0x28, 1, 100);
227
228 if (ret) {
229 dev_err(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
230 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
231 }
232
233 return 0;
234}
235
236/* Disable CPU control of SPTP power power collapse */
237static void a6xx_sptprac_disable(struct a6xx_gmu *gmu)
238{
239 u32 val;
240 int ret;
241
242 /* Make sure retention is on */
243 gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
244
245 gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001);
246
247 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
248 (val & 0x04), 100, 10000);
249
250 if (ret)
251 dev_err(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
252 gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
253}
254
255/* Let the GMU know we are starting a boot sequence */
256static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu)
257{
258 u32 vote;
259
260 /* Let the GMU know we are getting ready for boot */
261 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0);
262
263 /* Choose the "default" power level as the highest available */
264 vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1];
265
266 gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff);
267 gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff);
268
269 /* Let the GMU know the boot sequence has started */
270 return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
271}
272
273/* Let the GMU know that we are about to go into slumber */
274static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
275{
276 int ret;
277
278 /* Disable the power counter so the GMU isn't busy */
279 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
280
281 /* Disable SPTP_PC if the CPU is responsible for it */
282 if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
283 a6xx_sptprac_disable(gmu);
284
285 /* Tell the GMU to get ready to slumber */
286 gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
287
288 ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
289 a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
290
291 if (!ret) {
292 /* Check to see if the GMU really did slumber */
293 if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE)
294 != 0x0f) {
295 dev_err(gmu->dev, "The GMU did not go into slumber\n");
296 ret = -ETIMEDOUT;
297 }
298 }
299
300 /* Put fence into allow mode */
301 gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
302 return ret;
303}
304
305static int a6xx_rpmh_start(struct a6xx_gmu *gmu)
306{
307 int ret;
308 u32 val;
309
310 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1 << 1);
311 /* Wait for the register to finish posting */
312 wmb();
313
314 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val,
315 val & (1 << 1), 100, 10000);
316 if (ret) {
317 dev_err(gmu->dev, "Unable to power on the GPU RSC\n");
318 return ret;
319 }
320
321 ret = gmu_poll_timeout(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
322 !val, 100, 10000);
323
324 if (!ret) {
325 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
326
327 /* Re-enable the power counter */
328 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
329 return 0;
330 }
331
332 dev_err(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
333 return ret;
334}
335
336static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
337{
338 int ret;
339 u32 val;
340
341 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
342
343 ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
344 val, val & (1 << 16), 100, 10000);
345 if (ret)
346 dev_err(gmu->dev, "Unable to power off the GPU RSC\n");
347
348 gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
349}
350
351static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
352{
353 /* Disable SDE clock gating */
354 gmu_write(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
355
356 /* Setup RSC PDC handshake for sleep and wakeup */
357 gmu_write(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
358 gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
359 gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
360 gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
361 gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
362 gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000);
363 gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
364 gmu_write(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
365 gmu_write(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
366 gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
367 gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
368
369 /* Load RSC sequencer uCode for sleep and wakeup */
370 gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
371 gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
372 gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
373 gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
374 gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
375
376 /* Load PDC sequencer uCode for power up and power down sequence */
377 pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
378 pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
379 pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
380 pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
381 pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
382
383 /* Set TCS commands used by PDC sequence for low power modes */
384 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
385 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
386 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
387 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
388 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
389 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
390 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
391 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
392 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
393 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
394 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30080);
395 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
396 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
397 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
398 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
399 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
400 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
401 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
402 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
403 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
404 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
405 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
406 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30080);
407 pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
408
409 /* Setup GPU PDC */
410 pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
411 pdc_write(gmu, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
412
413 /* ensure no writes happen before the uCode is fully written */
414 wmb();
415}
416
417/*
418 * The lowest 16 bits of this value are the number of XO clock cycles for main
419 * hysteresis which is set at 0x1680 cycles (300 us). The higher 16 bits are
420 * for the shorter hysteresis that happens after main - this is 0xa (.5 us)
421 */
422
423#define GMU_PWR_COL_HYST 0x000a1680
424
425/* Set up the idle state for the GMU */
426static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
427{
428 /* Disable GMU WB/RB buffer */
429 gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
430
431 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
432
433 switch (gmu->idle_level) {
434 case GMU_IDLE_STATE_IFPC:
435 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
436 GMU_PWR_COL_HYST);
437 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
438 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
439 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE);
440 /* Fall through */
441 case GMU_IDLE_STATE_SPTP:
442 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST,
443 GMU_PWR_COL_HYST);
444 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
445 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
446 A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE);
447 }
448
449 /* Enable RPMh GPU client */
450 gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0,
451 A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE |
452 A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE |
453 A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE |
454 A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE |
455 A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE |
456 A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
457}
458
459static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
460{
461 static bool rpmh_init;
462 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
463 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
464 int i, ret;
465 u32 chipid;
466 u32 *image;
467
468 if (state == GMU_WARM_BOOT) {
469 ret = a6xx_rpmh_start(gmu);
470 if (ret)
471 return ret;
472 } else {
473 if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU],
474 "GMU firmware is not loaded\n"))
475 return -ENOENT;
476
477 /* Sanity check the size of the firmware that was loaded */
478 if (adreno_gpu->fw[ADRENO_FW_GMU]->size > 0x8000) {
479 dev_err(gmu->dev,
480 "GMU firmware is bigger than the available region\n");
481 return -EINVAL;
482 }
483
484 /* Turn on register retention */
485 gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
486
487 /* We only need to load the RPMh microcode once */
488 if (!rpmh_init) {
489 a6xx_gmu_rpmh_init(gmu);
490 rpmh_init = true;
491 } else if (state != GMU_RESET) {
492 ret = a6xx_rpmh_start(gmu);
493 if (ret)
494 return ret;
495 }
496
497 image = (u32 *) adreno_gpu->fw[ADRENO_FW_GMU]->data;
498
499 for (i = 0; i < adreno_gpu->fw[ADRENO_FW_GMU]->size >> 2; i++)
500 gmu_write(gmu, REG_A6XX_GMU_CM3_ITCM_START + i,
501 image[i]);
502 }
503
504 gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
505 gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
506
507 /* Write the iova of the HFI table */
508 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi->iova);
509 gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
510
511 gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
512 (1 << 31) | (0xa << 18) | (0xa0));
513
514 chipid = adreno_gpu->rev.core << 24;
515 chipid |= adreno_gpu->rev.major << 16;
516 chipid |= adreno_gpu->rev.minor << 12;
517 chipid |= adreno_gpu->rev.patchid << 8;
518
519 gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
520
521 /* Set up the lowest idle level on the GMU */
522 a6xx_gmu_power_config(gmu);
523
524 ret = a6xx_gmu_start(gmu);
525 if (ret)
526 return ret;
527
528 ret = a6xx_gmu_gfx_rail_on(gmu);
529 if (ret)
530 return ret;
531
532 /* Enable SPTP_PC if the CPU is responsible for it */
533 if (gmu->idle_level < GMU_IDLE_STATE_SPTP) {
534 ret = a6xx_sptprac_enable(gmu);
535 if (ret)
536 return ret;
537 }
538
539 ret = a6xx_gmu_hfi_start(gmu);
540 if (ret)
541 return ret;
542
543 /* FIXME: Do we need this wmb() here? */
544 wmb();
545
546 return 0;
547}
548
549#define A6XX_HFI_IRQ_MASK \
550 (A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ | \
551 A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
552
553#define A6XX_GMU_IRQ_MASK \
554 (A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \
555 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \
556 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
557
558static void a6xx_gmu_irq_enable(struct a6xx_gmu *gmu)
559{
560 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
561 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
562
563 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK,
564 ~A6XX_GMU_IRQ_MASK);
565 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
566 ~A6XX_HFI_IRQ_MASK);
567
568 enable_irq(gmu->gmu_irq);
569 enable_irq(gmu->hfi_irq);
570}
571
572static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
573{
574 disable_irq(gmu->gmu_irq);
575 disable_irq(gmu->hfi_irq);
576
577 gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0);
578 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
579}
580
581int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu)
582{
583 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
584 int ret;
585 u32 val;
586
587 /* Flush all the queues */
588 a6xx_hfi_stop(gmu);
589
590 /* Stop the interrupts */
591 a6xx_gmu_irq_disable(gmu);
592
593 /* Force off SPTP in case the GMU is managing it */
594 a6xx_sptprac_disable(gmu);
595
596 /* Make sure there are no outstanding RPMh votes */
597 gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
598 (val & 1), 100, 10000);
599 gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val,
600 (val & 1), 100, 10000);
601 gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val,
602 (val & 1), 100, 10000);
603 gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
604 (val & 1), 100, 1000);
605
606 /* Force off the GX GSDC */
607 regulator_force_disable(gmu->gx);
608
609 /* Disable the resources */
610 clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
611 pm_runtime_put_sync(gmu->dev);
612
613 /* Re-enable the resources */
614 pm_runtime_get_sync(gmu->dev);
615
616 /* Use a known rate to bring up the GMU */
617 clk_set_rate(gmu->core_clk, 200000000);
618 ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
619 if (ret)
620 goto out;
621
622 a6xx_gmu_irq_enable(gmu);
623
624 ret = a6xx_gmu_fw_start(gmu, GMU_RESET);
625 if (!ret)
626 ret = a6xx_hfi_start(gmu, GMU_COLD_BOOT);
627
628 /* Set the GPU back to the highest power frequency */
629 a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
630
631out:
632 if (ret)
633 a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
634
635 return ret;
636}
637
638int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
639{
640 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
641 int status, ret;
642
643 if (WARN(!gmu->mmio, "The GMU is not set up yet\n"))
644 return 0;
645
646 /* Turn on the resources */
647 pm_runtime_get_sync(gmu->dev);
648
649 /* Use a known rate to bring up the GMU */
650 clk_set_rate(gmu->core_clk, 200000000);
651 ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
652 if (ret)
653 goto out;
654
655 a6xx_gmu_irq_enable(gmu);
656
657 /* Check to see if we are doing a cold or warm boot */
658 status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
659 GMU_WARM_BOOT : GMU_COLD_BOOT;
660
661 ret = a6xx_gmu_fw_start(gmu, status);
662 if (ret)
663 goto out;
664
665 ret = a6xx_hfi_start(gmu, status);
666
667 /* Set the GPU to the highest power frequency */
668 a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
669
670out:
671 /* Make sure to turn off the boot OOB request on error */
672 if (ret)
673 a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
674
675 return ret;
676}
677
678bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
679{
680 u32 reg;
681
682 if (!gmu->mmio)
683 return true;
684
685 reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS);
686
687 if (reg & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB)
688 return false;
689
690 return true;
691}
692
693int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
694{
695 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
696 u32 val;
697
698 /*
699 * The GMU may still be in slumber unless the GPU started so check and
700 * skip putting it back into slumber if so
701 */
702 val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
703
704 if (val != 0xf) {
705 int ret = a6xx_gmu_wait_for_idle(a6xx_gpu);
706
707 /* Temporary until we can recover safely */
708 BUG_ON(ret);
709
710 /* tell the GMU we want to slumber */
711 a6xx_gmu_notify_slumber(gmu);
712
713 ret = gmu_poll_timeout(gmu,
714 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val,
715 !(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB),
716 100, 10000);
717
718 /*
719 * Let the user know we failed to slumber but don't worry too
720 * much because we are powering down anyway
721 */
722
723 if (ret)
724 dev_err(gmu->dev,
725 "Unable to slumber GMU: status = 0%x/0%x\n",
726 gmu_read(gmu,
727 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS),
728 gmu_read(gmu,
729 REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2));
730 }
731
732 /* Turn off HFI */
733 a6xx_hfi_stop(gmu);
734
735 /* Stop the interrupts and mask the hardware */
736 a6xx_gmu_irq_disable(gmu);
737
738 /* Tell RPMh to power off the GPU */
739 a6xx_rpmh_stop(gmu);
740
741 clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
742
743 pm_runtime_put_sync(gmu->dev);
744
745 return 0;
746}
747
748static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo)
749{
750 int count, i;
751 u64 iova;
752
753 if (IS_ERR_OR_NULL(bo))
754 return;
755
756 count = bo->size >> PAGE_SHIFT;
757 iova = bo->iova;
758
759 for (i = 0; i < count; i++, iova += PAGE_SIZE) {
760 iommu_unmap(gmu->domain, iova, PAGE_SIZE);
761 __free_pages(bo->pages[i], 0);
762 }
763
764 kfree(bo->pages);
765 kfree(bo);
766}
767
768static struct a6xx_gmu_bo *a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu,
769 size_t size)
770{
771 struct a6xx_gmu_bo *bo;
772 int ret, count, i;
773
774 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
775 if (!bo)
776 return ERR_PTR(-ENOMEM);
777
778 bo->size = PAGE_ALIGN(size);
779
780 count = bo->size >> PAGE_SHIFT;
781
782 bo->pages = kcalloc(count, sizeof(struct page *), GFP_KERNEL);
783 if (!bo->pages) {
784 kfree(bo);
785 return ERR_PTR(-ENOMEM);
786 }
787
788 for (i = 0; i < count; i++) {
789 bo->pages[i] = alloc_page(GFP_KERNEL);
790 if (!bo->pages[i])
791 goto err;
792 }
793
794 bo->iova = gmu->uncached_iova_base;
795
796 for (i = 0; i < count; i++) {
797 ret = iommu_map(gmu->domain,
798 bo->iova + (PAGE_SIZE * i),
799 page_to_phys(bo->pages[i]), PAGE_SIZE,
800 IOMMU_READ | IOMMU_WRITE);
801
802 if (ret) {
803 dev_err(gmu->dev, "Unable to map GMU buffer object\n");
804
805 for (i = i - 1 ; i >= 0; i--)
806 iommu_unmap(gmu->domain,
807 bo->iova + (PAGE_SIZE * i),
808 PAGE_SIZE);
809
810 goto err;
811 }
812 }
813
814 bo->virt = vmap(bo->pages, count, VM_IOREMAP,
815 pgprot_writecombine(PAGE_KERNEL));
816 if (!bo->virt)
817 goto err;
818
819 /* Align future IOVA addresses on 1MB boundaries */
820 gmu->uncached_iova_base += ALIGN(size, SZ_1M);
821
822 return bo;
823
824err:
825 for (i = 0; i < count; i++) {
826 if (bo->pages[i])
827 __free_pages(bo->pages[i], 0);
828 }
829
830 kfree(bo->pages);
831 kfree(bo);
832
833 return ERR_PTR(-ENOMEM);
834}
835
836static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
837{
838 int ret;
839
840 /*
841 * The GMU address space is hardcoded to treat the range
842 * 0x60000000 - 0x80000000 as un-cached memory. All buffers shared
843 * between the GMU and the CPU will live in this space
844 */
845 gmu->uncached_iova_base = 0x60000000;
846
847
848 gmu->domain = iommu_domain_alloc(&platform_bus_type);
849 if (!gmu->domain)
850 return -ENODEV;
851
852 ret = iommu_attach_device(gmu->domain, gmu->dev);
853
854 if (ret) {
855 iommu_domain_free(gmu->domain);
856 gmu->domain = NULL;
857 }
858
859 return ret;
860}
861
862/* Get the list of RPMh voltage levels from cmd-db */
863static int a6xx_gmu_rpmh_arc_cmds(const char *id, void *vals, int size)
864{
865 u32 len = cmd_db_read_aux_data_len(id);
866
867 if (!len)
868 return 0;
869
870 if (WARN_ON(len > size))
871 return -EINVAL;
872
873 cmd_db_read_aux_data(id, vals, len);
874
875 /*
876 * The data comes back as an array of unsigned shorts so adjust the
877 * count accordingly
878 */
879 return len >> 1;
880}
881
882/* Return the 'arc-level' for the given frequency */
883static u32 a6xx_gmu_get_arc_level(struct device *dev, unsigned long freq)
884{
885 struct dev_pm_opp *opp;
886 struct device_node *np;
887 u32 val = 0;
888
889 if (!freq)
890 return 0;
891
892 opp = dev_pm_opp_find_freq_exact(dev, freq, true);
893 if (IS_ERR(opp))
894 return 0;
895
896 np = dev_pm_opp_get_of_node(opp);
897
898 if (np) {
899 of_property_read_u32(np, "qcom,level", &val);
900 of_node_put(np);
901 }
902
903 dev_pm_opp_put(opp);
904
905 return val;
906}
907
908static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
909 unsigned long *freqs, int freqs_count,
910 u16 *pri, int pri_count,
911 u16 *sec, int sec_count)
912{
913 int i, j;
914
915 /* Construct a vote for each frequency */
916 for (i = 0; i < freqs_count; i++) {
917 u8 pindex = 0, sindex = 0;
918 u32 level = a6xx_gmu_get_arc_level(dev, freqs[i]);
919
920 /* Get the primary index that matches the arc level */
921 for (j = 0; j < pri_count; j++) {
922 if (pri[j] >= level) {
923 pindex = j;
924 break;
925 }
926 }
927
928 if (j == pri_count) {
929 dev_err(dev,
930 "Level %u not found in in the RPMh list\n",
931 level);
932 dev_err(dev, "Available levels:\n");
933 for (j = 0; j < pri_count; j++)
934 dev_err(dev, " %u\n", pri[j]);
935
936 return -EINVAL;
937 }
938
939 /*
940 * Look for a level in in the secondary list that matches. If
941 * nothing fits, use the maximum non zero vote
942 */
943
944 for (j = 0; j < sec_count; j++) {
945 if (sec[j] >= level) {
946 sindex = j;
947 break;
948 } else if (sec[j]) {
949 sindex = j;
950 }
951 }
952
953 /* Construct the vote */
954 votes[i] = ((pri[pindex] & 0xffff) << 16) |
955 (sindex << 8) | pindex;
956 }
957
958 return 0;
959}
960
961/*
962 * The GMU votes with the RPMh for itself and on behalf of the GPU but we need
963 * to construct the list of votes on the CPU and send it over. Query the RPMh
964 * voltage levels and build the votes
965 */
966
967static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu)
968{
969 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
970 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
971 struct msm_gpu *gpu = &adreno_gpu->base;
972
973 u16 gx[16], cx[16], mx[16];
974 u32 gxcount, cxcount, mxcount;
975 int ret;
976
977 /* Get the list of available voltage levels for each component */
978 gxcount = a6xx_gmu_rpmh_arc_cmds("gfx.lvl", gx, sizeof(gx));
979 cxcount = a6xx_gmu_rpmh_arc_cmds("cx.lvl", cx, sizeof(cx));
980 mxcount = a6xx_gmu_rpmh_arc_cmds("mx.lvl", mx, sizeof(mx));
981
982 /* Build the GX votes */
983 ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes,
984 gmu->gpu_freqs, gmu->nr_gpu_freqs,
985 gx, gxcount, mx, mxcount);
986
987 /* Build the CX votes */
988 ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes,
989 gmu->gmu_freqs, gmu->nr_gmu_freqs,
990 cx, cxcount, mx, mxcount);
991
992 return ret;
993}
994
995static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs,
996 u32 size)
997{
998 int count = dev_pm_opp_get_opp_count(dev);
999 struct dev_pm_opp *opp;
1000 int i, index = 0;
1001 unsigned long freq = 1;
1002
1003 /*
1004 * The OPP table doesn't contain the "off" frequency level so we need to
1005 * add 1 to the table size to account for it
1006 */
1007
1008 if (WARN(count + 1 > size,
1009 "The GMU frequency table is being truncated\n"))
1010 count = size - 1;
1011
1012 /* Set the "off" frequency */
1013 freqs[index++] = 0;
1014
1015 for (i = 0; i < count; i++) {
1016 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
1017 if (IS_ERR(opp))
1018 break;
1019
1020 dev_pm_opp_put(opp);
1021 freqs[index++] = freq++;
1022 }
1023
1024 return index;
1025}
1026
1027static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu)
1028{
1029 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1030 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1031 struct msm_gpu *gpu = &adreno_gpu->base;
1032
1033 int ret = 0;
1034
1035 /*
1036 * The GMU handles its own frequency switching so build a list of
1037 * available frequencies to send during initalization
1038 */
1039 ret = dev_pm_opp_of_add_table(gmu->dev);
1040 if (ret) {
1041 dev_err(gmu->dev, "Unable to set the OPP table for the GMU\n");
1042 return ret;
1043 }
1044
1045 gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev,
1046 gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs));
1047
1048 /*
1049 * The GMU also handles GPU frequency switching so build a list
1050 * from the GPU OPP table
1051 */
1052 gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev,
1053 gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs));
1054
1055 /* Build the list of RPMh votes that we'll send to the GMU */
1056 return a6xx_gmu_rpmh_votes_init(gmu);
1057}
1058
1059static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu)
1060{
1061 int ret = msm_clk_bulk_get(gmu->dev, &gmu->clocks);
1062
1063 if (ret < 1)
1064 return ret;
1065
1066 gmu->nr_clocks = ret;
1067
1068 gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks,
1069 gmu->nr_clocks, "gmu");
1070
1071 return 0;
1072}
1073
1074static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
1075 const char *name)
1076{
1077 void __iomem *ret;
1078 struct resource *res = platform_get_resource_byname(pdev,
1079 IORESOURCE_MEM, name);
1080
1081 if (!res) {
1082 dev_err(&pdev->dev, "Unable to find the %s registers\n", name);
1083 return ERR_PTR(-EINVAL);
1084 }
1085
1086 ret = devm_ioremap(&pdev->dev, res->start, resource_size(res));
1087 if (!ret) {
1088 dev_err(&pdev->dev, "Unable to map the %s registers\n", name);
1089 return ERR_PTR(-EINVAL);
1090 }
1091
1092 return ret;
1093}
1094
1095static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev,
1096 const char *name, irq_handler_t handler)
1097{
1098 int irq, ret;
1099
1100 irq = platform_get_irq_byname(pdev, name);
1101
1102 ret = devm_request_irq(&pdev->dev, irq, handler, IRQF_TRIGGER_HIGH,
1103 name, gmu);
1104 if (ret) {
1105 dev_err(&pdev->dev, "Unable to get interrupt %s\n", name);
1106 return ret;
1107 }
1108
1109 disable_irq(irq);
1110
1111 return irq;
1112}
1113
1114void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
1115{
1116 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1117
1118 if (IS_ERR_OR_NULL(gmu->mmio))
1119 return;
1120
1121 pm_runtime_disable(gmu->dev);
1122 a6xx_gmu_stop(a6xx_gpu);
1123
1124 a6xx_gmu_irq_disable(gmu);
1125 a6xx_gmu_memory_free(gmu, gmu->hfi);
1126
1127 iommu_detach_device(gmu->domain, gmu->dev);
1128
1129 iommu_domain_free(gmu->domain);
1130}
1131
1132int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
1133{
1134 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1135 struct platform_device *pdev = of_find_device_by_node(node);
1136 int ret;
1137
1138 if (!pdev)
1139 return -ENODEV;
1140
1141 gmu->dev = &pdev->dev;
1142
1143 of_dma_configure(gmu->dev, node, false);
1144
1145 /* Fow now, don't do anything fancy until we get our feet under us */
1146 gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
1147
1148 pm_runtime_enable(gmu->dev);
1149 gmu->gx = devm_regulator_get(gmu->dev, "vdd");
1150
1151 /* Get the list of clocks */
1152 ret = a6xx_gmu_clocks_probe(gmu);
1153 if (ret)
1154 return ret;
1155
1156 /* Set up the IOMMU context bank */
1157 ret = a6xx_gmu_memory_probe(gmu);
1158 if (ret)
1159 return ret;
1160
1161 /* Allocate memory for for the HFI queues */
1162 gmu->hfi = a6xx_gmu_memory_alloc(gmu, SZ_16K);
1163 if (IS_ERR(gmu->hfi))
1164 goto err;
1165
1166 /* Allocate memory for the GMU debug region */
1167 gmu->debug = a6xx_gmu_memory_alloc(gmu, SZ_16K);
1168 if (IS_ERR(gmu->debug))
1169 goto err;
1170
1171 /* Map the GMU registers */
1172 gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
1173
1174 /* Map the GPU power domain controller registers */
1175 gmu->pdc_mmio = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
1176
1177 if (IS_ERR(gmu->mmio) || IS_ERR(gmu->pdc_mmio))
1178 goto err;
1179
1180 /* Get the HFI and GMU interrupts */
1181 gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
1182 gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
1183
1184 if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0)
1185 goto err;
1186
1187 /* Set up a tasklet to handle GMU HFI responses */
1188 tasklet_init(&gmu->hfi_tasklet, a6xx_hfi_task, (unsigned long) gmu);
1189
1190 /* Get the power levels for the GMU and GPU */
1191 a6xx_gmu_pwrlevels_probe(gmu);
1192
1193 /* Set up the HFI queues */
1194 a6xx_hfi_init(gmu);
1195
1196 return 0;
1197err:
1198 a6xx_gmu_memory_free(gmu, gmu->hfi);
1199
1200 if (gmu->domain) {
1201 iommu_detach_device(gmu->domain, gmu->dev);
1202
1203 iommu_domain_free(gmu->domain);
1204 }
1205
1206 return -ENODEV;
1207}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
new file mode 100644
index 000000000000..d9a386c18799
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -0,0 +1,162 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */
3
4#ifndef _A6XX_GMU_H_
5#define _A6XX_GMU_H_
6
7#include <linux/interrupt.h>
8#include "msm_drv.h"
9#include "a6xx_hfi.h"
10
11struct a6xx_gmu_bo {
12 void *virt;
13 size_t size;
14 u64 iova;
15 struct page **pages;
16};
17
18/*
19 * These define the different GMU wake up options - these define how both the
20 * CPU and the GMU bring up the hardware
21 */
22
23/* THe GMU has already been booted and the rentention registers are active */
24#define GMU_WARM_BOOT 0
25
26/* the GMU is coming up for the first time or back from a power collapse */
27#define GMU_COLD_BOOT 1
28
29/* The GMU is being soft reset after a fault */
30#define GMU_RESET 2
31
32/*
33 * These define the level of control that the GMU has - the higher the number
34 * the more things that the GMU hardware controls on its own.
35 */
36
37/* The GMU does not do any idle state management */
38#define GMU_IDLE_STATE_ACTIVE 0
39
40/* The GMU manages SPTP power collapse */
41#define GMU_IDLE_STATE_SPTP 2
42
43/* The GMU does automatic IFPC (intra-frame power collapse) */
44#define GMU_IDLE_STATE_IFPC 3
45
46struct a6xx_gmu {
47 struct device *dev;
48
49 void * __iomem mmio;
50 void * __iomem pdc_mmio;
51
52 int hfi_irq;
53 int gmu_irq;
54
55 struct regulator *gx;
56
57 struct iommu_domain *domain;
58 u64 uncached_iova_base;
59
60 int idle_level;
61
62 struct a6xx_gmu_bo *hfi;
63 struct a6xx_gmu_bo *debug;
64
65 int nr_clocks;
66 struct clk_bulk_data *clocks;
67 struct clk *core_clk;
68
69 int nr_gpu_freqs;
70 unsigned long gpu_freqs[16];
71 u32 gx_arc_votes[16];
72
73 int nr_gmu_freqs;
74 unsigned long gmu_freqs[4];
75 u32 cx_arc_votes[4];
76
77 struct a6xx_hfi_queue queues[2];
78
79 struct tasklet_struct hfi_tasklet;
80};
81
82static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
83{
84 return msm_readl(gmu->mmio + (offset << 2));
85}
86
87static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value)
88{
89 return msm_writel(value, gmu->mmio + (offset << 2));
90}
91
92static inline void pdc_write(struct a6xx_gmu *gmu, u32 offset, u32 value)
93{
94 return msm_writel(value, gmu->pdc_mmio + (offset << 2));
95}
96
97static inline void gmu_rmw(struct a6xx_gmu *gmu, u32 reg, u32 mask, u32 or)
98{
99 u32 val = gmu_read(gmu, reg);
100
101 val &= ~mask;
102
103 gmu_write(gmu, reg, val | or);
104}
105
106#define gmu_poll_timeout(gmu, addr, val, cond, interval, timeout) \
107 readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \
108 interval, timeout)
109
110/*
111 * These are the available OOB (out of band requests) to the GMU where "out of
112 * band" means that the CPU talks to the GMU directly and not through HFI.
113 * Normally this works by writing a ITCM/DTCM register and then triggering a
114 * interrupt (the "request" bit) and waiting for an acknowledgment (the "ack"
115 * bit). The state is cleared by writing the "clear' bit to the GMU interrupt.
116 *
117 * These are used to force the GMU/GPU to stay on during a critical sequence or
118 * for hardware workarounds.
119 */
120
121enum a6xx_gmu_oob_state {
122 GMU_OOB_BOOT_SLUMBER = 0,
123 GMU_OOB_GPU_SET,
124 GMU_OOB_DCVS_SET,
125};
126
127/* These are the interrupt / ack bits for each OOB request that are set
128 * in a6xx_gmu_set_oob and a6xx_clear_oob
129 */
130
131/*
132 * Let the GMU know that a boot or slumber operation has started. The value in
133 * REG_A6XX_GMU_BOOT_SLUMBER_OPTION lets the GMU know which operation we are
134 * doing
135 */
136#define GMU_OOB_BOOT_SLUMBER_REQUEST 22
137#define GMU_OOB_BOOT_SLUMBER_ACK 30
138#define GMU_OOB_BOOT_SLUMBER_CLEAR 30
139
140/*
141 * Set a new power level for the GPU when the CPU is doing frequency scaling
142 */
143#define GMU_OOB_DCVS_REQUEST 23
144#define GMU_OOB_DCVS_ACK 31
145#define GMU_OOB_DCVS_CLEAR 31
146
147/*
148 * Let the GMU know to not turn off any GPU registers while the CPU is in a
149 * critical section
150 */
151#define GMU_OOB_GPU_SET_REQUEST 16
152#define GMU_OOB_GPU_SET_ACK 24
153#define GMU_OOB_GPU_SET_CLEAR 24
154
155
156void a6xx_hfi_init(struct a6xx_gmu *gmu);
157int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state);
158void a6xx_hfi_stop(struct a6xx_gmu *gmu);
159
160void a6xx_hfi_task(unsigned long data);
161
162#endif
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
new file mode 100644
index 000000000000..c629f742a1d1
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -0,0 +1,818 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
3
4
5#include "msm_gem.h"
6#include "msm_mmu.h"
7#include "a6xx_gpu.h"
8#include "a6xx_gmu.xml.h"
9
10static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
11{
12 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
13 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
14
15 /* Check that the GMU is idle */
16 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))
17 return false;
18
19 /* Check tha the CX master is idle */
20 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
21 ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
22 return false;
23
24 return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
25 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
26}
27
28bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
29{
30 /* wait for CP to drain ringbuffer: */
31 if (!adreno_idle(gpu, ring))
32 return false;
33
34 if (spin_until(_a6xx_check_idle(gpu))) {
35 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
36 gpu->name, __builtin_return_address(0),
37 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
38 gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
39 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
40 gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
41 return false;
42 }
43
44 return true;
45}
46
47static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
48{
49 uint32_t wptr;
50 unsigned long flags;
51
52 spin_lock_irqsave(&ring->lock, flags);
53
54 /* Copy the shadow to the actual register */
55 ring->cur = ring->next;
56
57 /* Make sure to wrap wptr if we need to */
58 wptr = get_wptr(ring);
59
60 spin_unlock_irqrestore(&ring->lock, flags);
61
62 /* Make sure everything is posted before making a decision */
63 mb();
64
65 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
66}
67
68static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
69 struct msm_file_private *ctx)
70{
71 struct msm_drm_private *priv = gpu->dev->dev_private;
72 struct msm_ringbuffer *ring = submit->ring;
73 unsigned int i;
74
75 /* Invalidate CCU depth and color */
76 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
77 OUT_RING(ring, PC_CCU_INVALIDATE_DEPTH);
78
79 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
80 OUT_RING(ring, PC_CCU_INVALIDATE_COLOR);
81
82 /* Submit the commands */
83 for (i = 0; i < submit->nr_cmds; i++) {
84 switch (submit->cmd[i].type) {
85 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
86 break;
87 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
88 if (priv->lastctx == ctx)
89 break;
90 case MSM_SUBMIT_CMD_BUF:
91 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
92 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
93 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
94 OUT_RING(ring, submit->cmd[i].size);
95 break;
96 }
97 }
98
99 /* Write the fence to the scratch register */
100 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
101 OUT_RING(ring, submit->seqno);
102
103 /*
104 * Execute a CACHE_FLUSH_TS event. This will ensure that the
105 * timestamp is written to the memory and then triggers the interrupt
106 */
107 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
108 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
109 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
110 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
111 OUT_RING(ring, submit->seqno);
112
113 a6xx_flush(gpu, ring);
114}
115
116static const struct {
117 u32 offset;
118 u32 value;
119} a6xx_hwcg[] = {
120 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
121 {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
122 {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
123 {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
124 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
125 {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
126 {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
127 {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
128 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
129 {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
130 {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
131 {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
132 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
133 {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
134 {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
135 {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
136 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
137 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
138 {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
139 {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
140 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
141 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
142 {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
143 {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
144 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
145 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
146 {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
147 {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
148 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
149 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
150 {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
151 {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
152 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
153 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
154 {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
155 {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
156 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
157 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
158 {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
159 {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
160 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
161 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
162 {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
163 {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
164 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
165 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
166 {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
167 {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
168 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
169 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
170 {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
171 {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
172 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
173 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
174 {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
175 {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
176 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
177 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
178 {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
179 {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
180 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
181 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
182 {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
183 {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
184 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
185 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
186 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
187 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
188 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
189 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
190 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
191 {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
192 {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
193 {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
194 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
195 {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
196 {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
197 {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
198 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
199 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
200 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
201 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
202 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
203 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
204 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
205 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
206 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
207 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
208 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
209 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
210 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
211 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
212 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
213 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
214 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
215 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
216 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
217 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
218 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
219 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
220 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
221 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
222 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
223 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
224 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}
225};
226
227static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
228{
229 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
230 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
231 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
232 unsigned int i;
233 u32 val;
234
235 val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
236
237 /* Don't re-program the registers if they are already correct */
238 if ((!state && !val) || (state && (val == 0x8aa8aa02)))
239 return;
240
241 /* Disable SP clock before programming HWCG registers */
242 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
243
244 for (i = 0; i < ARRAY_SIZE(a6xx_hwcg); i++)
245 gpu_write(gpu, a6xx_hwcg[i].offset,
246 state ? a6xx_hwcg[i].value : 0);
247
248 /* Enable SP clock */
249 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
250
251 gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0);
252}
253
254static int a6xx_cp_init(struct msm_gpu *gpu)
255{
256 struct msm_ringbuffer *ring = gpu->rb[0];
257
258 OUT_PKT7(ring, CP_ME_INIT, 8);
259
260 OUT_RING(ring, 0x0000002f);
261
262 /* Enable multiple hardware contexts */
263 OUT_RING(ring, 0x00000003);
264
265 /* Enable error detection */
266 OUT_RING(ring, 0x20000000);
267
268 /* Don't enable header dump */
269 OUT_RING(ring, 0x00000000);
270 OUT_RING(ring, 0x00000000);
271
272 /* No workarounds enabled */
273 OUT_RING(ring, 0x00000000);
274
275 /* Pad rest of the cmds with 0's */
276 OUT_RING(ring, 0x00000000);
277 OUT_RING(ring, 0x00000000);
278
279 a6xx_flush(gpu, ring);
280 return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
281}
282
283static int a6xx_ucode_init(struct msm_gpu *gpu)
284{
285 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
286 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
287
288 if (!a6xx_gpu->sqe_bo) {
289 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
290 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
291
292 if (IS_ERR(a6xx_gpu->sqe_bo)) {
293 int ret = PTR_ERR(a6xx_gpu->sqe_bo);
294
295 a6xx_gpu->sqe_bo = NULL;
296 DRM_DEV_ERROR(&gpu->pdev->dev,
297 "Could not allocate SQE ucode: %d\n", ret);
298
299 return ret;
300 }
301 }
302
303 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO,
304 REG_A6XX_CP_SQE_INSTR_BASE_HI, a6xx_gpu->sqe_iova);
305
306 return 0;
307}
308
309#define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
310 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
311 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
312 A6XX_RBBM_INT_0_MASK_CP_IB2 | \
313 A6XX_RBBM_INT_0_MASK_CP_IB1 | \
314 A6XX_RBBM_INT_0_MASK_CP_RB | \
315 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
316 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
317 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
318 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
319 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
320
321static int a6xx_hw_init(struct msm_gpu *gpu)
322{
323 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
324 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
325 int ret;
326
327 /* Make sure the GMU keeps the GPU on while we set it up */
328 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
329
330 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
331
332 /*
333 * Disable the trusted memory range - we don't actually supported secure
334 * memory rendering at this point in time and we don't want to block off
335 * part of the virtual memory space.
336 */
337 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
338 REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
339 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
340
341 /* enable hardware clockgating */
342 a6xx_set_hwcg(gpu, true);
343
344 /* VBIF start */
345 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
346 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
347
348 /* Make all blocks contribute to the GPU BUSY perf counter */
349 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
350
351 /* Disable L2 bypass in the UCHE */
352 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);
353 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);
354 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);
355 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
356 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
357 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
358
359 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
360 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
361 REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
362
363 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
364 REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
365 0x00100000 + adreno_gpu->gmem - 1);
366
367 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
368 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
369
370 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
371 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
372
373 /* Setting the mem pool size */
374 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
375
376 /* Setting the primFifo thresholds default values */
377 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));
378
379 /* Set the AHB default slave response to "ERROR" */
380 gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
381
382 /* Turn on performance counters */
383 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
384
385 /* Select CP0 to always count cycles */
386 gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
387
388 /* FIXME: not sure if this should live here or in a6xx_gmu.c */
389 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK,
390 0xff000000);
391 gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0,
392 0xff, 0x20);
393 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE,
394 0x01);
395
396 gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1);
397 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1);
398 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1);
399 gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21);
400
401 /* Enable fault detection */
402 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,
403 (1 << 30) | 0x1fffff);
404
405 gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);
406
407 /* Protect registers from the CP */
408 gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003);
409
410 gpu_write(gpu, REG_A6XX_CP_PROTECT(0),
411 A6XX_PROTECT_RDONLY(0x600, 0x51));
412 gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2));
413 gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13));
414 gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8));
415 gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1));
416 gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187));
417 gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810));
418 gpu_write(gpu, REG_A6XX_CP_PROTECT(7),
419 A6XX_PROTECT_RDONLY(0xfc00, 0x3));
420 gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0));
421 gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0));
422 gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0));
423 gpu_write(gpu, REG_A6XX_CP_PROTECT(11),
424 A6XX_PROTECT_RDONLY(0x0, 0x4f9));
425 gpu_write(gpu, REG_A6XX_CP_PROTECT(12),
426 A6XX_PROTECT_RDONLY(0x501, 0xa));
427 gpu_write(gpu, REG_A6XX_CP_PROTECT(13),
428 A6XX_PROTECT_RDONLY(0x511, 0x44));
429 gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe));
430 gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0));
431 gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf));
432 gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0));
433 gpu_write(gpu, REG_A6XX_CP_PROTECT(18),
434 A6XX_PROTECT_RW(0xbe20, 0x11f3));
435 gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82));
436 gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8));
437 gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19));
438 gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d));
439 gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76));
440 gpu_write(gpu, REG_A6XX_CP_PROTECT(24),
441 A6XX_PROTECT_RDONLY(0x8d0, 0x23));
442 gpu_write(gpu, REG_A6XX_CP_PROTECT(25),
443 A6XX_PROTECT_RDONLY(0x980, 0x4));
444 gpu_write(gpu, REG_A6XX_CP_PROTECT(26), A6XX_PROTECT_RW(0xa630, 0x0));
445
446 /* Enable interrupts */
447 gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);
448
449 ret = adreno_hw_init(gpu);
450 if (ret)
451 goto out;
452
453 ret = a6xx_ucode_init(gpu);
454 if (ret)
455 goto out;
456
457 /* Always come up on rb 0 */
458 a6xx_gpu->cur_ring = gpu->rb[0];
459
460 /* Enable the SQE_to start the CP engine */
461 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
462
463 ret = a6xx_cp_init(gpu);
464 if (ret)
465 goto out;
466
467 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
468
469out:
470 /*
471 * Tell the GMU that we are done touching the GPU and it can start power
472 * management
473 */
474 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
475
476 /* Take the GMU out of its special boot mode */
477 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
478
479 return ret;
480}
481
482static void a6xx_dump(struct msm_gpu *gpu)
483{
484 dev_info(&gpu->pdev->dev, "status: %08x\n",
485 gpu_read(gpu, REG_A6XX_RBBM_STATUS));
486 adreno_dump(gpu);
487}
488
489#define VBIF_RESET_ACK_TIMEOUT 100
490#define VBIF_RESET_ACK_MASK 0x00f0
491
492static void a6xx_recover(struct msm_gpu *gpu)
493{
494 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
495 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
496 int i;
497
498 adreno_dump_info(gpu);
499
500 for (i = 0; i < 8; i++)
501 dev_info(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
502 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
503
504 if (hang_debug)
505 a6xx_dump(gpu);
506
507 /*
508 * Turn off keep alive that might have been enabled by the hang
509 * interrupt
510 */
511 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
512
513 gpu->funcs->pm_suspend(gpu);
514 gpu->funcs->pm_resume(gpu);
515
516 msm_gpu_hw_init(gpu);
517}
518
519static int a6xx_fault_handler(void *arg, unsigned long iova, int flags)
520{
521 struct msm_gpu *gpu = arg;
522
523 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
524 iova, flags,
525 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
526 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
527 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
528 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
529
530 return -EFAULT;
531}
532
533static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
534{
535 u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
536
537 if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
538 u32 val;
539
540 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
541 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
542 dev_err_ratelimited(&gpu->pdev->dev,
543 "CP | opcode error | possible opcode=0x%8.8X\n",
544 val);
545 }
546
547 if (status & A6XX_CP_INT_CP_UCODE_ERROR)
548 dev_err_ratelimited(&gpu->pdev->dev,
549 "CP ucode error interrupt\n");
550
551 if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
552 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
553 gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
554
555 if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
556 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
557
558 dev_err_ratelimited(&gpu->pdev->dev,
559 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
560 val & (1 << 20) ? "READ" : "WRITE",
561 (val & 0x3ffff), val);
562 }
563
564 if (status & A6XX_CP_INT_CP_AHB_ERROR)
565 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
566
567 if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
568 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
569
570 if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
571 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
572
573}
574
575static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
576{
577 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
578 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
579 struct drm_device *dev = gpu->dev;
580 struct msm_drm_private *priv = dev->dev_private;
581 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
582
583 /*
584 * Force the GPU to stay on until after we finish
585 * collecting information
586 */
587 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
588
589 DRM_DEV_ERROR(&gpu->pdev->dev,
590 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
591 ring ? ring->id : -1, ring ? ring->seqno : 0,
592 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
593 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
594 gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
595 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),
596 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
597 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),
598 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
599
600 /* Turn off the hangcheck timer to keep it from bothering us */
601 del_timer(&gpu->hangcheck_timer);
602
603 queue_work(priv->wq, &gpu->recover_work);
604}
605
606static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
607{
608 u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
609
610 gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
611
612 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
613 a6xx_fault_detect_irq(gpu);
614
615 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
616 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
617
618 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
619 a6xx_cp_hw_err_irq(gpu);
620
621 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
622 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
623
624 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
625 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
626
627 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
628 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
629
630 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
631 msm_gpu_retire(gpu);
632
633 return IRQ_HANDLED;
634}
635
636static const u32 a6xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
637 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A6XX_CP_RB_BASE),
638 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A6XX_CP_RB_BASE_HI),
639 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR,
640 REG_A6XX_CP_RB_RPTR_ADDR_LO),
641 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
642 REG_A6XX_CP_RB_RPTR_ADDR_HI),
643 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A6XX_CP_RB_RPTR),
644 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A6XX_CP_RB_WPTR),
645 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A6XX_CP_RB_CNTL),
646};
647
648static const u32 a6xx_registers[] = {
649 0x0000, 0x0002, 0x0010, 0x0010, 0x0012, 0x0012, 0x0018, 0x001b,
650 0x001e, 0x0032, 0x0038, 0x003c, 0x0042, 0x0042, 0x0044, 0x0044,
651 0x0047, 0x0047, 0x0056, 0x0056, 0x00ad, 0x00ae, 0x00b0, 0x00fb,
652 0x0100, 0x011d, 0x0200, 0x020d, 0x0210, 0x0213, 0x0218, 0x023d,
653 0x0400, 0x04f9, 0x0500, 0x0500, 0x0505, 0x050b, 0x050e, 0x0511,
654 0x0533, 0x0533, 0x0540, 0x0555, 0x0800, 0x0808, 0x0810, 0x0813,
655 0x0820, 0x0821, 0x0823, 0x0827, 0x0830, 0x0833, 0x0840, 0x0843,
656 0x084f, 0x086f, 0x0880, 0x088a, 0x08a0, 0x08ab, 0x08c0, 0x08c4,
657 0x08d0, 0x08dd, 0x08f0, 0x08f3, 0x0900, 0x0903, 0x0908, 0x0911,
658 0x0928, 0x093e, 0x0942, 0x094d, 0x0980, 0x0984, 0x098d, 0x0996,
659 0x0998, 0x099e, 0x09a0, 0x09a6, 0x09a8, 0x09ae, 0x09b0, 0x09b1,
660 0x09c2, 0x09c8, 0x0a00, 0x0a03, 0x0c00, 0x0c04, 0x0c06, 0x0c06,
661 0x0c10, 0x0cd9, 0x0e00, 0x0e0e, 0x0e10, 0x0e13, 0x0e17, 0x0e19,
662 0x0e1c, 0x0e2b, 0x0e30, 0x0e32, 0x0e38, 0x0e39, 0x8600, 0x8601,
663 0x8610, 0x861b, 0x8620, 0x8620, 0x8628, 0x862b, 0x8630, 0x8637,
664 0x8e01, 0x8e01, 0x8e04, 0x8e05, 0x8e07, 0x8e08, 0x8e0c, 0x8e0c,
665 0x8e10, 0x8e1c, 0x8e20, 0x8e25, 0x8e28, 0x8e28, 0x8e2c, 0x8e2f,
666 0x8e3b, 0x8e3e, 0x8e40, 0x8e43, 0x8e50, 0x8e5e, 0x8e70, 0x8e77,
667 0x9600, 0x9604, 0x9624, 0x9637, 0x9e00, 0x9e01, 0x9e03, 0x9e0e,
668 0x9e11, 0x9e16, 0x9e19, 0x9e19, 0x9e1c, 0x9e1c, 0x9e20, 0x9e23,
669 0x9e30, 0x9e31, 0x9e34, 0x9e34, 0x9e70, 0x9e72, 0x9e78, 0x9e79,
670 0x9e80, 0x9fff, 0xa600, 0xa601, 0xa603, 0xa603, 0xa60a, 0xa60a,
671 0xa610, 0xa617, 0xa630, 0xa630,
672 ~0
673};
674
675static int a6xx_pm_resume(struct msm_gpu *gpu)
676{
677 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
678 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
679 int ret;
680
681 ret = a6xx_gmu_resume(a6xx_gpu);
682
683 gpu->needs_hw_init = true;
684
685 return ret;
686}
687
688static int a6xx_pm_suspend(struct msm_gpu *gpu)
689{
690 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
691 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
692
693 /*
694 * Make sure the GMU is idle before continuing (because some transitions
695 * may use VBIF
696 */
697 a6xx_gmu_wait_for_idle(a6xx_gpu);
698
699 /* Clear the VBIF pipe before shutting down */
700 /* FIXME: This accesses the GPU - do we need to make sure it is on? */
701 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
702 spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf);
703 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
704
705 return a6xx_gmu_stop(a6xx_gpu);
706}
707
708static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
709{
710 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
711 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
712
713 /* Force the GPU power on so we can read this register */
714 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
715
716 *value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO,
717 REG_A6XX_RBBM_PERFCTR_CP_0_HI);
718
719 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
720 return 0;
721}
722
723#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
724static void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
725 struct drm_printer *p)
726{
727 adreno_show(gpu, state, p);
728}
729#endif
730
731static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
732{
733 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
734 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
735
736 return a6xx_gpu->cur_ring;
737}
738
739static void a6xx_destroy(struct msm_gpu *gpu)
740{
741 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
742 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
743
744 if (a6xx_gpu->sqe_bo) {
745 if (a6xx_gpu->sqe_iova)
746 msm_gem_put_iova(a6xx_gpu->sqe_bo, gpu->aspace);
747 drm_gem_object_unreference_unlocked(a6xx_gpu->sqe_bo);
748 }
749
750 a6xx_gmu_remove(a6xx_gpu);
751
752 adreno_gpu_cleanup(adreno_gpu);
753 kfree(a6xx_gpu);
754}
755
756static const struct adreno_gpu_funcs funcs = {
757 .base = {
758 .get_param = adreno_get_param,
759 .hw_init = a6xx_hw_init,
760 .pm_suspend = a6xx_pm_suspend,
761 .pm_resume = a6xx_pm_resume,
762 .recover = a6xx_recover,
763 .submit = a6xx_submit,
764 .flush = a6xx_flush,
765 .active_ring = a6xx_active_ring,
766 .irq = a6xx_irq,
767 .destroy = a6xx_destroy,
768#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
769 .show = a6xx_show,
770#endif
771 },
772 .get_timestamp = a6xx_get_timestamp,
773};
774
775struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
776{
777 struct msm_drm_private *priv = dev->dev_private;
778 struct platform_device *pdev = priv->gpu_pdev;
779 struct device_node *node;
780 struct a6xx_gpu *a6xx_gpu;
781 struct adreno_gpu *adreno_gpu;
782 struct msm_gpu *gpu;
783 int ret;
784
785 a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
786 if (!a6xx_gpu)
787 return ERR_PTR(-ENOMEM);
788
789 adreno_gpu = &a6xx_gpu->base;
790 gpu = &adreno_gpu->base;
791
792 adreno_gpu->registers = a6xx_registers;
793 adreno_gpu->reg_offsets = a6xx_register_offsets;
794
795 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
796 if (ret) {
797 a6xx_destroy(&(a6xx_gpu->base.base));
798 return ERR_PTR(ret);
799 }
800
801 /* Check if there is a GMU phandle and set it up */
802 node = of_parse_phandle(pdev->dev.of_node, "gmu", 0);
803
804 /* FIXME: How do we gracefully handle this? */
805 BUG_ON(!node);
806
807 ret = a6xx_gmu_probe(a6xx_gpu, node);
808 if (ret) {
809 a6xx_destroy(&(a6xx_gpu->base.base));
810 return ERR_PTR(ret);
811 }
812
813 if (gpu->aspace)
814 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
815 a6xx_fault_handler);
816
817 return gpu;
818}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
new file mode 100644
index 000000000000..dd69e5b0e692
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -0,0 +1,60 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */
3
4#ifndef __A6XX_GPU_H__
5#define __A6XX_GPU_H__
6
7
8#include "adreno_gpu.h"
9#include "a6xx.xml.h"
10
11#include "a6xx_gmu.h"
12
13extern bool hang_debug;
14
15struct a6xx_gpu {
16 struct adreno_gpu base;
17
18 struct drm_gem_object *sqe_bo;
19 uint64_t sqe_iova;
20
21 struct msm_ringbuffer *cur_ring;
22
23 struct a6xx_gmu gmu;
24};
25
26#define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base)
27
28/*
29 * Given a register and a count, return a value to program into
30 * REG_CP_PROTECT_REG(n) - this will block both reads and writes for _len
31 * registers starting at _reg.
32 */
33#define A6XX_PROTECT_RW(_reg, _len) \
34 ((1 << 31) | \
35 (((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF))
36
37/*
38 * Same as above, but allow reads over the range. For areas of mixed use (such
39 * as performance counters) this allows us to protect a much larger range with a
40 * single register
41 */
42#define A6XX_PROTECT_RDONLY(_reg, _len) \
43 ((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF))
44
45
46int a6xx_gmu_resume(struct a6xx_gpu *gpu);
47int a6xx_gmu_stop(struct a6xx_gpu *gpu);
48
49int a6xx_gmu_wait_for_idle(struct a6xx_gpu *gpu);
50
51int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu);
52bool a6xx_gmu_isidle(struct a6xx_gmu *gmu);
53
54int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
55void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
56
57int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node);
58void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu);
59
60#endif /* __A6XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
new file mode 100644
index 000000000000..f19ef4cb6ea4
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
@@ -0,0 +1,435 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
3
4#include <linux/completion.h>
5#include <linux/circ_buf.h>
6#include <linux/list.h>
7
8#include "a6xx_gmu.h"
9#include "a6xx_gmu.xml.h"
10
11#define HFI_MSG_ID(val) [val] = #val
12
13static const char * const a6xx_hfi_msg_id[] = {
14 HFI_MSG_ID(HFI_H2F_MSG_INIT),
15 HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION),
16 HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
17 HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
18 HFI_MSG_ID(HFI_H2F_MSG_TEST),
19};
20
21static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data,
22 u32 dwords)
23{
24 struct a6xx_hfi_queue_header *header = queue->header;
25 u32 i, hdr, index = header->read_index;
26
27 if (header->read_index == header->write_index) {
28 header->rx_request = 1;
29 return 0;
30 }
31
32 hdr = queue->data[index];
33
34 /*
35 * If we are to assume that the GMU firmware is in fact a rational actor
36 * and is programmed to not send us a larger response than we expect
37 * then we can also assume that if the header size is unexpectedly large
38 * that it is due to memory corruption and/or hardware failure. In this
39 * case the only reasonable course of action is to BUG() to help harden
40 * the failure.
41 */
42
43 BUG_ON(HFI_HEADER_SIZE(hdr) > dwords);
44
45 for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {
46 data[i] = queue->data[index];
47 index = (index + 1) % header->size;
48 }
49
50 header->read_index = index;
51 return HFI_HEADER_SIZE(hdr);
52}
53
54static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
55 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
56{
57 struct a6xx_hfi_queue_header *header = queue->header;
58 u32 i, space, index = header->write_index;
59
60 spin_lock(&queue->lock);
61
62 space = CIRC_SPACE(header->write_index, header->read_index,
63 header->size);
64 if (space < dwords) {
65 header->dropped++;
66 spin_unlock(&queue->lock);
67 return -ENOSPC;
68 }
69
70 for (i = 0; i < dwords; i++) {
71 queue->data[index] = data[i];
72 index = (index + 1) % header->size;
73 }
74
75 header->write_index = index;
76 spin_unlock(&queue->lock);
77
78 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);
79 return 0;
80}
81
82struct a6xx_hfi_response {
83 u32 id;
84 u32 seqnum;
85 struct list_head node;
86 struct completion complete;
87
88 u32 error;
89 u32 payload[16];
90};
91
92/*
93 * Incoming HFI ack messages can come in out of order so we need to store all
94 * the pending messages on a list until they are handled.
95 */
96static spinlock_t hfi_ack_lock = __SPIN_LOCK_UNLOCKED(message_lock);
97static LIST_HEAD(hfi_ack_list);
98
99static void a6xx_hfi_handle_ack(struct a6xx_gmu *gmu,
100 struct a6xx_hfi_msg_response *msg)
101{
102 struct a6xx_hfi_response *resp;
103 u32 id, seqnum;
104
105 /* msg->ret_header contains the header of the message being acked */
106 id = HFI_HEADER_ID(msg->ret_header);
107 seqnum = HFI_HEADER_SEQNUM(msg->ret_header);
108
109 spin_lock(&hfi_ack_lock);
110 list_for_each_entry(resp, &hfi_ack_list, node) {
111 if (resp->id == id && resp->seqnum == seqnum) {
112 resp->error = msg->error;
113 memcpy(resp->payload, msg->payload,
114 sizeof(resp->payload));
115
116 complete(&resp->complete);
117 spin_unlock(&hfi_ack_lock);
118 return;
119 }
120 }
121 spin_unlock(&hfi_ack_lock);
122
123 dev_err(gmu->dev, "Nobody was waiting for HFI message %d\n", seqnum);
124}
125
126static void a6xx_hfi_handle_error(struct a6xx_gmu *gmu,
127 struct a6xx_hfi_msg_response *msg)
128{
129 struct a6xx_hfi_msg_error *error = (struct a6xx_hfi_msg_error *) msg;
130
131 dev_err(gmu->dev, "GMU firmware error %d\n", error->code);
132}
133
134void a6xx_hfi_task(unsigned long data)
135{
136 struct a6xx_gmu *gmu = (struct a6xx_gmu *) data;
137 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE];
138 struct a6xx_hfi_msg_response resp;
139
140 for (;;) {
141 u32 id;
142 int ret = a6xx_hfi_queue_read(queue, (u32 *) &resp,
143 sizeof(resp) >> 2);
144
145 /* Returns the number of bytes copied or negative on error */
146 if (ret <= 0) {
147 if (ret < 0)
148 dev_err(gmu->dev,
149 "Unable to read the HFI message queue\n");
150 break;
151 }
152
153 id = HFI_HEADER_ID(resp.header);
154
155 if (id == HFI_F2H_MSG_ACK)
156 a6xx_hfi_handle_ack(gmu, &resp);
157 else if (id == HFI_F2H_MSG_ERROR)
158 a6xx_hfi_handle_error(gmu, &resp);
159 }
160}
161
162static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id,
163 void *data, u32 size, u32 *payload, u32 payload_size)
164{
165 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE];
166 struct a6xx_hfi_response resp = { 0 };
167 int ret, dwords = size >> 2;
168 u32 seqnum;
169
170 seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;
171
172 /* First dword of the message is the message header - fill it in */
173 *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) |
174 (dwords << 8) | id;
175
176 init_completion(&resp.complete);
177 resp.id = id;
178 resp.seqnum = seqnum;
179
180 spin_lock_bh(&hfi_ack_lock);
181 list_add_tail(&resp.node, &hfi_ack_list);
182 spin_unlock_bh(&hfi_ack_lock);
183
184 ret = a6xx_hfi_queue_write(gmu, queue, data, dwords);
185 if (ret) {
186 dev_err(gmu->dev, "Unable to send message %s id %d\n",
187 a6xx_hfi_msg_id[id], seqnum);
188 goto out;
189 }
190
191 /* Wait up to 5 seconds for the response */
192 ret = wait_for_completion_timeout(&resp.complete,
193 msecs_to_jiffies(5000));
194 if (!ret) {
195 dev_err(gmu->dev,
196 "Message %s id %d timed out waiting for response\n",
197 a6xx_hfi_msg_id[id], seqnum);
198 ret = -ETIMEDOUT;
199 } else
200 ret = 0;
201
202out:
203 spin_lock_bh(&hfi_ack_lock);
204 list_del(&resp.node);
205 spin_unlock_bh(&hfi_ack_lock);
206
207 if (ret)
208 return ret;
209
210 if (resp.error) {
211 dev_err(gmu->dev, "Message %s id %d returned error %d\n",
212 a6xx_hfi_msg_id[id], seqnum, resp.error);
213 return -EINVAL;
214 }
215
216 if (payload && payload_size) {
217 int copy = min_t(u32, payload_size, sizeof(resp.payload));
218
219 memcpy(payload, resp.payload, copy);
220 }
221
222 return 0;
223}
224
225static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
226{
227 struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
228
229 msg.dbg_buffer_addr = (u32) gmu->debug->iova;
230 msg.dbg_buffer_size = (u32) gmu->debug->size;
231 msg.boot_state = boot_state;
232
233 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
234 NULL, 0);
235}
236
237static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
238{
239 struct a6xx_hfi_msg_fw_version msg = { 0 };
240
241 /* Currently supporting version 1.1 */
242 msg.supported_version = (1 << 28) | (1 << 16);
243
244 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg),
245 version, sizeof(*version));
246}
247
248static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
249{
250 struct a6xx_hfi_msg_perf_table msg = { 0 };
251 int i;
252
253 msg.num_gpu_levels = gmu->nr_gpu_freqs;
254 msg.num_gmu_levels = gmu->nr_gmu_freqs;
255
256 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
257 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
258 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
259 }
260
261 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
262 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
263 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
264 }
265
266 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
267 NULL, 0);
268}
269
270static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
271{
272 struct a6xx_hfi_msg_bw_table msg = { 0 };
273
274 /*
275 * The sdm845 GMU doesn't do bus frequency scaling on its own but it
276 * does need at least one entry in the list because it might be accessed
277 * when the GMU is shutting down. Send a single "off" entry.
278 */
279
280 msg.bw_level_num = 1;
281
282 msg.ddr_cmds_num = 3;
283 msg.ddr_wait_bitmask = 0x07;
284
285 msg.ddr_cmds_addrs[0] = 0x50000;
286 msg.ddr_cmds_addrs[1] = 0x5005c;
287 msg.ddr_cmds_addrs[2] = 0x5000c;
288
289 msg.ddr_cmds_data[0][0] = 0x40000000;
290 msg.ddr_cmds_data[0][1] = 0x40000000;
291 msg.ddr_cmds_data[0][2] = 0x40000000;
292
293 /*
294 * These are the CX (CNOC) votes. This is used but the values for the
295 * sdm845 GMU are known and fixed so we can hard code them.
296 */
297
298 msg.cnoc_cmds_num = 3;
299 msg.cnoc_wait_bitmask = 0x05;
300
301 msg.cnoc_cmds_addrs[0] = 0x50034;
302 msg.cnoc_cmds_addrs[1] = 0x5007c;
303 msg.cnoc_cmds_addrs[2] = 0x5004c;
304
305 msg.cnoc_cmds_data[0][0] = 0x40000000;
306 msg.cnoc_cmds_data[0][1] = 0x00000000;
307 msg.cnoc_cmds_data[0][2] = 0x40000000;
308
309 msg.cnoc_cmds_data[1][0] = 0x60000001;
310 msg.cnoc_cmds_data[1][1] = 0x20000001;
311 msg.cnoc_cmds_data[1][2] = 0x60000001;
312
313 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
314 NULL, 0);
315}
316
317static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
318{
319 struct a6xx_hfi_msg_test msg = { 0 };
320
321 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg),
322 NULL, 0);
323}
324
325int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
326{
327 int ret;
328
329 ret = a6xx_hfi_send_gmu_init(gmu, boot_state);
330 if (ret)
331 return ret;
332
333 ret = a6xx_hfi_get_fw_version(gmu, NULL);
334 if (ret)
335 return ret;
336
337 /*
338 * We have to get exchange version numbers per the sequence but at this
339 * point th kernel driver doesn't need to know the exact version of
340 * the GMU firmware
341 */
342
343 ret = a6xx_hfi_send_perf_table(gmu);
344 if (ret)
345 return ret;
346
347 ret = a6xx_hfi_send_bw_table(gmu);
348 if (ret)
349 return ret;
350
351 /*
352 * Let the GMU know that there won't be any more HFI messages until next
353 * boot
354 */
355 a6xx_hfi_send_test(gmu);
356
357 return 0;
358}
359
360void a6xx_hfi_stop(struct a6xx_gmu *gmu)
361{
362 int i;
363
364 for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
365 struct a6xx_hfi_queue *queue = &gmu->queues[i];
366
367 if (!queue->header)
368 continue;
369
370 if (queue->header->read_index != queue->header->write_index)
371 dev_err(gmu->dev, "HFI queue %d is not empty\n", i);
372
373 queue->header->read_index = 0;
374 queue->header->write_index = 0;
375 }
376}
377
378static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
379 struct a6xx_hfi_queue_header *header, void *virt, u64 iova,
380 u32 id)
381{
382 spin_lock_init(&queue->lock);
383 queue->header = header;
384 queue->data = virt;
385 atomic_set(&queue->seqnum, 0);
386
387 /* Set up the shared memory header */
388 header->iova = iova;
389 header->type = 10 << 8 | id;
390 header->status = 1;
391 header->size = SZ_4K >> 2;
392 header->msg_size = 0;
393 header->dropped = 0;
394 header->rx_watermark = 1;
395 header->tx_watermark = 1;
396 header->rx_request = 1;
397 header->tx_request = 0;
398 header->read_index = 0;
399 header->write_index = 0;
400}
401
402void a6xx_hfi_init(struct a6xx_gmu *gmu)
403{
404 struct a6xx_gmu_bo *hfi = gmu->hfi;
405 struct a6xx_hfi_queue_table_header *table = hfi->virt;
406 struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
407 u64 offset;
408 int table_size;
409
410 /*
411 * The table size is the size of the table header plus all of the queue
412 * headers
413 */
414 table_size = sizeof(*table);
415 table_size += (ARRAY_SIZE(gmu->queues) *
416 sizeof(struct a6xx_hfi_queue_header));
417
418 table->version = 0;
419 table->size = table_size;
420 /* First queue header is located immediately after the table header */
421 table->qhdr0_offset = sizeof(*table) >> 2;
422 table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2;
423 table->num_queues = ARRAY_SIZE(gmu->queues);
424 table->active_queues = ARRAY_SIZE(gmu->queues);
425
426 /* Command queue */
427 offset = SZ_4K;
428 a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset,
429 hfi->iova + offset, 0);
430
431 /* GMU response queue */
432 offset += SZ_4K;
433 a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset,
434 hfi->iova + offset, 4);
435}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
new file mode 100644
index 000000000000..60d1319fa44f
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
@@ -0,0 +1,127 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */
3
4#ifndef _A6XX_HFI_H_
5#define _A6XX_HFI_H_
6
7struct a6xx_hfi_queue_table_header {
8 u32 version;
9 u32 size; /* Size of the queue table in dwords */
10 u32 qhdr0_offset; /* Offset of the first queue header */
11 u32 qhdr_size; /* Size of the queue headers */
12 u32 num_queues; /* Number of total queues */
13 u32 active_queues; /* Number of active queues */
14};
15
16struct a6xx_hfi_queue_header {
17 u32 status;
18 u32 iova;
19 u32 type;
20 u32 size;
21 u32 msg_size;
22 u32 dropped;
23 u32 rx_watermark;
24 u32 tx_watermark;
25 u32 rx_request;
26 u32 tx_request;
27 u32 read_index;
28 u32 write_index;
29};
30
31struct a6xx_hfi_queue {
32 struct a6xx_hfi_queue_header *header;
33 spinlock_t lock;
34 u32 *data;
35 atomic_t seqnum;
36};
37
38/* This is the outgoing queue to the GMU */
39#define HFI_COMMAND_QUEUE 0
40
41/* THis is the incoming response queue from the GMU */
42#define HFI_RESPONSE_QUEUE 1
43
44#define HFI_HEADER_ID(msg) ((msg) & 0xff)
45#define HFI_HEADER_SIZE(msg) (((msg) >> 8) & 0xff)
46#define HFI_HEADER_SEQNUM(msg) (((msg) >> 20) & 0xfff)
47
48/* FIXME: Do we need this or can we use ARRAY_SIZE? */
49#define HFI_RESPONSE_PAYLOAD_SIZE 16
50
51/* HFI message types */
52
53#define HFI_MSG_CMD 0
54#define HFI_MSG_ACK 2
55
56#define HFI_F2H_MSG_ACK 126
57
58struct a6xx_hfi_msg_response {
59 u32 header;
60 u32 ret_header;
61 u32 error;
62 u32 payload[HFI_RESPONSE_PAYLOAD_SIZE];
63};
64
65#define HFI_F2H_MSG_ERROR 100
66
67struct a6xx_hfi_msg_error {
68 u32 header;
69 u32 code;
70 u32 payload[2];
71};
72
73#define HFI_H2F_MSG_INIT 0
74
75struct a6xx_hfi_msg_gmu_init_cmd {
76 u32 header;
77 u32 seg_id;
78 u32 dbg_buffer_addr;
79 u32 dbg_buffer_size;
80 u32 boot_state;
81};
82
83#define HFI_H2F_MSG_FW_VERSION 1
84
85struct a6xx_hfi_msg_fw_version {
86 u32 header;
87 u32 supported_version;
88};
89
90#define HFI_H2F_MSG_PERF_TABLE 4
91
92struct perf_level {
93 u32 vote;
94 u32 freq;
95};
96
97struct a6xx_hfi_msg_perf_table {
98 u32 header;
99 u32 num_gpu_levels;
100 u32 num_gmu_levels;
101
102 struct perf_level gx_votes[16];
103 struct perf_level cx_votes[4];
104};
105
106#define HFI_H2F_MSG_BW_TABLE 3
107
108struct a6xx_hfi_msg_bw_table {
109 u32 header;
110 u32 bw_level_num;
111 u32 cnoc_cmds_num;
112 u32 ddr_cmds_num;
113 u32 cnoc_wait_bitmask;
114 u32 ddr_wait_bitmask;
115 u32 cnoc_cmds_addrs[6];
116 u32 cnoc_cmds_data[2][6];
117 u32 ddr_cmds_addrs[8];
118 u32 ddr_cmds_data[16][8];
119};
120
121#define HFI_H2F_MSG_TEST 5
122
123struct a6xx_hfi_msg_test {
124 u32 header;
125};
126
127#endif
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 37746f1d54cf..7d3e9a129ac7 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -111,6 +111,16 @@ static const struct adreno_info gpulist[] = {
111 ADRENO_QUIRK_FAULT_DETECT_MASK, 111 ADRENO_QUIRK_FAULT_DETECT_MASK,
112 .init = a5xx_gpu_init, 112 .init = a5xx_gpu_init,
113 .zapfw = "a530_zap.mdt", 113 .zapfw = "a530_zap.mdt",
114 }, {
115 .rev = ADRENO_REV(6, 3, 0, ANY_ID),
116 .revn = 630,
117 .name = "A630",
118 .fw = {
119 [ADRENO_FW_SQE] = "a630_sqe.fw",
120 [ADRENO_FW_GMU] = "a630_gmu.bin",
121 },
122 .gmem = SZ_1M,
123 .init = a6xx_gpu_init,
114 }, 124 },
115}; 125};
116 126
@@ -127,6 +137,8 @@ MODULE_FIRMWARE("qcom/a530_zap.mdt");
127MODULE_FIRMWARE("qcom/a530_zap.b00"); 137MODULE_FIRMWARE("qcom/a530_zap.b00");
128MODULE_FIRMWARE("qcom/a530_zap.b01"); 138MODULE_FIRMWARE("qcom/a530_zap.b01");
129MODULE_FIRMWARE("qcom/a530_zap.b02"); 139MODULE_FIRMWARE("qcom/a530_zap.b02");
140MODULE_FIRMWARE("qcom/a630_sqe.fw");
141MODULE_FIRMWARE("qcom/a630_gmu.bin");
130 142
131static inline bool _rev_match(uint8_t entry, uint8_t id) 143static inline bool _rev_match(uint8_t entry, uint8_t id)
132{ 144{
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index d391ff377612..de6e6ee42fba 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -50,7 +50,9 @@ enum adreno_regs {
50 50
51enum { 51enum {
52 ADRENO_FW_PM4 = 0, 52 ADRENO_FW_PM4 = 0,
53 ADRENO_FW_SQE = 0, /* a6xx */
53 ADRENO_FW_PFP = 1, 54 ADRENO_FW_PFP = 1,
55 ADRENO_FW_GMU = 1, /* a6xx */
54 ADRENO_FW_GPMU = 2, 56 ADRENO_FW_GPMU = 2,
55 ADRENO_FW_MAX, 57 ADRENO_FW_MAX,
56}; 58};
@@ -335,6 +337,7 @@ static inline void adreno_gpu_write(struct adreno_gpu *gpu,
335struct msm_gpu *a3xx_gpu_init(struct drm_device *dev); 337struct msm_gpu *a3xx_gpu_init(struct drm_device *dev);
336struct msm_gpu *a4xx_gpu_init(struct drm_device *dev); 338struct msm_gpu *a4xx_gpu_init(struct drm_device *dev);
337struct msm_gpu *a5xx_gpu_init(struct drm_device *dev); 339struct msm_gpu *a5xx_gpu_init(struct drm_device *dev);
340struct msm_gpu *a6xx_gpu_init(struct drm_device *dev);
338 341
339static inline void adreno_gpu_write64(struct adreno_gpu *gpu, 342static inline void adreno_gpu_write64(struct adreno_gpu *gpu,
340 enum adreno_regs lo, enum adreno_regs hi, u64 data) 343 enum adreno_regs lo, enum adreno_regs hi, u64 data)
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index ca368490b3ee..5e808cfec345 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -88,7 +88,7 @@ static struct devfreq_dev_profile msm_devfreq_profile = {
88static void msm_devfreq_init(struct msm_gpu *gpu) 88static void msm_devfreq_init(struct msm_gpu *gpu)
89{ 89{
90 /* We need target support to do devfreq */ 90 /* We need target support to do devfreq */
91 if (!gpu->funcs->gpu_busy) 91 if (!gpu->funcs->gpu_busy || !gpu->core_clk)
92 return; 92 return;
93 93
94 msm_devfreq_profile.initial_freq = gpu->fast_rate; 94 msm_devfreq_profile.initial_freq = gpu->fast_rate;
generated by cgit v1.2.2 (git 2.25.0) at 2026-07-08 08:14:03 -0400