diff options
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | 4 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.c | 378 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/channel_gk20a.h | 16 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 344 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | 18 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.h | 6 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gm20b/fifo_gm20b.c | 22 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gp10b/fifo_gp10b.c | 7 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/vgpu/fifo_vgpu.c | 3 |
9 files changed, 395 insertions, 403 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c index 5dfd2309..418572a1 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | |||
@@ -488,7 +488,7 @@ u32 gk20a_ce_create_context_with_cb(struct device *dev, | |||
488 | 488 | ||
489 | /* -1 means default channel priority */ | 489 | /* -1 means default channel priority */ |
490 | if (priority != -1) { | 490 | if (priority != -1) { |
491 | err = gk20a_channel_set_priority(ce_ctx->ch, priority); | 491 | err = gk20a_fifo_set_priority(ce_ctx->ch, priority); |
492 | if (err) { | 492 | if (err) { |
493 | gk20a_err(ce_ctx->dev, | 493 | gk20a_err(ce_ctx->dev, |
494 | "ce: could not set the channel priority for CE context"); | 494 | "ce: could not set the channel priority for CE context"); |
@@ -498,7 +498,7 @@ u32 gk20a_ce_create_context_with_cb(struct device *dev, | |||
498 | 498 | ||
499 | /* -1 means default channel timeslice value */ | 499 | /* -1 means default channel timeslice value */ |
500 | if (timeslice != -1) { | 500 | if (timeslice != -1) { |
501 | err = gk20a_channel_set_timeslice(ce_ctx->ch, timeslice); | 501 | err = gk20a_fifo_set_timeslice(ce_ctx->ch, timeslice); |
502 | if (err) { | 502 | if (err) { |
503 | gk20a_err(ce_ctx->dev, | 503 | gk20a_err(ce_ctx->dev, |
504 | "ce: could not set the channel timeslice value for CE context"); | 504 | "ce: could not set the channel timeslice value for CE context"); |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index 88495bde..2facb595 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c | |||
@@ -38,11 +38,7 @@ | |||
38 | #include "dbg_gpu_gk20a.h" | 38 | #include "dbg_gpu_gk20a.h" |
39 | #include "fence_gk20a.h" | 39 | #include "fence_gk20a.h" |
40 | 40 | ||
41 | #include <nvgpu/hw/gk20a/hw_ram_gk20a.h> | ||
42 | #include <nvgpu/hw/gk20a/hw_fifo_gk20a.h> | ||
43 | #include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> | 41 | #include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> |
44 | #include <nvgpu/hw/gk20a/hw_ccsr_gk20a.h> | ||
45 | #include <nvgpu/hw/gk20a/hw_ltc_gk20a.h> | ||
46 | 42 | ||
47 | #define NVMAP_HANDLE_PARAM_SIZE 1 | 43 | #define NVMAP_HANDLE_PARAM_SIZE 1 |
48 | 44 | ||
@@ -78,11 +74,6 @@ static void channel_gk20a_joblist_delete(struct channel_gk20a *c, | |||
78 | static struct channel_gk20a_job *channel_gk20a_joblist_peek( | 74 | static struct channel_gk20a_job *channel_gk20a_joblist_peek( |
79 | struct channel_gk20a *c); | 75 | struct channel_gk20a *c); |
80 | 76 | ||
81 | static int channel_gk20a_commit_userd(struct channel_gk20a *c); | ||
82 | static int channel_gk20a_setup_userd(struct channel_gk20a *c); | ||
83 | |||
84 | static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a); | ||
85 | |||
86 | static int channel_gk20a_update_runlist(struct channel_gk20a *c, | 77 | static int channel_gk20a_update_runlist(struct channel_gk20a *c, |
87 | bool add); | 78 | bool add); |
88 | static void gk20a_free_error_notifiers(struct channel_gk20a *ch); | 79 | static void gk20a_free_error_notifiers(struct channel_gk20a *ch); |
@@ -159,34 +150,6 @@ int channel_gk20a_commit_va(struct channel_gk20a *c) | |||
159 | return 0; | 150 | return 0; |
160 | } | 151 | } |
161 | 152 | ||
162 | static int channel_gk20a_commit_userd(struct channel_gk20a *c) | ||
163 | { | ||
164 | u32 addr_lo; | ||
165 | u32 addr_hi; | ||
166 | struct gk20a *g = c->g; | ||
167 | |||
168 | gk20a_dbg_fn(""); | ||
169 | |||
170 | addr_lo = u64_lo32(c->userd_iova >> ram_userd_base_shift_v()); | ||
171 | addr_hi = u64_hi32(c->userd_iova); | ||
172 | |||
173 | gk20a_dbg_info("channel %d : set ramfc userd 0x%16llx", | ||
174 | c->hw_chid, (u64)c->userd_iova); | ||
175 | |||
176 | gk20a_mem_wr32(g, &c->inst_block, | ||
177 | ram_in_ramfc_w() + ram_fc_userd_w(), | ||
178 | gk20a_aperture_mask(g, &g->fifo.userd, | ||
179 | pbdma_userd_target_sys_mem_ncoh_f(), | ||
180 | pbdma_userd_target_vid_mem_f()) | | ||
181 | pbdma_userd_addr_f(addr_lo)); | ||
182 | |||
183 | gk20a_mem_wr32(g, &c->inst_block, | ||
184 | ram_in_ramfc_w() + ram_fc_userd_hi_w(), | ||
185 | pbdma_userd_hi_addr_f(addr_hi)); | ||
186 | |||
187 | return 0; | ||
188 | } | ||
189 | |||
190 | int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g, | 153 | int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g, |
191 | int timeslice_period, | 154 | int timeslice_period, |
192 | int *__timeslice_timeout, int *__timeslice_scale) | 155 | int *__timeslice_timeout, int *__timeslice_scale) |
@@ -215,255 +178,11 @@ int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g, | |||
215 | return 0; | 178 | return 0; |
216 | } | 179 | } |
217 | 180 | ||
218 | static int channel_gk20a_set_schedule_params(struct channel_gk20a *c) | ||
219 | { | ||
220 | int shift = 0, value = 0; | ||
221 | |||
222 | gk20a_channel_get_timescale_from_timeslice(c->g, | ||
223 | c->timeslice_us, &value, &shift); | ||
224 | |||
225 | /* disable channel */ | ||
226 | c->g->ops.fifo.disable_channel(c); | ||
227 | |||
228 | /* preempt the channel */ | ||
229 | WARN_ON(c->g->ops.fifo.preempt_channel(c->g, c->hw_chid)); | ||
230 | |||
231 | /* set new timeslice */ | ||
232 | gk20a_mem_wr32(c->g, &c->inst_block, ram_fc_runlist_timeslice_w(), | ||
233 | value | (shift << 12) | | ||
234 | fifo_runlist_timeslice_enable_true_f()); | ||
235 | |||
236 | /* enable channel */ | ||
237 | gk20a_writel(c->g, ccsr_channel_r(c->hw_chid), | ||
238 | gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) | | ||
239 | ccsr_channel_enable_set_true_f()); | ||
240 | |||
241 | return 0; | ||
242 | } | ||
243 | |||
244 | u32 channel_gk20a_pbdma_acquire_val(struct channel_gk20a *c) | ||
245 | { | ||
246 | u32 val, exp, man; | ||
247 | u64 timeout; | ||
248 | unsigned int val_len; | ||
249 | |||
250 | val = pbdma_acquire_retry_man_2_f() | | ||
251 | pbdma_acquire_retry_exp_2_f(); | ||
252 | |||
253 | if (!c->g->timeouts_enabled || !c->wdt_enabled) | ||
254 | return val; | ||
255 | |||
256 | timeout = gk20a_get_channel_watchdog_timeout(c); | ||
257 | timeout *= 80UL; | ||
258 | do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */ | ||
259 | timeout *= 1000000UL; /* ms -> ns */ | ||
260 | do_div(timeout, 1024); /* in unit of 1024ns */ | ||
261 | val_len = fls(timeout >> 32) + 32; | ||
262 | if (val_len == 32) | ||
263 | val_len = fls(timeout); | ||
264 | if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */ | ||
265 | exp = pbdma_acquire_timeout_exp_max_v(); | ||
266 | man = pbdma_acquire_timeout_man_max_v(); | ||
267 | } else if (val_len > 16) { | ||
268 | exp = val_len - 16; | ||
269 | man = timeout >> exp; | ||
270 | } else { | ||
271 | exp = 0; | ||
272 | man = timeout; | ||
273 | } | ||
274 | |||
275 | val |= pbdma_acquire_timeout_exp_f(exp) | | ||
276 | pbdma_acquire_timeout_man_f(man) | | ||
277 | pbdma_acquire_timeout_en_enable_f(); | ||
278 | |||
279 | return val; | ||
280 | } | ||
281 | |||
282 | void gk20a_channel_setup_ramfc_for_privileged_channel(struct channel_gk20a *c) | ||
283 | { | ||
284 | struct gk20a *g = c->g; | ||
285 | struct mem_desc *mem = &c->inst_block; | ||
286 | |||
287 | gk20a_dbg_info("channel %d : set ramfc privileged_channel", c->hw_chid); | ||
288 | |||
289 | /* Enable HCE priv mode for phys mode transfer */ | ||
290 | gk20a_mem_wr32(g, mem, ram_fc_hce_ctrl_w(), | ||
291 | pbdma_hce_ctrl_hce_priv_mode_yes_f()); | ||
292 | } | ||
293 | |||
294 | int channel_gk20a_setup_ramfc(struct channel_gk20a *c, | ||
295 | u64 gpfifo_base, u32 gpfifo_entries, u32 flags) | ||
296 | { | ||
297 | struct gk20a *g = c->g; | ||
298 | struct mem_desc *mem = &c->inst_block; | ||
299 | |||
300 | gk20a_dbg_fn(""); | ||
301 | |||
302 | gk20a_memset(g, mem, 0, 0, ram_fc_size_val_v()); | ||
303 | |||
304 | gk20a_mem_wr32(g, mem, ram_fc_gp_base_w(), | ||
305 | pbdma_gp_base_offset_f( | ||
306 | u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s()))); | ||
307 | |||
308 | gk20a_mem_wr32(g, mem, ram_fc_gp_base_hi_w(), | ||
309 | pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) | | ||
310 | pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries))); | ||
311 | |||
312 | gk20a_mem_wr32(g, mem, ram_fc_signature_w(), | ||
313 | c->g->ops.fifo.get_pbdma_signature(c->g)); | ||
314 | |||
315 | gk20a_mem_wr32(g, mem, ram_fc_formats_w(), | ||
316 | pbdma_formats_gp_fermi0_f() | | ||
317 | pbdma_formats_pb_fermi1_f() | | ||
318 | pbdma_formats_mp_fermi0_f()); | ||
319 | |||
320 | gk20a_mem_wr32(g, mem, ram_fc_pb_header_w(), | ||
321 | pbdma_pb_header_priv_user_f() | | ||
322 | pbdma_pb_header_method_zero_f() | | ||
323 | pbdma_pb_header_subchannel_zero_f() | | ||
324 | pbdma_pb_header_level_main_f() | | ||
325 | pbdma_pb_header_first_true_f() | | ||
326 | pbdma_pb_header_type_inc_f()); | ||
327 | |||
328 | gk20a_mem_wr32(g, mem, ram_fc_subdevice_w(), | ||
329 | pbdma_subdevice_id_f(1) | | ||
330 | pbdma_subdevice_status_active_f() | | ||
331 | pbdma_subdevice_channel_dma_enable_f()); | ||
332 | |||
333 | gk20a_mem_wr32(g, mem, ram_fc_target_w(), pbdma_target_engine_sw_f()); | ||
334 | |||
335 | gk20a_mem_wr32(g, mem, ram_fc_acquire_w(), | ||
336 | channel_gk20a_pbdma_acquire_val(c)); | ||
337 | |||
338 | gk20a_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(), | ||
339 | fifo_runlist_timeslice_timeout_128_f() | | ||
340 | fifo_runlist_timeslice_timescale_3_f() | | ||
341 | fifo_runlist_timeslice_enable_true_f()); | ||
342 | |||
343 | gk20a_mem_wr32(g, mem, ram_fc_pb_timeslice_w(), | ||
344 | fifo_pb_timeslice_timeout_16_f() | | ||
345 | fifo_pb_timeslice_timescale_0_f() | | ||
346 | fifo_pb_timeslice_enable_true_f()); | ||
347 | |||
348 | gk20a_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->hw_chid)); | ||
349 | |||
350 | if (c->is_privileged_channel) | ||
351 | gk20a_channel_setup_ramfc_for_privileged_channel(c); | ||
352 | |||
353 | return channel_gk20a_commit_userd(c); | ||
354 | } | ||
355 | |||
356 | static int channel_gk20a_setup_userd(struct channel_gk20a *c) | ||
357 | { | ||
358 | struct gk20a *g = c->g; | ||
359 | struct mem_desc *mem = &g->fifo.userd; | ||
360 | u32 offset = c->hw_chid * g->fifo.userd_entry_size / sizeof(u32); | ||
361 | |||
362 | gk20a_dbg_fn(""); | ||
363 | |||
364 | gk20a_mem_wr32(g, mem, offset + ram_userd_put_w(), 0); | ||
365 | gk20a_mem_wr32(g, mem, offset + ram_userd_get_w(), 0); | ||
366 | gk20a_mem_wr32(g, mem, offset + ram_userd_ref_w(), 0); | ||
367 | gk20a_mem_wr32(g, mem, offset + ram_userd_put_hi_w(), 0); | ||
368 | gk20a_mem_wr32(g, mem, offset + ram_userd_ref_threshold_w(), 0); | ||
369 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_top_level_get_w(), 0); | ||
370 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_top_level_get_hi_w(), 0); | ||
371 | gk20a_mem_wr32(g, mem, offset + ram_userd_get_hi_w(), 0); | ||
372 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_get_w(), 0); | ||
373 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_put_w(), 0); | ||
374 | |||
375 | return 0; | ||
376 | } | ||
377 | |||
378 | static void channel_gk20a_bind(struct channel_gk20a *c) | ||
379 | { | ||
380 | struct gk20a *g = c->g; | ||
381 | u32 inst_ptr = gk20a_mm_inst_block_addr(g, &c->inst_block) | ||
382 | >> ram_in_base_shift_v(); | ||
383 | |||
384 | gk20a_dbg_info("bind channel %d inst ptr 0x%08x", | ||
385 | c->hw_chid, inst_ptr); | ||
386 | |||
387 | |||
388 | gk20a_writel(g, ccsr_channel_r(c->hw_chid), | ||
389 | (gk20a_readl(g, ccsr_channel_r(c->hw_chid)) & | ||
390 | ~ccsr_channel_runlist_f(~0)) | | ||
391 | ccsr_channel_runlist_f(c->runlist_id)); | ||
392 | |||
393 | gk20a_writel(g, ccsr_channel_inst_r(c->hw_chid), | ||
394 | ccsr_channel_inst_ptr_f(inst_ptr) | | ||
395 | gk20a_aperture_mask(g, &c->inst_block, | ||
396 | ccsr_channel_inst_target_sys_mem_ncoh_f(), | ||
397 | ccsr_channel_inst_target_vid_mem_f()) | | ||
398 | ccsr_channel_inst_bind_true_f()); | ||
399 | |||
400 | gk20a_writel(g, ccsr_channel_r(c->hw_chid), | ||
401 | (gk20a_readl(g, ccsr_channel_r(c->hw_chid)) & | ||
402 | ~ccsr_channel_enable_set_f(~0)) | | ||
403 | ccsr_channel_enable_set_true_f()); | ||
404 | |||
405 | wmb(); | ||
406 | atomic_set(&c->bound, true); | ||
407 | |||
408 | } | ||
409 | |||
410 | void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a) | ||
411 | { | ||
412 | struct gk20a *g = ch_gk20a->g; | ||
413 | |||
414 | gk20a_dbg_fn(""); | ||
415 | |||
416 | if (atomic_cmpxchg(&ch_gk20a->bound, true, false)) { | ||
417 | gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid), | ||
418 | ccsr_channel_inst_ptr_f(0) | | ||
419 | ccsr_channel_inst_bind_false_f()); | ||
420 | } | ||
421 | } | ||
422 | |||
423 | int channel_gk20a_alloc_inst(struct gk20a *g, struct channel_gk20a *ch) | ||
424 | { | ||
425 | int err; | ||
426 | |||
427 | gk20a_dbg_fn(""); | ||
428 | |||
429 | err = gk20a_alloc_inst_block(g, &ch->inst_block); | ||
430 | if (err) | ||
431 | return err; | ||
432 | |||
433 | gk20a_dbg_info("channel %d inst block physical addr: 0x%16llx", | ||
434 | ch->hw_chid, gk20a_mm_inst_block_addr(g, &ch->inst_block)); | ||
435 | |||
436 | gk20a_dbg_fn("done"); | ||
437 | return 0; | ||
438 | } | ||
439 | |||
440 | void channel_gk20a_free_inst(struct gk20a *g, struct channel_gk20a *ch) | ||
441 | { | ||
442 | gk20a_free_inst_block(g, &ch->inst_block); | ||
443 | } | ||
444 | |||
445 | static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add) | 181 | static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add) |
446 | { | 182 | { |
447 | return c->g->ops.fifo.update_runlist(c->g, c->runlist_id, c->hw_chid, add, true); | 183 | return c->g->ops.fifo.update_runlist(c->g, c->runlist_id, c->hw_chid, add, true); |
448 | } | 184 | } |
449 | 185 | ||
450 | void channel_gk20a_enable(struct channel_gk20a *ch) | ||
451 | { | ||
452 | /* enable channel */ | ||
453 | gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid), | ||
454 | gk20a_readl(ch->g, ccsr_channel_r(ch->hw_chid)) | | ||
455 | ccsr_channel_enable_set_true_f()); | ||
456 | } | ||
457 | |||
458 | void channel_gk20a_disable(struct channel_gk20a *ch) | ||
459 | { | ||
460 | /* disable channel */ | ||
461 | gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid), | ||
462 | gk20a_readl(ch->g, | ||
463 | ccsr_channel_r(ch->hw_chid)) | | ||
464 | ccsr_channel_enable_clr_true_f()); | ||
465 | } | ||
466 | |||
467 | int gk20a_enable_channel_tsg(struct gk20a *g, struct channel_gk20a *ch) | 186 | int gk20a_enable_channel_tsg(struct gk20a *g, struct channel_gk20a *ch) |
468 | { | 187 | { |
469 | struct tsg_gk20a *tsg; | 188 | struct tsg_gk20a *tsg; |
@@ -991,8 +710,6 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) | |||
991 | 710 | ||
992 | gk20a_gr_flush_channel_tlb(gr); | 711 | gk20a_gr_flush_channel_tlb(gr); |
993 | 712 | ||
994 | memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub)); | ||
995 | |||
996 | gk20a_gmmu_unmap_free(ch_vm, &ch->gpfifo.mem); | 713 | gk20a_gmmu_unmap_free(ch_vm, &ch->gpfifo.mem); |
997 | nvgpu_big_free(g, ch->gpfifo.pipe); | 714 | nvgpu_big_free(g, ch->gpfifo.pipe); |
998 | memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc)); | 715 | memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc)); |
@@ -1834,6 +1551,7 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, | |||
1834 | struct vm_gk20a *ch_vm; | 1551 | struct vm_gk20a *ch_vm; |
1835 | u32 gpfifo_size; | 1552 | u32 gpfifo_size; |
1836 | int err = 0; | 1553 | int err = 0; |
1554 | unsigned long acquire_timeout; | ||
1837 | 1555 | ||
1838 | gpfifo_size = args->num_entries; | 1556 | gpfifo_size = args->num_entries; |
1839 | 1557 | ||
@@ -1852,9 +1570,6 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, | |||
1852 | } | 1570 | } |
1853 | ch_vm = c->vm; | 1571 | ch_vm = c->vm; |
1854 | 1572 | ||
1855 | c->ramfc.offset = 0; | ||
1856 | c->ramfc.size = ram_in_ramfc_s() / 8; | ||
1857 | |||
1858 | if (c->gpfifo.mem.size) { | 1573 | if (c->gpfifo.mem.size) { |
1859 | gk20a_err(d, "channel %d :" | 1574 | gk20a_err(d, "channel %d :" |
1860 | "gpfifo already allocated", c->hw_chid); | 1575 | "gpfifo already allocated", c->hw_chid); |
@@ -1884,7 +1599,7 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, | |||
1884 | gk20a_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d", | 1599 | gk20a_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d", |
1885 | c->hw_chid, c->gpfifo.mem.gpu_va, c->gpfifo.entry_num); | 1600 | c->hw_chid, c->gpfifo.mem.gpu_va, c->gpfifo.entry_num); |
1886 | 1601 | ||
1887 | channel_gk20a_setup_userd(c); | 1602 | g->ops.fifo.setup_userd(c); |
1888 | 1603 | ||
1889 | if (!platform->aggressive_sync_destroy_thresh) { | 1604 | if (!platform->aggressive_sync_destroy_thresh) { |
1890 | nvgpu_mutex_acquire(&c->sync_lock); | 1605 | nvgpu_mutex_acquire(&c->sync_lock); |
@@ -1903,8 +1618,14 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, | |||
1903 | } | 1618 | } |
1904 | } | 1619 | } |
1905 | 1620 | ||
1621 | if (!c->g->timeouts_enabled || !c->wdt_enabled) | ||
1622 | acquire_timeout = 0; | ||
1623 | else | ||
1624 | acquire_timeout = gk20a_get_channel_watchdog_timeout(c); | ||
1625 | |||
1906 | err = g->ops.fifo.setup_ramfc(c, c->gpfifo.mem.gpu_va, | 1626 | err = g->ops.fifo.setup_ramfc(c, c->gpfifo.mem.gpu_va, |
1907 | c->gpfifo.entry_num, args->flags); | 1627 | c->gpfifo.entry_num, |
1628 | acquire_timeout, args->flags); | ||
1908 | if (err) | 1629 | if (err) |
1909 | goto clean_up_sync; | 1630 | goto clean_up_sync; |
1910 | 1631 | ||
@@ -1949,19 +1670,6 @@ clean_up: | |||
1949 | return err; | 1670 | return err; |
1950 | } | 1671 | } |
1951 | 1672 | ||
1952 | u32 gk20a_userd_gp_get(struct gk20a *g, struct channel_gk20a *c) | ||
1953 | { | ||
1954 | return gk20a_bar1_readl(g, | ||
1955 | c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w()); | ||
1956 | } | ||
1957 | |||
1958 | void gk20a_userd_gp_put(struct gk20a *g, struct channel_gk20a *c) | ||
1959 | { | ||
1960 | gk20a_bar1_writel(g, | ||
1961 | c->userd_gpu_va + sizeof(u32) * ram_userd_gp_put_w(), | ||
1962 | c->gpfifo.put); | ||
1963 | } | ||
1964 | |||
1965 | /* Update with this periodically to determine how the gpfifo is draining. */ | 1673 | /* Update with this periodically to determine how the gpfifo is draining. */ |
1966 | static inline u32 update_gp_get(struct gk20a *g, | 1674 | static inline u32 update_gp_get(struct gk20a *g, |
1967 | struct channel_gk20a *c) | 1675 | struct channel_gk20a *c) |
@@ -2093,7 +1801,7 @@ static void trace_write_pushbuffer_range(struct channel_gk20a *c, | |||
2093 | 1801 | ||
2094 | static void __gk20a_channel_timeout_start(struct channel_gk20a *ch) | 1802 | static void __gk20a_channel_timeout_start(struct channel_gk20a *ch) |
2095 | { | 1803 | { |
2096 | ch->timeout.gp_get = gk20a_userd_gp_get(ch->g, ch); | 1804 | ch->timeout.gp_get = ch->g->ops.fifo.userd_gp_get(ch->g, ch); |
2097 | ch->timeout.running = true; | 1805 | ch->timeout.running = true; |
2098 | nvgpu_timeout_init(ch->g, &ch->timeout.timer, | 1806 | nvgpu_timeout_init(ch->g, &ch->timeout.timer, |
2099 | gk20a_get_channel_watchdog_timeout(ch), | 1807 | gk20a_get_channel_watchdog_timeout(ch), |
@@ -2225,7 +1933,7 @@ static void gk20a_channel_timeout_handler(struct channel_gk20a *ch) | |||
2225 | ch->timeout.running = false; | 1933 | ch->timeout.running = false; |
2226 | nvgpu_raw_spinlock_release(&ch->timeout.lock); | 1934 | nvgpu_raw_spinlock_release(&ch->timeout.lock); |
2227 | 1935 | ||
2228 | if (gk20a_userd_gp_get(ch->g, ch) != gp_get) { | 1936 | if (g->ops.fifo.userd_gp_get(ch->g, ch) != gp_get) { |
2229 | /* Channel has advanced, reschedule */ | 1937 | /* Channel has advanced, reschedule */ |
2230 | gk20a_channel_timeout_start(ch); | 1938 | gk20a_channel_timeout_start(ch); |
2231 | return; | 1939 | return; |
@@ -3693,55 +3401,6 @@ static int gk20a_channel_event_id_ctrl(struct channel_gk20a *ch, | |||
3693 | return err; | 3401 | return err; |
3694 | } | 3402 | } |
3695 | 3403 | ||
3696 | int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority) | ||
3697 | { | ||
3698 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
3699 | gk20a_err(dev_from_gk20a(ch->g), | ||
3700 | "invalid operation for TSG!\n"); | ||
3701 | return -EINVAL; | ||
3702 | } | ||
3703 | |||
3704 | /* set priority of graphics channel */ | ||
3705 | switch (priority) { | ||
3706 | case NVGPU_PRIORITY_LOW: | ||
3707 | ch->timeslice_us = ch->g->timeslice_low_priority_us; | ||
3708 | break; | ||
3709 | case NVGPU_PRIORITY_MEDIUM: | ||
3710 | ch->timeslice_us = ch->g->timeslice_medium_priority_us; | ||
3711 | break; | ||
3712 | case NVGPU_PRIORITY_HIGH: | ||
3713 | ch->timeslice_us = ch->g->timeslice_high_priority_us; | ||
3714 | break; | ||
3715 | default: | ||
3716 | pr_err("Unsupported priority"); | ||
3717 | return -EINVAL; | ||
3718 | } | ||
3719 | |||
3720 | return channel_gk20a_set_schedule_params(ch); | ||
3721 | } | ||
3722 | |||
3723 | int gk20a_channel_set_timeslice(struct channel_gk20a *ch, u32 timeslice) | ||
3724 | { | ||
3725 | struct gk20a *g = ch->g; | ||
3726 | |||
3727 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
3728 | gk20a_err(dev_from_gk20a(ch->g), | ||
3729 | "invalid operation for TSG!\n"); | ||
3730 | return -EINVAL; | ||
3731 | } | ||
3732 | |||
3733 | if (timeslice < g->min_timeslice_us || | ||
3734 | timeslice > g->max_timeslice_us) | ||
3735 | return -EINVAL; | ||
3736 | |||
3737 | ch->timeslice_us = timeslice; | ||
3738 | |||
3739 | gk20a_dbg(gpu_dbg_sched, "chid=%u timeslice=%u us", | ||
3740 | ch->hw_chid, timeslice); | ||
3741 | |||
3742 | return channel_gk20a_set_schedule_params(ch); | ||
3743 | } | ||
3744 | |||
3745 | static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, | 3404 | static int gk20a_channel_zcull_bind(struct channel_gk20a *ch, |
3746 | struct nvgpu_zcull_bind_args *args) | 3405 | struct nvgpu_zcull_bind_args *args) |
3747 | { | 3406 | { |
@@ -3924,21 +3583,6 @@ clean_up: | |||
3924 | return ret; | 3583 | return ret; |
3925 | } | 3584 | } |
3926 | 3585 | ||
3927 | void gk20a_init_channel(struct gpu_ops *gops) | ||
3928 | { | ||
3929 | gops->fifo.bind_channel = channel_gk20a_bind; | ||
3930 | gops->fifo.unbind_channel = channel_gk20a_unbind; | ||
3931 | gops->fifo.disable_channel = channel_gk20a_disable; | ||
3932 | gops->fifo.enable_channel = channel_gk20a_enable; | ||
3933 | gops->fifo.alloc_inst = channel_gk20a_alloc_inst; | ||
3934 | gops->fifo.free_inst = channel_gk20a_free_inst; | ||
3935 | gops->fifo.setup_ramfc = channel_gk20a_setup_ramfc; | ||
3936 | gops->fifo.channel_set_priority = gk20a_channel_set_priority; | ||
3937 | gops->fifo.channel_set_timeslice = gk20a_channel_set_timeslice; | ||
3938 | gops->fifo.userd_gp_get = gk20a_userd_gp_get; | ||
3939 | gops->fifo.userd_gp_put = gk20a_userd_gp_put; | ||
3940 | } | ||
3941 | |||
3942 | long gk20a_channel_ioctl(struct file *filp, | 3586 | long gk20a_channel_ioctl(struct file *filp, |
3943 | unsigned int cmd, unsigned long arg) | 3587 | unsigned int cmd, unsigned long arg) |
3944 | { | 3588 | { |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h index 42550632..d530f47d 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h | |||
@@ -201,7 +201,6 @@ struct channel_gk20a { | |||
201 | struct channel_ctx_gk20a ch_ctx; | 201 | struct channel_ctx_gk20a ch_ctx; |
202 | 202 | ||
203 | struct mem_desc inst_block; | 203 | struct mem_desc inst_block; |
204 | struct mem_desc_sub ramfc; | ||
205 | 204 | ||
206 | u64 userd_iova; | 205 | u64 userd_iova; |
207 | u64 userd_gpu_va; | 206 | u64 userd_gpu_va; |
@@ -314,8 +313,6 @@ int gk20a_channel_release(struct inode *inode, struct file *filp); | |||
314 | struct channel_gk20a *gk20a_get_channel_from_file(int fd); | 313 | struct channel_gk20a *gk20a_get_channel_from_file(int fd); |
315 | void gk20a_channel_update(struct channel_gk20a *c); | 314 | void gk20a_channel_update(struct channel_gk20a *c); |
316 | 315 | ||
317 | void gk20a_init_channel(struct gpu_ops *gops); | ||
318 | |||
319 | /* returns ch if reference was obtained */ | 316 | /* returns ch if reference was obtained */ |
320 | struct channel_gk20a *__must_check _gk20a_channel_get(struct channel_gk20a *ch, | 317 | struct channel_gk20a *__must_check _gk20a_channel_get(struct channel_gk20a *ch, |
321 | const char *caller); | 318 | const char *caller); |
@@ -336,7 +333,6 @@ struct channel_gk20a *gk20a_open_new_channel_with_cb(struct gk20a *g, | |||
336 | void *update_fn_data, | 333 | void *update_fn_data, |
337 | int runlist_id, | 334 | int runlist_id, |
338 | bool is_privileged_channel); | 335 | bool is_privileged_channel); |
339 | void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a); | ||
340 | 336 | ||
341 | int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | 337 | int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, |
342 | struct nvgpu_gpfifo *gpfifo, | 338 | struct nvgpu_gpfifo *gpfifo, |
@@ -351,14 +347,6 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c, | |||
351 | int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, | 347 | int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, |
352 | struct nvgpu_alloc_gpfifo_ex_args *args); | 348 | struct nvgpu_alloc_gpfifo_ex_args *args); |
353 | 349 | ||
354 | void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a); | ||
355 | void channel_gk20a_disable(struct channel_gk20a *ch); | ||
356 | int channel_gk20a_alloc_inst(struct gk20a *g, struct channel_gk20a *ch); | ||
357 | void channel_gk20a_free_inst(struct gk20a *g, struct channel_gk20a *ch); | ||
358 | u32 channel_gk20a_pbdma_acquire_val(struct channel_gk20a *c); | ||
359 | int channel_gk20a_setup_ramfc(struct channel_gk20a *c, | ||
360 | u64 gpfifo_base, u32 gpfifo_entries, u32 flags); | ||
361 | void channel_gk20a_enable(struct channel_gk20a *ch); | ||
362 | void gk20a_channel_timeout_restart_all_channels(struct gk20a *g); | 350 | void gk20a_channel_timeout_restart_all_channels(struct gk20a *g); |
363 | 351 | ||
364 | bool channel_gk20a_is_prealloc_enabled(struct channel_gk20a *c); | 352 | bool channel_gk20a_is_prealloc_enabled(struct channel_gk20a *c); |
@@ -369,13 +357,9 @@ bool channel_gk20a_joblist_is_empty(struct channel_gk20a *c); | |||
369 | int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g, | 357 | int gk20a_channel_get_timescale_from_timeslice(struct gk20a *g, |
370 | int timeslice_period, | 358 | int timeslice_period, |
371 | int *__timeslice_timeout, int *__timeslice_scale); | 359 | int *__timeslice_timeout, int *__timeslice_scale); |
372 | int gk20a_channel_set_priority(struct channel_gk20a *ch, u32 priority); | ||
373 | int gk20a_channel_set_timeslice(struct channel_gk20a *ch, unsigned int timeslice); | ||
374 | int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, | 360 | int gk20a_channel_set_runlist_interleave(struct channel_gk20a *ch, |
375 | u32 level); | 361 | u32 level); |
376 | void gk20a_channel_event_id_post_event(struct channel_gk20a *ch, | 362 | void gk20a_channel_event_id_post_event(struct channel_gk20a *ch, |
377 | u32 event_id); | 363 | u32 event_id); |
378 | 364 | ||
379 | void gk20a_channel_setup_ramfc_for_privileged_channel(struct channel_gk20a *c); | ||
380 | |||
381 | #endif /* CHANNEL_GK20A_H */ | 365 | #endif /* CHANNEL_GK20A_H */ |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index fdac40de..743bc1f5 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | |||
@@ -3852,9 +3852,342 @@ void gk20a_dump_eng_status(struct gk20a *g, | |||
3852 | gk20a_debug_output(o, "\n"); | 3852 | gk20a_debug_output(o, "\n"); |
3853 | } | 3853 | } |
3854 | 3854 | ||
3855 | void gk20a_fifo_enable_channel(struct channel_gk20a *ch) | ||
3856 | { | ||
3857 | gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid), | ||
3858 | gk20a_readl(ch->g, ccsr_channel_r(ch->hw_chid)) | | ||
3859 | ccsr_channel_enable_set_true_f()); | ||
3860 | } | ||
3861 | |||
3862 | void gk20a_fifo_disable_channel(struct channel_gk20a *ch) | ||
3863 | { | ||
3864 | gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid), | ||
3865 | gk20a_readl(ch->g, | ||
3866 | ccsr_channel_r(ch->hw_chid)) | | ||
3867 | ccsr_channel_enable_clr_true_f()); | ||
3868 | } | ||
3869 | |||
3870 | static void gk20a_fifo_channel_bind(struct channel_gk20a *c) | ||
3871 | { | ||
3872 | struct gk20a *g = c->g; | ||
3873 | u32 inst_ptr = gk20a_mm_inst_block_addr(g, &c->inst_block) >> | ||
3874 | ram_in_base_shift_v(); | ||
3875 | |||
3876 | gk20a_dbg_info("bind channel %d inst ptr 0x%08x", | ||
3877 | c->hw_chid, inst_ptr); | ||
3878 | |||
3879 | |||
3880 | gk20a_writel(g, ccsr_channel_r(c->hw_chid), | ||
3881 | (gk20a_readl(g, ccsr_channel_r(c->hw_chid)) & | ||
3882 | ~ccsr_channel_runlist_f(~0)) | | ||
3883 | ccsr_channel_runlist_f(c->runlist_id)); | ||
3884 | |||
3885 | gk20a_writel(g, ccsr_channel_inst_r(c->hw_chid), | ||
3886 | ccsr_channel_inst_ptr_f(inst_ptr) | | ||
3887 | gk20a_aperture_mask(g, &c->inst_block, | ||
3888 | ccsr_channel_inst_target_sys_mem_ncoh_f(), | ||
3889 | ccsr_channel_inst_target_vid_mem_f()) | | ||
3890 | ccsr_channel_inst_bind_true_f()); | ||
3891 | |||
3892 | gk20a_writel(g, ccsr_channel_r(c->hw_chid), | ||
3893 | (gk20a_readl(g, ccsr_channel_r(c->hw_chid)) & | ||
3894 | ~ccsr_channel_enable_set_f(~0)) | | ||
3895 | ccsr_channel_enable_set_true_f()); | ||
3896 | |||
3897 | wmb(); | ||
3898 | atomic_set(&c->bound, true); | ||
3899 | |||
3900 | } | ||
3901 | |||
3902 | void gk20a_fifo_channel_unbind(struct channel_gk20a *ch_gk20a) | ||
3903 | { | ||
3904 | struct gk20a *g = ch_gk20a->g; | ||
3905 | |||
3906 | gk20a_dbg_fn(""); | ||
3907 | |||
3908 | if (atomic_cmpxchg(&ch_gk20a->bound, true, false)) { | ||
3909 | gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid), | ||
3910 | ccsr_channel_inst_ptr_f(0) | | ||
3911 | ccsr_channel_inst_bind_false_f()); | ||
3912 | } | ||
3913 | } | ||
3914 | |||
3915 | static int gk20a_fifo_commit_userd(struct channel_gk20a *c) | ||
3916 | { | ||
3917 | u32 addr_lo; | ||
3918 | u32 addr_hi; | ||
3919 | struct gk20a *g = c->g; | ||
3920 | |||
3921 | gk20a_dbg_fn(""); | ||
3922 | |||
3923 | addr_lo = u64_lo32(c->userd_iova >> ram_userd_base_shift_v()); | ||
3924 | addr_hi = u64_hi32(c->userd_iova); | ||
3925 | |||
3926 | gk20a_dbg_info("channel %d : set ramfc userd 0x%16llx", | ||
3927 | c->hw_chid, (u64)c->userd_iova); | ||
3928 | |||
3929 | gk20a_mem_wr32(g, &c->inst_block, | ||
3930 | ram_in_ramfc_w() + ram_fc_userd_w(), | ||
3931 | gk20a_aperture_mask(g, &g->fifo.userd, | ||
3932 | pbdma_userd_target_sys_mem_ncoh_f(), | ||
3933 | pbdma_userd_target_vid_mem_f()) | | ||
3934 | pbdma_userd_addr_f(addr_lo)); | ||
3935 | |||
3936 | gk20a_mem_wr32(g, &c->inst_block, | ||
3937 | ram_in_ramfc_w() + ram_fc_userd_hi_w(), | ||
3938 | pbdma_userd_hi_addr_f(addr_hi)); | ||
3939 | |||
3940 | return 0; | ||
3941 | } | ||
3942 | |||
3943 | int gk20a_fifo_setup_ramfc(struct channel_gk20a *c, | ||
3944 | u64 gpfifo_base, u32 gpfifo_entries, | ||
3945 | unsigned long timeout, | ||
3946 | u32 flags) | ||
3947 | { | ||
3948 | struct gk20a *g = c->g; | ||
3949 | struct mem_desc *mem = &c->inst_block; | ||
3950 | |||
3951 | gk20a_dbg_fn(""); | ||
3952 | |||
3953 | gk20a_memset(g, mem, 0, 0, ram_fc_size_val_v()); | ||
3954 | |||
3955 | gk20a_mem_wr32(g, mem, ram_fc_gp_base_w(), | ||
3956 | pbdma_gp_base_offset_f( | ||
3957 | u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s()))); | ||
3958 | |||
3959 | gk20a_mem_wr32(g, mem, ram_fc_gp_base_hi_w(), | ||
3960 | pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) | | ||
3961 | pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries))); | ||
3962 | |||
3963 | gk20a_mem_wr32(g, mem, ram_fc_signature_w(), | ||
3964 | c->g->ops.fifo.get_pbdma_signature(c->g)); | ||
3965 | |||
3966 | gk20a_mem_wr32(g, mem, ram_fc_formats_w(), | ||
3967 | pbdma_formats_gp_fermi0_f() | | ||
3968 | pbdma_formats_pb_fermi1_f() | | ||
3969 | pbdma_formats_mp_fermi0_f()); | ||
3970 | |||
3971 | gk20a_mem_wr32(g, mem, ram_fc_pb_header_w(), | ||
3972 | pbdma_pb_header_priv_user_f() | | ||
3973 | pbdma_pb_header_method_zero_f() | | ||
3974 | pbdma_pb_header_subchannel_zero_f() | | ||
3975 | pbdma_pb_header_level_main_f() | | ||
3976 | pbdma_pb_header_first_true_f() | | ||
3977 | pbdma_pb_header_type_inc_f()); | ||
3978 | |||
3979 | gk20a_mem_wr32(g, mem, ram_fc_subdevice_w(), | ||
3980 | pbdma_subdevice_id_f(1) | | ||
3981 | pbdma_subdevice_status_active_f() | | ||
3982 | pbdma_subdevice_channel_dma_enable_f()); | ||
3983 | |||
3984 | gk20a_mem_wr32(g, mem, ram_fc_target_w(), pbdma_target_engine_sw_f()); | ||
3985 | |||
3986 | gk20a_mem_wr32(g, mem, ram_fc_acquire_w(), | ||
3987 | g->ops.fifo.pbdma_acquire_val(timeout)); | ||
3988 | |||
3989 | gk20a_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(), | ||
3990 | fifo_runlist_timeslice_timeout_128_f() | | ||
3991 | fifo_runlist_timeslice_timescale_3_f() | | ||
3992 | fifo_runlist_timeslice_enable_true_f()); | ||
3993 | |||
3994 | gk20a_mem_wr32(g, mem, ram_fc_pb_timeslice_w(), | ||
3995 | fifo_pb_timeslice_timeout_16_f() | | ||
3996 | fifo_pb_timeslice_timescale_0_f() | | ||
3997 | fifo_pb_timeslice_enable_true_f()); | ||
3998 | |||
3999 | gk20a_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->hw_chid)); | ||
4000 | |||
4001 | if (c->is_privileged_channel) | ||
4002 | gk20a_fifo_setup_ramfc_for_privileged_channel(c); | ||
4003 | |||
4004 | return gk20a_fifo_commit_userd(c); | ||
4005 | } | ||
4006 | |||
4007 | static int channel_gk20a_set_schedule_params(struct channel_gk20a *c) | ||
4008 | { | ||
4009 | int shift = 0, value = 0; | ||
4010 | |||
4011 | gk20a_channel_get_timescale_from_timeslice(c->g, | ||
4012 | c->timeslice_us, &value, &shift); | ||
4013 | |||
4014 | /* disable channel */ | ||
4015 | c->g->ops.fifo.disable_channel(c); | ||
4016 | |||
4017 | /* preempt the channel */ | ||
4018 | WARN_ON(c->g->ops.fifo.preempt_channel(c->g, c->hw_chid)); | ||
4019 | |||
4020 | /* set new timeslice */ | ||
4021 | gk20a_mem_wr32(c->g, &c->inst_block, ram_fc_runlist_timeslice_w(), | ||
4022 | value | (shift << 12) | | ||
4023 | fifo_runlist_timeslice_enable_true_f()); | ||
4024 | |||
4025 | /* enable channel */ | ||
4026 | c->g->ops.fifo.enable_channel(c); | ||
4027 | |||
4028 | return 0; | ||
4029 | } | ||
4030 | |||
4031 | int gk20a_fifo_set_timeslice(struct channel_gk20a *ch, u32 timeslice) | ||
4032 | { | ||
4033 | struct gk20a *g = ch->g; | ||
4034 | |||
4035 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
4036 | gk20a_err(dev_from_gk20a(ch->g), | ||
4037 | "invalid operation for TSG!\n"); | ||
4038 | return -EINVAL; | ||
4039 | } | ||
4040 | |||
4041 | if (timeslice < g->min_timeslice_us || | ||
4042 | timeslice > g->max_timeslice_us) | ||
4043 | return -EINVAL; | ||
4044 | |||
4045 | ch->timeslice_us = timeslice; | ||
4046 | |||
4047 | gk20a_dbg(gpu_dbg_sched, "chid=%u timeslice=%u us", | ||
4048 | ch->hw_chid, timeslice); | ||
4049 | |||
4050 | return channel_gk20a_set_schedule_params(ch); | ||
4051 | } | ||
4052 | |||
4053 | int gk20a_fifo_set_priority(struct channel_gk20a *ch, u32 priority) | ||
4054 | { | ||
4055 | if (gk20a_is_channel_marked_as_tsg(ch)) { | ||
4056 | gk20a_err(dev_from_gk20a(ch->g), | ||
4057 | "invalid operation for TSG!\n"); | ||
4058 | return -EINVAL; | ||
4059 | } | ||
4060 | |||
4061 | /* set priority of graphics channel */ | ||
4062 | switch (priority) { | ||
4063 | case NVGPU_PRIORITY_LOW: | ||
4064 | ch->timeslice_us = ch->g->timeslice_low_priority_us; | ||
4065 | break; | ||
4066 | case NVGPU_PRIORITY_MEDIUM: | ||
4067 | ch->timeslice_us = ch->g->timeslice_medium_priority_us; | ||
4068 | break; | ||
4069 | case NVGPU_PRIORITY_HIGH: | ||
4070 | ch->timeslice_us = ch->g->timeslice_high_priority_us; | ||
4071 | break; | ||
4072 | default: | ||
4073 | pr_err("Unsupported priority"); | ||
4074 | return -EINVAL; | ||
4075 | } | ||
4076 | |||
4077 | return channel_gk20a_set_schedule_params(ch); | ||
4078 | } | ||
4079 | |||
4080 | void gk20a_fifo_setup_ramfc_for_privileged_channel(struct channel_gk20a *c) | ||
4081 | { | ||
4082 | struct gk20a *g = c->g; | ||
4083 | struct mem_desc *mem = &c->inst_block; | ||
4084 | |||
4085 | gk20a_dbg_info("channel %d : set ramfc privileged_channel", c->hw_chid); | ||
4086 | |||
4087 | /* Enable HCE priv mode for phys mode transfer */ | ||
4088 | gk20a_mem_wr32(g, mem, ram_fc_hce_ctrl_w(), | ||
4089 | pbdma_hce_ctrl_hce_priv_mode_yes_f()); | ||
4090 | } | ||
4091 | |||
4092 | int gk20a_fifo_setup_userd(struct channel_gk20a *c) | ||
4093 | { | ||
4094 | struct gk20a *g = c->g; | ||
4095 | struct mem_desc *mem = &g->fifo.userd; | ||
4096 | u32 offset = c->hw_chid * g->fifo.userd_entry_size / sizeof(u32); | ||
4097 | |||
4098 | gk20a_dbg_fn(""); | ||
4099 | |||
4100 | gk20a_mem_wr32(g, mem, offset + ram_userd_put_w(), 0); | ||
4101 | gk20a_mem_wr32(g, mem, offset + ram_userd_get_w(), 0); | ||
4102 | gk20a_mem_wr32(g, mem, offset + ram_userd_ref_w(), 0); | ||
4103 | gk20a_mem_wr32(g, mem, offset + ram_userd_put_hi_w(), 0); | ||
4104 | gk20a_mem_wr32(g, mem, offset + ram_userd_ref_threshold_w(), 0); | ||
4105 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_top_level_get_w(), 0); | ||
4106 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_top_level_get_hi_w(), 0); | ||
4107 | gk20a_mem_wr32(g, mem, offset + ram_userd_get_hi_w(), 0); | ||
4108 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_get_w(), 0); | ||
4109 | gk20a_mem_wr32(g, mem, offset + ram_userd_gp_put_w(), 0); | ||
4110 | |||
4111 | return 0; | ||
4112 | } | ||
4113 | |||
4114 | int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch) | ||
4115 | { | ||
4116 | int err; | ||
4117 | |||
4118 | gk20a_dbg_fn(""); | ||
4119 | |||
4120 | err = gk20a_alloc_inst_block(g, &ch->inst_block); | ||
4121 | if (err) | ||
4122 | return err; | ||
4123 | |||
4124 | gk20a_dbg_info("channel %d inst block physical addr: 0x%16llx", | ||
4125 | ch->hw_chid, gk20a_mm_inst_block_addr(g, &ch->inst_block)); | ||
4126 | |||
4127 | gk20a_dbg_fn("done"); | ||
4128 | return 0; | ||
4129 | } | ||
4130 | |||
4131 | void gk20a_fifo_free_inst(struct gk20a *g, struct channel_gk20a *ch) | ||
4132 | { | ||
4133 | gk20a_free_inst_block(g, &ch->inst_block); | ||
4134 | } | ||
4135 | |||
4136 | u32 gk20a_fifo_userd_gp_get(struct gk20a *g, struct channel_gk20a *c) | ||
4137 | { | ||
4138 | return gk20a_bar1_readl(g, | ||
4139 | c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w()); | ||
4140 | } | ||
4141 | |||
4142 | void gk20a_fifo_userd_gp_put(struct gk20a *g, struct channel_gk20a *c) | ||
4143 | { | ||
4144 | gk20a_bar1_writel(g, | ||
4145 | c->userd_gpu_va + sizeof(u32) * ram_userd_gp_put_w(), | ||
4146 | c->gpfifo.put); | ||
4147 | } | ||
4148 | |||
4149 | u32 gk20a_fifo_pbdma_acquire_val(u64 timeout) | ||
4150 | { | ||
4151 | u32 val, exp, man; | ||
4152 | unsigned int val_len; | ||
4153 | |||
4154 | val = pbdma_acquire_retry_man_2_f() | | ||
4155 | pbdma_acquire_retry_exp_2_f(); | ||
4156 | |||
4157 | if (!timeout) | ||
4158 | return val; | ||
4159 | |||
4160 | timeout *= 80UL; | ||
4161 | do_div(timeout, 100); /* set acquire timeout to 80% of channel wdt */ | ||
4162 | timeout *= 1000000UL; /* ms -> ns */ | ||
4163 | do_div(timeout, 1024); /* in unit of 1024ns */ | ||
4164 | val_len = fls(timeout >> 32) + 32; | ||
4165 | if (val_len == 32) | ||
4166 | val_len = fls(timeout); | ||
4167 | if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */ | ||
4168 | exp = pbdma_acquire_timeout_exp_max_v(); | ||
4169 | man = pbdma_acquire_timeout_man_max_v(); | ||
4170 | } else if (val_len > 16) { | ||
4171 | exp = val_len - 16; | ||
4172 | man = timeout >> exp; | ||
4173 | } else { | ||
4174 | exp = 0; | ||
4175 | man = timeout; | ||
4176 | } | ||
4177 | |||
4178 | val |= pbdma_acquire_timeout_exp_f(exp) | | ||
4179 | pbdma_acquire_timeout_man_f(man) | | ||
4180 | pbdma_acquire_timeout_en_enable_f(); | ||
4181 | |||
4182 | return val; | ||
4183 | } | ||
4184 | |||
3855 | void gk20a_init_fifo(struct gpu_ops *gops) | 4185 | void gk20a_init_fifo(struct gpu_ops *gops) |
3856 | { | 4186 | { |
3857 | gk20a_init_channel(gops); | 4187 | gops->fifo.disable_channel = gk20a_fifo_disable_channel; |
4188 | gops->fifo.enable_channel = gk20a_fifo_enable_channel; | ||
4189 | gops->fifo.bind_channel = gk20a_fifo_channel_bind; | ||
4190 | gops->fifo.unbind_channel = gk20a_fifo_channel_unbind; | ||
3858 | gops->fifo.init_fifo_setup_hw = gk20a_init_fifo_setup_hw; | 4191 | gops->fifo.init_fifo_setup_hw = gk20a_init_fifo_setup_hw; |
3859 | gops->fifo.preempt_channel = gk20a_fifo_preempt_channel; | 4192 | gops->fifo.preempt_channel = gk20a_fifo_preempt_channel; |
3860 | gops->fifo.preempt_tsg = gk20a_fifo_preempt_tsg; | 4193 | gops->fifo.preempt_tsg = gk20a_fifo_preempt_tsg; |
@@ -3883,4 +4216,13 @@ void gk20a_init_fifo(struct gpu_ops *gops) | |||
3883 | gops->fifo.is_preempt_pending = gk20a_fifo_is_preempt_pending; | 4216 | gops->fifo.is_preempt_pending = gk20a_fifo_is_preempt_pending; |
3884 | gops->fifo.init_pbdma_intr_descs = gk20a_fifo_init_pbdma_intr_descs; | 4217 | gops->fifo.init_pbdma_intr_descs = gk20a_fifo_init_pbdma_intr_descs; |
3885 | gops->fifo.reset_enable_hw = gk20a_init_fifo_reset_enable_hw; | 4218 | gops->fifo.reset_enable_hw = gk20a_init_fifo_reset_enable_hw; |
4219 | gops->fifo.setup_ramfc = gk20a_fifo_setup_ramfc; | ||
4220 | gops->fifo.channel_set_priority = gk20a_fifo_set_priority; | ||
4221 | gops->fifo.channel_set_timeslice = gk20a_fifo_set_timeslice; | ||
4222 | gops->fifo.alloc_inst = gk20a_fifo_alloc_inst; | ||
4223 | gops->fifo.free_inst = gk20a_fifo_free_inst; | ||
4224 | gops->fifo.setup_userd = gk20a_fifo_setup_userd; | ||
4225 | gops->fifo.userd_gp_get = gk20a_fifo_userd_gp_get; | ||
4226 | gops->fifo.userd_gp_put = gk20a_fifo_userd_gp_put; | ||
4227 | gops->fifo.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val; | ||
3886 | } | 4228 | } |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h index a9703385..06269fa5 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | |||
@@ -331,8 +331,8 @@ void gk20a_get_ch_runlist_entry(struct channel_gk20a *ch, u32 *runlist); | |||
331 | void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask, | 331 | void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask, |
332 | u32 runlist_state, int runlist_mutex_state); | 332 | u32 runlist_state, int runlist_mutex_state); |
333 | 333 | ||
334 | u32 gk20a_userd_gp_get(struct gk20a *g, struct channel_gk20a *c); | 334 | u32 gk20a_fifo_userd_gp_get(struct gk20a *g, struct channel_gk20a *c); |
335 | void gk20a_userd_gp_put(struct gk20a *g, struct channel_gk20a *c); | 335 | void gk20a_fifo_userd_gp_put(struct gk20a *g, struct channel_gk20a *c); |
336 | 336 | ||
337 | bool gk20a_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid); | 337 | bool gk20a_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid); |
338 | #ifdef CONFIG_DEBUG_FS | 338 | #ifdef CONFIG_DEBUG_FS |
@@ -351,8 +351,11 @@ void gk20a_dump_eng_status(struct gk20a *g, | |||
351 | struct gk20a_debug_output *o); | 351 | struct gk20a_debug_output *o); |
352 | const char *gk20a_decode_ccsr_chan_status(u32 index); | 352 | const char *gk20a_decode_ccsr_chan_status(u32 index); |
353 | const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index); | 353 | const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index); |
354 | void gk20a_fifo_enable_channel(struct channel_gk20a *ch); | ||
355 | void gk20a_fifo_disable_channel(struct channel_gk20a *ch); | ||
354 | 356 | ||
355 | struct channel_gk20a *gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr); | 357 | struct channel_gk20a *gk20a_refch_from_inst_ptr(struct gk20a *g, u64 inst_ptr); |
358 | void gk20a_fifo_channel_unbind(struct channel_gk20a *ch_gk20a); | ||
356 | 359 | ||
357 | u32 gk20a_fifo_intr_0_error_mask(struct gk20a *g); | 360 | u32 gk20a_fifo_intr_0_error_mask(struct gk20a *g); |
358 | 361 | ||
@@ -361,5 +364,16 @@ int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, unsigned int id_type, | |||
361 | int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg); | 364 | int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg); |
362 | void __locked_fifo_preempt_timeout_rc(struct gk20a *g, u32 id, | 365 | void __locked_fifo_preempt_timeout_rc(struct gk20a *g, u32 id, |
363 | unsigned int id_type); | 366 | unsigned int id_type); |
367 | int gk20a_fifo_setup_ramfc(struct channel_gk20a *c, | ||
368 | u64 gpfifo_base, u32 gpfifo_entries, | ||
369 | unsigned long timeout, u32 flags); | ||
370 | int gk20a_fifo_set_priority(struct channel_gk20a *ch, u32 priority); | ||
371 | int gk20a_fifo_set_timeslice(struct channel_gk20a *ch, unsigned int timeslice); | ||
372 | void gk20a_fifo_setup_ramfc_for_privileged_channel(struct channel_gk20a *c); | ||
373 | int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch); | ||
374 | void gk20a_fifo_free_inst(struct gk20a *g, struct channel_gk20a *ch); | ||
375 | int gk20a_fifo_setup_userd(struct channel_gk20a *c); | ||
376 | u32 gk20a_fifo_pbdma_acquire_val(u64 timeout); | ||
377 | |||
364 | 378 | ||
365 | #endif /*__GR_GK20A_H__*/ | 379 | #endif /*__GR_GK20A_H__*/ |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index c8932d63..4f50ae36 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h | |||
@@ -398,7 +398,9 @@ struct gpu_ops { | |||
398 | int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch); | 398 | int (*alloc_inst)(struct gk20a *g, struct channel_gk20a *ch); |
399 | void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch); | 399 | void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch); |
400 | int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base, | 400 | int (*setup_ramfc)(struct channel_gk20a *c, u64 gpfifo_base, |
401 | u32 gpfifo_entries, u32 flags); | 401 | u32 gpfifo_entries, |
402 | unsigned long acquire_timeout, | ||
403 | u32 flags); | ||
402 | int (*resetup_ramfc)(struct channel_gk20a *c); | 404 | int (*resetup_ramfc)(struct channel_gk20a *c); |
403 | int (*preempt_channel)(struct gk20a *g, u32 hw_chid); | 405 | int (*preempt_channel)(struct gk20a *g, u32 hw_chid); |
404 | int (*preempt_tsg)(struct gk20a *g, u32 tsgid); | 406 | int (*preempt_tsg)(struct gk20a *g, u32 tsgid); |
@@ -456,6 +458,8 @@ struct gpu_ops { | |||
456 | unsigned int id_type, unsigned int timeout_rc_type); | 458 | unsigned int id_type, unsigned int timeout_rc_type); |
457 | void (*init_pbdma_intr_descs)(struct fifo_gk20a *f); | 459 | void (*init_pbdma_intr_descs)(struct fifo_gk20a *f); |
458 | int (*reset_enable_hw)(struct gk20a *g); | 460 | int (*reset_enable_hw)(struct gk20a *g); |
461 | int (*setup_userd)(struct channel_gk20a *c); | ||
462 | u32 (*pbdma_acquire_val)(u64 timeout); | ||
459 | } fifo; | 463 | } fifo; |
460 | struct pmu_v { | 464 | struct pmu_v { |
461 | /*used for change of enum zbc update cmd id from ver 0 to ver1*/ | 465 | /*used for change of enum zbc update cmd id from ver 0 to ver1*/ |
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c index f09da825..fc653357 100644 --- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c | |||
@@ -188,16 +188,18 @@ void gm20b_init_fifo(struct gpu_ops *gops) | |||
188 | { | 188 | { |
189 | gops->fifo.init_fifo_setup_hw = gk20a_init_fifo_setup_hw; | 189 | gops->fifo.init_fifo_setup_hw = gk20a_init_fifo_setup_hw; |
190 | gops->fifo.bind_channel = channel_gm20b_bind; | 190 | gops->fifo.bind_channel = channel_gm20b_bind; |
191 | gops->fifo.unbind_channel = channel_gk20a_unbind; | 191 | gops->fifo.unbind_channel = gk20a_fifo_channel_unbind; |
192 | gops->fifo.disable_channel = channel_gk20a_disable; | 192 | gops->fifo.disable_channel = gk20a_fifo_disable_channel; |
193 | gops->fifo.enable_channel = channel_gk20a_enable; | 193 | gops->fifo.enable_channel = gk20a_fifo_enable_channel; |
194 | gops->fifo.alloc_inst = channel_gk20a_alloc_inst; | 194 | gops->fifo.alloc_inst = gk20a_fifo_alloc_inst; |
195 | gops->fifo.free_inst = channel_gk20a_free_inst; | 195 | gops->fifo.free_inst = gk20a_fifo_free_inst; |
196 | gops->fifo.setup_ramfc = channel_gk20a_setup_ramfc; | 196 | gops->fifo.setup_ramfc = gk20a_fifo_setup_ramfc; |
197 | gops->fifo.channel_set_priority = gk20a_channel_set_priority; | 197 | gops->fifo.channel_set_priority = gk20a_fifo_set_priority; |
198 | gops->fifo.channel_set_timeslice = gk20a_channel_set_timeslice; | 198 | gops->fifo.channel_set_timeslice = gk20a_fifo_set_timeslice; |
199 | gops->fifo.userd_gp_get = gk20a_userd_gp_get; | 199 | gops->fifo.setup_userd = gk20a_fifo_setup_userd; |
200 | gops->fifo.userd_gp_put = gk20a_userd_gp_put; | 200 | gops->fifo.userd_gp_get = gk20a_fifo_userd_gp_get; |
201 | gops->fifo.userd_gp_put = gk20a_fifo_userd_gp_put; | ||
202 | gops->fifo.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val; | ||
201 | 203 | ||
202 | gops->fifo.preempt_channel = gk20a_fifo_preempt_channel; | 204 | gops->fifo.preempt_channel = gk20a_fifo_preempt_channel; |
203 | gops->fifo.preempt_tsg = gk20a_fifo_preempt_tsg; | 205 | gops->fifo.preempt_tsg = gk20a_fifo_preempt_tsg; |
diff --git a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c index 77ea1b47..6f576e3f 100644 --- a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c | |||
@@ -80,7 +80,8 @@ int channel_gp10b_commit_userd(struct channel_gk20a *c) | |||
80 | } | 80 | } |
81 | 81 | ||
82 | static int channel_gp10b_setup_ramfc(struct channel_gk20a *c, | 82 | static int channel_gp10b_setup_ramfc(struct channel_gk20a *c, |
83 | u64 gpfifo_base, u32 gpfifo_entries, u32 flags) | 83 | u64 gpfifo_base, u32 gpfifo_entries, |
84 | unsigned long acquire_timeout, u32 flags) | ||
84 | { | 85 | { |
85 | struct gk20a *g = c->g; | 86 | struct gk20a *g = c->g; |
86 | struct mem_desc *mem = &c->inst_block; | 87 | struct mem_desc *mem = &c->inst_block; |
@@ -121,7 +122,7 @@ static int channel_gp10b_setup_ramfc(struct channel_gk20a *c, | |||
121 | gk20a_mem_wr32(g, mem, ram_fc_target_w(), pbdma_target_engine_sw_f()); | 122 | gk20a_mem_wr32(g, mem, ram_fc_target_w(), pbdma_target_engine_sw_f()); |
122 | 123 | ||
123 | gk20a_mem_wr32(g, mem, ram_fc_acquire_w(), | 124 | gk20a_mem_wr32(g, mem, ram_fc_acquire_w(), |
124 | channel_gk20a_pbdma_acquire_val(c)); | 125 | g->ops.fifo.pbdma_acquire_val(acquire_timeout)); |
125 | 126 | ||
126 | gk20a_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(), | 127 | gk20a_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(), |
127 | pbdma_runlist_timeslice_timeout_128_f() | | 128 | pbdma_runlist_timeslice_timeout_128_f() | |
@@ -139,7 +140,7 @@ static int channel_gp10b_setup_ramfc(struct channel_gk20a *c, | |||
139 | gk20a_mem_wr32(g, mem, ram_fc_config_w(), | 140 | gk20a_mem_wr32(g, mem, ram_fc_config_w(), |
140 | pbdma_config_auth_level_privileged_f()); | 141 | pbdma_config_auth_level_privileged_f()); |
141 | 142 | ||
142 | gk20a_channel_setup_ramfc_for_privileged_channel(c); | 143 | gk20a_fifo_setup_ramfc_for_privileged_channel(c); |
143 | } | 144 | } |
144 | 145 | ||
145 | return channel_gp10b_commit_userd(c); | 146 | return channel_gp10b_commit_userd(c); |
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c index 0c93a2ed..497f8c91 100644 --- a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c | |||
@@ -134,7 +134,8 @@ static void vgpu_channel_disable(struct channel_gk20a *ch) | |||
134 | } | 134 | } |
135 | 135 | ||
136 | static int vgpu_channel_setup_ramfc(struct channel_gk20a *ch, u64 gpfifo_base, | 136 | static int vgpu_channel_setup_ramfc(struct channel_gk20a *ch, u64 gpfifo_base, |
137 | u32 gpfifo_entries, u32 flags) | 137 | u32 gpfifo_entries, |
138 | unsigned long acquire_timeout, u32 flags) | ||
138 | { | 139 | { |
139 | struct device __maybe_unused *d = dev_from_gk20a(ch->g); | 140 | struct device __maybe_unused *d = dev_from_gk20a(ch->g); |
140 | struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d); | 141 | struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d); |