summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gk20a
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/gk20a')
-rw-r--r--drivers/gpu/nvgpu/gk20a/cde_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/ce2_gk20a.c3
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_gk20a.c6
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c7
-rw-r--r--drivers/gpu/nvgpu/gk20a/debug_gk20a.c6
-rw-r--r--drivers/gpu/nvgpu/gk20a/fence_gk20a.c5
-rw-r--r--drivers/gpu/nvgpu/gk20a/fifo_gk20a.c2
-rw-r--r--drivers/gpu/nvgpu/gk20a/gr_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/mm_gk20a.c2
-rw-r--r--drivers/gpu/nvgpu/gk20a/mm_gk20a.h6
-rw-r--r--drivers/gpu/nvgpu/gk20a/pmu_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c466
-rw-r--r--drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h318
-rw-r--r--drivers/gpu/nvgpu/gk20a/sync_gk20a.c11
14 files changed, 27 insertions, 808 deletions
diff --git a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
index b4a1f6f4..2a9ad40d 100644
--- a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
@@ -34,7 +34,6 @@
34#include "fence_gk20a.h" 34#include "fence_gk20a.h"
35#include "gr_gk20a.h" 35#include "gr_gk20a.h"
36#include "debug_gk20a.h" 36#include "debug_gk20a.h"
37#include "semaphore_gk20a.h"
38 37
39#include <nvgpu/hw/gk20a/hw_ccsr_gk20a.h> 38#include <nvgpu/hw/gk20a/hw_ccsr_gk20a.h>
40#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> 39#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h>
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
index 7afed41f..62b0a05e 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * GK20A Graphics Copy Engine (gr host) 2 * GK20A Graphics Copy Engine (gr host)
3 * 3 *
4 * Copyright (c) 2011-2016, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -28,7 +28,6 @@
28 28
29#include "gk20a.h" 29#include "gk20a.h"
30#include "debug_gk20a.h" 30#include "debug_gk20a.h"
31#include "semaphore_gk20a.h"
32 31
33#include <nvgpu/hw/gk20a/hw_ce2_gk20a.h> 32#include <nvgpu/hw/gk20a/hw_ce2_gk20a.h>
34#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> 33#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h>
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index c8b1c105..3fa6bb25 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -28,13 +28,13 @@
28#include <linux/vmalloc.h> 28#include <linux/vmalloc.h>
29#include <linux/circ_buf.h> 29#include <linux/circ_buf.h>
30 30
31#include "debug_gk20a.h" 31#include <nvgpu/semaphore.h>
32#include "ctxsw_trace_gk20a.h"
33 32
34#include "gk20a.h" 33#include "gk20a.h"
34#include "debug_gk20a.h"
35#include "ctxsw_trace_gk20a.h"
35#include "dbg_gpu_gk20a.h" 36#include "dbg_gpu_gk20a.h"
36#include "fence_gk20a.h" 37#include "fence_gk20a.h"
37#include "semaphore_gk20a.h"
38 38
39#include <nvgpu/timers.h> 39#include <nvgpu/timers.h>
40 40
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
index c3c6fbb8..0eba1c30 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
@@ -1,9 +1,7 @@
1/* 1/*
2 * drivers/video/tegra/host/gk20a/channel_sync_gk20a.c
3 *
4 * GK20A Channel Synchronization Abstraction 2 * GK20A Channel Synchronization Abstraction
5 * 3 *
6 * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved.
7 * 5 *
8 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -20,10 +18,11 @@
20#include <linux/list.h> 18#include <linux/list.h>
21#include <linux/version.h> 19#include <linux/version.h>
22 20
21#include <nvgpu/semaphore.h>
22
23#include "channel_sync_gk20a.h" 23#include "channel_sync_gk20a.h"
24#include "gk20a.h" 24#include "gk20a.h"
25#include "fence_gk20a.h" 25#include "fence_gk20a.h"
26#include "semaphore_gk20a.h"
27#include "sync_gk20a.h" 26#include "sync_gk20a.h"
28#include "mm_gk20a.h" 27#include "mm_gk20a.h"
29 28
diff --git a/drivers/gpu/nvgpu/gk20a/debug_gk20a.c b/drivers/gpu/nvgpu/gk20a/debug_gk20a.c
index 37ba720a..83fdc05d 100644
--- a/drivers/gpu/nvgpu/gk20a/debug_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/debug_gk20a.c
@@ -1,6 +1,4 @@
1/* 1/*
2 * drivers/video/tegra/host/t20/debug_gk20a.c
3 *
4 * Copyright (C) 2011-2017 NVIDIA Corporation. All rights reserved. 2 * Copyright (C) 2011-2017 NVIDIA Corporation. All rights reserved.
5 * 3 *
6 * This software is licensed under the terms of the GNU General Public 4 * This software is licensed under the terms of the GNU General Public
@@ -20,12 +18,12 @@
20 18
21#include <linux/debugfs.h> 19#include <linux/debugfs.h>
22#include <linux/seq_file.h> 20#include <linux/seq_file.h>
23
24#include <linux/io.h> 21#include <linux/io.h>
25 22
23#include <nvgpu/semaphore.h>
24
26#include "gk20a.h" 25#include "gk20a.h"
27#include "debug_gk20a.h" 26#include "debug_gk20a.h"
28#include "semaphore_gk20a.h"
29 27
30#include <nvgpu/hw/gk20a/hw_ram_gk20a.h> 28#include <nvgpu/hw/gk20a/hw_ram_gk20a.h>
31#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h> 29#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
diff --git a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c
index b8a1dcbc..6bd59067 100644
--- a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -17,8 +17,9 @@
17#include <linux/file.h> 17#include <linux/file.h>
18#include <linux/version.h> 18#include <linux/version.h>
19 19
20#include <nvgpu/semaphore.h>
21
20#include "gk20a.h" 22#include "gk20a.h"
21#include "semaphore_gk20a.h"
22#include "channel_gk20a.h" 23#include "channel_gk20a.h"
23#include "sync_gk20a.h" 24#include "sync_gk20a.h"
24 25
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 469148c2..c6b444f9 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -25,11 +25,11 @@
25#include <linux/nvhost.h> 25#include <linux/nvhost.h>
26 26
27#include <nvgpu/timers.h> 27#include <nvgpu/timers.h>
28#include <nvgpu/semaphore.h>
28 29
29#include "gk20a.h" 30#include "gk20a.h"
30#include "debug_gk20a.h" 31#include "debug_gk20a.h"
31#include "ctxsw_trace_gk20a.h" 32#include "ctxsw_trace_gk20a.h"
32#include "semaphore_gk20a.h"
33 33
34#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h> 34#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
35#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h> 35#include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h>
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index cddb3316..0e1c88a4 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -41,7 +41,6 @@
41#include "regops_gk20a.h" 41#include "regops_gk20a.h"
42#include "dbg_gpu_gk20a.h" 42#include "dbg_gpu_gk20a.h"
43#include "debug_gk20a.h" 43#include "debug_gk20a.h"
44#include "semaphore_gk20a.h"
45#include "platform_gk20a.h" 44#include "platform_gk20a.h"
46#include "ctxsw_trace_gk20a.h" 45#include "ctxsw_trace_gk20a.h"
47 46
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index ea5ea73f..cafb1233 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -33,13 +33,13 @@
33 33
34#include <nvgpu/timers.h> 34#include <nvgpu/timers.h>
35#include <nvgpu/allocator.h> 35#include <nvgpu/allocator.h>
36#include <nvgpu/semaphore.h>
36#include <nvgpu/page_allocator.h> 37#include <nvgpu/page_allocator.h>
37 38
38#include "gk20a.h" 39#include "gk20a.h"
39#include "mm_gk20a.h" 40#include "mm_gk20a.h"
40#include "fence_gk20a.h" 41#include "fence_gk20a.h"
41#include "kind_gk20a.h" 42#include "kind_gk20a.h"
42#include "semaphore_gk20a.h"
43 43
44#include <nvgpu/hw/gk20a/hw_gmmu_gk20a.h> 44#include <nvgpu/hw/gk20a/hw_gmmu_gk20a.h>
45#include <nvgpu/hw/gk20a/hw_fb_gk20a.h> 45#include <nvgpu/hw/gk20a/hw_fb_gk20a.h>
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index f3dffa46..d39ca2d0 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -52,6 +52,12 @@ enum gk20a_aperture {
52 APERTURE_VIDMEM 52 APERTURE_VIDMEM
53}; 53};
54 54
55enum gk20a_mem_rw_flag {
56 gk20a_mem_flag_none = 0,
57 gk20a_mem_flag_read_only = 1,
58 gk20a_mem_flag_write_only = 2,
59};
60
55static inline const char *gk20a_aperture_str(enum gk20a_aperture aperture) 61static inline const char *gk20a_aperture_str(enum gk20a_aperture aperture)
56{ 62{
57 switch (aperture) { 63 switch (aperture) {
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
index 9924e48f..d53cf09b 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -29,7 +29,6 @@
29 29
30#include "gk20a.h" 30#include "gk20a.h"
31#include "gr_gk20a.h" 31#include "gr_gk20a.h"
32#include "semaphore_gk20a.h"
33 32
34#include <nvgpu/hw/gk20a/hw_mc_gk20a.h> 33#include <nvgpu/hw/gk20a/hw_mc_gk20a.h>
35#include <nvgpu/hw/gk20a/hw_pwr_gk20a.h> 34#include <nvgpu/hw/gk20a/hw_pwr_gk20a.h>
diff --git a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c b/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c
deleted file mode 100644
index 2038e300..00000000
--- a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.c
+++ /dev/null
@@ -1,466 +0,0 @@
1/*
2 * drivers/video/tegra/host/gk20a/semaphore_gk20a.c
3 *
4 * GK20A Semaphores
5 *
6 * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 */
17
18#define pr_fmt(fmt) "gpu_sema: " fmt
19
20#include <linux/dma-mapping.h>
21#include <linux/highmem.h>
22#include <linux/slab.h>
23
24#include <asm/pgtable.h>
25
26#include "gk20a.h"
27#include "mm_gk20a.h"
28#include "semaphore_gk20a.h"
29
30#define __lock_sema_sea(s) \
31 do { \
32 gpu_sema_verbose_dbg("Acquiring sema lock..."); \
33 mutex_lock(&s->sea_lock); \
34 gpu_sema_verbose_dbg("Sema lock aquried!"); \
35 } while (0)
36
37#define __unlock_sema_sea(s) \
38 do { \
39 mutex_unlock(&s->sea_lock); \
40 gpu_sema_verbose_dbg("Released sema lock"); \
41 } while (0)
42
43/*
44 * Return the sema_sea pointer.
45 */
46struct gk20a_semaphore_sea *gk20a_semaphore_get_sea(struct gk20a *g)
47{
48 return g->sema_sea;
49}
50
51static int __gk20a_semaphore_sea_grow(struct gk20a_semaphore_sea *sea)
52{
53 int ret = 0;
54 struct gk20a *gk20a = sea->gk20a;
55
56 __lock_sema_sea(sea);
57
58 ret = gk20a_gmmu_alloc_attr_sys(gk20a, DMA_ATTR_NO_KERNEL_MAPPING,
59 PAGE_SIZE * SEMAPHORE_POOL_COUNT,
60 &sea->sea_mem);
61 if (ret)
62 goto out;
63
64 sea->ro_sg_table = sea->sea_mem.sgt;
65 sea->size = SEMAPHORE_POOL_COUNT;
66 sea->map_size = SEMAPHORE_POOL_COUNT * PAGE_SIZE;
67
68out:
69 __unlock_sema_sea(sea);
70 return ret;
71}
72
73/*
74 * Create the semaphore sea. Only create it once - subsequent calls to this will
75 * return the originally created sea pointer.
76 */
77struct gk20a_semaphore_sea *gk20a_semaphore_sea_create(struct gk20a *g)
78{
79 if (g->sema_sea)
80 return g->sema_sea;
81
82 g->sema_sea = kzalloc(sizeof(*g->sema_sea), GFP_KERNEL);
83 if (!g->sema_sea)
84 return NULL;
85
86 g->sema_sea->size = 0;
87 g->sema_sea->page_count = 0;
88 g->sema_sea->gk20a = g;
89 INIT_LIST_HEAD(&g->sema_sea->pool_list);
90 mutex_init(&g->sema_sea->sea_lock);
91
92 if (__gk20a_semaphore_sea_grow(g->sema_sea))
93 goto cleanup;
94
95 gpu_sema_dbg("Created semaphore sea!");
96 return g->sema_sea;
97
98cleanup:
99 kfree(g->sema_sea);
100 g->sema_sea = NULL;
101 gpu_sema_dbg("Failed to creat semaphore sea!");
102 return NULL;
103}
104
105static int __semaphore_bitmap_alloc(unsigned long *bitmap, unsigned long len)
106{
107 unsigned long idx = find_first_zero_bit(bitmap, len);
108
109 if (idx == len)
110 return -ENOSPC;
111
112 set_bit(idx, bitmap);
113
114 return (int)idx;
115}
116
117/*
118 * Allocate a pool from the sea.
119 */
120struct gk20a_semaphore_pool *gk20a_semaphore_pool_alloc(
121 struct gk20a_semaphore_sea *sea)
122{
123 struct gk20a_semaphore_pool *p;
124 unsigned long page_idx;
125 int ret, err = 0;
126
127 p = kzalloc(sizeof(*p), GFP_KERNEL);
128 if (!p)
129 return ERR_PTR(-ENOMEM);
130
131 __lock_sema_sea(sea);
132
133 ret = __semaphore_bitmap_alloc(sea->pools_alloced, SEMAPHORE_POOL_COUNT);
134 if (ret < 0) {
135 err = ret;
136 goto fail;
137 }
138
139 page_idx = (unsigned long)ret;
140
141 p->page = sea->sea_mem.pages[page_idx];
142 p->ro_sg_table = sea->ro_sg_table;
143 p->page_idx = page_idx;
144 p->sema_sea = sea;
145 INIT_LIST_HEAD(&p->hw_semas);
146 kref_init(&p->ref);
147 mutex_init(&p->pool_lock);
148
149 sea->page_count++;
150 list_add(&p->pool_list_entry, &sea->pool_list);
151 __unlock_sema_sea(sea);
152
153 gpu_sema_dbg("Allocated semaphore pool: page-idx=%d", p->page_idx);
154
155 return p;
156
157fail:
158 __unlock_sema_sea(sea);
159 kfree(p);
160 gpu_sema_dbg("Failed to allocate semaphore pool!");
161 return ERR_PTR(err);
162}
163
164/*
165 * Map a pool into the passed vm's address space. This handles both the fixed
166 * global RO mapping and the non-fixed private RW mapping.
167 */
168int gk20a_semaphore_pool_map(struct gk20a_semaphore_pool *p,
169 struct vm_gk20a *vm)
170{
171 int ents, err = 0;
172 u64 addr;
173
174 gpu_sema_dbg("Mapping sempahore pool! (idx=%d)", p->page_idx);
175
176 p->cpu_va = vmap(&p->page, 1, 0,
177 pgprot_writecombine(PAGE_KERNEL));
178
179 gpu_sema_dbg(" %d: CPU VA = 0x%p!", p->page_idx, p->cpu_va);
180
181 /* First do the RW mapping. */
182 p->rw_sg_table = kzalloc(sizeof(*p->rw_sg_table), GFP_KERNEL);
183 if (!p->rw_sg_table)
184 return -ENOMEM;
185
186 err = sg_alloc_table_from_pages(p->rw_sg_table, &p->page, 1, 0,
187 PAGE_SIZE, GFP_KERNEL);
188 if (err) {
189 err = -ENOMEM;
190 goto fail;
191 }
192
193 /* Add IOMMU mapping... */
194 ents = dma_map_sg(dev_from_vm(vm), p->rw_sg_table->sgl, 1,
195 DMA_BIDIRECTIONAL);
196 if (ents != 1) {
197 err = -ENOMEM;
198 goto fail_free_sgt;
199 }
200
201 gpu_sema_dbg(" %d: DMA addr = 0x%pad", p->page_idx,
202 &sg_dma_address(p->rw_sg_table->sgl));
203
204 /* Map into the GPU... Doesn't need to be fixed. */
205 p->gpu_va = gk20a_gmmu_map(vm, &p->rw_sg_table, PAGE_SIZE,
206 0, gk20a_mem_flag_none, false,
207 APERTURE_SYSMEM);
208 if (!p->gpu_va) {
209 err = -ENOMEM;
210 goto fail_unmap_sgt;
211 }
212
213 gpu_sema_dbg(" %d: GPU read-write VA = 0x%llx", p->page_idx,
214 p->gpu_va);
215
216 /*
217 * And now the global mapping. Take the sea lock so that we don't race
218 * with a concurrent remap.
219 */
220 __lock_sema_sea(p->sema_sea);
221
222 BUG_ON(p->mapped);
223 addr = gk20a_gmmu_fixed_map(vm, &p->sema_sea->ro_sg_table,
224 p->sema_sea->gpu_va, p->sema_sea->map_size,
225 0,
226 gk20a_mem_flag_read_only,
227 false,
228 APERTURE_SYSMEM);
229 if (!addr) {
230 err = -ENOMEM;
231 BUG();
232 goto fail_unlock;
233 }
234 p->gpu_va_ro = addr;
235 p->mapped = 1;
236
237 gpu_sema_dbg(" %d: GPU read-only VA = 0x%llx", p->page_idx,
238 p->gpu_va_ro);
239
240 __unlock_sema_sea(p->sema_sea);
241
242 return 0;
243
244fail_unlock:
245 __unlock_sema_sea(p->sema_sea);
246fail_unmap_sgt:
247 dma_unmap_sg(dev_from_vm(vm), p->rw_sg_table->sgl, 1,
248 DMA_BIDIRECTIONAL);
249fail_free_sgt:
250 sg_free_table(p->rw_sg_table);
251fail:
252 kfree(p->rw_sg_table);
253 p->rw_sg_table = NULL;
254 gpu_sema_dbg(" %d: Failed to map semaphore pool!", p->page_idx);
255 return err;
256}
257
258/*
259 * Unmap a semaphore_pool.
260 */
261void gk20a_semaphore_pool_unmap(struct gk20a_semaphore_pool *p,
262 struct vm_gk20a *vm)
263{
264 struct gk20a_semaphore_int *hw_sema;
265
266 kunmap(p->cpu_va);
267
268 /* First the global RO mapping... */
269 __lock_sema_sea(p->sema_sea);
270 gk20a_gmmu_unmap(vm, p->gpu_va_ro,
271 p->sema_sea->map_size, gk20a_mem_flag_none);
272 p->ro_sg_table = NULL;
273 __unlock_sema_sea(p->sema_sea);
274
275 /* And now the private RW mapping. */
276 gk20a_gmmu_unmap(vm, p->gpu_va, PAGE_SIZE, gk20a_mem_flag_none);
277 p->gpu_va = 0;
278
279 dma_unmap_sg(dev_from_vm(vm), p->rw_sg_table->sgl, 1,
280 DMA_BIDIRECTIONAL);
281
282 sg_free_table(p->rw_sg_table);
283 kfree(p->rw_sg_table);
284 p->rw_sg_table = NULL;
285
286 list_for_each_entry(hw_sema, &p->hw_semas, hw_sema_list)
287 /*
288 * Make sure the mem addresses are all NULL so if this gets
289 * reused we will fault.
290 */
291 hw_sema->value = NULL;
292
293 gpu_sema_dbg("Unmapped semaphore pool! (idx=%d)", p->page_idx);
294}
295
296/*
297 * Completely free a sempahore_pool. You should make sure this pool is not
298 * mapped otherwise there's going to be a memory leak.
299 */
300static void gk20a_semaphore_pool_free(struct kref *ref)
301{
302 struct gk20a_semaphore_pool *p =
303 container_of(ref, struct gk20a_semaphore_pool, ref);
304 struct gk20a_semaphore_sea *s = p->sema_sea;
305 struct gk20a_semaphore_int *hw_sema, *tmp;
306
307 WARN_ON(p->gpu_va || p->rw_sg_table || p->ro_sg_table);
308
309 __lock_sema_sea(s);
310 list_del(&p->pool_list_entry);
311 clear_bit(p->page_idx, s->pools_alloced);
312 s->page_count--;
313 __unlock_sema_sea(s);
314
315 list_for_each_entry_safe(hw_sema, tmp, &p->hw_semas, hw_sema_list)
316 kfree(hw_sema);
317
318 gpu_sema_dbg("Freed semaphore pool! (idx=%d)", p->page_idx);
319 kfree(p);
320}
321
322void gk20a_semaphore_pool_get(struct gk20a_semaphore_pool *p)
323{
324 kref_get(&p->ref);
325}
326
327void gk20a_semaphore_pool_put(struct gk20a_semaphore_pool *p)
328{
329 kref_put(&p->ref, gk20a_semaphore_pool_free);
330}
331
332/*
333 * Get the address for a semaphore_pool - if global is true then return the
334 * global RO address instead of the RW address owned by the semaphore's VM.
335 */
336u64 __gk20a_semaphore_pool_gpu_va(struct gk20a_semaphore_pool *p, bool global)
337{
338 if (!global)
339 return p->gpu_va;
340
341 return p->gpu_va_ro + (PAGE_SIZE * p->page_idx);
342}
343
344static int __gk20a_init_hw_sema(struct channel_gk20a *ch)
345{
346 int hw_sema_idx;
347 int ret = 0;
348 struct gk20a_semaphore_int *hw_sema;
349 struct gk20a_semaphore_pool *p = ch->vm->sema_pool;
350
351 BUG_ON(!p);
352
353 mutex_lock(&p->pool_lock);
354
355 /* Find an available HW semaphore. */
356 hw_sema_idx = __semaphore_bitmap_alloc(p->semas_alloced,
357 PAGE_SIZE / SEMAPHORE_SIZE);
358 if (hw_sema_idx < 0) {
359 ret = hw_sema_idx;
360 goto fail;
361 }
362
363 hw_sema = kzalloc(sizeof(struct gk20a_semaphore_int), GFP_KERNEL);
364 if (!hw_sema) {
365 ret = -ENOMEM;
366 goto fail_free_idx;
367 }
368
369 ch->hw_sema = hw_sema;
370 hw_sema->ch = ch;
371 hw_sema->p = p;
372 hw_sema->idx = hw_sema_idx;
373 hw_sema->offset = SEMAPHORE_SIZE * hw_sema_idx;
374 atomic_set(&hw_sema->next_value, 0);
375 hw_sema->value = p->cpu_va + hw_sema->offset;
376 writel(0, hw_sema->value);
377
378 list_add(&hw_sema->hw_sema_list, &p->hw_semas);
379
380 mutex_unlock(&p->pool_lock);
381
382 return 0;
383
384fail_free_idx:
385 clear_bit(hw_sema_idx, p->semas_alloced);
386fail:
387 mutex_unlock(&p->pool_lock);
388 return ret;
389}
390
391/*
392 * Free the channel used semaphore index
393 */
394void gk20a_semaphore_free_hw_sema(struct channel_gk20a *ch)
395{
396 struct gk20a_semaphore_pool *p = ch->vm->sema_pool;
397
398 BUG_ON(!p);
399
400 mutex_lock(&p->pool_lock);
401
402 clear_bit(ch->hw_sema->idx, p->semas_alloced);
403
404 /* Make sure that when the ch is re-opened it will get a new HW sema. */
405 list_del(&ch->hw_sema->hw_sema_list);
406 kfree(ch->hw_sema);
407 ch->hw_sema = NULL;
408
409 mutex_unlock(&p->pool_lock);
410}
411
412/*
413 * Allocate a semaphore from the passed pool.
414 *
415 * Since semaphores are ref-counted there's no explicit free for external code
416 * to use. When the ref-count hits 0 the internal free will happen.
417 */
418struct gk20a_semaphore *gk20a_semaphore_alloc(struct channel_gk20a *ch)
419{
420 struct gk20a_semaphore *s;
421 int ret;
422
423 if (!ch->hw_sema) {
424 ret = __gk20a_init_hw_sema(ch);
425 if (ret)
426 return NULL;
427 }
428
429 s = kzalloc(sizeof(*s), GFP_KERNEL);
430 if (!s)
431 return NULL;
432
433 kref_init(&s->ref);
434 s->hw_sema = ch->hw_sema;
435 atomic_set(&s->value, 0);
436
437 /*
438 * Take a ref on the pool so that we can keep this pool alive for
439 * as long as this semaphore is alive.
440 */
441 gk20a_semaphore_pool_get(s->hw_sema->p);
442
443 gpu_sema_dbg("Allocated semaphore (c=%d)", ch->hw_chid);
444
445 return s;
446}
447
448static void gk20a_semaphore_free(struct kref *ref)
449{
450 struct gk20a_semaphore *s =
451 container_of(ref, struct gk20a_semaphore, ref);
452
453 gk20a_semaphore_pool_put(s->hw_sema->p);
454
455 kfree(s);
456}
457
458void gk20a_semaphore_put(struct gk20a_semaphore *s)
459{
460 kref_put(&s->ref, gk20a_semaphore_free);
461}
462
463void gk20a_semaphore_get(struct gk20a_semaphore *s)
464{
465 kref_get(&s->ref);
466}
diff --git a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h b/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h
deleted file mode 100644
index 8e09fcfc..00000000
--- a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h
+++ /dev/null
@@ -1,318 +0,0 @@
1/*
2 * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 */
13
14#ifndef SEMAPHORE_GK20A_H
15#define SEMAPHORE_GK20A_H
16
17#include <linux/kref.h>
18#include <linux/list.h>
19#include <linux/delay.h>
20
21#include <nvgpu/allocator.h>
22
23#include "gk20a.h"
24#include "mm_gk20a.h"
25#include "channel_gk20a.h"
26
27#define gpu_sema_dbg(fmt, args...) \
28 gk20a_dbg(gpu_dbg_sema, fmt, ##args)
29#define gpu_sema_verbose_dbg(fmt, args...) \
30 gk20a_dbg(gpu_dbg_sema_v, fmt, ##args)
31
32/*
33 * Max number of channels that can be used is 512. This of course needs to be
34 * fixed to be dynamic but still fast.
35 */
36#define SEMAPHORE_POOL_COUNT 512
37#define SEMAPHORE_SIZE 16
38#define SEMAPHORE_SEA_GROWTH_RATE 32
39
40struct gk20a_semaphore_sea;
41
42/*
43 * Underlying semaphore data structure. This semaphore can be shared amongst
44 * other semaphore instances.
45 */
46struct gk20a_semaphore_int {
47 int idx; /* Semaphore index. */
48 u32 offset; /* Offset into the pool. */
49 atomic_t next_value; /* Next available value. */
50 u32 *value; /* Current value (access w/ readl()). */
51 u32 nr_incrs; /* Number of increments programmed. */
52 struct gk20a_semaphore_pool *p; /* Pool that owns this sema. */
53 struct channel_gk20a *ch; /* Channel that owns this sema. */
54 struct list_head hw_sema_list; /* List of HW semaphores. */
55};
56
57/*
58 * A semaphore which the rest of the driver actually uses. This consists of a
59 * pointer to a real semaphore and a value to wait for. This allows one physical
60 * semaphore to be shared among an essentially infinite number of submits.
61 */
62struct gk20a_semaphore {
63 struct gk20a_semaphore_int *hw_sema;
64
65 atomic_t value;
66 int incremented;
67
68 struct kref ref;
69};
70
71/*
72 * A semaphore pool. Each address space will own exactly one of these.
73 */
74struct gk20a_semaphore_pool {
75 struct page *page; /* This pool's page of memory */
76 struct list_head pool_list_entry; /* Node for list of pools. */
77 void *cpu_va; /* CPU access to the pool. */
78 u64 gpu_va; /* GPU access to the pool. */
79 u64 gpu_va_ro; /* GPU access to the pool. */
80 int page_idx; /* Index into sea bitmap. */
81
82 struct list_head hw_semas; /* List of HW semas. */
83 DECLARE_BITMAP(semas_alloced, PAGE_SIZE / SEMAPHORE_SIZE);
84
85 struct gk20a_semaphore_sea *sema_sea; /* Sea that owns this pool. */
86
87 struct mutex pool_lock;
88
89 /*
90 * This is the address spaces's personal RW table. Other channels will
91 * ultimately map this page as RO.
92 */
93 struct sg_table *rw_sg_table;
94
95 /*
96 * This is to keep track of whether the pool has had its sg_table
97 * updated during sea resizing.
98 */
99 struct sg_table *ro_sg_table;
100
101 int mapped;
102
103 /*
104 * Sometimes a channel can be released before other channels are
105 * done waiting on it. This ref count ensures that the pool doesn't
106 * go away until all semaphores using this pool are cleaned up first.
107 */
108 struct kref ref;
109};
110
111/*
112 * A sea of semaphores pools. Each pool is owned by a single VM. Since multiple
113 * channels can share a VM each channel gets it's own HW semaphore from the
114 * pool. Channels then allocate regular semaphores - basically just a value that
115 * signifies when a particular job is done.
116 */
117struct gk20a_semaphore_sea {
118 struct list_head pool_list; /* List of pools in this sea. */
119 struct gk20a *gk20a;
120
121 size_t size; /* Number of pages available. */
122 u64 gpu_va; /* GPU virtual address of sema sea. */
123 u64 map_size; /* Size of the mapping. */
124
125 /*
126 * TODO:
127 * List of pages that we use to back the pools. The number of pages
128 * can grow dynamically since allocating 512 pages for all channels at
129 * once would be a tremendous waste.
130 */
131 int page_count; /* Pages allocated to pools. */
132
133 struct sg_table *ro_sg_table;
134 /*
135 struct page *pages[SEMAPHORE_POOL_COUNT];
136 */
137
138 struct mem_desc sea_mem;
139
140 /*
141 * Can't use a regular allocator here since the full range of pools are
142 * not always allocated. Instead just use a bitmap.
143 */
144 DECLARE_BITMAP(pools_alloced, SEMAPHORE_POOL_COUNT);
145
146 struct mutex sea_lock; /* Lock alloc/free calls. */
147};
148
149enum gk20a_mem_rw_flag {
150 gk20a_mem_flag_none = 0,
151 gk20a_mem_flag_read_only = 1,
152 gk20a_mem_flag_write_only = 2,
153};
154
155/*
156 * Semaphore sea functions.
157 */
158struct gk20a_semaphore_sea *gk20a_semaphore_sea_create(struct gk20a *gk20a);
159int gk20a_semaphore_sea_map(struct gk20a_semaphore_pool *sea,
160 struct vm_gk20a *vm);
161void gk20a_semaphore_sea_unmap(struct gk20a_semaphore_pool *sea,
162 struct vm_gk20a *vm);
163struct gk20a_semaphore_sea *gk20a_semaphore_get_sea(struct gk20a *g);
164
165/*
166 * Semaphore pool functions.
167 */
168struct gk20a_semaphore_pool *gk20a_semaphore_pool_alloc(
169 struct gk20a_semaphore_sea *sea);
170int gk20a_semaphore_pool_map(struct gk20a_semaphore_pool *pool,
171 struct vm_gk20a *vm);
172void gk20a_semaphore_pool_unmap(struct gk20a_semaphore_pool *pool,
173 struct vm_gk20a *vm);
174u64 __gk20a_semaphore_pool_gpu_va(struct gk20a_semaphore_pool *p, bool global);
175void gk20a_semaphore_pool_get(struct gk20a_semaphore_pool *p);
176void gk20a_semaphore_pool_put(struct gk20a_semaphore_pool *p);
177
178/*
179 * Semaphore functions.
180 */
181struct gk20a_semaphore *gk20a_semaphore_alloc(struct channel_gk20a *ch);
182void gk20a_semaphore_put(struct gk20a_semaphore *s);
183void gk20a_semaphore_get(struct gk20a_semaphore *s);
184void gk20a_semaphore_free_hw_sema(struct channel_gk20a *ch);
185
186/*
187 * Return the address of a specific semaphore.
188 *
189 * Don't call this on a semaphore you don't own - the VA returned will make no
190 * sense in your specific channel's VM.
191 */
192static inline u64 gk20a_semaphore_gpu_rw_va(struct gk20a_semaphore *s)
193{
194 return __gk20a_semaphore_pool_gpu_va(s->hw_sema->p, false) +
195 s->hw_sema->offset;
196}
197
198/*
199 * Get the global RO address for the semaphore. Can be called on any semaphore
200 * regardless of whether you own it.
201 */
202static inline u64 gk20a_semaphore_gpu_ro_va(struct gk20a_semaphore *s)
203{
204 return __gk20a_semaphore_pool_gpu_va(s->hw_sema->p, true) +
205 s->hw_sema->offset;
206}
207
208static inline u64 gk20a_hw_sema_addr(struct gk20a_semaphore_int *hw_sema)
209{
210 return __gk20a_semaphore_pool_gpu_va(hw_sema->p, true) +
211 hw_sema->offset;
212}
213
214/*
215 * TODO: handle wrap around... Hmm, how to do this?
216 */
217static inline bool gk20a_semaphore_is_released(struct gk20a_semaphore *s)
218{
219 u32 sema_val = readl(s->hw_sema->value);
220
221 /*
222 * If the underlying semaphore value is greater than or equal to
223 * the value of the semaphore then the semaphore has been signaled
224 * (a.k.a. released).
225 */
226 return (int)sema_val >= atomic_read(&s->value);
227}
228
229static inline bool gk20a_semaphore_is_acquired(struct gk20a_semaphore *s)
230{
231 return !gk20a_semaphore_is_released(s);
232}
233
234/*
235 * Read the underlying value from a semaphore.
236 */
237static inline u32 gk20a_semaphore_read(struct gk20a_semaphore *s)
238{
239 return readl(s->hw_sema->value);
240}
241
242static inline u32 gk20a_semaphore_get_value(struct gk20a_semaphore *s)
243{
244 return (u32)atomic_read(&s->value);
245}
246
247static inline u32 gk20a_semaphore_next_value(struct gk20a_semaphore *s)
248{
249 return (u32)atomic_read(&s->hw_sema->next_value);
250}
251
252/*
253 * If @force is set then this will not wait for the underlying semaphore to
254 * catch up to the passed semaphore.
255 */
256static inline void __gk20a_semaphore_release(struct gk20a_semaphore *s,
257 bool force)
258{
259 u32 current_val;
260 u32 val = gk20a_semaphore_get_value(s);
261 int attempts = 0;
262
263 /*
264 * Wait until the sema value is 1 less than the write value. That
265 * way this function is essentially an increment.
266 *
267 * TODO: tune the wait a little better.
268 */
269 while ((current_val = gk20a_semaphore_read(s)) < (val - 1)) {
270 if (force)
271 break;
272 msleep(100);
273 attempts += 1;
274 if (attempts > 100) {
275 WARN(1, "Stall on sema release!");
276 return;
277 }
278 }
279
280 /*
281 * If the semaphore has already passed the value we would write then
282 * this is really just a NO-OP.
283 */
284 if (current_val >= val)
285 return;
286
287 writel(val, s->hw_sema->value);
288
289 gpu_sema_verbose_dbg("(c=%d) WRITE %u",
290 s->hw_sema->ch->hw_chid, val);
291}
292
293static inline void gk20a_semaphore_release(struct gk20a_semaphore *s)
294{
295 __gk20a_semaphore_release(s, false);
296}
297
298/*
299 * Configure a software based increment on this semaphore. This is useful for
300 * when we want the GPU to wait on a SW event before processing a channel.
301 * Another way to describe this is when the GPU needs to wait on a SW pre-fence.
302 * The pre-fence signals SW which in turn calls gk20a_semaphore_release() which
303 * then allows the GPU to continue.
304 *
305 * Also used to prep a semaphore for an INCR by the GPU.
306 */
307static inline void gk20a_semaphore_incr(struct gk20a_semaphore *s)
308{
309 BUG_ON(s->incremented);
310
311 atomic_set(&s->value, atomic_add_return(1, &s->hw_sema->next_value));
312 s->incremented = 1;
313
314 gpu_sema_verbose_dbg("INCR sema for c=%d (%u)",
315 s->hw_sema->ch->hw_chid,
316 gk20a_semaphore_next_value(s));
317}
318#endif
diff --git a/drivers/gpu/nvgpu/gk20a/sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/sync_gk20a.c
index b642981c..e7bacac8 100644
--- a/drivers/gpu/nvgpu/gk20a/sync_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/sync_gk20a.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * GK20A Sync Framework Integration 2 * GK20A Sync Framework Integration
3 * 3 *
4 * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved. 4 * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify it 6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 7 * under the terms and conditions of the GNU General Public License,
@@ -13,8 +13,6 @@
13 * more details. 13 * more details.
14 */ 14 */
15 15
16#include "sync_gk20a.h"
17
18#include <linux/version.h> 16#include <linux/version.h>
19#include <linux/kernel.h> 17#include <linux/kernel.h>
20#include <linux/file.h> 18#include <linux/file.h>
@@ -23,9 +21,14 @@
23#include <linux/module.h> 21#include <linux/module.h>
24#include <linux/slab.h> 22#include <linux/slab.h>
25#include <linux/spinlock.h> 23#include <linux/spinlock.h>
24
26#include <uapi/linux/nvgpu.h> 25#include <uapi/linux/nvgpu.h>
26
27#include <nvgpu/semaphore.h>
28
27#include "../drivers/staging/android/sync.h" 29#include "../drivers/staging/android/sync.h"
28#include "semaphore_gk20a.h" 30
31#include "sync_gk20a.h"
29 32
30static const struct sync_timeline_ops gk20a_sync_timeline_ops; 33static const struct sync_timeline_ops gk20a_sync_timeline_ops;
31 34