aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/video/tegra/nvmap
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/video/tegra/nvmap')
-rw-r--r--drivers/video/tegra/nvmap/Makefile7
-rw-r--r--drivers/video/tegra/nvmap/nvmap.c871
-rw-r--r--drivers/video/tegra/nvmap/nvmap.h271
-rw-r--r--drivers/video/tegra/nvmap/nvmap_common.h38
-rw-r--r--drivers/video/tegra/nvmap/nvmap_dev.c1436
-rw-r--r--drivers/video/tegra/nvmap/nvmap_handle.c1020
-rw-r--r--drivers/video/tegra/nvmap/nvmap_heap.c1113
-rw-r--r--drivers/video/tegra/nvmap/nvmap_heap.h68
-rw-r--r--drivers/video/tegra/nvmap/nvmap_ioctl.c749
-rw-r--r--drivers/video/tegra/nvmap/nvmap_ioctl.h159
-rw-r--r--drivers/video/tegra/nvmap/nvmap_mru.c187
-rw-r--r--drivers/video/tegra/nvmap/nvmap_mru.h84
12 files changed, 6003 insertions, 0 deletions
diff --git a/drivers/video/tegra/nvmap/Makefile b/drivers/video/tegra/nvmap/Makefile
new file mode 100644
index 00000000000..95d7f68836a
--- /dev/null
+++ b/drivers/video/tegra/nvmap/Makefile
@@ -0,0 +1,7 @@
1GCOV_PROFILE := y
2obj-y += nvmap.o
3obj-y += nvmap_dev.o
4obj-y += nvmap_handle.o
5obj-y += nvmap_heap.o
6obj-y += nvmap_ioctl.o
7obj-${CONFIG_NVMAP_RECLAIM_UNPINNED_VM} += nvmap_mru.o \ No newline at end of file
diff --git a/drivers/video/tegra/nvmap/nvmap.c b/drivers/video/tegra/nvmap/nvmap.c
new file mode 100644
index 00000000000..b4b6241618d
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap.c
@@ -0,0 +1,871 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap.c
3 *
4 * Memory manager for Tegra GPU
5 *
6 * Copyright (c) 2009-2011, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#include <linux/err.h>
24#include <linux/highmem.h>
25#include <linux/io.h>
26#include <linux/rbtree.h>
27#include <linux/vmalloc.h>
28#include <linux/wait.h>
29#include <linux/slab.h>
30
31#include <asm/pgtable.h>
32#include <asm/tlbflush.h>
33
34#include <mach/iovmm.h>
35#include <mach/nvmap.h>
36
37#include "nvmap.h"
38#include "nvmap_mru.h"
39
40/* private nvmap_handle flag for pinning duplicate detection */
41#define NVMAP_HANDLE_VISITED (0x1ul << 31)
42
43/* map the backing pages for a heap_pgalloc handle into its IOVMM area */
44static void map_iovmm_area(struct nvmap_handle *h)
45{
46 tegra_iovmm_addr_t va;
47 unsigned long i;
48
49 BUG_ON(!h->heap_pgalloc || !h->pgalloc.area);
50 BUG_ON(h->size & ~PAGE_MASK);
51 WARN_ON(!h->pgalloc.dirty);
52
53 for (va = h->pgalloc.area->iovm_start, i = 0;
54 va < (h->pgalloc.area->iovm_start + h->size);
55 i++, va += PAGE_SIZE) {
56 BUG_ON(!pfn_valid(page_to_pfn(h->pgalloc.pages[i])));
57 tegra_iovmm_vm_insert_pfn(h->pgalloc.area, va,
58 page_to_pfn(h->pgalloc.pages[i]));
59 }
60 h->pgalloc.dirty = false;
61}
62
63/* must be called inside nvmap_pin_lock, to ensure that an entire stream
64 * of pins will complete without racing with a second stream. handle should
65 * have nvmap_handle_get (or nvmap_validate_get) called before calling
66 * this function. */
67static int pin_locked(struct nvmap_client *client, struct nvmap_handle *h)
68{
69 struct tegra_iovmm_area *area;
70 BUG_ON(!h->alloc);
71
72 nvmap_mru_lock(client->share);
73 if (atomic_inc_return(&h->pin) == 1) {
74 if (h->heap_pgalloc && !h->pgalloc.contig) {
75 area = nvmap_handle_iovmm_locked(client, h);
76 if (!area) {
77 /* no race here, inside the pin mutex */
78 atomic_dec(&h->pin);
79 nvmap_mru_unlock(client->share);
80 return -ENOMEM;
81 }
82 if (area != h->pgalloc.area)
83 h->pgalloc.dirty = true;
84 h->pgalloc.area = area;
85 }
86 }
87 nvmap_mru_unlock(client->share);
88 return 0;
89}
90
91/* doesn't need to be called inside nvmap_pin_lock, since this will only
92 * expand the available VM area */
93static int handle_unpin(struct nvmap_client *client,
94 struct nvmap_handle *h, int free_vm)
95{
96 int ret = 0;
97 nvmap_mru_lock(client->share);
98
99 if (atomic_read(&h->pin) == 0) {
100 nvmap_err(client, "%s unpinning unpinned handle %p\n",
101 current->group_leader->comm, h);
102 nvmap_mru_unlock(client->share);
103 return 0;
104 }
105
106 BUG_ON(!h->alloc);
107
108 if (!atomic_dec_return(&h->pin)) {
109 if (h->heap_pgalloc && h->pgalloc.area) {
110 /* if a secure handle is clean (i.e., mapped into
111 * IOVMM, it needs to be zapped on unpin. */
112 if (h->secure && !h->pgalloc.dirty) {
113 tegra_iovmm_zap_vm(h->pgalloc.area);
114 h->pgalloc.dirty = true;
115 }
116 if (free_vm) {
117 tegra_iovmm_free_vm(h->pgalloc.area);
118 h->pgalloc.area = NULL;
119 } else
120 nvmap_mru_insert_locked(client->share, h);
121 ret = 1;
122 }
123 }
124
125 nvmap_mru_unlock(client->share);
126 nvmap_handle_put(h);
127 return ret;
128}
129
130static int pin_array_locked(struct nvmap_client *client,
131 struct nvmap_handle **h, int count)
132{
133 int pinned;
134 int i;
135 int err = 0;
136
137 for (pinned = 0; pinned < count; pinned++) {
138 err = pin_locked(client, h[pinned]);
139 if (err)
140 break;
141 }
142
143 if (err) {
144 /* unpin pinned handles */
145 for (i = 0; i < pinned; i++) {
146 /* inc ref counter, because
147 * handle_unpin decrements it */
148 nvmap_handle_get(h[i]);
149 /* unpin handles and free vm */
150 handle_unpin(client, h[i], true);
151 }
152 }
153
154 if (err && tegra_iovmm_get_max_free(client->share->iovmm) >=
155 client->iovm_limit) {
156 /* First attempt to pin in empty iovmm
157 * may still fail because of fragmentation caused by
158 * placing handles in MRU areas. After such failure
159 * all MRU gets cleaned and iovm space is freed.
160 *
161 * We have to do pinning again here since there might be is
162 * no more incoming pin_wait wakeup calls from unpin
163 * operations */
164 for (pinned = 0; pinned < count; pinned++) {
165 err = pin_locked(client, h[pinned]);
166 if (err)
167 break;
168 }
169 if (err) {
170 pr_err("Pinning in empty iovmm failed!!!\n");
171 BUG_ON(1);
172 }
173 }
174 return err;
175}
176
177static int wait_pin_array_locked(struct nvmap_client *client,
178 struct nvmap_handle **h, int count)
179{
180 int ret = 0;
181
182 ret = pin_array_locked(client, h, count);
183
184 if (ret) {
185 ret = wait_event_interruptible(client->share->pin_wait,
186 !pin_array_locked(client, h, count));
187 }
188 return ret ? -EINTR : 0;
189}
190
191static int handle_unpin_noref(struct nvmap_client *client, unsigned long id)
192{
193 struct nvmap_handle *h;
194 int w;
195
196 h = nvmap_validate_get(client, id);
197 if (unlikely(!h)) {
198 nvmap_err(client, "%s attempting to unpin invalid handle %p\n",
199 current->group_leader->comm, (void *)id);
200 return 0;
201 }
202
203 nvmap_err(client, "%s unpinning unreferenced handle %p\n",
204 current->group_leader->comm, h);
205 WARN_ON(1);
206
207 w = handle_unpin(client, h, false);
208 nvmap_handle_put(h);
209 return w;
210}
211
212void nvmap_unpin_ids(struct nvmap_client *client,
213 unsigned int nr, const unsigned long *ids)
214{
215 unsigned int i;
216 int do_wake = 0;
217
218 for (i = 0; i < nr; i++) {
219 struct nvmap_handle_ref *ref;
220
221 if (!ids[i])
222 continue;
223
224 nvmap_ref_lock(client);
225 ref = _nvmap_validate_id_locked(client, ids[i]);
226 if (ref) {
227 struct nvmap_handle *h = ref->handle;
228 int e = atomic_add_unless(&ref->pin, -1, 0);
229
230 nvmap_ref_unlock(client);
231
232 if (!e) {
233 nvmap_err(client, "%s unpinning unpinned "
234 "handle %08lx\n",
235 current->group_leader->comm, ids[i]);
236 } else {
237 do_wake |= handle_unpin(client, h, false);
238 }
239 } else {
240 nvmap_ref_unlock(client);
241 if (client->super)
242 do_wake |= handle_unpin_noref(client, ids[i]);
243 else
244 nvmap_err(client, "%s unpinning invalid "
245 "handle %08lx\n",
246 current->group_leader->comm, ids[i]);
247 }
248 }
249
250 if (do_wake)
251 wake_up(&client->share->pin_wait);
252}
253
254/* pins a list of handle_ref objects; same conditions apply as to
255 * _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */
256int nvmap_pin_ids(struct nvmap_client *client,
257 unsigned int nr, const unsigned long *ids)
258{
259 int ret = 0;
260 unsigned int i;
261 struct nvmap_handle **h = (struct nvmap_handle **)ids;
262 struct nvmap_handle_ref *ref;
263
264 /* to optimize for the common case (client provided valid handle
265 * references and the pin succeeds), increment the handle_ref pin
266 * count during validation. in error cases, the tree will need to
267 * be re-walked, since the handle_ref is discarded so that an
268 * allocation isn't required. if a handle_ref is not found,
269 * locally validate that the caller has permission to pin the handle;
270 * handle_refs are not created in this case, so it is possible that
271 * if the caller crashes after pinning a global handle, the handle
272 * will be permanently leaked. */
273 nvmap_ref_lock(client);
274 for (i = 0; i < nr && !ret; i++) {
275 ref = _nvmap_validate_id_locked(client, ids[i]);
276 if (ref) {
277 atomic_inc(&ref->pin);
278 nvmap_handle_get(h[i]);
279 } else {
280 struct nvmap_handle *verify;
281 nvmap_ref_unlock(client);
282 verify = nvmap_validate_get(client, ids[i]);
283 if (verify)
284 nvmap_warn(client, "%s pinning unreferenced "
285 "handle %p\n",
286 current->group_leader->comm, h[i]);
287 else
288 ret = -EPERM;
289 nvmap_ref_lock(client);
290 }
291 }
292 nvmap_ref_unlock(client);
293
294 nr = i;
295
296 if (ret)
297 goto out;
298
299 ret = mutex_lock_interruptible(&client->share->pin_lock);
300 if (WARN_ON(ret))
301 goto out;
302
303 ret = wait_pin_array_locked(client, h, nr);
304
305 mutex_unlock(&client->share->pin_lock);
306
307 if (ret) {
308 ret = -EINTR;
309 } else {
310 for (i = 0; i < nr; i++) {
311 if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
312 map_iovmm_area(h[i]);
313 }
314 }
315
316out:
317 if (ret) {
318 nvmap_ref_lock(client);
319 for (i = 0; i < nr; i++) {
320 ref = _nvmap_validate_id_locked(client, ids[i]);
321 if (!ref) {
322 nvmap_warn(client, "%s freed handle %p "
323 "during pinning\n",
324 current->group_leader->comm,
325 (void *)ids[i]);
326 continue;
327 }
328 atomic_dec(&ref->pin);
329 }
330 nvmap_ref_unlock(client);
331
332 for (i = 0; i < nr; i++)
333 nvmap_handle_put(h[i]);
334 }
335
336 return ret;
337}
338
339static phys_addr_t handle_phys(struct nvmap_handle *h)
340{
341 phys_addr_t addr;
342
343 if (h->heap_pgalloc && h->pgalloc.contig) {
344 addr = page_to_phys(h->pgalloc.pages[0]);
345 } else if (h->heap_pgalloc) {
346 BUG_ON(!h->pgalloc.area);
347 addr = h->pgalloc.area->iovm_start;
348 } else {
349 addr = h->carveout->base;
350 }
351
352 return addr;
353}
354
355/* stores the physical address (+offset) of each handle relocation entry
356 * into its output location. see nvmap_pin_array for more details.
357 *
358 * each entry in arr (i.e., each relocation request) specifies two handles:
359 * the handle to pin (pin), and the handle where the address of pin should be
360 * written (patch). in pseudocode, this loop basically looks like:
361 *
362 * for (i = 0; i < nr; i++) {
363 * (pin, pin_offset, patch, patch_offset) = arr[i];
364 * patch[patch_offset] = address_of(pin) + pin_offset;
365 * }
366 */
367static int nvmap_reloc_pin_array(struct nvmap_client *client,
368 const struct nvmap_pinarray_elem *arr,
369 int nr, struct nvmap_handle *gather)
370{
371 struct nvmap_handle *last_patch = NULL;
372 unsigned int last_pfn = 0;
373 pte_t **pte;
374 void *addr;
375 int i;
376
377 pte = nvmap_alloc_pte(client->dev, &addr);
378 if (IS_ERR(pte))
379 return PTR_ERR(pte);
380
381 for (i = 0; i < nr; i++) {
382 struct nvmap_handle *patch;
383 struct nvmap_handle *pin;
384 phys_addr_t reloc_addr;
385 phys_addr_t phys;
386 unsigned int pfn;
387
388 /* all of the handles are validated and get'ted prior to
389 * calling this function, so casting is safe here */
390 pin = (struct nvmap_handle *)arr[i].pin_mem;
391
392 if (arr[i].patch_mem == (unsigned long)last_patch) {
393 patch = last_patch;
394 } else if (arr[i].patch_mem == (unsigned long)gather) {
395 patch = gather;
396 } else {
397 if (last_patch)
398 nvmap_handle_put(last_patch);
399
400 patch = nvmap_get_handle_id(client, arr[i].patch_mem);
401 if (!patch) {
402 nvmap_free_pte(client->dev, pte);
403 return -EPERM;
404 }
405 last_patch = patch;
406 }
407
408 if (patch->heap_pgalloc) {
409 unsigned int page = arr[i].patch_offset >> PAGE_SHIFT;
410 phys = page_to_phys(patch->pgalloc.pages[page]);
411 phys += (arr[i].patch_offset & ~PAGE_MASK);
412 } else {
413 phys = patch->carveout->base + arr[i].patch_offset;
414 }
415
416 pfn = __phys_to_pfn(phys);
417 if (pfn != last_pfn) {
418 pgprot_t prot = nvmap_pgprot(patch, pgprot_kernel);
419 phys_addr_t kaddr = (phys_addr_t)addr;
420 set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, prot));
421 flush_tlb_kernel_page(kaddr);
422 last_pfn = pfn;
423 }
424
425 reloc_addr = handle_phys(pin) + arr[i].pin_offset;
426 reloc_addr >>= arr[i].reloc_shift;
427 __raw_writel(reloc_addr, addr + (phys & ~PAGE_MASK));
428 }
429
430 nvmap_free_pte(client->dev, pte);
431
432 if (last_patch)
433 nvmap_handle_put(last_patch);
434
435 wmb();
436
437 return 0;
438}
439
440static int nvmap_validate_get_pin_array(struct nvmap_client *client,
441 const struct nvmap_pinarray_elem *arr,
442 int nr, struct nvmap_handle **h)
443{
444 int i;
445 int ret = 0;
446 int count = 0;
447
448 nvmap_ref_lock(client);
449
450 for (i = 0; i < nr; i++) {
451 struct nvmap_handle_ref *ref;
452
453 if (need_resched()) {
454 nvmap_ref_unlock(client);
455 schedule();
456 nvmap_ref_lock(client);
457 }
458
459 ref = _nvmap_validate_id_locked(client, arr[i].pin_mem);
460
461 if (!ref)
462 nvmap_warn(client, "falied to validate id\n");
463 else if (!ref->handle)
464 nvmap_warn(client, "id had no associated handle\n");
465 else if (!ref->handle->alloc)
466 nvmap_warn(client, "handle had no allocation\n");
467
468 if (!ref || !ref->handle || !ref->handle->alloc) {
469 ret = -EPERM;
470 break;
471 }
472
473 /* a handle may be referenced multiple times in arr, but
474 * it will only be pinned once; this ensures that the
475 * minimum number of sync-queue slots in the host driver
476 * are dedicated to storing unpin lists, which allows
477 * for greater parallelism between the CPU and graphics
478 * processor */
479 if (ref->handle->flags & NVMAP_HANDLE_VISITED)
480 continue;
481
482 ref->handle->flags |= NVMAP_HANDLE_VISITED;
483
484 h[count] = nvmap_handle_get(ref->handle);
485 BUG_ON(!h[count]);
486 count++;
487 }
488
489 nvmap_ref_unlock(client);
490
491 if (ret) {
492 for (i = 0; i < count; i++) {
493 h[i]->flags &= ~NVMAP_HANDLE_VISITED;
494 nvmap_handle_put(h[i]);
495 }
496 }
497
498 return ret ?: count;
499}
500
501/* a typical mechanism host1x clients use for using the Tegra graphics
502 * processor is to build a command buffer which contains relocatable
503 * memory handle commands, and rely on the kernel to convert these in-place
504 * to addresses which are understood by the GPU hardware.
505 *
506 * this is implemented by having clients provide a sideband array
507 * of relocatable handles (+ offsets) and the location in the command
508 * buffer handle to patch with the GPU address when the client submits
509 * its command buffer to the host1x driver.
510 *
511 * the host driver also uses this relocation mechanism internally to
512 * relocate the client's (unpinned) command buffers into host-addressable
513 * memory.
514 *
515 * @client: nvmap_client which should be used for validation; should be
516 * owned by the process which is submitting command buffers
517 * @gather: special handle for relocated command buffer outputs used
518 * internally by the host driver. if this handle is encountered
519 * as an output handle in the relocation array, it is assumed
520 * to be a known-good output and is not validated.
521 * @arr: array of ((relocatable handle, offset), (output handle, offset))
522 * tuples.
523 * @nr: number of entries in arr
524 * @unique_arr: list of nvmap_handle objects which were pinned by
525 * nvmap_pin_array. must be unpinned by the caller after the
526 * command buffers referenced in gather have completed.
527 */
528int nvmap_pin_array(struct nvmap_client *client, struct nvmap_handle *gather,
529 const struct nvmap_pinarray_elem *arr, int nr,
530 struct nvmap_handle **unique_arr)
531{
532 int count = 0;
533 int ret = 0;
534 int i;
535
536 if (mutex_lock_interruptible(&client->share->pin_lock)) {
537 nvmap_warn(client, "%s interrupted when acquiring pin lock\n",
538 current->group_leader->comm);
539 return -EINTR;
540 }
541
542 count = nvmap_validate_get_pin_array(client, arr, nr, unique_arr);
543 if (count < 0) {
544 mutex_unlock(&client->share->pin_lock);
545 nvmap_warn(client, "failed to validate pin array\n");
546 return count;
547 }
548
549 for (i = 0; i < count; i++)
550 unique_arr[i]->flags &= ~NVMAP_HANDLE_VISITED;
551
552 ret = wait_pin_array_locked(client, unique_arr, count);
553
554 mutex_unlock(&client->share->pin_lock);
555
556 if (!ret)
557 ret = nvmap_reloc_pin_array(client, arr, nr, gather);
558
559 if (WARN_ON(ret)) {
560 for (i = 0; i < count; i++)
561 nvmap_handle_put(unique_arr[i]);
562 return ret;
563 } else {
564 for (i = 0; i < count; i++) {
565 if (unique_arr[i]->heap_pgalloc &&
566 unique_arr[i]->pgalloc.dirty)
567 map_iovmm_area(unique_arr[i]);
568 }
569 }
570
571 return count;
572}
573
574phys_addr_t nvmap_pin(struct nvmap_client *client,
575 struct nvmap_handle_ref *ref)
576{
577 struct nvmap_handle *h;
578 phys_addr_t phys;
579 int ret = 0;
580
581 h = nvmap_handle_get(ref->handle);
582 if (WARN_ON(!h))
583 return -EINVAL;
584
585 atomic_inc(&ref->pin);
586
587 if (WARN_ON(mutex_lock_interruptible(&client->share->pin_lock))) {
588 ret = -EINTR;
589 } else {
590 ret = wait_pin_array_locked(client, &h, 1);
591 mutex_unlock(&client->share->pin_lock);
592 }
593
594 if (ret) {
595 atomic_dec(&ref->pin);
596 nvmap_handle_put(h);
597 } else {
598 if (h->heap_pgalloc && h->pgalloc.dirty)
599 map_iovmm_area(h);
600 phys = handle_phys(h);
601 }
602
603 return ret ?: phys;
604}
605
606phys_addr_t nvmap_handle_address(struct nvmap_client *c, unsigned long id)
607{
608 struct nvmap_handle *h;
609 phys_addr_t phys;
610
611 h = nvmap_get_handle_id(c, id);
612 if (!h)
613 return -EPERM;
614 mutex_lock(&h->lock);
615 phys = handle_phys(h);
616 mutex_unlock(&h->lock);
617 nvmap_handle_put(h);
618
619 return phys;
620}
621
622void nvmap_unpin(struct nvmap_client *client, struct nvmap_handle_ref *ref)
623{
624 if (!ref)
625 return;
626
627 atomic_dec(&ref->pin);
628 if (handle_unpin(client, ref->handle, false))
629 wake_up(&client->share->pin_wait);
630}
631
632void nvmap_unpin_handles(struct nvmap_client *client,
633 struct nvmap_handle **h, int nr)
634{
635 int i;
636 int do_wake = 0;
637
638 for (i = 0; i < nr; i++) {
639 if (WARN_ON(!h[i]))
640 continue;
641 do_wake |= handle_unpin(client, h[i], false);
642 }
643
644 if (do_wake)
645 wake_up(&client->share->pin_wait);
646}
647
648void *nvmap_mmap(struct nvmap_handle_ref *ref)
649{
650 struct nvmap_handle *h;
651 pgprot_t prot;
652 unsigned long adj_size;
653 unsigned long offs;
654 struct vm_struct *v;
655 void *p;
656
657 h = nvmap_handle_get(ref->handle);
658 if (!h)
659 return NULL;
660
661 prot = nvmap_pgprot(h, pgprot_kernel);
662
663 if (h->heap_pgalloc)
664 return vm_map_ram(h->pgalloc.pages, h->size >> PAGE_SHIFT,
665 -1, prot);
666
667 /* carveout - explicitly map the pfns into a vmalloc area */
668
669 nvmap_usecount_inc(h);
670
671 adj_size = h->carveout->base & ~PAGE_MASK;
672 adj_size += h->size;
673 adj_size = PAGE_ALIGN(adj_size);
674
675 v = alloc_vm_area(adj_size);
676 if (!v) {
677 nvmap_usecount_dec(h);
678 nvmap_handle_put(h);
679 return NULL;
680 }
681
682 p = v->addr + (h->carveout->base & ~PAGE_MASK);
683
684 for (offs = 0; offs < adj_size; offs += PAGE_SIZE) {
685 unsigned long addr = (unsigned long) v->addr + offs;
686 unsigned int pfn;
687 pgd_t *pgd;
688 pud_t *pud;
689 pmd_t *pmd;
690 pte_t *pte;
691
692 pfn = __phys_to_pfn(h->carveout->base + offs);
693 pgd = pgd_offset_k(addr);
694 pud = pud_alloc(&init_mm, pgd, addr);
695 if (!pud)
696 break;
697 pmd = pmd_alloc(&init_mm, pud, addr);
698 if (!pmd)
699 break;
700 pte = pte_alloc_kernel(pmd, addr);
701 if (!pte)
702 break;
703 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
704 flush_tlb_kernel_page(addr);
705 }
706
707 if (offs != adj_size) {
708 free_vm_area(v);
709 nvmap_usecount_dec(h);
710 nvmap_handle_put(h);
711 return NULL;
712 }
713
714 /* leave the handle ref count incremented by 1, so that
715 * the handle will not be freed while the kernel mapping exists.
716 * nvmap_handle_put will be called by unmapping this address */
717 return p;
718}
719
720void nvmap_munmap(struct nvmap_handle_ref *ref, void *addr)
721{
722 struct nvmap_handle *h;
723
724 if (!ref)
725 return;
726
727 h = ref->handle;
728
729 if (h->heap_pgalloc) {
730 vm_unmap_ram(addr, h->size >> PAGE_SHIFT);
731 } else {
732 struct vm_struct *vm;
733 addr -= (h->carveout->base & ~PAGE_MASK);
734 vm = remove_vm_area(addr);
735 BUG_ON(!vm);
736 kfree(vm);
737 nvmap_usecount_dec(h);
738 }
739 nvmap_handle_put(h);
740}
741
742struct nvmap_handle_ref *nvmap_alloc(struct nvmap_client *client, size_t size,
743 size_t align, unsigned int flags,
744 unsigned int heap_mask)
745{
746 const unsigned int default_heap = (NVMAP_HEAP_SYSMEM |
747 NVMAP_HEAP_CARVEOUT_GENERIC);
748 struct nvmap_handle_ref *r = NULL;
749 int err;
750
751 if (heap_mask == 0)
752 heap_mask = default_heap;
753
754 r = nvmap_create_handle(client, size);
755 if (IS_ERR(r))
756 return r;
757
758 err = nvmap_alloc_handle_id(client, nvmap_ref_to_id(r),
759 heap_mask, align, flags);
760
761 if (err) {
762 nvmap_free_handle_id(client, nvmap_ref_to_id(r));
763 return ERR_PTR(err);
764 }
765
766 return r;
767}
768
769/* allocates memory with specifed iovm_start address. */
770struct nvmap_handle_ref *nvmap_alloc_iovm(struct nvmap_client *client,
771 size_t size, size_t align, unsigned int flags, unsigned int iovm_start)
772{
773 int err;
774 struct nvmap_handle *h;
775 struct nvmap_handle_ref *r;
776 const unsigned int default_heap = NVMAP_HEAP_IOVMM;
777
778 /* size need to be more than one page.
779 * otherwise heap preference would change to system heap.
780 */
781 if (size <= PAGE_SIZE)
782 size = PAGE_SIZE << 1;
783 r = nvmap_create_handle(client, size);
784 if (IS_ERR_OR_NULL(r))
785 return r;
786
787 h = r->handle;
788 h->pgalloc.iovm_addr = iovm_start;
789 err = nvmap_alloc_handle_id(client, nvmap_ref_to_id(r),
790 default_heap, align, flags);
791 if (err)
792 goto fail;
793
794 err = mutex_lock_interruptible(&client->share->pin_lock);
795 if (WARN_ON(err))
796 goto fail;
797 err = pin_locked(client, h);
798 mutex_unlock(&client->share->pin_lock);
799 if (err)
800 goto fail;
801 return r;
802
803fail:
804 nvmap_free_handle_id(client, nvmap_ref_to_id(r));
805 return ERR_PTR(err);
806}
807
808void nvmap_free_iovm(struct nvmap_client *client, struct nvmap_handle_ref *r)
809{
810 unsigned long ref_id = nvmap_ref_to_id(r);
811
812 nvmap_unpin_ids(client, 1, &ref_id);
813 nvmap_free_handle_id(client, ref_id);
814}
815
816void nvmap_free(struct nvmap_client *client, struct nvmap_handle_ref *r)
817{
818 if (!r)
819 return;
820
821 nvmap_free_handle_id(client, nvmap_ref_to_id(r));
822}
823
824/*
825 * create a mapping to the user's buffer and write it
826 * (uses similar logic from nvmap_reloc_pin_array to map the cmdbuf)
827 */
828int nvmap_patch_word(struct nvmap_client *client,
829 struct nvmap_handle *patch,
830 u32 patch_offset, u32 patch_value)
831{
832 phys_addr_t phys;
833 unsigned long kaddr;
834 unsigned int pfn;
835 void *addr;
836 pte_t **pte;
837 pgprot_t prot;
838
839 if (patch_offset >= patch->size) {
840 nvmap_warn(client, "read/write outside of handle\n");
841 return -EFAULT;
842 }
843
844 pte = nvmap_alloc_pte(client->dev, &addr);
845 if (IS_ERR(pte))
846 return PTR_ERR(pte);
847
848 /* derive physaddr of cmdbuf WAIT to patch */
849 if (patch->heap_pgalloc) {
850 unsigned int page = patch_offset >> PAGE_SHIFT;
851 phys = page_to_phys(patch->pgalloc.pages[page]);
852 phys += (patch_offset & ~PAGE_MASK);
853 } else {
854 phys = patch->carveout->base + patch_offset;
855 }
856
857 pfn = __phys_to_pfn(phys);
858 prot = nvmap_pgprot(patch, pgprot_kernel);
859 kaddr = (unsigned long)addr;
860
861 /* write PTE, so addr points to cmdbuf PFN */
862 set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, prot));
863 flush_tlb_kernel_page(kaddr);
864
865 /* write patch_value to addr + page offset */
866 __raw_writel(patch_value, addr + (phys & ~PAGE_MASK));
867
868 nvmap_free_pte(client->dev, pte);
869 wmb();
870 return 0;
871}
diff --git a/drivers/video/tegra/nvmap/nvmap.h b/drivers/video/tegra/nvmap/nvmap.h
new file mode 100644
index 00000000000..44a0d86b603
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap.h
@@ -0,0 +1,271 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap.h
3 *
4 * GPU memory management driver for Tegra
5 *
6 * Copyright (c) 2010-2012, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *'
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#ifndef __VIDEO_TEGRA_NVMAP_NVMAP_H
24#define __VIDEO_TEGRA_NVMAP_NVMAP_H
25
26#include <linux/list.h>
27#include <linux/mm.h>
28#include <linux/mutex.h>
29#include <linux/rbtree.h>
30#include <linux/sched.h>
31#include <linux/wait.h>
32#include <linux/atomic.h>
33#include <mach/nvmap.h>
34#include "nvmap_heap.h"
35
36struct nvmap_device;
37struct page;
38struct tegra_iovmm_area;
39
40#if defined(CONFIG_TEGRA_NVMAP)
41#define nvmap_err(_client, _fmt, ...) \
42 dev_err(nvmap_client_to_device(_client), \
43 "%s: "_fmt, __func__, ##__VA_ARGS__)
44
45#define nvmap_warn(_client, _fmt, ...) \
46 dev_warn(nvmap_client_to_device(_client), \
47 "%s: "_fmt, __func__, ##__VA_ARGS__)
48
49#define nvmap_debug(_client, _fmt, ...) \
50 dev_dbg(nvmap_client_to_device(_client), \
51 "%s: "_fmt, __func__, ##__VA_ARGS__)
52
53#define nvmap_ref_to_id(_ref) ((unsigned long)(_ref)->handle)
54
55/* handles allocated using shared system memory (either IOVMM- or high-order
56 * page allocations */
57struct nvmap_pgalloc {
58 struct page **pages;
59 struct tegra_iovmm_area *area;
60 struct list_head mru_list; /* MRU entry for IOVMM reclamation */
61 bool contig; /* contiguous system memory */
62 bool dirty; /* area is invalid and needs mapping */
63 u32 iovm_addr; /* is non-zero, if client need specific iova mapping */
64};
65
66struct nvmap_handle {
67 struct rb_node node; /* entry on global handle tree */
68 atomic_t ref; /* reference count (i.e., # of duplications) */
69 atomic_t pin; /* pin count */
70 unsigned int usecount; /* how often is used */
71 unsigned long flags;
72 size_t size; /* padded (as-allocated) size */
73 size_t orig_size; /* original (as-requested) size */
74 size_t align;
75 struct nvmap_client *owner;
76 struct nvmap_device *dev;
77 union {
78 struct nvmap_pgalloc pgalloc;
79 struct nvmap_heap_block *carveout;
80 };
81 bool global; /* handle may be duplicated by other clients */
82 bool secure; /* zap IOVMM area on unpin */
83 bool heap_pgalloc; /* handle is page allocated (sysmem / iovmm) */
84 bool alloc; /* handle has memory allocated */
85 unsigned int userflags; /* flags passed from userspace */
86 struct mutex lock;
87};
88
89#define NVMAP_DEFAULT_PAGE_POOL_SIZE 8192
90#define NVMAP_UC_POOL NVMAP_HANDLE_UNCACHEABLE
91#define NVMAP_WC_POOL NVMAP_HANDLE_WRITE_COMBINE
92#define NVMAP_IWB_POOL NVMAP_HANDLE_INNER_CACHEABLE
93#define NVMAP_WB_POOL NVMAP_HANDLE_CACHEABLE
94#define NVMAP_NUM_POOLS (NVMAP_HANDLE_CACHEABLE + 1)
95
96struct nvmap_page_pool {
97 struct mutex lock;
98 int npages;
99 struct page **page_array;
100 struct page **shrink_array;
101 int max_pages;
102 int flags;
103};
104
105int nvmap_page_pool_init(struct nvmap_page_pool *pool, int flags);
106
107struct nvmap_share {
108 struct tegra_iovmm_client *iovmm;
109 wait_queue_head_t pin_wait;
110 struct mutex pin_lock;
111 union {
112 struct nvmap_page_pool pools[NVMAP_NUM_POOLS];
113 struct {
114 struct nvmap_page_pool uc_pool;
115 struct nvmap_page_pool wc_pool;
116 struct nvmap_page_pool iwb_pool;
117 struct nvmap_page_pool wb_pool;
118 };
119 };
120#ifdef CONFIG_NVMAP_RECLAIM_UNPINNED_VM
121 struct mutex mru_lock;
122 struct list_head *mru_lists;
123 int nr_mru;
124#endif
125};
126
127struct nvmap_carveout_commit {
128 size_t commit;
129 struct list_head list;
130};
131
132struct nvmap_client {
133 const char *name;
134 struct nvmap_device *dev;
135 struct nvmap_share *share;
136 struct rb_root handle_refs;
137 atomic_t iovm_commit;
138 size_t iovm_limit;
139 struct mutex ref_lock;
140 bool super;
141 atomic_t count;
142 struct task_struct *task;
143 struct list_head list;
144 struct nvmap_carveout_commit carveout_commit[0];
145};
146
147struct nvmap_vma_priv {
148 struct nvmap_handle *handle;
149 size_t offs;
150 atomic_t count; /* number of processes cloning the VMA */
151};
152
153static inline void nvmap_ref_lock(struct nvmap_client *priv)
154{
155 mutex_lock(&priv->ref_lock);
156}
157
158static inline void nvmap_ref_unlock(struct nvmap_client *priv)
159{
160 mutex_unlock(&priv->ref_lock);
161}
162#endif /* CONFIG_TEGRA_NVMAP */
163
164struct device *nvmap_client_to_device(struct nvmap_client *client);
165
166pte_t **nvmap_alloc_pte(struct nvmap_device *dev, void **vaddr);
167
168pte_t **nvmap_alloc_pte_irq(struct nvmap_device *dev, void **vaddr);
169
170void nvmap_free_pte(struct nvmap_device *dev, pte_t **pte);
171
172void nvmap_usecount_inc(struct nvmap_handle *h);
173void nvmap_usecount_dec(struct nvmap_handle *h);
174
175struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *dev,
176 struct nvmap_handle *handle,
177 unsigned long type);
178
179unsigned long nvmap_carveout_usage(struct nvmap_client *c,
180 struct nvmap_heap_block *b);
181
182struct nvmap_carveout_node;
183void nvmap_carveout_commit_add(struct nvmap_client *client,
184 struct nvmap_carveout_node *node, size_t len);
185
186void nvmap_carveout_commit_subtract(struct nvmap_client *client,
187 struct nvmap_carveout_node *node,
188 size_t len);
189
190struct nvmap_share *nvmap_get_share_from_dev(struct nvmap_device *dev);
191
192struct nvmap_handle *nvmap_validate_get(struct nvmap_client *client,
193 unsigned long handle);
194
195struct nvmap_handle_ref *_nvmap_validate_id_locked(struct nvmap_client *priv,
196 unsigned long id);
197
198struct nvmap_handle *nvmap_get_handle_id(struct nvmap_client *client,
199 unsigned long id);
200
201struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
202 size_t size);
203
204struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
205 unsigned long id);
206
207int nvmap_alloc_handle_id(struct nvmap_client *client,
208 unsigned long id, unsigned int heap_mask,
209 size_t align, unsigned int flags);
210
211void nvmap_free_handle_id(struct nvmap_client *c, unsigned long id);
212
213int nvmap_pin_ids(struct nvmap_client *client,
214 unsigned int nr, const unsigned long *ids);
215
216void nvmap_unpin_ids(struct nvmap_client *priv,
217 unsigned int nr, const unsigned long *ids);
218
219void _nvmap_handle_free(struct nvmap_handle *h);
220
221int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h);
222
223void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h);
224
225#if defined(CONFIG_TEGRA_NVMAP)
226static inline struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h)
227{
228 if (unlikely(atomic_inc_return(&h->ref) <= 1)) {
229 pr_err("%s: %s getting a freed handle\n",
230 __func__, current->group_leader->comm);
231 if (atomic_read(&h->ref) <= 0)
232 return NULL;
233 }
234 return h;
235}
236
237static inline void nvmap_handle_put(struct nvmap_handle *h)
238{
239 int cnt = atomic_dec_return(&h->ref);
240
241 if (WARN_ON(cnt < 0)) {
242 pr_err("%s: %s put to negative references\n",
243 __func__, current->comm);
244 } else if (cnt == 0)
245 _nvmap_handle_free(h);
246}
247
248static inline pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot)
249{
250 if (h->flags == NVMAP_HANDLE_UNCACHEABLE)
251 return pgprot_noncached(prot);
252 else if (h->flags == NVMAP_HANDLE_WRITE_COMBINE)
253 return pgprot_writecombine(prot);
254 else if (h->flags == NVMAP_HANDLE_INNER_CACHEABLE)
255 return pgprot_inner_writeback(prot);
256 return prot;
257}
258#else /* CONFIG_TEGRA_NVMAP */
259struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h);
260void nvmap_handle_put(struct nvmap_handle *h);
261pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot);
262#endif /* !CONFIG_TEGRA_NVMAP */
263
264int is_nvmap_vma(struct vm_area_struct *vma);
265
266struct nvmap_handle_ref *nvmap_alloc_iovm(struct nvmap_client *client,
267 size_t size, size_t align, unsigned int flags, unsigned int iova_start);
268
269void nvmap_free_iovm(struct nvmap_client *client, struct nvmap_handle_ref *r);
270
271#endif
diff --git a/drivers/video/tegra/nvmap/nvmap_common.h b/drivers/video/tegra/nvmap/nvmap_common.h
new file mode 100644
index 00000000000..6da010720bb
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_common.h
@@ -0,0 +1,38 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_common.h
3 *
4 * GPU memory management driver for Tegra
5 *
6 * Copyright (c) 2011, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *'
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23extern void v7_flush_kern_cache_all(void *);
24extern void v7_clean_kern_cache_all(void *);
25
26#define FLUSH_CLEAN_BY_SET_WAY_THRESHOLD (8 * PAGE_SIZE)
27
28static inline void inner_flush_cache_all(void)
29{
30 on_each_cpu(v7_flush_kern_cache_all, NULL, 1);
31}
32
33static inline void inner_clean_cache_all(void)
34{
35 on_each_cpu(v7_clean_kern_cache_all, NULL, 1);
36}
37
38extern void __flush_dcache_page(struct address_space *, struct page *);
diff --git a/drivers/video/tegra/nvmap/nvmap_dev.c b/drivers/video/tegra/nvmap/nvmap_dev.c
new file mode 100644
index 00000000000..f84f38c93aa
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_dev.c
@@ -0,0 +1,1436 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_dev.c
3 *
4 * User-space interface to nvmap
5 *
6 * Copyright (c) 2011-2012, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#include <linux/backing-dev.h>
24#include <linux/bitmap.h>
25#include <linux/debugfs.h>
26#include <linux/delay.h>
27#include <linux/kernel.h>
28#include <linux/miscdevice.h>
29#include <linux/mm.h>
30#include <linux/oom.h>
31#include <linux/platform_device.h>
32#include <linux/seq_file.h>
33#include <linux/slab.h>
34#include <linux/spinlock.h>
35#include <linux/uaccess.h>
36#include <linux/vmalloc.h>
37
38#include <asm/cacheflush.h>
39#include <asm/tlbflush.h>
40
41#include <mach/iovmm.h>
42#include <mach/nvmap.h>
43
44#include "nvmap.h"
45#include "nvmap_ioctl.h"
46#include "nvmap_mru.h"
47#include "nvmap_common.h"
48
49#define NVMAP_NUM_PTES 64
50#define NVMAP_CARVEOUT_KILLER_RETRY_TIME 100 /* msecs */
51
52#ifdef CONFIG_NVMAP_CARVEOUT_KILLER
53static bool carveout_killer = true;
54#else
55static bool carveout_killer;
56#endif
57module_param(carveout_killer, bool, 0640);
58
59struct nvmap_carveout_node {
60 unsigned int heap_bit;
61 struct nvmap_heap *carveout;
62 int index;
63 struct list_head clients;
64 spinlock_t clients_lock;
65};
66
67struct nvmap_device {
68 struct vm_struct *vm_rgn;
69 pte_t *ptes[NVMAP_NUM_PTES];
70 unsigned long ptebits[NVMAP_NUM_PTES / BITS_PER_LONG];
71 unsigned int lastpte;
72 spinlock_t ptelock;
73
74 struct rb_root handles;
75 spinlock_t handle_lock;
76 wait_queue_head_t pte_wait;
77 struct miscdevice dev_super;
78 struct miscdevice dev_user;
79 struct nvmap_carveout_node *heaps;
80 int nr_carveouts;
81 struct nvmap_share iovmm_master;
82 struct list_head clients;
83 spinlock_t clients_lock;
84};
85
86struct nvmap_device *nvmap_dev;
87
88static struct backing_dev_info nvmap_bdi = {
89 .ra_pages = 0,
90 .capabilities = (BDI_CAP_NO_ACCT_AND_WRITEBACK |
91 BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP),
92};
93
94static int nvmap_open(struct inode *inode, struct file *filp);
95static int nvmap_release(struct inode *inode, struct file *filp);
96static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
97static int nvmap_map(struct file *filp, struct vm_area_struct *vma);
98static void nvmap_vma_open(struct vm_area_struct *vma);
99static void nvmap_vma_close(struct vm_area_struct *vma);
100static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
101
102static const struct file_operations nvmap_user_fops = {
103 .owner = THIS_MODULE,
104 .open = nvmap_open,
105 .release = nvmap_release,
106 .unlocked_ioctl = nvmap_ioctl,
107 .mmap = nvmap_map,
108};
109
110static const struct file_operations nvmap_super_fops = {
111 .owner = THIS_MODULE,
112 .open = nvmap_open,
113 .release = nvmap_release,
114 .unlocked_ioctl = nvmap_ioctl,
115 .mmap = nvmap_map,
116};
117
118static struct vm_operations_struct nvmap_vma_ops = {
119 .open = nvmap_vma_open,
120 .close = nvmap_vma_close,
121 .fault = nvmap_vma_fault,
122};
123
124int is_nvmap_vma(struct vm_area_struct *vma)
125{
126 return vma->vm_ops == &nvmap_vma_ops;
127}
128
129struct device *nvmap_client_to_device(struct nvmap_client *client)
130{
131 if (client->super)
132 return client->dev->dev_super.this_device;
133 else
134 return client->dev->dev_user.this_device;
135}
136
137struct nvmap_share *nvmap_get_share_from_dev(struct nvmap_device *dev)
138{
139 return &dev->iovmm_master;
140}
141
142/* allocates a PTE for the caller's use; returns the PTE pointer or
143 * a negative errno. may be called from IRQs */
144pte_t **nvmap_alloc_pte_irq(struct nvmap_device *dev, void **vaddr)
145{
146 unsigned long flags;
147 unsigned long bit;
148
149 spin_lock_irqsave(&dev->ptelock, flags);
150 bit = find_next_zero_bit(dev->ptebits, NVMAP_NUM_PTES, dev->lastpte);
151 if (bit == NVMAP_NUM_PTES) {
152 bit = find_first_zero_bit(dev->ptebits, dev->lastpte);
153 if (bit == dev->lastpte)
154 bit = NVMAP_NUM_PTES;
155 }
156
157 if (bit == NVMAP_NUM_PTES) {
158 spin_unlock_irqrestore(&dev->ptelock, flags);
159 return ERR_PTR(-ENOMEM);
160 }
161
162 dev->lastpte = bit;
163 set_bit(bit, dev->ptebits);
164 spin_unlock_irqrestore(&dev->ptelock, flags);
165
166 *vaddr = dev->vm_rgn->addr + bit * PAGE_SIZE;
167 return &(dev->ptes[bit]);
168}
169
170/* allocates a PTE for the caller's use; returns the PTE pointer or
171 * a negative errno. must be called from sleepable contexts */
172pte_t **nvmap_alloc_pte(struct nvmap_device *dev, void **vaddr)
173{
174 int ret;
175 pte_t **pte;
176 ret = wait_event_interruptible(dev->pte_wait,
177 !IS_ERR(pte = nvmap_alloc_pte_irq(dev, vaddr)));
178
179 if (ret == -ERESTARTSYS)
180 return ERR_PTR(-EINTR);
181
182 return pte;
183}
184
185/* frees a PTE */
186void nvmap_free_pte(struct nvmap_device *dev, pte_t **pte)
187{
188 unsigned long addr;
189 unsigned int bit = pte - dev->ptes;
190 unsigned long flags;
191
192 if (WARN_ON(bit >= NVMAP_NUM_PTES))
193 return;
194
195 addr = (unsigned long)dev->vm_rgn->addr + bit * PAGE_SIZE;
196 set_pte_at(&init_mm, addr, *pte, 0);
197
198 spin_lock_irqsave(&dev->ptelock, flags);
199 clear_bit(bit, dev->ptebits);
200 spin_unlock_irqrestore(&dev->ptelock, flags);
201 wake_up(&dev->pte_wait);
202}
203
204/* verifies that the handle ref value "ref" is a valid handle ref for the
205 * file. caller must hold the file's ref_lock prior to calling this function */
206struct nvmap_handle_ref *_nvmap_validate_id_locked(struct nvmap_client *c,
207 unsigned long id)
208{
209 struct rb_node *n = c->handle_refs.rb_node;
210
211 while (n) {
212 struct nvmap_handle_ref *ref;
213 ref = rb_entry(n, struct nvmap_handle_ref, node);
214 if ((unsigned long)ref->handle == id)
215 return ref;
216 else if (id > (unsigned long)ref->handle)
217 n = n->rb_right;
218 else
219 n = n->rb_left;
220 }
221
222 return NULL;
223}
224
225struct nvmap_handle *nvmap_get_handle_id(struct nvmap_client *client,
226 unsigned long id)
227{
228 struct nvmap_handle_ref *ref;
229 struct nvmap_handle *h = NULL;
230
231 nvmap_ref_lock(client);
232 ref = _nvmap_validate_id_locked(client, id);
233 if (ref)
234 h = ref->handle;
235 if (h)
236 h = nvmap_handle_get(h);
237 nvmap_ref_unlock(client);
238 return h;
239}
240
241unsigned long nvmap_carveout_usage(struct nvmap_client *c,
242 struct nvmap_heap_block *b)
243{
244 struct nvmap_heap *h = nvmap_block_to_heap(b);
245 struct nvmap_carveout_node *n;
246 int i;
247
248 for (i = 0; i < c->dev->nr_carveouts; i++) {
249 n = &c->dev->heaps[i];
250 if (n->carveout == h)
251 return n->heap_bit;
252 }
253 return 0;
254}
255
256/*
257 * This routine is used to flush the carveout memory from cache.
258 * Why cache flush is needed for carveout? Consider the case, where a piece of
259 * carveout is allocated as cached and released. After this, if the same memory is
260 * allocated for uncached request and the memory is not flushed out from cache.
261 * In this case, the client might pass this to H/W engine and it could start modify
262 * the memory. As this was cached earlier, it might have some portion of it in cache.
263 * During cpu request to read/write other memory, the cached portion of this memory
264 * might get flushed back to main memory and would cause corruptions, if it happens
265 * after H/W writes data to memory.
266 *
267 * But flushing out the memory blindly on each carveout allocation is redundant.
268 *
269 * In order to optimize the carveout buffer cache flushes, the following
270 * strategy is used.
271 *
272 * The whole Carveout is flushed out from cache during its initialization.
273 * During allocation, carveout buffers are not flused from cache.
274 * During deallocation, carveout buffers are flushed, if they were allocated as cached.
275 * if they were allocated as uncached/writecombined, no cache flush is needed.
276 * Just draining store buffers is enough.
277 */
278int nvmap_flush_heap_block(struct nvmap_client *client,
279 struct nvmap_heap_block *block, size_t len, unsigned int prot)
280{
281 pte_t **pte;
282 void *addr;
283 phys_addr_t kaddr;
284 phys_addr_t phys = block->base;
285 phys_addr_t end = block->base + len;
286
287 if (prot == NVMAP_HANDLE_UNCACHEABLE || prot == NVMAP_HANDLE_WRITE_COMBINE)
288 goto out;
289
290 if (len >= FLUSH_CLEAN_BY_SET_WAY_THRESHOLD) {
291 inner_flush_cache_all();
292 if (prot != NVMAP_HANDLE_INNER_CACHEABLE)
293 outer_flush_range(block->base, block->base + len);
294 goto out;
295 }
296
297 pte = nvmap_alloc_pte((client ? client->dev : nvmap_dev), &addr);
298 if (IS_ERR(pte))
299 return PTR_ERR(pte);
300
301 kaddr = (phys_addr_t)addr;
302
303 while (phys < end) {
304 phys_addr_t next = (phys + PAGE_SIZE) & PAGE_MASK;
305 unsigned long pfn = __phys_to_pfn(phys);
306 void *base = (void *)kaddr + (phys & ~PAGE_MASK);
307
308 next = min(next, end);
309 set_pte_at(&init_mm, kaddr, *pte, pfn_pte(pfn, pgprot_kernel));
310 flush_tlb_kernel_page(kaddr);
311 __cpuc_flush_dcache_area(base, next - phys);
312 phys = next;
313 }
314
315 if (prot != NVMAP_HANDLE_INNER_CACHEABLE)
316 outer_flush_range(block->base, block->base + len);
317
318 nvmap_free_pte((client ? client->dev : nvmap_dev), pte);
319out:
320 wmb();
321 return 0;
322}
323
324void nvmap_carveout_commit_add(struct nvmap_client *client,
325 struct nvmap_carveout_node *node,
326 size_t len)
327{
328 unsigned long flags;
329
330 nvmap_ref_lock(client);
331 spin_lock_irqsave(&node->clients_lock, flags);
332 BUG_ON(list_empty(&client->carveout_commit[node->index].list) &&
333 client->carveout_commit[node->index].commit != 0);
334
335 client->carveout_commit[node->index].commit += len;
336 /* if this client isn't already on the list of nodes for this heap,
337 add it */
338 if (list_empty(&client->carveout_commit[node->index].list)) {
339 list_add(&client->carveout_commit[node->index].list,
340 &node->clients);
341 }
342 spin_unlock_irqrestore(&node->clients_lock, flags);
343 nvmap_ref_unlock(client);
344}
345
346void nvmap_carveout_commit_subtract(struct nvmap_client *client,
347 struct nvmap_carveout_node *node,
348 size_t len)
349{
350 unsigned long flags;
351
352 if (!client)
353 return;
354
355 spin_lock_irqsave(&node->clients_lock, flags);
356 BUG_ON(client->carveout_commit[node->index].commit < len);
357 client->carveout_commit[node->index].commit -= len;
358 /* if no more allocation in this carveout for this node, delete it */
359 if (!client->carveout_commit[node->index].commit)
360 list_del_init(&client->carveout_commit[node->index].list);
361 spin_unlock_irqrestore(&node->clients_lock, flags);
362}
363
364static struct nvmap_client *get_client_from_carveout_commit(
365 struct nvmap_carveout_node *node, struct nvmap_carveout_commit *commit)
366{
367 struct nvmap_carveout_commit *first_commit = commit - node->index;
368 return (void *)first_commit - offsetof(struct nvmap_client,
369 carveout_commit);
370}
371
372static DECLARE_WAIT_QUEUE_HEAD(wait_reclaim);
373static int wait_count;
374bool nvmap_shrink_carveout(struct nvmap_carveout_node *node)
375{
376 struct nvmap_carveout_commit *commit;
377 size_t selected_size = 0;
378 int selected_oom_adj = OOM_ADJUST_MIN;
379 struct task_struct *selected_task = NULL;
380 unsigned long flags;
381 bool wait = false;
382 int current_oom_adj = OOM_ADJUST_MIN;
383
384 task_lock(current);
385 if (current->signal)
386 current_oom_adj = current->signal->oom_adj;
387 task_unlock(current);
388
389 spin_lock_irqsave(&node->clients_lock, flags);
390 /* find the task with the smallest oom_adj (lowest priority)
391 * and largest carveout allocation -- ignore kernel allocations,
392 * there's no way to handle them */
393 list_for_each_entry(commit, &node->clients, list) {
394 struct nvmap_client *client =
395 get_client_from_carveout_commit(node, commit);
396 size_t size = commit->commit;
397 struct task_struct *task = client->task;
398 struct signal_struct *sig;
399
400 if (!task)
401 continue;
402
403 task_lock(task);
404 sig = task->signal;
405 if (!task->mm || !sig)
406 goto end;
407 /* don't try to kill current */
408 if (task == current->group_leader)
409 goto end;
410 /* don't try to kill higher priority tasks */
411 if (sig->oom_adj < current_oom_adj)
412 goto end;
413 if (sig->oom_adj < selected_oom_adj)
414 goto end;
415 if (sig->oom_adj == selected_oom_adj &&
416 size <= selected_size)
417 goto end;
418 selected_oom_adj = sig->oom_adj;
419 selected_size = size;
420 selected_task = task;
421end:
422 task_unlock(task);
423 }
424 if (selected_task) {
425 wait = true;
426 if (fatal_signal_pending(selected_task)) {
427 pr_warning("carveout_killer: process %d dying "
428 "slowly\n", selected_task->pid);
429 goto out;
430 }
431 pr_info("carveout_killer: killing process %d with oom_adj %d "
432 "to reclaim %d (for process with oom_adj %d)\n",
433 selected_task->pid, selected_oom_adj,
434 selected_size, current_oom_adj);
435 force_sig(SIGKILL, selected_task);
436 }
437out:
438 spin_unlock_irqrestore(&node->clients_lock, flags);
439 return wait;
440}
441
442static
443struct nvmap_heap_block *do_nvmap_carveout_alloc(struct nvmap_client *client,
444 struct nvmap_handle *handle,
445 unsigned long type)
446{
447 struct nvmap_carveout_node *co_heap;
448 struct nvmap_device *dev = client->dev;
449 int i;
450
451 for (i = 0; i < dev->nr_carveouts; i++) {
452 struct nvmap_heap_block *block;
453 co_heap = &dev->heaps[i];
454
455 if (!(co_heap->heap_bit & type))
456 continue;
457
458 block = nvmap_heap_alloc(co_heap->carveout, handle);
459 if (block)
460 return block;
461 }
462 return NULL;
463}
464
465static bool nvmap_carveout_freed(int count)
466{
467 smp_rmb();
468 return count != wait_count;
469}
470
471struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *client,
472 struct nvmap_handle *handle,
473 unsigned long type)
474{
475 struct nvmap_heap_block *block;
476 struct nvmap_carveout_node *co_heap;
477 struct nvmap_device *dev = client->dev;
478 int i;
479 unsigned long end = jiffies +
480 msecs_to_jiffies(NVMAP_CARVEOUT_KILLER_RETRY_TIME);
481 int count = 0;
482
483 do {
484 block = do_nvmap_carveout_alloc(client, handle, type);
485 if (!carveout_killer)
486 return block;
487
488 if (block)
489 return block;
490
491 if (!count++) {
492 char task_comm[TASK_COMM_LEN];
493 if (client->task)
494 get_task_comm(task_comm, client->task);
495 else
496 task_comm[0] = 0;
497 pr_info("%s: failed to allocate %u bytes for "
498 "process %s, firing carveout "
499 "killer!\n", __func__, handle->size, task_comm);
500
501 } else {
502 pr_info("%s: still can't allocate %u bytes, "
503 "attempt %d!\n", __func__, handle->size, count);
504 }
505
506 /* shrink carveouts that matter and try again */
507 for (i = 0; i < dev->nr_carveouts; i++) {
508 int count;
509 co_heap = &dev->heaps[i];
510
511 if (!(co_heap->heap_bit & type))
512 continue;
513
514 count = wait_count;
515 /* indicates we didn't find anything to kill,
516 might as well stop trying */
517 if (!nvmap_shrink_carveout(co_heap))
518 return NULL;
519
520 if (time_is_after_jiffies(end))
521 wait_event_interruptible_timeout(wait_reclaim,
522 nvmap_carveout_freed(count),
523 end - jiffies);
524 }
525 } while (time_is_after_jiffies(end));
526
527 if (time_is_before_jiffies(end))
528 pr_info("carveout_killer: timeout expired without "
529 "allocation succeeding.\n");
530
531 return NULL;
532}
533
534/* remove a handle from the device's tree of all handles; called
535 * when freeing handles. */
536int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h)
537{
538 spin_lock(&dev->handle_lock);
539
540 /* re-test inside the spinlock if the handle really has no clients;
541 * only remove the handle if it is unreferenced */
542 if (atomic_add_return(0, &h->ref) > 0) {
543 spin_unlock(&dev->handle_lock);
544 return -EBUSY;
545 }
546 smp_rmb();
547 BUG_ON(atomic_read(&h->ref) < 0);
548 BUG_ON(atomic_read(&h->pin) != 0);
549
550 rb_erase(&h->node, &dev->handles);
551
552 spin_unlock(&dev->handle_lock);
553 return 0;
554}
555
556/* adds a newly-created handle to the device master tree */
557void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h)
558{
559 struct rb_node **p;
560 struct rb_node *parent = NULL;
561
562 spin_lock(&dev->handle_lock);
563 p = &dev->handles.rb_node;
564 while (*p) {
565 struct nvmap_handle *b;
566
567 parent = *p;
568 b = rb_entry(parent, struct nvmap_handle, node);
569 if (h > b)
570 p = &parent->rb_right;
571 else
572 p = &parent->rb_left;
573 }
574 rb_link_node(&h->node, parent, p);
575 rb_insert_color(&h->node, &dev->handles);
576 spin_unlock(&dev->handle_lock);
577}
578
579/* validates that a handle is in the device master tree, and that the
580 * client has permission to access it */
581struct nvmap_handle *nvmap_validate_get(struct nvmap_client *client,
582 unsigned long id)
583{
584 struct nvmap_handle *h = NULL;
585 struct rb_node *n;
586
587 spin_lock(&client->dev->handle_lock);
588
589 n = client->dev->handles.rb_node;
590
591 while (n) {
592 h = rb_entry(n, struct nvmap_handle, node);
593 if ((unsigned long)h == id) {
594 if (client->super || h->global || (h->owner == client))
595 h = nvmap_handle_get(h);
596 else
597 h = NULL;
598 spin_unlock(&client->dev->handle_lock);
599 return h;
600 }
601 if (id > (unsigned long)h)
602 n = n->rb_right;
603 else
604 n = n->rb_left;
605 }
606 spin_unlock(&client->dev->handle_lock);
607 return NULL;
608}
609
610struct nvmap_client *nvmap_create_client(struct nvmap_device *dev,
611 const char *name)
612{
613 struct nvmap_client *client;
614 struct task_struct *task;
615 int i;
616
617 if (WARN_ON(!dev))
618 return NULL;
619
620 client = kzalloc(sizeof(*client) + (sizeof(struct nvmap_carveout_commit)
621 * dev->nr_carveouts), GFP_KERNEL);
622 if (!client)
623 return NULL;
624
625 client->name = name;
626 client->super = true;
627 client->dev = dev;
628 /* TODO: allocate unique IOVMM client for each nvmap client */
629 client->share = &dev->iovmm_master;
630 client->handle_refs = RB_ROOT;
631
632 atomic_set(&client->iovm_commit, 0);
633
634 client->iovm_limit = nvmap_mru_vm_size(client->share->iovmm);
635
636 for (i = 0; i < dev->nr_carveouts; i++) {
637 INIT_LIST_HEAD(&client->carveout_commit[i].list);
638 client->carveout_commit[i].commit = 0;
639 }
640
641 get_task_struct(current->group_leader);
642 task_lock(current->group_leader);
643 /* don't bother to store task struct for kernel threads,
644 they can't be killed anyway */
645 if (current->flags & PF_KTHREAD) {
646 put_task_struct(current->group_leader);
647 task = NULL;
648 } else {
649 task = current->group_leader;
650 }
651 task_unlock(current->group_leader);
652 client->task = task;
653
654 mutex_init(&client->ref_lock);
655 atomic_set(&client->count, 1);
656
657 spin_lock(&dev->clients_lock);
658 list_add(&client->list, &dev->clients);
659 spin_unlock(&dev->clients_lock);
660 return client;
661}
662
663static void destroy_client(struct nvmap_client *client)
664{
665 struct rb_node *n;
666 int i;
667
668 if (!client)
669 return;
670
671
672 while ((n = rb_first(&client->handle_refs))) {
673 struct nvmap_handle_ref *ref;
674 int pins, dupes;
675
676 ref = rb_entry(n, struct nvmap_handle_ref, node);
677 rb_erase(&ref->node, &client->handle_refs);
678
679 smp_rmb();
680 pins = atomic_read(&ref->pin);
681
682 if (ref->handle->owner == client)
683 ref->handle->owner = NULL;
684
685 while (pins--)
686 nvmap_unpin_handles(client, &ref->handle, 1);
687
688 dupes = atomic_read(&ref->dupes);
689 while (dupes--)
690 nvmap_handle_put(ref->handle);
691
692 kfree(ref);
693 }
694
695 if (carveout_killer) {
696 wait_count++;
697 smp_wmb();
698 wake_up_all(&wait_reclaim);
699 }
700
701 for (i = 0; i < client->dev->nr_carveouts; i++)
702 list_del(&client->carveout_commit[i].list);
703
704 if (client->task)
705 put_task_struct(client->task);
706
707 spin_lock(&client->dev->clients_lock);
708 list_del(&client->list);
709 spin_unlock(&client->dev->clients_lock);
710 kfree(client);
711}
712
713struct nvmap_client *nvmap_client_get(struct nvmap_client *client)
714{
715 if (WARN_ON(!client))
716 return NULL;
717
718 if (WARN_ON(!atomic_add_unless(&client->count, 1, 0)))
719 return NULL;
720
721 return client;
722}
723
724struct nvmap_client *nvmap_client_get_file(int fd)
725{
726 struct nvmap_client *client = ERR_PTR(-EFAULT);
727 struct file *f = fget(fd);
728 if (!f)
729 return ERR_PTR(-EINVAL);
730
731 if ((f->f_op == &nvmap_user_fops) || (f->f_op == &nvmap_super_fops)) {
732 client = f->private_data;
733 atomic_inc(&client->count);
734 }
735
736 fput(f);
737 return client;
738}
739
740void nvmap_client_put(struct nvmap_client *client)
741{
742 if (!client)
743 return;
744
745 if (!atomic_dec_return(&client->count))
746 destroy_client(client);
747}
748
749static int nvmap_open(struct inode *inode, struct file *filp)
750{
751 struct miscdevice *miscdev = filp->private_data;
752 struct nvmap_device *dev = dev_get_drvdata(miscdev->parent);
753 struct nvmap_client *priv;
754 int ret;
755
756 ret = nonseekable_open(inode, filp);
757 if (unlikely(ret))
758 return ret;
759
760 BUG_ON(dev != nvmap_dev);
761 priv = nvmap_create_client(dev, "user");
762 if (!priv)
763 return -ENOMEM;
764
765 priv->super = (filp->f_op == &nvmap_super_fops);
766
767 filp->f_mapping->backing_dev_info = &nvmap_bdi;
768
769 filp->private_data = priv;
770 return 0;
771}
772
773static int nvmap_release(struct inode *inode, struct file *filp)
774{
775 nvmap_client_put(filp->private_data);
776 return 0;
777}
778
779static int nvmap_map(struct file *filp, struct vm_area_struct *vma)
780{
781 struct nvmap_vma_priv *priv;
782
783 /* after NVMAP_IOC_MMAP, the handle that is mapped by this VMA
784 * will be stored in vm_private_data and faulted in. until the
785 * ioctl is made, the VMA is mapped no-access */
786 vma->vm_private_data = NULL;
787
788 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
789 if (!priv)
790 return -ENOMEM;
791
792 priv->offs = 0;
793 priv->handle = NULL;
794 atomic_set(&priv->count, 1);
795
796 vma->vm_flags |= VM_SHARED;
797 vma->vm_flags |= (VM_IO | VM_DONTEXPAND | VM_MIXEDMAP | VM_RESERVED);
798 vma->vm_ops = &nvmap_vma_ops;
799 vma->vm_private_data = priv;
800
801 return 0;
802}
803
804static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
805{
806 int err = 0;
807 void __user *uarg = (void __user *)arg;
808
809 if (_IOC_TYPE(cmd) != NVMAP_IOC_MAGIC)
810 return -ENOTTY;
811
812 if (_IOC_NR(cmd) > NVMAP_IOC_MAXNR)
813 return -ENOTTY;
814
815 if (_IOC_DIR(cmd) & _IOC_READ)
816 err = !access_ok(VERIFY_WRITE, uarg, _IOC_SIZE(cmd));
817 if (_IOC_DIR(cmd) & _IOC_WRITE)
818 err = !access_ok(VERIFY_READ, uarg, _IOC_SIZE(cmd));
819
820 if (err)
821 return -EFAULT;
822
823 switch (cmd) {
824 case NVMAP_IOC_CLAIM:
825 nvmap_warn(filp->private_data, "preserved handles not"
826 "supported\n");
827 err = -ENODEV;
828 break;
829 case NVMAP_IOC_CREATE:
830 case NVMAP_IOC_FROM_ID:
831 err = nvmap_ioctl_create(filp, cmd, uarg);
832 break;
833
834 case NVMAP_IOC_GET_ID:
835 err = nvmap_ioctl_getid(filp, uarg);
836 break;
837
838 case NVMAP_IOC_PARAM:
839 err = nvmap_ioctl_get_param(filp, uarg);
840 break;
841
842 case NVMAP_IOC_UNPIN_MULT:
843 case NVMAP_IOC_PIN_MULT:
844 err = nvmap_ioctl_pinop(filp, cmd == NVMAP_IOC_PIN_MULT, uarg);
845 break;
846
847 case NVMAP_IOC_ALLOC:
848 err = nvmap_ioctl_alloc(filp, uarg);
849 break;
850
851 case NVMAP_IOC_FREE:
852 err = nvmap_ioctl_free(filp, arg);
853 break;
854
855 case NVMAP_IOC_MMAP:
856 err = nvmap_map_into_caller_ptr(filp, uarg);
857 break;
858
859 case NVMAP_IOC_WRITE:
860 case NVMAP_IOC_READ:
861 err = nvmap_ioctl_rw_handle(filp, cmd == NVMAP_IOC_READ, uarg);
862 break;
863
864 case NVMAP_IOC_CACHE:
865 err = nvmap_ioctl_cache_maint(filp, uarg);
866 break;
867
868 default:
869 return -ENOTTY;
870 }
871 return err;
872}
873
874/* to ensure that the backing store for the VMA isn't freed while a fork'd
875 * reference still exists, nvmap_vma_open increments the reference count on
876 * the handle, and nvmap_vma_close decrements it. alternatively, we could
877 * disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
878*/
879static void nvmap_vma_open(struct vm_area_struct *vma)
880{
881 struct nvmap_vma_priv *priv;
882
883 priv = vma->vm_private_data;
884
885 BUG_ON(!priv);
886
887 atomic_inc(&priv->count);
888}
889
890static void nvmap_vma_close(struct vm_area_struct *vma)
891{
892 struct nvmap_vma_priv *priv = vma->vm_private_data;
893
894 if (priv) {
895 if (priv->handle) {
896 nvmap_usecount_dec(priv->handle);
897 BUG_ON(priv->handle->usecount < 0);
898 }
899 if (!atomic_dec_return(&priv->count)) {
900 if (priv->handle)
901 nvmap_handle_put(priv->handle);
902 kfree(priv);
903 }
904 }
905 vma->vm_private_data = NULL;
906}
907
908static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
909{
910 struct nvmap_vma_priv *priv;
911 unsigned long offs;
912
913 offs = (unsigned long)(vmf->virtual_address - vma->vm_start);
914 priv = vma->vm_private_data;
915 if (!priv || !priv->handle || !priv->handle->alloc)
916 return VM_FAULT_SIGBUS;
917
918 offs += priv->offs;
919 /* if the VMA was split for some reason, vm_pgoff will be the VMA's
920 * offset from the original VMA */
921 offs += (vma->vm_pgoff << PAGE_SHIFT);
922
923 if (offs >= priv->handle->size)
924 return VM_FAULT_SIGBUS;
925
926 if (!priv->handle->heap_pgalloc) {
927 unsigned long pfn;
928 BUG_ON(priv->handle->carveout->base & ~PAGE_MASK);
929 pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT);
930 vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
931 return VM_FAULT_NOPAGE;
932 } else {
933 struct page *page;
934 offs >>= PAGE_SHIFT;
935 page = priv->handle->pgalloc.pages[offs];
936 if (page)
937 get_page(page);
938 vmf->page = page;
939 return (page) ? 0 : VM_FAULT_SIGBUS;
940 }
941}
942
943static ssize_t attr_show_usage(struct device *dev,
944 struct device_attribute *attr, char *buf)
945{
946 struct nvmap_carveout_node *node = nvmap_heap_device_to_arg(dev);
947
948 return sprintf(buf, "%08x\n", node->heap_bit);
949}
950
951static struct device_attribute heap_attr_show_usage =
952 __ATTR(usage, S_IRUGO, attr_show_usage, NULL);
953
954static struct attribute *heap_extra_attrs[] = {
955 &heap_attr_show_usage.attr,
956 NULL,
957};
958
959static struct attribute_group heap_extra_attr_group = {
960 .attrs = heap_extra_attrs,
961};
962
963static void client_stringify(struct nvmap_client *client, struct seq_file *s)
964{
965 char task_comm[TASK_COMM_LEN];
966 if (!client->task) {
967 seq_printf(s, "%-18s %18s %8u", client->name, "kernel", 0);
968 return;
969 }
970 get_task_comm(task_comm, client->task);
971 seq_printf(s, "%-18s %18s %8u", client->name, task_comm,
972 client->task->pid);
973}
974
975static void allocations_stringify(struct nvmap_client *client,
976 struct seq_file *s)
977{
978 unsigned long base = 0;
979 struct rb_node *n = rb_first(&client->handle_refs);
980
981 for (; n != NULL; n = rb_next(n)) {
982 struct nvmap_handle_ref *ref =
983 rb_entry(n, struct nvmap_handle_ref, node);
984 struct nvmap_handle *handle = ref->handle;
985 if (handle->alloc && !handle->heap_pgalloc) {
986 seq_printf(s, "%-18s %-18s %8lx %10u %8x\n", "", "",
987 (unsigned long)(handle->carveout->base),
988 handle->size, handle->userflags);
989 } else if (handle->alloc && handle->heap_pgalloc) {
990 seq_printf(s, "%-18s %-18s %8lx %10u %8x\n", "", "",
991 base, handle->size, handle->userflags);
992 }
993 }
994}
995
996static int nvmap_debug_allocations_show(struct seq_file *s, void *unused)
997{
998 struct nvmap_carveout_node *node = s->private;
999 struct nvmap_carveout_commit *commit;
1000 unsigned long flags;
1001 unsigned int total = 0;
1002
1003 spin_lock_irqsave(&node->clients_lock, flags);
1004 seq_printf(s, "%-18s %18s %8s %10s %8s\n", "CLIENT", "PROCESS", "PID",
1005 "SIZE", "FLAGS");
1006 seq_printf(s, "%-18s %18s %8s %10s\n", "", "",
1007 "BASE", "SIZE");
1008 list_for_each_entry(commit, &node->clients, list) {
1009 struct nvmap_client *client =
1010 get_client_from_carveout_commit(node, commit);
1011 client_stringify(client, s);
1012 seq_printf(s, " %10u\n", commit->commit);
1013 allocations_stringify(client, s);
1014 seq_printf(s, "\n");
1015 total += commit->commit;
1016 }
1017 seq_printf(s, "%-18s %-18s %8u %10u\n", "total", "", 0, total);
1018 spin_unlock_irqrestore(&node->clients_lock, flags);
1019
1020 return 0;
1021}
1022
1023static int nvmap_debug_allocations_open(struct inode *inode, struct file *file)
1024{
1025 return single_open(file, nvmap_debug_allocations_show,
1026 inode->i_private);
1027}
1028
1029static const struct file_operations debug_allocations_fops = {
1030 .open = nvmap_debug_allocations_open,
1031 .read = seq_read,
1032 .llseek = seq_lseek,
1033 .release = single_release,
1034};
1035
1036static int nvmap_debug_clients_show(struct seq_file *s, void *unused)
1037{
1038 struct nvmap_carveout_node *node = s->private;
1039 struct nvmap_carveout_commit *commit;
1040 unsigned long flags;
1041 unsigned int total = 0;
1042
1043 spin_lock_irqsave(&node->clients_lock, flags);
1044 seq_printf(s, "%-18s %18s %8s %10s\n", "CLIENT", "PROCESS", "PID",
1045 "SIZE");
1046 list_for_each_entry(commit, &node->clients, list) {
1047 struct nvmap_client *client =
1048 get_client_from_carveout_commit(node, commit);
1049 client_stringify(client, s);
1050 seq_printf(s, " %10u\n", commit->commit);
1051 total += commit->commit;
1052 }
1053 seq_printf(s, "%-18s %18s %8u %10u\n", "total", "", 0, total);
1054 spin_unlock_irqrestore(&node->clients_lock, flags);
1055
1056 return 0;
1057}
1058
1059static int nvmap_debug_clients_open(struct inode *inode, struct file *file)
1060{
1061 return single_open(file, nvmap_debug_clients_show, inode->i_private);
1062}
1063
1064static const struct file_operations debug_clients_fops = {
1065 .open = nvmap_debug_clients_open,
1066 .read = seq_read,
1067 .llseek = seq_lseek,
1068 .release = single_release,
1069};
1070
1071static int nvmap_debug_iovmm_clients_show(struct seq_file *s, void *unused)
1072{
1073 unsigned long flags;
1074 unsigned int total = 0;
1075 struct nvmap_client *client;
1076 struct nvmap_device *dev = s->private;
1077
1078 spin_lock_irqsave(&dev->clients_lock, flags);
1079 seq_printf(s, "%-18s %18s %8s %10s\n", "CLIENT", "PROCESS", "PID",
1080 "SIZE");
1081 list_for_each_entry(client, &dev->clients, list) {
1082 client_stringify(client, s);
1083 seq_printf(s, " %10u\n", atomic_read(&client->iovm_commit));
1084 total += atomic_read(&client->iovm_commit);
1085 }
1086 seq_printf(s, "%-18s %18s %8u %10u\n", "total", "", 0, total);
1087 spin_unlock_irqrestore(&dev->clients_lock, flags);
1088
1089 return 0;
1090}
1091
1092static int nvmap_debug_iovmm_clients_open(struct inode *inode,
1093 struct file *file)
1094{
1095 return single_open(file, nvmap_debug_iovmm_clients_show,
1096 inode->i_private);
1097}
1098
1099static const struct file_operations debug_iovmm_clients_fops = {
1100 .open = nvmap_debug_iovmm_clients_open,
1101 .read = seq_read,
1102 .llseek = seq_lseek,
1103 .release = single_release,
1104};
1105
1106static int nvmap_debug_iovmm_allocations_show(struct seq_file *s, void *unused)
1107{
1108 unsigned long flags;
1109 unsigned int total = 0;
1110 struct nvmap_client *client;
1111 struct nvmap_device *dev = s->private;
1112
1113 spin_lock_irqsave(&dev->clients_lock, flags);
1114 seq_printf(s, "%-18s %18s %8s %10s\n", "CLIENT", "PROCESS", "PID",
1115 "SIZE");
1116 seq_printf(s, "%-18s %18s %8s %10s\n", "", "",
1117 "BASE", "SIZE");
1118 list_for_each_entry(client, &dev->clients, list) {
1119 client_stringify(client, s);
1120 seq_printf(s, " %10u\n", atomic_read(&client->iovm_commit));
1121 allocations_stringify(client, s);
1122 seq_printf(s, "\n");
1123 total += atomic_read(&client->iovm_commit);
1124 }
1125 seq_printf(s, "%-18s %-18s %8u %10u\n", "total", "", 0, total);
1126 spin_unlock_irqrestore(&dev->clients_lock, flags);
1127
1128 return 0;
1129}
1130
1131static int nvmap_debug_iovmm_allocations_open(struct inode *inode,
1132 struct file *file)
1133{
1134 return single_open(file, nvmap_debug_iovmm_allocations_show,
1135 inode->i_private);
1136}
1137
1138static const struct file_operations debug_iovmm_allocations_fops = {
1139 .open = nvmap_debug_iovmm_allocations_open,
1140 .read = seq_read,
1141 .llseek = seq_lseek,
1142 .release = single_release,
1143};
1144
1145static int nvmap_probe(struct platform_device *pdev)
1146{
1147 struct nvmap_platform_data *plat = pdev->dev.platform_data;
1148 struct nvmap_device *dev;
1149 struct dentry *nvmap_debug_root;
1150 unsigned int i;
1151 int e;
1152
1153 if (!plat) {
1154 dev_err(&pdev->dev, "no platform data?\n");
1155 return -ENODEV;
1156 }
1157
1158 if (WARN_ON(nvmap_dev != NULL)) {
1159 dev_err(&pdev->dev, "only one nvmap device may be present\n");
1160 return -ENODEV;
1161 }
1162
1163 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1164 if (!dev) {
1165 dev_err(&pdev->dev, "out of memory for device\n");
1166 return -ENOMEM;
1167 }
1168
1169 dev->dev_user.minor = MISC_DYNAMIC_MINOR;
1170 dev->dev_user.name = "nvmap";
1171 dev->dev_user.fops = &nvmap_user_fops;
1172 dev->dev_user.parent = &pdev->dev;
1173
1174 dev->dev_super.minor = MISC_DYNAMIC_MINOR;
1175 dev->dev_super.name = "knvmap";
1176 dev->dev_super.fops = &nvmap_super_fops;
1177 dev->dev_super.parent = &pdev->dev;
1178
1179 dev->handles = RB_ROOT;
1180
1181 init_waitqueue_head(&dev->pte_wait);
1182
1183 init_waitqueue_head(&dev->iovmm_master.pin_wait);
1184 mutex_init(&dev->iovmm_master.pin_lock);
1185 for (i = 0; i < NVMAP_NUM_POOLS; i++)
1186 nvmap_page_pool_init(&dev->iovmm_master.pools[i], i);
1187
1188 dev->iovmm_master.iovmm =
1189 tegra_iovmm_alloc_client(dev_name(&pdev->dev), NULL,
1190 &(dev->dev_user));
1191#ifdef CONFIG_TEGRA_IOVMM
1192 if (!dev->iovmm_master.iovmm) {
1193 e = PTR_ERR(dev->iovmm_master.iovmm);
1194 dev_err(&pdev->dev, "couldn't create iovmm client\n");
1195 goto fail;
1196 }
1197#endif
1198 dev->vm_rgn = alloc_vm_area(NVMAP_NUM_PTES * PAGE_SIZE);
1199 if (!dev->vm_rgn) {
1200 e = -ENOMEM;
1201 dev_err(&pdev->dev, "couldn't allocate remapping region\n");
1202 goto fail;
1203 }
1204 e = nvmap_mru_init(&dev->iovmm_master);
1205 if (e) {
1206 dev_err(&pdev->dev, "couldn't initialize MRU lists\n");
1207 goto fail;
1208 }
1209
1210 spin_lock_init(&dev->ptelock);
1211 spin_lock_init(&dev->handle_lock);
1212 INIT_LIST_HEAD(&dev->clients);
1213 spin_lock_init(&dev->clients_lock);
1214
1215 for (i = 0; i < NVMAP_NUM_PTES; i++) {
1216 unsigned long addr;
1217 pgd_t *pgd;
1218 pud_t *pud;
1219 pmd_t *pmd;
1220
1221 addr = (unsigned long)dev->vm_rgn->addr + (i * PAGE_SIZE);
1222 pgd = pgd_offset_k(addr);
1223 pud = pud_alloc(&init_mm, pgd, addr);
1224 if (!pud) {
1225 e = -ENOMEM;
1226 dev_err(&pdev->dev, "couldn't allocate page tables\n");
1227 goto fail;
1228 }
1229 pmd = pmd_alloc(&init_mm, pud, addr);
1230 if (!pmd) {
1231 e = -ENOMEM;
1232 dev_err(&pdev->dev, "couldn't allocate page tables\n");
1233 goto fail;
1234 }
1235 dev->ptes[i] = pte_alloc_kernel(pmd, addr);
1236 if (!dev->ptes[i]) {
1237 e = -ENOMEM;
1238 dev_err(&pdev->dev, "couldn't allocate page tables\n");
1239 goto fail;
1240 }
1241 }
1242
1243 e = misc_register(&dev->dev_user);
1244 if (e) {
1245 dev_err(&pdev->dev, "unable to register miscdevice %s\n",
1246 dev->dev_user.name);
1247 goto fail;
1248 }
1249
1250 e = misc_register(&dev->dev_super);
1251 if (e) {
1252 dev_err(&pdev->dev, "unable to register miscdevice %s\n",
1253 dev->dev_super.name);
1254 goto fail;
1255 }
1256
1257 dev->nr_carveouts = 0;
1258 dev->heaps = kzalloc(sizeof(struct nvmap_carveout_node) *
1259 plat->nr_carveouts, GFP_KERNEL);
1260 if (!dev->heaps) {
1261 e = -ENOMEM;
1262 dev_err(&pdev->dev, "couldn't allocate carveout memory\n");
1263 goto fail;
1264 }
1265
1266 nvmap_debug_root = debugfs_create_dir("nvmap", NULL);
1267 if (IS_ERR_OR_NULL(nvmap_debug_root))
1268 dev_err(&pdev->dev, "couldn't create debug files\n");
1269
1270 for (i = 0; i < plat->nr_carveouts; i++) {
1271 struct nvmap_carveout_node *node = &dev->heaps[dev->nr_carveouts];
1272 const struct nvmap_platform_carveout *co = &plat->carveouts[i];
1273 if (!co->size)
1274 continue;
1275 node->carveout = nvmap_heap_create(dev->dev_user.this_device,
1276 co->name, co->base, co->size,
1277 co->buddy_size, node);
1278 if (!node->carveout) {
1279 e = -ENOMEM;
1280 dev_err(&pdev->dev, "couldn't create %s\n", co->name);
1281 goto fail_heaps;
1282 }
1283 node->index = dev->nr_carveouts;
1284 dev->nr_carveouts++;
1285 spin_lock_init(&node->clients_lock);
1286 INIT_LIST_HEAD(&node->clients);
1287 node->heap_bit = co->usage_mask;
1288 if (nvmap_heap_create_group(node->carveout,
1289 &heap_extra_attr_group))
1290 dev_warn(&pdev->dev, "couldn't add extra attributes\n");
1291
1292 dev_info(&pdev->dev, "created carveout %s (%uKiB)\n",
1293 co->name, co->size / 1024);
1294
1295 if (!IS_ERR_OR_NULL(nvmap_debug_root)) {
1296 struct dentry *heap_root =
1297 debugfs_create_dir(co->name, nvmap_debug_root);
1298 if (!IS_ERR_OR_NULL(heap_root)) {
1299 debugfs_create_file("clients", 0664, heap_root,
1300 node, &debug_clients_fops);
1301 debugfs_create_file("allocations", 0664,
1302 heap_root, node, &debug_allocations_fops);
1303 }
1304 }
1305 }
1306 if (!IS_ERR_OR_NULL(nvmap_debug_root)) {
1307 struct dentry *iovmm_root =
1308 debugfs_create_dir("iovmm", nvmap_debug_root);
1309 if (!IS_ERR_OR_NULL(iovmm_root)) {
1310 debugfs_create_file("clients", 0664, iovmm_root,
1311 dev, &debug_iovmm_clients_fops);
1312 debugfs_create_file("allocations", 0664, iovmm_root,
1313 dev, &debug_iovmm_allocations_fops);
1314 for (i = 0; i < NVMAP_NUM_POOLS; i++) {
1315 char name[40];
1316 char *memtype_string[] = {"uc", "wc",
1317 "iwb", "wb"};
1318 sprintf(name, "%s_page_pool_available_pages",
1319 memtype_string[i]);
1320 debugfs_create_u32(name, S_IRUGO|S_IWUSR,
1321 iovmm_root,
1322 &dev->iovmm_master.pools[i].npages);
1323 }
1324 }
1325 }
1326
1327 platform_set_drvdata(pdev, dev);
1328 nvmap_dev = dev;
1329
1330 return 0;
1331fail_heaps:
1332 for (i = 0; i < dev->nr_carveouts; i++) {
1333 struct nvmap_carveout_node *node = &dev->heaps[i];
1334 nvmap_heap_remove_group(node->carveout, &heap_extra_attr_group);
1335 nvmap_heap_destroy(node->carveout);
1336 }
1337fail:
1338 kfree(dev->heaps);
1339 nvmap_mru_destroy(&dev->iovmm_master);
1340 if (dev->dev_super.minor != MISC_DYNAMIC_MINOR)
1341 misc_deregister(&dev->dev_super);
1342 if (dev->dev_user.minor != MISC_DYNAMIC_MINOR)
1343 misc_deregister(&dev->dev_user);
1344 if (!IS_ERR_OR_NULL(dev->iovmm_master.iovmm))
1345 tegra_iovmm_free_client(dev->iovmm_master.iovmm);
1346 if (dev->vm_rgn)
1347 free_vm_area(dev->vm_rgn);
1348 kfree(dev);
1349 nvmap_dev = NULL;
1350 return e;
1351}
1352
1353static int nvmap_remove(struct platform_device *pdev)
1354{
1355 struct nvmap_device *dev = platform_get_drvdata(pdev);
1356 struct rb_node *n;
1357 struct nvmap_handle *h;
1358 int i;
1359
1360 misc_deregister(&dev->dev_super);
1361 misc_deregister(&dev->dev_user);
1362
1363 while ((n = rb_first(&dev->handles))) {
1364 h = rb_entry(n, struct nvmap_handle, node);
1365 rb_erase(&h->node, &dev->handles);
1366 kfree(h);
1367 }
1368
1369 if (!IS_ERR_OR_NULL(dev->iovmm_master.iovmm))
1370 tegra_iovmm_free_client(dev->iovmm_master.iovmm);
1371
1372 nvmap_mru_destroy(&dev->iovmm_master);
1373
1374 for (i = 0; i < dev->nr_carveouts; i++) {
1375 struct nvmap_carveout_node *node = &dev->heaps[i];
1376 nvmap_heap_remove_group(node->carveout, &heap_extra_attr_group);
1377 nvmap_heap_destroy(node->carveout);
1378 }
1379 kfree(dev->heaps);
1380
1381 free_vm_area(dev->vm_rgn);
1382 kfree(dev);
1383 nvmap_dev = NULL;
1384 return 0;
1385}
1386
1387static int nvmap_suspend(struct platform_device *pdev, pm_message_t state)
1388{
1389 return 0;
1390}
1391
1392static int nvmap_resume(struct platform_device *pdev)
1393{
1394 return 0;
1395}
1396
1397static struct platform_driver nvmap_driver = {
1398 .probe = nvmap_probe,
1399 .remove = nvmap_remove,
1400 .suspend = nvmap_suspend,
1401 .resume = nvmap_resume,
1402
1403 .driver = {
1404 .name = "tegra-nvmap",
1405 .owner = THIS_MODULE,
1406 },
1407};
1408
1409static int __init nvmap_init_driver(void)
1410{
1411 int e;
1412
1413 nvmap_dev = NULL;
1414
1415 e = nvmap_heap_init();
1416 if (e)
1417 goto fail;
1418
1419 e = platform_driver_register(&nvmap_driver);
1420 if (e) {
1421 nvmap_heap_deinit();
1422 goto fail;
1423 }
1424
1425fail:
1426 return e;
1427}
1428fs_initcall(nvmap_init_driver);
1429
1430static void __exit nvmap_exit_driver(void)
1431{
1432 platform_driver_unregister(&nvmap_driver);
1433 nvmap_heap_deinit();
1434 nvmap_dev = NULL;
1435}
1436module_exit(nvmap_exit_driver);
diff --git a/drivers/video/tegra/nvmap/nvmap_handle.c b/drivers/video/tegra/nvmap/nvmap_handle.c
new file mode 100644
index 00000000000..539b7ce9801
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_handle.c
@@ -0,0 +1,1020 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_handle.c
3 *
4 * Handle allocation and freeing routines for nvmap
5 *
6 * Copyright (c) 2009-2012, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#define pr_fmt(fmt) "%s: " fmt, __func__
24
25#include <linux/err.h>
26#include <linux/kernel.h>
27#include <linux/list.h>
28#include <linux/mm.h>
29#include <linux/rbtree.h>
30#include <linux/slab.h>
31#include <linux/vmalloc.h>
32#include <linux/fs.h>
33
34#include <asm/cacheflush.h>
35#include <asm/outercache.h>
36#include <asm/pgtable.h>
37
38#include <mach/iovmm.h>
39#include <mach/nvmap.h>
40
41#include <linux/vmstat.h>
42#include <linux/swap.h>
43#include <linux/shrinker.h>
44#include <linux/moduleparam.h>
45
46#include "nvmap.h"
47#include "nvmap_mru.h"
48#include "nvmap_common.h"
49
50#define PRINT_CARVEOUT_CONVERSION 0
51#if PRINT_CARVEOUT_CONVERSION
52#define PR_INFO pr_info
53#else
54#define PR_INFO(...)
55#endif
56
57#define NVMAP_SECURE_HEAPS (NVMAP_HEAP_CARVEOUT_IRAM | NVMAP_HEAP_IOVMM | \
58 NVMAP_HEAP_CARVEOUT_VPR)
59#ifdef CONFIG_NVMAP_HIGHMEM_ONLY
60#define GFP_NVMAP (__GFP_HIGHMEM | __GFP_NOWARN)
61#else
62#define GFP_NVMAP (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
63#endif
64/* handles may be arbitrarily large (16+MiB), and any handle allocated from
65 * the kernel (i.e., not a carveout handle) includes its array of pages. to
66 * preserve kmalloc space, if the array of pages exceeds PAGELIST_VMALLOC_MIN,
67 * the array is allocated using vmalloc. */
68#define PAGELIST_VMALLOC_MIN (PAGE_SIZE * 2)
69#define NVMAP_TEST_PAGE_POOL_SHRINKER 1
70static bool enable_pp = 1;
71static int pool_size[NVMAP_NUM_POOLS];
72
73static char *s_memtype_str[] = {
74 "uc",
75 "wc",
76 "iwb",
77 "wb",
78};
79
80static inline void nvmap_page_pool_lock(struct nvmap_page_pool *pool)
81{
82 mutex_lock(&pool->lock);
83}
84
85static inline void nvmap_page_pool_unlock(struct nvmap_page_pool *pool)
86{
87 mutex_unlock(&pool->lock);
88}
89
90static struct page *nvmap_page_pool_alloc_locked(struct nvmap_page_pool *pool)
91{
92 struct page *page = NULL;
93
94 if (pool->npages > 0)
95 page = pool->page_array[--pool->npages];
96 return page;
97}
98
99static struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool)
100{
101 struct page *page = NULL;
102
103 if (pool) {
104 nvmap_page_pool_lock(pool);
105 page = nvmap_page_pool_alloc_locked(pool);
106 nvmap_page_pool_unlock(pool);
107 }
108 return page;
109}
110
111static bool nvmap_page_pool_release_locked(struct nvmap_page_pool *pool,
112 struct page *page)
113{
114 int ret = false;
115
116 if (enable_pp && pool->npages < pool->max_pages) {
117 pool->page_array[pool->npages++] = page;
118 ret = true;
119 }
120 return ret;
121}
122
123static bool nvmap_page_pool_release(struct nvmap_page_pool *pool,
124 struct page *page)
125{
126 int ret = false;
127
128 if (pool) {
129 nvmap_page_pool_lock(pool);
130 ret = nvmap_page_pool_release_locked(pool, page);
131 nvmap_page_pool_unlock(pool);
132 }
133 return ret;
134}
135
136static int nvmap_page_pool_get_available_count(struct nvmap_page_pool *pool)
137{
138 return pool->npages;
139}
140
141static int nvmap_page_pool_free(struct nvmap_page_pool *pool, int nr_free)
142{
143 int i = nr_free;
144 int idx = 0;
145 struct page *page;
146
147 if (!nr_free)
148 return nr_free;
149 nvmap_page_pool_lock(pool);
150 while (i) {
151 page = nvmap_page_pool_alloc_locked(pool);
152 if (!page)
153 break;
154 pool->shrink_array[idx++] = page;
155 i--;
156 }
157
158 if (idx)
159 set_pages_array_wb(pool->shrink_array, idx);
160 while (idx--)
161 __free_page(pool->shrink_array[idx]);
162 nvmap_page_pool_unlock(pool);
163 return i;
164}
165
166static int nvmap_page_pool_get_unused_pages(void)
167{
168 unsigned int i;
169 int total = 0;
170 struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev);
171
172 for (i = 0; i < NVMAP_NUM_POOLS; i++)
173 total += nvmap_page_pool_get_available_count(&share->pools[i]);
174
175 return total;
176}
177
178static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, int size)
179{
180 int available_pages;
181 int pages_to_release = 0;
182 struct page **page_array = NULL;
183 struct page **shrink_array = NULL;
184
185 if (size == pool->max_pages)
186 return;
187repeat:
188 nvmap_page_pool_free(pool, pages_to_release);
189 nvmap_page_pool_lock(pool);
190 available_pages = nvmap_page_pool_get_available_count(pool);
191 if (available_pages > size) {
192 nvmap_page_pool_unlock(pool);
193 pages_to_release = available_pages - size;
194 goto repeat;
195 }
196
197 if (size == 0) {
198 vfree(pool->page_array);
199 vfree(pool->shrink_array);
200 pool->page_array = pool->shrink_array = NULL;
201 goto out;
202 }
203
204 page_array = vmalloc(sizeof(struct page *) * size);
205 shrink_array = vmalloc(sizeof(struct page *) * size);
206 if (!page_array || !shrink_array)
207 goto fail;
208
209 memcpy(page_array, pool->page_array,
210 pool->npages * sizeof(struct page *));
211 vfree(pool->page_array);
212 vfree(pool->shrink_array);
213 pool->page_array = page_array;
214 pool->shrink_array = shrink_array;
215out:
216 pr_debug("%s pool resized to %d from %d pages",
217 s_memtype_str[pool->flags], size, pool->max_pages);
218 pool->max_pages = size;
219 goto exit;
220fail:
221 vfree(page_array);
222 vfree(shrink_array);
223 pr_err("failed");
224exit:
225 nvmap_page_pool_unlock(pool);
226}
227
228static int nvmap_page_pool_shrink(struct shrinker *shrinker,
229 struct shrink_control *sc)
230{
231 unsigned int i;
232 unsigned int pool_offset;
233 struct nvmap_page_pool *pool;
234 int shrink_pages = sc->nr_to_scan;
235 static atomic_t start_pool = ATOMIC_INIT(-1);
236 struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev);
237
238 if (!shrink_pages)
239 goto out;
240
241 pr_debug("sh_pages=%d", shrink_pages);
242
243 for (i = 0; i < NVMAP_NUM_POOLS && shrink_pages; i++) {
244 pool_offset = atomic_add_return(1, &start_pool) %
245 NVMAP_NUM_POOLS;
246 pool = &share->pools[pool_offset];
247 shrink_pages = nvmap_page_pool_free(pool, shrink_pages);
248 }
249out:
250 return nvmap_page_pool_get_unused_pages();
251}
252
253static struct shrinker nvmap_page_pool_shrinker = {
254 .shrink = nvmap_page_pool_shrink,
255 .seeks = 1,
256};
257
258static void shrink_page_pools(int *total_pages, int *available_pages)
259{
260 struct shrink_control sc;
261
262 sc.gfp_mask = GFP_KERNEL;
263 sc.nr_to_scan = 0;
264 *total_pages = nvmap_page_pool_shrink(NULL, &sc);
265 sc.nr_to_scan = *total_pages * 2;
266 *available_pages = nvmap_page_pool_shrink(NULL, &sc);
267}
268
269#if NVMAP_TEST_PAGE_POOL_SHRINKER
270static bool shrink_pp;
271static int shrink_set(const char *arg, const struct kernel_param *kp)
272{
273 int cpu = smp_processor_id();
274 unsigned long long t1, t2;
275 int total_pages, available_pages;
276
277 param_set_bool(arg, kp);
278
279 if (shrink_pp) {
280 t1 = cpu_clock(cpu);
281 shrink_page_pools(&total_pages, &available_pages);
282 t2 = cpu_clock(cpu);
283 pr_info("shrink page pools: time=%lldns, "
284 "total_pages_released=%d, free_pages_available=%d",
285 t2-t1, total_pages, available_pages);
286 }
287 return 0;
288}
289
290static int shrink_get(char *buff, const struct kernel_param *kp)
291{
292 return param_get_bool(buff, kp);
293}
294
295static struct kernel_param_ops shrink_ops = {
296 .get = shrink_get,
297 .set = shrink_set,
298};
299
300module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
301#endif
302
303static int enable_pp_set(const char *arg, const struct kernel_param *kp)
304{
305 int total_pages, available_pages;
306
307 param_set_bool(arg, kp);
308
309 if (!enable_pp) {
310 shrink_page_pools(&total_pages, &available_pages);
311 pr_info("disabled page pools and released pages, "
312 "total_pages_released=%d, free_pages_available=%d",
313 total_pages, available_pages);
314 }
315 return 0;
316}
317
318static int enable_pp_get(char *buff, const struct kernel_param *kp)
319{
320 return param_get_int(buff, kp);
321}
322
323static struct kernel_param_ops enable_pp_ops = {
324 .get = enable_pp_get,
325 .set = enable_pp_set,
326};
327
328module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
329
330#define POOL_SIZE_SET(m, i) \
331static int pool_size_##m##_set(const char *arg, const struct kernel_param *kp) \
332{ \
333 struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev); \
334 param_set_int(arg, kp); \
335 nvmap_page_pool_resize(&share->pools[i], pool_size[i]); \
336 return 0; \
337}
338
339#define POOL_SIZE_GET(m) \
340static int pool_size_##m##_get(char *buff, const struct kernel_param *kp) \
341{ \
342 return param_get_int(buff, kp); \
343}
344
345#define POOL_SIZE_OPS(m) \
346static struct kernel_param_ops pool_size_##m##_ops = { \
347 .get = pool_size_##m##_get, \
348 .set = pool_size_##m##_set, \
349};
350
351#define POOL_SIZE_MOUDLE_PARAM_CB(m, i) \
352module_param_cb(m##_pool_size, &pool_size_##m##_ops, &pool_size[i], 0644)
353
354POOL_SIZE_SET(uc, NVMAP_HANDLE_UNCACHEABLE);
355POOL_SIZE_GET(uc);
356POOL_SIZE_OPS(uc);
357POOL_SIZE_MOUDLE_PARAM_CB(uc, NVMAP_HANDLE_UNCACHEABLE);
358
359POOL_SIZE_SET(wc, NVMAP_HANDLE_WRITE_COMBINE);
360POOL_SIZE_GET(wc);
361POOL_SIZE_OPS(wc);
362POOL_SIZE_MOUDLE_PARAM_CB(wc, NVMAP_HANDLE_WRITE_COMBINE);
363
364POOL_SIZE_SET(iwb, NVMAP_HANDLE_INNER_CACHEABLE);
365POOL_SIZE_GET(iwb);
366POOL_SIZE_OPS(iwb);
367POOL_SIZE_MOUDLE_PARAM_CB(iwb, NVMAP_HANDLE_INNER_CACHEABLE);
368
369POOL_SIZE_SET(wb, NVMAP_HANDLE_CACHEABLE);
370POOL_SIZE_GET(wb);
371POOL_SIZE_OPS(wb);
372POOL_SIZE_MOUDLE_PARAM_CB(wb, NVMAP_HANDLE_CACHEABLE);
373
374int nvmap_page_pool_init(struct nvmap_page_pool *pool, int flags)
375{
376 struct page *page;
377 int i;
378 static int reg = 1;
379 struct sysinfo info;
380 typedef int (*set_pages_array) (struct page **pages, int addrinarray);
381 set_pages_array s_cpa[] = {
382 set_pages_array_uc,
383 set_pages_array_wc,
384 set_pages_array_iwb,
385 set_pages_array_wb
386 };
387
388 BUG_ON(flags >= NVMAP_NUM_POOLS);
389 memset(pool, 0x0, sizeof(*pool));
390 mutex_init(&pool->lock);
391 pool->flags = flags;
392
393 /* No default pool for cached memory. */
394 if (flags == NVMAP_HANDLE_CACHEABLE)
395 return 0;
396
397 si_meminfo(&info);
398 if (!pool_size[flags]) {
399 /* Use 3/8th of total ram for page pools.
400 * 1/8th for uc, 1/8th for wc and 1/8th for iwb.
401 */
402 pool->max_pages = info.totalram >> 3;
403 }
404 if (pool->max_pages <= 0 || pool->max_pages >= info.totalram)
405 pool->max_pages = NVMAP_DEFAULT_PAGE_POOL_SIZE;
406 pool_size[flags] = pool->max_pages;
407 pr_info("nvmap %s page pool size=%d pages",
408 s_memtype_str[flags], pool->max_pages);
409 pool->page_array = vmalloc(sizeof(void *) * pool->max_pages);
410 pool->shrink_array = vmalloc(sizeof(struct page *) * pool->max_pages);
411 if (!pool->page_array || !pool->shrink_array)
412 goto fail;
413
414 if (reg) {
415 reg = 0;
416 register_shrinker(&nvmap_page_pool_shrinker);
417 }
418
419 nvmap_page_pool_lock(pool);
420 for (i = 0; i < pool->max_pages; i++) {
421 page = alloc_page(GFP_NVMAP);
422 if (!page)
423 goto do_cpa;
424 if (!nvmap_page_pool_release_locked(pool, page)) {
425 __free_page(page);
426 goto do_cpa;
427 }
428 }
429do_cpa:
430 (*s_cpa[flags])(pool->page_array, pool->npages);
431 nvmap_page_pool_unlock(pool);
432 return 0;
433fail:
434 pool->max_pages = 0;
435 vfree(pool->shrink_array);
436 vfree(pool->page_array);
437 return -ENOMEM;
438}
439
440static inline void *altalloc(size_t len)
441{
442 if (len >= PAGELIST_VMALLOC_MIN)
443 return vmalloc(len);
444 else
445 return kmalloc(len, GFP_KERNEL);
446}
447
448static inline void altfree(void *ptr, size_t len)
449{
450 if (!ptr)
451 return;
452
453 if (len >= PAGELIST_VMALLOC_MIN)
454 vfree(ptr);
455 else
456 kfree(ptr);
457}
458
459void _nvmap_handle_free(struct nvmap_handle *h)
460{
461 struct nvmap_share *share = nvmap_get_share_from_dev(h->dev);
462 unsigned int i, nr_page, page_index = 0;
463 struct nvmap_page_pool *pool = NULL;
464
465 if (nvmap_handle_remove(h->dev, h) != 0)
466 return;
467
468 if (!h->alloc)
469 goto out;
470
471 if (!h->heap_pgalloc) {
472 nvmap_usecount_inc(h);
473 nvmap_heap_free(h->carveout);
474 goto out;
475 }
476
477 nr_page = DIV_ROUND_UP(h->size, PAGE_SIZE);
478
479 BUG_ON(h->size & ~PAGE_MASK);
480 BUG_ON(!h->pgalloc.pages);
481
482 nvmap_mru_remove(share, h);
483
484 if (h->flags < NVMAP_NUM_POOLS)
485 pool = &share->pools[h->flags];
486
487 while (page_index < nr_page) {
488 if (!nvmap_page_pool_release(pool,
489 h->pgalloc.pages[page_index]))
490 break;
491 page_index++;
492 }
493
494 if (page_index == nr_page)
495 goto skip_attr_restore;
496
497 /* Restore page attributes. */
498 if (h->flags == NVMAP_HANDLE_WRITE_COMBINE ||
499 h->flags == NVMAP_HANDLE_UNCACHEABLE ||
500 h->flags == NVMAP_HANDLE_INNER_CACHEABLE)
501 set_pages_array_wb(&h->pgalloc.pages[page_index],
502 nr_page - page_index);
503
504skip_attr_restore:
505 if (h->pgalloc.area)
506 tegra_iovmm_free_vm(h->pgalloc.area);
507
508 for (i = page_index; i < nr_page; i++)
509 __free_page(h->pgalloc.pages[i]);
510
511 altfree(h->pgalloc.pages, nr_page * sizeof(struct page *));
512
513out:
514 kfree(h);
515}
516
517static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size)
518{
519 struct page *page, *p, *e;
520 unsigned int order;
521
522 size = PAGE_ALIGN(size);
523 order = get_order(size);
524 page = alloc_pages(gfp, order);
525
526 if (!page)
527 return NULL;
528
529 split_page(page, order);
530 e = page + (1 << order);
531 for (p = page + (size >> PAGE_SHIFT); p < e; p++)
532 __free_page(p);
533
534 return page;
535}
536
537static int handle_page_alloc(struct nvmap_client *client,
538 struct nvmap_handle *h, bool contiguous)
539{
540 size_t size = PAGE_ALIGN(h->size);
541 struct nvmap_share *share = nvmap_get_share_from_dev(h->dev);
542 unsigned int nr_page = size >> PAGE_SHIFT;
543 pgprot_t prot;
544 unsigned int i = 0, page_index = 0;
545 struct page **pages;
546 struct nvmap_page_pool *pool = NULL;
547
548 pages = altalloc(nr_page * sizeof(*pages));
549 if (!pages)
550 return -ENOMEM;
551
552 prot = nvmap_pgprot(h, pgprot_kernel);
553
554 h->pgalloc.area = NULL;
555 if (contiguous) {
556 struct page *page;
557 page = nvmap_alloc_pages_exact(GFP_NVMAP, size);
558 if (!page)
559 goto fail;
560
561 for (i = 0; i < nr_page; i++)
562 pages[i] = nth_page(page, i);
563
564 } else {
565 if (h->flags < NVMAP_NUM_POOLS)
566 pool = &share->pools[h->flags];
567
568 for (i = 0; i < nr_page; i++) {
569 /* Get pages from pool, if available. */
570 pages[i] = nvmap_page_pool_alloc(pool);
571 if (!pages[i])
572 break;
573 page_index++;
574 }
575
576 for (; i < nr_page; i++) {
577 pages[i] = nvmap_alloc_pages_exact(GFP_NVMAP,
578 PAGE_SIZE);
579 if (!pages[i])
580 goto fail;
581 }
582
583#ifndef CONFIG_NVMAP_RECLAIM_UNPINNED_VM
584 h->pgalloc.area = tegra_iovmm_create_vm(client->share->iovmm,
585 NULL, size, h->align, prot,
586 h->pgalloc.iovm_addr);
587 if (!h->pgalloc.area)
588 goto fail;
589
590 h->pgalloc.dirty = true;
591#endif
592 }
593
594 if (nr_page == page_index)
595 goto skip_attr_change;
596
597 /* Update the pages mapping in kernel page table. */
598 if (h->flags == NVMAP_HANDLE_WRITE_COMBINE)
599 set_pages_array_wc(&pages[page_index],
600 nr_page - page_index);
601 else if (h->flags == NVMAP_HANDLE_UNCACHEABLE)
602 set_pages_array_uc(&pages[page_index],
603 nr_page - page_index);
604 else if (h->flags == NVMAP_HANDLE_INNER_CACHEABLE)
605 set_pages_array_iwb(&pages[page_index],
606 nr_page - page_index);
607
608skip_attr_change:
609 h->size = size;
610 h->pgalloc.pages = pages;
611 h->pgalloc.contig = contiguous;
612 INIT_LIST_HEAD(&h->pgalloc.mru_list);
613 return 0;
614
615fail:
616 while (i--) {
617 set_pages_array_wb(&pages[i], 1);
618 __free_page(pages[i]);
619 }
620 altfree(pages, nr_page * sizeof(*pages));
621 wmb();
622 return -ENOMEM;
623}
624
625static void alloc_handle(struct nvmap_client *client,
626 struct nvmap_handle *h, unsigned int type)
627{
628 BUG_ON(type & (type - 1));
629
630#ifdef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
631#define __NVMAP_HEAP_CARVEOUT (NVMAP_HEAP_CARVEOUT_IRAM | NVMAP_HEAP_CARVEOUT_VPR)
632#define __NVMAP_HEAP_IOVMM (NVMAP_HEAP_IOVMM | NVMAP_HEAP_CARVEOUT_GENERIC)
633 if (type & NVMAP_HEAP_CARVEOUT_GENERIC) {
634#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
635 if (h->size <= PAGE_SIZE) {
636 PR_INFO("###CARVEOUT CONVERTED TO SYSMEM "
637 "0x%x bytes %s(%d)###\n",
638 h->size, current->comm, current->pid);
639 goto sysheap;
640 }
641#endif
642 PR_INFO("###CARVEOUT CONVERTED TO IOVM "
643 "0x%x bytes %s(%d)###\n",
644 h->size, current->comm, current->pid);
645 }
646#else
647#define __NVMAP_HEAP_CARVEOUT NVMAP_HEAP_CARVEOUT_MASK
648#define __NVMAP_HEAP_IOVMM NVMAP_HEAP_IOVMM
649#endif
650
651 if (type & __NVMAP_HEAP_CARVEOUT) {
652 struct nvmap_heap_block *b;
653#ifdef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
654 PR_INFO("###IRAM REQUEST RETAINED "
655 "0x%x bytes %s(%d)###\n",
656 h->size, current->comm, current->pid);
657#endif
658 /* Protect handle from relocation */
659 nvmap_usecount_inc(h);
660
661 b = nvmap_carveout_alloc(client, h, type);
662 if (b) {
663 h->heap_pgalloc = false;
664 h->alloc = true;
665 nvmap_carveout_commit_add(client,
666 nvmap_heap_to_arg(nvmap_block_to_heap(b)),
667 h->size);
668 }
669 nvmap_usecount_dec(h);
670
671 } else if (type & __NVMAP_HEAP_IOVMM) {
672 size_t reserved = PAGE_ALIGN(h->size);
673 int commit = 0;
674 int ret;
675
676 /* increment the committed IOVM space prior to allocation
677 * to avoid race conditions with other threads simultaneously
678 * allocating. */
679 commit = atomic_add_return(reserved,
680 &client->iovm_commit);
681
682 if (commit < client->iovm_limit)
683 ret = handle_page_alloc(client, h, false);
684 else
685 ret = -ENOMEM;
686
687 if (!ret) {
688 h->heap_pgalloc = true;
689 h->alloc = true;
690 } else {
691 atomic_sub(reserved, &client->iovm_commit);
692 }
693
694 } else if (type & NVMAP_HEAP_SYSMEM) {
695#if defined(CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM) && \
696 defined(CONFIG_NVMAP_ALLOW_SYSMEM)
697sysheap:
698#endif
699 if (handle_page_alloc(client, h, true) == 0) {
700 BUG_ON(!h->pgalloc.contig);
701 h->heap_pgalloc = true;
702 h->alloc = true;
703 }
704 }
705}
706
707/* small allocations will try to allocate from generic OS memory before
708 * any of the limited heaps, to increase the effective memory for graphics
709 * allocations, and to reduce fragmentation of the graphics heaps with
710 * sub-page splinters */
711static const unsigned int heap_policy_small[] = {
712 NVMAP_HEAP_CARVEOUT_VPR,
713 NVMAP_HEAP_CARVEOUT_IRAM,
714#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
715 NVMAP_HEAP_SYSMEM,
716#endif
717 NVMAP_HEAP_CARVEOUT_MASK,
718 NVMAP_HEAP_IOVMM,
719 0,
720};
721
722static const unsigned int heap_policy_large[] = {
723 NVMAP_HEAP_CARVEOUT_VPR,
724 NVMAP_HEAP_CARVEOUT_IRAM,
725 NVMAP_HEAP_IOVMM,
726 NVMAP_HEAP_CARVEOUT_MASK,
727#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
728 NVMAP_HEAP_SYSMEM,
729#endif
730 0,
731};
732
733/* Do not override single page policy if there is not much space to
734avoid invoking system oom killer. */
735#define NVMAP_SMALL_POLICY_SYSMEM_THRESHOLD 50000000
736
737int nvmap_alloc_handle_id(struct nvmap_client *client,
738 unsigned long id, unsigned int heap_mask,
739 size_t align, unsigned int flags)
740{
741 struct nvmap_handle *h = NULL;
742 const unsigned int *alloc_policy;
743 int nr_page;
744 int err = -ENOMEM;
745
746 h = nvmap_get_handle_id(client, id);
747
748 if (!h)
749 return -EINVAL;
750
751 if (h->alloc)
752 goto out;
753
754 h->userflags = flags;
755 nr_page = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT);
756 h->secure = !!(flags & NVMAP_HANDLE_SECURE);
757 h->flags = (flags & NVMAP_HANDLE_CACHE_FLAG);
758 h->align = max_t(size_t, align, L1_CACHE_BYTES);
759
760#ifndef CONFIG_TEGRA_IOVMM
761 if (heap_mask & NVMAP_HEAP_IOVMM) {
762 heap_mask &= NVMAP_HEAP_IOVMM;
763 heap_mask |= NVMAP_HEAP_CARVEOUT_GENERIC;
764 }
765#endif
766#ifndef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
767#ifdef CONFIG_NVMAP_ALLOW_SYSMEM
768 /* Allow single pages allocations in system memory to save
769 * carveout space and avoid extra iovm mappings */
770 if (nr_page == 1) {
771 if (heap_mask & NVMAP_HEAP_IOVMM)
772 heap_mask |= NVMAP_HEAP_SYSMEM;
773 else if (heap_mask & NVMAP_HEAP_CARVEOUT_GENERIC) {
774 /* Calculate size of free physical pages
775 * managed by kernel */
776 unsigned long freeMem =
777 (global_page_state(NR_FREE_PAGES) +
778 global_page_state(NR_FILE_PAGES) -
779 total_swapcache_pages) << PAGE_SHIFT;
780
781 if (freeMem > NVMAP_SMALL_POLICY_SYSMEM_THRESHOLD)
782 heap_mask |= NVMAP_HEAP_SYSMEM;
783 }
784 }
785#endif
786
787 /* This restriction is deprecated as alignments greater than
788 PAGE_SIZE are now correctly handled, but it is retained for
789 AP20 compatibility. */
790 if (h->align > PAGE_SIZE)
791 heap_mask &= NVMAP_HEAP_CARVEOUT_MASK;
792#endif
793 /* secure allocations can only be served from secure heaps */
794 if (h->secure)
795 heap_mask &= NVMAP_SECURE_HEAPS;
796
797 if (!heap_mask) {
798 err = -EINVAL;
799 goto out;
800 }
801
802 alloc_policy = (nr_page == 1) ? heap_policy_small : heap_policy_large;
803
804 while (!h->alloc && *alloc_policy) {
805 unsigned int heap_type;
806
807 heap_type = *alloc_policy++;
808 heap_type &= heap_mask;
809
810 if (!heap_type)
811 continue;
812
813 heap_mask &= ~heap_type;
814
815 while (heap_type && !h->alloc) {
816 unsigned int heap;
817
818 /* iterate possible heaps MSB-to-LSB, since higher-
819 * priority carveouts will have higher usage masks */
820 heap = 1 << __fls(heap_type);
821 alloc_handle(client, h, heap);
822 heap_type &= ~heap;
823 }
824 }
825
826out:
827 err = (h->alloc) ? 0 : err;
828 nvmap_handle_put(h);
829 return err;
830}
831
832void nvmap_free_handle_id(struct nvmap_client *client, unsigned long id)
833{
834 struct nvmap_handle_ref *ref;
835 struct nvmap_handle *h;
836 int pins;
837
838 nvmap_ref_lock(client);
839
840 ref = _nvmap_validate_id_locked(client, id);
841 if (!ref) {
842 nvmap_ref_unlock(client);
843 return;
844 }
845
846 BUG_ON(!ref->handle);
847 h = ref->handle;
848
849 if (atomic_dec_return(&ref->dupes)) {
850 nvmap_ref_unlock(client);
851 goto out;
852 }
853
854 smp_rmb();
855 pins = atomic_read(&ref->pin);
856 rb_erase(&ref->node, &client->handle_refs);
857
858 if (h->alloc && h->heap_pgalloc && !h->pgalloc.contig)
859 atomic_sub(h->size, &client->iovm_commit);
860
861 if (h->alloc && !h->heap_pgalloc) {
862 mutex_lock(&h->lock);
863 nvmap_carveout_commit_subtract(client,
864 nvmap_heap_to_arg(nvmap_block_to_heap(h->carveout)),
865 h->size);
866 mutex_unlock(&h->lock);
867 }
868
869 nvmap_ref_unlock(client);
870
871 if (pins)
872 nvmap_err(client, "%s freeing pinned handle %p\n",
873 current->group_leader->comm, h);
874
875 while (pins--)
876 nvmap_unpin_handles(client, &ref->handle, 1);
877
878 if (h->owner == client)
879 h->owner = NULL;
880
881 kfree(ref);
882
883out:
884 BUG_ON(!atomic_read(&h->ref));
885 nvmap_handle_put(h);
886}
887
888static void add_handle_ref(struct nvmap_client *client,
889 struct nvmap_handle_ref *ref)
890{
891 struct rb_node **p, *parent = NULL;
892
893 nvmap_ref_lock(client);
894 p = &client->handle_refs.rb_node;
895 while (*p) {
896 struct nvmap_handle_ref *node;
897 parent = *p;
898 node = rb_entry(parent, struct nvmap_handle_ref, node);
899 if (ref->handle > node->handle)
900 p = &parent->rb_right;
901 else
902 p = &parent->rb_left;
903 }
904 rb_link_node(&ref->node, parent, p);
905 rb_insert_color(&ref->node, &client->handle_refs);
906 nvmap_ref_unlock(client);
907}
908
909struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
910 size_t size)
911{
912 struct nvmap_handle *h;
913 struct nvmap_handle_ref *ref = NULL;
914
915 if (!client)
916 return ERR_PTR(-EINVAL);
917
918 if (!size)
919 return ERR_PTR(-EINVAL);
920
921 h = kzalloc(sizeof(*h), GFP_KERNEL);
922 if (!h)
923 return ERR_PTR(-ENOMEM);
924
925 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
926 if (!ref) {
927 kfree(h);
928 return ERR_PTR(-ENOMEM);
929 }
930
931 atomic_set(&h->ref, 1);
932 atomic_set(&h->pin, 0);
933 h->owner = client;
934 h->dev = client->dev;
935 BUG_ON(!h->owner);
936 h->size = h->orig_size = size;
937 h->flags = NVMAP_HANDLE_WRITE_COMBINE;
938 mutex_init(&h->lock);
939
940 nvmap_handle_add(client->dev, h);
941
942 atomic_set(&ref->dupes, 1);
943 ref->handle = h;
944 atomic_set(&ref->pin, 0);
945 add_handle_ref(client, ref);
946 return ref;
947}
948
949struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
950 unsigned long id)
951{
952 struct nvmap_handle_ref *ref = NULL;
953 struct nvmap_handle *h = NULL;
954
955 BUG_ON(!client || client->dev != nvmap_dev);
956 /* on success, the reference count for the handle should be
957 * incremented, so the success paths will not call nvmap_handle_put */
958 h = nvmap_validate_get(client, id);
959
960 if (!h) {
961 nvmap_debug(client, "%s duplicate handle failed\n",
962 current->group_leader->comm);
963 return ERR_PTR(-EPERM);
964 }
965
966 if (!h->alloc) {
967 nvmap_err(client, "%s duplicating unallocated handle\n",
968 current->group_leader->comm);
969 nvmap_handle_put(h);
970 return ERR_PTR(-EINVAL);
971 }
972
973 nvmap_ref_lock(client);
974 ref = _nvmap_validate_id_locked(client, (unsigned long)h);
975
976 if (ref) {
977 /* handle already duplicated in client; just increment
978 * the reference count rather than re-duplicating it */
979 atomic_inc(&ref->dupes);
980 nvmap_ref_unlock(client);
981 return ref;
982 }
983
984 nvmap_ref_unlock(client);
985
986 /* verify that adding this handle to the process' access list
987 * won't exceed the IOVM limit */
988 if (h->heap_pgalloc && !h->pgalloc.contig) {
989 int oc;
990 oc = atomic_add_return(h->size, &client->iovm_commit);
991 if (oc > client->iovm_limit && !client->super) {
992 atomic_sub(h->size, &client->iovm_commit);
993 nvmap_handle_put(h);
994 nvmap_err(client, "duplicating %p in %s over-commits"
995 " IOVMM space\n", (void *)id,
996 current->group_leader->comm);
997 return ERR_PTR(-ENOMEM);
998 }
999 }
1000
1001 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
1002 if (!ref) {
1003 nvmap_handle_put(h);
1004 return ERR_PTR(-ENOMEM);
1005 }
1006
1007 if (!h->heap_pgalloc) {
1008 mutex_lock(&h->lock);
1009 nvmap_carveout_commit_add(client,
1010 nvmap_heap_to_arg(nvmap_block_to_heap(h->carveout)),
1011 h->size);
1012 mutex_unlock(&h->lock);
1013 }
1014
1015 atomic_set(&ref->dupes, 1);
1016 ref->handle = h;
1017 atomic_set(&ref->pin, 0);
1018 add_handle_ref(client, ref);
1019 return ref;
1020}
diff --git a/drivers/video/tegra/nvmap/nvmap_heap.c b/drivers/video/tegra/nvmap/nvmap_heap.c
new file mode 100644
index 00000000000..7474f31534f
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_heap.c
@@ -0,0 +1,1113 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_heap.c
3 *
4 * GPU heap allocator.
5 *
6 * Copyright (c) 2011, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#include <linux/device.h>
24#include <linux/kernel.h>
25#include <linux/list.h>
26#include <linux/mm.h>
27#include <linux/mutex.h>
28#include <linux/slab.h>
29#include <linux/err.h>
30
31#include <mach/nvmap.h>
32#include "nvmap.h"
33#include "nvmap_heap.h"
34#include "nvmap_common.h"
35
36#include <asm/tlbflush.h>
37#include <asm/cacheflush.h>
38
39/*
40 * "carveouts" are platform-defined regions of physically contiguous memory
41 * which are not managed by the OS. a platform may specify multiple carveouts,
42 * for either small special-purpose memory regions (like IRAM on Tegra SoCs)
43 * or reserved regions of main system memory.
44 *
45 * the carveout allocator returns allocations which are physically contiguous.
46 * to reduce external fragmentation, the allocation algorithm implemented in
47 * this file employs 3 strategies for keeping allocations of similar size
48 * grouped together inside the larger heap: the "small", "normal" and "huge"
49 * strategies. the size thresholds (in bytes) for determining which strategy
50 * to employ should be provided by the platform for each heap. it is possible
51 * for a platform to define a heap where only the "normal" strategy is used.
52 *
53 * o "normal" allocations use an address-order first-fit allocator (called
54 * BOTTOM_UP in the code below). each allocation is rounded up to be
55 * an integer multiple of the "small" allocation size.
56 *
57 * o "huge" allocations use an address-order last-fit allocator (called
58 * TOP_DOWN in the code below). like "normal" allocations, each allocation
59 * is rounded up to be an integer multiple of the "small" allocation size.
60 *
61 * o "small" allocations are treated differently: the heap manager maintains
62 * a pool of "small"-sized blocks internally from which allocations less
63 * than 1/2 of the "small" size are buddy-allocated. if a "small" allocation
64 * is requested and none of the buddy sub-heaps is able to service it,
65 * the heap manager will try to allocate a new buddy-heap.
66 *
67 * this allocator is intended to keep "splinters" colocated in the carveout,
68 * and to ensure that the minimum free block size in the carveout (i.e., the
69 * "small" threshold) is still a meaningful size.
70 *
71 */
72
73#define MAX_BUDDY_NR 128 /* maximum buddies in a buddy allocator */
74
75enum direction {
76 TOP_DOWN,
77 BOTTOM_UP
78};
79
80enum block_type {
81 BLOCK_FIRST_FIT, /* block was allocated directly from the heap */
82 BLOCK_BUDDY, /* block was allocated from a buddy sub-heap */
83 BLOCK_EMPTY,
84};
85
86struct heap_stat {
87 size_t free; /* total free size */
88 size_t free_largest; /* largest free block */
89 size_t free_count; /* number of free blocks */
90 size_t total; /* total size */
91 size_t largest; /* largest unique block */
92 size_t count; /* total number of blocks */
93 /* fast compaction attempt counter */
94 unsigned int compaction_count_fast;
95 /* full compaction attempt counter */
96 unsigned int compaction_count_full;
97};
98
99struct buddy_heap;
100
101struct buddy_block {
102 struct nvmap_heap_block block;
103 struct buddy_heap *heap;
104};
105
106struct list_block {
107 struct nvmap_heap_block block;
108 struct list_head all_list;
109 unsigned int mem_prot;
110 unsigned long orig_addr;
111 size_t size;
112 size_t align;
113 struct nvmap_heap *heap;
114 struct list_head free_list;
115};
116
117struct combo_block {
118 union {
119 struct list_block lb;
120 struct buddy_block bb;
121 };
122};
123
124struct buddy_bits {
125 unsigned int alloc:1;
126 unsigned int order:7; /* log2(MAX_BUDDY_NR); */
127};
128
129struct buddy_heap {
130 struct list_block *heap_base;
131 unsigned int nr_buddies;
132 struct list_head buddy_list;
133 struct buddy_bits bitmap[MAX_BUDDY_NR];
134};
135
136struct nvmap_heap {
137 struct list_head all_list;
138 struct list_head free_list;
139 struct mutex lock;
140 struct list_head buddy_list;
141 unsigned int min_buddy_shift;
142 unsigned int buddy_heap_size;
143 unsigned int small_alloc;
144 const char *name;
145 void *arg;
146 struct device dev;
147};
148
149static struct kmem_cache *buddy_heap_cache;
150static struct kmem_cache *block_cache;
151
152static inline struct nvmap_heap *parent_of(struct buddy_heap *heap)
153{
154 return heap->heap_base->heap;
155}
156
157static inline unsigned int order_of(size_t len, size_t min_shift)
158{
159 len = 2 * DIV_ROUND_UP(len, (1 << min_shift)) - 1;
160 return fls(len)-1;
161}
162
163/* returns the free size in bytes of the buddy heap; must be called while
164 * holding the parent heap's lock. */
165static void buddy_stat(struct buddy_heap *heap, struct heap_stat *stat)
166{
167 unsigned int index;
168 unsigned int shift = parent_of(heap)->min_buddy_shift;
169
170 for (index = 0; index < heap->nr_buddies;
171 index += (1 << heap->bitmap[index].order)) {
172 size_t curr = 1 << (heap->bitmap[index].order + shift);
173
174 stat->largest = max(stat->largest, curr);
175 stat->total += curr;
176 stat->count++;
177
178 if (!heap->bitmap[index].alloc) {
179 stat->free += curr;
180 stat->free_largest = max(stat->free_largest, curr);
181 stat->free_count++;
182 }
183 }
184}
185
186/* returns the free size of the heap (including any free blocks in any
187 * buddy-heap suballocators; must be called while holding the parent
188 * heap's lock. */
189static unsigned long heap_stat(struct nvmap_heap *heap, struct heap_stat *stat)
190{
191 struct buddy_heap *bh;
192 struct list_block *l = NULL;
193 unsigned long base = -1ul;
194
195 memset(stat, 0, sizeof(*stat));
196 mutex_lock(&heap->lock);
197 list_for_each_entry(l, &heap->all_list, all_list) {
198 stat->total += l->size;
199 stat->largest = max(l->size, stat->largest);
200 stat->count++;
201 base = min(base, l->orig_addr);
202 }
203
204 list_for_each_entry(bh, &heap->buddy_list, buddy_list) {
205 buddy_stat(bh, stat);
206 /* the total counts are double-counted for buddy heaps
207 * since the blocks allocated for buddy heaps exist in the
208 * all_list; subtract out the doubly-added stats */
209 stat->total -= bh->heap_base->size;
210 stat->count--;
211 }
212
213 list_for_each_entry(l, &heap->free_list, free_list) {
214 stat->free += l->size;
215 stat->free_count++;
216 stat->free_largest = max(l->size, stat->free_largest);
217 }
218 mutex_unlock(&heap->lock);
219
220 return base;
221}
222
223static ssize_t heap_name_show(struct device *dev,
224 struct device_attribute *attr, char *buf);
225
226static ssize_t heap_stat_show(struct device *dev,
227 struct device_attribute *attr, char *buf);
228
229static struct device_attribute heap_stat_total_max =
230 __ATTR(total_max, S_IRUGO, heap_stat_show, NULL);
231
232static struct device_attribute heap_stat_total_count =
233 __ATTR(total_count, S_IRUGO, heap_stat_show, NULL);
234
235static struct device_attribute heap_stat_total_size =
236 __ATTR(total_size, S_IRUGO, heap_stat_show, NULL);
237
238static struct device_attribute heap_stat_free_max =
239 __ATTR(free_max, S_IRUGO, heap_stat_show, NULL);
240
241static struct device_attribute heap_stat_free_count =
242 __ATTR(free_count, S_IRUGO, heap_stat_show, NULL);
243
244static struct device_attribute heap_stat_free_size =
245 __ATTR(free_size, S_IRUGO, heap_stat_show, NULL);
246
247static struct device_attribute heap_stat_base =
248 __ATTR(base, S_IRUGO, heap_stat_show, NULL);
249
250static struct device_attribute heap_attr_name =
251 __ATTR(name, S_IRUGO, heap_name_show, NULL);
252
253static struct attribute *heap_stat_attrs[] = {
254 &heap_stat_total_max.attr,
255 &heap_stat_total_count.attr,
256 &heap_stat_total_size.attr,
257 &heap_stat_free_max.attr,
258 &heap_stat_free_count.attr,
259 &heap_stat_free_size.attr,
260 &heap_stat_base.attr,
261 &heap_attr_name.attr,
262 NULL,
263};
264
265static struct attribute_group heap_stat_attr_group = {
266 .attrs = heap_stat_attrs,
267};
268
269static ssize_t heap_name_show(struct device *dev,
270 struct device_attribute *attr, char *buf)
271{
272
273 struct nvmap_heap *heap = container_of(dev, struct nvmap_heap, dev);
274 return sprintf(buf, "%s\n", heap->name);
275}
276
277static ssize_t heap_stat_show(struct device *dev,
278 struct device_attribute *attr, char *buf)
279{
280 struct nvmap_heap *heap = container_of(dev, struct nvmap_heap, dev);
281 struct heap_stat stat;
282 unsigned long base;
283
284 base = heap_stat(heap, &stat);
285
286 if (attr == &heap_stat_total_max)
287 return sprintf(buf, "%u\n", stat.largest);
288 else if (attr == &heap_stat_total_count)
289 return sprintf(buf, "%u\n", stat.count);
290 else if (attr == &heap_stat_total_size)
291 return sprintf(buf, "%u\n", stat.total);
292 else if (attr == &heap_stat_free_max)
293 return sprintf(buf, "%u\n", stat.free_largest);
294 else if (attr == &heap_stat_free_count)
295 return sprintf(buf, "%u\n", stat.free_count);
296 else if (attr == &heap_stat_free_size)
297 return sprintf(buf, "%u\n", stat.free);
298 else if (attr == &heap_stat_base)
299 return sprintf(buf, "%08lx\n", base);
300 else
301 return -EINVAL;
302}
303#ifndef CONFIG_NVMAP_CARVEOUT_COMPACTOR
304static struct nvmap_heap_block *buddy_alloc(struct buddy_heap *heap,
305 size_t size, size_t align,
306 unsigned int mem_prot)
307{
308 unsigned int index = 0;
309 unsigned int min_shift = parent_of(heap)->min_buddy_shift;
310 unsigned int order = order_of(size, min_shift);
311 unsigned int align_mask;
312 unsigned int best = heap->nr_buddies;
313 struct buddy_block *b;
314
315 if (heap->heap_base->mem_prot != mem_prot)
316 return NULL;
317
318 align = max(align, (size_t)(1 << min_shift));
319 align_mask = (align >> min_shift) - 1;
320
321 for (index = 0; index < heap->nr_buddies;
322 index += (1 << heap->bitmap[index].order)) {
323
324 if (heap->bitmap[index].alloc || (index & align_mask) ||
325 (heap->bitmap[index].order < order))
326 continue;
327
328 if (best == heap->nr_buddies ||
329 heap->bitmap[index].order < heap->bitmap[best].order)
330 best = index;
331
332 if (heap->bitmap[best].order == order)
333 break;
334 }
335
336 if (best == heap->nr_buddies)
337 return NULL;
338
339 b = kmem_cache_zalloc(block_cache, GFP_KERNEL);
340 if (!b)
341 return NULL;
342
343 while (heap->bitmap[best].order != order) {
344 unsigned int buddy;
345 heap->bitmap[best].order--;
346 buddy = best ^ (1 << heap->bitmap[best].order);
347 heap->bitmap[buddy].order = heap->bitmap[best].order;
348 heap->bitmap[buddy].alloc = 0;
349 }
350 heap->bitmap[best].alloc = 1;
351 b->block.base = heap->heap_base->block.base + (best << min_shift);
352 b->heap = heap;
353 b->block.type = BLOCK_BUDDY;
354 return &b->block;
355}
356#endif
357
358static struct buddy_heap *do_buddy_free(struct nvmap_heap_block *block)
359{
360 struct buddy_block *b = container_of(block, struct buddy_block, block);
361 struct buddy_heap *h = b->heap;
362 unsigned int min_shift = parent_of(h)->min_buddy_shift;
363 unsigned int index;
364
365 index = (block->base - h->heap_base->block.base) >> min_shift;
366 h->bitmap[index].alloc = 0;
367
368 for (;;) {
369 unsigned int buddy = index ^ (1 << h->bitmap[index].order);
370 if (buddy >= h->nr_buddies || h->bitmap[buddy].alloc ||
371 h->bitmap[buddy].order != h->bitmap[index].order)
372 break;
373
374 h->bitmap[buddy].order++;
375 h->bitmap[index].order++;
376 index = min(buddy, index);
377 }
378
379 kmem_cache_free(block_cache, b);
380 if ((1 << h->bitmap[0].order) == h->nr_buddies)
381 return h;
382
383 return NULL;
384}
385
386
387/*
388 * base_max limits position of allocated chunk in memory.
389 * if base_max is 0 then there is no such limitation.
390 */
391static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap,
392 size_t len, size_t align,
393 unsigned int mem_prot,
394 unsigned long base_max)
395{
396 struct list_block *b = NULL;
397 struct list_block *i = NULL;
398 struct list_block *rem = NULL;
399 unsigned long fix_base;
400 enum direction dir;
401
402 /* since pages are only mappable with one cache attribute,
403 * and most allocations from carveout heaps are DMA coherent
404 * (i.e., non-cacheable), round cacheable allocations up to
405 * a page boundary to ensure that the physical pages will
406 * only be mapped one way. */
407 if (mem_prot == NVMAP_HANDLE_CACHEABLE ||
408 mem_prot == NVMAP_HANDLE_INNER_CACHEABLE) {
409 align = max_t(size_t, align, PAGE_SIZE);
410 len = PAGE_ALIGN(len);
411 }
412
413#ifdef CONFIG_NVMAP_CARVEOUT_COMPACTOR
414 dir = BOTTOM_UP;
415#else
416 dir = (len <= heap->small_alloc) ? BOTTOM_UP : TOP_DOWN;
417#endif
418
419 if (dir == BOTTOM_UP) {
420 list_for_each_entry(i, &heap->free_list, free_list) {
421 size_t fix_size;
422 fix_base = ALIGN(i->block.base, align);
423 fix_size = i->size - (fix_base - i->block.base);
424
425 /* needed for compaction. relocated chunk
426 * should never go up */
427 if (base_max && fix_base > base_max)
428 break;
429
430 if (fix_size >= len) {
431 b = i;
432 break;
433 }
434 }
435 } else {
436 list_for_each_entry_reverse(i, &heap->free_list, free_list) {
437 if (i->size >= len) {
438 fix_base = i->block.base + i->size - len;
439 fix_base &= ~(align-1);
440 if (fix_base >= i->block.base) {
441 b = i;
442 break;
443 }
444 }
445 }
446 }
447
448 if (!b)
449 return NULL;
450
451 if (dir == BOTTOM_UP)
452 b->block.type = BLOCK_FIRST_FIT;
453
454 /* split free block */
455 if (b->block.base != fix_base) {
456 /* insert a new free block before allocated */
457 rem = kmem_cache_zalloc(block_cache, GFP_KERNEL);
458 if (!rem) {
459 b->orig_addr = b->block.base;
460 b->block.base = fix_base;
461 b->size -= (b->block.base - b->orig_addr);
462 goto out;
463 }
464
465 rem->block.type = BLOCK_EMPTY;
466 rem->block.base = b->block.base;
467 rem->orig_addr = rem->block.base;
468 rem->size = fix_base - rem->block.base;
469 b->block.base = fix_base;
470 b->orig_addr = fix_base;
471 b->size -= rem->size;
472 list_add_tail(&rem->all_list, &b->all_list);
473 list_add_tail(&rem->free_list, &b->free_list);
474 }
475
476 b->orig_addr = b->block.base;
477
478 if (b->size > len) {
479 /* insert a new free block after allocated */
480 rem = kmem_cache_zalloc(block_cache, GFP_KERNEL);
481 if (!rem)
482 goto out;
483
484 rem->block.type = BLOCK_EMPTY;
485 rem->block.base = b->block.base + len;
486 rem->size = b->size - len;
487 BUG_ON(rem->size > b->size);
488 rem->orig_addr = rem->block.base;
489 b->size = len;
490 list_add(&rem->all_list, &b->all_list);
491 list_add(&rem->free_list, &b->free_list);
492 }
493
494out:
495 list_del(&b->free_list);
496 b->heap = heap;
497 b->mem_prot = mem_prot;
498 b->align = align;
499 return &b->block;
500}
501
502#ifdef DEBUG_FREE_LIST
503static void freelist_debug(struct nvmap_heap *heap, const char *title,
504 struct list_block *token)
505{
506 int i;
507 struct list_block *n;
508
509 dev_debug(&heap->dev, "%s\n", title);
510 i = 0;
511 list_for_each_entry(n, &heap->free_list, free_list) {
512 dev_debug(&heap->dev, "\t%d [%p..%p]%s\n", i, (void *)n->orig_addr,
513 (void *)(n->orig_addr + n->size),
514 (n == token) ? "<--" : "");
515 i++;
516 }
517}
518#else
519#define freelist_debug(_heap, _title, _token) do { } while (0)
520#endif
521
522static struct list_block *do_heap_free(struct nvmap_heap_block *block)
523{
524 struct list_block *b = container_of(block, struct list_block, block);
525 struct list_block *n = NULL;
526 struct nvmap_heap *heap = b->heap;
527
528 BUG_ON(b->block.base > b->orig_addr);
529 b->size += (b->block.base - b->orig_addr);
530 b->block.base = b->orig_addr;
531
532 freelist_debug(heap, "free list before", b);
533
534 /* Find position of first free block to the right of freed one */
535 list_for_each_entry(n, &heap->free_list, free_list) {
536 if (n->block.base > b->block.base)
537 break;
538 }
539
540 /* Add freed block before found free one */
541 list_add_tail(&b->free_list, &n->free_list);
542 BUG_ON(list_empty(&b->all_list));
543
544 freelist_debug(heap, "free list pre-merge", b);
545
546 /* merge freed block with next if they connect
547 * freed block becomes bigger, next one is destroyed */
548 if (!list_is_last(&b->free_list, &heap->free_list)) {
549 n = list_first_entry(&b->free_list, struct list_block, free_list);
550 if (n->block.base == b->block.base + b->size) {
551 list_del(&n->all_list);
552 list_del(&n->free_list);
553 BUG_ON(b->orig_addr >= n->orig_addr);
554 b->size += n->size;
555 kmem_cache_free(block_cache, n);
556 }
557 }
558
559 /* merge freed block with prev if they connect
560 * previous free block becomes bigger, freed one is destroyed */
561 if (b->free_list.prev != &heap->free_list) {
562 n = list_entry(b->free_list.prev, struct list_block, free_list);
563 if (n->block.base + n->size == b->block.base) {
564 list_del(&b->all_list);
565 list_del(&b->free_list);
566 BUG_ON(n->orig_addr >= b->orig_addr);
567 n->size += b->size;
568 kmem_cache_free(block_cache, b);
569 b = n;
570 }
571 }
572
573 freelist_debug(heap, "free list after", b);
574 b->block.type = BLOCK_EMPTY;
575 return b;
576}
577
578#ifndef CONFIG_NVMAP_CARVEOUT_COMPACTOR
579
580static struct nvmap_heap_block *do_buddy_alloc(struct nvmap_heap *h,
581 size_t len, size_t align,
582 unsigned int mem_prot)
583{
584 struct buddy_heap *bh;
585 struct nvmap_heap_block *b = NULL;
586
587 list_for_each_entry(bh, &h->buddy_list, buddy_list) {
588 b = buddy_alloc(bh, len, align, mem_prot);
589 if (b)
590 return b;
591 }
592
593 /* no buddy heaps could service this allocation: try to create a new
594 * buddy heap instead */
595 bh = kmem_cache_zalloc(buddy_heap_cache, GFP_KERNEL);
596 if (!bh)
597 return NULL;
598
599 b = do_heap_alloc(h, h->buddy_heap_size,
600 h->buddy_heap_size, mem_prot, 0);
601 if (!b) {
602 kmem_cache_free(buddy_heap_cache, bh);
603 return NULL;
604 }
605
606 bh->heap_base = container_of(b, struct list_block, block);
607 bh->nr_buddies = h->buddy_heap_size >> h->min_buddy_shift;
608 bh->bitmap[0].alloc = 0;
609 bh->bitmap[0].order = order_of(h->buddy_heap_size, h->min_buddy_shift);
610 list_add_tail(&bh->buddy_list, &h->buddy_list);
611 return buddy_alloc(bh, len, align, mem_prot);
612}
613
614#endif
615
616#ifdef CONFIG_NVMAP_CARVEOUT_COMPACTOR
617
618static int do_heap_copy_listblock(struct nvmap_device *dev,
619 unsigned long dst_base, unsigned long src_base, size_t len)
620{
621 pte_t **pte_src = NULL;
622 pte_t **pte_dst = NULL;
623 void *addr_src = NULL;
624 void *addr_dst = NULL;
625 unsigned long kaddr_src;
626 unsigned long kaddr_dst;
627 unsigned long phys_src = src_base;
628 unsigned long phys_dst = dst_base;
629 unsigned long pfn_src;
630 unsigned long pfn_dst;
631 int error = 0;
632
633 pgprot_t prot = pgprot_writecombine(pgprot_kernel);
634
635 int page;
636
637 pte_src = nvmap_alloc_pte(dev, &addr_src);
638 if (IS_ERR(pte_src)) {
639 pr_err("Error when allocating pte_src\n");
640 pte_src = NULL;
641 error = -1;
642 goto fail;
643 }
644
645 pte_dst = nvmap_alloc_pte(dev, &addr_dst);
646 if (IS_ERR(pte_dst)) {
647 pr_err("Error while allocating pte_dst\n");
648 pte_dst = NULL;
649 error = -1;
650 goto fail;
651 }
652
653 kaddr_src = (unsigned long)addr_src;
654 kaddr_dst = (unsigned long)addr_dst;
655
656 BUG_ON(phys_dst > phys_src);
657 BUG_ON((phys_src & PAGE_MASK) != phys_src);
658 BUG_ON((phys_dst & PAGE_MASK) != phys_dst);
659 BUG_ON((len & PAGE_MASK) != len);
660
661 for (page = 0; page < (len >> PAGE_SHIFT) ; page++) {
662
663 pfn_src = __phys_to_pfn(phys_src) + page;
664 pfn_dst = __phys_to_pfn(phys_dst) + page;
665
666 set_pte_at(&init_mm, kaddr_src, *pte_src,
667 pfn_pte(pfn_src, prot));
668 flush_tlb_kernel_page(kaddr_src);
669
670 set_pte_at(&init_mm, kaddr_dst, *pte_dst,
671 pfn_pte(pfn_dst, prot));
672 flush_tlb_kernel_page(kaddr_dst);
673
674 memcpy(addr_dst, addr_src, PAGE_SIZE);
675 }
676
677fail:
678 if (pte_src)
679 nvmap_free_pte(dev, pte_src);
680 if (pte_dst)
681 nvmap_free_pte(dev, pte_dst);
682 return error;
683}
684
685
686static struct nvmap_heap_block *do_heap_relocate_listblock(
687 struct list_block *block, bool fast)
688{
689 struct nvmap_heap_block *heap_block = &block->block;
690 struct nvmap_heap_block *heap_block_new = NULL;
691 struct nvmap_heap *heap = block->heap;
692 struct nvmap_handle *handle = heap_block->handle;
693 unsigned long src_base = heap_block->base;
694 unsigned long dst_base;
695 size_t src_size = block->size;
696 size_t src_align = block->align;
697 unsigned int src_prot = block->mem_prot;
698 int error = 0;
699 struct nvmap_share *share;
700
701 if (!handle) {
702 pr_err("INVALID HANDLE!\n");
703 return NULL;
704 }
705
706 mutex_lock(&handle->lock);
707
708 share = nvmap_get_share_from_dev(handle->dev);
709
710 /* TODO: It is possible to use only handle lock and no share
711 * pin_lock, but then we'll need to lock every handle during
712 * each pinning operation. Need to estimate performance impact
713 * if we decide to simplify locking this way. */
714 mutex_lock(&share->pin_lock);
715
716 /* abort if block is pinned */
717 if (atomic_read(&handle->pin))
718 goto fail;
719 /* abort if block is mapped */
720 if (handle->usecount)
721 goto fail;
722
723 if (fast) {
724 /* Fast compaction path - first allocate, then free. */
725 heap_block_new = do_heap_alloc(heap, src_size, src_align,
726 src_prot, src_base);
727 if (heap_block_new)
728 do_heap_free(heap_block);
729 else
730 goto fail;
731 } else {
732 /* Full compaction path, first free, then allocate
733 * It is slower but provide best compaction results */
734 do_heap_free(heap_block);
735 heap_block_new = do_heap_alloc(heap, src_size, src_align,
736 src_prot, src_base);
737 /* Allocation should always succeed*/
738 BUG_ON(!heap_block_new);
739 }
740
741 /* update handle */
742 handle->carveout = heap_block_new;
743 heap_block_new->handle = handle;
744
745 /* copy source data to new block location */
746 dst_base = heap_block_new->base;
747
748 /* new allocation should always go lower addresses */
749 BUG_ON(dst_base >= src_base);
750
751 error = do_heap_copy_listblock(handle->dev,
752 dst_base, src_base, src_size);
753 BUG_ON(error);
754
755fail:
756 mutex_unlock(&share->pin_lock);
757 mutex_unlock(&handle->lock);
758 return heap_block_new;
759}
760
761static void nvmap_heap_compact(struct nvmap_heap *heap,
762 size_t requested_size, bool fast)
763{
764 struct list_block *block_current = NULL;
765 struct list_block *block_prev = NULL;
766 struct list_block *block_next = NULL;
767
768 struct list_head *ptr, *ptr_prev, *ptr_next;
769 int relocation_count = 0;
770
771 ptr = heap->all_list.next;
772
773 /* walk through all blocks */
774 while (ptr != &heap->all_list) {
775 block_current = list_entry(ptr, struct list_block, all_list);
776
777 ptr_prev = ptr->prev;
778 ptr_next = ptr->next;
779
780 if (block_current->block.type != BLOCK_EMPTY) {
781 ptr = ptr_next;
782 continue;
783 }
784
785 if (fast && block_current->size >= requested_size)
786 break;
787
788 /* relocate prev block */
789 if (ptr_prev != &heap->all_list) {
790
791 block_prev = list_entry(ptr_prev,
792 struct list_block, all_list);
793
794 BUG_ON(block_prev->block.type != BLOCK_FIRST_FIT);
795
796 if (do_heap_relocate_listblock(block_prev, true)) {
797
798 /* After relocation current free block can be
799 * destroyed when it is merged with previous
800 * free block. Updated pointer to new free
801 * block can be obtained from the next block */
802 relocation_count++;
803 ptr = ptr_next->prev;
804 continue;
805 }
806 }
807
808 if (ptr_next != &heap->all_list) {
809
810 block_next = list_entry(ptr_next,
811 struct list_block, all_list);
812
813 BUG_ON(block_next->block.type != BLOCK_FIRST_FIT);
814
815 if (do_heap_relocate_listblock(block_next, fast)) {
816 ptr = ptr_prev->next;
817 relocation_count++;
818 continue;
819 }
820 }
821 ptr = ptr_next;
822 }
823 pr_err("Relocated %d chunks\n", relocation_count);
824}
825#endif
826
827void nvmap_usecount_inc(struct nvmap_handle *h)
828{
829 if (h->alloc && !h->heap_pgalloc) {
830 mutex_lock(&h->lock);
831 h->usecount++;
832 mutex_unlock(&h->lock);
833 } else {
834 h->usecount++;
835 }
836}
837
838
839void nvmap_usecount_dec(struct nvmap_handle *h)
840{
841 h->usecount--;
842}
843
844/* nvmap_heap_alloc: allocates a block of memory of len bytes, aligned to
845 * align bytes. */
846struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *h,
847 struct nvmap_handle *handle)
848{
849 struct nvmap_heap_block *b;
850 size_t len = handle->size;
851 size_t align = handle->align;
852 unsigned int prot = handle->flags;
853
854 mutex_lock(&h->lock);
855
856#ifdef CONFIG_NVMAP_CARVEOUT_COMPACTOR
857 /* Align to page size */
858 align = ALIGN(align, PAGE_SIZE);
859 len = ALIGN(len, PAGE_SIZE);
860 b = do_heap_alloc(h, len, align, prot, 0);
861 if (!b) {
862 pr_err("Compaction triggered!\n");
863 nvmap_heap_compact(h, len, true);
864 b = do_heap_alloc(h, len, align, prot, 0);
865 if (!b) {
866 pr_err("Full compaction triggered!\n");
867 nvmap_heap_compact(h, len, false);
868 b = do_heap_alloc(h, len, align, prot, 0);
869 }
870 }
871#else
872 if (len <= h->buddy_heap_size / 2) {
873 b = do_buddy_alloc(h, len, align, prot);
874 } else {
875 if (h->buddy_heap_size)
876 len = ALIGN(len, h->buddy_heap_size);
877 align = max(align, (size_t)L1_CACHE_BYTES);
878 b = do_heap_alloc(h, len, align, prot, 0);
879 }
880#endif
881
882 if (b) {
883 b->handle = handle;
884 handle->carveout = b;
885 }
886 mutex_unlock(&h->lock);
887 return b;
888}
889
890struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b)
891{
892 if (b->type == BLOCK_BUDDY) {
893 struct buddy_block *bb;
894 bb = container_of(b, struct buddy_block, block);
895 return parent_of(bb->heap);
896 } else {
897 struct list_block *lb;
898 lb = container_of(b, struct list_block, block);
899 return lb->heap;
900 }
901}
902
903/* nvmap_heap_free: frees block b*/
904void nvmap_heap_free(struct nvmap_heap_block *b)
905{
906 struct buddy_heap *bh = NULL;
907 struct nvmap_heap *h = nvmap_block_to_heap(b);
908 struct list_block *lb;
909
910 mutex_lock(&h->lock);
911 if (b->type == BLOCK_BUDDY)
912 bh = do_buddy_free(b);
913 else {
914 lb = container_of(b, struct list_block, block);
915 nvmap_flush_heap_block(NULL, b, lb->size, lb->mem_prot);
916 do_heap_free(b);
917 }
918
919 if (bh) {
920 list_del(&bh->buddy_list);
921 mutex_unlock(&h->lock);
922 nvmap_heap_free(&bh->heap_base->block);
923 kmem_cache_free(buddy_heap_cache, bh);
924 } else
925 mutex_unlock(&h->lock);
926}
927
928
929static void heap_release(struct device *heap)
930{
931}
932
933/* nvmap_heap_create: create a heap object of len bytes, starting from
934 * address base.
935 *
936 * if buddy_size is >= NVMAP_HEAP_MIN_BUDDY_SIZE, then allocations <= 1/2
937 * of the buddy heap size will use a buddy sub-allocator, where each buddy
938 * heap is buddy_size bytes (should be a power of 2). all other allocations
939 * will be rounded up to be a multiple of buddy_size bytes.
940 */
941struct nvmap_heap *nvmap_heap_create(struct device *parent, const char *name,
942 phys_addr_t base, size_t len,
943 size_t buddy_size, void *arg)
944{
945 struct nvmap_heap *h = NULL;
946 struct list_block *l = NULL;
947
948 if (WARN_ON(buddy_size && buddy_size < NVMAP_HEAP_MIN_BUDDY_SIZE)) {
949 dev_warn(parent, "%s: buddy_size %u too small\n", __func__,
950 buddy_size);
951 buddy_size = 0;
952 } else if (WARN_ON(buddy_size >= len)) {
953 dev_warn(parent, "%s: buddy_size %u too large\n", __func__,
954 buddy_size);
955 buddy_size = 0;
956 } else if (WARN_ON(buddy_size & (buddy_size - 1))) {
957 dev_warn(parent, "%s: buddy_size %u not a power of 2\n",
958 __func__, buddy_size);
959 buddy_size = 1 << (ilog2(buddy_size) + 1);
960 }
961
962 if (WARN_ON(buddy_size && (base & (buddy_size - 1)))) {
963 unsigned long orig = base;
964 dev_warn(parent, "%s: base address %p not aligned to "
965 "buddy_size %u\n", __func__, (void *)base, buddy_size);
966 base = ALIGN(base, buddy_size);
967 len -= (base - orig);
968 }
969
970 if (WARN_ON(buddy_size && (len & (buddy_size - 1)))) {
971 dev_warn(parent, "%s: length %u not aligned to "
972 "buddy_size %u\n", __func__, len, buddy_size);
973 len &= ~(buddy_size - 1);
974 }
975
976 h = kzalloc(sizeof(*h), GFP_KERNEL);
977 if (!h) {
978 dev_err(parent, "%s: out of memory\n", __func__);
979 goto fail_alloc;
980 }
981
982 l = kmem_cache_zalloc(block_cache, GFP_KERNEL);
983 if (!l) {
984 dev_err(parent, "%s: out of memory\n", __func__);
985 goto fail_alloc;
986 }
987
988 dev_set_name(&h->dev, "heap-%s", name);
989 h->name = name;
990 h->arg = arg;
991 h->dev.parent = parent;
992 h->dev.driver = NULL;
993 h->dev.release = heap_release;
994 if (device_register(&h->dev)) {
995 dev_err(parent, "%s: failed to register %s\n", __func__,
996 dev_name(&h->dev));
997 goto fail_alloc;
998 }
999 if (sysfs_create_group(&h->dev.kobj, &heap_stat_attr_group)) {
1000 dev_err(&h->dev, "%s: failed to create attributes\n", __func__);
1001 goto fail_register;
1002 }
1003 h->small_alloc = max(2 * buddy_size, len / 256);
1004 h->buddy_heap_size = buddy_size;
1005 if (buddy_size)
1006 h->min_buddy_shift = ilog2(buddy_size / MAX_BUDDY_NR);
1007 INIT_LIST_HEAD(&h->free_list);
1008 INIT_LIST_HEAD(&h->buddy_list);
1009 INIT_LIST_HEAD(&h->all_list);
1010 mutex_init(&h->lock);
1011 l->block.base = base;
1012 l->block.type = BLOCK_EMPTY;
1013 l->size = len;
1014 l->orig_addr = base;
1015 list_add_tail(&l->free_list, &h->free_list);
1016 list_add_tail(&l->all_list, &h->all_list);
1017
1018 inner_flush_cache_all();
1019 outer_flush_range(base, base + len);
1020 wmb();
1021 return h;
1022
1023fail_register:
1024 device_unregister(&h->dev);
1025fail_alloc:
1026 if (l)
1027 kmem_cache_free(block_cache, l);
1028 kfree(h);
1029 return NULL;
1030}
1031
1032void *nvmap_heap_device_to_arg(struct device *dev)
1033{
1034 struct nvmap_heap *heap = container_of(dev, struct nvmap_heap, dev);
1035 return heap->arg;
1036}
1037
1038void *nvmap_heap_to_arg(struct nvmap_heap *heap)
1039{
1040 return heap->arg;
1041}
1042
1043/* nvmap_heap_destroy: frees all resources in heap */
1044void nvmap_heap_destroy(struct nvmap_heap *heap)
1045{
1046 WARN_ON(!list_empty(&heap->buddy_list));
1047
1048 sysfs_remove_group(&heap->dev.kobj, &heap_stat_attr_group);
1049 device_unregister(&heap->dev);
1050
1051 while (!list_empty(&heap->buddy_list)) {
1052 struct buddy_heap *b;
1053 b = list_first_entry(&heap->buddy_list, struct buddy_heap,
1054 buddy_list);
1055 list_del(&heap->buddy_list);
1056 nvmap_heap_free(&b->heap_base->block);
1057 kmem_cache_free(buddy_heap_cache, b);
1058 }
1059
1060 WARN_ON(!list_is_singular(&heap->all_list));
1061 while (!list_empty(&heap->all_list)) {
1062 struct list_block *l;
1063 l = list_first_entry(&heap->all_list, struct list_block,
1064 all_list);
1065 list_del(&l->all_list);
1066 kmem_cache_free(block_cache, l);
1067 }
1068
1069 kfree(heap);
1070}
1071
1072/* nvmap_heap_create_group: adds the attribute_group grp to the heap kobject */
1073int nvmap_heap_create_group(struct nvmap_heap *heap,
1074 const struct attribute_group *grp)
1075{
1076 return sysfs_create_group(&heap->dev.kobj, grp);
1077}
1078
1079/* nvmap_heap_remove_group: removes the attribute_group grp */
1080void nvmap_heap_remove_group(struct nvmap_heap *heap,
1081 const struct attribute_group *grp)
1082{
1083 sysfs_remove_group(&heap->dev.kobj, grp);
1084}
1085
1086int nvmap_heap_init(void)
1087{
1088 BUG_ON(buddy_heap_cache != NULL);
1089 buddy_heap_cache = KMEM_CACHE(buddy_heap, 0);
1090 if (!buddy_heap_cache) {
1091 pr_err("%s: unable to create buddy heap cache\n", __func__);
1092 return -ENOMEM;
1093 }
1094
1095 block_cache = KMEM_CACHE(combo_block, 0);
1096 if (!block_cache) {
1097 kmem_cache_destroy(buddy_heap_cache);
1098 pr_err("%s: unable to create block cache\n", __func__);
1099 return -ENOMEM;
1100 }
1101 return 0;
1102}
1103
1104void nvmap_heap_deinit(void)
1105{
1106 if (buddy_heap_cache)
1107 kmem_cache_destroy(buddy_heap_cache);
1108 if (block_cache)
1109 kmem_cache_destroy(block_cache);
1110
1111 block_cache = NULL;
1112 buddy_heap_cache = NULL;
1113}
diff --git a/drivers/video/tegra/nvmap/nvmap_heap.h b/drivers/video/tegra/nvmap/nvmap_heap.h
new file mode 100644
index 00000000000..158a1fa3d33
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_heap.h
@@ -0,0 +1,68 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_heap.h
3 *
4 * GPU heap allocator.
5 *
6 * Copyright (c) 2010-2011, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#ifndef __NVMAP_HEAP_H
24#define __NVMAP_HEAP_H
25
26struct device;
27struct nvmap_heap;
28struct attribute_group;
29
30struct nvmap_heap_block {
31 phys_addr_t base;
32 unsigned int type;
33 struct nvmap_handle *handle;
34};
35
36#define NVMAP_HEAP_MIN_BUDDY_SIZE 8192
37
38struct nvmap_heap *nvmap_heap_create(struct device *parent, const char *name,
39 phys_addr_t base, size_t len,
40 unsigned int buddy_size, void *arg);
41
42void nvmap_heap_destroy(struct nvmap_heap *heap);
43
44void *nvmap_heap_device_to_arg(struct device *dev);
45
46void *nvmap_heap_to_arg(struct nvmap_heap *heap);
47
48struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *heap,
49 struct nvmap_handle *handle);
50
51struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b);
52
53void nvmap_heap_free(struct nvmap_heap_block *block);
54
55int nvmap_heap_create_group(struct nvmap_heap *heap,
56 const struct attribute_group *grp);
57
58void nvmap_heap_remove_group(struct nvmap_heap *heap,
59 const struct attribute_group *grp);
60
61int __init nvmap_heap_init(void);
62
63void nvmap_heap_deinit(void);
64
65int nvmap_flush_heap_block(struct nvmap_client *client,
66 struct nvmap_heap_block *block, size_t len, unsigned int prot);
67
68#endif
diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c
new file mode 100644
index 00000000000..58bc71d5046
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c
@@ -0,0 +1,749 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_ioctl.c
3 *
4 * User-space interface to nvmap
5 *
6 * Copyright (c) 2011, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#include <linux/dma-mapping.h>
24#include <linux/fs.h>
25#include <linux/kernel.h>
26#include <linux/slab.h>
27#include <linux/uaccess.h>
28
29#include <asm/cacheflush.h>
30#include <asm/outercache.h>
31#include <asm/tlbflush.h>
32
33#include <mach/iovmm.h>
34#include <mach/nvmap.h>
35
36#include "nvmap_ioctl.h"
37#include "nvmap.h"
38#include "nvmap_common.h"
39
40static ssize_t rw_handle(struct nvmap_client *client, struct nvmap_handle *h,
41 int is_read, unsigned long h_offs,
42 unsigned long sys_addr, unsigned long h_stride,
43 unsigned long sys_stride, unsigned long elem_size,
44 unsigned long count);
45
46static int cache_maint(struct nvmap_client *client, struct nvmap_handle *h,
47 unsigned long start, unsigned long end, unsigned int op);
48
49
50int nvmap_ioctl_pinop(struct file *filp, bool is_pin, void __user *arg)
51{
52 struct nvmap_pin_handle op;
53 struct nvmap_handle *h;
54 unsigned long on_stack[16];
55 unsigned long *refs;
56 unsigned long __user *output;
57 unsigned int i;
58 int err = 0;
59
60 if (copy_from_user(&op, arg, sizeof(op)))
61 return -EFAULT;
62
63 if (!op.count)
64 return -EINVAL;
65
66 if (op.count > 1) {
67 size_t bytes = op.count * sizeof(unsigned long *);
68
69 if (op.count > ARRAY_SIZE(on_stack))
70 refs = kmalloc(op.count * sizeof(*refs), GFP_KERNEL);
71 else
72 refs = on_stack;
73
74 if (!refs)
75 return -ENOMEM;
76
77 if (copy_from_user(refs, (void *)op.handles, bytes)) {
78 err = -EFAULT;
79 goto out;
80 }
81 } else {
82 refs = on_stack;
83 on_stack[0] = (unsigned long)op.handles;
84 }
85
86 if (is_pin)
87 err = nvmap_pin_ids(filp->private_data, op.count, refs);
88 else
89 nvmap_unpin_ids(filp->private_data, op.count, refs);
90
91 /* skip the output stage on unpin */
92 if (err || !is_pin)
93 goto out;
94
95 /* it is guaranteed that if nvmap_pin_ids returns 0 that
96 * all of the handle_ref objects are valid, so dereferencing
97 * directly here is safe */
98 if (op.count > 1)
99 output = (unsigned long __user *)op.addr;
100 else {
101 struct nvmap_pin_handle __user *tmp = arg;
102 output = (unsigned long __user *)&(tmp->addr);
103 }
104
105 if (!output)
106 goto out;
107
108 for (i = 0; i < op.count && !err; i++) {
109 unsigned long addr;
110
111 h = (struct nvmap_handle *)refs[i];
112
113 if (h->heap_pgalloc && h->pgalloc.contig)
114 addr = page_to_phys(h->pgalloc.pages[0]);
115 else if (h->heap_pgalloc)
116 addr = h->pgalloc.area->iovm_start;
117 else
118 addr = h->carveout->base;
119
120 err = put_user(addr, &output[i]);
121 }
122
123 if (err)
124 nvmap_unpin_ids(filp->private_data, op.count, refs);
125
126out:
127 if (refs != on_stack)
128 kfree(refs);
129
130 return err;
131}
132
133int nvmap_ioctl_getid(struct file *filp, void __user *arg)
134{
135 struct nvmap_client *client = filp->private_data;
136 struct nvmap_create_handle op;
137 struct nvmap_handle *h = NULL;
138
139 if (copy_from_user(&op, arg, sizeof(op)))
140 return -EFAULT;
141
142 if (!op.handle)
143 return -EINVAL;
144
145 h = nvmap_get_handle_id(client, op.handle);
146
147 if (!h)
148 return -EPERM;
149
150 op.id = (__u32)h;
151 if (client == h->owner)
152 h->global = true;
153
154 nvmap_handle_put(h);
155
156 return copy_to_user(arg, &op, sizeof(op)) ? -EFAULT : 0;
157}
158
159int nvmap_ioctl_alloc(struct file *filp, void __user *arg)
160{
161 struct nvmap_alloc_handle op;
162 struct nvmap_client *client = filp->private_data;
163
164 if (copy_from_user(&op, arg, sizeof(op)))
165 return -EFAULT;
166
167 if (!op.handle)
168 return -EINVAL;
169
170 if (op.align & (op.align - 1))
171 return -EINVAL;
172
173 /* user-space handles are aligned to page boundaries, to prevent
174 * data leakage. */
175 op.align = max_t(size_t, op.align, PAGE_SIZE);
176
177 return nvmap_alloc_handle_id(client, op.handle, op.heap_mask,
178 op.align, op.flags);
179}
180
181int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg)
182{
183 struct nvmap_create_handle op;
184 struct nvmap_handle_ref *ref = NULL;
185 struct nvmap_client *client = filp->private_data;
186 int err = 0;
187
188 if (copy_from_user(&op, arg, sizeof(op)))
189 return -EFAULT;
190
191 if (!client)
192 return -ENODEV;
193
194 if (cmd == NVMAP_IOC_CREATE) {
195 ref = nvmap_create_handle(client, PAGE_ALIGN(op.size));
196 if (!IS_ERR(ref))
197 ref->handle->orig_size = op.size;
198 } else if (cmd == NVMAP_IOC_FROM_ID) {
199 ref = nvmap_duplicate_handle_id(client, op.id);
200 } else {
201 return -EINVAL;
202 }
203
204 if (IS_ERR(ref))
205 return PTR_ERR(ref);
206
207 op.handle = nvmap_ref_to_id(ref);
208 if (copy_to_user(arg, &op, sizeof(op))) {
209 err = -EFAULT;
210 nvmap_free_handle_id(client, op.handle);
211 }
212
213 return err;
214}
215
216int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg)
217{
218 struct nvmap_client *client = filp->private_data;
219 struct nvmap_map_caller op;
220 struct nvmap_vma_priv *vpriv;
221 struct vm_area_struct *vma;
222 struct nvmap_handle *h = NULL;
223 unsigned int cache_flags;
224 int err = 0;
225
226 if (copy_from_user(&op, arg, sizeof(op)))
227 return -EFAULT;
228
229 if (!op.handle)
230 return -EINVAL;
231
232 h = nvmap_get_handle_id(client, op.handle);
233
234 if (!h)
235 return -EPERM;
236
237 down_read(&current->mm->mmap_sem);
238
239 vma = find_vma(current->mm, op.addr);
240 if (!vma || !vma->vm_private_data) {
241 err = -ENOMEM;
242 goto out;
243 }
244
245 if (op.offset & ~PAGE_MASK) {
246 err = -EFAULT;
247 goto out;
248 }
249
250 if ((op.offset + op.length) > h->size) {
251 err = -EADDRNOTAVAIL;
252 goto out;
253 }
254
255 vpriv = vma->vm_private_data;
256 BUG_ON(!vpriv);
257
258 /* the VMA must exactly match the requested mapping operation, and the
259 * VMA that is targetted must have been created by this driver
260 */
261 if ((vma->vm_start != op.addr) || !is_nvmap_vma(vma) ||
262 (vma->vm_end-vma->vm_start != op.length)) {
263 err = -EPERM;
264 goto out;
265 }
266
267 /* verify that each mmap() system call creates a unique VMA */
268
269 if (vpriv->handle && (h == vpriv->handle)) {
270 goto out;
271 } else if (vpriv->handle) {
272 err = -EADDRNOTAVAIL;
273 goto out;
274 }
275
276 nvmap_usecount_inc(h);
277
278 if (!h->heap_pgalloc && (h->carveout->base & ~PAGE_MASK)) {
279 nvmap_usecount_dec(h);
280 err = -EFAULT;
281 goto out;
282 }
283
284 vpriv->handle = h;
285 vpriv->offs = op.offset;
286
287 cache_flags = op.flags & NVMAP_HANDLE_CACHE_FLAG;
288 if ((cache_flags == NVMAP_HANDLE_INNER_CACHEABLE ||
289 cache_flags == NVMAP_HANDLE_CACHEABLE) &&
290 (h->flags == NVMAP_HANDLE_UNCACHEABLE ||
291 h->flags == NVMAP_HANDLE_WRITE_COMBINE)) {
292 if (h->size & ~PAGE_MASK) {
293 pr_err("\n%s:attempt to convert a buffer from uc/wc to"
294 " wb, whose size is not a multiple of page size."
295 " request ignored.\n", __func__);
296 } else {
297 unsigned int nr_page = h->size >> PAGE_SHIFT;
298 wmb();
299 /* override allocation time cache coherency attributes. */
300 h->flags &= ~NVMAP_HANDLE_CACHE_FLAG;
301 h->flags |= cache_flags;
302
303 /* Update page attributes, if the memory is allocated
304 * from system heap pages.
305 */
306 if (cache_flags == NVMAP_HANDLE_INNER_CACHEABLE &&
307 h->heap_pgalloc)
308 set_pages_array_iwb(h->pgalloc.pages, nr_page);
309 else if (h->heap_pgalloc)
310 set_pages_array_wb(h->pgalloc.pages, nr_page);
311 }
312 }
313 vma->vm_page_prot = nvmap_pgprot(h, vma->vm_page_prot);
314
315out:
316 up_read(&current->mm->mmap_sem);
317
318 if (err)
319 nvmap_handle_put(h);
320 return err;
321}
322
323int nvmap_ioctl_get_param(struct file *filp, void __user* arg)
324{
325 struct nvmap_handle_param op;
326 struct nvmap_client *client = filp->private_data;
327 struct nvmap_handle *h;
328 int err = 0;
329
330 if (copy_from_user(&op, arg, sizeof(op)))
331 return -EFAULT;
332
333 h = nvmap_get_handle_id(client, op.handle);
334 if (!h)
335 return -EINVAL;
336
337 switch (op.param) {
338 case NVMAP_HANDLE_PARAM_SIZE:
339 op.result = h->orig_size;
340 break;
341 case NVMAP_HANDLE_PARAM_ALIGNMENT:
342 mutex_lock(&h->lock);
343 if (!h->alloc)
344 op.result = 0;
345 else if (h->heap_pgalloc)
346 op.result = PAGE_SIZE;
347 else if (h->carveout->base)
348 op.result = (h->carveout->base & -h->carveout->base);
349 else
350 op.result = SZ_4M;
351 mutex_unlock(&h->lock);
352 break;
353 case NVMAP_HANDLE_PARAM_BASE:
354 if (WARN_ON(!h->alloc || !atomic_add_return(0, &h->pin)))
355 op.result = -1ul;
356 else if (!h->heap_pgalloc) {
357 mutex_lock(&h->lock);
358 op.result = h->carveout->base;
359 mutex_unlock(&h->lock);
360 } else if (h->pgalloc.contig)
361 op.result = page_to_phys(h->pgalloc.pages[0]);
362 else if (h->pgalloc.area)
363 op.result = h->pgalloc.area->iovm_start;
364 else
365 op.result = -1ul;
366 break;
367 case NVMAP_HANDLE_PARAM_HEAP:
368 if (!h->alloc)
369 op.result = 0;
370 else if (!h->heap_pgalloc) {
371 mutex_lock(&h->lock);
372 op.result = nvmap_carveout_usage(client, h->carveout);
373 mutex_unlock(&h->lock);
374 } else if (h->pgalloc.contig)
375 op.result = NVMAP_HEAP_SYSMEM;
376 else
377 op.result = NVMAP_HEAP_IOVMM;
378 break;
379 default:
380 err = -EINVAL;
381 break;
382 }
383
384 if (!err && copy_to_user(arg, &op, sizeof(op)))
385 err = -EFAULT;
386
387 nvmap_handle_put(h);
388 return err;
389}
390
391int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user* arg)
392{
393 struct nvmap_client *client = filp->private_data;
394 struct nvmap_rw_handle __user *uarg = arg;
395 struct nvmap_rw_handle op;
396 struct nvmap_handle *h;
397 ssize_t copied;
398 int err = 0;
399
400 if (copy_from_user(&op, arg, sizeof(op)))
401 return -EFAULT;
402
403 if (!op.handle || !op.addr || !op.count || !op.elem_size)
404 return -EINVAL;
405
406 h = nvmap_get_handle_id(client, op.handle);
407 if (!h)
408 return -EPERM;
409
410 nvmap_usecount_inc(h);
411
412 copied = rw_handle(client, h, is_read, op.offset,
413 (unsigned long)op.addr, op.hmem_stride,
414 op.user_stride, op.elem_size, op.count);
415
416 if (copied < 0) {
417 err = copied;
418 copied = 0;
419 } else if (copied < (op.count * op.elem_size))
420 err = -EINTR;
421
422 __put_user(copied, &uarg->count);
423
424 nvmap_usecount_dec(h);
425
426 nvmap_handle_put(h);
427
428 return err;
429}
430
431int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg)
432{
433 struct nvmap_client *client = filp->private_data;
434 struct nvmap_cache_op op;
435 struct vm_area_struct *vma;
436 struct nvmap_vma_priv *vpriv;
437 unsigned long start;
438 unsigned long end;
439 int err = 0;
440
441 if (copy_from_user(&op, arg, sizeof(op)))
442 return -EFAULT;
443
444 if (!op.handle || !op.addr || op.op < NVMAP_CACHE_OP_WB ||
445 op.op > NVMAP_CACHE_OP_WB_INV)
446 return -EINVAL;
447
448 down_read(&current->mm->mmap_sem);
449
450 vma = find_vma(current->active_mm, (unsigned long)op.addr);
451 if (!vma || !is_nvmap_vma(vma) ||
452 (unsigned long)op.addr + op.len > vma->vm_end) {
453 err = -EADDRNOTAVAIL;
454 goto out;
455 }
456
457 vpriv = (struct nvmap_vma_priv *)vma->vm_private_data;
458
459 if ((unsigned long)vpriv->handle != op.handle) {
460 err = -EFAULT;
461 goto out;
462 }
463
464 start = (unsigned long)op.addr - vma->vm_start;
465 end = start + op.len;
466
467 err = cache_maint(client, vpriv->handle, start, end, op.op);
468out:
469 up_read(&current->mm->mmap_sem);
470 return err;
471}
472
473int nvmap_ioctl_free(struct file *filp, unsigned long arg)
474{
475 struct nvmap_client *client = filp->private_data;
476
477 if (!arg)
478 return 0;
479
480 nvmap_free_handle_id(client, arg);
481 return 0;
482}
483
484static void inner_cache_maint(unsigned int op, void *vaddr, size_t size)
485{
486 if (op == NVMAP_CACHE_OP_WB_INV)
487 dmac_flush_range(vaddr, vaddr + size);
488 else if (op == NVMAP_CACHE_OP_INV)
489 dmac_map_area(vaddr, size, DMA_FROM_DEVICE);
490 else
491 dmac_map_area(vaddr, size, DMA_TO_DEVICE);
492}
493
494static void outer_cache_maint(unsigned int op, unsigned long paddr, size_t size)
495{
496 if (op == NVMAP_CACHE_OP_WB_INV)
497 outer_flush_range(paddr, paddr + size);
498 else if (op == NVMAP_CACHE_OP_INV)
499 outer_inv_range(paddr, paddr + size);
500 else
501 outer_clean_range(paddr, paddr + size);
502}
503
504static void heap_page_cache_maint(struct nvmap_client *client,
505 struct nvmap_handle *h, unsigned long start, unsigned long end,
506 unsigned int op, bool inner, bool outer, pte_t **pte,
507 unsigned long kaddr, pgprot_t prot)
508{
509 struct page *page;
510 unsigned long paddr;
511 unsigned long next;
512 unsigned long off;
513 size_t size;
514
515 while (start < end) {
516 page = h->pgalloc.pages[start >> PAGE_SHIFT];
517 next = min(((start + PAGE_SIZE) & PAGE_MASK), end);
518 off = start & ~PAGE_MASK;
519 size = next - start;
520 paddr = page_to_phys(page) + off;
521
522 if (inner) {
523 void *vaddr = (void *)kaddr + off;
524 BUG_ON(!pte);
525 BUG_ON(!kaddr);
526 set_pte_at(&init_mm, kaddr, *pte,
527 pfn_pte(__phys_to_pfn(paddr), prot));
528 flush_tlb_kernel_page(kaddr);
529 inner_cache_maint(op, vaddr, size);
530 }
531
532 if (outer)
533 outer_cache_maint(op, paddr, size);
534 start = next;
535 }
536}
537
538static bool fast_cache_maint(struct nvmap_client *client, struct nvmap_handle *h,
539 unsigned long start, unsigned long end, unsigned int op)
540{
541 int ret = false;
542
543 if ((op == NVMAP_CACHE_OP_INV) ||
544 ((end - start) < FLUSH_CLEAN_BY_SET_WAY_THRESHOLD))
545 goto out;
546
547 if (op == NVMAP_CACHE_OP_WB_INV)
548 inner_flush_cache_all();
549 else if (op == NVMAP_CACHE_OP_WB)
550 inner_clean_cache_all();
551
552 if (h->heap_pgalloc && (h->flags != NVMAP_HANDLE_INNER_CACHEABLE)) {
553 heap_page_cache_maint(client, h, start, end, op,
554 false, true, NULL, 0, 0);
555 } else if (h->flags != NVMAP_HANDLE_INNER_CACHEABLE) {
556 start += h->carveout->base;
557 end += h->carveout->base;
558 outer_cache_maint(op, start, end - start);
559 }
560 ret = true;
561out:
562 return ret;
563}
564
565static int cache_maint(struct nvmap_client *client, struct nvmap_handle *h,
566 unsigned long start, unsigned long end, unsigned int op)
567{
568 pgprot_t prot;
569 pte_t **pte = NULL;
570 unsigned long kaddr;
571 unsigned long loop;
572 int err = 0;
573
574 h = nvmap_handle_get(h);
575 if (!h)
576 return -EFAULT;
577
578 if (!h->alloc) {
579 err = -EFAULT;
580 goto out;
581 }
582
583 wmb();
584 if (h->flags == NVMAP_HANDLE_UNCACHEABLE ||
585 h->flags == NVMAP_HANDLE_WRITE_COMBINE || start == end)
586 goto out;
587
588 if (fast_cache_maint(client, h, start, end, op))
589 goto out;
590
591 prot = nvmap_pgprot(h, pgprot_kernel);
592 pte = nvmap_alloc_pte(client->dev, (void **)&kaddr);
593 if (IS_ERR(pte)) {
594 err = PTR_ERR(pte);
595 pte = NULL;
596 goto out;
597 }
598
599 if (h->heap_pgalloc) {
600 heap_page_cache_maint(client, h, start, end, op, true,
601 (h->flags == NVMAP_HANDLE_INNER_CACHEABLE) ? false : true,
602 pte, kaddr, prot);
603 goto out;
604 }
605
606 if (start > h->size || end > h->size) {
607 nvmap_warn(client, "cache maintenance outside handle\n");
608 return -EINVAL;
609 }
610
611 /* lock carveout from relocation by mapcount */
612 nvmap_usecount_inc(h);
613
614 start += h->carveout->base;
615 end += h->carveout->base;
616
617 loop = start;
618
619 while (loop < end) {
620 unsigned long next = (loop + PAGE_SIZE) & PAGE_MASK;
621 void *base = (void *)kaddr + (loop & ~PAGE_MASK);
622 next = min(next, end);
623
624 set_pte_at(&init_mm, kaddr, *pte,
625 pfn_pte(__phys_to_pfn(loop), prot));
626 flush_tlb_kernel_page(kaddr);
627
628 inner_cache_maint(op, base, next - loop);
629 loop = next;
630 }
631
632 if (h->flags != NVMAP_HANDLE_INNER_CACHEABLE)
633 outer_cache_maint(op, start, end - start);
634
635 /* unlock carveout */
636 nvmap_usecount_dec(h);
637
638out:
639 if (pte)
640 nvmap_free_pte(client->dev, pte);
641 nvmap_handle_put(h);
642 return err;
643}
644
645static int rw_handle_page(struct nvmap_handle *h, int is_read,
646 phys_addr_t start, unsigned long rw_addr,
647 unsigned long bytes, unsigned long kaddr, pte_t *pte)
648{
649 pgprot_t prot = nvmap_pgprot(h, pgprot_kernel);
650 unsigned long end = start + bytes;
651 int err = 0;
652
653 while (!err && start < end) {
654 struct page *page = NULL;
655 phys_addr_t phys;
656 size_t count;
657 void *src;
658
659 if (!h->heap_pgalloc) {
660 phys = h->carveout->base + start;
661 } else {
662 page = h->pgalloc.pages[start >> PAGE_SHIFT];
663 BUG_ON(!page);
664 get_page(page);
665 phys = page_to_phys(page) + (start & ~PAGE_MASK);
666 }
667
668 set_pte_at(&init_mm, kaddr, pte,
669 pfn_pte(__phys_to_pfn(phys), prot));
670 flush_tlb_kernel_page(kaddr);
671
672 src = (void *)kaddr + (phys & ~PAGE_MASK);
673 phys = PAGE_SIZE - (phys & ~PAGE_MASK);
674 count = min_t(size_t, end - start, phys);
675
676 if (is_read)
677 err = copy_to_user((void *)rw_addr, src, count);
678 else
679 err = copy_from_user(src, (void *)rw_addr, count);
680
681 if (err)
682 err = -EFAULT;
683
684 rw_addr += count;
685 start += count;
686
687 if (page)
688 put_page(page);
689 }
690
691 return err;
692}
693
694static ssize_t rw_handle(struct nvmap_client *client, struct nvmap_handle *h,
695 int is_read, unsigned long h_offs,
696 unsigned long sys_addr, unsigned long h_stride,
697 unsigned long sys_stride, unsigned long elem_size,
698 unsigned long count)
699{
700 ssize_t copied = 0;
701 pte_t **pte;
702 void *addr;
703 int ret = 0;
704
705 if (!elem_size)
706 return -EINVAL;
707
708 if (!h->alloc)
709 return -EFAULT;
710
711 if (elem_size == h_stride && elem_size == sys_stride) {
712 elem_size *= count;
713 h_stride = elem_size;
714 sys_stride = elem_size;
715 count = 1;
716 }
717
718 pte = nvmap_alloc_pte(client->dev, &addr);
719 if (IS_ERR(pte))
720 return PTR_ERR(pte);
721
722 while (count--) {
723 if (h_offs + elem_size > h->size) {
724 nvmap_warn(client, "read/write outside of handle\n");
725 ret = -EFAULT;
726 break;
727 }
728 if (is_read)
729 cache_maint(client, h, h_offs,
730 h_offs + elem_size, NVMAP_CACHE_OP_INV);
731
732 ret = rw_handle_page(h, is_read, h_offs, sys_addr,
733 elem_size, (unsigned long)addr, *pte);
734
735 if (ret)
736 break;
737
738 if (!is_read)
739 cache_maint(client, h, h_offs,
740 h_offs + elem_size, NVMAP_CACHE_OP_WB);
741
742 copied += elem_size;
743 sys_addr += sys_stride;
744 h_offs += h_stride;
745 }
746
747 nvmap_free_pte(client->dev, pte);
748 return ret ?: copied;
749}
diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.h b/drivers/video/tegra/nvmap/nvmap_ioctl.h
new file mode 100644
index 00000000000..c802cd4dd7a
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_ioctl.h
@@ -0,0 +1,159 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_ioctl.h
3 *
4 * ioctl declarations for nvmap
5 *
6 * Copyright (c) 2010, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#ifndef __VIDEO_TEGRA_NVMAP_IOCTL_H
24#define __VIDEO_TEGRA_NVMAP_IOCTL_H
25
26#include <linux/ioctl.h>
27#include <linux/file.h>
28
29#include <mach/nvmap.h>
30
31enum {
32 NVMAP_HANDLE_PARAM_SIZE = 1,
33 NVMAP_HANDLE_PARAM_ALIGNMENT,
34 NVMAP_HANDLE_PARAM_BASE,
35 NVMAP_HANDLE_PARAM_HEAP,
36};
37
38enum {
39 NVMAP_CACHE_OP_WB = 0,
40 NVMAP_CACHE_OP_INV,
41 NVMAP_CACHE_OP_WB_INV,
42};
43
44
45struct nvmap_create_handle {
46 union {
47 __u32 key; /* ClaimPreservedHandle */
48 __u32 id; /* FromId */
49 __u32 size; /* CreateHandle */
50 };
51 __u32 handle;
52};
53
54struct nvmap_alloc_handle {
55 __u32 handle;
56 __u32 heap_mask;
57 __u32 flags;
58 __u32 align;
59};
60
61struct nvmap_map_caller {
62 __u32 handle; /* hmem */
63 __u32 offset; /* offset into hmem; should be page-aligned */
64 __u32 length; /* number of bytes to map */
65 __u32 flags;
66 unsigned long addr; /* user pointer */
67};
68
69struct nvmap_rw_handle {
70 unsigned long addr; /* user pointer */
71 __u32 handle; /* hmem */
72 __u32 offset; /* offset into hmem */
73 __u32 elem_size; /* individual atom size */
74 __u32 hmem_stride; /* delta in bytes between atoms in hmem */
75 __u32 user_stride; /* delta in bytes between atoms in user */
76 __u32 count; /* number of atoms to copy */
77};
78
79struct nvmap_pin_handle {
80 unsigned long handles; /* array of handles to pin/unpin */
81 unsigned long addr; /* array of addresses to return */
82 __u32 count; /* number of entries in handles */
83};
84
85struct nvmap_handle_param {
86 __u32 handle;
87 __u32 param;
88 unsigned long result;
89};
90
91struct nvmap_cache_op {
92 unsigned long addr;
93 __u32 handle;
94 __u32 len;
95 __s32 op;
96};
97
98#define NVMAP_IOC_MAGIC 'N'
99
100/* Creates a new memory handle. On input, the argument is the size of the new
101 * handle; on return, the argument is the name of the new handle
102 */
103#define NVMAP_IOC_CREATE _IOWR(NVMAP_IOC_MAGIC, 0, struct nvmap_create_handle)
104#define NVMAP_IOC_CLAIM _IOWR(NVMAP_IOC_MAGIC, 1, struct nvmap_create_handle)
105#define NVMAP_IOC_FROM_ID _IOWR(NVMAP_IOC_MAGIC, 2, struct nvmap_create_handle)
106
107/* Actually allocates memory for the specified handle */
108#define NVMAP_IOC_ALLOC _IOW(NVMAP_IOC_MAGIC, 3, struct nvmap_alloc_handle)
109
110/* Frees a memory handle, unpinning any pinned pages and unmapping any mappings
111 */
112#define NVMAP_IOC_FREE _IO(NVMAP_IOC_MAGIC, 4)
113
114/* Maps the region of the specified handle into a user-provided virtual address
115 * that was previously created via an mmap syscall on this fd */
116#define NVMAP_IOC_MMAP _IOWR(NVMAP_IOC_MAGIC, 5, struct nvmap_map_caller)
117
118/* Reads/writes data (possibly strided) from a user-provided buffer into the
119 * hmem at the specified offset */
120#define NVMAP_IOC_WRITE _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle)
121#define NVMAP_IOC_READ _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle)
122
123#define NVMAP_IOC_PARAM _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param)
124
125/* Pins a list of memory handles into IO-addressable memory (either IOVMM
126 * space or physical memory, depending on the allocation), and returns the
127 * address. Handles may be pinned recursively. */
128#define NVMAP_IOC_PIN_MULT _IOWR(NVMAP_IOC_MAGIC, 10, struct nvmap_pin_handle)
129#define NVMAP_IOC_UNPIN_MULT _IOW(NVMAP_IOC_MAGIC, 11, struct nvmap_pin_handle)
130
131#define NVMAP_IOC_CACHE _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op)
132
133/* Returns a global ID usable to allow a remote process to create a handle
134 * reference to the same handle */
135#define NVMAP_IOC_GET_ID _IOWR(NVMAP_IOC_MAGIC, 13, struct nvmap_create_handle)
136
137#define NVMAP_IOC_MAXNR (_IOC_NR(NVMAP_IOC_GET_ID))
138
139int nvmap_ioctl_pinop(struct file *filp, bool is_pin, void __user *arg);
140
141int nvmap_ioctl_get_param(struct file *filp, void __user* arg);
142
143int nvmap_ioctl_getid(struct file *filp, void __user *arg);
144
145int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
146
147int nvmap_ioctl_free(struct file *filp, unsigned long arg);
148
149int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg);
150
151int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg);
152
153int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg);
154
155int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user* arg);
156
157
158
159#endif
diff --git a/drivers/video/tegra/nvmap/nvmap_mru.c b/drivers/video/tegra/nvmap/nvmap_mru.c
new file mode 100644
index 00000000000..f54d44923eb
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_mru.c
@@ -0,0 +1,187 @@
1/*
2 * drivers/video/tegra/nvmap/nvmap_mru.c
3 *
4 * IOVMM virtualization support for nvmap
5 *
6 * Copyright (c) 2009-2011, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23#include <linux/list.h>
24#include <linux/slab.h>
25
26#include <asm/pgtable.h>
27
28#include <mach/iovmm.h>
29
30#include "nvmap.h"
31#include "nvmap_mru.h"
32
33/* if IOVMM reclamation is enabled (CONFIG_NVMAP_RECLAIM_UNPINNED_VM),
34 * unpinned handles are placed onto a most-recently-used eviction list;
35 * multiple lists are maintained, segmented by size (sizes were chosen to
36 * roughly correspond with common sizes for graphics surfaces).
37 *
38 * if a handle is located on the MRU list, then the code below may
39 * steal its IOVMM area at any time to satisfy a pin operation if no
40 * free IOVMM space is available
41 */
42
43static const size_t mru_cutoff[] = {
44 262144, 393216, 786432, 1048576, 1572864
45};
46
47static inline struct list_head *mru_list(struct nvmap_share *share, size_t size)
48{
49 unsigned int i;
50
51 BUG_ON(!share->mru_lists);
52 for (i = 0; i < ARRAY_SIZE(mru_cutoff); i++)
53 if (size <= mru_cutoff[i])
54 break;
55
56 return &share->mru_lists[i];
57}
58
59size_t nvmap_mru_vm_size(struct tegra_iovmm_client *iovmm)
60{
61 size_t vm_size = tegra_iovmm_get_vm_size(iovmm);
62 return (vm_size >> 2) * 3;
63}
64
65/* nvmap_mru_vma_lock should be acquired by the caller before calling this */
66void nvmap_mru_insert_locked(struct nvmap_share *share, struct nvmap_handle *h)
67{
68 size_t len = h->pgalloc.area->iovm_length;
69 list_add(&h->pgalloc.mru_list, mru_list(share, len));
70}
71
72void nvmap_mru_remove(struct nvmap_share *s, struct nvmap_handle *h)
73{
74 nvmap_mru_lock(s);
75 if (!list_empty(&h->pgalloc.mru_list))
76 list_del(&h->pgalloc.mru_list);
77 nvmap_mru_unlock(s);
78 INIT_LIST_HEAD(&h->pgalloc.mru_list);
79}
80
81/* returns a tegra_iovmm_area for a handle. if the handle already has
82 * an iovmm_area allocated, the handle is simply removed from its MRU list
83 * and the existing iovmm_area is returned.
84 *
85 * if no existing allocation exists, try to allocate a new IOVMM area.
86 *
87 * if a new area can not be allocated, try to re-use the most-recently-unpinned
88 * handle's allocation.
89 *
90 * and if that fails, iteratively evict handles from the MRU lists and free
91 * their allocations, until the new allocation succeeds.
92 */
93struct tegra_iovmm_area *nvmap_handle_iovmm_locked(struct nvmap_client *c,
94 struct nvmap_handle *h)
95{
96 struct list_head *mru;
97 struct nvmap_handle *evict = NULL;
98 struct tegra_iovmm_area *vm = NULL;
99 unsigned int i, idx;
100 pgprot_t prot;
101
102 BUG_ON(!h || !c || !c->share);
103
104 prot = nvmap_pgprot(h, pgprot_kernel);
105
106 if (h->pgalloc.area) {
107 BUG_ON(list_empty(&h->pgalloc.mru_list));
108 list_del(&h->pgalloc.mru_list);
109 INIT_LIST_HEAD(&h->pgalloc.mru_list);
110 return h->pgalloc.area;
111 }
112
113 vm = tegra_iovmm_create_vm(c->share->iovmm, NULL,
114 h->size, h->align, prot,
115 h->pgalloc.iovm_addr);
116
117 if (vm) {
118 INIT_LIST_HEAD(&h->pgalloc.mru_list);
119 return vm;
120 }
121 /* if client is looking for specific iovm address, return from here. */
122 if ((vm == NULL) && (h->pgalloc.iovm_addr != 0))
123 return NULL;
124 /* attempt to re-use the most recently unpinned IOVMM area in the
125 * same size bin as the current handle. If that fails, iteratively
126 * evict handles (starting from the current bin) until an allocation
127 * succeeds or no more areas can be evicted */
128 mru = mru_list(c->share, h->size);
129 if (!list_empty(mru))
130 evict = list_first_entry(mru, struct nvmap_handle,
131 pgalloc.mru_list);
132
133 if (evict && evict->pgalloc.area->iovm_length >= h->size) {
134 list_del(&evict->pgalloc.mru_list);
135 vm = evict->pgalloc.area;
136 evict->pgalloc.area = NULL;
137 INIT_LIST_HEAD(&evict->pgalloc.mru_list);
138 return vm;
139 }
140
141 idx = mru - c->share->mru_lists;
142
143 for (i = 0; i < c->share->nr_mru && !vm; i++, idx++) {
144 if (idx >= c->share->nr_mru)
145 idx = 0;
146 mru = &c->share->mru_lists[idx];
147 while (!list_empty(mru) && !vm) {
148 evict = list_first_entry(mru, struct nvmap_handle,
149 pgalloc.mru_list);
150
151 BUG_ON(atomic_read(&evict->pin) != 0);
152 BUG_ON(!evict->pgalloc.area);
153 list_del(&evict->pgalloc.mru_list);
154 INIT_LIST_HEAD(&evict->pgalloc.mru_list);
155 tegra_iovmm_free_vm(evict->pgalloc.area);
156 evict->pgalloc.area = NULL;
157 vm = tegra_iovmm_create_vm(c->share->iovmm,
158 NULL, h->size, h->align,
159 prot, h->pgalloc.iovm_addr);
160 }
161 }
162 return vm;
163}
164
165int nvmap_mru_init(struct nvmap_share *share)
166{
167 int i;
168 mutex_init(&share->mru_lock);
169 share->nr_mru = ARRAY_SIZE(mru_cutoff) + 1;
170
171 share->mru_lists = kzalloc(sizeof(struct list_head) * share->nr_mru,
172 GFP_KERNEL);
173
174 if (!share->mru_lists)
175 return -ENOMEM;
176
177 for (i = 0; i < share->nr_mru; i++)
178 INIT_LIST_HEAD(&share->mru_lists[i]);
179
180 return 0;
181}
182
183void nvmap_mru_destroy(struct nvmap_share *share)
184{
185 kfree(share->mru_lists);
186 share->mru_lists = NULL;
187}
diff --git a/drivers/video/tegra/nvmap/nvmap_mru.h b/drivers/video/tegra/nvmap/nvmap_mru.h
new file mode 100644
index 00000000000..6c94630bc3e
--- /dev/null
+++ b/drivers/video/tegra/nvmap/nvmap_mru.h
@@ -0,0 +1,84 @@
1/*
2 * drivers/video/tegra/nvmap_mru.c
3 *
4 * IOVMM virtualization support for nvmap
5 *
6 * Copyright (c) 2009-2010, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23
24#ifndef __VIDEO_TEGRA_NVMAP_MRU_H
25#define __VIDEO_TEGRA_NVMAP_MRU_H
26
27#include <linux/spinlock.h>
28
29#include "nvmap.h"
30
31struct tegra_iovmm_area;
32struct tegra_iovmm_client;
33
34#ifdef CONFIG_NVMAP_RECLAIM_UNPINNED_VM
35
36static inline void nvmap_mru_lock(struct nvmap_share *share)
37{
38 mutex_lock(&share->mru_lock);
39}
40
41static inline void nvmap_mru_unlock(struct nvmap_share *share)
42{
43 mutex_unlock(&share->mru_lock);
44}
45
46int nvmap_mru_init(struct nvmap_share *share);
47
48void nvmap_mru_destroy(struct nvmap_share *share);
49
50size_t nvmap_mru_vm_size(struct tegra_iovmm_client *iovmm);
51
52void nvmap_mru_insert_locked(struct nvmap_share *share, struct nvmap_handle *h);
53
54void nvmap_mru_remove(struct nvmap_share *s, struct nvmap_handle *h);
55
56struct tegra_iovmm_area *nvmap_handle_iovmm_locked(struct nvmap_client *c,
57 struct nvmap_handle *h);
58
59#else
60
61#define nvmap_mru_lock(_s) do { } while (0)
62#define nvmap_mru_unlock(_s) do { } while (0)
63#define nvmap_mru_init(_s) 0
64#define nvmap_mru_destroy(_s) do { } while (0)
65#define nvmap_mru_vm_size(_a) tegra_iovmm_get_vm_size(_a)
66
67static inline void nvmap_mru_insert_locked(struct nvmap_share *share,
68 struct nvmap_handle *h)
69{ }
70
71static inline void nvmap_mru_remove(struct nvmap_share *s,
72 struct nvmap_handle *h)
73{ }
74
75static inline struct tegra_iovmm_area *nvmap_handle_iovmm_locked(struct nvmap_client *c,
76 struct nvmap_handle *h)
77{
78 BUG_ON(!h->pgalloc.area);
79 return h->pgalloc.area;
80}
81
82#endif
83
84#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-14 22:56:14 -0400