aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/ttm
diff options
context:
space:
mode:
authorThomas Hellstrom <thellstrom@vmware.com>2009-06-10 09:20:19 -0400
committerDave Airlie <airlied@redhat.com>2009-06-14 19:37:57 -0400
commitba4e7d973dd09b66912ac4c0856add8b0703a997 (patch)
tree32a87edb83a427ffd22645c5f77e6cec8be4e719 /drivers/gpu/drm/ttm
parente6c03c5b40314d787f7053f631594d6b1bd609e8 (diff)
drm: Add the TTM GPU memory manager subsystem.
TTM is a GPU memory manager subsystem designed for use with GPU devices with various memory types (On-card VRAM, AGP, PCI apertures etc.). It's essentially a helper library that assists the DRM driver in creating and managing persistent buffer objects. TTM manages placement of data and CPU map setup and teardown on data movement. It can also optionally manage synchronization of data on a per-buffer-object level. TTM takes care to provide an always valid virtual user-space address to a buffer object which makes user-space sub-allocation of big buffer objects feasible. TTM uses a fine-grained per buffer-object locking scheme, taking care to release all relevant locks when waiting for the GPU. Although this implies some locking overhead, it's probably a big win for devices with multiple command submission mechanisms, since the lock contention will be minimal. TTM can be used with whatever user-space interface the driver chooses, including GEM. It's used by the upcoming Radeon KMS DRM driver and is also the GPU memory management core of various new experimental DRM drivers. Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com> Signed-off-by: Jerome Glisse <jglisse@redhat.com> Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r--drivers/gpu/drm/ttm/Makefile8
-rw-r--r--drivers/gpu/drm/ttm/ttm_agp_backend.c150
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c1698
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_util.c561
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_vm.c454
-rw-r--r--drivers/gpu/drm/ttm/ttm_global.c114
-rw-r--r--drivers/gpu/drm/ttm/ttm_memory.c234
-rw-r--r--drivers/gpu/drm/ttm/ttm_module.c50
-rw-r--r--drivers/gpu/drm/ttm/ttm_tt.c635
9 files changed, 3904 insertions, 0 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile
new file mode 100644
index 000000000000..b0a9de7a57c2
--- /dev/null
+++ b/drivers/gpu/drm/ttm/Makefile
@@ -0,0 +1,8 @@
1#
2# Makefile for the drm device driver. This driver provides support for the
3
4ccflags-y := -Iinclude/drm
5ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \
6 ttm_bo_util.o ttm_bo_vm.o ttm_module.o ttm_global.o
7
8obj-$(CONFIG_DRM_TTM) += ttm.o
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
new file mode 100644
index 000000000000..e8f6d2229d8c
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -0,0 +1,150 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 * Keith Packard.
30 */
31
32#include "ttm/ttm_module.h"
33#include "ttm/ttm_bo_driver.h"
34#ifdef TTM_HAS_AGP
35#include "ttm/ttm_placement.h"
36#include <linux/agp_backend.h>
37#include <linux/module.h>
38#include <linux/io.h>
39#include <asm/agp.h>
40
41struct ttm_agp_backend {
42 struct ttm_backend backend;
43 struct agp_memory *mem;
44 struct agp_bridge_data *bridge;
45};
46
47static int ttm_agp_populate(struct ttm_backend *backend,
48 unsigned long num_pages, struct page **pages,
49 struct page *dummy_read_page)
50{
51 struct ttm_agp_backend *agp_be =
52 container_of(backend, struct ttm_agp_backend, backend);
53 struct page **cur_page, **last_page = pages + num_pages;
54 struct agp_memory *mem;
55
56 mem = agp_allocate_memory(agp_be->bridge, num_pages, AGP_USER_MEMORY);
57 if (unlikely(mem == NULL))
58 return -ENOMEM;
59
60 mem->page_count = 0;
61 for (cur_page = pages; cur_page < last_page; ++cur_page) {
62 struct page *page = *cur_page;
63 if (!page)
64 page = dummy_read_page;
65
66 mem->memory[mem->page_count++] =
67 phys_to_gart(page_to_phys(page));
68 }
69 agp_be->mem = mem;
70 return 0;
71}
72
73static int ttm_agp_bind(struct ttm_backend *backend, struct ttm_mem_reg *bo_mem)
74{
75 struct ttm_agp_backend *agp_be =
76 container_of(backend, struct ttm_agp_backend, backend);
77 struct agp_memory *mem = agp_be->mem;
78 int cached = (bo_mem->placement & TTM_PL_FLAG_CACHED);
79 int ret;
80
81 mem->is_flushed = 1;
82 mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY;
83
84 ret = agp_bind_memory(mem, bo_mem->mm_node->start);
85 if (ret)
86 printk(KERN_ERR TTM_PFX "AGP Bind memory failed.\n");
87
88 return ret;
89}
90
91static int ttm_agp_unbind(struct ttm_backend *backend)
92{
93 struct ttm_agp_backend *agp_be =
94 container_of(backend, struct ttm_agp_backend, backend);
95
96 if (agp_be->mem->is_bound)
97 return agp_unbind_memory(agp_be->mem);
98 else
99 return 0;
100}
101
102static void ttm_agp_clear(struct ttm_backend *backend)
103{
104 struct ttm_agp_backend *agp_be =
105 container_of(backend, struct ttm_agp_backend, backend);
106 struct agp_memory *mem = agp_be->mem;
107
108 if (mem) {
109 ttm_agp_unbind(backend);
110 agp_free_memory(mem);
111 }
112 agp_be->mem = NULL;
113}
114
115static void ttm_agp_destroy(struct ttm_backend *backend)
116{
117 struct ttm_agp_backend *agp_be =
118 container_of(backend, struct ttm_agp_backend, backend);
119
120 if (agp_be->mem)
121 ttm_agp_clear(backend);
122 kfree(agp_be);
123}
124
125static struct ttm_backend_func ttm_agp_func = {
126 .populate = ttm_agp_populate,
127 .clear = ttm_agp_clear,
128 .bind = ttm_agp_bind,
129 .unbind = ttm_agp_unbind,
130 .destroy = ttm_agp_destroy,
131};
132
133struct ttm_backend *ttm_agp_backend_init(struct ttm_bo_device *bdev,
134 struct agp_bridge_data *bridge)
135{
136 struct ttm_agp_backend *agp_be;
137
138 agp_be = kmalloc(sizeof(*agp_be), GFP_KERNEL);
139 if (!agp_be)
140 return NULL;
141
142 agp_be->mem = NULL;
143 agp_be->bridge = bridge;
144 agp_be->backend.func = &ttm_agp_func;
145 agp_be->backend.bdev = bdev;
146 return &agp_be->backend;
147}
148EXPORT_SYMBOL(ttm_agp_backend_init);
149
150#endif
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
new file mode 100644
index 000000000000..1587aeca7bea
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -0,0 +1,1698 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31#include "ttm/ttm_module.h"
32#include "ttm/ttm_bo_driver.h"
33#include "ttm/ttm_placement.h"
34#include <linux/jiffies.h>
35#include <linux/slab.h>
36#include <linux/sched.h>
37#include <linux/mm.h>
38#include <linux/file.h>
39#include <linux/module.h>
40
41#define TTM_ASSERT_LOCKED(param)
42#define TTM_DEBUG(fmt, arg...)
43#define TTM_BO_HASH_ORDER 13
44
45static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
46static void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo);
47static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
48
49static inline uint32_t ttm_bo_type_flags(unsigned type)
50{
51 return 1 << (type);
52}
53
54static void ttm_bo_release_list(struct kref *list_kref)
55{
56 struct ttm_buffer_object *bo =
57 container_of(list_kref, struct ttm_buffer_object, list_kref);
58 struct ttm_bo_device *bdev = bo->bdev;
59
60 BUG_ON(atomic_read(&bo->list_kref.refcount));
61 BUG_ON(atomic_read(&bo->kref.refcount));
62 BUG_ON(atomic_read(&bo->cpu_writers));
63 BUG_ON(bo->sync_obj != NULL);
64 BUG_ON(bo->mem.mm_node != NULL);
65 BUG_ON(!list_empty(&bo->lru));
66 BUG_ON(!list_empty(&bo->ddestroy));
67
68 if (bo->ttm)
69 ttm_tt_destroy(bo->ttm);
70 if (bo->destroy)
71 bo->destroy(bo);
72 else {
73 ttm_mem_global_free(bdev->mem_glob, bo->acc_size, false);
74 kfree(bo);
75 }
76}
77
78int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
79{
80
81 if (interruptible) {
82 int ret = 0;
83
84 ret = wait_event_interruptible(bo->event_queue,
85 atomic_read(&bo->reserved) == 0);
86 if (unlikely(ret != 0))
87 return -ERESTART;
88 } else {
89 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
90 }
91 return 0;
92}
93
94static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
95{
96 struct ttm_bo_device *bdev = bo->bdev;
97 struct ttm_mem_type_manager *man;
98
99 BUG_ON(!atomic_read(&bo->reserved));
100
101 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
102
103 BUG_ON(!list_empty(&bo->lru));
104
105 man = &bdev->man[bo->mem.mem_type];
106 list_add_tail(&bo->lru, &man->lru);
107 kref_get(&bo->list_kref);
108
109 if (bo->ttm != NULL) {
110 list_add_tail(&bo->swap, &bdev->swap_lru);
111 kref_get(&bo->list_kref);
112 }
113 }
114}
115
116/**
117 * Call with the lru_lock held.
118 */
119
120static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
121{
122 int put_count = 0;
123
124 if (!list_empty(&bo->swap)) {
125 list_del_init(&bo->swap);
126 ++put_count;
127 }
128 if (!list_empty(&bo->lru)) {
129 list_del_init(&bo->lru);
130 ++put_count;
131 }
132
133 /*
134 * TODO: Add a driver hook to delete from
135 * driver-specific LRU's here.
136 */
137
138 return put_count;
139}
140
141int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
142 bool interruptible,
143 bool no_wait, bool use_sequence, uint32_t sequence)
144{
145 struct ttm_bo_device *bdev = bo->bdev;
146 int ret;
147
148 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
149 if (use_sequence && bo->seq_valid &&
150 (sequence - bo->val_seq < (1 << 31))) {
151 return -EAGAIN;
152 }
153
154 if (no_wait)
155 return -EBUSY;
156
157 spin_unlock(&bdev->lru_lock);
158 ret = ttm_bo_wait_unreserved(bo, interruptible);
159 spin_lock(&bdev->lru_lock);
160
161 if (unlikely(ret))
162 return ret;
163 }
164
165 if (use_sequence) {
166 bo->val_seq = sequence;
167 bo->seq_valid = true;
168 } else {
169 bo->seq_valid = false;
170 }
171
172 return 0;
173}
174EXPORT_SYMBOL(ttm_bo_reserve);
175
176static void ttm_bo_ref_bug(struct kref *list_kref)
177{
178 BUG();
179}
180
181int ttm_bo_reserve(struct ttm_buffer_object *bo,
182 bool interruptible,
183 bool no_wait, bool use_sequence, uint32_t sequence)
184{
185 struct ttm_bo_device *bdev = bo->bdev;
186 int put_count = 0;
187 int ret;
188
189 spin_lock(&bdev->lru_lock);
190 ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
191 sequence);
192 if (likely(ret == 0))
193 put_count = ttm_bo_del_from_lru(bo);
194 spin_unlock(&bdev->lru_lock);
195
196 while (put_count--)
197 kref_put(&bo->list_kref, ttm_bo_ref_bug);
198
199 return ret;
200}
201
202void ttm_bo_unreserve(struct ttm_buffer_object *bo)
203{
204 struct ttm_bo_device *bdev = bo->bdev;
205
206 spin_lock(&bdev->lru_lock);
207 ttm_bo_add_to_lru(bo);
208 atomic_set(&bo->reserved, 0);
209 wake_up_all(&bo->event_queue);
210 spin_unlock(&bdev->lru_lock);
211}
212EXPORT_SYMBOL(ttm_bo_unreserve);
213
214/*
215 * Call bo->mutex locked.
216 */
217
218static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
219{
220 struct ttm_bo_device *bdev = bo->bdev;
221 int ret = 0;
222 uint32_t page_flags = 0;
223
224 TTM_ASSERT_LOCKED(&bo->mutex);
225 bo->ttm = NULL;
226
227 switch (bo->type) {
228 case ttm_bo_type_device:
229 if (zero_alloc)
230 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
231 case ttm_bo_type_kernel:
232 bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
233 page_flags, bdev->dummy_read_page);
234 if (unlikely(bo->ttm == NULL))
235 ret = -ENOMEM;
236 break;
237 case ttm_bo_type_user:
238 bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
239 page_flags | TTM_PAGE_FLAG_USER,
240 bdev->dummy_read_page);
241 if (unlikely(bo->ttm == NULL))
242 ret = -ENOMEM;
243 break;
244
245 ret = ttm_tt_set_user(bo->ttm, current,
246 bo->buffer_start, bo->num_pages);
247 if (unlikely(ret != 0))
248 ttm_tt_destroy(bo->ttm);
249 break;
250 default:
251 printk(KERN_ERR TTM_PFX "Illegal buffer object type\n");
252 ret = -EINVAL;
253 break;
254 }
255
256 return ret;
257}
258
259static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
260 struct ttm_mem_reg *mem,
261 bool evict, bool interruptible, bool no_wait)
262{
263 struct ttm_bo_device *bdev = bo->bdev;
264 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
265 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
266 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
267 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
268 int ret = 0;
269
270 if (old_is_pci || new_is_pci ||
271 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0))
272 ttm_bo_unmap_virtual(bo);
273
274 /*
275 * Create and bind a ttm if required.
276 */
277
278 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm == NULL)) {
279 ret = ttm_bo_add_ttm(bo, false);
280 if (ret)
281 goto out_err;
282
283 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
284 if (ret)
285 return ret;
286
287 if (mem->mem_type != TTM_PL_SYSTEM) {
288 ret = ttm_tt_bind(bo->ttm, mem);
289 if (ret)
290 goto out_err;
291 }
292
293 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
294
295 struct ttm_mem_reg *old_mem = &bo->mem;
296 uint32_t save_flags = old_mem->placement;
297
298 *old_mem = *mem;
299 mem->mm_node = NULL;
300 ttm_flag_masked(&save_flags, mem->placement,
301 TTM_PL_MASK_MEMTYPE);
302 goto moved;
303 }
304
305 }
306
307 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
308 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
309 ret = ttm_bo_move_ttm(bo, evict, no_wait, mem);
310 else if (bdev->driver->move)
311 ret = bdev->driver->move(bo, evict, interruptible,
312 no_wait, mem);
313 else
314 ret = ttm_bo_move_memcpy(bo, evict, no_wait, mem);
315
316 if (ret)
317 goto out_err;
318
319moved:
320 if (bo->evicted) {
321 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
322 if (ret)
323 printk(KERN_ERR TTM_PFX "Can not flush read caches\n");
324 bo->evicted = false;
325 }
326
327 if (bo->mem.mm_node) {
328 spin_lock(&bo->lock);
329 bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) +
330 bdev->man[bo->mem.mem_type].gpu_offset;
331 bo->cur_placement = bo->mem.placement;
332 spin_unlock(&bo->lock);
333 }
334
335 return 0;
336
337out_err:
338 new_man = &bdev->man[bo->mem.mem_type];
339 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
340 ttm_tt_unbind(bo->ttm);
341 ttm_tt_destroy(bo->ttm);
342 bo->ttm = NULL;
343 }
344
345 return ret;
346}
347
348/**
349 * If bo idle, remove from delayed- and lru lists, and unref.
350 * If not idle, and already on delayed list, do nothing.
351 * If not idle, and not on delayed list, put on delayed list,
352 * up the list_kref and schedule a delayed list check.
353 */
354
355static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
356{
357 struct ttm_bo_device *bdev = bo->bdev;
358 struct ttm_bo_driver *driver = bdev->driver;
359 int ret;
360
361 spin_lock(&bo->lock);
362 (void) ttm_bo_wait(bo, false, false, !remove_all);
363
364 if (!bo->sync_obj) {
365 int put_count;
366
367 spin_unlock(&bo->lock);
368
369 spin_lock(&bdev->lru_lock);
370 ret = ttm_bo_reserve_locked(bo, false, false, false, 0);
371 BUG_ON(ret);
372 if (bo->ttm)
373 ttm_tt_unbind(bo->ttm);
374
375 if (!list_empty(&bo->ddestroy)) {
376 list_del_init(&bo->ddestroy);
377 kref_put(&bo->list_kref, ttm_bo_ref_bug);
378 }
379 if (bo->mem.mm_node) {
380 drm_mm_put_block(bo->mem.mm_node);
381 bo->mem.mm_node = NULL;
382 }
383 put_count = ttm_bo_del_from_lru(bo);
384 spin_unlock(&bdev->lru_lock);
385
386 atomic_set(&bo->reserved, 0);
387
388 while (put_count--)
389 kref_put(&bo->list_kref, ttm_bo_release_list);
390
391 return 0;
392 }
393
394 spin_lock(&bdev->lru_lock);
395 if (list_empty(&bo->ddestroy)) {
396 void *sync_obj = bo->sync_obj;
397 void *sync_obj_arg = bo->sync_obj_arg;
398
399 kref_get(&bo->list_kref);
400 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
401 spin_unlock(&bdev->lru_lock);
402 spin_unlock(&bo->lock);
403
404 if (sync_obj)
405 driver->sync_obj_flush(sync_obj, sync_obj_arg);
406 schedule_delayed_work(&bdev->wq,
407 ((HZ / 100) < 1) ? 1 : HZ / 100);
408 ret = 0;
409
410 } else {
411 spin_unlock(&bdev->lru_lock);
412 spin_unlock(&bo->lock);
413 ret = -EBUSY;
414 }
415
416 return ret;
417}
418
419/**
420 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
421 * encountered buffers.
422 */
423
424static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
425{
426 struct ttm_buffer_object *entry, *nentry;
427 struct list_head *list, *next;
428 int ret;
429
430 spin_lock(&bdev->lru_lock);
431 list_for_each_safe(list, next, &bdev->ddestroy) {
432 entry = list_entry(list, struct ttm_buffer_object, ddestroy);
433 nentry = NULL;
434
435 /*
436 * Protect the next list entry from destruction while we
437 * unlock the lru_lock.
438 */
439
440 if (next != &bdev->ddestroy) {
441 nentry = list_entry(next, struct ttm_buffer_object,
442 ddestroy);
443 kref_get(&nentry->list_kref);
444 }
445 kref_get(&entry->list_kref);
446
447 spin_unlock(&bdev->lru_lock);
448 ret = ttm_bo_cleanup_refs(entry, remove_all);
449 kref_put(&entry->list_kref, ttm_bo_release_list);
450
451 spin_lock(&bdev->lru_lock);
452 if (nentry) {
453 bool next_onlist = !list_empty(next);
454 spin_unlock(&bdev->lru_lock);
455 kref_put(&nentry->list_kref, ttm_bo_release_list);
456 spin_lock(&bdev->lru_lock);
457 /*
458 * Someone might have raced us and removed the
459 * next entry from the list. We don't bother restarting
460 * list traversal.
461 */
462
463 if (!next_onlist)
464 break;
465 }
466 if (ret)
467 break;
468 }
469 ret = !list_empty(&bdev->ddestroy);
470 spin_unlock(&bdev->lru_lock);
471
472 return ret;
473}
474
475static void ttm_bo_delayed_workqueue(struct work_struct *work)
476{
477 struct ttm_bo_device *bdev =
478 container_of(work, struct ttm_bo_device, wq.work);
479
480 if (ttm_bo_delayed_delete(bdev, false)) {
481 schedule_delayed_work(&bdev->wq,
482 ((HZ / 100) < 1) ? 1 : HZ / 100);
483 }
484}
485
486static void ttm_bo_release(struct kref *kref)
487{
488 struct ttm_buffer_object *bo =
489 container_of(kref, struct ttm_buffer_object, kref);
490 struct ttm_bo_device *bdev = bo->bdev;
491
492 if (likely(bo->vm_node != NULL)) {
493 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
494 drm_mm_put_block(bo->vm_node);
495 bo->vm_node = NULL;
496 }
497 write_unlock(&bdev->vm_lock);
498 ttm_bo_cleanup_refs(bo, false);
499 kref_put(&bo->list_kref, ttm_bo_release_list);
500 write_lock(&bdev->vm_lock);
501}
502
503void ttm_bo_unref(struct ttm_buffer_object **p_bo)
504{
505 struct ttm_buffer_object *bo = *p_bo;
506 struct ttm_bo_device *bdev = bo->bdev;
507
508 *p_bo = NULL;
509 write_lock(&bdev->vm_lock);
510 kref_put(&bo->kref, ttm_bo_release);
511 write_unlock(&bdev->vm_lock);
512}
513EXPORT_SYMBOL(ttm_bo_unref);
514
515static int ttm_bo_evict(struct ttm_buffer_object *bo, unsigned mem_type,
516 bool interruptible, bool no_wait)
517{
518 int ret = 0;
519 struct ttm_bo_device *bdev = bo->bdev;
520 struct ttm_mem_reg evict_mem;
521 uint32_t proposed_placement;
522
523 if (bo->mem.mem_type != mem_type)
524 goto out;
525
526 spin_lock(&bo->lock);
527 ret = ttm_bo_wait(bo, false, interruptible, no_wait);
528 spin_unlock(&bo->lock);
529
530 if (ret && ret != -ERESTART) {
531 printk(KERN_ERR TTM_PFX "Failed to expire sync object before "
532 "buffer eviction.\n");
533 goto out;
534 }
535
536 BUG_ON(!atomic_read(&bo->reserved));
537
538 evict_mem = bo->mem;
539 evict_mem.mm_node = NULL;
540
541 proposed_placement = bdev->driver->evict_flags(bo);
542
543 ret = ttm_bo_mem_space(bo, proposed_placement,
544 &evict_mem, interruptible, no_wait);
545 if (unlikely(ret != 0 && ret != -ERESTART))
546 ret = ttm_bo_mem_space(bo, TTM_PL_FLAG_SYSTEM,
547 &evict_mem, interruptible, no_wait);
548
549 if (ret) {
550 if (ret != -ERESTART)
551 printk(KERN_ERR TTM_PFX
552 "Failed to find memory space for "
553 "buffer 0x%p eviction.\n", bo);
554 goto out;
555 }
556
557 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
558 no_wait);
559 if (ret) {
560 if (ret != -ERESTART)
561 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
562 goto out;
563 }
564
565 spin_lock(&bdev->lru_lock);
566 if (evict_mem.mm_node) {
567 drm_mm_put_block(evict_mem.mm_node);
568 evict_mem.mm_node = NULL;
569 }
570 spin_unlock(&bdev->lru_lock);
571 bo->evicted = true;
572out:
573 return ret;
574}
575
576/**
577 * Repeatedly evict memory from the LRU for @mem_type until we create enough
578 * space, or we've evicted everything and there isn't enough space.
579 */
580static int ttm_bo_mem_force_space(struct ttm_bo_device *bdev,
581 struct ttm_mem_reg *mem,
582 uint32_t mem_type,
583 bool interruptible, bool no_wait)
584{
585 struct drm_mm_node *node;
586 struct ttm_buffer_object *entry;
587 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
588 struct list_head *lru;
589 unsigned long num_pages = mem->num_pages;
590 int put_count = 0;
591 int ret;
592
593retry_pre_get:
594 ret = drm_mm_pre_get(&man->manager);
595 if (unlikely(ret != 0))
596 return ret;
597
598 spin_lock(&bdev->lru_lock);
599 do {
600 node = drm_mm_search_free(&man->manager, num_pages,
601 mem->page_alignment, 1);
602 if (node)
603 break;
604
605 lru = &man->lru;
606 if (list_empty(lru))
607 break;
608
609 entry = list_first_entry(lru, struct ttm_buffer_object, lru);
610 kref_get(&entry->list_kref);
611
612 ret =
613 ttm_bo_reserve_locked(entry, interruptible, no_wait,
614 false, 0);
615
616 if (likely(ret == 0))
617 put_count = ttm_bo_del_from_lru(entry);
618
619 spin_unlock(&bdev->lru_lock);
620
621 if (unlikely(ret != 0))
622 return ret;
623
624 while (put_count--)
625 kref_put(&entry->list_kref, ttm_bo_ref_bug);
626
627 ret = ttm_bo_evict(entry, mem_type, interruptible, no_wait);
628
629 ttm_bo_unreserve(entry);
630
631 kref_put(&entry->list_kref, ttm_bo_release_list);
632 if (ret)
633 return ret;
634
635 spin_lock(&bdev->lru_lock);
636 } while (1);
637
638 if (!node) {
639 spin_unlock(&bdev->lru_lock);
640 return -ENOMEM;
641 }
642
643 node = drm_mm_get_block_atomic(node, num_pages, mem->page_alignment);
644 if (unlikely(!node)) {
645 spin_unlock(&bdev->lru_lock);
646 goto retry_pre_get;
647 }
648
649 spin_unlock(&bdev->lru_lock);
650 mem->mm_node = node;
651 mem->mem_type = mem_type;
652 return 0;
653}
654
655static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
656 bool disallow_fixed,
657 uint32_t mem_type,
658 uint32_t mask, uint32_t *res_mask)
659{
660 uint32_t cur_flags = ttm_bo_type_flags(mem_type);
661
662 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && disallow_fixed)
663 return false;
664
665 if ((cur_flags & mask & TTM_PL_MASK_MEM) == 0)
666 return false;
667
668 if ((mask & man->available_caching) == 0)
669 return false;
670 if (mask & man->default_caching)
671 cur_flags |= man->default_caching;
672 else if (mask & TTM_PL_FLAG_CACHED)
673 cur_flags |= TTM_PL_FLAG_CACHED;
674 else if (mask & TTM_PL_FLAG_WC)
675 cur_flags |= TTM_PL_FLAG_WC;
676 else
677 cur_flags |= TTM_PL_FLAG_UNCACHED;
678
679 *res_mask = cur_flags;
680 return true;
681}
682
683/**
684 * Creates space for memory region @mem according to its type.
685 *
686 * This function first searches for free space in compatible memory types in
687 * the priority order defined by the driver. If free space isn't found, then
688 * ttm_bo_mem_force_space is attempted in priority order to evict and find
689 * space.
690 */
691int ttm_bo_mem_space(struct ttm_buffer_object *bo,
692 uint32_t proposed_placement,
693 struct ttm_mem_reg *mem,
694 bool interruptible, bool no_wait)
695{
696 struct ttm_bo_device *bdev = bo->bdev;
697 struct ttm_mem_type_manager *man;
698
699 uint32_t num_prios = bdev->driver->num_mem_type_prio;
700 const uint32_t *prios = bdev->driver->mem_type_prio;
701 uint32_t i;
702 uint32_t mem_type = TTM_PL_SYSTEM;
703 uint32_t cur_flags = 0;
704 bool type_found = false;
705 bool type_ok = false;
706 bool has_eagain = false;
707 struct drm_mm_node *node = NULL;
708 int ret;
709
710 mem->mm_node = NULL;
711 for (i = 0; i < num_prios; ++i) {
712 mem_type = prios[i];
713 man = &bdev->man[mem_type];
714
715 type_ok = ttm_bo_mt_compatible(man,
716 bo->type == ttm_bo_type_user,
717 mem_type, proposed_placement,
718 &cur_flags);
719
720 if (!type_ok)
721 continue;
722
723 if (mem_type == TTM_PL_SYSTEM)
724 break;
725
726 if (man->has_type && man->use_type) {
727 type_found = true;
728 do {
729 ret = drm_mm_pre_get(&man->manager);
730 if (unlikely(ret))
731 return ret;
732
733 spin_lock(&bdev->lru_lock);
734 node = drm_mm_search_free(&man->manager,
735 mem->num_pages,
736 mem->page_alignment,
737 1);
738 if (unlikely(!node)) {
739 spin_unlock(&bdev->lru_lock);
740 break;
741 }
742 node = drm_mm_get_block_atomic(node,
743 mem->num_pages,
744 mem->
745 page_alignment);
746 spin_unlock(&bdev->lru_lock);
747 } while (!node);
748 }
749 if (node)
750 break;
751 }
752
753 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) {
754 mem->mm_node = node;
755 mem->mem_type = mem_type;
756 mem->placement = cur_flags;
757 return 0;
758 }
759
760 if (!type_found)
761 return -EINVAL;
762
763 num_prios = bdev->driver->num_mem_busy_prio;
764 prios = bdev->driver->mem_busy_prio;
765
766 for (i = 0; i < num_prios; ++i) {
767 mem_type = prios[i];
768 man = &bdev->man[mem_type];
769
770 if (!man->has_type)
771 continue;
772
773 if (!ttm_bo_mt_compatible(man,
774 bo->type == ttm_bo_type_user,
775 mem_type,
776 proposed_placement, &cur_flags))
777 continue;
778
779 ret = ttm_bo_mem_force_space(bdev, mem, mem_type,
780 interruptible, no_wait);
781
782 if (ret == 0 && mem->mm_node) {
783 mem->placement = cur_flags;
784 return 0;
785 }
786
787 if (ret == -ERESTART)
788 has_eagain = true;
789 }
790
791 ret = (has_eagain) ? -ERESTART : -ENOMEM;
792 return ret;
793}
794EXPORT_SYMBOL(ttm_bo_mem_space);
795
796int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait)
797{
798 int ret = 0;
799
800 if ((atomic_read(&bo->cpu_writers) > 0) && no_wait)
801 return -EBUSY;
802
803 ret = wait_event_interruptible(bo->event_queue,
804 atomic_read(&bo->cpu_writers) == 0);
805
806 if (ret == -ERESTARTSYS)
807 ret = -ERESTART;
808
809 return ret;
810}
811
812int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
813 uint32_t proposed_placement,
814 bool interruptible, bool no_wait)
815{
816 struct ttm_bo_device *bdev = bo->bdev;
817 int ret = 0;
818 struct ttm_mem_reg mem;
819
820 BUG_ON(!atomic_read(&bo->reserved));
821
822 /*
823 * FIXME: It's possible to pipeline buffer moves.
824 * Have the driver move function wait for idle when necessary,
825 * instead of doing it here.
826 */
827
828 spin_lock(&bo->lock);
829 ret = ttm_bo_wait(bo, false, interruptible, no_wait);
830 spin_unlock(&bo->lock);
831
832 if (ret)
833 return ret;
834
835 mem.num_pages = bo->num_pages;
836 mem.size = mem.num_pages << PAGE_SHIFT;
837 mem.page_alignment = bo->mem.page_alignment;
838
839 /*
840 * Determine where to move the buffer.
841 */
842
843 ret = ttm_bo_mem_space(bo, proposed_placement, &mem,
844 interruptible, no_wait);
845 if (ret)
846 goto out_unlock;
847
848 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait);
849
850out_unlock:
851 if (ret && mem.mm_node) {
852 spin_lock(&bdev->lru_lock);
853 drm_mm_put_block(mem.mm_node);
854 spin_unlock(&bdev->lru_lock);
855 }
856 return ret;
857}
858
859static int ttm_bo_mem_compat(uint32_t proposed_placement,
860 struct ttm_mem_reg *mem)
861{
862 if ((proposed_placement & mem->placement & TTM_PL_MASK_MEM) == 0)
863 return 0;
864 if ((proposed_placement & mem->placement & TTM_PL_MASK_CACHING) == 0)
865 return 0;
866
867 return 1;
868}
869
870int ttm_buffer_object_validate(struct ttm_buffer_object *bo,
871 uint32_t proposed_placement,
872 bool interruptible, bool no_wait)
873{
874 int ret;
875
876 BUG_ON(!atomic_read(&bo->reserved));
877 bo->proposed_placement = proposed_placement;
878
879 TTM_DEBUG("Proposed placement 0x%08lx, Old flags 0x%08lx\n",
880 (unsigned long)proposed_placement,
881 (unsigned long)bo->mem.placement);
882
883 /*
884 * Check whether we need to move buffer.
885 */
886
887 if (!ttm_bo_mem_compat(bo->proposed_placement, &bo->mem)) {
888 ret = ttm_bo_move_buffer(bo, bo->proposed_placement,
889 interruptible, no_wait);
890 if (ret) {
891 if (ret != -ERESTART)
892 printk(KERN_ERR TTM_PFX
893 "Failed moving buffer. "
894 "Proposed placement 0x%08x\n",
895 bo->proposed_placement);
896 if (ret == -ENOMEM)
897 printk(KERN_ERR TTM_PFX
898 "Out of aperture space or "
899 "DRM memory quota.\n");
900 return ret;
901 }
902 }
903
904 /*
905 * We might need to add a TTM.
906 */
907
908 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
909 ret = ttm_bo_add_ttm(bo, true);
910 if (ret)
911 return ret;
912 }
913 /*
914 * Validation has succeeded, move the access and other
915 * non-mapping-related flag bits from the proposed flags to
916 * the active flags
917 */
918
919 ttm_flag_masked(&bo->mem.placement, bo->proposed_placement,
920 ~TTM_PL_MASK_MEMTYPE);
921
922 return 0;
923}
924EXPORT_SYMBOL(ttm_buffer_object_validate);
925
926int
927ttm_bo_check_placement(struct ttm_buffer_object *bo,
928 uint32_t set_flags, uint32_t clr_flags)
929{
930 uint32_t new_mask = set_flags | clr_flags;
931
932 if ((bo->type == ttm_bo_type_user) &&
933 (clr_flags & TTM_PL_FLAG_CACHED)) {
934 printk(KERN_ERR TTM_PFX
935 "User buffers require cache-coherent memory.\n");
936 return -EINVAL;
937 }
938
939 if (!capable(CAP_SYS_ADMIN)) {
940 if (new_mask & TTM_PL_FLAG_NO_EVICT) {
941 printk(KERN_ERR TTM_PFX "Need to be root to modify"
942 " NO_EVICT status.\n");
943 return -EINVAL;
944 }
945
946 if ((clr_flags & bo->mem.placement & TTM_PL_MASK_MEMTYPE) &&
947 (bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
948 printk(KERN_ERR TTM_PFX
949 "Incompatible memory specification"
950 " for NO_EVICT buffer.\n");
951 return -EINVAL;
952 }
953 }
954 return 0;
955}
956
957int ttm_buffer_object_init(struct ttm_bo_device *bdev,
958 struct ttm_buffer_object *bo,
959 unsigned long size,
960 enum ttm_bo_type type,
961 uint32_t flags,
962 uint32_t page_alignment,
963 unsigned long buffer_start,
964 bool interruptible,
965 struct file *persistant_swap_storage,
966 size_t acc_size,
967 void (*destroy) (struct ttm_buffer_object *))
968{
969 int ret = 0;
970 unsigned long num_pages;
971
972 size += buffer_start & ~PAGE_MASK;
973 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
974 if (num_pages == 0) {
975 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n");
976 return -EINVAL;
977 }
978 bo->destroy = destroy;
979
980 spin_lock_init(&bo->lock);
981 kref_init(&bo->kref);
982 kref_init(&bo->list_kref);
983 atomic_set(&bo->cpu_writers, 0);
984 atomic_set(&bo->reserved, 1);
985 init_waitqueue_head(&bo->event_queue);
986 INIT_LIST_HEAD(&bo->lru);
987 INIT_LIST_HEAD(&bo->ddestroy);
988 INIT_LIST_HEAD(&bo->swap);
989 bo->bdev = bdev;
990 bo->type = type;
991 bo->num_pages = num_pages;
992 bo->mem.mem_type = TTM_PL_SYSTEM;
993 bo->mem.num_pages = bo->num_pages;
994 bo->mem.mm_node = NULL;
995 bo->mem.page_alignment = page_alignment;
996 bo->buffer_start = buffer_start & PAGE_MASK;
997 bo->priv_flags = 0;
998 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
999 bo->seq_valid = false;
1000 bo->persistant_swap_storage = persistant_swap_storage;
1001 bo->acc_size = acc_size;
1002
1003 ret = ttm_bo_check_placement(bo, flags, 0ULL);
1004 if (unlikely(ret != 0))
1005 goto out_err;
1006
1007 /*
1008 * If no caching attributes are set, accept any form of caching.
1009 */
1010
1011 if ((flags & TTM_PL_MASK_CACHING) == 0)
1012 flags |= TTM_PL_MASK_CACHING;
1013
1014 /*
1015 * For ttm_bo_type_device buffers, allocate
1016 * address space from the device.
1017 */
1018
1019 if (bo->type == ttm_bo_type_device) {
1020 ret = ttm_bo_setup_vm(bo);
1021 if (ret)
1022 goto out_err;
1023 }
1024
1025 ret = ttm_buffer_object_validate(bo, flags, interruptible, false);
1026 if (ret)
1027 goto out_err;
1028
1029 ttm_bo_unreserve(bo);
1030 return 0;
1031
1032out_err:
1033 ttm_bo_unreserve(bo);
1034 ttm_bo_unref(&bo);
1035
1036 return ret;
1037}
1038EXPORT_SYMBOL(ttm_buffer_object_init);
1039
1040static inline size_t ttm_bo_size(struct ttm_bo_device *bdev,
1041 unsigned long num_pages)
1042{
1043 size_t page_array_size = (num_pages * sizeof(void *) + PAGE_SIZE - 1) &
1044 PAGE_MASK;
1045
1046 return bdev->ttm_bo_size + 2 * page_array_size;
1047}
1048
1049int ttm_buffer_object_create(struct ttm_bo_device *bdev,
1050 unsigned long size,
1051 enum ttm_bo_type type,
1052 uint32_t flags,
1053 uint32_t page_alignment,
1054 unsigned long buffer_start,
1055 bool interruptible,
1056 struct file *persistant_swap_storage,
1057 struct ttm_buffer_object **p_bo)
1058{
1059 struct ttm_buffer_object *bo;
1060 int ret;
1061 struct ttm_mem_global *mem_glob = bdev->mem_glob;
1062
1063 size_t acc_size =
1064 ttm_bo_size(bdev, (size + PAGE_SIZE - 1) >> PAGE_SHIFT);
1065 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false, false);
1066 if (unlikely(ret != 0))
1067 return ret;
1068
1069 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1070
1071 if (unlikely(bo == NULL)) {
1072 ttm_mem_global_free(mem_glob, acc_size, false);
1073 return -ENOMEM;
1074 }
1075
1076 ret = ttm_buffer_object_init(bdev, bo, size, type, flags,
1077 page_alignment, buffer_start,
1078 interruptible,
1079 persistant_swap_storage, acc_size, NULL);
1080 if (likely(ret == 0))
1081 *p_bo = bo;
1082
1083 return ret;
1084}
1085
1086static int ttm_bo_leave_list(struct ttm_buffer_object *bo,
1087 uint32_t mem_type, bool allow_errors)
1088{
1089 int ret;
1090
1091 spin_lock(&bo->lock);
1092 ret = ttm_bo_wait(bo, false, false, false);
1093 spin_unlock(&bo->lock);
1094
1095 if (ret && allow_errors)
1096 goto out;
1097
1098 if (bo->mem.mem_type == mem_type)
1099 ret = ttm_bo_evict(bo, mem_type, false, false);
1100
1101 if (ret) {
1102 if (allow_errors) {
1103 goto out;
1104 } else {
1105 ret = 0;
1106 printk(KERN_ERR TTM_PFX "Cleanup eviction failed\n");
1107 }
1108 }
1109
1110out:
1111 return ret;
1112}
1113
1114static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1115 struct list_head *head,
1116 unsigned mem_type, bool allow_errors)
1117{
1118 struct ttm_buffer_object *entry;
1119 int ret;
1120 int put_count;
1121
1122 /*
1123 * Can't use standard list traversal since we're unlocking.
1124 */
1125
1126 spin_lock(&bdev->lru_lock);
1127
1128 while (!list_empty(head)) {
1129 entry = list_first_entry(head, struct ttm_buffer_object, lru);
1130 kref_get(&entry->list_kref);
1131 ret = ttm_bo_reserve_locked(entry, false, false, false, 0);
1132 put_count = ttm_bo_del_from_lru(entry);
1133 spin_unlock(&bdev->lru_lock);
1134 while (put_count--)
1135 kref_put(&entry->list_kref, ttm_bo_ref_bug);
1136 BUG_ON(ret);
1137 ret = ttm_bo_leave_list(entry, mem_type, allow_errors);
1138 ttm_bo_unreserve(entry);
1139 kref_put(&entry->list_kref, ttm_bo_release_list);
1140 spin_lock(&bdev->lru_lock);
1141 }
1142
1143 spin_unlock(&bdev->lru_lock);
1144
1145 return 0;
1146}
1147
1148int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1149{
1150 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1151 int ret = -EINVAL;
1152
1153 if (mem_type >= TTM_NUM_MEM_TYPES) {
1154 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type);
1155 return ret;
1156 }
1157
1158 if (!man->has_type) {
1159 printk(KERN_ERR TTM_PFX "Trying to take down uninitialized "
1160 "memory manager type %u\n", mem_type);
1161 return ret;
1162 }
1163
1164 man->use_type = false;
1165 man->has_type = false;
1166
1167 ret = 0;
1168 if (mem_type > 0) {
1169 ttm_bo_force_list_clean(bdev, &man->lru, mem_type, false);
1170
1171 spin_lock(&bdev->lru_lock);
1172 if (drm_mm_clean(&man->manager))
1173 drm_mm_takedown(&man->manager);
1174 else
1175 ret = -EBUSY;
1176
1177 spin_unlock(&bdev->lru_lock);
1178 }
1179
1180 return ret;
1181}
1182EXPORT_SYMBOL(ttm_bo_clean_mm);
1183
1184int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1185{
1186 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1187
1188 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1189 printk(KERN_ERR TTM_PFX
1190 "Illegal memory manager memory type %u.\n",
1191 mem_type);
1192 return -EINVAL;
1193 }
1194
1195 if (!man->has_type) {
1196 printk(KERN_ERR TTM_PFX
1197 "Memory type %u has not been initialized.\n",
1198 mem_type);
1199 return 0;
1200 }
1201
1202 return ttm_bo_force_list_clean(bdev, &man->lru, mem_type, true);
1203}
1204EXPORT_SYMBOL(ttm_bo_evict_mm);
1205
1206int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1207 unsigned long p_offset, unsigned long p_size)
1208{
1209 int ret = -EINVAL;
1210 struct ttm_mem_type_manager *man;
1211
1212 if (type >= TTM_NUM_MEM_TYPES) {
1213 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type);
1214 return ret;
1215 }
1216
1217 man = &bdev->man[type];
1218 if (man->has_type) {
1219 printk(KERN_ERR TTM_PFX
1220 "Memory manager already initialized for type %d\n",
1221 type);
1222 return ret;
1223 }
1224
1225 ret = bdev->driver->init_mem_type(bdev, type, man);
1226 if (ret)
1227 return ret;
1228
1229 ret = 0;
1230 if (type != TTM_PL_SYSTEM) {
1231 if (!p_size) {
1232 printk(KERN_ERR TTM_PFX
1233 "Zero size memory manager type %d\n",
1234 type);
1235 return ret;
1236 }
1237 ret = drm_mm_init(&man->manager, p_offset, p_size);
1238 if (ret)
1239 return ret;
1240 }
1241 man->has_type = true;
1242 man->use_type = true;
1243 man->size = p_size;
1244
1245 INIT_LIST_HEAD(&man->lru);
1246
1247 return 0;
1248}
1249EXPORT_SYMBOL(ttm_bo_init_mm);
1250
1251int ttm_bo_device_release(struct ttm_bo_device *bdev)
1252{
1253 int ret = 0;
1254 unsigned i = TTM_NUM_MEM_TYPES;
1255 struct ttm_mem_type_manager *man;
1256
1257 while (i--) {
1258 man = &bdev->man[i];
1259 if (man->has_type) {
1260 man->use_type = false;
1261 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1262 ret = -EBUSY;
1263 printk(KERN_ERR TTM_PFX
1264 "DRM memory manager type %d "
1265 "is not clean.\n", i);
1266 }
1267 man->has_type = false;
1268 }
1269 }
1270
1271 if (!cancel_delayed_work(&bdev->wq))
1272 flush_scheduled_work();
1273
1274 while (ttm_bo_delayed_delete(bdev, true))
1275 ;
1276
1277 spin_lock(&bdev->lru_lock);
1278 if (list_empty(&bdev->ddestroy))
1279 TTM_DEBUG("Delayed destroy list was clean\n");
1280
1281 if (list_empty(&bdev->man[0].lru))
1282 TTM_DEBUG("Swap list was clean\n");
1283 spin_unlock(&bdev->lru_lock);
1284
1285 ttm_mem_unregister_shrink(bdev->mem_glob, &bdev->shrink);
1286 BUG_ON(!drm_mm_clean(&bdev->addr_space_mm));
1287 write_lock(&bdev->vm_lock);
1288 drm_mm_takedown(&bdev->addr_space_mm);
1289 write_unlock(&bdev->vm_lock);
1290
1291 __free_page(bdev->dummy_read_page);
1292 return ret;
1293}
1294EXPORT_SYMBOL(ttm_bo_device_release);
1295
1296/*
1297 * This function is intended to be called on drm driver load.
1298 * If you decide to call it from firstopen, you must protect the call
1299 * from a potentially racing ttm_bo_driver_finish in lastclose.
1300 * (This may happen on X server restart).
1301 */
1302
1303int ttm_bo_device_init(struct ttm_bo_device *bdev,
1304 struct ttm_mem_global *mem_glob,
1305 struct ttm_bo_driver *driver, uint64_t file_page_offset)
1306{
1307 int ret = -EINVAL;
1308
1309 bdev->dummy_read_page = NULL;
1310 rwlock_init(&bdev->vm_lock);
1311 spin_lock_init(&bdev->lru_lock);
1312
1313 bdev->driver = driver;
1314 bdev->mem_glob = mem_glob;
1315
1316 memset(bdev->man, 0, sizeof(bdev->man));
1317
1318 bdev->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1319 if (unlikely(bdev->dummy_read_page == NULL)) {
1320 ret = -ENOMEM;
1321 goto out_err0;
1322 }
1323
1324 /*
1325 * Initialize the system memory buffer type.
1326 * Other types need to be driver / IOCTL initialized.
1327 */
1328 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0, 0);
1329 if (unlikely(ret != 0))
1330 goto out_err1;
1331
1332 bdev->addr_space_rb = RB_ROOT;
1333 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
1334 if (unlikely(ret != 0))
1335 goto out_err2;
1336
1337 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1338 bdev->nice_mode = true;
1339 INIT_LIST_HEAD(&bdev->ddestroy);
1340 INIT_LIST_HEAD(&bdev->swap_lru);
1341 bdev->dev_mapping = NULL;
1342 ttm_mem_init_shrink(&bdev->shrink, ttm_bo_swapout);
1343 ret = ttm_mem_register_shrink(mem_glob, &bdev->shrink);
1344 if (unlikely(ret != 0)) {
1345 printk(KERN_ERR TTM_PFX
1346 "Could not register buffer object swapout.\n");
1347 goto out_err2;
1348 }
1349
1350 bdev->ttm_bo_extra_size =
1351 ttm_round_pot(sizeof(struct ttm_tt)) +
1352 ttm_round_pot(sizeof(struct ttm_backend));
1353
1354 bdev->ttm_bo_size = bdev->ttm_bo_extra_size +
1355 ttm_round_pot(sizeof(struct ttm_buffer_object));
1356
1357 return 0;
1358out_err2:
1359 ttm_bo_clean_mm(bdev, 0);
1360out_err1:
1361 __free_page(bdev->dummy_read_page);
1362out_err0:
1363 return ret;
1364}
1365EXPORT_SYMBOL(ttm_bo_device_init);
1366
1367/*
1368 * buffer object vm functions.
1369 */
1370
1371bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1372{
1373 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1374
1375 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1376 if (mem->mem_type == TTM_PL_SYSTEM)
1377 return false;
1378
1379 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1380 return false;
1381
1382 if (mem->placement & TTM_PL_FLAG_CACHED)
1383 return false;
1384 }
1385 return true;
1386}
1387
1388int ttm_bo_pci_offset(struct ttm_bo_device *bdev,
1389 struct ttm_mem_reg *mem,
1390 unsigned long *bus_base,
1391 unsigned long *bus_offset, unsigned long *bus_size)
1392{
1393 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1394
1395 *bus_size = 0;
1396 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1397 return -EINVAL;
1398
1399 if (ttm_mem_reg_is_pci(bdev, mem)) {
1400 *bus_offset = mem->mm_node->start << PAGE_SHIFT;
1401 *bus_size = mem->num_pages << PAGE_SHIFT;
1402 *bus_base = man->io_offset;
1403 }
1404
1405 return 0;
1406}
1407
1408void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1409{
1410 struct ttm_bo_device *bdev = bo->bdev;
1411 loff_t offset = (loff_t) bo->addr_space_offset;
1412 loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT;
1413
1414 if (!bdev->dev_mapping)
1415 return;
1416
1417 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
1418}
1419
1420static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
1421{
1422 struct ttm_bo_device *bdev = bo->bdev;
1423 struct rb_node **cur = &bdev->addr_space_rb.rb_node;
1424 struct rb_node *parent = NULL;
1425 struct ttm_buffer_object *cur_bo;
1426 unsigned long offset = bo->vm_node->start;
1427 unsigned long cur_offset;
1428
1429 while (*cur) {
1430 parent = *cur;
1431 cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb);
1432 cur_offset = cur_bo->vm_node->start;
1433 if (offset < cur_offset)
1434 cur = &parent->rb_left;
1435 else if (offset > cur_offset)
1436 cur = &parent->rb_right;
1437 else
1438 BUG();
1439 }
1440
1441 rb_link_node(&bo->vm_rb, parent, cur);
1442 rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb);
1443}
1444
1445/**
1446 * ttm_bo_setup_vm:
1447 *
1448 * @bo: the buffer to allocate address space for
1449 *
1450 * Allocate address space in the drm device so that applications
1451 * can mmap the buffer and access the contents. This only
1452 * applies to ttm_bo_type_device objects as others are not
1453 * placed in the drm device address space.
1454 */
1455
1456static int ttm_bo_setup_vm(struct ttm_buffer_object *bo)
1457{
1458 struct ttm_bo_device *bdev = bo->bdev;
1459 int ret;
1460
1461retry_pre_get:
1462 ret = drm_mm_pre_get(&bdev->addr_space_mm);
1463 if (unlikely(ret != 0))
1464 return ret;
1465
1466 write_lock(&bdev->vm_lock);
1467 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
1468 bo->mem.num_pages, 0, 0);
1469
1470 if (unlikely(bo->vm_node == NULL)) {
1471 ret = -ENOMEM;
1472 goto out_unlock;
1473 }
1474
1475 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node,
1476 bo->mem.num_pages, 0);
1477
1478 if (unlikely(bo->vm_node == NULL)) {
1479 write_unlock(&bdev->vm_lock);
1480 goto retry_pre_get;
1481 }
1482
1483 ttm_bo_vm_insert_rb(bo);
1484 write_unlock(&bdev->vm_lock);
1485 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT;
1486
1487 return 0;
1488out_unlock:
1489 write_unlock(&bdev->vm_lock);
1490 return ret;
1491}
1492
1493int ttm_bo_wait(struct ttm_buffer_object *bo,
1494 bool lazy, bool interruptible, bool no_wait)
1495{
1496 struct ttm_bo_driver *driver = bo->bdev->driver;
1497 void *sync_obj;
1498 void *sync_obj_arg;
1499 int ret = 0;
1500
1501 if (likely(bo->sync_obj == NULL))
1502 return 0;
1503
1504 while (bo->sync_obj) {
1505
1506 if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) {
1507 void *tmp_obj = bo->sync_obj;
1508 bo->sync_obj = NULL;
1509 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1510 spin_unlock(&bo->lock);
1511 driver->sync_obj_unref(&tmp_obj);
1512 spin_lock(&bo->lock);
1513 continue;
1514 }
1515
1516 if (no_wait)
1517 return -EBUSY;
1518
1519 sync_obj = driver->sync_obj_ref(bo->sync_obj);
1520 sync_obj_arg = bo->sync_obj_arg;
1521 spin_unlock(&bo->lock);
1522 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
1523 lazy, interruptible);
1524 if (unlikely(ret != 0)) {
1525 driver->sync_obj_unref(&sync_obj);
1526 spin_lock(&bo->lock);
1527 return ret;
1528 }
1529 spin_lock(&bo->lock);
1530 if (likely(bo->sync_obj == sync_obj &&
1531 bo->sync_obj_arg == sync_obj_arg)) {
1532 void *tmp_obj = bo->sync_obj;
1533 bo->sync_obj = NULL;
1534 clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1535 &bo->priv_flags);
1536 spin_unlock(&bo->lock);
1537 driver->sync_obj_unref(&sync_obj);
1538 driver->sync_obj_unref(&tmp_obj);
1539 spin_lock(&bo->lock);
1540 }
1541 }
1542 return 0;
1543}
1544EXPORT_SYMBOL(ttm_bo_wait);
1545
1546void ttm_bo_unblock_reservation(struct ttm_buffer_object *bo)
1547{
1548 atomic_set(&bo->reserved, 0);
1549 wake_up_all(&bo->event_queue);
1550}
1551
1552int ttm_bo_block_reservation(struct ttm_buffer_object *bo, bool interruptible,
1553 bool no_wait)
1554{
1555 int ret;
1556
1557 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
1558 if (no_wait)
1559 return -EBUSY;
1560 else if (interruptible) {
1561 ret = wait_event_interruptible
1562 (bo->event_queue, atomic_read(&bo->reserved) == 0);
1563 if (unlikely(ret != 0))
1564 return -ERESTART;
1565 } else {
1566 wait_event(bo->event_queue,
1567 atomic_read(&bo->reserved) == 0);
1568 }
1569 }
1570 return 0;
1571}
1572
1573int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1574{
1575 int ret = 0;
1576
1577 /*
1578 * Using ttm_bo_reserve instead of ttm_bo_block_reservation
1579 * makes sure the lru lists are updated.
1580 */
1581
1582 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1583 if (unlikely(ret != 0))
1584 return ret;
1585 spin_lock(&bo->lock);
1586 ret = ttm_bo_wait(bo, false, true, no_wait);
1587 spin_unlock(&bo->lock);
1588 if (likely(ret == 0))
1589 atomic_inc(&bo->cpu_writers);
1590 ttm_bo_unreserve(bo);
1591 return ret;
1592}
1593
1594void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1595{
1596 if (atomic_dec_and_test(&bo->cpu_writers))
1597 wake_up_all(&bo->event_queue);
1598}
1599
1600/**
1601 * A buffer object shrink method that tries to swap out the first
1602 * buffer object on the bo_global::swap_lru list.
1603 */
1604
1605static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1606{
1607 struct ttm_bo_device *bdev =
1608 container_of(shrink, struct ttm_bo_device, shrink);
1609 struct ttm_buffer_object *bo;
1610 int ret = -EBUSY;
1611 int put_count;
1612 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM);
1613
1614 spin_lock(&bdev->lru_lock);
1615 while (ret == -EBUSY) {
1616 if (unlikely(list_empty(&bdev->swap_lru))) {
1617 spin_unlock(&bdev->lru_lock);
1618 return -EBUSY;
1619 }
1620
1621 bo = list_first_entry(&bdev->swap_lru,
1622 struct ttm_buffer_object, swap);
1623 kref_get(&bo->list_kref);
1624
1625 /**
1626 * Reserve buffer. Since we unlock while sleeping, we need
1627 * to re-check that nobody removed us from the swap-list while
1628 * we slept.
1629 */
1630
1631 ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
1632 if (unlikely(ret == -EBUSY)) {
1633 spin_unlock(&bdev->lru_lock);
1634 ttm_bo_wait_unreserved(bo, false);
1635 kref_put(&bo->list_kref, ttm_bo_release_list);
1636 spin_lock(&bdev->lru_lock);
1637 }
1638 }
1639
1640 BUG_ON(ret != 0);
1641 put_count = ttm_bo_del_from_lru(bo);
1642 spin_unlock(&bdev->lru_lock);
1643
1644 while (put_count--)
1645 kref_put(&bo->list_kref, ttm_bo_ref_bug);
1646
1647 /**
1648 * Wait for GPU, then move to system cached.
1649 */
1650
1651 spin_lock(&bo->lock);
1652 ret = ttm_bo_wait(bo, false, false, false);
1653 spin_unlock(&bo->lock);
1654
1655 if (unlikely(ret != 0))
1656 goto out;
1657
1658 if ((bo->mem.placement & swap_placement) != swap_placement) {
1659 struct ttm_mem_reg evict_mem;
1660
1661 evict_mem = bo->mem;
1662 evict_mem.mm_node = NULL;
1663 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1664 evict_mem.mem_type = TTM_PL_SYSTEM;
1665
1666 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
1667 false, false);
1668 if (unlikely(ret != 0))
1669 goto out;
1670 }
1671
1672 ttm_bo_unmap_virtual(bo);
1673
1674 /**
1675 * Swap out. Buffer will be swapped in again as soon as
1676 * anyone tries to access a ttm page.
1677 */
1678
1679 ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage);
1680out:
1681
1682 /**
1683 *
1684 * Unreserve without putting on LRU to avoid swapping out an
1685 * already swapped buffer.
1686 */
1687
1688 atomic_set(&bo->reserved, 0);
1689 wake_up_all(&bo->event_queue);
1690 kref_put(&bo->list_kref, ttm_bo_release_list);
1691 return ret;
1692}
1693
1694void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1695{
1696 while (ttm_bo_swapout(&bdev->shrink) == 0)
1697 ;
1698}
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
new file mode 100644
index 000000000000..517c84559633
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -0,0 +1,561 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31#include "ttm/ttm_bo_driver.h"
32#include "ttm/ttm_placement.h"
33#include <linux/io.h>
34#include <linux/highmem.h>
35#include <linux/wait.h>
36#include <linux/vmalloc.h>
37#include <linux/version.h>
38#include <linux/module.h>
39
40void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
41{
42 struct ttm_mem_reg *old_mem = &bo->mem;
43
44 if (old_mem->mm_node) {
45 spin_lock(&bo->bdev->lru_lock);
46 drm_mm_put_block(old_mem->mm_node);
47 spin_unlock(&bo->bdev->lru_lock);
48 }
49 old_mem->mm_node = NULL;
50}
51
52int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
53 bool evict, bool no_wait, struct ttm_mem_reg *new_mem)
54{
55 struct ttm_tt *ttm = bo->ttm;
56 struct ttm_mem_reg *old_mem = &bo->mem;
57 uint32_t save_flags = old_mem->placement;
58 int ret;
59
60 if (old_mem->mem_type != TTM_PL_SYSTEM) {
61 ttm_tt_unbind(ttm);
62 ttm_bo_free_old_node(bo);
63 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
64 TTM_PL_MASK_MEM);
65 old_mem->mem_type = TTM_PL_SYSTEM;
66 save_flags = old_mem->placement;
67 }
68
69 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
70 if (unlikely(ret != 0))
71 return ret;
72
73 if (new_mem->mem_type != TTM_PL_SYSTEM) {
74 ret = ttm_tt_bind(ttm, new_mem);
75 if (unlikely(ret != 0))
76 return ret;
77 }
78
79 *old_mem = *new_mem;
80 new_mem->mm_node = NULL;
81 ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE);
82 return 0;
83}
84EXPORT_SYMBOL(ttm_bo_move_ttm);
85
86int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
87 void **virtual)
88{
89 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
90 unsigned long bus_offset;
91 unsigned long bus_size;
92 unsigned long bus_base;
93 int ret;
94 void *addr;
95
96 *virtual = NULL;
97 ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset, &bus_size);
98 if (ret || bus_size == 0)
99 return ret;
100
101 if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP))
102 addr = (void *)(((u8 *) man->io_addr) + bus_offset);
103 else {
104 if (mem->placement & TTM_PL_FLAG_WC)
105 addr = ioremap_wc(bus_base + bus_offset, bus_size);
106 else
107 addr = ioremap_nocache(bus_base + bus_offset, bus_size);
108 if (!addr)
109 return -ENOMEM;
110 }
111 *virtual = addr;
112 return 0;
113}
114
115void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
116 void *virtual)
117{
118 struct ttm_mem_type_manager *man;
119
120 man = &bdev->man[mem->mem_type];
121
122 if (virtual && (man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP))
123 iounmap(virtual);
124}
125
126static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
127{
128 uint32_t *dstP =
129 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
130 uint32_t *srcP =
131 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
132
133 int i;
134 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
135 iowrite32(ioread32(srcP++), dstP++);
136 return 0;
137}
138
139static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
140 unsigned long page)
141{
142 struct page *d = ttm_tt_get_page(ttm, page);
143 void *dst;
144
145 if (!d)
146 return -ENOMEM;
147
148 src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
149 dst = kmap(d);
150 if (!dst)
151 return -ENOMEM;
152
153 memcpy_fromio(dst, src, PAGE_SIZE);
154 kunmap(d);
155 return 0;
156}
157
158static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
159 unsigned long page)
160{
161 struct page *s = ttm_tt_get_page(ttm, page);
162 void *src;
163
164 if (!s)
165 return -ENOMEM;
166
167 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
168 src = kmap(s);
169 if (!src)
170 return -ENOMEM;
171
172 memcpy_toio(dst, src, PAGE_SIZE);
173 kunmap(s);
174 return 0;
175}
176
177int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
178 bool evict, bool no_wait, struct ttm_mem_reg *new_mem)
179{
180 struct ttm_bo_device *bdev = bo->bdev;
181 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
182 struct ttm_tt *ttm = bo->ttm;
183 struct ttm_mem_reg *old_mem = &bo->mem;
184 struct ttm_mem_reg old_copy = *old_mem;
185 void *old_iomap;
186 void *new_iomap;
187 int ret;
188 uint32_t save_flags = old_mem->placement;
189 unsigned long i;
190 unsigned long page;
191 unsigned long add = 0;
192 int dir;
193
194 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
195 if (ret)
196 return ret;
197 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
198 if (ret)
199 goto out;
200
201 if (old_iomap == NULL && new_iomap == NULL)
202 goto out2;
203 if (old_iomap == NULL && ttm == NULL)
204 goto out2;
205
206 add = 0;
207 dir = 1;
208
209 if ((old_mem->mem_type == new_mem->mem_type) &&
210 (new_mem->mm_node->start <
211 old_mem->mm_node->start + old_mem->mm_node->size)) {
212 dir = -1;
213 add = new_mem->num_pages - 1;
214 }
215
216 for (i = 0; i < new_mem->num_pages; ++i) {
217 page = i * dir + add;
218 if (old_iomap == NULL)
219 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page);
220 else if (new_iomap == NULL)
221 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page);
222 else
223 ret = ttm_copy_io_page(new_iomap, old_iomap, page);
224 if (ret)
225 goto out1;
226 }
227 mb();
228out2:
229 ttm_bo_free_old_node(bo);
230
231 *old_mem = *new_mem;
232 new_mem->mm_node = NULL;
233 ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE);
234
235 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
236 ttm_tt_unbind(ttm);
237 ttm_tt_destroy(ttm);
238 bo->ttm = NULL;
239 }
240
241out1:
242 ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
243out:
244 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
245 return ret;
246}
247EXPORT_SYMBOL(ttm_bo_move_memcpy);
248
249static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
250{
251 kfree(bo);
252}
253
254/**
255 * ttm_buffer_object_transfer
256 *
257 * @bo: A pointer to a struct ttm_buffer_object.
258 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
259 * holding the data of @bo with the old placement.
260 *
261 * This is a utility function that may be called after an accelerated move
262 * has been scheduled. A new buffer object is created as a placeholder for
263 * the old data while it's being copied. When that buffer object is idle,
264 * it can be destroyed, releasing the space of the old placement.
265 * Returns:
266 * !0: Failure.
267 */
268
269static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
270 struct ttm_buffer_object **new_obj)
271{
272 struct ttm_buffer_object *fbo;
273 struct ttm_bo_device *bdev = bo->bdev;
274 struct ttm_bo_driver *driver = bdev->driver;
275
276 fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
277 if (!fbo)
278 return -ENOMEM;
279
280 *fbo = *bo;
281
282 /**
283 * Fix up members that we shouldn't copy directly:
284 * TODO: Explicit member copy would probably be better here.
285 */
286
287 spin_lock_init(&fbo->lock);
288 init_waitqueue_head(&fbo->event_queue);
289 INIT_LIST_HEAD(&fbo->ddestroy);
290 INIT_LIST_HEAD(&fbo->lru);
291 INIT_LIST_HEAD(&fbo->swap);
292 fbo->vm_node = NULL;
293
294 fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
295 if (fbo->mem.mm_node)
296 fbo->mem.mm_node->private = (void *)fbo;
297 kref_init(&fbo->list_kref);
298 kref_init(&fbo->kref);
299 fbo->destroy = &ttm_transfered_destroy;
300
301 *new_obj = fbo;
302 return 0;
303}
304
305pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
306{
307#if defined(__i386__) || defined(__x86_64__)
308 if (caching_flags & TTM_PL_FLAG_WC)
309 tmp = pgprot_writecombine(tmp);
310 else if (boot_cpu_data.x86 > 3)
311 tmp = pgprot_noncached(tmp);
312
313#elif defined(__powerpc__)
314 if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
315 pgprot_val(tmp) |= _PAGE_NO_CACHE;
316 if (caching_flags & TTM_PL_FLAG_UNCACHED)
317 pgprot_val(tmp) |= _PAGE_GUARDED;
318 }
319#endif
320#if defined(__ia64__)
321 if (caching_flags & TTM_PL_FLAG_WC)
322 tmp = pgprot_writecombine(tmp);
323 else
324 tmp = pgprot_noncached(tmp);
325#endif
326#if defined(__sparc__)
327 if (!(caching_flags & TTM_PL_FLAG_CACHED))
328 tmp = pgprot_noncached(tmp);
329#endif
330 return tmp;
331}
332
333static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
334 unsigned long bus_base,
335 unsigned long bus_offset,
336 unsigned long bus_size,
337 struct ttm_bo_kmap_obj *map)
338{
339 struct ttm_bo_device *bdev = bo->bdev;
340 struct ttm_mem_reg *mem = &bo->mem;
341 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
342
343 if (!(man->flags & TTM_MEMTYPE_FLAG_NEEDS_IOREMAP)) {
344 map->bo_kmap_type = ttm_bo_map_premapped;
345 map->virtual = (void *)(((u8 *) man->io_addr) + bus_offset);
346 } else {
347 map->bo_kmap_type = ttm_bo_map_iomap;
348 if (mem->placement & TTM_PL_FLAG_WC)
349 map->virtual = ioremap_wc(bus_base + bus_offset,
350 bus_size);
351 else
352 map->virtual = ioremap_nocache(bus_base + bus_offset,
353 bus_size);
354 }
355 return (!map->virtual) ? -ENOMEM : 0;
356}
357
358static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
359 unsigned long start_page,
360 unsigned long num_pages,
361 struct ttm_bo_kmap_obj *map)
362{
363 struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
364 struct ttm_tt *ttm = bo->ttm;
365 struct page *d;
366 int i;
367
368 BUG_ON(!ttm);
369 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
370 /*
371 * We're mapping a single page, and the desired
372 * page protection is consistent with the bo.
373 */
374
375 map->bo_kmap_type = ttm_bo_map_kmap;
376 map->page = ttm_tt_get_page(ttm, start_page);
377 map->virtual = kmap(map->page);
378 } else {
379 /*
380 * Populate the part we're mapping;
381 */
382 for (i = start_page; i < start_page + num_pages; ++i) {
383 d = ttm_tt_get_page(ttm, i);
384 if (!d)
385 return -ENOMEM;
386 }
387
388 /*
389 * We need to use vmap to get the desired page protection
390 * or to make the buffer object look contigous.
391 */
392 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
393 PAGE_KERNEL :
394 ttm_io_prot(mem->placement, PAGE_KERNEL);
395 map->bo_kmap_type = ttm_bo_map_vmap;
396 map->virtual = vmap(ttm->pages + start_page, num_pages,
397 0, prot);
398 }
399 return (!map->virtual) ? -ENOMEM : 0;
400}
401
402int ttm_bo_kmap(struct ttm_buffer_object *bo,
403 unsigned long start_page, unsigned long num_pages,
404 struct ttm_bo_kmap_obj *map)
405{
406 int ret;
407 unsigned long bus_base;
408 unsigned long bus_offset;
409 unsigned long bus_size;
410
411 BUG_ON(!list_empty(&bo->swap));
412 map->virtual = NULL;
413 if (num_pages > bo->num_pages)
414 return -EINVAL;
415 if (start_page > bo->num_pages)
416 return -EINVAL;
417#if 0
418 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
419 return -EPERM;
420#endif
421 ret = ttm_bo_pci_offset(bo->bdev, &bo->mem, &bus_base,
422 &bus_offset, &bus_size);
423 if (ret)
424 return ret;
425 if (bus_size == 0) {
426 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
427 } else {
428 bus_offset += start_page << PAGE_SHIFT;
429 bus_size = num_pages << PAGE_SHIFT;
430 return ttm_bo_ioremap(bo, bus_base, bus_offset, bus_size, map);
431 }
432}
433EXPORT_SYMBOL(ttm_bo_kmap);
434
435void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
436{
437 if (!map->virtual)
438 return;
439 switch (map->bo_kmap_type) {
440 case ttm_bo_map_iomap:
441 iounmap(map->virtual);
442 break;
443 case ttm_bo_map_vmap:
444 vunmap(map->virtual);
445 break;
446 case ttm_bo_map_kmap:
447 kunmap(map->page);
448 break;
449 case ttm_bo_map_premapped:
450 break;
451 default:
452 BUG();
453 }
454 map->virtual = NULL;
455 map->page = NULL;
456}
457EXPORT_SYMBOL(ttm_bo_kunmap);
458
459int ttm_bo_pfn_prot(struct ttm_buffer_object *bo,
460 unsigned long dst_offset,
461 unsigned long *pfn, pgprot_t *prot)
462{
463 struct ttm_mem_reg *mem = &bo->mem;
464 struct ttm_bo_device *bdev = bo->bdev;
465 unsigned long bus_offset;
466 unsigned long bus_size;
467 unsigned long bus_base;
468 int ret;
469 ret = ttm_bo_pci_offset(bdev, mem, &bus_base, &bus_offset,
470 &bus_size);
471 if (ret)
472 return -EINVAL;
473 if (bus_size != 0)
474 *pfn = (bus_base + bus_offset + dst_offset) >> PAGE_SHIFT;
475 else
476 if (!bo->ttm)
477 return -EINVAL;
478 else
479 *pfn = page_to_pfn(ttm_tt_get_page(bo->ttm,
480 dst_offset >>
481 PAGE_SHIFT));
482 *prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
483 PAGE_KERNEL : ttm_io_prot(mem->placement, PAGE_KERNEL);
484
485 return 0;
486}
487
488int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
489 void *sync_obj,
490 void *sync_obj_arg,
491 bool evict, bool no_wait,
492 struct ttm_mem_reg *new_mem)
493{
494 struct ttm_bo_device *bdev = bo->bdev;
495 struct ttm_bo_driver *driver = bdev->driver;
496 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
497 struct ttm_mem_reg *old_mem = &bo->mem;
498 int ret;
499 uint32_t save_flags = old_mem->placement;
500 struct ttm_buffer_object *ghost_obj;
501 void *tmp_obj = NULL;
502
503 spin_lock(&bo->lock);
504 if (bo->sync_obj) {
505 tmp_obj = bo->sync_obj;
506 bo->sync_obj = NULL;
507 }
508 bo->sync_obj = driver->sync_obj_ref(sync_obj);
509 bo->sync_obj_arg = sync_obj_arg;
510 if (evict) {
511 ret = ttm_bo_wait(bo, false, false, false);
512 spin_unlock(&bo->lock);
513 driver->sync_obj_unref(&bo->sync_obj);
514
515 if (ret)
516 return ret;
517
518 ttm_bo_free_old_node(bo);
519 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
520 (bo->ttm != NULL)) {
521 ttm_tt_unbind(bo->ttm);
522 ttm_tt_destroy(bo->ttm);
523 bo->ttm = NULL;
524 }
525 } else {
526 /**
527 * This should help pipeline ordinary buffer moves.
528 *
529 * Hang old buffer memory on a new buffer object,
530 * and leave it to be released when the GPU
531 * operation has completed.
532 */
533
534 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
535 spin_unlock(&bo->lock);
536
537 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
538 if (ret)
539 return ret;
540
541 /**
542 * If we're not moving to fixed memory, the TTM object
543 * needs to stay alive. Otherwhise hang it on the ghost
544 * bo to be unbound and destroyed.
545 */
546
547 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
548 ghost_obj->ttm = NULL;
549 else
550 bo->ttm = NULL;
551
552 ttm_bo_unreserve(ghost_obj);
553 ttm_bo_unref(&ghost_obj);
554 }
555
556 *old_mem = *new_mem;
557 new_mem->mm_node = NULL;
558 ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE);
559 return 0;
560}
561EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
new file mode 100644
index 000000000000..27b146c54fbc
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -0,0 +1,454 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31#include <ttm/ttm_module.h>
32#include <ttm/ttm_bo_driver.h>
33#include <ttm/ttm_placement.h>
34#include <linux/mm.h>
35#include <linux/version.h>
36#include <linux/rbtree.h>
37#include <linux/module.h>
38#include <linux/uaccess.h>
39
40#define TTM_BO_VM_NUM_PREFAULT 16
41
42static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev,
43 unsigned long page_start,
44 unsigned long num_pages)
45{
46 struct rb_node *cur = bdev->addr_space_rb.rb_node;
47 unsigned long cur_offset;
48 struct ttm_buffer_object *bo;
49 struct ttm_buffer_object *best_bo = NULL;
50
51 while (likely(cur != NULL)) {
52 bo = rb_entry(cur, struct ttm_buffer_object, vm_rb);
53 cur_offset = bo->vm_node->start;
54 if (page_start >= cur_offset) {
55 cur = cur->rb_right;
56 best_bo = bo;
57 if (page_start == cur_offset)
58 break;
59 } else
60 cur = cur->rb_left;
61 }
62
63 if (unlikely(best_bo == NULL))
64 return NULL;
65
66 if (unlikely((best_bo->vm_node->start + best_bo->num_pages) <
67 (page_start + num_pages)))
68 return NULL;
69
70 return best_bo;
71}
72
73static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
74{
75 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
76 vma->vm_private_data;
77 struct ttm_bo_device *bdev = bo->bdev;
78 unsigned long bus_base;
79 unsigned long bus_offset;
80 unsigned long bus_size;
81 unsigned long page_offset;
82 unsigned long page_last;
83 unsigned long pfn;
84 struct ttm_tt *ttm = NULL;
85 struct page *page;
86 int ret;
87 int i;
88 bool is_iomem;
89 unsigned long address = (unsigned long)vmf->virtual_address;
90 int retval = VM_FAULT_NOPAGE;
91
92 /*
93 * Work around locking order reversal in fault / nopfn
94 * between mmap_sem and bo_reserve: Perform a trylock operation
95 * for reserve, and if it fails, retry the fault after scheduling.
96 */
97
98 ret = ttm_bo_reserve(bo, true, true, false, 0);
99 if (unlikely(ret != 0)) {
100 if (ret == -EBUSY)
101 set_need_resched();
102 return VM_FAULT_NOPAGE;
103 }
104
105 /*
106 * Wait for buffer data in transit, due to a pipelined
107 * move.
108 */
109
110 spin_lock(&bo->lock);
111 if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
112 ret = ttm_bo_wait(bo, false, true, false);
113 spin_unlock(&bo->lock);
114 if (unlikely(ret != 0)) {
115 retval = (ret != -ERESTART) ?
116 VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
117 goto out_unlock;
118 }
119 } else
120 spin_unlock(&bo->lock);
121
122
123 ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset,
124 &bus_size);
125 if (unlikely(ret != 0)) {
126 retval = VM_FAULT_SIGBUS;
127 goto out_unlock;
128 }
129
130 is_iomem = (bus_size != 0);
131
132 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
133 bo->vm_node->start - vma->vm_pgoff;
134 page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) +
135 bo->vm_node->start - vma->vm_pgoff;
136
137 if (unlikely(page_offset >= bo->num_pages)) {
138 retval = VM_FAULT_SIGBUS;
139 goto out_unlock;
140 }
141
142 /*
143 * Strictly, we're not allowed to modify vma->vm_page_prot here,
144 * since the mmap_sem is only held in read mode. However, we
145 * modify only the caching bits of vma->vm_page_prot and
146 * consider those bits protected by
147 * the bo->mutex, as we should be the only writers.
148 * There shouldn't really be any readers of these bits except
149 * within vm_insert_mixed()? fork?
150 *
151 * TODO: Add a list of vmas to the bo, and change the
152 * vma->vm_page_prot when the object changes caching policy, with
153 * the correct locks held.
154 */
155
156 if (is_iomem) {
157 vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
158 vma->vm_page_prot);
159 } else {
160 ttm = bo->ttm;
161 vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
162 vm_get_page_prot(vma->vm_flags) :
163 ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
164 }
165
166 /*
167 * Speculatively prefault a number of pages. Only error on
168 * first page.
169 */
170
171 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
172
173 if (is_iomem)
174 pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) +
175 page_offset;
176 else {
177 page = ttm_tt_get_page(ttm, page_offset);
178 if (unlikely(!page && i == 0)) {
179 retval = VM_FAULT_OOM;
180 goto out_unlock;
181 } else if (unlikely(!page)) {
182 break;
183 }
184 pfn = page_to_pfn(page);
185 }
186
187 ret = vm_insert_mixed(vma, address, pfn);
188 /*
189 * Somebody beat us to this PTE or prefaulting to
190 * an already populated PTE, or prefaulting error.
191 */
192
193 if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
194 break;
195 else if (unlikely(ret != 0)) {
196 retval =
197 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
198 goto out_unlock;
199
200 }
201
202 address += PAGE_SIZE;
203 if (unlikely(++page_offset >= page_last))
204 break;
205 }
206
207out_unlock:
208 ttm_bo_unreserve(bo);
209 return retval;
210}
211
212static void ttm_bo_vm_open(struct vm_area_struct *vma)
213{
214 struct ttm_buffer_object *bo =
215 (struct ttm_buffer_object *)vma->vm_private_data;
216
217 (void)ttm_bo_reference(bo);
218}
219
220static void ttm_bo_vm_close(struct vm_area_struct *vma)
221{
222 struct ttm_buffer_object *bo =
223 (struct ttm_buffer_object *)vma->vm_private_data;
224
225 ttm_bo_unref(&bo);
226 vma->vm_private_data = NULL;
227}
228
229static struct vm_operations_struct ttm_bo_vm_ops = {
230 .fault = ttm_bo_vm_fault,
231 .open = ttm_bo_vm_open,
232 .close = ttm_bo_vm_close
233};
234
235int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
236 struct ttm_bo_device *bdev)
237{
238 struct ttm_bo_driver *driver;
239 struct ttm_buffer_object *bo;
240 int ret;
241
242 read_lock(&bdev->vm_lock);
243 bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff,
244 (vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
245 if (likely(bo != NULL))
246 ttm_bo_reference(bo);
247 read_unlock(&bdev->vm_lock);
248
249 if (unlikely(bo == NULL)) {
250 printk(KERN_ERR TTM_PFX
251 "Could not find buffer object to map.\n");
252 return -EINVAL;
253 }
254
255 driver = bo->bdev->driver;
256 if (unlikely(!driver->verify_access)) {
257 ret = -EPERM;
258 goto out_unref;
259 }
260 ret = driver->verify_access(bo, filp);
261 if (unlikely(ret != 0))
262 goto out_unref;
263
264 vma->vm_ops = &ttm_bo_vm_ops;
265
266 /*
267 * Note: We're transferring the bo reference to
268 * vma->vm_private_data here.
269 */
270
271 vma->vm_private_data = bo;
272 vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
273 return 0;
274out_unref:
275 ttm_bo_unref(&bo);
276 return ret;
277}
278EXPORT_SYMBOL(ttm_bo_mmap);
279
280int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
281{
282 if (vma->vm_pgoff != 0)
283 return -EACCES;
284
285 vma->vm_ops = &ttm_bo_vm_ops;
286 vma->vm_private_data = ttm_bo_reference(bo);
287 vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
288 return 0;
289}
290EXPORT_SYMBOL(ttm_fbdev_mmap);
291
292
293ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
294 const char __user *wbuf, char __user *rbuf, size_t count,
295 loff_t *f_pos, bool write)
296{
297 struct ttm_buffer_object *bo;
298 struct ttm_bo_driver *driver;
299 struct ttm_bo_kmap_obj map;
300 unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
301 unsigned long kmap_offset;
302 unsigned long kmap_end;
303 unsigned long kmap_num;
304 size_t io_size;
305 unsigned int page_offset;
306 char *virtual;
307 int ret;
308 bool no_wait = false;
309 bool dummy;
310
311 read_lock(&bdev->vm_lock);
312 bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1);
313 if (likely(bo != NULL))
314 ttm_bo_reference(bo);
315 read_unlock(&bdev->vm_lock);
316
317 if (unlikely(bo == NULL))
318 return -EFAULT;
319
320 driver = bo->bdev->driver;
321 if (unlikely(driver->verify_access)) {
322 ret = -EPERM;
323 goto out_unref;
324 }
325
326 ret = driver->verify_access(bo, filp);
327 if (unlikely(ret != 0))
328 goto out_unref;
329
330 kmap_offset = dev_offset - bo->vm_node->start;
331 if (unlikely(kmap_offset) >= bo->num_pages) {
332 ret = -EFBIG;
333 goto out_unref;
334 }
335
336 page_offset = *f_pos & ~PAGE_MASK;
337 io_size = bo->num_pages - kmap_offset;
338 io_size = (io_size << PAGE_SHIFT) - page_offset;
339 if (count < io_size)
340 io_size = count;
341
342 kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
343 kmap_num = kmap_end - kmap_offset + 1;
344
345 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
346
347 switch (ret) {
348 case 0:
349 break;
350 case -ERESTART:
351 ret = -EINTR;
352 goto out_unref;
353 case -EBUSY:
354 ret = -EAGAIN;
355 goto out_unref;
356 default:
357 goto out_unref;
358 }
359
360 ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
361 if (unlikely(ret != 0)) {
362 ttm_bo_unreserve(bo);
363 goto out_unref;
364 }
365
366 virtual = ttm_kmap_obj_virtual(&map, &dummy);
367 virtual += page_offset;
368
369 if (write)
370 ret = copy_from_user(virtual, wbuf, io_size);
371 else
372 ret = copy_to_user(rbuf, virtual, io_size);
373
374 ttm_bo_kunmap(&map);
375 ttm_bo_unreserve(bo);
376 ttm_bo_unref(&bo);
377
378 if (unlikely(ret != 0))
379 return -EFBIG;
380
381 *f_pos += io_size;
382
383 return io_size;
384out_unref:
385 ttm_bo_unref(&bo);
386 return ret;
387}
388
389ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
390 char __user *rbuf, size_t count, loff_t *f_pos,
391 bool write)
392{
393 struct ttm_bo_kmap_obj map;
394 unsigned long kmap_offset;
395 unsigned long kmap_end;
396 unsigned long kmap_num;
397 size_t io_size;
398 unsigned int page_offset;
399 char *virtual;
400 int ret;
401 bool no_wait = false;
402 bool dummy;
403
404 kmap_offset = (*f_pos >> PAGE_SHIFT);
405 if (unlikely(kmap_offset) >= bo->num_pages)
406 return -EFBIG;
407
408 page_offset = *f_pos & ~PAGE_MASK;
409 io_size = bo->num_pages - kmap_offset;
410 io_size = (io_size << PAGE_SHIFT) - page_offset;
411 if (count < io_size)
412 io_size = count;
413
414 kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
415 kmap_num = kmap_end - kmap_offset + 1;
416
417 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
418
419 switch (ret) {
420 case 0:
421 break;
422 case -ERESTART:
423 return -EINTR;
424 case -EBUSY:
425 return -EAGAIN;
426 default:
427 return ret;
428 }
429
430 ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
431 if (unlikely(ret != 0)) {
432 ttm_bo_unreserve(bo);
433 return ret;
434 }
435
436 virtual = ttm_kmap_obj_virtual(&map, &dummy);
437 virtual += page_offset;
438
439 if (write)
440 ret = copy_from_user(virtual, wbuf, io_size);
441 else
442 ret = copy_to_user(rbuf, virtual, io_size);
443
444 ttm_bo_kunmap(&map);
445 ttm_bo_unreserve(bo);
446 ttm_bo_unref(&bo);
447
448 if (unlikely(ret != 0))
449 return ret;
450
451 *f_pos += io_size;
452
453 return io_size;
454}
diff --git a/drivers/gpu/drm/ttm/ttm_global.c b/drivers/gpu/drm/ttm/ttm_global.c
new file mode 100644
index 000000000000..0b14eb1972b8
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_global.c
@@ -0,0 +1,114 @@
1/**************************************************************************
2 *
3 * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31#include "ttm/ttm_module.h"
32#include <linux/mutex.h>
33#include <linux/slab.h>
34#include <linux/module.h>
35
36struct ttm_global_item {
37 struct mutex mutex;
38 void *object;
39 int refcount;
40};
41
42static struct ttm_global_item glob[TTM_GLOBAL_NUM];
43
44void ttm_global_init(void)
45{
46 int i;
47
48 for (i = 0; i < TTM_GLOBAL_NUM; ++i) {
49 struct ttm_global_item *item = &glob[i];
50 mutex_init(&item->mutex);
51 item->object = NULL;
52 item->refcount = 0;
53 }
54}
55
56void ttm_global_release(void)
57{
58 int i;
59 for (i = 0; i < TTM_GLOBAL_NUM; ++i) {
60 struct ttm_global_item *item = &glob[i];
61 BUG_ON(item->object != NULL);
62 BUG_ON(item->refcount != 0);
63 }
64}
65
66int ttm_global_item_ref(struct ttm_global_reference *ref)
67{
68 int ret;
69 struct ttm_global_item *item = &glob[ref->global_type];
70 void *object;
71
72 mutex_lock(&item->mutex);
73 if (item->refcount == 0) {
74 item->object = kmalloc(ref->size, GFP_KERNEL);
75 if (unlikely(item->object == NULL)) {
76 ret = -ENOMEM;
77 goto out_err;
78 }
79
80 ref->object = item->object;
81 ret = ref->init(ref);
82 if (unlikely(ret != 0))
83 goto out_err;
84
85 ++item->refcount;
86 }
87 ref->object = item->object;
88 object = item->object;
89 mutex_unlock(&item->mutex);
90 return 0;
91out_err:
92 kfree(item->object);
93 mutex_unlock(&item->mutex);
94 item->object = NULL;
95 return ret;
96}
97EXPORT_SYMBOL(ttm_global_item_ref);
98
99void ttm_global_item_unref(struct ttm_global_reference *ref)
100{
101 struct ttm_global_item *item = &glob[ref->global_type];
102
103 mutex_lock(&item->mutex);
104 BUG_ON(item->refcount == 0);
105 BUG_ON(ref->object != item->object);
106 if (--item->refcount == 0) {
107 ref->release(ref);
108 kfree(item->object);
109 item->object = NULL;
110 }
111 mutex_unlock(&item->mutex);
112}
113EXPORT_SYMBOL(ttm_global_item_unref);
114
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
new file mode 100644
index 000000000000..87323d4ff68d
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -0,0 +1,234 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28#include "ttm/ttm_memory.h"
29#include <linux/spinlock.h>
30#include <linux/sched.h>
31#include <linux/wait.h>
32#include <linux/mm.h>
33#include <linux/module.h>
34
35#define TTM_PFX "[TTM] "
36#define TTM_MEMORY_ALLOC_RETRIES 4
37
38/**
39 * At this point we only support a single shrink callback.
40 * Extend this if needed, perhaps using a linked list of callbacks.
41 * Note that this function is reentrant:
42 * many threads may try to swap out at any given time.
43 */
44
45static void ttm_shrink(struct ttm_mem_global *glob, bool from_workqueue,
46 uint64_t extra)
47{
48 int ret;
49 struct ttm_mem_shrink *shrink;
50 uint64_t target;
51 uint64_t total_target;
52
53 spin_lock(&glob->lock);
54 if (glob->shrink == NULL)
55 goto out;
56
57 if (from_workqueue) {
58 target = glob->swap_limit;
59 total_target = glob->total_memory_swap_limit;
60 } else if (capable(CAP_SYS_ADMIN)) {
61 total_target = glob->emer_total_memory;
62 target = glob->emer_memory;
63 } else {
64 total_target = glob->max_total_memory;
65 target = glob->max_memory;
66 }
67
68 total_target = (extra >= total_target) ? 0 : total_target - extra;
69 target = (extra >= target) ? 0 : target - extra;
70
71 while (glob->used_memory > target ||
72 glob->used_total_memory > total_target) {
73 shrink = glob->shrink;
74 spin_unlock(&glob->lock);
75 ret = shrink->do_shrink(shrink);
76 spin_lock(&glob->lock);
77 if (unlikely(ret != 0))
78 goto out;
79 }
80out:
81 spin_unlock(&glob->lock);
82}
83
84static void ttm_shrink_work(struct work_struct *work)
85{
86 struct ttm_mem_global *glob =
87 container_of(work, struct ttm_mem_global, work);
88
89 ttm_shrink(glob, true, 0ULL);
90}
91
92int ttm_mem_global_init(struct ttm_mem_global *glob)
93{
94 struct sysinfo si;
95 uint64_t mem;
96
97 spin_lock_init(&glob->lock);
98 glob->swap_queue = create_singlethread_workqueue("ttm_swap");
99 INIT_WORK(&glob->work, ttm_shrink_work);
100 init_waitqueue_head(&glob->queue);
101
102 si_meminfo(&si);
103
104 mem = si.totalram - si.totalhigh;
105 mem *= si.mem_unit;
106
107 glob->max_memory = mem >> 1;
108 glob->emer_memory = (mem >> 1) + (mem >> 2);
109 glob->swap_limit = glob->max_memory - (mem >> 3);
110 glob->used_memory = 0;
111 glob->used_total_memory = 0;
112 glob->shrink = NULL;
113
114 mem = si.totalram;
115 mem *= si.mem_unit;
116
117 glob->max_total_memory = mem >> 1;
118 glob->emer_total_memory = (mem >> 1) + (mem >> 2);
119
120 glob->total_memory_swap_limit = glob->max_total_memory - (mem >> 3);
121
122 printk(KERN_INFO TTM_PFX "TTM available graphics memory: %llu MiB\n",
123 glob->max_total_memory >> 20);
124 printk(KERN_INFO TTM_PFX "TTM available object memory: %llu MiB\n",
125 glob->max_memory >> 20);
126
127 return 0;
128}
129EXPORT_SYMBOL(ttm_mem_global_init);
130
131void ttm_mem_global_release(struct ttm_mem_global *glob)
132{
133 printk(KERN_INFO TTM_PFX "Used total memory is %llu bytes.\n",
134 (unsigned long long)glob->used_total_memory);
135 flush_workqueue(glob->swap_queue);
136 destroy_workqueue(glob->swap_queue);
137 glob->swap_queue = NULL;
138}
139EXPORT_SYMBOL(ttm_mem_global_release);
140
141static inline void ttm_check_swapping(struct ttm_mem_global *glob)
142{
143 bool needs_swapping;
144
145 spin_lock(&glob->lock);
146 needs_swapping = (glob->used_memory > glob->swap_limit ||
147 glob->used_total_memory >
148 glob->total_memory_swap_limit);
149 spin_unlock(&glob->lock);
150
151 if (unlikely(needs_swapping))
152 (void)queue_work(glob->swap_queue, &glob->work);
153
154}
155
156void ttm_mem_global_free(struct ttm_mem_global *glob,
157 uint64_t amount, bool himem)
158{
159 spin_lock(&glob->lock);
160 glob->used_total_memory -= amount;
161 if (!himem)
162 glob->used_memory -= amount;
163 wake_up_all(&glob->queue);
164 spin_unlock(&glob->lock);
165}
166
167static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
168 uint64_t amount, bool himem, bool reserve)
169{
170 uint64_t limit;
171 uint64_t lomem_limit;
172 int ret = -ENOMEM;
173
174 spin_lock(&glob->lock);
175
176 if (capable(CAP_SYS_ADMIN)) {
177 limit = glob->emer_total_memory;
178 lomem_limit = glob->emer_memory;
179 } else {
180 limit = glob->max_total_memory;
181 lomem_limit = glob->max_memory;
182 }
183
184 if (unlikely(glob->used_total_memory + amount > limit))
185 goto out_unlock;
186 if (unlikely(!himem && glob->used_memory + amount > lomem_limit))
187 goto out_unlock;
188
189 if (reserve) {
190 glob->used_total_memory += amount;
191 if (!himem)
192 glob->used_memory += amount;
193 }
194 ret = 0;
195out_unlock:
196 spin_unlock(&glob->lock);
197 ttm_check_swapping(glob);
198
199 return ret;
200}
201
202int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
203 bool no_wait, bool interruptible, bool himem)
204{
205 int count = TTM_MEMORY_ALLOC_RETRIES;
206
207 while (unlikely(ttm_mem_global_reserve(glob, memory, himem, true)
208 != 0)) {
209 if (no_wait)
210 return -ENOMEM;
211 if (unlikely(count-- == 0))
212 return -ENOMEM;
213 ttm_shrink(glob, false, memory + (memory >> 2) + 16);
214 }
215
216 return 0;
217}
218
219size_t ttm_round_pot(size_t size)
220{
221 if ((size & (size - 1)) == 0)
222 return size;
223 else if (size > PAGE_SIZE)
224 return PAGE_ALIGN(size);
225 else {
226 size_t tmp_size = 4;
227
228 while (tmp_size < size)
229 tmp_size <<= 1;
230
231 return tmp_size;
232 }
233 return 0;
234}
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
new file mode 100644
index 000000000000..59ce8191d584
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -0,0 +1,50 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 * Jerome Glisse
30 */
31#include <linux/module.h>
32#include <ttm/ttm_module.h>
33
34static int __init ttm_init(void)
35{
36 ttm_global_init();
37 return 0;
38}
39
40static void __exit ttm_exit(void)
41{
42 ttm_global_release();
43}
44
45module_init(ttm_init);
46module_exit(ttm_exit);
47
48MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse");
49MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)");
50MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
new file mode 100644
index 000000000000..c27ab3a877ad
--- /dev/null
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -0,0 +1,635 @@
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31#include <linux/version.h>
32#include <linux/vmalloc.h>
33#include <linux/sched.h>
34#include <linux/highmem.h>
35#include <linux/pagemap.h>
36#include <linux/file.h>
37#include <linux/swap.h>
38#include "ttm/ttm_module.h"
39#include "ttm/ttm_bo_driver.h"
40#include "ttm/ttm_placement.h"
41
42static int ttm_tt_swapin(struct ttm_tt *ttm);
43
44#if defined(CONFIG_X86)
45static void ttm_tt_clflush_page(struct page *page)
46{
47 uint8_t *page_virtual;
48 unsigned int i;
49
50 if (unlikely(page == NULL))
51 return;
52
53 page_virtual = kmap_atomic(page, KM_USER0);
54
55 for (i = 0; i < PAGE_SIZE; i += boot_cpu_data.x86_clflush_size)
56 clflush(page_virtual + i);
57
58 kunmap_atomic(page_virtual, KM_USER0);
59}
60
61static void ttm_tt_cache_flush_clflush(struct page *pages[],
62 unsigned long num_pages)
63{
64 unsigned long i;
65
66 mb();
67 for (i = 0; i < num_pages; ++i)
68 ttm_tt_clflush_page(*pages++);
69 mb();
70}
71#else
72static void ttm_tt_ipi_handler(void *null)
73{
74 ;
75}
76#endif
77
78void ttm_tt_cache_flush(struct page *pages[], unsigned long num_pages)
79{
80
81#if defined(CONFIG_X86)
82 if (cpu_has_clflush) {
83 ttm_tt_cache_flush_clflush(pages, num_pages);
84 return;
85 }
86#else
87 if (on_each_cpu(ttm_tt_ipi_handler, NULL, 1) != 0)
88 printk(KERN_ERR TTM_PFX
89 "Timed out waiting for drm cache flush.\n");
90#endif
91}
92
93/**
94 * Allocates storage for pointers to the pages that back the ttm.
95 *
96 * Uses kmalloc if possible. Otherwise falls back to vmalloc.
97 */
98static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
99{
100 unsigned long size = ttm->num_pages * sizeof(*ttm->pages);
101 ttm->pages = NULL;
102
103 if (size <= PAGE_SIZE)
104 ttm->pages = kzalloc(size, GFP_KERNEL);
105
106 if (!ttm->pages) {
107 ttm->pages = vmalloc_user(size);
108 if (ttm->pages)
109 ttm->page_flags |= TTM_PAGE_FLAG_VMALLOC;
110 }
111}
112
113static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
114{
115 if (ttm->page_flags & TTM_PAGE_FLAG_VMALLOC) {
116 vfree(ttm->pages);
117 ttm->page_flags &= ~TTM_PAGE_FLAG_VMALLOC;
118 } else {
119 kfree(ttm->pages);
120 }
121 ttm->pages = NULL;
122}
123
124static struct page *ttm_tt_alloc_page(unsigned page_flags)
125{
126 if (page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
127 return alloc_page(GFP_HIGHUSER | __GFP_ZERO);
128
129 return alloc_page(GFP_HIGHUSER);
130}
131
132static void ttm_tt_free_user_pages(struct ttm_tt *ttm)
133{
134 int write;
135 int dirty;
136 struct page *page;
137 int i;
138 struct ttm_backend *be = ttm->be;
139
140 BUG_ON(!(ttm->page_flags & TTM_PAGE_FLAG_USER));
141 write = ((ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0);
142 dirty = ((ttm->page_flags & TTM_PAGE_FLAG_USER_DIRTY) != 0);
143
144 if (be)
145 be->func->clear(be);
146
147 for (i = 0; i < ttm->num_pages; ++i) {
148 page = ttm->pages[i];
149 if (page == NULL)
150 continue;
151
152 if (page == ttm->dummy_read_page) {
153 BUG_ON(write);
154 continue;
155 }
156
157 if (write && dirty && !PageReserved(page))
158 set_page_dirty_lock(page);
159
160 ttm->pages[i] = NULL;
161 ttm_mem_global_free(ttm->bdev->mem_glob, PAGE_SIZE, false);
162 put_page(page);
163 }
164 ttm->state = tt_unpopulated;
165 ttm->first_himem_page = ttm->num_pages;
166 ttm->last_lomem_page = -1;
167}
168
169static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
170{
171 struct page *p;
172 struct ttm_bo_device *bdev = ttm->bdev;
173 struct ttm_mem_global *mem_glob = bdev->mem_glob;
174 int ret;
175
176 while (NULL == (p = ttm->pages[index])) {
177 p = ttm_tt_alloc_page(ttm->page_flags);
178
179 if (!p)
180 return NULL;
181
182 if (PageHighMem(p)) {
183 ret =
184 ttm_mem_global_alloc(mem_glob, PAGE_SIZE,
185 false, false, true);
186 if (unlikely(ret != 0))
187 goto out_err;
188 ttm->pages[--ttm->first_himem_page] = p;
189 } else {
190 ret =
191 ttm_mem_global_alloc(mem_glob, PAGE_SIZE,
192 false, false, false);
193 if (unlikely(ret != 0))
194 goto out_err;
195 ttm->pages[++ttm->last_lomem_page] = p;
196 }
197 }
198 return p;
199out_err:
200 put_page(p);
201 return NULL;
202}
203
204struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index)
205{
206 int ret;
207
208 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
209 ret = ttm_tt_swapin(ttm);
210 if (unlikely(ret != 0))
211 return NULL;
212 }
213 return __ttm_tt_get_page(ttm, index);
214}
215
216int ttm_tt_populate(struct ttm_tt *ttm)
217{
218 struct page *page;
219 unsigned long i;
220 struct ttm_backend *be;
221 int ret;
222
223 if (ttm->state != tt_unpopulated)
224 return 0;
225
226 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
227 ret = ttm_tt_swapin(ttm);
228 if (unlikely(ret != 0))
229 return ret;
230 }
231
232 be = ttm->be;
233
234 for (i = 0; i < ttm->num_pages; ++i) {
235 page = __ttm_tt_get_page(ttm, i);
236 if (!page)
237 return -ENOMEM;
238 }
239
240 be->func->populate(be, ttm->num_pages, ttm->pages,
241 ttm->dummy_read_page);
242 ttm->state = tt_unbound;
243 return 0;
244}
245
246#ifdef CONFIG_X86
247static inline int ttm_tt_set_page_caching(struct page *p,
248 enum ttm_caching_state c_state)
249{
250 if (PageHighMem(p))
251 return 0;
252
253 switch (c_state) {
254 case tt_cached:
255 return set_pages_wb(p, 1);
256 case tt_wc:
257 return set_memory_wc((unsigned long) page_address(p), 1);
258 default:
259 return set_pages_uc(p, 1);
260 }
261}
262#else /* CONFIG_X86 */
263static inline int ttm_tt_set_page_caching(struct page *p,
264 enum ttm_caching_state c_state)
265{
266 return 0;
267}
268#endif /* CONFIG_X86 */
269
270/*
271 * Change caching policy for the linear kernel map
272 * for range of pages in a ttm.
273 */
274
275static int ttm_tt_set_caching(struct ttm_tt *ttm,
276 enum ttm_caching_state c_state)
277{
278 int i, j;
279 struct page *cur_page;
280 int ret;
281
282 if (ttm->caching_state == c_state)
283 return 0;
284
285 if (c_state != tt_cached) {
286 ret = ttm_tt_populate(ttm);
287 if (unlikely(ret != 0))
288 return ret;
289 }
290
291 if (ttm->caching_state == tt_cached)
292 ttm_tt_cache_flush(ttm->pages, ttm->num_pages);
293
294 for (i = 0; i < ttm->num_pages; ++i) {
295 cur_page = ttm->pages[i];
296 if (likely(cur_page != NULL)) {
297 ret = ttm_tt_set_page_caching(cur_page, c_state);
298 if (unlikely(ret != 0))
299 goto out_err;
300 }
301 }
302
303 ttm->caching_state = c_state;
304
305 return 0;
306
307out_err:
308 for (j = 0; j < i; ++j) {
309 cur_page = ttm->pages[j];
310 if (likely(cur_page != NULL)) {
311 (void)ttm_tt_set_page_caching(cur_page,
312 ttm->caching_state);
313 }
314 }
315
316 return ret;
317}
318
319int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
320{
321 enum ttm_caching_state state;
322
323 if (placement & TTM_PL_FLAG_WC)
324 state = tt_wc;
325 else if (placement & TTM_PL_FLAG_UNCACHED)
326 state = tt_uncached;
327 else
328 state = tt_cached;
329
330 return ttm_tt_set_caching(ttm, state);
331}
332
333static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
334{
335 int i;
336 struct page *cur_page;
337 struct ttm_backend *be = ttm->be;
338
339 if (be)
340 be->func->clear(be);
341 (void)ttm_tt_set_caching(ttm, tt_cached);
342 for (i = 0; i < ttm->num_pages; ++i) {
343 cur_page = ttm->pages[i];
344 ttm->pages[i] = NULL;
345 if (cur_page) {
346 if (page_count(cur_page) != 1)
347 printk(KERN_ERR TTM_PFX
348 "Erroneous page count. "
349 "Leaking pages.\n");
350 ttm_mem_global_free(ttm->bdev->mem_glob, PAGE_SIZE,
351 PageHighMem(cur_page));
352 __free_page(cur_page);
353 }
354 }
355 ttm->state = tt_unpopulated;
356 ttm->first_himem_page = ttm->num_pages;
357 ttm->last_lomem_page = -1;
358}
359
360void ttm_tt_destroy(struct ttm_tt *ttm)
361{
362 struct ttm_backend *be;
363
364 if (unlikely(ttm == NULL))
365 return;
366
367 be = ttm->be;
368 if (likely(be != NULL)) {
369 be->func->destroy(be);
370 ttm->be = NULL;
371 }
372
373 if (likely(ttm->pages != NULL)) {
374 if (ttm->page_flags & TTM_PAGE_FLAG_USER)
375 ttm_tt_free_user_pages(ttm);
376 else
377 ttm_tt_free_alloced_pages(ttm);
378
379 ttm_tt_free_page_directory(ttm);
380 }
381
382 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP) &&
383 ttm->swap_storage)
384 fput(ttm->swap_storage);
385
386 kfree(ttm);
387}
388
389int ttm_tt_set_user(struct ttm_tt *ttm,
390 struct task_struct *tsk,
391 unsigned long start, unsigned long num_pages)
392{
393 struct mm_struct *mm = tsk->mm;
394 int ret;
395 int write = (ttm->page_flags & TTM_PAGE_FLAG_WRITE) != 0;
396 struct ttm_mem_global *mem_glob = ttm->bdev->mem_glob;
397
398 BUG_ON(num_pages != ttm->num_pages);
399 BUG_ON((ttm->page_flags & TTM_PAGE_FLAG_USER) == 0);
400
401 /**
402 * Account user pages as lowmem pages for now.
403 */
404
405 ret = ttm_mem_global_alloc(mem_glob, num_pages * PAGE_SIZE,
406 false, false, false);
407 if (unlikely(ret != 0))
408 return ret;
409
410 down_read(&mm->mmap_sem);
411 ret = get_user_pages(tsk, mm, start, num_pages,
412 write, 0, ttm->pages, NULL);
413 up_read(&mm->mmap_sem);
414
415 if (ret != num_pages && write) {
416 ttm_tt_free_user_pages(ttm);
417 ttm_mem_global_free(mem_glob, num_pages * PAGE_SIZE, false);
418 return -ENOMEM;
419 }
420
421 ttm->tsk = tsk;
422 ttm->start = start;
423 ttm->state = tt_unbound;
424
425 return 0;
426}
427
428struct ttm_tt *ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
429 uint32_t page_flags, struct page *dummy_read_page)
430{
431 struct ttm_bo_driver *bo_driver = bdev->driver;
432 struct ttm_tt *ttm;
433
434 if (!bo_driver)
435 return NULL;
436
437 ttm = kzalloc(sizeof(*ttm), GFP_KERNEL);
438 if (!ttm)
439 return NULL;
440
441 ttm->bdev = bdev;
442
443 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
444 ttm->first_himem_page = ttm->num_pages;
445 ttm->last_lomem_page = -1;
446 ttm->caching_state = tt_cached;
447 ttm->page_flags = page_flags;
448
449 ttm->dummy_read_page = dummy_read_page;
450
451 ttm_tt_alloc_page_directory(ttm);
452 if (!ttm->pages) {
453 ttm_tt_destroy(ttm);
454 printk(KERN_ERR TTM_PFX "Failed allocating page table\n");
455 return NULL;
456 }
457 ttm->be = bo_driver->create_ttm_backend_entry(bdev);
458 if (!ttm->be) {
459 ttm_tt_destroy(ttm);
460 printk(KERN_ERR TTM_PFX "Failed creating ttm backend entry\n");
461 return NULL;
462 }
463 ttm->state = tt_unpopulated;
464 return ttm;
465}
466
467void ttm_tt_unbind(struct ttm_tt *ttm)
468{
469 int ret;
470 struct ttm_backend *be = ttm->be;
471
472 if (ttm->state == tt_bound) {
473 ret = be->func->unbind(be);
474 BUG_ON(ret);
475 ttm->state = tt_unbound;
476 }
477}
478
479int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
480{
481 int ret = 0;
482 struct ttm_backend *be;
483
484 if (!ttm)
485 return -EINVAL;
486
487 if (ttm->state == tt_bound)
488 return 0;
489
490 be = ttm->be;
491
492 ret = ttm_tt_populate(ttm);
493 if (ret)
494 return ret;
495
496 ret = be->func->bind(be, bo_mem);
497 if (ret) {
498 printk(KERN_ERR TTM_PFX "Couldn't bind backend.\n");
499 return ret;
500 }
501
502 ttm->state = tt_bound;
503
504 if (ttm->page_flags & TTM_PAGE_FLAG_USER)
505 ttm->page_flags |= TTM_PAGE_FLAG_USER_DIRTY;
506 return 0;
507}
508EXPORT_SYMBOL(ttm_tt_bind);
509
510static int ttm_tt_swapin(struct ttm_tt *ttm)
511{
512 struct address_space *swap_space;
513 struct file *swap_storage;
514 struct page *from_page;
515 struct page *to_page;
516 void *from_virtual;
517 void *to_virtual;
518 int i;
519 int ret;
520
521 if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
522 ret = ttm_tt_set_user(ttm, ttm->tsk, ttm->start,
523 ttm->num_pages);
524 if (unlikely(ret != 0))
525 return ret;
526
527 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
528 return 0;
529 }
530
531 swap_storage = ttm->swap_storage;
532 BUG_ON(swap_storage == NULL);
533
534 swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
535
536 for (i = 0; i < ttm->num_pages; ++i) {
537 from_page = read_mapping_page(swap_space, i, NULL);
538 if (IS_ERR(from_page))
539 goto out_err;
540 to_page = __ttm_tt_get_page(ttm, i);
541 if (unlikely(to_page == NULL))
542 goto out_err;
543
544 preempt_disable();
545 from_virtual = kmap_atomic(from_page, KM_USER0);
546 to_virtual = kmap_atomic(to_page, KM_USER1);
547 memcpy(to_virtual, from_virtual, PAGE_SIZE);
548 kunmap_atomic(to_virtual, KM_USER1);
549 kunmap_atomic(from_virtual, KM_USER0);
550 preempt_enable();
551 page_cache_release(from_page);
552 }
553
554 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTANT_SWAP))
555 fput(swap_storage);
556 ttm->swap_storage = NULL;
557 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
558
559 return 0;
560out_err:
561 ttm_tt_free_alloced_pages(ttm);
562 return -ENOMEM;
563}
564
565int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
566{
567 struct address_space *swap_space;
568 struct file *swap_storage;
569 struct page *from_page;
570 struct page *to_page;
571 void *from_virtual;
572 void *to_virtual;
573 int i;
574
575 BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
576 BUG_ON(ttm->caching_state != tt_cached);
577
578 /*
579 * For user buffers, just unpin the pages, as there should be
580 * vma references.
581 */
582
583 if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
584 ttm_tt_free_user_pages(ttm);
585 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
586 ttm->swap_storage = NULL;
587 return 0;
588 }
589
590 if (!persistant_swap_storage) {
591 swap_storage = shmem_file_setup("ttm swap",
592 ttm->num_pages << PAGE_SHIFT,
593 0);
594 if (unlikely(IS_ERR(swap_storage))) {
595 printk(KERN_ERR "Failed allocating swap storage.\n");
596 return -ENOMEM;
597 }
598 } else
599 swap_storage = persistant_swap_storage;
600
601 swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
602
603 for (i = 0; i < ttm->num_pages; ++i) {
604 from_page = ttm->pages[i];
605 if (unlikely(from_page == NULL))
606 continue;
607 to_page = read_mapping_page(swap_space, i, NULL);
608 if (unlikely(to_page == NULL))
609 goto out_err;
610
611 preempt_disable();
612 from_virtual = kmap_atomic(from_page, KM_USER0);
613 to_virtual = kmap_atomic(to_page, KM_USER1);
614 memcpy(to_virtual, from_virtual, PAGE_SIZE);
615 kunmap_atomic(to_virtual, KM_USER1);
616 kunmap_atomic(from_virtual, KM_USER0);
617 preempt_enable();
618 set_page_dirty(to_page);
619 mark_page_accessed(to_page);
620 page_cache_release(to_page);
621 }
622
623 ttm_tt_free_alloced_pages(ttm);
624 ttm->swap_storage = swap_storage;
625 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
626 if (persistant_swap_storage)
627 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP;
628
629 return 0;
630out_err:
631 if (!persistant_swap_storage)
632 fput(swap_storage);
633
634 return -ENOMEM;
635}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-10 07:18:44 -0500