aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/i915_gem.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c365
1 files changed, 101 insertions, 264 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 0758c7802e6..16fca1d1799 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -34,7 +34,9 @@
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/swap.h> 35#include <linux/swap.h>
36#include <linux/pci.h> 36#include <linux/pci.h>
37#include <linux/intel-gtt.h>
37 38
39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
38static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); 40static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
39static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 41static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
40static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 42static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
@@ -48,8 +50,6 @@ static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
48static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, 50static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
49 unsigned alignment); 51 unsigned alignment);
50static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); 52static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
51static int i915_gem_evict_something(struct drm_device *dev, int min_size);
52static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
53static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 53static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
54 struct drm_i915_gem_pwrite *args, 54 struct drm_i915_gem_pwrite *args,
55 struct drm_file *file_priv); 55 struct drm_file *file_priv);
@@ -58,6 +58,14 @@ static void i915_gem_free_object_tail(struct drm_gem_object *obj);
58static LIST_HEAD(shrink_list); 58static LIST_HEAD(shrink_list);
59static DEFINE_SPINLOCK(shrink_list_lock); 59static DEFINE_SPINLOCK(shrink_list_lock);
60 60
61static inline bool
62i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
63{
64 return obj_priv->gtt_space &&
65 !obj_priv->active &&
66 obj_priv->pin_count == 0;
67}
68
61int i915_gem_do_init(struct drm_device *dev, unsigned long start, 69int i915_gem_do_init(struct drm_device *dev, unsigned long start,
62 unsigned long end) 70 unsigned long end)
63{ 71{
@@ -128,12 +136,15 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
128 return -ENOMEM; 136 return -ENOMEM;
129 137
130 ret = drm_gem_handle_create(file_priv, obj, &handle); 138 ret = drm_gem_handle_create(file_priv, obj, &handle);
131 drm_gem_object_unreference_unlocked(obj); 139 if (ret) {
132 if (ret) 140 drm_gem_object_unreference_unlocked(obj);
133 return ret; 141 return ret;
142 }
134 143
135 args->handle = handle; 144 /* Sink the floating reference from kref_init(handlecount) */
145 drm_gem_object_handle_unreference_unlocked(obj);
136 146
147 args->handle = handle;
137 return 0; 148 return 0;
138} 149}
139 150
@@ -313,7 +324,8 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
313 if (ret == -ENOMEM) { 324 if (ret == -ENOMEM) {
314 struct drm_device *dev = obj->dev; 325 struct drm_device *dev = obj->dev;
315 326
316 ret = i915_gem_evict_something(dev, obj->size); 327 ret = i915_gem_evict_something(dev, obj->size,
328 i915_gem_get_gtt_alignment(obj));
317 if (ret) 329 if (ret)
318 return ret; 330 return ret;
319 331
@@ -1036,6 +1048,11 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1036 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1048 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1037 } 1049 }
1038 1050
1051
1052 /* Maintain LRU order of "inactive" objects */
1053 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1054 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1055
1039 drm_gem_object_unreference(obj); 1056 drm_gem_object_unreference(obj);
1040 mutex_unlock(&dev->struct_mutex); 1057 mutex_unlock(&dev->struct_mutex);
1041 return ret; 1058 return ret;
@@ -1137,7 +1154,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1137{ 1154{
1138 struct drm_gem_object *obj = vma->vm_private_data; 1155 struct drm_gem_object *obj = vma->vm_private_data;
1139 struct drm_device *dev = obj->dev; 1156 struct drm_device *dev = obj->dev;
1140 struct drm_i915_private *dev_priv = dev->dev_private; 1157 drm_i915_private_t *dev_priv = dev->dev_private;
1141 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1158 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1142 pgoff_t page_offset; 1159 pgoff_t page_offset;
1143 unsigned long pfn; 1160 unsigned long pfn;
@@ -1155,8 +1172,6 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1155 if (ret) 1172 if (ret)
1156 goto unlock; 1173 goto unlock;
1157 1174
1158 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1159
1160 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1175 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1161 if (ret) 1176 if (ret)
1162 goto unlock; 1177 goto unlock;
@@ -1169,6 +1184,9 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1169 goto unlock; 1184 goto unlock;
1170 } 1185 }
1171 1186
1187 if (i915_gem_object_is_inactive(obj_priv))
1188 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1189
1172 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1190 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1173 page_offset; 1191 page_offset;
1174 1192
@@ -1363,7 +1381,6 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1363 struct drm_file *file_priv) 1381 struct drm_file *file_priv)
1364{ 1382{
1365 struct drm_i915_gem_mmap_gtt *args = data; 1383 struct drm_i915_gem_mmap_gtt *args = data;
1366 struct drm_i915_private *dev_priv = dev->dev_private;
1367 struct drm_gem_object *obj; 1384 struct drm_gem_object *obj;
1368 struct drm_i915_gem_object *obj_priv; 1385 struct drm_i915_gem_object *obj_priv;
1369 int ret; 1386 int ret;
@@ -1409,7 +1426,6 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1409 mutex_unlock(&dev->struct_mutex); 1426 mutex_unlock(&dev->struct_mutex);
1410 return ret; 1427 return ret;
1411 } 1428 }
1412 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1413 } 1429 }
1414 1430
1415 drm_gem_object_unreference(obj); 1431 drm_gem_object_unreference(obj);
@@ -1493,9 +1509,16 @@ i915_gem_object_truncate(struct drm_gem_object *obj)
1493 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1509 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1494 struct inode *inode; 1510 struct inode *inode;
1495 1511
1512 /* Our goal here is to return as much of the memory as
1513 * is possible back to the system as we are called from OOM.
1514 * To do this we must instruct the shmfs to drop all of its
1515 * backing pages, *now*. Here we mirror the actions taken
1516 * when by shmem_delete_inode() to release the backing store.
1517 */
1496 inode = obj->filp->f_path.dentry->d_inode; 1518 inode = obj->filp->f_path.dentry->d_inode;
1497 if (inode->i_op->truncate) 1519 truncate_inode_pages(inode->i_mapping, 0);
1498 inode->i_op->truncate (inode); 1520 if (inode->i_op->truncate_range)
1521 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1499 1522
1500 obj_priv->madv = __I915_MADV_PURGED; 1523 obj_priv->madv = __I915_MADV_PURGED;
1501} 1524}
@@ -1887,19 +1910,6 @@ i915_gem_flush(struct drm_device *dev,
1887 flush_domains); 1910 flush_domains);
1888} 1911}
1889 1912
1890static void
1891i915_gem_flush_ring(struct drm_device *dev,
1892 uint32_t invalidate_domains,
1893 uint32_t flush_domains,
1894 struct intel_ring_buffer *ring)
1895{
1896 if (flush_domains & I915_GEM_DOMAIN_CPU)
1897 drm_agp_chipset_flush(dev);
1898 ring->flush(dev, ring,
1899 invalidate_domains,
1900 flush_domains);
1901}
1902
1903/** 1913/**
1904 * Ensures that all rendering to the object has completed and the object is 1914 * Ensures that all rendering to the object has completed and the object is
1905 * safe to unbind from the GTT or access from the CPU. 1915 * safe to unbind from the GTT or access from the CPU.
@@ -1973,8 +1983,6 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
1973 * cause memory corruption through use-after-free. 1983 * cause memory corruption through use-after-free.
1974 */ 1984 */
1975 1985
1976 BUG_ON(obj_priv->active);
1977
1978 /* release the fence reg _after_ flushing */ 1986 /* release the fence reg _after_ flushing */
1979 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) 1987 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1980 i915_gem_clear_fence_reg(obj); 1988 i915_gem_clear_fence_reg(obj);
@@ -2010,34 +2018,7 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2010 return ret; 2018 return ret;
2011} 2019}
2012 2020
2013static struct drm_gem_object * 2021int
2014i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2015{
2016 drm_i915_private_t *dev_priv = dev->dev_private;
2017 struct drm_i915_gem_object *obj_priv;
2018 struct drm_gem_object *best = NULL;
2019 struct drm_gem_object *first = NULL;
2020
2021 /* Try to find the smallest clean object */
2022 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2023 struct drm_gem_object *obj = &obj_priv->base;
2024 if (obj->size >= min_size) {
2025 if ((!obj_priv->dirty ||
2026 i915_gem_object_is_purgeable(obj_priv)) &&
2027 (!best || obj->size < best->size)) {
2028 best = obj;
2029 if (best->size == min_size)
2030 return best;
2031 }
2032 if (!first)
2033 first = obj;
2034 }
2035 }
2036
2037 return best ? best : first;
2038}
2039
2040static int
2041i915_gpu_idle(struct drm_device *dev) 2022i915_gpu_idle(struct drm_device *dev)
2042{ 2023{
2043 drm_i915_private_t *dev_priv = dev->dev_private; 2024 drm_i915_private_t *dev_priv = dev->dev_private;
@@ -2078,155 +2059,6 @@ i915_gpu_idle(struct drm_device *dev)
2078 return ret; 2059 return ret;
2079} 2060}
2080 2061
2081static int
2082i915_gem_evict_everything(struct drm_device *dev)
2083{
2084 drm_i915_private_t *dev_priv = dev->dev_private;
2085 int ret;
2086 bool lists_empty;
2087
2088 spin_lock(&dev_priv->mm.active_list_lock);
2089 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2090 list_empty(&dev_priv->mm.flushing_list) &&
2091 list_empty(&dev_priv->render_ring.active_list) &&
2092 (!HAS_BSD(dev)
2093 || list_empty(&dev_priv->bsd_ring.active_list)));
2094 spin_unlock(&dev_priv->mm.active_list_lock);
2095
2096 if (lists_empty)
2097 return -ENOSPC;
2098
2099 /* Flush everything (on to the inactive lists) and evict */
2100 ret = i915_gpu_idle(dev);
2101 if (ret)
2102 return ret;
2103
2104 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2105
2106 ret = i915_gem_evict_from_inactive_list(dev);
2107 if (ret)
2108 return ret;
2109
2110 spin_lock(&dev_priv->mm.active_list_lock);
2111 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2112 list_empty(&dev_priv->mm.flushing_list) &&
2113 list_empty(&dev_priv->render_ring.active_list) &&
2114 (!HAS_BSD(dev)
2115 || list_empty(&dev_priv->bsd_ring.active_list)));
2116 spin_unlock(&dev_priv->mm.active_list_lock);
2117 BUG_ON(!lists_empty);
2118
2119 return 0;
2120}
2121
2122static int
2123i915_gem_evict_something(struct drm_device *dev, int min_size)
2124{
2125 drm_i915_private_t *dev_priv = dev->dev_private;
2126 struct drm_gem_object *obj;
2127 int ret;
2128
2129 struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
2130 struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
2131 for (;;) {
2132 i915_gem_retire_requests(dev);
2133
2134 /* If there's an inactive buffer available now, grab it
2135 * and be done.
2136 */
2137 obj = i915_gem_find_inactive_object(dev, min_size);
2138 if (obj) {
2139 struct drm_i915_gem_object *obj_priv;
2140
2141#if WATCH_LRU
2142 DRM_INFO("%s: evicting %p\n", __func__, obj);
2143#endif
2144 obj_priv = to_intel_bo(obj);
2145 BUG_ON(obj_priv->pin_count != 0);
2146 BUG_ON(obj_priv->active);
2147
2148 /* Wait on the rendering and unbind the buffer. */
2149 return i915_gem_object_unbind(obj);
2150 }
2151
2152 /* If we didn't get anything, but the ring is still processing
2153 * things, wait for the next to finish and hopefully leave us
2154 * a buffer to evict.
2155 */
2156 if (!list_empty(&render_ring->request_list)) {
2157 struct drm_i915_gem_request *request;
2158
2159 request = list_first_entry(&render_ring->request_list,
2160 struct drm_i915_gem_request,
2161 list);
2162
2163 ret = i915_wait_request(dev,
2164 request->seqno, request->ring);
2165 if (ret)
2166 return ret;
2167
2168 continue;
2169 }
2170
2171 if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) {
2172 struct drm_i915_gem_request *request;
2173
2174 request = list_first_entry(&bsd_ring->request_list,
2175 struct drm_i915_gem_request,
2176 list);
2177
2178 ret = i915_wait_request(dev,
2179 request->seqno, request->ring);
2180 if (ret)
2181 return ret;
2182
2183 continue;
2184 }
2185
2186 /* If we didn't have anything on the request list but there
2187 * are buffers awaiting a flush, emit one and try again.
2188 * When we wait on it, those buffers waiting for that flush
2189 * will get moved to inactive.
2190 */
2191 if (!list_empty(&dev_priv->mm.flushing_list)) {
2192 struct drm_i915_gem_object *obj_priv;
2193
2194 /* Find an object that we can immediately reuse */
2195 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2196 obj = &obj_priv->base;
2197 if (obj->size >= min_size)
2198 break;
2199
2200 obj = NULL;
2201 }
2202
2203 if (obj != NULL) {
2204 uint32_t seqno;
2205
2206 i915_gem_flush_ring(dev,
2207 obj->write_domain,
2208 obj->write_domain,
2209 obj_priv->ring);
2210 seqno = i915_add_request(dev, NULL,
2211 obj->write_domain,
2212 obj_priv->ring);
2213 if (seqno == 0)
2214 return -ENOMEM;
2215 continue;
2216 }
2217 }
2218
2219 /* If we didn't do any of the above, there's no single buffer
2220 * large enough to swap out for the new one, so just evict
2221 * everything and start again. (This should be rare.)
2222 */
2223 if (!list_empty (&dev_priv->mm.inactive_list))
2224 return i915_gem_evict_from_inactive_list(dev);
2225 else
2226 return i915_gem_evict_everything(dev);
2227 }
2228}
2229
2230int 2062int
2231i915_gem_object_get_pages(struct drm_gem_object *obj, 2063i915_gem_object_get_pages(struct drm_gem_object *obj,
2232 gfp_t gfpmask) 2064 gfp_t gfpmask)
@@ -2666,7 +2498,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2666#if WATCH_LRU 2498#if WATCH_LRU
2667 DRM_INFO("%s: GTT full, evicting something\n", __func__); 2499 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2668#endif 2500#endif
2669 ret = i915_gem_evict_something(dev, obj->size); 2501 ret = i915_gem_evict_something(dev, obj->size, alignment);
2670 if (ret) 2502 if (ret)
2671 return ret; 2503 return ret;
2672 2504
@@ -2684,7 +2516,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2684 2516
2685 if (ret == -ENOMEM) { 2517 if (ret == -ENOMEM) {
2686 /* first try to clear up some space from the GTT */ 2518 /* first try to clear up some space from the GTT */
2687 ret = i915_gem_evict_something(dev, obj->size); 2519 ret = i915_gem_evict_something(dev, obj->size,
2520 alignment);
2688 if (ret) { 2521 if (ret) {
2689 /* now try to shrink everyone else */ 2522 /* now try to shrink everyone else */
2690 if (gfpmask) { 2523 if (gfpmask) {
@@ -2714,7 +2547,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2714 drm_mm_put_block(obj_priv->gtt_space); 2547 drm_mm_put_block(obj_priv->gtt_space);
2715 obj_priv->gtt_space = NULL; 2548 obj_priv->gtt_space = NULL;
2716 2549
2717 ret = i915_gem_evict_something(dev, obj->size); 2550 ret = i915_gem_evict_something(dev, obj->size, alignment);
2718 if (ret) 2551 if (ret)
2719 return ret; 2552 return ret;
2720 2553
@@ -2723,6 +2556,9 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2723 atomic_inc(&dev->gtt_count); 2556 atomic_inc(&dev->gtt_count);
2724 atomic_add(obj->size, &dev->gtt_memory); 2557 atomic_add(obj->size, &dev->gtt_memory);
2725 2558
2559 /* keep track of bounds object by adding it to the inactive list */
2560 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2561
2726 /* Assert that the object is not currently in any GPU domain. As it 2562 /* Assert that the object is not currently in any GPU domain. As it
2727 * wasn't in the GTT, there shouldn't be any way it could have been in 2563 * wasn't in the GTT, there shouldn't be any way it could have been in
2728 * a GPU cache 2564 * a GPU cache
@@ -3117,6 +2953,7 @@ static void
3117i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) 2953i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3118{ 2954{
3119 struct drm_device *dev = obj->dev; 2955 struct drm_device *dev = obj->dev;
2956 drm_i915_private_t *dev_priv = dev->dev_private;
3120 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2957 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3121 uint32_t invalidate_domains = 0; 2958 uint32_t invalidate_domains = 0;
3122 uint32_t flush_domains = 0; 2959 uint32_t flush_domains = 0;
@@ -3179,6 +3016,13 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3179 obj->pending_write_domain = obj->write_domain; 3016 obj->pending_write_domain = obj->write_domain;
3180 obj->read_domains = obj->pending_read_domains; 3017 obj->read_domains = obj->pending_read_domains;
3181 3018
3019 if (flush_domains & I915_GEM_GPU_DOMAINS) {
3020 if (obj_priv->ring == &dev_priv->render_ring)
3021 dev_priv->flush_rings |= FLUSH_RENDER_RING;
3022 else if (obj_priv->ring == &dev_priv->bsd_ring)
3023 dev_priv->flush_rings |= FLUSH_BSD_RING;
3024 }
3025
3182 dev->invalidate_domains |= invalidate_domains; 3026 dev->invalidate_domains |= invalidate_domains;
3183 dev->flush_domains |= flush_domains; 3027 dev->flush_domains |= flush_domains;
3184#if WATCH_BUF 3028#if WATCH_BUF
@@ -3718,7 +3562,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3718 ring = &dev_priv->render_ring; 3562 ring = &dev_priv->render_ring;
3719 } 3563 }
3720 3564
3721
3722 if (args->buffer_count < 1) { 3565 if (args->buffer_count < 1) {
3723 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); 3566 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3724 return -EINVAL; 3567 return -EINVAL;
@@ -3746,6 +3589,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3746 if (ret != 0) { 3589 if (ret != 0) {
3747 DRM_ERROR("copy %d cliprects failed: %d\n", 3590 DRM_ERROR("copy %d cliprects failed: %d\n",
3748 args->num_cliprects, ret); 3591 args->num_cliprects, ret);
3592 ret = -EFAULT;
3749 goto pre_mutex_err; 3593 goto pre_mutex_err;
3750 } 3594 }
3751 } 3595 }
@@ -3892,6 +3736,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3892 */ 3736 */
3893 dev->invalidate_domains = 0; 3737 dev->invalidate_domains = 0;
3894 dev->flush_domains = 0; 3738 dev->flush_domains = 0;
3739 dev_priv->flush_rings = 0;
3895 3740
3896 for (i = 0; i < args->buffer_count; i++) { 3741 for (i = 0; i < args->buffer_count; i++) {
3897 struct drm_gem_object *obj = object_list[i]; 3742 struct drm_gem_object *obj = object_list[i];
@@ -3912,16 +3757,14 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3912 i915_gem_flush(dev, 3757 i915_gem_flush(dev,
3913 dev->invalidate_domains, 3758 dev->invalidate_domains,
3914 dev->flush_domains); 3759 dev->flush_domains);
3915 if (dev->flush_domains & I915_GEM_GPU_DOMAINS) { 3760 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3916 (void)i915_add_request(dev, file_priv, 3761 (void)i915_add_request(dev, file_priv,
3917 dev->flush_domains, 3762 dev->flush_domains,
3918 &dev_priv->render_ring); 3763 &dev_priv->render_ring);
3919 3764 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3920 if (HAS_BSD(dev)) 3765 (void)i915_add_request(dev, file_priv,
3921 (void)i915_add_request(dev, file_priv, 3766 dev->flush_domains,
3922 dev->flush_domains, 3767 &dev_priv->bsd_ring);
3923 &dev_priv->bsd_ring);
3924 }
3925 } 3768 }
3926 3769
3927 for (i = 0; i < args->buffer_count; i++) { 3770 for (i = 0; i < args->buffer_count; i++) {
@@ -4192,6 +4035,10 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4192 if (alignment == 0) 4035 if (alignment == 0)
4193 alignment = i915_gem_get_gtt_alignment(obj); 4036 alignment = i915_gem_get_gtt_alignment(obj);
4194 if (obj_priv->gtt_offset & (alignment - 1)) { 4037 if (obj_priv->gtt_offset & (alignment - 1)) {
4038 WARN(obj_priv->pin_count,
4039 "bo is already pinned with incorrect alignment:"
4040 " offset=%x, req.alignment=%x\n",
4041 obj_priv->gtt_offset, alignment);
4195 ret = i915_gem_object_unbind(obj); 4042 ret = i915_gem_object_unbind(obj);
4196 if (ret) 4043 if (ret)
4197 return ret; 4044 return ret;
@@ -4213,8 +4060,7 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4213 atomic_inc(&dev->pin_count); 4060 atomic_inc(&dev->pin_count);
4214 atomic_add(obj->size, &dev->pin_memory); 4061 atomic_add(obj->size, &dev->pin_memory);
4215 if (!obj_priv->active && 4062 if (!obj_priv->active &&
4216 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 && 4063 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4217 !list_empty(&obj_priv->list))
4218 list_del_init(&obj_priv->list); 4064 list_del_init(&obj_priv->list);
4219 } 4065 }
4220 i915_verify_inactive(dev, __FILE__, __LINE__); 4066 i915_verify_inactive(dev, __FILE__, __LINE__);
@@ -4359,22 +4205,34 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4359 } 4205 }
4360 4206
4361 mutex_lock(&dev->struct_mutex); 4207 mutex_lock(&dev->struct_mutex);
4362 /* Update the active list for the hardware's current position.
4363 * Otherwise this only updates on a delayed timer or when irqs are
4364 * actually unmasked, and our working set ends up being larger than
4365 * required.
4366 */
4367 i915_gem_retire_requests(dev);
4368 4208
4369 obj_priv = to_intel_bo(obj); 4209 /* Count all active objects as busy, even if they are currently not used
4370 /* Don't count being on the flushing list against the object being 4210 * by the gpu. Users of this interface expect objects to eventually
4371 * done. Otherwise, a buffer left on the flushing list but not getting 4211 * become non-busy without any further actions, therefore emit any
4372 * flushed (because nobody's flushing that domain) won't ever return 4212 * necessary flushes here.
4373 * unbusy and get reused by libdrm's bo cache. The other expected
4374 * consumer of this interface, OpenGL's occlusion queries, also specs
4375 * that the objects get unbusy "eventually" without any interference.
4376 */ 4213 */
4377 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0; 4214 obj_priv = to_intel_bo(obj);
4215 args->busy = obj_priv->active;
4216 if (args->busy) {
4217 /* Unconditionally flush objects, even when the gpu still uses this
4218 * object. Userspace calling this function indicates that it wants to
4219 * use this buffer rather sooner than later, so issuing the required
4220 * flush earlier is beneficial.
4221 */
4222 if (obj->write_domain) {
4223 i915_gem_flush(dev, 0, obj->write_domain);
4224 (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4225 }
4226
4227 /* Update the active list for the hardware's current position.
4228 * Otherwise this only updates on a delayed timer or when irqs
4229 * are actually unmasked, and our working set ends up being
4230 * larger than required.
4231 */
4232 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4233
4234 args->busy = obj_priv->active;
4235 }
4378 4236
4379 drm_gem_object_unreference(obj); 4237 drm_gem_object_unreference(obj);
4380 mutex_unlock(&dev->struct_mutex); 4238 mutex_unlock(&dev->struct_mutex);
@@ -4514,30 +4372,6 @@ void i915_gem_free_object(struct drm_gem_object *obj)
4514 i915_gem_free_object_tail(obj); 4372 i915_gem_free_object_tail(obj);
4515} 4373}
4516 4374
4517/** Unbinds all inactive objects. */
4518static int
4519i915_gem_evict_from_inactive_list(struct drm_device *dev)
4520{
4521 drm_i915_private_t *dev_priv = dev->dev_private;
4522
4523 while (!list_empty(&dev_priv->mm.inactive_list)) {
4524 struct drm_gem_object *obj;
4525 int ret;
4526
4527 obj = &list_first_entry(&dev_priv->mm.inactive_list,
4528 struct drm_i915_gem_object,
4529 list)->base;
4530
4531 ret = i915_gem_object_unbind(obj);
4532 if (ret != 0) {
4533 DRM_ERROR("Error unbinding object: %d\n", ret);
4534 return ret;
4535 }
4536 }
4537
4538 return 0;
4539}
4540
4541int 4375int
4542i915_gem_idle(struct drm_device *dev) 4376i915_gem_idle(struct drm_device *dev)
4543{ 4377{
@@ -4562,7 +4396,7 @@ i915_gem_idle(struct drm_device *dev)
4562 4396
4563 /* Under UMS, be paranoid and evict. */ 4397 /* Under UMS, be paranoid and evict. */
4564 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 4398 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4565 ret = i915_gem_evict_from_inactive_list(dev); 4399 ret = i915_gem_evict_inactive(dev);
4566 if (ret) { 4400 if (ret) {
4567 mutex_unlock(&dev->struct_mutex); 4401 mutex_unlock(&dev->struct_mutex);
4568 return ret; 4402 return ret;
@@ -4680,6 +4514,8 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
4680 goto cleanup_render_ring; 4514 goto cleanup_render_ring;
4681 } 4515 }
4682 4516
4517 dev_priv->next_seqno = 1;
4518
4683 return 0; 4519 return 0;
4684 4520
4685cleanup_render_ring: 4521cleanup_render_ring:
@@ -4841,7 +4677,7 @@ i915_gem_load(struct drm_device *dev)
4841 * e.g. for cursor + overlay regs 4677 * e.g. for cursor + overlay regs
4842 */ 4678 */
4843int i915_gem_init_phys_object(struct drm_device *dev, 4679int i915_gem_init_phys_object(struct drm_device *dev,
4844 int id, int size) 4680 int id, int size, int align)
4845{ 4681{
4846 drm_i915_private_t *dev_priv = dev->dev_private; 4682 drm_i915_private_t *dev_priv = dev->dev_private;
4847 struct drm_i915_gem_phys_object *phys_obj; 4683 struct drm_i915_gem_phys_object *phys_obj;
@@ -4856,7 +4692,7 @@ int i915_gem_init_phys_object(struct drm_device *dev,
4856 4692
4857 phys_obj->id = id; 4693 phys_obj->id = id;
4858 4694
4859 phys_obj->handle = drm_pci_alloc(dev, size, 0); 4695 phys_obj->handle = drm_pci_alloc(dev, size, align);
4860 if (!phys_obj->handle) { 4696 if (!phys_obj->handle) {
4861 ret = -ENOMEM; 4697 ret = -ENOMEM;
4862 goto kfree_obj; 4698 goto kfree_obj;
@@ -4938,7 +4774,9 @@ out:
4938 4774
4939int 4775int
4940i915_gem_attach_phys_object(struct drm_device *dev, 4776i915_gem_attach_phys_object(struct drm_device *dev,
4941 struct drm_gem_object *obj, int id) 4777 struct drm_gem_object *obj,
4778 int id,
4779 int align)
4942{ 4780{
4943 drm_i915_private_t *dev_priv = dev->dev_private; 4781 drm_i915_private_t *dev_priv = dev->dev_private;
4944 struct drm_i915_gem_object *obj_priv; 4782 struct drm_i915_gem_object *obj_priv;
@@ -4957,11 +4795,10 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4957 i915_gem_detach_phys_object(dev, obj); 4795 i915_gem_detach_phys_object(dev, obj);
4958 } 4796 }
4959 4797
4960
4961 /* create a new object */ 4798 /* create a new object */
4962 if (!dev_priv->mm.phys_objs[id - 1]) { 4799 if (!dev_priv->mm.phys_objs[id - 1]) {
4963 ret = i915_gem_init_phys_object(dev, id, 4800 ret = i915_gem_init_phys_object(dev, id,
4964 obj->size); 4801 obj->size, align);
4965 if (ret) { 4802 if (ret) {
4966 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); 4803 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4967 goto out; 4804 goto out;