diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 365 |
1 files changed, 101 insertions, 264 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 0758c7802e6..16fca1d1799 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c | |||
@@ -34,7 +34,9 @@ | |||
34 | #include <linux/slab.h> | 34 | #include <linux/slab.h> |
35 | #include <linux/swap.h> | 35 | #include <linux/swap.h> |
36 | #include <linux/pci.h> | 36 | #include <linux/pci.h> |
37 | #include <linux/intel-gtt.h> | ||
37 | 38 | ||
39 | static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); | ||
38 | static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); | 40 | static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); |
39 | static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); | 41 | static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); |
40 | static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); | 42 | static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); |
@@ -48,8 +50,6 @@ static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); | |||
48 | static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, | 50 | static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, |
49 | unsigned alignment); | 51 | unsigned alignment); |
50 | static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); | 52 | static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); |
51 | static int i915_gem_evict_something(struct drm_device *dev, int min_size); | ||
52 | static int i915_gem_evict_from_inactive_list(struct drm_device *dev); | ||
53 | static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, | 53 | static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, |
54 | struct drm_i915_gem_pwrite *args, | 54 | struct drm_i915_gem_pwrite *args, |
55 | struct drm_file *file_priv); | 55 | struct drm_file *file_priv); |
@@ -58,6 +58,14 @@ static void i915_gem_free_object_tail(struct drm_gem_object *obj); | |||
58 | static LIST_HEAD(shrink_list); | 58 | static LIST_HEAD(shrink_list); |
59 | static DEFINE_SPINLOCK(shrink_list_lock); | 59 | static DEFINE_SPINLOCK(shrink_list_lock); |
60 | 60 | ||
61 | static inline bool | ||
62 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) | ||
63 | { | ||
64 | return obj_priv->gtt_space && | ||
65 | !obj_priv->active && | ||
66 | obj_priv->pin_count == 0; | ||
67 | } | ||
68 | |||
61 | int i915_gem_do_init(struct drm_device *dev, unsigned long start, | 69 | int i915_gem_do_init(struct drm_device *dev, unsigned long start, |
62 | unsigned long end) | 70 | unsigned long end) |
63 | { | 71 | { |
@@ -128,12 +136,15 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, | |||
128 | return -ENOMEM; | 136 | return -ENOMEM; |
129 | 137 | ||
130 | ret = drm_gem_handle_create(file_priv, obj, &handle); | 138 | ret = drm_gem_handle_create(file_priv, obj, &handle); |
131 | drm_gem_object_unreference_unlocked(obj); | 139 | if (ret) { |
132 | if (ret) | 140 | drm_gem_object_unreference_unlocked(obj); |
133 | return ret; | 141 | return ret; |
142 | } | ||
134 | 143 | ||
135 | args->handle = handle; | 144 | /* Sink the floating reference from kref_init(handlecount) */ |
145 | drm_gem_object_handle_unreference_unlocked(obj); | ||
136 | 146 | ||
147 | args->handle = handle; | ||
137 | return 0; | 148 | return 0; |
138 | } | 149 | } |
139 | 150 | ||
@@ -313,7 +324,8 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj) | |||
313 | if (ret == -ENOMEM) { | 324 | if (ret == -ENOMEM) { |
314 | struct drm_device *dev = obj->dev; | 325 | struct drm_device *dev = obj->dev; |
315 | 326 | ||
316 | ret = i915_gem_evict_something(dev, obj->size); | 327 | ret = i915_gem_evict_something(dev, obj->size, |
328 | i915_gem_get_gtt_alignment(obj)); | ||
317 | if (ret) | 329 | if (ret) |
318 | return ret; | 330 | return ret; |
319 | 331 | ||
@@ -1036,6 +1048,11 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, | |||
1036 | ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); | 1048 | ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); |
1037 | } | 1049 | } |
1038 | 1050 | ||
1051 | |||
1052 | /* Maintain LRU order of "inactive" objects */ | ||
1053 | if (ret == 0 && i915_gem_object_is_inactive(obj_priv)) | ||
1054 | list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); | ||
1055 | |||
1039 | drm_gem_object_unreference(obj); | 1056 | drm_gem_object_unreference(obj); |
1040 | mutex_unlock(&dev->struct_mutex); | 1057 | mutex_unlock(&dev->struct_mutex); |
1041 | return ret; | 1058 | return ret; |
@@ -1137,7 +1154,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
1137 | { | 1154 | { |
1138 | struct drm_gem_object *obj = vma->vm_private_data; | 1155 | struct drm_gem_object *obj = vma->vm_private_data; |
1139 | struct drm_device *dev = obj->dev; | 1156 | struct drm_device *dev = obj->dev; |
1140 | struct drm_i915_private *dev_priv = dev->dev_private; | 1157 | drm_i915_private_t *dev_priv = dev->dev_private; |
1141 | struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); | 1158 | struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); |
1142 | pgoff_t page_offset; | 1159 | pgoff_t page_offset; |
1143 | unsigned long pfn; | 1160 | unsigned long pfn; |
@@ -1155,8 +1172,6 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
1155 | if (ret) | 1172 | if (ret) |
1156 | goto unlock; | 1173 | goto unlock; |
1157 | 1174 | ||
1158 | list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); | ||
1159 | |||
1160 | ret = i915_gem_object_set_to_gtt_domain(obj, write); | 1175 | ret = i915_gem_object_set_to_gtt_domain(obj, write); |
1161 | if (ret) | 1176 | if (ret) |
1162 | goto unlock; | 1177 | goto unlock; |
@@ -1169,6 +1184,9 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
1169 | goto unlock; | 1184 | goto unlock; |
1170 | } | 1185 | } |
1171 | 1186 | ||
1187 | if (i915_gem_object_is_inactive(obj_priv)) | ||
1188 | list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); | ||
1189 | |||
1172 | pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + | 1190 | pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + |
1173 | page_offset; | 1191 | page_offset; |
1174 | 1192 | ||
@@ -1363,7 +1381,6 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, | |||
1363 | struct drm_file *file_priv) | 1381 | struct drm_file *file_priv) |
1364 | { | 1382 | { |
1365 | struct drm_i915_gem_mmap_gtt *args = data; | 1383 | struct drm_i915_gem_mmap_gtt *args = data; |
1366 | struct drm_i915_private *dev_priv = dev->dev_private; | ||
1367 | struct drm_gem_object *obj; | 1384 | struct drm_gem_object *obj; |
1368 | struct drm_i915_gem_object *obj_priv; | 1385 | struct drm_i915_gem_object *obj_priv; |
1369 | int ret; | 1386 | int ret; |
@@ -1409,7 +1426,6 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, | |||
1409 | mutex_unlock(&dev->struct_mutex); | 1426 | mutex_unlock(&dev->struct_mutex); |
1410 | return ret; | 1427 | return ret; |
1411 | } | 1428 | } |
1412 | list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); | ||
1413 | } | 1429 | } |
1414 | 1430 | ||
1415 | drm_gem_object_unreference(obj); | 1431 | drm_gem_object_unreference(obj); |
@@ -1493,9 +1509,16 @@ i915_gem_object_truncate(struct drm_gem_object *obj) | |||
1493 | struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); | 1509 | struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); |
1494 | struct inode *inode; | 1510 | struct inode *inode; |
1495 | 1511 | ||
1512 | /* Our goal here is to return as much of the memory as | ||
1513 | * is possible back to the system as we are called from OOM. | ||
1514 | * To do this we must instruct the shmfs to drop all of its | ||
1515 | * backing pages, *now*. Here we mirror the actions taken | ||
1516 | * when by shmem_delete_inode() to release the backing store. | ||
1517 | */ | ||
1496 | inode = obj->filp->f_path.dentry->d_inode; | 1518 | inode = obj->filp->f_path.dentry->d_inode; |
1497 | if (inode->i_op->truncate) | 1519 | truncate_inode_pages(inode->i_mapping, 0); |
1498 | inode->i_op->truncate (inode); | 1520 | if (inode->i_op->truncate_range) |
1521 | inode->i_op->truncate_range(inode, 0, (loff_t)-1); | ||
1499 | 1522 | ||
1500 | obj_priv->madv = __I915_MADV_PURGED; | 1523 | obj_priv->madv = __I915_MADV_PURGED; |
1501 | } | 1524 | } |
@@ -1887,19 +1910,6 @@ i915_gem_flush(struct drm_device *dev, | |||
1887 | flush_domains); | 1910 | flush_domains); |
1888 | } | 1911 | } |
1889 | 1912 | ||
1890 | static void | ||
1891 | i915_gem_flush_ring(struct drm_device *dev, | ||
1892 | uint32_t invalidate_domains, | ||
1893 | uint32_t flush_domains, | ||
1894 | struct intel_ring_buffer *ring) | ||
1895 | { | ||
1896 | if (flush_domains & I915_GEM_DOMAIN_CPU) | ||
1897 | drm_agp_chipset_flush(dev); | ||
1898 | ring->flush(dev, ring, | ||
1899 | invalidate_domains, | ||
1900 | flush_domains); | ||
1901 | } | ||
1902 | |||
1903 | /** | 1913 | /** |
1904 | * Ensures that all rendering to the object has completed and the object is | 1914 | * Ensures that all rendering to the object has completed and the object is |
1905 | * safe to unbind from the GTT or access from the CPU. | 1915 | * safe to unbind from the GTT or access from the CPU. |
@@ -1973,8 +1983,6 @@ i915_gem_object_unbind(struct drm_gem_object *obj) | |||
1973 | * cause memory corruption through use-after-free. | 1983 | * cause memory corruption through use-after-free. |
1974 | */ | 1984 | */ |
1975 | 1985 | ||
1976 | BUG_ON(obj_priv->active); | ||
1977 | |||
1978 | /* release the fence reg _after_ flushing */ | 1986 | /* release the fence reg _after_ flushing */ |
1979 | if (obj_priv->fence_reg != I915_FENCE_REG_NONE) | 1987 | if (obj_priv->fence_reg != I915_FENCE_REG_NONE) |
1980 | i915_gem_clear_fence_reg(obj); | 1988 | i915_gem_clear_fence_reg(obj); |
@@ -2010,34 +2018,7 @@ i915_gem_object_unbind(struct drm_gem_object *obj) | |||
2010 | return ret; | 2018 | return ret; |
2011 | } | 2019 | } |
2012 | 2020 | ||
2013 | static struct drm_gem_object * | 2021 | int |
2014 | i915_gem_find_inactive_object(struct drm_device *dev, int min_size) | ||
2015 | { | ||
2016 | drm_i915_private_t *dev_priv = dev->dev_private; | ||
2017 | struct drm_i915_gem_object *obj_priv; | ||
2018 | struct drm_gem_object *best = NULL; | ||
2019 | struct drm_gem_object *first = NULL; | ||
2020 | |||
2021 | /* Try to find the smallest clean object */ | ||
2022 | list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) { | ||
2023 | struct drm_gem_object *obj = &obj_priv->base; | ||
2024 | if (obj->size >= min_size) { | ||
2025 | if ((!obj_priv->dirty || | ||
2026 | i915_gem_object_is_purgeable(obj_priv)) && | ||
2027 | (!best || obj->size < best->size)) { | ||
2028 | best = obj; | ||
2029 | if (best->size == min_size) | ||
2030 | return best; | ||
2031 | } | ||
2032 | if (!first) | ||
2033 | first = obj; | ||
2034 | } | ||
2035 | } | ||
2036 | |||
2037 | return best ? best : first; | ||
2038 | } | ||
2039 | |||
2040 | static int | ||
2041 | i915_gpu_idle(struct drm_device *dev) | 2022 | i915_gpu_idle(struct drm_device *dev) |
2042 | { | 2023 | { |
2043 | drm_i915_private_t *dev_priv = dev->dev_private; | 2024 | drm_i915_private_t *dev_priv = dev->dev_private; |
@@ -2078,155 +2059,6 @@ i915_gpu_idle(struct drm_device *dev) | |||
2078 | return ret; | 2059 | return ret; |
2079 | } | 2060 | } |
2080 | 2061 | ||
2081 | static int | ||
2082 | i915_gem_evict_everything(struct drm_device *dev) | ||
2083 | { | ||
2084 | drm_i915_private_t *dev_priv = dev->dev_private; | ||
2085 | int ret; | ||
2086 | bool lists_empty; | ||
2087 | |||
2088 | spin_lock(&dev_priv->mm.active_list_lock); | ||
2089 | lists_empty = (list_empty(&dev_priv->mm.inactive_list) && | ||
2090 | list_empty(&dev_priv->mm.flushing_list) && | ||
2091 | list_empty(&dev_priv->render_ring.active_list) && | ||
2092 | (!HAS_BSD(dev) | ||
2093 | || list_empty(&dev_priv->bsd_ring.active_list))); | ||
2094 | spin_unlock(&dev_priv->mm.active_list_lock); | ||
2095 | |||
2096 | if (lists_empty) | ||
2097 | return -ENOSPC; | ||
2098 | |||
2099 | /* Flush everything (on to the inactive lists) and evict */ | ||
2100 | ret = i915_gpu_idle(dev); | ||
2101 | if (ret) | ||
2102 | return ret; | ||
2103 | |||
2104 | BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); | ||
2105 | |||
2106 | ret = i915_gem_evict_from_inactive_list(dev); | ||
2107 | if (ret) | ||
2108 | return ret; | ||
2109 | |||
2110 | spin_lock(&dev_priv->mm.active_list_lock); | ||
2111 | lists_empty = (list_empty(&dev_priv->mm.inactive_list) && | ||
2112 | list_empty(&dev_priv->mm.flushing_list) && | ||
2113 | list_empty(&dev_priv->render_ring.active_list) && | ||
2114 | (!HAS_BSD(dev) | ||
2115 | || list_empty(&dev_priv->bsd_ring.active_list))); | ||
2116 | spin_unlock(&dev_priv->mm.active_list_lock); | ||
2117 | BUG_ON(!lists_empty); | ||
2118 | |||
2119 | return 0; | ||
2120 | } | ||
2121 | |||
2122 | static int | ||
2123 | i915_gem_evict_something(struct drm_device *dev, int min_size) | ||
2124 | { | ||
2125 | drm_i915_private_t *dev_priv = dev->dev_private; | ||
2126 | struct drm_gem_object *obj; | ||
2127 | int ret; | ||
2128 | |||
2129 | struct intel_ring_buffer *render_ring = &dev_priv->render_ring; | ||
2130 | struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring; | ||
2131 | for (;;) { | ||
2132 | i915_gem_retire_requests(dev); | ||
2133 | |||
2134 | /* If there's an inactive buffer available now, grab it | ||
2135 | * and be done. | ||
2136 | */ | ||
2137 | obj = i915_gem_find_inactive_object(dev, min_size); | ||
2138 | if (obj) { | ||
2139 | struct drm_i915_gem_object *obj_priv; | ||
2140 | |||
2141 | #if WATCH_LRU | ||
2142 | DRM_INFO("%s: evicting %p\n", __func__, obj); | ||
2143 | #endif | ||
2144 | obj_priv = to_intel_bo(obj); | ||
2145 | BUG_ON(obj_priv->pin_count != 0); | ||
2146 | BUG_ON(obj_priv->active); | ||
2147 | |||
2148 | /* Wait on the rendering and unbind the buffer. */ | ||
2149 | return i915_gem_object_unbind(obj); | ||
2150 | } | ||
2151 | |||
2152 | /* If we didn't get anything, but the ring is still processing | ||
2153 | * things, wait for the next to finish and hopefully leave us | ||
2154 | * a buffer to evict. | ||
2155 | */ | ||
2156 | if (!list_empty(&render_ring->request_list)) { | ||
2157 | struct drm_i915_gem_request *request; | ||
2158 | |||
2159 | request = list_first_entry(&render_ring->request_list, | ||
2160 | struct drm_i915_gem_request, | ||
2161 | list); | ||
2162 | |||
2163 | ret = i915_wait_request(dev, | ||
2164 | request->seqno, request->ring); | ||
2165 | if (ret) | ||
2166 | return ret; | ||
2167 | |||
2168 | continue; | ||
2169 | } | ||
2170 | |||
2171 | if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) { | ||
2172 | struct drm_i915_gem_request *request; | ||
2173 | |||
2174 | request = list_first_entry(&bsd_ring->request_list, | ||
2175 | struct drm_i915_gem_request, | ||
2176 | list); | ||
2177 | |||
2178 | ret = i915_wait_request(dev, | ||
2179 | request->seqno, request->ring); | ||
2180 | if (ret) | ||
2181 | return ret; | ||
2182 | |||
2183 | continue; | ||
2184 | } | ||
2185 | |||
2186 | /* If we didn't have anything on the request list but there | ||
2187 | * are buffers awaiting a flush, emit one and try again. | ||
2188 | * When we wait on it, those buffers waiting for that flush | ||
2189 | * will get moved to inactive. | ||
2190 | */ | ||
2191 | if (!list_empty(&dev_priv->mm.flushing_list)) { | ||
2192 | struct drm_i915_gem_object *obj_priv; | ||
2193 | |||
2194 | /* Find an object that we can immediately reuse */ | ||
2195 | list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) { | ||
2196 | obj = &obj_priv->base; | ||
2197 | if (obj->size >= min_size) | ||
2198 | break; | ||
2199 | |||
2200 | obj = NULL; | ||
2201 | } | ||
2202 | |||
2203 | if (obj != NULL) { | ||
2204 | uint32_t seqno; | ||
2205 | |||
2206 | i915_gem_flush_ring(dev, | ||
2207 | obj->write_domain, | ||
2208 | obj->write_domain, | ||
2209 | obj_priv->ring); | ||
2210 | seqno = i915_add_request(dev, NULL, | ||
2211 | obj->write_domain, | ||
2212 | obj_priv->ring); | ||
2213 | if (seqno == 0) | ||
2214 | return -ENOMEM; | ||
2215 | continue; | ||
2216 | } | ||
2217 | } | ||
2218 | |||
2219 | /* If we didn't do any of the above, there's no single buffer | ||
2220 | * large enough to swap out for the new one, so just evict | ||
2221 | * everything and start again. (This should be rare.) | ||
2222 | */ | ||
2223 | if (!list_empty (&dev_priv->mm.inactive_list)) | ||
2224 | return i915_gem_evict_from_inactive_list(dev); | ||
2225 | else | ||
2226 | return i915_gem_evict_everything(dev); | ||
2227 | } | ||
2228 | } | ||
2229 | |||
2230 | int | 2062 | int |
2231 | i915_gem_object_get_pages(struct drm_gem_object *obj, | 2063 | i915_gem_object_get_pages(struct drm_gem_object *obj, |
2232 | gfp_t gfpmask) | 2064 | gfp_t gfpmask) |
@@ -2666,7 +2498,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) | |||
2666 | #if WATCH_LRU | 2498 | #if WATCH_LRU |
2667 | DRM_INFO("%s: GTT full, evicting something\n", __func__); | 2499 | DRM_INFO("%s: GTT full, evicting something\n", __func__); |
2668 | #endif | 2500 | #endif |
2669 | ret = i915_gem_evict_something(dev, obj->size); | 2501 | ret = i915_gem_evict_something(dev, obj->size, alignment); |
2670 | if (ret) | 2502 | if (ret) |
2671 | return ret; | 2503 | return ret; |
2672 | 2504 | ||
@@ -2684,7 +2516,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) | |||
2684 | 2516 | ||
2685 | if (ret == -ENOMEM) { | 2517 | if (ret == -ENOMEM) { |
2686 | /* first try to clear up some space from the GTT */ | 2518 | /* first try to clear up some space from the GTT */ |
2687 | ret = i915_gem_evict_something(dev, obj->size); | 2519 | ret = i915_gem_evict_something(dev, obj->size, |
2520 | alignment); | ||
2688 | if (ret) { | 2521 | if (ret) { |
2689 | /* now try to shrink everyone else */ | 2522 | /* now try to shrink everyone else */ |
2690 | if (gfpmask) { | 2523 | if (gfpmask) { |
@@ -2714,7 +2547,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) | |||
2714 | drm_mm_put_block(obj_priv->gtt_space); | 2547 | drm_mm_put_block(obj_priv->gtt_space); |
2715 | obj_priv->gtt_space = NULL; | 2548 | obj_priv->gtt_space = NULL; |
2716 | 2549 | ||
2717 | ret = i915_gem_evict_something(dev, obj->size); | 2550 | ret = i915_gem_evict_something(dev, obj->size, alignment); |
2718 | if (ret) | 2551 | if (ret) |
2719 | return ret; | 2552 | return ret; |
2720 | 2553 | ||
@@ -2723,6 +2556,9 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) | |||
2723 | atomic_inc(&dev->gtt_count); | 2556 | atomic_inc(&dev->gtt_count); |
2724 | atomic_add(obj->size, &dev->gtt_memory); | 2557 | atomic_add(obj->size, &dev->gtt_memory); |
2725 | 2558 | ||
2559 | /* keep track of bounds object by adding it to the inactive list */ | ||
2560 | list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); | ||
2561 | |||
2726 | /* Assert that the object is not currently in any GPU domain. As it | 2562 | /* Assert that the object is not currently in any GPU domain. As it |
2727 | * wasn't in the GTT, there shouldn't be any way it could have been in | 2563 | * wasn't in the GTT, there shouldn't be any way it could have been in |
2728 | * a GPU cache | 2564 | * a GPU cache |
@@ -3117,6 +2953,7 @@ static void | |||
3117 | i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) | 2953 | i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) |
3118 | { | 2954 | { |
3119 | struct drm_device *dev = obj->dev; | 2955 | struct drm_device *dev = obj->dev; |
2956 | drm_i915_private_t *dev_priv = dev->dev_private; | ||
3120 | struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); | 2957 | struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); |
3121 | uint32_t invalidate_domains = 0; | 2958 | uint32_t invalidate_domains = 0; |
3122 | uint32_t flush_domains = 0; | 2959 | uint32_t flush_domains = 0; |
@@ -3179,6 +3016,13 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) | |||
3179 | obj->pending_write_domain = obj->write_domain; | 3016 | obj->pending_write_domain = obj->write_domain; |
3180 | obj->read_domains = obj->pending_read_domains; | 3017 | obj->read_domains = obj->pending_read_domains; |
3181 | 3018 | ||
3019 | if (flush_domains & I915_GEM_GPU_DOMAINS) { | ||
3020 | if (obj_priv->ring == &dev_priv->render_ring) | ||
3021 | dev_priv->flush_rings |= FLUSH_RENDER_RING; | ||
3022 | else if (obj_priv->ring == &dev_priv->bsd_ring) | ||
3023 | dev_priv->flush_rings |= FLUSH_BSD_RING; | ||
3024 | } | ||
3025 | |||
3182 | dev->invalidate_domains |= invalidate_domains; | 3026 | dev->invalidate_domains |= invalidate_domains; |
3183 | dev->flush_domains |= flush_domains; | 3027 | dev->flush_domains |= flush_domains; |
3184 | #if WATCH_BUF | 3028 | #if WATCH_BUF |
@@ -3718,7 +3562,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, | |||
3718 | ring = &dev_priv->render_ring; | 3562 | ring = &dev_priv->render_ring; |
3719 | } | 3563 | } |
3720 | 3564 | ||
3721 | |||
3722 | if (args->buffer_count < 1) { | 3565 | if (args->buffer_count < 1) { |
3723 | DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); | 3566 | DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); |
3724 | return -EINVAL; | 3567 | return -EINVAL; |
@@ -3746,6 +3589,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, | |||
3746 | if (ret != 0) { | 3589 | if (ret != 0) { |
3747 | DRM_ERROR("copy %d cliprects failed: %d\n", | 3590 | DRM_ERROR("copy %d cliprects failed: %d\n", |
3748 | args->num_cliprects, ret); | 3591 | args->num_cliprects, ret); |
3592 | ret = -EFAULT; | ||
3749 | goto pre_mutex_err; | 3593 | goto pre_mutex_err; |
3750 | } | 3594 | } |
3751 | } | 3595 | } |
@@ -3892,6 +3736,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, | |||
3892 | */ | 3736 | */ |
3893 | dev->invalidate_domains = 0; | 3737 | dev->invalidate_domains = 0; |
3894 | dev->flush_domains = 0; | 3738 | dev->flush_domains = 0; |
3739 | dev_priv->flush_rings = 0; | ||
3895 | 3740 | ||
3896 | for (i = 0; i < args->buffer_count; i++) { | 3741 | for (i = 0; i < args->buffer_count; i++) { |
3897 | struct drm_gem_object *obj = object_list[i]; | 3742 | struct drm_gem_object *obj = object_list[i]; |
@@ -3912,16 +3757,14 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, | |||
3912 | i915_gem_flush(dev, | 3757 | i915_gem_flush(dev, |
3913 | dev->invalidate_domains, | 3758 | dev->invalidate_domains, |
3914 | dev->flush_domains); | 3759 | dev->flush_domains); |
3915 | if (dev->flush_domains & I915_GEM_GPU_DOMAINS) { | 3760 | if (dev_priv->flush_rings & FLUSH_RENDER_RING) |
3916 | (void)i915_add_request(dev, file_priv, | 3761 | (void)i915_add_request(dev, file_priv, |
3917 | dev->flush_domains, | 3762 | dev->flush_domains, |
3918 | &dev_priv->render_ring); | 3763 | &dev_priv->render_ring); |
3919 | 3764 | if (dev_priv->flush_rings & FLUSH_BSD_RING) | |
3920 | if (HAS_BSD(dev)) | 3765 | (void)i915_add_request(dev, file_priv, |
3921 | (void)i915_add_request(dev, file_priv, | 3766 | dev->flush_domains, |
3922 | dev->flush_domains, | 3767 | &dev_priv->bsd_ring); |
3923 | &dev_priv->bsd_ring); | ||
3924 | } | ||
3925 | } | 3768 | } |
3926 | 3769 | ||
3927 | for (i = 0; i < args->buffer_count; i++) { | 3770 | for (i = 0; i < args->buffer_count; i++) { |
@@ -4192,6 +4035,10 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) | |||
4192 | if (alignment == 0) | 4035 | if (alignment == 0) |
4193 | alignment = i915_gem_get_gtt_alignment(obj); | 4036 | alignment = i915_gem_get_gtt_alignment(obj); |
4194 | if (obj_priv->gtt_offset & (alignment - 1)) { | 4037 | if (obj_priv->gtt_offset & (alignment - 1)) { |
4038 | WARN(obj_priv->pin_count, | ||
4039 | "bo is already pinned with incorrect alignment:" | ||
4040 | " offset=%x, req.alignment=%x\n", | ||
4041 | obj_priv->gtt_offset, alignment); | ||
4195 | ret = i915_gem_object_unbind(obj); | 4042 | ret = i915_gem_object_unbind(obj); |
4196 | if (ret) | 4043 | if (ret) |
4197 | return ret; | 4044 | return ret; |
@@ -4213,8 +4060,7 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) | |||
4213 | atomic_inc(&dev->pin_count); | 4060 | atomic_inc(&dev->pin_count); |
4214 | atomic_add(obj->size, &dev->pin_memory); | 4061 | atomic_add(obj->size, &dev->pin_memory); |
4215 | if (!obj_priv->active && | 4062 | if (!obj_priv->active && |
4216 | (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 && | 4063 | (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) |
4217 | !list_empty(&obj_priv->list)) | ||
4218 | list_del_init(&obj_priv->list); | 4064 | list_del_init(&obj_priv->list); |
4219 | } | 4065 | } |
4220 | i915_verify_inactive(dev, __FILE__, __LINE__); | 4066 | i915_verify_inactive(dev, __FILE__, __LINE__); |
@@ -4359,22 +4205,34 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, | |||
4359 | } | 4205 | } |
4360 | 4206 | ||
4361 | mutex_lock(&dev->struct_mutex); | 4207 | mutex_lock(&dev->struct_mutex); |
4362 | /* Update the active list for the hardware's current position. | ||
4363 | * Otherwise this only updates on a delayed timer or when irqs are | ||
4364 | * actually unmasked, and our working set ends up being larger than | ||
4365 | * required. | ||
4366 | */ | ||
4367 | i915_gem_retire_requests(dev); | ||
4368 | 4208 | ||
4369 | obj_priv = to_intel_bo(obj); | 4209 | /* Count all active objects as busy, even if they are currently not used |
4370 | /* Don't count being on the flushing list against the object being | 4210 | * by the gpu. Users of this interface expect objects to eventually |
4371 | * done. Otherwise, a buffer left on the flushing list but not getting | 4211 | * become non-busy without any further actions, therefore emit any |
4372 | * flushed (because nobody's flushing that domain) won't ever return | 4212 | * necessary flushes here. |
4373 | * unbusy and get reused by libdrm's bo cache. The other expected | ||
4374 | * consumer of this interface, OpenGL's occlusion queries, also specs | ||
4375 | * that the objects get unbusy "eventually" without any interference. | ||
4376 | */ | 4213 | */ |
4377 | args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0; | 4214 | obj_priv = to_intel_bo(obj); |
4215 | args->busy = obj_priv->active; | ||
4216 | if (args->busy) { | ||
4217 | /* Unconditionally flush objects, even when the gpu still uses this | ||
4218 | * object. Userspace calling this function indicates that it wants to | ||
4219 | * use this buffer rather sooner than later, so issuing the required | ||
4220 | * flush earlier is beneficial. | ||
4221 | */ | ||
4222 | if (obj->write_domain) { | ||
4223 | i915_gem_flush(dev, 0, obj->write_domain); | ||
4224 | (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring); | ||
4225 | } | ||
4226 | |||
4227 | /* Update the active list for the hardware's current position. | ||
4228 | * Otherwise this only updates on a delayed timer or when irqs | ||
4229 | * are actually unmasked, and our working set ends up being | ||
4230 | * larger than required. | ||
4231 | */ | ||
4232 | i915_gem_retire_requests_ring(dev, obj_priv->ring); | ||
4233 | |||
4234 | args->busy = obj_priv->active; | ||
4235 | } | ||
4378 | 4236 | ||
4379 | drm_gem_object_unreference(obj); | 4237 | drm_gem_object_unreference(obj); |
4380 | mutex_unlock(&dev->struct_mutex); | 4238 | mutex_unlock(&dev->struct_mutex); |
@@ -4514,30 +4372,6 @@ void i915_gem_free_object(struct drm_gem_object *obj) | |||
4514 | i915_gem_free_object_tail(obj); | 4372 | i915_gem_free_object_tail(obj); |
4515 | } | 4373 | } |
4516 | 4374 | ||
4517 | /** Unbinds all inactive objects. */ | ||
4518 | static int | ||
4519 | i915_gem_evict_from_inactive_list(struct drm_device *dev) | ||
4520 | { | ||
4521 | drm_i915_private_t *dev_priv = dev->dev_private; | ||
4522 | |||
4523 | while (!list_empty(&dev_priv->mm.inactive_list)) { | ||
4524 | struct drm_gem_object *obj; | ||
4525 | int ret; | ||
4526 | |||
4527 | obj = &list_first_entry(&dev_priv->mm.inactive_list, | ||
4528 | struct drm_i915_gem_object, | ||
4529 | list)->base; | ||
4530 | |||
4531 | ret = i915_gem_object_unbind(obj); | ||
4532 | if (ret != 0) { | ||
4533 | DRM_ERROR("Error unbinding object: %d\n", ret); | ||
4534 | return ret; | ||
4535 | } | ||
4536 | } | ||
4537 | |||
4538 | return 0; | ||
4539 | } | ||
4540 | |||
4541 | int | 4375 | int |
4542 | i915_gem_idle(struct drm_device *dev) | 4376 | i915_gem_idle(struct drm_device *dev) |
4543 | { | 4377 | { |
@@ -4562,7 +4396,7 @@ i915_gem_idle(struct drm_device *dev) | |||
4562 | 4396 | ||
4563 | /* Under UMS, be paranoid and evict. */ | 4397 | /* Under UMS, be paranoid and evict. */ |
4564 | if (!drm_core_check_feature(dev, DRIVER_MODESET)) { | 4398 | if (!drm_core_check_feature(dev, DRIVER_MODESET)) { |
4565 | ret = i915_gem_evict_from_inactive_list(dev); | 4399 | ret = i915_gem_evict_inactive(dev); |
4566 | if (ret) { | 4400 | if (ret) { |
4567 | mutex_unlock(&dev->struct_mutex); | 4401 | mutex_unlock(&dev->struct_mutex); |
4568 | return ret; | 4402 | return ret; |
@@ -4680,6 +4514,8 @@ i915_gem_init_ringbuffer(struct drm_device *dev) | |||
4680 | goto cleanup_render_ring; | 4514 | goto cleanup_render_ring; |
4681 | } | 4515 | } |
4682 | 4516 | ||
4517 | dev_priv->next_seqno = 1; | ||
4518 | |||
4683 | return 0; | 4519 | return 0; |
4684 | 4520 | ||
4685 | cleanup_render_ring: | 4521 | cleanup_render_ring: |
@@ -4841,7 +4677,7 @@ i915_gem_load(struct drm_device *dev) | |||
4841 | * e.g. for cursor + overlay regs | 4677 | * e.g. for cursor + overlay regs |
4842 | */ | 4678 | */ |
4843 | int i915_gem_init_phys_object(struct drm_device *dev, | 4679 | int i915_gem_init_phys_object(struct drm_device *dev, |
4844 | int id, int size) | 4680 | int id, int size, int align) |
4845 | { | 4681 | { |
4846 | drm_i915_private_t *dev_priv = dev->dev_private; | 4682 | drm_i915_private_t *dev_priv = dev->dev_private; |
4847 | struct drm_i915_gem_phys_object *phys_obj; | 4683 | struct drm_i915_gem_phys_object *phys_obj; |
@@ -4856,7 +4692,7 @@ int i915_gem_init_phys_object(struct drm_device *dev, | |||
4856 | 4692 | ||
4857 | phys_obj->id = id; | 4693 | phys_obj->id = id; |
4858 | 4694 | ||
4859 | phys_obj->handle = drm_pci_alloc(dev, size, 0); | 4695 | phys_obj->handle = drm_pci_alloc(dev, size, align); |
4860 | if (!phys_obj->handle) { | 4696 | if (!phys_obj->handle) { |
4861 | ret = -ENOMEM; | 4697 | ret = -ENOMEM; |
4862 | goto kfree_obj; | 4698 | goto kfree_obj; |
@@ -4938,7 +4774,9 @@ out: | |||
4938 | 4774 | ||
4939 | int | 4775 | int |
4940 | i915_gem_attach_phys_object(struct drm_device *dev, | 4776 | i915_gem_attach_phys_object(struct drm_device *dev, |
4941 | struct drm_gem_object *obj, int id) | 4777 | struct drm_gem_object *obj, |
4778 | int id, | ||
4779 | int align) | ||
4942 | { | 4780 | { |
4943 | drm_i915_private_t *dev_priv = dev->dev_private; | 4781 | drm_i915_private_t *dev_priv = dev->dev_private; |
4944 | struct drm_i915_gem_object *obj_priv; | 4782 | struct drm_i915_gem_object *obj_priv; |
@@ -4957,11 +4795,10 @@ i915_gem_attach_phys_object(struct drm_device *dev, | |||
4957 | i915_gem_detach_phys_object(dev, obj); | 4795 | i915_gem_detach_phys_object(dev, obj); |
4958 | } | 4796 | } |
4959 | 4797 | ||
4960 | |||
4961 | /* create a new object */ | 4798 | /* create a new object */ |
4962 | if (!dev_priv->mm.phys_objs[id - 1]) { | 4799 | if (!dev_priv->mm.phys_objs[id - 1]) { |
4963 | ret = i915_gem_init_phys_object(dev, id, | 4800 | ret = i915_gem_init_phys_object(dev, id, |
4964 | obj->size); | 4801 | obj->size, align); |
4965 | if (ret) { | 4802 | if (ret) { |
4966 | DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); | 4803 | DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); |
4967 | goto out; | 4804 | goto out; |