aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/char/agp/agp.h1
-rw-r--r--drivers/char/agp/compat_ioctl.c1
-rw-r--r--drivers/char/agp/compat_ioctl.h1
-rw-r--r--drivers/char/agp/frontend.c8
-rw-r--r--drivers/char/agp/generic.c27
-rw-r--r--drivers/char/agp/intel-agp.c5
-rw-r--r--drivers/char/agp/intel-agp.h14
-rw-r--r--drivers/char/agp/intel-gtt.c778
-rw-r--r--drivers/gpu/drm/drm_agpsupport.c6
-rw-r--r--drivers/gpu/drm/drm_mm.c40
-rw-r--r--drivers/gpu/drm/i915/Makefile2
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.c417
-rw-r--r--drivers/gpu/drm/i915/i915_dma.c782
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c65
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h596
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c3760
-rw-r--r--drivers/gpu/drm/i915/i915_gem_debug.c23
-rw-r--r--drivers/gpu/drm/i915/i915_gem_evict.c125
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c1310
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c99
-rw-r--r--drivers/gpu/drm/i915/i915_gem_tiling.c139
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c590
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h136
-rw-r--r--drivers/gpu/drm/i915/i915_suspend.c93
-rw-r--r--drivers/gpu/drm/i915/i915_trace.h91
-rw-r--r--drivers/gpu/drm/i915/intel_display.c583
-rw-r--r--drivers/gpu/drm/i915/intel_dp.c3
-rw-r--r--drivers/gpu/drm/i915/intel_drv.h20
-rw-r--r--drivers/gpu/drm/i915/intel_fb.c25
-rw-r--r--drivers/gpu/drm/i915/intel_i2c.c21
-rw-r--r--drivers/gpu/drm/i915/intel_lvds.c43
-rw-r--r--drivers/gpu/drm/i915/intel_opregion.c8
-rw-r--r--drivers/gpu/drm/i915/intel_overlay.c116
-rw-r--r--drivers/gpu/drm/i915/intel_panel.c52
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c853
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.h136
-rw-r--r--drivers/gpu/drm/i915/intel_tv.c14
37 files changed, 5963 insertions, 5020 deletions
diff --git a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h
index 5259065f3c79..3e67ddde9e16 100644
--- a/drivers/char/agp/agp.h
+++ b/drivers/char/agp/agp.h
@@ -120,7 +120,6 @@ struct agp_bridge_driver {
120 void (*agp_destroy_page)(struct page *, int flags); 120 void (*agp_destroy_page)(struct page *, int flags);
121 void (*agp_destroy_pages)(struct agp_memory *); 121 void (*agp_destroy_pages)(struct agp_memory *);
122 int (*agp_type_to_mask_type) (struct agp_bridge_data *, int); 122 int (*agp_type_to_mask_type) (struct agp_bridge_data *, int);
123 void (*chipset_flush)(struct agp_bridge_data *);
124}; 123};
125 124
126struct agp_bridge_data { 125struct agp_bridge_data {
diff --git a/drivers/char/agp/compat_ioctl.c b/drivers/char/agp/compat_ioctl.c
index 9d2c97a69cdd..a48e05b31593 100644
--- a/drivers/char/agp/compat_ioctl.c
+++ b/drivers/char/agp/compat_ioctl.c
@@ -276,7 +276,6 @@ long compat_agp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
276 break; 276 break;
277 277
278 case AGPIOC_CHIPSET_FLUSH32: 278 case AGPIOC_CHIPSET_FLUSH32:
279 ret_val = agpioc_chipset_flush_wrap(curr_priv);
280 break; 279 break;
281 } 280 }
282 281
diff --git a/drivers/char/agp/compat_ioctl.h b/drivers/char/agp/compat_ioctl.h
index 0c9678ac0371..f30e0fd97963 100644
--- a/drivers/char/agp/compat_ioctl.h
+++ b/drivers/char/agp/compat_ioctl.h
@@ -102,6 +102,5 @@ void agp_free_memory_wrap(struct agp_memory *memory);
102struct agp_memory *agp_allocate_memory_wrap(size_t pg_count, u32 type); 102struct agp_memory *agp_allocate_memory_wrap(size_t pg_count, u32 type);
103struct agp_memory *agp_find_mem_by_key(int key); 103struct agp_memory *agp_find_mem_by_key(int key);
104struct agp_client *agp_find_client_by_pid(pid_t id); 104struct agp_client *agp_find_client_by_pid(pid_t id);
105int agpioc_chipset_flush_wrap(struct agp_file_private *priv);
106 105
107#endif /* _AGP_COMPAT_H */ 106#endif /* _AGP_COMPAT_H */
diff --git a/drivers/char/agp/frontend.c b/drivers/char/agp/frontend.c
index 3cb4539a98b2..2e044338753c 100644
--- a/drivers/char/agp/frontend.c
+++ b/drivers/char/agp/frontend.c
@@ -957,13 +957,6 @@ static int agpioc_unbind_wrap(struct agp_file_private *priv, void __user *arg)
957 return agp_unbind_memory(memory); 957 return agp_unbind_memory(memory);
958} 958}
959 959
960int agpioc_chipset_flush_wrap(struct agp_file_private *priv)
961{
962 DBG("");
963 agp_flush_chipset(agp_bridge);
964 return 0;
965}
966
967static long agp_ioctl(struct file *file, 960static long agp_ioctl(struct file *file,
968 unsigned int cmd, unsigned long arg) 961 unsigned int cmd, unsigned long arg)
969{ 962{
@@ -1039,7 +1032,6 @@ static long agp_ioctl(struct file *file,
1039 break; 1032 break;
1040 1033
1041 case AGPIOC_CHIPSET_FLUSH: 1034 case AGPIOC_CHIPSET_FLUSH:
1042 ret_val = agpioc_chipset_flush_wrap(curr_priv);
1043 break; 1035 break;
1044 } 1036 }
1045 1037
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index 4956f1c8f9d5..012cba0d6d96 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -81,13 +81,6 @@ static int agp_get_key(void)
81 return -1; 81 return -1;
82} 82}
83 83
84void agp_flush_chipset(struct agp_bridge_data *bridge)
85{
86 if (bridge->driver->chipset_flush)
87 bridge->driver->chipset_flush(bridge);
88}
89EXPORT_SYMBOL(agp_flush_chipset);
90
91/* 84/*
92 * Use kmalloc if possible for the page list. Otherwise fall back to 85 * Use kmalloc if possible for the page list. Otherwise fall back to
93 * vmalloc. This speeds things up and also saves memory for small AGP 86 * vmalloc. This speeds things up and also saves memory for small AGP
@@ -487,26 +480,6 @@ int agp_unbind_memory(struct agp_memory *curr)
487} 480}
488EXPORT_SYMBOL(agp_unbind_memory); 481EXPORT_SYMBOL(agp_unbind_memory);
489 482
490/**
491 * agp_rebind_emmory - Rewrite the entire GATT, useful on resume
492 */
493int agp_rebind_memory(void)
494{
495 struct agp_memory *curr;
496 int ret_val = 0;
497
498 spin_lock(&agp_bridge->mapped_lock);
499 list_for_each_entry(curr, &agp_bridge->mapped_list, mapped_list) {
500 ret_val = curr->bridge->driver->insert_memory(curr,
501 curr->pg_start,
502 curr->type);
503 if (ret_val != 0)
504 break;
505 }
506 spin_unlock(&agp_bridge->mapped_lock);
507 return ret_val;
508}
509EXPORT_SYMBOL(agp_rebind_memory);
510 483
511/* End - Routines for handling swapping of agp_memory into the GATT */ 484/* End - Routines for handling swapping of agp_memory into the GATT */
512 485
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index e72f49d52202..07e9796fead7 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -828,14 +828,9 @@ static void __devexit agp_intel_remove(struct pci_dev *pdev)
828static int agp_intel_resume(struct pci_dev *pdev) 828static int agp_intel_resume(struct pci_dev *pdev)
829{ 829{
830 struct agp_bridge_data *bridge = pci_get_drvdata(pdev); 830 struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
831 int ret_val;
832 831
833 bridge->driver->configure(); 832 bridge->driver->configure();
834 833
835 ret_val = agp_rebind_memory();
836 if (ret_val != 0)
837 return ret_val;
838
839 return 0; 834 return 0;
840} 835}
841#endif 836#endif
diff --git a/drivers/char/agp/intel-agp.h b/drivers/char/agp/intel-agp.h
index 90539df02504..010e3defd6c3 100644
--- a/drivers/char/agp/intel-agp.h
+++ b/drivers/char/agp/intel-agp.h
@@ -75,6 +75,8 @@
75#define I810_GMS_DISABLE 0x00000000 75#define I810_GMS_DISABLE 0x00000000
76#define I810_PGETBL_CTL 0x2020 76#define I810_PGETBL_CTL 0x2020
77#define I810_PGETBL_ENABLED 0x00000001 77#define I810_PGETBL_ENABLED 0x00000001
78/* Note: PGETBL_CTL2 has a different offset on G33. */
79#define I965_PGETBL_CTL2 0x20c4
78#define I965_PGETBL_SIZE_MASK 0x0000000e 80#define I965_PGETBL_SIZE_MASK 0x0000000e
79#define I965_PGETBL_SIZE_512KB (0 << 1) 81#define I965_PGETBL_SIZE_512KB (0 << 1)
80#define I965_PGETBL_SIZE_256KB (1 << 1) 82#define I965_PGETBL_SIZE_256KB (1 << 1)
@@ -82,9 +84,15 @@
82#define I965_PGETBL_SIZE_1MB (3 << 1) 84#define I965_PGETBL_SIZE_1MB (3 << 1)
83#define I965_PGETBL_SIZE_2MB (4 << 1) 85#define I965_PGETBL_SIZE_2MB (4 << 1)
84#define I965_PGETBL_SIZE_1_5MB (5 << 1) 86#define I965_PGETBL_SIZE_1_5MB (5 << 1)
85#define G33_PGETBL_SIZE_MASK (3 << 8) 87#define G33_GMCH_SIZE_MASK (3 << 8)
86#define G33_PGETBL_SIZE_1M (1 << 8) 88#define G33_GMCH_SIZE_1M (1 << 8)
87#define G33_PGETBL_SIZE_2M (2 << 8) 89#define G33_GMCH_SIZE_2M (2 << 8)
90#define G4x_GMCH_SIZE_MASK (0xf << 8)
91#define G4x_GMCH_SIZE_1M (0x1 << 8)
92#define G4x_GMCH_SIZE_2M (0x3 << 8)
93#define G4x_GMCH_SIZE_VT_1M (0x9 << 8)
94#define G4x_GMCH_SIZE_VT_1_5M (0xa << 8)
95#define G4x_GMCH_SIZE_VT_2M (0xc << 8)
88 96
89#define I810_DRAM_CTL 0x3000 97#define I810_DRAM_CTL 0x3000
90#define I810_DRAM_ROW_0 0x00000001 98#define I810_DRAM_ROW_0 0x00000001
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index 29ac6d499fa6..356f73e0d17e 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -24,7 +24,6 @@
24#include <asm/smp.h> 24#include <asm/smp.h>
25#include "agp.h" 25#include "agp.h"
26#include "intel-agp.h" 26#include "intel-agp.h"
27#include <linux/intel-gtt.h>
28#include <drm/intel-gtt.h> 27#include <drm/intel-gtt.h>
29 28
30/* 29/*
@@ -39,40 +38,12 @@
39#define USE_PCI_DMA_API 0 38#define USE_PCI_DMA_API 0
40#endif 39#endif
41 40
42/* Max amount of stolen space, anything above will be returned to Linux */
43int intel_max_stolen = 32 * 1024 * 1024;
44
45static const struct aper_size_info_fixed intel_i810_sizes[] =
46{
47 {64, 16384, 4},
48 /* The 32M mode still requires a 64k gatt */
49 {32, 8192, 4}
50};
51
52#define AGP_DCACHE_MEMORY 1
53#define AGP_PHYS_MEMORY 2
54#define INTEL_AGP_CACHED_MEMORY 3
55
56static struct gatt_mask intel_i810_masks[] =
57{
58 {.mask = I810_PTE_VALID, .type = 0},
59 {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
60 {.mask = I810_PTE_VALID, .type = 0},
61 {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
62 .type = INTEL_AGP_CACHED_MEMORY}
63};
64
65#define INTEL_AGP_UNCACHED_MEMORY 0
66#define INTEL_AGP_CACHED_MEMORY_LLC 1
67#define INTEL_AGP_CACHED_MEMORY_LLC_GFDT 2
68#define INTEL_AGP_CACHED_MEMORY_LLC_MLC 3
69#define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT 4
70
71struct intel_gtt_driver { 41struct intel_gtt_driver {
72 unsigned int gen : 8; 42 unsigned int gen : 8;
73 unsigned int is_g33 : 1; 43 unsigned int is_g33 : 1;
74 unsigned int is_pineview : 1; 44 unsigned int is_pineview : 1;
75 unsigned int is_ironlake : 1; 45 unsigned int is_ironlake : 1;
46 unsigned int has_pgtbl_enable : 1;
76 unsigned int dma_mask_size : 8; 47 unsigned int dma_mask_size : 8;
77 /* Chipset specific GTT setup */ 48 /* Chipset specific GTT setup */
78 int (*setup)(void); 49 int (*setup)(void);
@@ -95,13 +66,14 @@ static struct _intel_private {
95 u8 __iomem *registers; 66 u8 __iomem *registers;
96 phys_addr_t gtt_bus_addr; 67 phys_addr_t gtt_bus_addr;
97 phys_addr_t gma_bus_addr; 68 phys_addr_t gma_bus_addr;
98 phys_addr_t pte_bus_addr; 69 u32 PGETBL_save;
99 u32 __iomem *gtt; /* I915G */ 70 u32 __iomem *gtt; /* I915G */
100 int num_dcache_entries; 71 int num_dcache_entries;
101 union { 72 union {
102 void __iomem *i9xx_flush_page; 73 void __iomem *i9xx_flush_page;
103 void *i8xx_flush_page; 74 void *i8xx_flush_page;
104 }; 75 };
76 char *i81x_gtt_table;
105 struct page *i8xx_page; 77 struct page *i8xx_page;
106 struct resource ifp_resource; 78 struct resource ifp_resource;
107 int resource_valid; 79 int resource_valid;
@@ -113,42 +85,31 @@ static struct _intel_private {
113#define IS_G33 intel_private.driver->is_g33 85#define IS_G33 intel_private.driver->is_g33
114#define IS_PINEVIEW intel_private.driver->is_pineview 86#define IS_PINEVIEW intel_private.driver->is_pineview
115#define IS_IRONLAKE intel_private.driver->is_ironlake 87#define IS_IRONLAKE intel_private.driver->is_ironlake
88#define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable
116 89
117static void intel_agp_free_sglist(struct agp_memory *mem) 90int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
118{ 91 struct scatterlist **sg_list, int *num_sg)
119 struct sg_table st;
120
121 st.sgl = mem->sg_list;
122 st.orig_nents = st.nents = mem->page_count;
123
124 sg_free_table(&st);
125
126 mem->sg_list = NULL;
127 mem->num_sg = 0;
128}
129
130static int intel_agp_map_memory(struct agp_memory *mem)
131{ 92{
132 struct sg_table st; 93 struct sg_table st;
133 struct scatterlist *sg; 94 struct scatterlist *sg;
134 int i; 95 int i;
135 96
136 if (mem->sg_list) 97 if (*sg_list)
137 return 0; /* already mapped (for e.g. resume */ 98 return 0; /* already mapped (for e.g. resume */
138 99
139 DBG("try mapping %lu pages\n", (unsigned long)mem->page_count); 100 DBG("try mapping %lu pages\n", (unsigned long)num_entries);
140 101
141 if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL)) 102 if (sg_alloc_table(&st, num_entries, GFP_KERNEL))
142 goto err; 103 goto err;
143 104
144 mem->sg_list = sg = st.sgl; 105 *sg_list = sg = st.sgl;
145 106
146 for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg)) 107 for (i = 0 ; i < num_entries; i++, sg = sg_next(sg))
147 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0); 108 sg_set_page(sg, pages[i], PAGE_SIZE, 0);
148 109
149 mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list, 110 *num_sg = pci_map_sg(intel_private.pcidev, *sg_list,
150 mem->page_count, PCI_DMA_BIDIRECTIONAL); 111 num_entries, PCI_DMA_BIDIRECTIONAL);
151 if (unlikely(!mem->num_sg)) 112 if (unlikely(!*num_sg))
152 goto err; 113 goto err;
153 114
154 return 0; 115 return 0;
@@ -157,90 +118,22 @@ err:
157 sg_free_table(&st); 118 sg_free_table(&st);
158 return -ENOMEM; 119 return -ENOMEM;
159} 120}
121EXPORT_SYMBOL(intel_gtt_map_memory);
160 122
161static void intel_agp_unmap_memory(struct agp_memory *mem) 123void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
162{ 124{
125 struct sg_table st;
163 DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count); 126 DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
164 127
165 pci_unmap_sg(intel_private.pcidev, mem->sg_list, 128 pci_unmap_sg(intel_private.pcidev, sg_list,
166 mem->page_count, PCI_DMA_BIDIRECTIONAL); 129 num_sg, PCI_DMA_BIDIRECTIONAL);
167 intel_agp_free_sglist(mem);
168}
169
170static int intel_i810_fetch_size(void)
171{
172 u32 smram_miscc;
173 struct aper_size_info_fixed *values;
174
175 pci_read_config_dword(intel_private.bridge_dev,
176 I810_SMRAM_MISCC, &smram_miscc);
177 values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
178
179 if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
180 dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
181 return 0;
182 }
183 if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
184 agp_bridge->current_size = (void *) (values + 1);
185 agp_bridge->aperture_size_idx = 1;
186 return values[1].size;
187 } else {
188 agp_bridge->current_size = (void *) (values);
189 agp_bridge->aperture_size_idx = 0;
190 return values[0].size;
191 }
192
193 return 0;
194}
195
196static int intel_i810_configure(void)
197{
198 struct aper_size_info_fixed *current_size;
199 u32 temp;
200 int i;
201
202 current_size = A_SIZE_FIX(agp_bridge->current_size);
203 130
204 if (!intel_private.registers) { 131 st.sgl = sg_list;
205 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp); 132 st.orig_nents = st.nents = num_sg;
206 temp &= 0xfff80000;
207 133
208 intel_private.registers = ioremap(temp, 128 * 4096); 134 sg_free_table(&st);
209 if (!intel_private.registers) {
210 dev_err(&intel_private.pcidev->dev,
211 "can't remap memory\n");
212 return -ENOMEM;
213 }
214 }
215
216 if ((readl(intel_private.registers+I810_DRAM_CTL)
217 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
218 /* This will need to be dynamically assigned */
219 dev_info(&intel_private.pcidev->dev,
220 "detected 4MB dedicated video ram\n");
221 intel_private.num_dcache_entries = 1024;
222 }
223 pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
224 agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
225 writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
226 readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
227
228 if (agp_bridge->driver->needs_scratch_page) {
229 for (i = 0; i < current_size->num_entries; i++) {
230 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
231 }
232 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
233 }
234 global_cache_flush();
235 return 0;
236}
237
238static void intel_i810_cleanup(void)
239{
240 writel(0, intel_private.registers+I810_PGETBL_CTL);
241 readl(intel_private.registers); /* PCI Posting. */
242 iounmap(intel_private.registers);
243} 135}
136EXPORT_SYMBOL(intel_gtt_unmap_memory);
244 137
245static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode) 138static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
246{ 139{
@@ -277,80 +170,64 @@ static void i8xx_destroy_pages(struct page *page)
277 atomic_dec(&agp_bridge->current_memory_agp); 170 atomic_dec(&agp_bridge->current_memory_agp);
278} 171}
279 172
280static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start, 173#define I810_GTT_ORDER 4
281 int type) 174static int i810_setup(void)
282{ 175{
283 int i, j, num_entries; 176 u32 reg_addr;
284 void *temp; 177 char *gtt_table;
285 int ret = -EINVAL;
286 int mask_type;
287
288 if (mem->page_count == 0)
289 goto out;
290
291 temp = agp_bridge->current_size;
292 num_entries = A_SIZE_FIX(temp)->num_entries;
293 178
294 if ((pg_start + mem->page_count) > num_entries) 179 /* i81x does not preallocate the gtt. It's always 64kb in size. */
295 goto out_err; 180 gtt_table = alloc_gatt_pages(I810_GTT_ORDER);
181 if (gtt_table == NULL)
182 return -ENOMEM;
183 intel_private.i81x_gtt_table = gtt_table;
296 184
185 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
186 reg_addr &= 0xfff80000;
297 187
298 for (j = pg_start; j < (pg_start + mem->page_count); j++) { 188 intel_private.registers = ioremap(reg_addr, KB(64));
299 if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) { 189 if (!intel_private.registers)
300 ret = -EBUSY; 190 return -ENOMEM;
301 goto out_err;
302 }
303 }
304 191
305 if (type != mem->type) 192 writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED,
306 goto out_err; 193 intel_private.registers+I810_PGETBL_CTL);
307 194
308 mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type); 195 intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
309 196
310 switch (mask_type) { 197 if ((readl(intel_private.registers+I810_DRAM_CTL)
311 case AGP_DCACHE_MEMORY: 198 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
312 if (!mem->is_flushed) 199 dev_info(&intel_private.pcidev->dev,
313 global_cache_flush(); 200 "detected 4MB dedicated video ram\n");
314 for (i = pg_start; i < (pg_start + mem->page_count); i++) { 201 intel_private.num_dcache_entries = 1024;
315 writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
316 intel_private.registers+I810_PTE_BASE+(i*4));
317 }
318 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
319 break;
320 case AGP_PHYS_MEMORY:
321 case AGP_NORMAL_MEMORY:
322 if (!mem->is_flushed)
323 global_cache_flush();
324 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
325 writel(agp_bridge->driver->mask_memory(agp_bridge,
326 page_to_phys(mem->pages[i]), mask_type),
327 intel_private.registers+I810_PTE_BASE+(j*4));
328 }
329 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
330 break;
331 default:
332 goto out_err;
333 } 202 }
334 203
335out: 204 return 0;
336 ret = 0;
337out_err:
338 mem->is_flushed = true;
339 return ret;
340} 205}
341 206
342static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start, 207static void i810_cleanup(void)
343 int type) 208{
209 writel(0, intel_private.registers+I810_PGETBL_CTL);
210 free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER);
211}
212
213static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start,
214 int type)
344{ 215{
345 int i; 216 int i;
346 217
347 if (mem->page_count == 0) 218 if ((pg_start + mem->page_count)
348 return 0; 219 > intel_private.num_dcache_entries)
220 return -EINVAL;
221
222 if (!mem->is_flushed)
223 global_cache_flush();
349 224
350 for (i = pg_start; i < (mem->page_count + pg_start); i++) { 225 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
351 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4)); 226 dma_addr_t addr = i << PAGE_SHIFT;
227 intel_private.driver->write_entry(addr,
228 i, type);
352 } 229 }
353 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); 230 readl(intel_private.gtt+i-1);
354 231
355 return 0; 232 return 0;
356} 233}
@@ -397,29 +274,6 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
397 return new; 274 return new;
398} 275}
399 276
400static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
401{
402 struct agp_memory *new;
403
404 if (type == AGP_DCACHE_MEMORY) {
405 if (pg_count != intel_private.num_dcache_entries)
406 return NULL;
407
408 new = agp_create_memory(1);
409 if (new == NULL)
410 return NULL;
411
412 new->type = AGP_DCACHE_MEMORY;
413 new->page_count = pg_count;
414 new->num_scratch_pages = 0;
415 agp_free_page_array(new);
416 return new;
417 }
418 if (type == AGP_PHYS_MEMORY)
419 return alloc_agpphysmem_i8xx(pg_count, type);
420 return NULL;
421}
422
423static void intel_i810_free_by_type(struct agp_memory *curr) 277static void intel_i810_free_by_type(struct agp_memory *curr)
424{ 278{
425 agp_free_key(curr->key); 279 agp_free_key(curr->key);
@@ -437,13 +291,6 @@ static void intel_i810_free_by_type(struct agp_memory *curr)
437 kfree(curr); 291 kfree(curr);
438} 292}
439 293
440static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
441 dma_addr_t addr, int type)
442{
443 /* Type checking must be done elsewhere */
444 return addr | bridge->driver->masks[type].mask;
445}
446
447static int intel_gtt_setup_scratch_page(void) 294static int intel_gtt_setup_scratch_page(void)
448{ 295{
449 struct page *page; 296 struct page *page;
@@ -455,7 +302,7 @@ static int intel_gtt_setup_scratch_page(void)
455 get_page(page); 302 get_page(page);
456 set_pages_uc(page, 1); 303 set_pages_uc(page, 1);
457 304
458 if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) { 305 if (intel_private.base.needs_dmar) {
459 dma_addr = pci_map_page(intel_private.pcidev, page, 0, 306 dma_addr = pci_map_page(intel_private.pcidev, page, 0,
460 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 307 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
461 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr)) 308 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
@@ -470,34 +317,45 @@ static int intel_gtt_setup_scratch_page(void)
470 return 0; 317 return 0;
471} 318}
472 319
473static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = { 320static void i810_write_entry(dma_addr_t addr, unsigned int entry,
321 unsigned int flags)
322{
323 u32 pte_flags = I810_PTE_VALID;
324
325 switch (flags) {
326 case AGP_DCACHE_MEMORY:
327 pte_flags |= I810_PTE_LOCAL;
328 break;
329 case AGP_USER_CACHED_MEMORY:
330 pte_flags |= I830_PTE_SYSTEM_CACHED;
331 break;
332 }
333
334 writel(addr | pte_flags, intel_private.gtt + entry);
335}
336
337static const struct aper_size_info_fixed intel_fake_agp_sizes[] = {
338 {32, 8192, 3},
339 {64, 16384, 4},
474 {128, 32768, 5}, 340 {128, 32768, 5},
475 /* The 64M mode still requires a 128k gatt */
476 {64, 16384, 5},
477 {256, 65536, 6}, 341 {256, 65536, 6},
478 {512, 131072, 7}, 342 {512, 131072, 7},
479}; 343};
480 344
481static unsigned int intel_gtt_stolen_entries(void) 345static unsigned int intel_gtt_stolen_size(void)
482{ 346{
483 u16 gmch_ctrl; 347 u16 gmch_ctrl;
484 u8 rdct; 348 u8 rdct;
485 int local = 0; 349 int local = 0;
486 static const int ddt[4] = { 0, 16, 32, 64 }; 350 static const int ddt[4] = { 0, 16, 32, 64 };
487 unsigned int overhead_entries, stolen_entries;
488 unsigned int stolen_size = 0; 351 unsigned int stolen_size = 0;
489 352
353 if (INTEL_GTT_GEN == 1)
354 return 0; /* no stolen mem on i81x */
355
490 pci_read_config_word(intel_private.bridge_dev, 356 pci_read_config_word(intel_private.bridge_dev,
491 I830_GMCH_CTRL, &gmch_ctrl); 357 I830_GMCH_CTRL, &gmch_ctrl);
492 358
493 if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
494 overhead_entries = 0;
495 else
496 overhead_entries = intel_private.base.gtt_mappable_entries
497 / 1024;
498
499 overhead_entries += 1; /* BIOS popup */
500
501 if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB || 359 if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
502 intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) { 360 intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
503 switch (gmch_ctrl & I830_GMCH_GMS_MASK) { 361 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
@@ -623,12 +481,7 @@ static unsigned int intel_gtt_stolen_entries(void)
623 } 481 }
624 } 482 }
625 483
626 if (!local && stolen_size > intel_max_stolen) { 484 if (stolen_size > 0) {
627 dev_info(&intel_private.bridge_dev->dev,
628 "detected %dK stolen memory, trimming to %dK\n",
629 stolen_size / KB(1), intel_max_stolen / KB(1));
630 stolen_size = intel_max_stolen;
631 } else if (stolen_size > 0) {
632 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n", 485 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
633 stolen_size / KB(1), local ? "local" : "stolen"); 486 stolen_size / KB(1), local ? "local" : "stolen");
634 } else { 487 } else {
@@ -637,46 +490,88 @@ static unsigned int intel_gtt_stolen_entries(void)
637 stolen_size = 0; 490 stolen_size = 0;
638 } 491 }
639 492
640 stolen_entries = stolen_size/KB(4) - overhead_entries; 493 return stolen_size;
494}
641 495
642 return stolen_entries; 496static void i965_adjust_pgetbl_size(unsigned int size_flag)
497{
498 u32 pgetbl_ctl, pgetbl_ctl2;
499
500 /* ensure that ppgtt is disabled */
501 pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2);
502 pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
503 writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2);
504
505 /* write the new ggtt size */
506 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
507 pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
508 pgetbl_ctl |= size_flag;
509 writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL);
643} 510}
644 511
645static unsigned int intel_gtt_total_entries(void) 512static unsigned int i965_gtt_total_entries(void)
646{ 513{
647 int size; 514 int size;
515 u32 pgetbl_ctl;
516 u16 gmch_ctl;
648 517
649 if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) { 518 pci_read_config_word(intel_private.bridge_dev,
650 u32 pgetbl_ctl; 519 I830_GMCH_CTRL, &gmch_ctl);
651 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
652 520
653 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { 521 if (INTEL_GTT_GEN == 5) {
654 case I965_PGETBL_SIZE_128KB: 522 switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
655 size = KB(128); 523 case G4x_GMCH_SIZE_1M:
656 break; 524 case G4x_GMCH_SIZE_VT_1M:
657 case I965_PGETBL_SIZE_256KB: 525 i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB);
658 size = KB(256);
659 break;
660 case I965_PGETBL_SIZE_512KB:
661 size = KB(512);
662 break;
663 case I965_PGETBL_SIZE_1MB:
664 size = KB(1024);
665 break; 526 break;
666 case I965_PGETBL_SIZE_2MB: 527 case G4x_GMCH_SIZE_VT_1_5M:
667 size = KB(2048); 528 i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB);
668 break; 529 break;
669 case I965_PGETBL_SIZE_1_5MB: 530 case G4x_GMCH_SIZE_2M:
670 size = KB(1024 + 512); 531 case G4x_GMCH_SIZE_VT_2M:
532 i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB);
671 break; 533 break;
672 default:
673 dev_info(&intel_private.pcidev->dev,
674 "unknown page table size, assuming 512KB\n");
675 size = KB(512);
676 } 534 }
535 }
677 536
678 return size/4; 537 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
679 } else if (INTEL_GTT_GEN == 6) { 538
539 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
540 case I965_PGETBL_SIZE_128KB:
541 size = KB(128);
542 break;
543 case I965_PGETBL_SIZE_256KB:
544 size = KB(256);
545 break;
546 case I965_PGETBL_SIZE_512KB:
547 size = KB(512);
548 break;
549 /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
550 case I965_PGETBL_SIZE_1MB:
551 size = KB(1024);
552 break;
553 case I965_PGETBL_SIZE_2MB:
554 size = KB(2048);
555 break;
556 case I965_PGETBL_SIZE_1_5MB:
557 size = KB(1024 + 512);
558 break;
559 default:
560 dev_info(&intel_private.pcidev->dev,
561 "unknown page table size, assuming 512KB\n");
562 size = KB(512);
563 }
564
565 return size/4;
566}
567
568static unsigned int intel_gtt_total_entries(void)
569{
570 int size;
571
572 if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)
573 return i965_gtt_total_entries();
574 else if (INTEL_GTT_GEN == 6) {
680 u16 snb_gmch_ctl; 575 u16 snb_gmch_ctl;
681 576
682 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl); 577 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
@@ -706,7 +601,18 @@ static unsigned int intel_gtt_mappable_entries(void)
706{ 601{
707 unsigned int aperture_size; 602 unsigned int aperture_size;
708 603
709 if (INTEL_GTT_GEN == 2) { 604 if (INTEL_GTT_GEN == 1) {
605 u32 smram_miscc;
606
607 pci_read_config_dword(intel_private.bridge_dev,
608 I810_SMRAM_MISCC, &smram_miscc);
609
610 if ((smram_miscc & I810_GFX_MEM_WIN_SIZE)
611 == I810_GFX_MEM_WIN_32M)
612 aperture_size = MB(32);
613 else
614 aperture_size = MB(64);
615 } else if (INTEL_GTT_GEN == 2) {
710 u16 gmch_ctrl; 616 u16 gmch_ctrl;
711 617
712 pci_read_config_word(intel_private.bridge_dev, 618 pci_read_config_word(intel_private.bridge_dev,
@@ -739,7 +645,7 @@ static void intel_gtt_cleanup(void)
739 645
740 iounmap(intel_private.gtt); 646 iounmap(intel_private.gtt);
741 iounmap(intel_private.registers); 647 iounmap(intel_private.registers);
742 648
743 intel_gtt_teardown_scratch_page(); 649 intel_gtt_teardown_scratch_page();
744} 650}
745 651
@@ -755,6 +661,14 @@ static int intel_gtt_init(void)
755 intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries(); 661 intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
756 intel_private.base.gtt_total_entries = intel_gtt_total_entries(); 662 intel_private.base.gtt_total_entries = intel_gtt_total_entries();
757 663
664 /* save the PGETBL reg for resume */
665 intel_private.PGETBL_save =
666 readl(intel_private.registers+I810_PGETBL_CTL)
667 & ~I810_PGETBL_ENABLED;
668 /* we only ever restore the register when enabling the PGTBL... */
669 if (HAS_PGTBL_EN)
670 intel_private.PGETBL_save |= I810_PGETBL_ENABLED;
671
758 dev_info(&intel_private.bridge_dev->dev, 672 dev_info(&intel_private.bridge_dev->dev,
759 "detected gtt size: %dK total, %dK mappable\n", 673 "detected gtt size: %dK total, %dK mappable\n",
760 intel_private.base.gtt_total_entries * 4, 674 intel_private.base.gtt_total_entries * 4,
@@ -772,14 +686,7 @@ static int intel_gtt_init(void)
772 686
773 global_cache_flush(); /* FIXME: ? */ 687 global_cache_flush(); /* FIXME: ? */
774 688
775 /* we have to call this as early as possible after the MMIO base address is known */ 689 intel_private.base.stolen_size = intel_gtt_stolen_size();
776 intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
777 if (intel_private.base.gtt_stolen_entries == 0) {
778 intel_private.driver->cleanup();
779 iounmap(intel_private.registers);
780 iounmap(intel_private.gtt);
781 return -ENOMEM;
782 }
783 690
784 ret = intel_gtt_setup_scratch_page(); 691 ret = intel_gtt_setup_scratch_page();
785 if (ret != 0) { 692 if (ret != 0) {
@@ -787,6 +694,8 @@ static int intel_gtt_init(void)
787 return ret; 694 return ret;
788 } 695 }
789 696
697 intel_private.base.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2;
698
790 return 0; 699 return 0;
791} 700}
792 701
@@ -862,25 +771,19 @@ static void i830_write_entry(dma_addr_t addr, unsigned int entry,
862 unsigned int flags) 771 unsigned int flags)
863{ 772{
864 u32 pte_flags = I810_PTE_VALID; 773 u32 pte_flags = I810_PTE_VALID;
865 774
866 switch (flags) { 775 if (flags == AGP_USER_CACHED_MEMORY)
867 case AGP_DCACHE_MEMORY:
868 pte_flags |= I810_PTE_LOCAL;
869 break;
870 case AGP_USER_CACHED_MEMORY:
871 pte_flags |= I830_PTE_SYSTEM_CACHED; 776 pte_flags |= I830_PTE_SYSTEM_CACHED;
872 break;
873 }
874 777
875 writel(addr | pte_flags, intel_private.gtt + entry); 778 writel(addr | pte_flags, intel_private.gtt + entry);
876} 779}
877 780
878static void intel_enable_gtt(void) 781static bool intel_enable_gtt(void)
879{ 782{
880 u32 gma_addr; 783 u32 gma_addr;
881 u16 gmch_ctrl; 784 u8 __iomem *reg;
882 785
883 if (INTEL_GTT_GEN == 2) 786 if (INTEL_GTT_GEN <= 2)
884 pci_read_config_dword(intel_private.pcidev, I810_GMADDR, 787 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
885 &gma_addr); 788 &gma_addr);
886 else 789 else
@@ -889,13 +792,38 @@ static void intel_enable_gtt(void)
889 792
890 intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK); 793 intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
891 794
892 pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl); 795 if (INTEL_GTT_GEN >= 6)
893 gmch_ctrl |= I830_GMCH_ENABLED; 796 return true;
894 pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);
895 797
896 writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED, 798 if (INTEL_GTT_GEN == 2) {
897 intel_private.registers+I810_PGETBL_CTL); 799 u16 gmch_ctrl;
898 readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */ 800
801 pci_read_config_word(intel_private.bridge_dev,
802 I830_GMCH_CTRL, &gmch_ctrl);
803 gmch_ctrl |= I830_GMCH_ENABLED;
804 pci_write_config_word(intel_private.bridge_dev,
805 I830_GMCH_CTRL, gmch_ctrl);
806
807 pci_read_config_word(intel_private.bridge_dev,
808 I830_GMCH_CTRL, &gmch_ctrl);
809 if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) {
810 dev_err(&intel_private.pcidev->dev,
811 "failed to enable the GTT: GMCH_CTRL=%x\n",
812 gmch_ctrl);
813 return false;
814 }
815 }
816
817 reg = intel_private.registers+I810_PGETBL_CTL;
818 writel(intel_private.PGETBL_save, reg);
819 if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) {
820 dev_err(&intel_private.pcidev->dev,
821 "failed to enable the GTT: PGETBL=%x [expected %x]\n",
822 readl(reg), intel_private.PGETBL_save);
823 return false;
824 }
825
826 return true;
899} 827}
900 828
901static int i830_setup(void) 829static int i830_setup(void)
@@ -910,8 +838,6 @@ static int i830_setup(void)
910 return -ENOMEM; 838 return -ENOMEM;
911 839
912 intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; 840 intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
913 intel_private.pte_bus_addr =
914 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
915 841
916 intel_i830_setup_flush(); 842 intel_i830_setup_flush();
917 843
@@ -936,12 +862,12 @@ static int intel_fake_agp_configure(void)
936{ 862{
937 int i; 863 int i;
938 864
939 intel_enable_gtt(); 865 if (!intel_enable_gtt())
866 return -EIO;
940 867
941 agp_bridge->gart_bus_addr = intel_private.gma_bus_addr; 868 agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
942 869
943 for (i = intel_private.base.gtt_stolen_entries; 870 for (i = 0; i < intel_private.base.gtt_total_entries; i++) {
944 i < intel_private.base.gtt_total_entries; i++) {
945 intel_private.driver->write_entry(intel_private.scratch_page_dma, 871 intel_private.driver->write_entry(intel_private.scratch_page_dma,
946 i, 0); 872 i, 0);
947 } 873 }
@@ -965,10 +891,10 @@ static bool i830_check_flags(unsigned int flags)
965 return false; 891 return false;
966} 892}
967 893
968static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list, 894void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
969 unsigned int sg_len, 895 unsigned int sg_len,
970 unsigned int pg_start, 896 unsigned int pg_start,
971 unsigned int flags) 897 unsigned int flags)
972{ 898{
973 struct scatterlist *sg; 899 struct scatterlist *sg;
974 unsigned int len, m; 900 unsigned int len, m;
@@ -989,27 +915,34 @@ static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
989 } 915 }
990 readl(intel_private.gtt+j-1); 916 readl(intel_private.gtt+j-1);
991} 917}
918EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
919
920void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
921 struct page **pages, unsigned int flags)
922{
923 int i, j;
924
925 for (i = 0, j = first_entry; i < num_entries; i++, j++) {
926 dma_addr_t addr = page_to_phys(pages[i]);
927 intel_private.driver->write_entry(addr,
928 j, flags);
929 }
930 readl(intel_private.gtt+j-1);
931}
932EXPORT_SYMBOL(intel_gtt_insert_pages);
992 933
993static int intel_fake_agp_insert_entries(struct agp_memory *mem, 934static int intel_fake_agp_insert_entries(struct agp_memory *mem,
994 off_t pg_start, int type) 935 off_t pg_start, int type)
995{ 936{
996 int i, j;
997 int ret = -EINVAL; 937 int ret = -EINVAL;
998 938
939 if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY)
940 return i810_insert_dcache_entries(mem, pg_start, type);
941
999 if (mem->page_count == 0) 942 if (mem->page_count == 0)
1000 goto out; 943 goto out;
1001 944
1002 if (pg_start < intel_private.base.gtt_stolen_entries) { 945 if (pg_start + mem->page_count > intel_private.base.gtt_total_entries)
1003 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
1004 "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
1005 pg_start, intel_private.base.gtt_stolen_entries);
1006
1007 dev_info(&intel_private.pcidev->dev,
1008 "trying to insert into local/stolen memory\n");
1009 goto out_err;
1010 }
1011
1012 if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries)
1013 goto out_err; 946 goto out_err;
1014 947
1015 if (type != mem->type) 948 if (type != mem->type)
@@ -1021,21 +954,17 @@ static int intel_fake_agp_insert_entries(struct agp_memory *mem,
1021 if (!mem->is_flushed) 954 if (!mem->is_flushed)
1022 global_cache_flush(); 955 global_cache_flush();
1023 956
1024 if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) { 957 if (intel_private.base.needs_dmar) {
1025 ret = intel_agp_map_memory(mem); 958 ret = intel_gtt_map_memory(mem->pages, mem->page_count,
959 &mem->sg_list, &mem->num_sg);
1026 if (ret != 0) 960 if (ret != 0)
1027 return ret; 961 return ret;
1028 962
1029 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg, 963 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
1030 pg_start, type); 964 pg_start, type);
1031 } else { 965 } else
1032 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { 966 intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,
1033 dma_addr_t addr = page_to_phys(mem->pages[i]); 967 type);
1034 intel_private.driver->write_entry(addr,
1035 j, type);
1036 }
1037 readl(intel_private.gtt+j-1);
1038 }
1039 968
1040out: 969out:
1041 ret = 0; 970 ret = 0;
@@ -1044,40 +973,54 @@ out_err:
1044 return ret; 973 return ret;
1045} 974}
1046 975
976void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries)
977{
978 unsigned int i;
979
980 for (i = first_entry; i < (first_entry + num_entries); i++) {
981 intel_private.driver->write_entry(intel_private.scratch_page_dma,
982 i, 0);
983 }
984 readl(intel_private.gtt+i-1);
985}
986EXPORT_SYMBOL(intel_gtt_clear_range);
987
1047static int intel_fake_agp_remove_entries(struct agp_memory *mem, 988static int intel_fake_agp_remove_entries(struct agp_memory *mem,
1048 off_t pg_start, int type) 989 off_t pg_start, int type)
1049{ 990{
1050 int i;
1051
1052 if (mem->page_count == 0) 991 if (mem->page_count == 0)
1053 return 0; 992 return 0;
1054 993
1055 if (pg_start < intel_private.base.gtt_stolen_entries) { 994 if (intel_private.base.needs_dmar) {
1056 dev_info(&intel_private.pcidev->dev, 995 intel_gtt_unmap_memory(mem->sg_list, mem->num_sg);
1057 "trying to disable local/stolen memory\n"); 996 mem->sg_list = NULL;
1058 return -EINVAL; 997 mem->num_sg = 0;
1059 } 998 }
1060 999
1061 if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) 1000 intel_gtt_clear_range(pg_start, mem->page_count);
1062 intel_agp_unmap_memory(mem);
1063
1064 for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1065 intel_private.driver->write_entry(intel_private.scratch_page_dma,
1066 i, 0);
1067 }
1068 readl(intel_private.gtt+i-1);
1069 1001
1070 return 0; 1002 return 0;
1071} 1003}
1072 1004
1073static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge)
1074{
1075 intel_private.driver->chipset_flush();
1076}
1077
1078static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count, 1005static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
1079 int type) 1006 int type)
1080{ 1007{
1008 struct agp_memory *new;
1009
1010 if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) {
1011 if (pg_count != intel_private.num_dcache_entries)
1012 return NULL;
1013
1014 new = agp_create_memory(1);
1015 if (new == NULL)
1016 return NULL;
1017
1018 new->type = AGP_DCACHE_MEMORY;
1019 new->page_count = pg_count;
1020 new->num_scratch_pages = 0;
1021 agp_free_page_array(new);
1022 return new;
1023 }
1081 if (type == AGP_PHYS_MEMORY) 1024 if (type == AGP_PHYS_MEMORY)
1082 return alloc_agpphysmem_i8xx(pg_count, type); 1025 return alloc_agpphysmem_i8xx(pg_count, type);
1083 /* always return NULL for other allocation types for now */ 1026 /* always return NULL for other allocation types for now */
@@ -1274,40 +1217,11 @@ static int i9xx_setup(void)
1274 intel_private.gtt_bus_addr = reg_addr + gtt_offset; 1217 intel_private.gtt_bus_addr = reg_addr + gtt_offset;
1275 } 1218 }
1276 1219
1277 intel_private.pte_bus_addr =
1278 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1279
1280 intel_i9xx_setup_flush(); 1220 intel_i9xx_setup_flush();
1281 1221
1282 return 0; 1222 return 0;
1283} 1223}
1284 1224
1285static const struct agp_bridge_driver intel_810_driver = {
1286 .owner = THIS_MODULE,
1287 .aperture_sizes = intel_i810_sizes,
1288 .size_type = FIXED_APER_SIZE,
1289 .num_aperture_sizes = 2,
1290 .needs_scratch_page = true,
1291 .configure = intel_i810_configure,
1292 .fetch_size = intel_i810_fetch_size,
1293 .cleanup = intel_i810_cleanup,
1294 .mask_memory = intel_i810_mask_memory,
1295 .masks = intel_i810_masks,
1296 .agp_enable = intel_fake_agp_enable,
1297 .cache_flush = global_cache_flush,
1298 .create_gatt_table = agp_generic_create_gatt_table,
1299 .free_gatt_table = agp_generic_free_gatt_table,
1300 .insert_memory = intel_i810_insert_entries,
1301 .remove_memory = intel_i810_remove_entries,
1302 .alloc_by_type = intel_i810_alloc_by_type,
1303 .free_by_type = intel_i810_free_by_type,
1304 .agp_alloc_page = agp_generic_alloc_page,
1305 .agp_alloc_pages = agp_generic_alloc_pages,
1306 .agp_destroy_page = agp_generic_destroy_page,
1307 .agp_destroy_pages = agp_generic_destroy_pages,
1308 .agp_type_to_mask_type = agp_generic_type_to_mask_type,
1309};
1310
1311static const struct agp_bridge_driver intel_fake_agp_driver = { 1225static const struct agp_bridge_driver intel_fake_agp_driver = {
1312 .owner = THIS_MODULE, 1226 .owner = THIS_MODULE,
1313 .size_type = FIXED_APER_SIZE, 1227 .size_type = FIXED_APER_SIZE,
@@ -1328,15 +1242,20 @@ static const struct agp_bridge_driver intel_fake_agp_driver = {
1328 .agp_alloc_pages = agp_generic_alloc_pages, 1242 .agp_alloc_pages = agp_generic_alloc_pages,
1329 .agp_destroy_page = agp_generic_destroy_page, 1243 .agp_destroy_page = agp_generic_destroy_page,
1330 .agp_destroy_pages = agp_generic_destroy_pages, 1244 .agp_destroy_pages = agp_generic_destroy_pages,
1331 .chipset_flush = intel_fake_agp_chipset_flush,
1332}; 1245};
1333 1246
1334static const struct intel_gtt_driver i81x_gtt_driver = { 1247static const struct intel_gtt_driver i81x_gtt_driver = {
1335 .gen = 1, 1248 .gen = 1,
1249 .has_pgtbl_enable = 1,
1336 .dma_mask_size = 32, 1250 .dma_mask_size = 32,
1251 .setup = i810_setup,
1252 .cleanup = i810_cleanup,
1253 .check_flags = i830_check_flags,
1254 .write_entry = i810_write_entry,
1337}; 1255};
1338static const struct intel_gtt_driver i8xx_gtt_driver = { 1256static const struct intel_gtt_driver i8xx_gtt_driver = {
1339 .gen = 2, 1257 .gen = 2,
1258 .has_pgtbl_enable = 1,
1340 .setup = i830_setup, 1259 .setup = i830_setup,
1341 .cleanup = i830_cleanup, 1260 .cleanup = i830_cleanup,
1342 .write_entry = i830_write_entry, 1261 .write_entry = i830_write_entry,
@@ -1346,10 +1265,11 @@ static const struct intel_gtt_driver i8xx_gtt_driver = {
1346}; 1265};
1347static const struct intel_gtt_driver i915_gtt_driver = { 1266static const struct intel_gtt_driver i915_gtt_driver = {
1348 .gen = 3, 1267 .gen = 3,
1268 .has_pgtbl_enable = 1,
1349 .setup = i9xx_setup, 1269 .setup = i9xx_setup,
1350 .cleanup = i9xx_cleanup, 1270 .cleanup = i9xx_cleanup,
1351 /* i945 is the last gpu to need phys mem (for overlay and cursors). */ 1271 /* i945 is the last gpu to need phys mem (for overlay and cursors). */
1352 .write_entry = i830_write_entry, 1272 .write_entry = i830_write_entry,
1353 .dma_mask_size = 32, 1273 .dma_mask_size = 32,
1354 .check_flags = i830_check_flags, 1274 .check_flags = i830_check_flags,
1355 .chipset_flush = i9xx_chipset_flush, 1275 .chipset_flush = i9xx_chipset_flush,
@@ -1376,6 +1296,7 @@ static const struct intel_gtt_driver pineview_gtt_driver = {
1376}; 1296};
1377static const struct intel_gtt_driver i965_gtt_driver = { 1297static const struct intel_gtt_driver i965_gtt_driver = {
1378 .gen = 4, 1298 .gen = 4,
1299 .has_pgtbl_enable = 1,
1379 .setup = i9xx_setup, 1300 .setup = i9xx_setup,
1380 .cleanup = i9xx_cleanup, 1301 .cleanup = i9xx_cleanup,
1381 .write_entry = i965_write_entry, 1302 .write_entry = i965_write_entry,
@@ -1419,93 +1340,92 @@ static const struct intel_gtt_driver sandybridge_gtt_driver = {
1419static const struct intel_gtt_driver_description { 1340static const struct intel_gtt_driver_description {
1420 unsigned int gmch_chip_id; 1341 unsigned int gmch_chip_id;
1421 char *name; 1342 char *name;
1422 const struct agp_bridge_driver *gmch_driver;
1423 const struct intel_gtt_driver *gtt_driver; 1343 const struct intel_gtt_driver *gtt_driver;
1424} intel_gtt_chipsets[] = { 1344} intel_gtt_chipsets[] = {
1425 { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver, 1345 { PCI_DEVICE_ID_INTEL_82810_IG1, "i810",
1426 &i81x_gtt_driver}, 1346 &i81x_gtt_driver},
1427 { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver, 1347 { PCI_DEVICE_ID_INTEL_82810_IG3, "i810",
1428 &i81x_gtt_driver}, 1348 &i81x_gtt_driver},
1429 { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver, 1349 { PCI_DEVICE_ID_INTEL_82810E_IG, "i810",
1430 &i81x_gtt_driver}, 1350 &i81x_gtt_driver},
1431 { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver, 1351 { PCI_DEVICE_ID_INTEL_82815_CGC, "i815",
1432 &i81x_gtt_driver}, 1352 &i81x_gtt_driver},
1433 { PCI_DEVICE_ID_INTEL_82830_CGC, "830M", 1353 { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1434 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1354 &i8xx_gtt_driver},
1435 { PCI_DEVICE_ID_INTEL_82845G_IG, "830M", 1355 { PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1436 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1356 &i8xx_gtt_driver},
1437 { PCI_DEVICE_ID_INTEL_82854_IG, "854", 1357 { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1438 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1358 &i8xx_gtt_driver},
1439 { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM", 1359 { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1440 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1360 &i8xx_gtt_driver},
1441 { PCI_DEVICE_ID_INTEL_82865_IG, "865", 1361 { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1442 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1362 &i8xx_gtt_driver},
1443 { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)", 1363 { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1444 &intel_fake_agp_driver, &i915_gtt_driver }, 1364 &i915_gtt_driver },
1445 { PCI_DEVICE_ID_INTEL_82915G_IG, "915G", 1365 { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1446 &intel_fake_agp_driver, &i915_gtt_driver }, 1366 &i915_gtt_driver },
1447 { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM", 1367 { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1448 &intel_fake_agp_driver, &i915_gtt_driver }, 1368 &i915_gtt_driver },
1449 { PCI_DEVICE_ID_INTEL_82945G_IG, "945G", 1369 { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1450 &intel_fake_agp_driver, &i915_gtt_driver }, 1370 &i915_gtt_driver },
1451 { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM", 1371 { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1452 &intel_fake_agp_driver, &i915_gtt_driver }, 1372 &i915_gtt_driver },
1453 { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME", 1373 { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1454 &intel_fake_agp_driver, &i915_gtt_driver }, 1374 &i915_gtt_driver },
1455 { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ", 1375 { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1456 &intel_fake_agp_driver, &i965_gtt_driver }, 1376 &i965_gtt_driver },
1457 { PCI_DEVICE_ID_INTEL_82G35_IG, "G35", 1377 { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1458 &intel_fake_agp_driver, &i965_gtt_driver }, 1378 &i965_gtt_driver },
1459 { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q", 1379 { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1460 &intel_fake_agp_driver, &i965_gtt_driver }, 1380 &i965_gtt_driver },
1461 { PCI_DEVICE_ID_INTEL_82965G_IG, "965G", 1381 { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1462 &intel_fake_agp_driver, &i965_gtt_driver }, 1382 &i965_gtt_driver },
1463 { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM", 1383 { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1464 &intel_fake_agp_driver, &i965_gtt_driver }, 1384 &i965_gtt_driver },
1465 { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE", 1385 { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1466 &intel_fake_agp_driver, &i965_gtt_driver }, 1386 &i965_gtt_driver },
1467 { PCI_DEVICE_ID_INTEL_G33_IG, "G33", 1387 { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1468 &intel_fake_agp_driver, &g33_gtt_driver }, 1388 &g33_gtt_driver },
1469 { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35", 1389 { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1470 &intel_fake_agp_driver, &g33_gtt_driver }, 1390 &g33_gtt_driver },
1471 { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33", 1391 { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1472 &intel_fake_agp_driver, &g33_gtt_driver }, 1392 &g33_gtt_driver },
1473 { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150", 1393 { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1474 &intel_fake_agp_driver, &pineview_gtt_driver }, 1394 &pineview_gtt_driver },
1475 { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150", 1395 { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1476 &intel_fake_agp_driver, &pineview_gtt_driver }, 1396 &pineview_gtt_driver },
1477 { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45", 1397 { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1478 &intel_fake_agp_driver, &g4x_gtt_driver }, 1398 &g4x_gtt_driver },
1479 { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake", 1399 { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1480 &intel_fake_agp_driver, &g4x_gtt_driver }, 1400 &g4x_gtt_driver },
1481 { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43", 1401 { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1482 &intel_fake_agp_driver, &g4x_gtt_driver }, 1402 &g4x_gtt_driver },
1483 { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43", 1403 { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1484 &intel_fake_agp_driver, &g4x_gtt_driver }, 1404 &g4x_gtt_driver },
1485 { PCI_DEVICE_ID_INTEL_B43_IG, "B43", 1405 { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1486 &intel_fake_agp_driver, &g4x_gtt_driver }, 1406 &g4x_gtt_driver },
1487 { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43", 1407 { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1488 &intel_fake_agp_driver, &g4x_gtt_driver }, 1408 &g4x_gtt_driver },
1489 { PCI_DEVICE_ID_INTEL_G41_IG, "G41", 1409 { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1490 &intel_fake_agp_driver, &g4x_gtt_driver }, 1410 &g4x_gtt_driver },
1491 { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG, 1411 { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1492 "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver }, 1412 "HD Graphics", &ironlake_gtt_driver },
1493 { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG, 1413 { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1494 "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver }, 1414 "HD Graphics", &ironlake_gtt_driver },
1495 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG, 1415 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1496 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1416 "Sandybridge", &sandybridge_gtt_driver },
1497 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG, 1417 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1498 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1418 "Sandybridge", &sandybridge_gtt_driver },
1499 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG, 1419 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1500 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1420 "Sandybridge", &sandybridge_gtt_driver },
1501 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG, 1421 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1502 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1422 "Sandybridge", &sandybridge_gtt_driver },
1503 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG, 1423 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1504 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1424 "Sandybridge", &sandybridge_gtt_driver },
1505 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG, 1425 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1506 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1426 "Sandybridge", &sandybridge_gtt_driver },
1507 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG, 1427 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1508 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1428 "Sandybridge", &sandybridge_gtt_driver },
1509 { 0, NULL, NULL } 1429 { 0, NULL, NULL }
1510}; 1430};
1511 1431
@@ -1530,21 +1450,20 @@ int intel_gmch_probe(struct pci_dev *pdev,
1530 struct agp_bridge_data *bridge) 1450 struct agp_bridge_data *bridge)
1531{ 1451{
1532 int i, mask; 1452 int i, mask;
1533 bridge->driver = NULL; 1453 intel_private.driver = NULL;
1534 1454
1535 for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) { 1455 for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1536 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) { 1456 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1537 bridge->driver = 1457 intel_private.driver =
1538 intel_gtt_chipsets[i].gmch_driver;
1539 intel_private.driver =
1540 intel_gtt_chipsets[i].gtt_driver; 1458 intel_gtt_chipsets[i].gtt_driver;
1541 break; 1459 break;
1542 } 1460 }
1543 } 1461 }
1544 1462
1545 if (!bridge->driver) 1463 if (!intel_private.driver)
1546 return 0; 1464 return 0;
1547 1465
1466 bridge->driver = &intel_fake_agp_driver;
1548 bridge->dev_private_data = &intel_private; 1467 bridge->dev_private_data = &intel_private;
1549 bridge->dev = pdev; 1468 bridge->dev = pdev;
1550 1469
@@ -1560,8 +1479,8 @@ int intel_gmch_probe(struct pci_dev *pdev,
1560 pci_set_consistent_dma_mask(intel_private.pcidev, 1479 pci_set_consistent_dma_mask(intel_private.pcidev,
1561 DMA_BIT_MASK(mask)); 1480 DMA_BIT_MASK(mask));
1562 1481
1563 if (bridge->driver == &intel_810_driver) 1482 /*if (bridge->driver == &intel_810_driver)
1564 return 1; 1483 return 1;*/
1565 1484
1566 if (intel_gtt_init() != 0) 1485 if (intel_gtt_init() != 0)
1567 return 0; 1486 return 0;
@@ -1570,12 +1489,19 @@ int intel_gmch_probe(struct pci_dev *pdev,
1570} 1489}
1571EXPORT_SYMBOL(intel_gmch_probe); 1490EXPORT_SYMBOL(intel_gmch_probe);
1572 1491
1573struct intel_gtt *intel_gtt_get(void) 1492const struct intel_gtt *intel_gtt_get(void)
1574{ 1493{
1575 return &intel_private.base; 1494 return &intel_private.base;
1576} 1495}
1577EXPORT_SYMBOL(intel_gtt_get); 1496EXPORT_SYMBOL(intel_gtt_get);
1578 1497
1498void intel_gtt_chipset_flush(void)
1499{
1500 if (intel_private.driver->chipset_flush)
1501 intel_private.driver->chipset_flush();
1502}
1503EXPORT_SYMBOL(intel_gtt_chipset_flush);
1504
1579void intel_gmch_remove(struct pci_dev *pdev) 1505void intel_gmch_remove(struct pci_dev *pdev)
1580{ 1506{
1581 if (intel_private.pcidev) 1507 if (intel_private.pcidev)
diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c
index 252fdb98b73a..0cb2ba50af53 100644
--- a/drivers/gpu/drm/drm_agpsupport.c
+++ b/drivers/gpu/drm/drm_agpsupport.c
@@ -466,10 +466,4 @@ drm_agp_bind_pages(struct drm_device *dev,
466} 466}
467EXPORT_SYMBOL(drm_agp_bind_pages); 467EXPORT_SYMBOL(drm_agp_bind_pages);
468 468
469void drm_agp_chipset_flush(struct drm_device *dev)
470{
471 agp_flush_chipset(dev->agp->bridge);
472}
473EXPORT_SYMBOL(drm_agp_chipset_flush);
474
475#endif /* __OS_HAS_AGP */ 469#endif /* __OS_HAS_AGP */
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index a6bfc302ed90..c59515ba7e69 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -392,10 +392,36 @@ void drm_mm_init_scan(struct drm_mm *mm, unsigned long size,
392 mm->scanned_blocks = 0; 392 mm->scanned_blocks = 0;
393 mm->scan_hit_start = 0; 393 mm->scan_hit_start = 0;
394 mm->scan_hit_size = 0; 394 mm->scan_hit_size = 0;
395 mm->scan_check_range = 0;
395} 396}
396EXPORT_SYMBOL(drm_mm_init_scan); 397EXPORT_SYMBOL(drm_mm_init_scan);
397 398
398/** 399/**
400 * Initializa lru scanning.
401 *
402 * This simply sets up the scanning routines with the parameters for the desired
403 * hole. This version is for range-restricted scans.
404 *
405 * Warning: As long as the scan list is non-empty, no other operations than
406 * adding/removing nodes to/from the scan list are allowed.
407 */
408void drm_mm_init_scan_with_range(struct drm_mm *mm, unsigned long size,
409 unsigned alignment,
410 unsigned long start,
411 unsigned long end)
412{
413 mm->scan_alignment = alignment;
414 mm->scan_size = size;
415 mm->scanned_blocks = 0;
416 mm->scan_hit_start = 0;
417 mm->scan_hit_size = 0;
418 mm->scan_start = start;
419 mm->scan_end = end;
420 mm->scan_check_range = 1;
421}
422EXPORT_SYMBOL(drm_mm_init_scan_with_range);
423
424/**
399 * Add a node to the scan list that might be freed to make space for the desired 425 * Add a node to the scan list that might be freed to make space for the desired
400 * hole. 426 * hole.
401 * 427 *
@@ -406,6 +432,8 @@ int drm_mm_scan_add_block(struct drm_mm_node *node)
406 struct drm_mm *mm = node->mm; 432 struct drm_mm *mm = node->mm;
407 struct list_head *prev_free, *next_free; 433 struct list_head *prev_free, *next_free;
408 struct drm_mm_node *prev_node, *next_node; 434 struct drm_mm_node *prev_node, *next_node;
435 unsigned long adj_start;
436 unsigned long adj_end;
409 437
410 mm->scanned_blocks++; 438 mm->scanned_blocks++;
411 439
@@ -452,7 +480,17 @@ int drm_mm_scan_add_block(struct drm_mm_node *node)
452 node->free_stack.prev = prev_free; 480 node->free_stack.prev = prev_free;
453 node->free_stack.next = next_free; 481 node->free_stack.next = next_free;
454 482
455 if (check_free_hole(node->start, node->start + node->size, 483 if (mm->scan_check_range) {
484 adj_start = node->start < mm->scan_start ?
485 mm->scan_start : node->start;
486 adj_end = node->start + node->size > mm->scan_end ?
487 mm->scan_end : node->start + node->size;
488 } else {
489 adj_start = node->start;
490 adj_end = node->start + node->size;
491 }
492
493 if (check_free_hole(adj_start , adj_end,
456 mm->scan_size, mm->scan_alignment)) { 494 mm->scan_size, mm->scan_alignment)) {
457 mm->scan_hit_start = node->start; 495 mm->scan_hit_start = node->start;
458 mm->scan_hit_size = node->size; 496 mm->scan_hit_size = node->size;
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index fdc833d5cc7b..0ae6a7c5020f 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -9,6 +9,8 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
9 i915_gem.o \ 9 i915_gem.o \
10 i915_gem_debug.o \ 10 i915_gem_debug.o \
11 i915_gem_evict.o \ 11 i915_gem_evict.o \
12 i915_gem_execbuffer.o \
13 i915_gem_gtt.o \
12 i915_gem_tiling.o \ 14 i915_gem_tiling.o \
13 i915_trace_points.o \ 15 i915_trace_points.o \
14 intel_display.o \ 16 intel_display.o \
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 1f4f3ceb63c7..22821994b35a 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -32,6 +32,7 @@
32#include "drmP.h" 32#include "drmP.h"
33#include "drm.h" 33#include "drm.h"
34#include "intel_drv.h" 34#include "intel_drv.h"
35#include "intel_ringbuffer.h"
35#include "i915_drm.h" 36#include "i915_drm.h"
36#include "i915_drv.h" 37#include "i915_drv.h"
37 38
@@ -72,7 +73,6 @@ static int i915_capabilities(struct seq_file *m, void *data)
72 B(is_broadwater); 73 B(is_broadwater);
73 B(is_crestline); 74 B(is_crestline);
74 B(has_fbc); 75 B(has_fbc);
75 B(has_rc6);
76 B(has_pipe_cxsr); 76 B(has_pipe_cxsr);
77 B(has_hotplug); 77 B(has_hotplug);
78 B(cursor_needs_physical); 78 B(cursor_needs_physical);
@@ -86,19 +86,19 @@ static int i915_capabilities(struct seq_file *m, void *data)
86 return 0; 86 return 0;
87} 87}
88 88
89static const char *get_pin_flag(struct drm_i915_gem_object *obj_priv) 89static const char *get_pin_flag(struct drm_i915_gem_object *obj)
90{ 90{
91 if (obj_priv->user_pin_count > 0) 91 if (obj->user_pin_count > 0)
92 return "P"; 92 return "P";
93 else if (obj_priv->pin_count > 0) 93 else if (obj->pin_count > 0)
94 return "p"; 94 return "p";
95 else 95 else
96 return " "; 96 return " ";
97} 97}
98 98
99static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv) 99static const char *get_tiling_flag(struct drm_i915_gem_object *obj)
100{ 100{
101 switch (obj_priv->tiling_mode) { 101 switch (obj->tiling_mode) {
102 default: 102 default:
103 case I915_TILING_NONE: return " "; 103 case I915_TILING_NONE: return " ";
104 case I915_TILING_X: return "X"; 104 case I915_TILING_X: return "X";
@@ -109,7 +109,7 @@ static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv)
109static void 109static void
110describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) 110describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
111{ 111{
112 seq_printf(m, "%p: %s%s %8zd %08x %08x %d%s%s", 112 seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s",
113 &obj->base, 113 &obj->base,
114 get_pin_flag(obj), 114 get_pin_flag(obj),
115 get_tiling_flag(obj), 115 get_tiling_flag(obj),
@@ -117,6 +117,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
117 obj->base.read_domains, 117 obj->base.read_domains,
118 obj->base.write_domain, 118 obj->base.write_domain,
119 obj->last_rendering_seqno, 119 obj->last_rendering_seqno,
120 obj->last_fenced_seqno,
120 obj->dirty ? " dirty" : "", 121 obj->dirty ? " dirty" : "",
121 obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); 122 obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
122 if (obj->base.name) 123 if (obj->base.name)
@@ -124,7 +125,17 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
124 if (obj->fence_reg != I915_FENCE_REG_NONE) 125 if (obj->fence_reg != I915_FENCE_REG_NONE)
125 seq_printf(m, " (fence: %d)", obj->fence_reg); 126 seq_printf(m, " (fence: %d)", obj->fence_reg);
126 if (obj->gtt_space != NULL) 127 if (obj->gtt_space != NULL)
127 seq_printf(m, " (gtt_offset: %08x)", obj->gtt_offset); 128 seq_printf(m, " (gtt offset: %08x, size: %08x)",
129 obj->gtt_offset, (unsigned int)obj->gtt_space->size);
130 if (obj->pin_mappable || obj->fault_mappable) {
131 char s[3], *t = s;
132 if (obj->pin_mappable)
133 *t++ = 'p';
134 if (obj->fault_mappable)
135 *t++ = 'f';
136 *t = '\0';
137 seq_printf(m, " (%s mappable)", s);
138 }
128 if (obj->ring != NULL) 139 if (obj->ring != NULL)
129 seq_printf(m, " (%s)", obj->ring->name); 140 seq_printf(m, " (%s)", obj->ring->name);
130} 141}
@@ -136,7 +147,7 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
136 struct list_head *head; 147 struct list_head *head;
137 struct drm_device *dev = node->minor->dev; 148 struct drm_device *dev = node->minor->dev;
138 drm_i915_private_t *dev_priv = dev->dev_private; 149 drm_i915_private_t *dev_priv = dev->dev_private;
139 struct drm_i915_gem_object *obj_priv; 150 struct drm_i915_gem_object *obj;
140 size_t total_obj_size, total_gtt_size; 151 size_t total_obj_size, total_gtt_size;
141 int count, ret; 152 int count, ret;
142 153
@@ -171,12 +182,12 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
171 } 182 }
172 183
173 total_obj_size = total_gtt_size = count = 0; 184 total_obj_size = total_gtt_size = count = 0;
174 list_for_each_entry(obj_priv, head, mm_list) { 185 list_for_each_entry(obj, head, mm_list) {
175 seq_printf(m, " "); 186 seq_printf(m, " ");
176 describe_obj(m, obj_priv); 187 describe_obj(m, obj);
177 seq_printf(m, "\n"); 188 seq_printf(m, "\n");
178 total_obj_size += obj_priv->base.size; 189 total_obj_size += obj->base.size;
179 total_gtt_size += obj_priv->gtt_space->size; 190 total_gtt_size += obj->gtt_space->size;
180 count++; 191 count++;
181 } 192 }
182 mutex_unlock(&dev->struct_mutex); 193 mutex_unlock(&dev->struct_mutex);
@@ -186,24 +197,79 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
186 return 0; 197 return 0;
187} 198}
188 199
200#define count_objects(list, member) do { \
201 list_for_each_entry(obj, list, member) { \
202 size += obj->gtt_space->size; \
203 ++count; \
204 if (obj->map_and_fenceable) { \
205 mappable_size += obj->gtt_space->size; \
206 ++mappable_count; \
207 } \
208 } \
209} while(0)
210
189static int i915_gem_object_info(struct seq_file *m, void* data) 211static int i915_gem_object_info(struct seq_file *m, void* data)
190{ 212{
191 struct drm_info_node *node = (struct drm_info_node *) m->private; 213 struct drm_info_node *node = (struct drm_info_node *) m->private;
192 struct drm_device *dev = node->minor->dev; 214 struct drm_device *dev = node->minor->dev;
193 struct drm_i915_private *dev_priv = dev->dev_private; 215 struct drm_i915_private *dev_priv = dev->dev_private;
216 u32 count, mappable_count;
217 size_t size, mappable_size;
218 struct drm_i915_gem_object *obj;
194 int ret; 219 int ret;
195 220
196 ret = mutex_lock_interruptible(&dev->struct_mutex); 221 ret = mutex_lock_interruptible(&dev->struct_mutex);
197 if (ret) 222 if (ret)
198 return ret; 223 return ret;
199 224
200 seq_printf(m, "%u objects\n", dev_priv->mm.object_count); 225 seq_printf(m, "%u objects, %zu bytes\n",
201 seq_printf(m, "%zu object bytes\n", dev_priv->mm.object_memory); 226 dev_priv->mm.object_count,
202 seq_printf(m, "%u pinned\n", dev_priv->mm.pin_count); 227 dev_priv->mm.object_memory);
203 seq_printf(m, "%zu pin bytes\n", dev_priv->mm.pin_memory); 228
204 seq_printf(m, "%u objects in gtt\n", dev_priv->mm.gtt_count); 229 size = count = mappable_size = mappable_count = 0;
205 seq_printf(m, "%zu gtt bytes\n", dev_priv->mm.gtt_memory); 230 count_objects(&dev_priv->mm.gtt_list, gtt_list);
206 seq_printf(m, "%zu gtt total\n", dev_priv->mm.gtt_total); 231 seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
232 count, mappable_count, size, mappable_size);
233
234 size = count = mappable_size = mappable_count = 0;
235 count_objects(&dev_priv->mm.active_list, mm_list);
236 count_objects(&dev_priv->mm.flushing_list, mm_list);
237 seq_printf(m, " %u [%u] active objects, %zu [%zu] bytes\n",
238 count, mappable_count, size, mappable_size);
239
240 size = count = mappable_size = mappable_count = 0;
241 count_objects(&dev_priv->mm.pinned_list, mm_list);
242 seq_printf(m, " %u [%u] pinned objects, %zu [%zu] bytes\n",
243 count, mappable_count, size, mappable_size);
244
245 size = count = mappable_size = mappable_count = 0;
246 count_objects(&dev_priv->mm.inactive_list, mm_list);
247 seq_printf(m, " %u [%u] inactive objects, %zu [%zu] bytes\n",
248 count, mappable_count, size, mappable_size);
249
250 size = count = mappable_size = mappable_count = 0;
251 count_objects(&dev_priv->mm.deferred_free_list, mm_list);
252 seq_printf(m, " %u [%u] freed objects, %zu [%zu] bytes\n",
253 count, mappable_count, size, mappable_size);
254
255 size = count = mappable_size = mappable_count = 0;
256 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
257 if (obj->fault_mappable) {
258 size += obj->gtt_space->size;
259 ++count;
260 }
261 if (obj->pin_mappable) {
262 mappable_size += obj->gtt_space->size;
263 ++mappable_count;
264 }
265 }
266 seq_printf(m, "%u pinned mappable objects, %zu bytes\n",
267 mappable_count, mappable_size);
268 seq_printf(m, "%u fault mappable objects, %zu bytes\n",
269 count, size);
270
271 seq_printf(m, "%zu [%zu] gtt total\n",
272 dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total);
207 273
208 mutex_unlock(&dev->struct_mutex); 274 mutex_unlock(&dev->struct_mutex);
209 275
@@ -243,14 +309,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
243 seq_printf(m, "%d prepares\n", work->pending); 309 seq_printf(m, "%d prepares\n", work->pending);
244 310
245 if (work->old_fb_obj) { 311 if (work->old_fb_obj) {
246 struct drm_i915_gem_object *obj_priv = to_intel_bo(work->old_fb_obj); 312 struct drm_i915_gem_object *obj = work->old_fb_obj;
247 if(obj_priv) 313 if (obj)
248 seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset ); 314 seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
249 } 315 }
250 if (work->pending_flip_obj) { 316 if (work->pending_flip_obj) {
251 struct drm_i915_gem_object *obj_priv = to_intel_bo(work->pending_flip_obj); 317 struct drm_i915_gem_object *obj = work->pending_flip_obj;
252 if(obj_priv) 318 if (obj)
253 seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset ); 319 seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
254 } 320 }
255 } 321 }
256 spin_unlock_irqrestore(&dev->event_lock, flags); 322 spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -265,44 +331,80 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
265 struct drm_device *dev = node->minor->dev; 331 struct drm_device *dev = node->minor->dev;
266 drm_i915_private_t *dev_priv = dev->dev_private; 332 drm_i915_private_t *dev_priv = dev->dev_private;
267 struct drm_i915_gem_request *gem_request; 333 struct drm_i915_gem_request *gem_request;
268 int ret; 334 int ret, count;
269 335
270 ret = mutex_lock_interruptible(&dev->struct_mutex); 336 ret = mutex_lock_interruptible(&dev->struct_mutex);
271 if (ret) 337 if (ret)
272 return ret; 338 return ret;
273 339
274 seq_printf(m, "Request:\n"); 340 count = 0;
275 list_for_each_entry(gem_request, &dev_priv->render_ring.request_list, 341 if (!list_empty(&dev_priv->ring[RCS].request_list)) {
276 list) { 342 seq_printf(m, "Render requests:\n");
277 seq_printf(m, " %d @ %d\n", 343 list_for_each_entry(gem_request,
278 gem_request->seqno, 344 &dev_priv->ring[RCS].request_list,
279 (int) (jiffies - gem_request->emitted_jiffies)); 345 list) {
346 seq_printf(m, " %d @ %d\n",
347 gem_request->seqno,
348 (int) (jiffies - gem_request->emitted_jiffies));
349 }
350 count++;
351 }
352 if (!list_empty(&dev_priv->ring[VCS].request_list)) {
353 seq_printf(m, "BSD requests:\n");
354 list_for_each_entry(gem_request,
355 &dev_priv->ring[VCS].request_list,
356 list) {
357 seq_printf(m, " %d @ %d\n",
358 gem_request->seqno,
359 (int) (jiffies - gem_request->emitted_jiffies));
360 }
361 count++;
362 }
363 if (!list_empty(&dev_priv->ring[BCS].request_list)) {
364 seq_printf(m, "BLT requests:\n");
365 list_for_each_entry(gem_request,
366 &dev_priv->ring[BCS].request_list,
367 list) {
368 seq_printf(m, " %d @ %d\n",
369 gem_request->seqno,
370 (int) (jiffies - gem_request->emitted_jiffies));
371 }
372 count++;
280 } 373 }
281 mutex_unlock(&dev->struct_mutex); 374 mutex_unlock(&dev->struct_mutex);
282 375
376 if (count == 0)
377 seq_printf(m, "No requests\n");
378
283 return 0; 379 return 0;
284} 380}
285 381
382static void i915_ring_seqno_info(struct seq_file *m,
383 struct intel_ring_buffer *ring)
384{
385 if (ring->get_seqno) {
386 seq_printf(m, "Current sequence (%s): %d\n",
387 ring->name, ring->get_seqno(ring));
388 seq_printf(m, "Waiter sequence (%s): %d\n",
389 ring->name, ring->waiting_seqno);
390 seq_printf(m, "IRQ sequence (%s): %d\n",
391 ring->name, ring->irq_seqno);
392 }
393}
394
286static int i915_gem_seqno_info(struct seq_file *m, void *data) 395static int i915_gem_seqno_info(struct seq_file *m, void *data)
287{ 396{
288 struct drm_info_node *node = (struct drm_info_node *) m->private; 397 struct drm_info_node *node = (struct drm_info_node *) m->private;
289 struct drm_device *dev = node->minor->dev; 398 struct drm_device *dev = node->minor->dev;
290 drm_i915_private_t *dev_priv = dev->dev_private; 399 drm_i915_private_t *dev_priv = dev->dev_private;
291 int ret; 400 int ret, i;
292 401
293 ret = mutex_lock_interruptible(&dev->struct_mutex); 402 ret = mutex_lock_interruptible(&dev->struct_mutex);
294 if (ret) 403 if (ret)
295 return ret; 404 return ret;
296 405
297 if (dev_priv->render_ring.status_page.page_addr != NULL) { 406 for (i = 0; i < I915_NUM_RINGS; i++)
298 seq_printf(m, "Current sequence: %d\n", 407 i915_ring_seqno_info(m, &dev_priv->ring[i]);
299 dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring));
300 } else {
301 seq_printf(m, "Current sequence: hws uninitialized\n");
302 }
303 seq_printf(m, "Waiter sequence: %d\n",
304 dev_priv->mm.waiting_gem_seqno);
305 seq_printf(m, "IRQ sequence: %d\n", dev_priv->mm.irq_gem_seqno);
306 408
307 mutex_unlock(&dev->struct_mutex); 409 mutex_unlock(&dev->struct_mutex);
308 410
@@ -315,7 +417,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
315 struct drm_info_node *node = (struct drm_info_node *) m->private; 417 struct drm_info_node *node = (struct drm_info_node *) m->private;
316 struct drm_device *dev = node->minor->dev; 418 struct drm_device *dev = node->minor->dev;
317 drm_i915_private_t *dev_priv = dev->dev_private; 419 drm_i915_private_t *dev_priv = dev->dev_private;
318 int ret; 420 int ret, i;
319 421
320 ret = mutex_lock_interruptible(&dev->struct_mutex); 422 ret = mutex_lock_interruptible(&dev->struct_mutex);
321 if (ret) 423 if (ret)
@@ -354,16 +456,8 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
354 } 456 }
355 seq_printf(m, "Interrupts received: %d\n", 457 seq_printf(m, "Interrupts received: %d\n",
356 atomic_read(&dev_priv->irq_received)); 458 atomic_read(&dev_priv->irq_received));
357 if (dev_priv->render_ring.status_page.page_addr != NULL) { 459 for (i = 0; i < I915_NUM_RINGS; i++)
358 seq_printf(m, "Current sequence: %d\n", 460 i915_ring_seqno_info(m, &dev_priv->ring[i]);
359 dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring));
360 } else {
361 seq_printf(m, "Current sequence: hws uninitialized\n");
362 }
363 seq_printf(m, "Waiter sequence: %d\n",
364 dev_priv->mm.waiting_gem_seqno);
365 seq_printf(m, "IRQ sequence: %d\n",
366 dev_priv->mm.irq_gem_seqno);
367 mutex_unlock(&dev->struct_mutex); 461 mutex_unlock(&dev->struct_mutex);
368 462
369 return 0; 463 return 0;
@@ -383,29 +477,17 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
383 seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start); 477 seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start);
384 seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs); 478 seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
385 for (i = 0; i < dev_priv->num_fence_regs; i++) { 479 for (i = 0; i < dev_priv->num_fence_regs; i++) {
386 struct drm_gem_object *obj = dev_priv->fence_regs[i].obj; 480 struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
387 481
388 if (obj == NULL) { 482 seq_printf(m, "Fenced object[%2d] = ", i);
389 seq_printf(m, "Fenced object[%2d] = unused\n", i); 483 if (obj == NULL)
390 } else { 484 seq_printf(m, "unused");
391 struct drm_i915_gem_object *obj_priv; 485 else
392 486 describe_obj(m, obj);
393 obj_priv = to_intel_bo(obj); 487 seq_printf(m, "\n");
394 seq_printf(m, "Fenced object[%2d] = %p: %s "
395 "%08x %08zx %08x %s %08x %08x %d",
396 i, obj, get_pin_flag(obj_priv),
397 obj_priv->gtt_offset,
398 obj->size, obj_priv->stride,
399 get_tiling_flag(obj_priv),
400 obj->read_domains, obj->write_domain,
401 obj_priv->last_rendering_seqno);
402 if (obj->name)
403 seq_printf(m, " (name: %d)", obj->name);
404 seq_printf(m, "\n");
405 }
406 } 488 }
407 mutex_unlock(&dev->struct_mutex);
408 489
490 mutex_unlock(&dev->struct_mutex);
409 return 0; 491 return 0;
410} 492}
411 493
@@ -414,10 +496,12 @@ static int i915_hws_info(struct seq_file *m, void *data)
414 struct drm_info_node *node = (struct drm_info_node *) m->private; 496 struct drm_info_node *node = (struct drm_info_node *) m->private;
415 struct drm_device *dev = node->minor->dev; 497 struct drm_device *dev = node->minor->dev;
416 drm_i915_private_t *dev_priv = dev->dev_private; 498 drm_i915_private_t *dev_priv = dev->dev_private;
417 int i; 499 struct intel_ring_buffer *ring;
418 volatile u32 *hws; 500 volatile u32 *hws;
501 int i;
419 502
420 hws = (volatile u32 *)dev_priv->render_ring.status_page.page_addr; 503 ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
504 hws = (volatile u32 *)ring->status_page.page_addr;
421 if (hws == NULL) 505 if (hws == NULL)
422 return 0; 506 return 0;
423 507
@@ -431,14 +515,14 @@ static int i915_hws_info(struct seq_file *m, void *data)
431 515
432static void i915_dump_object(struct seq_file *m, 516static void i915_dump_object(struct seq_file *m,
433 struct io_mapping *mapping, 517 struct io_mapping *mapping,
434 struct drm_i915_gem_object *obj_priv) 518 struct drm_i915_gem_object *obj)
435{ 519{
436 int page, page_count, i; 520 int page, page_count, i;
437 521
438 page_count = obj_priv->base.size / PAGE_SIZE; 522 page_count = obj->base.size / PAGE_SIZE;
439 for (page = 0; page < page_count; page++) { 523 for (page = 0; page < page_count; page++) {
440 u32 *mem = io_mapping_map_wc(mapping, 524 u32 *mem = io_mapping_map_wc(mapping,
441 obj_priv->gtt_offset + page * PAGE_SIZE); 525 obj->gtt_offset + page * PAGE_SIZE);
442 for (i = 0; i < PAGE_SIZE; i += 4) 526 for (i = 0; i < PAGE_SIZE; i += 4)
443 seq_printf(m, "%08x : %08x\n", i, mem[i / 4]); 527 seq_printf(m, "%08x : %08x\n", i, mem[i / 4]);
444 io_mapping_unmap(mem); 528 io_mapping_unmap(mem);
@@ -450,25 +534,21 @@ static int i915_batchbuffer_info(struct seq_file *m, void *data)
450 struct drm_info_node *node = (struct drm_info_node *) m->private; 534 struct drm_info_node *node = (struct drm_info_node *) m->private;
451 struct drm_device *dev = node->minor->dev; 535 struct drm_device *dev = node->minor->dev;
452 drm_i915_private_t *dev_priv = dev->dev_private; 536 drm_i915_private_t *dev_priv = dev->dev_private;
453 struct drm_gem_object *obj; 537 struct drm_i915_gem_object *obj;
454 struct drm_i915_gem_object *obj_priv;
455 int ret; 538 int ret;
456 539
457 ret = mutex_lock_interruptible(&dev->struct_mutex); 540 ret = mutex_lock_interruptible(&dev->struct_mutex);
458 if (ret) 541 if (ret)
459 return ret; 542 return ret;
460 543
461 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 544 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
462 obj = &obj_priv->base; 545 if (obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) {
463 if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) { 546 seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
464 seq_printf(m, "--- gtt_offset = 0x%08x\n", 547 i915_dump_object(m, dev_priv->mm.gtt_mapping, obj);
465 obj_priv->gtt_offset);
466 i915_dump_object(m, dev_priv->mm.gtt_mapping, obj_priv);
467 } 548 }
468 } 549 }
469 550
470 mutex_unlock(&dev->struct_mutex); 551 mutex_unlock(&dev->struct_mutex);
471
472 return 0; 552 return 0;
473} 553}
474 554
@@ -477,19 +557,21 @@ static int i915_ringbuffer_data(struct seq_file *m, void *data)
477 struct drm_info_node *node = (struct drm_info_node *) m->private; 557 struct drm_info_node *node = (struct drm_info_node *) m->private;
478 struct drm_device *dev = node->minor->dev; 558 struct drm_device *dev = node->minor->dev;
479 drm_i915_private_t *dev_priv = dev->dev_private; 559 drm_i915_private_t *dev_priv = dev->dev_private;
560 struct intel_ring_buffer *ring;
480 int ret; 561 int ret;
481 562
482 ret = mutex_lock_interruptible(&dev->struct_mutex); 563 ret = mutex_lock_interruptible(&dev->struct_mutex);
483 if (ret) 564 if (ret)
484 return ret; 565 return ret;
485 566
486 if (!dev_priv->render_ring.gem_object) { 567 ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
568 if (!ring->obj) {
487 seq_printf(m, "No ringbuffer setup\n"); 569 seq_printf(m, "No ringbuffer setup\n");
488 } else { 570 } else {
489 u8 *virt = dev_priv->render_ring.virtual_start; 571 u8 *virt = ring->virtual_start;
490 uint32_t off; 572 uint32_t off;
491 573
492 for (off = 0; off < dev_priv->render_ring.size; off += 4) { 574 for (off = 0; off < ring->size; off += 4) {
493 uint32_t *ptr = (uint32_t *)(virt + off); 575 uint32_t *ptr = (uint32_t *)(virt + off);
494 seq_printf(m, "%08x : %08x\n", off, *ptr); 576 seq_printf(m, "%08x : %08x\n", off, *ptr);
495 } 577 }
@@ -504,19 +586,38 @@ static int i915_ringbuffer_info(struct seq_file *m, void *data)
504 struct drm_info_node *node = (struct drm_info_node *) m->private; 586 struct drm_info_node *node = (struct drm_info_node *) m->private;
505 struct drm_device *dev = node->minor->dev; 587 struct drm_device *dev = node->minor->dev;
506 drm_i915_private_t *dev_priv = dev->dev_private; 588 drm_i915_private_t *dev_priv = dev->dev_private;
507 unsigned int head, tail; 589 struct intel_ring_buffer *ring;
508 590
509 head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 591 ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
510 tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; 592 if (ring->size == 0)
593 return 0;
511 594
512 seq_printf(m, "RingHead : %08x\n", head); 595 seq_printf(m, "Ring %s:\n", ring->name);
513 seq_printf(m, "RingTail : %08x\n", tail); 596 seq_printf(m, " Head : %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR);
514 seq_printf(m, "RingSize : %08lx\n", dev_priv->render_ring.size); 597 seq_printf(m, " Tail : %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR);
515 seq_printf(m, "Acthd : %08x\n", I915_READ(INTEL_INFO(dev)->gen >= 4 ? ACTHD_I965 : ACTHD)); 598 seq_printf(m, " Size : %08x\n", ring->size);
599 seq_printf(m, " Active : %08x\n", intel_ring_get_active_head(ring));
600 seq_printf(m, " NOPID : %08x\n", I915_READ_NOPID(ring));
601 if (IS_GEN6(dev)) {
602 seq_printf(m, " Sync 0 : %08x\n", I915_READ_SYNC_0(ring));
603 seq_printf(m, " Sync 1 : %08x\n", I915_READ_SYNC_1(ring));
604 }
605 seq_printf(m, " Control : %08x\n", I915_READ_CTL(ring));
606 seq_printf(m, " Start : %08x\n", I915_READ_START(ring));
516 607
517 return 0; 608 return 0;
518} 609}
519 610
611static const char *ring_str(int ring)
612{
613 switch (ring) {
614 case RING_RENDER: return " render";
615 case RING_BSD: return " bsd";
616 case RING_BLT: return " blt";
617 default: return "";
618 }
619}
620
520static const char *pin_flag(int pinned) 621static const char *pin_flag(int pinned)
521{ 622{
522 if (pinned > 0) 623 if (pinned > 0)
@@ -547,6 +648,36 @@ static const char *purgeable_flag(int purgeable)
547 return purgeable ? " purgeable" : ""; 648 return purgeable ? " purgeable" : "";
548} 649}
549 650
651static void print_error_buffers(struct seq_file *m,
652 const char *name,
653 struct drm_i915_error_buffer *err,
654 int count)
655{
656 seq_printf(m, "%s [%d]:\n", name, count);
657
658 while (count--) {
659 seq_printf(m, " %08x %8zd %04x %04x %08x%s%s%s%s%s",
660 err->gtt_offset,
661 err->size,
662 err->read_domains,
663 err->write_domain,
664 err->seqno,
665 pin_flag(err->pinned),
666 tiling_flag(err->tiling),
667 dirty_flag(err->dirty),
668 purgeable_flag(err->purgeable),
669 ring_str(err->ring));
670
671 if (err->name)
672 seq_printf(m, " (name: %d)", err->name);
673 if (err->fence_reg != I915_FENCE_REG_NONE)
674 seq_printf(m, " (fence: %d)", err->fence_reg);
675
676 seq_printf(m, "\n");
677 err++;
678 }
679}
680
550static int i915_error_state(struct seq_file *m, void *unused) 681static int i915_error_state(struct seq_file *m, void *unused)
551{ 682{
552 struct drm_info_node *node = (struct drm_info_node *) m->private; 683 struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -568,41 +699,46 @@ static int i915_error_state(struct seq_file *m, void *unused)
568 error->time.tv_usec); 699 error->time.tv_usec);
569 seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device); 700 seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
570 seq_printf(m, "EIR: 0x%08x\n", error->eir); 701 seq_printf(m, "EIR: 0x%08x\n", error->eir);
571 seq_printf(m, " PGTBL_ER: 0x%08x\n", error->pgtbl_er); 702 seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
572 seq_printf(m, " INSTPM: 0x%08x\n", error->instpm); 703 if (INTEL_INFO(dev)->gen >= 6) {
704 seq_printf(m, "ERROR: 0x%08x\n", error->error);
705 seq_printf(m, "Blitter command stream:\n");
706 seq_printf(m, " ACTHD: 0x%08x\n", error->bcs_acthd);
707 seq_printf(m, " IPEIR: 0x%08x\n", error->bcs_ipeir);
708 seq_printf(m, " IPEHR: 0x%08x\n", error->bcs_ipehr);
709 seq_printf(m, " INSTDONE: 0x%08x\n", error->bcs_instdone);
710 seq_printf(m, " seqno: 0x%08x\n", error->bcs_seqno);
711 seq_printf(m, "Video (BSD) command stream:\n");
712 seq_printf(m, " ACTHD: 0x%08x\n", error->vcs_acthd);
713 seq_printf(m, " IPEIR: 0x%08x\n", error->vcs_ipeir);
714 seq_printf(m, " IPEHR: 0x%08x\n", error->vcs_ipehr);
715 seq_printf(m, " INSTDONE: 0x%08x\n", error->vcs_instdone);
716 seq_printf(m, " seqno: 0x%08x\n", error->vcs_seqno);
717 }
718 seq_printf(m, "Render command stream:\n");
719 seq_printf(m, " ACTHD: 0x%08x\n", error->acthd);
573 seq_printf(m, " IPEIR: 0x%08x\n", error->ipeir); 720 seq_printf(m, " IPEIR: 0x%08x\n", error->ipeir);
574 seq_printf(m, " IPEHR: 0x%08x\n", error->ipehr); 721 seq_printf(m, " IPEHR: 0x%08x\n", error->ipehr);
575 seq_printf(m, " INSTDONE: 0x%08x\n", error->instdone); 722 seq_printf(m, " INSTDONE: 0x%08x\n", error->instdone);
576 seq_printf(m, " ACTHD: 0x%08x\n", error->acthd);
577 if (INTEL_INFO(dev)->gen >= 4) { 723 if (INTEL_INFO(dev)->gen >= 4) {
578 seq_printf(m, " INSTPS: 0x%08x\n", error->instps);
579 seq_printf(m, " INSTDONE1: 0x%08x\n", error->instdone1); 724 seq_printf(m, " INSTDONE1: 0x%08x\n", error->instdone1);
725 seq_printf(m, " INSTPS: 0x%08x\n", error->instps);
580 } 726 }
581 seq_printf(m, "seqno: 0x%08x\n", error->seqno); 727 seq_printf(m, " INSTPM: 0x%08x\n", error->instpm);
582 728 seq_printf(m, " seqno: 0x%08x\n", error->seqno);
583 if (error->active_bo_count) { 729
584 seq_printf(m, "Buffers [%d]:\n", error->active_bo_count); 730 for (i = 0; i < 16; i++)
585 731 seq_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]);
586 for (i = 0; i < error->active_bo_count; i++) { 732
587 seq_printf(m, " %08x %8zd %08x %08x %08x%s%s%s%s", 733 if (error->active_bo)
588 error->active_bo[i].gtt_offset, 734 print_error_buffers(m, "Active",
589 error->active_bo[i].size, 735 error->active_bo,
590 error->active_bo[i].read_domains, 736 error->active_bo_count);
591 error->active_bo[i].write_domain, 737
592 error->active_bo[i].seqno, 738 if (error->pinned_bo)
593 pin_flag(error->active_bo[i].pinned), 739 print_error_buffers(m, "Pinned",
594 tiling_flag(error->active_bo[i].tiling), 740 error->pinned_bo,
595 dirty_flag(error->active_bo[i].dirty), 741 error->pinned_bo_count);
596 purgeable_flag(error->active_bo[i].purgeable));
597
598 if (error->active_bo[i].name)
599 seq_printf(m, " (name: %d)", error->active_bo[i].name);
600 if (error->active_bo[i].fence_reg != I915_FENCE_REG_NONE)
601 seq_printf(m, " (fence: %d)", error->active_bo[i].fence_reg);
602
603 seq_printf(m, "\n");
604 }
605 }
606 742
607 for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) { 743 for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) {
608 if (error->batchbuffer[i]) { 744 if (error->batchbuffer[i]) {
@@ -635,6 +771,9 @@ static int i915_error_state(struct seq_file *m, void *unused)
635 if (error->overlay) 771 if (error->overlay)
636 intel_overlay_print_error_state(m, error->overlay); 772 intel_overlay_print_error_state(m, error->overlay);
637 773
774 if (error->display)
775 intel_display_print_error_state(m, dev, error->display);
776
638out: 777out:
639 spin_unlock_irqrestore(&dev_priv->error_lock, flags); 778 spin_unlock_irqrestore(&dev_priv->error_lock, flags);
640 779
@@ -886,7 +1025,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
886 fb->base.height, 1025 fb->base.height,
887 fb->base.depth, 1026 fb->base.depth,
888 fb->base.bits_per_pixel); 1027 fb->base.bits_per_pixel);
889 describe_obj(m, to_intel_bo(fb->obj)); 1028 describe_obj(m, fb->obj);
890 seq_printf(m, "\n"); 1029 seq_printf(m, "\n");
891 1030
892 list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) { 1031 list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) {
@@ -898,7 +1037,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
898 fb->base.height, 1037 fb->base.height,
899 fb->base.depth, 1038 fb->base.depth,
900 fb->base.bits_per_pixel); 1039 fb->base.bits_per_pixel);
901 describe_obj(m, to_intel_bo(fb->obj)); 1040 describe_obj(m, fb->obj);
902 seq_printf(m, "\n"); 1041 seq_printf(m, "\n");
903 } 1042 }
904 1043
@@ -943,7 +1082,6 @@ i915_wedged_write(struct file *filp,
943 loff_t *ppos) 1082 loff_t *ppos)
944{ 1083{
945 struct drm_device *dev = filp->private_data; 1084 struct drm_device *dev = filp->private_data;
946 drm_i915_private_t *dev_priv = dev->dev_private;
947 char buf[20]; 1085 char buf[20];
948 int val = 1; 1086 int val = 1;
949 1087
@@ -959,12 +1097,7 @@ i915_wedged_write(struct file *filp,
959 } 1097 }
960 1098
961 DRM_INFO("Manually setting wedged to %d\n", val); 1099 DRM_INFO("Manually setting wedged to %d\n", val);
962 1100 i915_handle_error(dev, val);
963 atomic_set(&dev_priv->mm.wedged, val);
964 if (val) {
965 wake_up_all(&dev_priv->irq_queue);
966 queue_work(dev_priv->wq, &dev_priv->error_work);
967 }
968 1101
969 return cnt; 1102 return cnt;
970} 1103}
@@ -1028,9 +1161,15 @@ static struct drm_info_list i915_debugfs_list[] = {
1028 {"i915_gem_seqno", i915_gem_seqno_info, 0}, 1161 {"i915_gem_seqno", i915_gem_seqno_info, 0},
1029 {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0}, 1162 {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
1030 {"i915_gem_interrupt", i915_interrupt_info, 0}, 1163 {"i915_gem_interrupt", i915_interrupt_info, 0},
1031 {"i915_gem_hws", i915_hws_info, 0}, 1164 {"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
1032 {"i915_ringbuffer_data", i915_ringbuffer_data, 0}, 1165 {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
1033 {"i915_ringbuffer_info", i915_ringbuffer_info, 0}, 1166 {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
1167 {"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS},
1168 {"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS},
1169 {"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS},
1170 {"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS},
1171 {"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS},
1172 {"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS},
1034 {"i915_batchbuffers", i915_batchbuffer_info, 0}, 1173 {"i915_batchbuffers", i915_batchbuffer_info, 0},
1035 {"i915_error_state", i915_error_state, 0}, 1174 {"i915_error_state", i915_error_state, 0},
1036 {"i915_rstdby_delays", i915_rstdby_delays, 0}, 1175 {"i915_rstdby_delays", i915_rstdby_delays, 0},
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index e6800819bca8..e9fb8953c606 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -49,6 +49,8 @@
49static int i915_init_phys_hws(struct drm_device *dev) 49static int i915_init_phys_hws(struct drm_device *dev)
50{ 50{
51 drm_i915_private_t *dev_priv = dev->dev_private; 51 drm_i915_private_t *dev_priv = dev->dev_private;
52 struct intel_ring_buffer *ring = LP_RING(dev_priv);
53
52 /* Program Hardware Status Page */ 54 /* Program Hardware Status Page */
53 dev_priv->status_page_dmah = 55 dev_priv->status_page_dmah =
54 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE); 56 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE);
@@ -57,11 +59,10 @@ static int i915_init_phys_hws(struct drm_device *dev)
57 DRM_ERROR("Can not allocate hardware status page\n"); 59 DRM_ERROR("Can not allocate hardware status page\n");
58 return -ENOMEM; 60 return -ENOMEM;
59 } 61 }
60 dev_priv->render_ring.status_page.page_addr 62 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
61 = dev_priv->status_page_dmah->vaddr;
62 dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr; 63 dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
63 64
64 memset(dev_priv->render_ring.status_page.page_addr, 0, PAGE_SIZE); 65 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
65 66
66 if (INTEL_INFO(dev)->gen >= 4) 67 if (INTEL_INFO(dev)->gen >= 4)
67 dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) & 68 dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) &
@@ -79,13 +80,15 @@ static int i915_init_phys_hws(struct drm_device *dev)
79static void i915_free_hws(struct drm_device *dev) 80static void i915_free_hws(struct drm_device *dev)
80{ 81{
81 drm_i915_private_t *dev_priv = dev->dev_private; 82 drm_i915_private_t *dev_priv = dev->dev_private;
83 struct intel_ring_buffer *ring = LP_RING(dev_priv);
84
82 if (dev_priv->status_page_dmah) { 85 if (dev_priv->status_page_dmah) {
83 drm_pci_free(dev, dev_priv->status_page_dmah); 86 drm_pci_free(dev, dev_priv->status_page_dmah);
84 dev_priv->status_page_dmah = NULL; 87 dev_priv->status_page_dmah = NULL;
85 } 88 }
86 89
87 if (dev_priv->render_ring.status_page.gfx_addr) { 90 if (ring->status_page.gfx_addr) {
88 dev_priv->render_ring.status_page.gfx_addr = 0; 91 ring->status_page.gfx_addr = 0;
89 drm_core_ioremapfree(&dev_priv->hws_map, dev); 92 drm_core_ioremapfree(&dev_priv->hws_map, dev);
90 } 93 }
91 94
@@ -97,7 +100,7 @@ void i915_kernel_lost_context(struct drm_device * dev)
97{ 100{
98 drm_i915_private_t *dev_priv = dev->dev_private; 101 drm_i915_private_t *dev_priv = dev->dev_private;
99 struct drm_i915_master_private *master_priv; 102 struct drm_i915_master_private *master_priv;
100 struct intel_ring_buffer *ring = &dev_priv->render_ring; 103 struct intel_ring_buffer *ring = LP_RING(dev_priv);
101 104
102 /* 105 /*
103 * We should never lose context on the ring with modesetting 106 * We should never lose context on the ring with modesetting
@@ -106,8 +109,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
106 if (drm_core_check_feature(dev, DRIVER_MODESET)) 109 if (drm_core_check_feature(dev, DRIVER_MODESET))
107 return; 110 return;
108 111
109 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 112 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
110 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; 113 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
111 ring->space = ring->head - (ring->tail + 8); 114 ring->space = ring->head - (ring->tail + 8);
112 if (ring->space < 0) 115 if (ring->space < 0)
113 ring->space += ring->size; 116 ring->space += ring->size;
@@ -123,6 +126,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
123static int i915_dma_cleanup(struct drm_device * dev) 126static int i915_dma_cleanup(struct drm_device * dev)
124{ 127{
125 drm_i915_private_t *dev_priv = dev->dev_private; 128 drm_i915_private_t *dev_priv = dev->dev_private;
129 int i;
130
126 /* Make sure interrupts are disabled here because the uninstall ioctl 131 /* Make sure interrupts are disabled here because the uninstall ioctl
127 * may not have been called from userspace and after dev_private 132 * may not have been called from userspace and after dev_private
128 * is freed, it's too late. 133 * is freed, it's too late.
@@ -131,9 +136,8 @@ static int i915_dma_cleanup(struct drm_device * dev)
131 drm_irq_uninstall(dev); 136 drm_irq_uninstall(dev);
132 137
133 mutex_lock(&dev->struct_mutex); 138 mutex_lock(&dev->struct_mutex);
134 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 139 for (i = 0; i < I915_NUM_RINGS; i++)
135 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 140 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
136 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
137 mutex_unlock(&dev->struct_mutex); 141 mutex_unlock(&dev->struct_mutex);
138 142
139 /* Clear the HWS virtual address at teardown */ 143 /* Clear the HWS virtual address at teardown */
@@ -147,6 +151,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
147{ 151{
148 drm_i915_private_t *dev_priv = dev->dev_private; 152 drm_i915_private_t *dev_priv = dev->dev_private;
149 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; 153 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
154 struct intel_ring_buffer *ring = LP_RING(dev_priv);
150 155
151 master_priv->sarea = drm_getsarea(dev); 156 master_priv->sarea = drm_getsarea(dev);
152 if (master_priv->sarea) { 157 if (master_priv->sarea) {
@@ -157,24 +162,24 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
157 } 162 }
158 163
159 if (init->ring_size != 0) { 164 if (init->ring_size != 0) {
160 if (dev_priv->render_ring.gem_object != NULL) { 165 if (ring->obj != NULL) {
161 i915_dma_cleanup(dev); 166 i915_dma_cleanup(dev);
162 DRM_ERROR("Client tried to initialize ringbuffer in " 167 DRM_ERROR("Client tried to initialize ringbuffer in "
163 "GEM mode\n"); 168 "GEM mode\n");
164 return -EINVAL; 169 return -EINVAL;
165 } 170 }
166 171
167 dev_priv->render_ring.size = init->ring_size; 172 ring->size = init->ring_size;
168 173
169 dev_priv->render_ring.map.offset = init->ring_start; 174 ring->map.offset = init->ring_start;
170 dev_priv->render_ring.map.size = init->ring_size; 175 ring->map.size = init->ring_size;
171 dev_priv->render_ring.map.type = 0; 176 ring->map.type = 0;
172 dev_priv->render_ring.map.flags = 0; 177 ring->map.flags = 0;
173 dev_priv->render_ring.map.mtrr = 0; 178 ring->map.mtrr = 0;
174 179
175 drm_core_ioremap_wc(&dev_priv->render_ring.map, dev); 180 drm_core_ioremap_wc(&ring->map, dev);
176 181
177 if (dev_priv->render_ring.map.handle == NULL) { 182 if (ring->map.handle == NULL) {
178 i915_dma_cleanup(dev); 183 i915_dma_cleanup(dev);
179 DRM_ERROR("can not ioremap virtual address for" 184 DRM_ERROR("can not ioremap virtual address for"
180 " ring buffer\n"); 185 " ring buffer\n");
@@ -182,7 +187,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
182 } 187 }
183 } 188 }
184 189
185 dev_priv->render_ring.virtual_start = dev_priv->render_ring.map.handle; 190 ring->virtual_start = ring->map.handle;
186 191
187 dev_priv->cpp = init->cpp; 192 dev_priv->cpp = init->cpp;
188 dev_priv->back_offset = init->back_offset; 193 dev_priv->back_offset = init->back_offset;
@@ -201,12 +206,10 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
201static int i915_dma_resume(struct drm_device * dev) 206static int i915_dma_resume(struct drm_device * dev)
202{ 207{
203 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 208 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
209 struct intel_ring_buffer *ring = LP_RING(dev_priv);
204 210
205 struct intel_ring_buffer *ring;
206 DRM_DEBUG_DRIVER("%s\n", __func__); 211 DRM_DEBUG_DRIVER("%s\n", __func__);
207 212
208 ring = &dev_priv->render_ring;
209
210 if (ring->map.handle == NULL) { 213 if (ring->map.handle == NULL) {
211 DRM_ERROR("can not ioremap virtual address for" 214 DRM_ERROR("can not ioremap virtual address for"
212 " ring buffer\n"); 215 " ring buffer\n");
@@ -221,7 +224,7 @@ static int i915_dma_resume(struct drm_device * dev)
221 DRM_DEBUG_DRIVER("hw status page @ %p\n", 224 DRM_DEBUG_DRIVER("hw status page @ %p\n",
222 ring->status_page.page_addr); 225 ring->status_page.page_addr);
223 if (ring->status_page.gfx_addr != 0) 226 if (ring->status_page.gfx_addr != 0)
224 intel_ring_setup_status_page(dev, ring); 227 intel_ring_setup_status_page(ring);
225 else 228 else
226 I915_WRITE(HWS_PGA, dev_priv->dma_status_page); 229 I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
227 230
@@ -263,7 +266,7 @@ static int i915_dma_init(struct drm_device *dev, void *data,
263 * instruction detected will be given a size of zero, which is a 266 * instruction detected will be given a size of zero, which is a
264 * signal to abort the rest of the buffer. 267 * signal to abort the rest of the buffer.
265 */ 268 */
266static int do_validate_cmd(int cmd) 269static int validate_cmd(int cmd)
267{ 270{
268 switch (((cmd >> 29) & 0x7)) { 271 switch (((cmd >> 29) & 0x7)) {
269 case 0x0: 272 case 0x0:
@@ -321,40 +324,27 @@ static int do_validate_cmd(int cmd)
321 return 0; 324 return 0;
322} 325}
323 326
324static int validate_cmd(int cmd)
325{
326 int ret = do_validate_cmd(cmd);
327
328/* printk("validate_cmd( %x ): %d\n", cmd, ret); */
329
330 return ret;
331}
332
333static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords) 327static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
334{ 328{
335 drm_i915_private_t *dev_priv = dev->dev_private; 329 drm_i915_private_t *dev_priv = dev->dev_private;
336 int i; 330 int i, ret;
337 331
338 if ((dwords+1) * sizeof(int) >= dev_priv->render_ring.size - 8) 332 if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8)
339 return -EINVAL; 333 return -EINVAL;
340 334
341 BEGIN_LP_RING((dwords+1)&~1);
342
343 for (i = 0; i < dwords;) { 335 for (i = 0; i < dwords;) {
344 int cmd, sz; 336 int sz = validate_cmd(buffer[i]);
345 337 if (sz == 0 || i + sz > dwords)
346 cmd = buffer[i];
347
348 if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
349 return -EINVAL; 338 return -EINVAL;
350 339 i += sz;
351 OUT_RING(cmd);
352
353 while (++i, --sz) {
354 OUT_RING(buffer[i]);
355 }
356 } 340 }
357 341
342 ret = BEGIN_LP_RING((dwords+1)&~1);
343 if (ret)
344 return ret;
345
346 for (i = 0; i < dwords; i++)
347 OUT_RING(buffer[i]);
358 if (dwords & 1) 348 if (dwords & 1)
359 OUT_RING(0); 349 OUT_RING(0);
360 350
@@ -365,34 +355,41 @@ static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
365 355
366int 356int
367i915_emit_box(struct drm_device *dev, 357i915_emit_box(struct drm_device *dev,
368 struct drm_clip_rect *boxes, 358 struct drm_clip_rect *box,
369 int i, int DR1, int DR4) 359 int DR1, int DR4)
370{ 360{
371 struct drm_clip_rect box = boxes[i]; 361 struct drm_i915_private *dev_priv = dev->dev_private;
362 int ret;
372 363
373 if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) { 364 if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
365 box->y2 <= 0 || box->x2 <= 0) {
374 DRM_ERROR("Bad box %d,%d..%d,%d\n", 366 DRM_ERROR("Bad box %d,%d..%d,%d\n",
375 box.x1, box.y1, box.x2, box.y2); 367 box->x1, box->y1, box->x2, box->y2);
376 return -EINVAL; 368 return -EINVAL;
377 } 369 }
378 370
379 if (INTEL_INFO(dev)->gen >= 4) { 371 if (INTEL_INFO(dev)->gen >= 4) {
380 BEGIN_LP_RING(4); 372 ret = BEGIN_LP_RING(4);
373 if (ret)
374 return ret;
375
381 OUT_RING(GFX_OP_DRAWRECT_INFO_I965); 376 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
382 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); 377 OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
383 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); 378 OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
384 OUT_RING(DR4); 379 OUT_RING(DR4);
385 ADVANCE_LP_RING();
386 } else { 380 } else {
387 BEGIN_LP_RING(6); 381 ret = BEGIN_LP_RING(6);
382 if (ret)
383 return ret;
384
388 OUT_RING(GFX_OP_DRAWRECT_INFO); 385 OUT_RING(GFX_OP_DRAWRECT_INFO);
389 OUT_RING(DR1); 386 OUT_RING(DR1);
390 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); 387 OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
391 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); 388 OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
392 OUT_RING(DR4); 389 OUT_RING(DR4);
393 OUT_RING(0); 390 OUT_RING(0);
394 ADVANCE_LP_RING();
395 } 391 }
392 ADVANCE_LP_RING();
396 393
397 return 0; 394 return 0;
398} 395}
@@ -412,12 +409,13 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
412 if (master_priv->sarea_priv) 409 if (master_priv->sarea_priv)
413 master_priv->sarea_priv->last_enqueue = dev_priv->counter; 410 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
414 411
415 BEGIN_LP_RING(4); 412 if (BEGIN_LP_RING(4) == 0) {
416 OUT_RING(MI_STORE_DWORD_INDEX); 413 OUT_RING(MI_STORE_DWORD_INDEX);
417 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 414 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
418 OUT_RING(dev_priv->counter); 415 OUT_RING(dev_priv->counter);
419 OUT_RING(0); 416 OUT_RING(0);
420 ADVANCE_LP_RING(); 417 ADVANCE_LP_RING();
418 }
421} 419}
422 420
423static int i915_dispatch_cmdbuffer(struct drm_device * dev, 421static int i915_dispatch_cmdbuffer(struct drm_device * dev,
@@ -439,7 +437,7 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev,
439 437
440 for (i = 0; i < count; i++) { 438 for (i = 0; i < count; i++) {
441 if (i < nbox) { 439 if (i < nbox) {
442 ret = i915_emit_box(dev, cliprects, i, 440 ret = i915_emit_box(dev, &cliprects[i],
443 cmd->DR1, cmd->DR4); 441 cmd->DR1, cmd->DR4);
444 if (ret) 442 if (ret)
445 return ret; 443 return ret;
@@ -458,8 +456,9 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
458 drm_i915_batchbuffer_t * batch, 456 drm_i915_batchbuffer_t * batch,
459 struct drm_clip_rect *cliprects) 457 struct drm_clip_rect *cliprects)
460{ 458{
459 struct drm_i915_private *dev_priv = dev->dev_private;
461 int nbox = batch->num_cliprects; 460 int nbox = batch->num_cliprects;
462 int i = 0, count; 461 int i, count, ret;
463 462
464 if ((batch->start | batch->used) & 0x7) { 463 if ((batch->start | batch->used) & 0x7) {
465 DRM_ERROR("alignment"); 464 DRM_ERROR("alignment");
@@ -469,17 +468,19 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
469 i915_kernel_lost_context(dev); 468 i915_kernel_lost_context(dev);
470 469
471 count = nbox ? nbox : 1; 470 count = nbox ? nbox : 1;
472
473 for (i = 0; i < count; i++) { 471 for (i = 0; i < count; i++) {
474 if (i < nbox) { 472 if (i < nbox) {
475 int ret = i915_emit_box(dev, cliprects, i, 473 ret = i915_emit_box(dev, &cliprects[i],
476 batch->DR1, batch->DR4); 474 batch->DR1, batch->DR4);
477 if (ret) 475 if (ret)
478 return ret; 476 return ret;
479 } 477 }
480 478
481 if (!IS_I830(dev) && !IS_845G(dev)) { 479 if (!IS_I830(dev) && !IS_845G(dev)) {
482 BEGIN_LP_RING(2); 480 ret = BEGIN_LP_RING(2);
481 if (ret)
482 return ret;
483
483 if (INTEL_INFO(dev)->gen >= 4) { 484 if (INTEL_INFO(dev)->gen >= 4) {
484 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); 485 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
485 OUT_RING(batch->start); 486 OUT_RING(batch->start);
@@ -487,26 +488,29 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
487 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); 488 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
488 OUT_RING(batch->start | MI_BATCH_NON_SECURE); 489 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
489 } 490 }
490 ADVANCE_LP_RING();
491 } else { 491 } else {
492 BEGIN_LP_RING(4); 492 ret = BEGIN_LP_RING(4);
493 if (ret)
494 return ret;
495
493 OUT_RING(MI_BATCH_BUFFER); 496 OUT_RING(MI_BATCH_BUFFER);
494 OUT_RING(batch->start | MI_BATCH_NON_SECURE); 497 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
495 OUT_RING(batch->start + batch->used - 4); 498 OUT_RING(batch->start + batch->used - 4);
496 OUT_RING(0); 499 OUT_RING(0);
497 ADVANCE_LP_RING();
498 } 500 }
501 ADVANCE_LP_RING();
499 } 502 }
500 503
501 504
502 if (IS_G4X(dev) || IS_GEN5(dev)) { 505 if (IS_G4X(dev) || IS_GEN5(dev)) {
503 BEGIN_LP_RING(2); 506 if (BEGIN_LP_RING(2) == 0) {
504 OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP); 507 OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
505 OUT_RING(MI_NOOP); 508 OUT_RING(MI_NOOP);
506 ADVANCE_LP_RING(); 509 ADVANCE_LP_RING();
510 }
507 } 511 }
508 i915_emit_breadcrumb(dev);
509 512
513 i915_emit_breadcrumb(dev);
510 return 0; 514 return 0;
511} 515}
512 516
@@ -515,6 +519,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
515 drm_i915_private_t *dev_priv = dev->dev_private; 519 drm_i915_private_t *dev_priv = dev->dev_private;
516 struct drm_i915_master_private *master_priv = 520 struct drm_i915_master_private *master_priv =
517 dev->primary->master->driver_priv; 521 dev->primary->master->driver_priv;
522 int ret;
518 523
519 if (!master_priv->sarea_priv) 524 if (!master_priv->sarea_priv)
520 return -EINVAL; 525 return -EINVAL;
@@ -526,12 +531,13 @@ static int i915_dispatch_flip(struct drm_device * dev)
526 531
527 i915_kernel_lost_context(dev); 532 i915_kernel_lost_context(dev);
528 533
529 BEGIN_LP_RING(2); 534 ret = BEGIN_LP_RING(10);
535 if (ret)
536 return ret;
537
530 OUT_RING(MI_FLUSH | MI_READ_FLUSH); 538 OUT_RING(MI_FLUSH | MI_READ_FLUSH);
531 OUT_RING(0); 539 OUT_RING(0);
532 ADVANCE_LP_RING();
533 540
534 BEGIN_LP_RING(6);
535 OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP); 541 OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
536 OUT_RING(0); 542 OUT_RING(0);
537 if (dev_priv->current_page == 0) { 543 if (dev_priv->current_page == 0) {
@@ -542,33 +548,32 @@ static int i915_dispatch_flip(struct drm_device * dev)
542 dev_priv->current_page = 0; 548 dev_priv->current_page = 0;
543 } 549 }
544 OUT_RING(0); 550 OUT_RING(0);
545 ADVANCE_LP_RING();
546 551
547 BEGIN_LP_RING(2);
548 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP); 552 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
549 OUT_RING(0); 553 OUT_RING(0);
554
550 ADVANCE_LP_RING(); 555 ADVANCE_LP_RING();
551 556
552 master_priv->sarea_priv->last_enqueue = dev_priv->counter++; 557 master_priv->sarea_priv->last_enqueue = dev_priv->counter++;
553 558
554 BEGIN_LP_RING(4); 559 if (BEGIN_LP_RING(4) == 0) {
555 OUT_RING(MI_STORE_DWORD_INDEX); 560 OUT_RING(MI_STORE_DWORD_INDEX);
556 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 561 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
557 OUT_RING(dev_priv->counter); 562 OUT_RING(dev_priv->counter);
558 OUT_RING(0); 563 OUT_RING(0);
559 ADVANCE_LP_RING(); 564 ADVANCE_LP_RING();
565 }
560 566
561 master_priv->sarea_priv->pf_current_page = dev_priv->current_page; 567 master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
562 return 0; 568 return 0;
563} 569}
564 570
565static int i915_quiescent(struct drm_device * dev) 571static int i915_quiescent(struct drm_device *dev)
566{ 572{
567 drm_i915_private_t *dev_priv = dev->dev_private; 573 struct intel_ring_buffer *ring = LP_RING(dev->dev_private);
568 574
569 i915_kernel_lost_context(dev); 575 i915_kernel_lost_context(dev);
570 return intel_wait_ring_buffer(dev, &dev_priv->render_ring, 576 return intel_wait_ring_buffer(ring, ring->size - 8);
571 dev_priv->render_ring.size - 8);
572} 577}
573 578
574static int i915_flush_ioctl(struct drm_device *dev, void *data, 579static int i915_flush_ioctl(struct drm_device *dev, void *data,
@@ -767,6 +772,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
767 case I915_PARAM_HAS_BLT: 772 case I915_PARAM_HAS_BLT:
768 value = HAS_BLT(dev); 773 value = HAS_BLT(dev);
769 break; 774 break;
775 case I915_PARAM_HAS_RELAXED_FENCING:
776 value = 1;
777 break;
770 case I915_PARAM_HAS_COHERENT_RINGS: 778 case I915_PARAM_HAS_COHERENT_RINGS:
771 value = 1; 779 value = 1;
772 break; 780 break;
@@ -825,7 +833,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
825{ 833{
826 drm_i915_private_t *dev_priv = dev->dev_private; 834 drm_i915_private_t *dev_priv = dev->dev_private;
827 drm_i915_hws_addr_t *hws = data; 835 drm_i915_hws_addr_t *hws = data;
828 struct intel_ring_buffer *ring = &dev_priv->render_ring; 836 struct intel_ring_buffer *ring = LP_RING(dev_priv);
829 837
830 if (!I915_NEED_GFX_HWS(dev)) 838 if (!I915_NEED_GFX_HWS(dev))
831 return -EINVAL; 839 return -EINVAL;
@@ -1004,73 +1012,47 @@ intel_teardown_mchbar(struct drm_device *dev)
1004#define PTE_VALID (1 << 0) 1012#define PTE_VALID (1 << 0)
1005 1013
1006/** 1014/**
1007 * i915_gtt_to_phys - take a GTT address and turn it into a physical one 1015 * i915_stolen_to_phys - take an offset into stolen memory and turn it into
1016 * a physical one
1008 * @dev: drm device 1017 * @dev: drm device
1009 * @gtt_addr: address to translate 1018 * @offset: address to translate
1010 * 1019 *
1011 * Some chip functions require allocations from stolen space but need the 1020 * Some chip functions require allocations from stolen space and need the
1012 * physical address of the memory in question. We use this routine 1021 * physical address of the memory in question.
1013 * to get a physical address suitable for register programming from a given
1014 * GTT address.
1015 */ 1022 */
1016static unsigned long i915_gtt_to_phys(struct drm_device *dev, 1023static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset)
1017 unsigned long gtt_addr)
1018{ 1024{
1019 unsigned long *gtt; 1025 struct drm_i915_private *dev_priv = dev->dev_private;
1020 unsigned long entry, phys; 1026 struct pci_dev *pdev = dev_priv->bridge_dev;
1021 int gtt_bar = IS_GEN2(dev) ? 1 : 0; 1027 u32 base;
1022 int gtt_offset, gtt_size; 1028
1023 1029#if 0
1024 if (INTEL_INFO(dev)->gen >= 4) { 1030 /* On the machines I have tested the Graphics Base of Stolen Memory
1025 if (IS_G4X(dev) || INTEL_INFO(dev)->gen > 4) { 1031 * is unreliable, so compute the base by subtracting the stolen memory
1026 gtt_offset = 2*1024*1024; 1032 * from the Top of Low Usable DRAM which is where the BIOS places
1027 gtt_size = 2*1024*1024; 1033 * the graphics stolen memory.
1028 } else { 1034 */
1029 gtt_offset = 512*1024; 1035 if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
1030 gtt_size = 512*1024; 1036 /* top 32bits are reserved = 0 */
1031 } 1037 pci_read_config_dword(pdev, 0xA4, &base);
1032 } else { 1038 } else {
1033 gtt_bar = 3; 1039 /* XXX presume 8xx is the same as i915 */
1034 gtt_offset = 0; 1040 pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base);
1035 gtt_size = pci_resource_len(dev->pdev, gtt_bar); 1041 }
1036 } 1042#else
1037 1043 if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
1038 gtt = ioremap_wc(pci_resource_start(dev->pdev, gtt_bar) + gtt_offset, 1044 u16 val;
1039 gtt_size); 1045 pci_read_config_word(pdev, 0xb0, &val);
1040 if (!gtt) { 1046 base = val >> 4 << 20;
1041 DRM_ERROR("ioremap of GTT failed\n"); 1047 } else {
1042 return 0; 1048 u8 val;
1043 } 1049 pci_read_config_byte(pdev, 0x9c, &val);
1044 1050 base = val >> 3 << 27;
1045 entry = *(volatile u32 *)(gtt + (gtt_addr / 1024));
1046
1047 DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, PTE: 0x%08lx\n", gtt_addr, entry);
1048
1049 /* Mask out these reserved bits on this hardware. */
1050 if (INTEL_INFO(dev)->gen < 4 && !IS_G33(dev))
1051 entry &= ~PTE_ADDRESS_MASK_HIGH;
1052
1053 /* If it's not a mapping type we know, then bail. */
1054 if ((entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_UNCACHED &&
1055 (entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_CACHED) {
1056 iounmap(gtt);
1057 return 0;
1058 }
1059
1060 if (!(entry & PTE_VALID)) {
1061 DRM_ERROR("bad GTT entry in stolen space\n");
1062 iounmap(gtt);
1063 return 0;
1064 } 1051 }
1052 base -= dev_priv->mm.gtt->stolen_size;
1053#endif
1065 1054
1066 iounmap(gtt); 1055 return base + offset;
1067
1068 phys =(entry & PTE_ADDRESS_MASK) |
1069 ((uint64_t)(entry & PTE_ADDRESS_MASK_HIGH) << (32 - 4));
1070
1071 DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, phys addr: 0x%08lx\n", gtt_addr, phys);
1072
1073 return phys;
1074} 1056}
1075 1057
1076static void i915_warn_stolen(struct drm_device *dev) 1058static void i915_warn_stolen(struct drm_device *dev)
@@ -1086,47 +1068,28 @@ static void i915_setup_compression(struct drm_device *dev, int size)
1086 unsigned long cfb_base; 1068 unsigned long cfb_base;
1087 unsigned long ll_base = 0; 1069 unsigned long ll_base = 0;
1088 1070
1089 /* Leave 1M for line length buffer & misc. */ 1071 compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
1090 compressed_fb = drm_mm_search_free(&dev_priv->mm.vram, size, 4096, 0); 1072 if (compressed_fb)
1091 if (!compressed_fb) { 1073 compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
1092 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; 1074 if (!compressed_fb)
1093 i915_warn_stolen(dev); 1075 goto err;
1094 return;
1095 }
1096
1097 compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
1098 if (!compressed_fb) {
1099 i915_warn_stolen(dev);
1100 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
1101 return;
1102 }
1103 1076
1104 cfb_base = i915_gtt_to_phys(dev, compressed_fb->start); 1077 cfb_base = i915_stolen_to_phys(dev, compressed_fb->start);
1105 if (!cfb_base) { 1078 if (!cfb_base)
1106 DRM_ERROR("failed to get stolen phys addr, disabling FBC\n"); 1079 goto err_fb;
1107 drm_mm_put_block(compressed_fb);
1108 }
1109 1080
1110 if (!(IS_GM45(dev) || IS_IRONLAKE_M(dev))) { 1081 if (!(IS_GM45(dev) || IS_IRONLAKE_M(dev))) {
1111 compressed_llb = drm_mm_search_free(&dev_priv->mm.vram, 4096, 1082 compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
1112 4096, 0); 1083 4096, 4096, 0);
1113 if (!compressed_llb) { 1084 if (compressed_llb)
1114 i915_warn_stolen(dev); 1085 compressed_llb = drm_mm_get_block(compressed_llb,
1115 return; 1086 4096, 4096);
1116 } 1087 if (!compressed_llb)
1117 1088 goto err_fb;
1118 compressed_llb = drm_mm_get_block(compressed_llb, 4096, 4096);
1119 if (!compressed_llb) {
1120 i915_warn_stolen(dev);
1121 return;
1122 }
1123 1089
1124 ll_base = i915_gtt_to_phys(dev, compressed_llb->start); 1090 ll_base = i915_stolen_to_phys(dev, compressed_llb->start);
1125 if (!ll_base) { 1091 if (!ll_base)
1126 DRM_ERROR("failed to get stolen phys addr, disabling FBC\n"); 1092 goto err_llb;
1127 drm_mm_put_block(compressed_fb);
1128 drm_mm_put_block(compressed_llb);
1129 }
1130 } 1093 }
1131 1094
1132 dev_priv->cfb_size = size; 1095 dev_priv->cfb_size = size;
@@ -1143,8 +1106,17 @@ static void i915_setup_compression(struct drm_device *dev, int size)
1143 dev_priv->compressed_llb = compressed_llb; 1106 dev_priv->compressed_llb = compressed_llb;
1144 } 1107 }
1145 1108
1146 DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", cfb_base, 1109 DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n",
1147 ll_base, size >> 20); 1110 cfb_base, ll_base, size >> 20);
1111 return;
1112
1113err_llb:
1114 drm_mm_put_block(compressed_llb);
1115err_fb:
1116 drm_mm_put_block(compressed_fb);
1117err:
1118 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
1119 i915_warn_stolen(dev);
1148} 1120}
1149 1121
1150static void i915_cleanup_compression(struct drm_device *dev) 1122static void i915_cleanup_compression(struct drm_device *dev)
@@ -1195,17 +1167,20 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
1195 return can_switch; 1167 return can_switch;
1196} 1168}
1197 1169
1198static int i915_load_modeset_init(struct drm_device *dev, 1170static int i915_load_modeset_init(struct drm_device *dev)
1199 unsigned long prealloc_size,
1200 unsigned long agp_size)
1201{ 1171{
1202 struct drm_i915_private *dev_priv = dev->dev_private; 1172 struct drm_i915_private *dev_priv = dev->dev_private;
1173 unsigned long prealloc_size, gtt_size, mappable_size;
1203 int ret = 0; 1174 int ret = 0;
1204 1175
1205 /* Basic memrange allocator for stolen space (aka mm.vram) */ 1176 prealloc_size = dev_priv->mm.gtt->stolen_size;
1206 drm_mm_init(&dev_priv->mm.vram, 0, prealloc_size); 1177 gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
1178 mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
1207 1179
1208 /* Let GEM Manage from end of prealloc space to end of aperture. 1180 /* Basic memrange allocator for stolen space */
1181 drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
1182
1183 /* Let GEM Manage all of the aperture.
1209 * 1184 *
1210 * However, leave one page at the end still bound to the scratch page. 1185 * However, leave one page at the end still bound to the scratch page.
1211 * There are a number of places where the hardware apparently 1186 * There are a number of places where the hardware apparently
@@ -1214,7 +1189,7 @@ static int i915_load_modeset_init(struct drm_device *dev,
1214 * at the last page of the aperture. One page should be enough to 1189 * at the last page of the aperture. One page should be enough to
1215 * keep any prefetching inside of the aperture. 1190 * keep any prefetching inside of the aperture.
1216 */ 1191 */
1217 i915_gem_do_init(dev, prealloc_size, agp_size - 4096); 1192 i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE);
1218 1193
1219 mutex_lock(&dev->struct_mutex); 1194 mutex_lock(&dev->struct_mutex);
1220 ret = i915_gem_init_ringbuffer(dev); 1195 ret = i915_gem_init_ringbuffer(dev);
@@ -1226,16 +1201,17 @@ static int i915_load_modeset_init(struct drm_device *dev,
1226 if (I915_HAS_FBC(dev) && i915_powersave) { 1201 if (I915_HAS_FBC(dev) && i915_powersave) {
1227 int cfb_size; 1202 int cfb_size;
1228 1203
1229 /* Try to get an 8M buffer... */ 1204 /* Leave 1M for line length buffer & misc. */
1230 if (prealloc_size > (9*1024*1024)) 1205
1231 cfb_size = 8*1024*1024; 1206 /* Try to get a 32M buffer... */
1207 if (prealloc_size > (36*1024*1024))
1208 cfb_size = 32*1024*1024;
1232 else /* fall back to 7/8 of the stolen space */ 1209 else /* fall back to 7/8 of the stolen space */
1233 cfb_size = prealloc_size * 7 / 8; 1210 cfb_size = prealloc_size * 7 / 8;
1234 i915_setup_compression(dev, cfb_size); 1211 i915_setup_compression(dev, cfb_size);
1235 } 1212 }
1236 1213
1237 /* Allow hardware batchbuffers unless told otherwise. 1214 /* Allow hardware batchbuffers unless told otherwise. */
1238 */
1239 dev_priv->allow_batchbuffer = 1; 1215 dev_priv->allow_batchbuffer = 1;
1240 1216
1241 ret = intel_parse_bios(dev); 1217 ret = intel_parse_bios(dev);
@@ -1425,152 +1401,12 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev)
1425 } 1401 }
1426} 1402}
1427 1403
1428struct v_table { 1404static const struct cparams {
1429 u8 vid; 1405 u16 i;
1430 unsigned long vd; /* in .1 mil */ 1406 u16 t;
1431 unsigned long vm; /* in .1 mil */ 1407 u16 m;
1432 u8 pvid; 1408 u16 c;
1433}; 1409} cparams[] = {
1434
1435static struct v_table v_table[] = {
1436 { 0, 16125, 15000, 0x7f, },
1437 { 1, 16000, 14875, 0x7e, },
1438 { 2, 15875, 14750, 0x7d, },
1439 { 3, 15750, 14625, 0x7c, },
1440 { 4, 15625, 14500, 0x7b, },
1441 { 5, 15500, 14375, 0x7a, },
1442 { 6, 15375, 14250, 0x79, },
1443 { 7, 15250, 14125, 0x78, },
1444 { 8, 15125, 14000, 0x77, },
1445 { 9, 15000, 13875, 0x76, },
1446 { 10, 14875, 13750, 0x75, },
1447 { 11, 14750, 13625, 0x74, },
1448 { 12, 14625, 13500, 0x73, },
1449 { 13, 14500, 13375, 0x72, },
1450 { 14, 14375, 13250, 0x71, },
1451 { 15, 14250, 13125, 0x70, },
1452 { 16, 14125, 13000, 0x6f, },
1453 { 17, 14000, 12875, 0x6e, },
1454 { 18, 13875, 12750, 0x6d, },
1455 { 19, 13750, 12625, 0x6c, },
1456 { 20, 13625, 12500, 0x6b, },
1457 { 21, 13500, 12375, 0x6a, },
1458 { 22, 13375, 12250, 0x69, },
1459 { 23, 13250, 12125, 0x68, },
1460 { 24, 13125, 12000, 0x67, },
1461 { 25, 13000, 11875, 0x66, },
1462 { 26, 12875, 11750, 0x65, },
1463 { 27, 12750, 11625, 0x64, },
1464 { 28, 12625, 11500, 0x63, },
1465 { 29, 12500, 11375, 0x62, },
1466 { 30, 12375, 11250, 0x61, },
1467 { 31, 12250, 11125, 0x60, },
1468 { 32, 12125, 11000, 0x5f, },
1469 { 33, 12000, 10875, 0x5e, },
1470 { 34, 11875, 10750, 0x5d, },
1471 { 35, 11750, 10625, 0x5c, },
1472 { 36, 11625, 10500, 0x5b, },
1473 { 37, 11500, 10375, 0x5a, },
1474 { 38, 11375, 10250, 0x59, },
1475 { 39, 11250, 10125, 0x58, },
1476 { 40, 11125, 10000, 0x57, },
1477 { 41, 11000, 9875, 0x56, },
1478 { 42, 10875, 9750, 0x55, },
1479 { 43, 10750, 9625, 0x54, },
1480 { 44, 10625, 9500, 0x53, },
1481 { 45, 10500, 9375, 0x52, },
1482 { 46, 10375, 9250, 0x51, },
1483 { 47, 10250, 9125, 0x50, },
1484 { 48, 10125, 9000, 0x4f, },
1485 { 49, 10000, 8875, 0x4e, },
1486 { 50, 9875, 8750, 0x4d, },
1487 { 51, 9750, 8625, 0x4c, },
1488 { 52, 9625, 8500, 0x4b, },
1489 { 53, 9500, 8375, 0x4a, },
1490 { 54, 9375, 8250, 0x49, },
1491 { 55, 9250, 8125, 0x48, },
1492 { 56, 9125, 8000, 0x47, },
1493 { 57, 9000, 7875, 0x46, },
1494 { 58, 8875, 7750, 0x45, },
1495 { 59, 8750, 7625, 0x44, },
1496 { 60, 8625, 7500, 0x43, },
1497 { 61, 8500, 7375, 0x42, },
1498 { 62, 8375, 7250, 0x41, },
1499 { 63, 8250, 7125, 0x40, },
1500 { 64, 8125, 7000, 0x3f, },
1501 { 65, 8000, 6875, 0x3e, },
1502 { 66, 7875, 6750, 0x3d, },
1503 { 67, 7750, 6625, 0x3c, },
1504 { 68, 7625, 6500, 0x3b, },
1505 { 69, 7500, 6375, 0x3a, },
1506 { 70, 7375, 6250, 0x39, },
1507 { 71, 7250, 6125, 0x38, },
1508 { 72, 7125, 6000, 0x37, },
1509 { 73, 7000, 5875, 0x36, },
1510 { 74, 6875, 5750, 0x35, },
1511 { 75, 6750, 5625, 0x34, },
1512 { 76, 6625, 5500, 0x33, },
1513 { 77, 6500, 5375, 0x32, },
1514 { 78, 6375, 5250, 0x31, },
1515 { 79, 6250, 5125, 0x30, },
1516 { 80, 6125, 5000, 0x2f, },
1517 { 81, 6000, 4875, 0x2e, },
1518 { 82, 5875, 4750, 0x2d, },
1519 { 83, 5750, 4625, 0x2c, },
1520 { 84, 5625, 4500, 0x2b, },
1521 { 85, 5500, 4375, 0x2a, },
1522 { 86, 5375, 4250, 0x29, },
1523 { 87, 5250, 4125, 0x28, },
1524 { 88, 5125, 4000, 0x27, },
1525 { 89, 5000, 3875, 0x26, },
1526 { 90, 4875, 3750, 0x25, },
1527 { 91, 4750, 3625, 0x24, },
1528 { 92, 4625, 3500, 0x23, },
1529 { 93, 4500, 3375, 0x22, },
1530 { 94, 4375, 3250, 0x21, },
1531 { 95, 4250, 3125, 0x20, },
1532 { 96, 4125, 3000, 0x1f, },
1533 { 97, 4125, 3000, 0x1e, },
1534 { 98, 4125, 3000, 0x1d, },
1535 { 99, 4125, 3000, 0x1c, },
1536 { 100, 4125, 3000, 0x1b, },
1537 { 101, 4125, 3000, 0x1a, },
1538 { 102, 4125, 3000, 0x19, },
1539 { 103, 4125, 3000, 0x18, },
1540 { 104, 4125, 3000, 0x17, },
1541 { 105, 4125, 3000, 0x16, },
1542 { 106, 4125, 3000, 0x15, },
1543 { 107, 4125, 3000, 0x14, },
1544 { 108, 4125, 3000, 0x13, },
1545 { 109, 4125, 3000, 0x12, },
1546 { 110, 4125, 3000, 0x11, },
1547 { 111, 4125, 3000, 0x10, },
1548 { 112, 4125, 3000, 0x0f, },
1549 { 113, 4125, 3000, 0x0e, },
1550 { 114, 4125, 3000, 0x0d, },
1551 { 115, 4125, 3000, 0x0c, },
1552 { 116, 4125, 3000, 0x0b, },
1553 { 117, 4125, 3000, 0x0a, },
1554 { 118, 4125, 3000, 0x09, },
1555 { 119, 4125, 3000, 0x08, },
1556 { 120, 1125, 0, 0x07, },
1557 { 121, 1000, 0, 0x06, },
1558 { 122, 875, 0, 0x05, },
1559 { 123, 750, 0, 0x04, },
1560 { 124, 625, 0, 0x03, },
1561 { 125, 500, 0, 0x02, },
1562 { 126, 375, 0, 0x01, },
1563 { 127, 0, 0, 0x00, },
1564};
1565
1566struct cparams {
1567 int i;
1568 int t;
1569 int m;
1570 int c;
1571};
1572
1573static struct cparams cparams[] = {
1574 { 1, 1333, 301, 28664 }, 1410 { 1, 1333, 301, 28664 },
1575 { 1, 1066, 294, 24460 }, 1411 { 1, 1066, 294, 24460 },
1576 { 1, 800, 294, 25192 }, 1412 { 1, 800, 294, 25192 },
@@ -1636,21 +1472,145 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
1636 return ((m * x) / 127) - b; 1472 return ((m * x) / 127) - b;
1637} 1473}
1638 1474
1639static unsigned long pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) 1475static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
1640{ 1476{
1641 unsigned long val = 0; 1477 static const struct v_table {
1642 int i; 1478 u16 vd; /* in .1 mil */
1643 1479 u16 vm; /* in .1 mil */
1644 for (i = 0; i < ARRAY_SIZE(v_table); i++) { 1480 } v_table[] = {
1645 if (v_table[i].pvid == pxvid) { 1481 { 0, 0, },
1646 if (IS_MOBILE(dev_priv->dev)) 1482 { 375, 0, },
1647 val = v_table[i].vm; 1483 { 500, 0, },
1648 else 1484 { 625, 0, },
1649 val = v_table[i].vd; 1485 { 750, 0, },
1650 } 1486 { 875, 0, },
1651 } 1487 { 1000, 0, },
1652 1488 { 1125, 0, },
1653 return val; 1489 { 4125, 3000, },
1490 { 4125, 3000, },
1491 { 4125, 3000, },
1492 { 4125, 3000, },
1493 { 4125, 3000, },
1494 { 4125, 3000, },
1495 { 4125, 3000, },
1496 { 4125, 3000, },
1497 { 4125, 3000, },
1498 { 4125, 3000, },
1499 { 4125, 3000, },
1500 { 4125, 3000, },
1501 { 4125, 3000, },
1502 { 4125, 3000, },
1503 { 4125, 3000, },
1504 { 4125, 3000, },
1505 { 4125, 3000, },
1506 { 4125, 3000, },
1507 { 4125, 3000, },
1508 { 4125, 3000, },
1509 { 4125, 3000, },
1510 { 4125, 3000, },
1511 { 4125, 3000, },
1512 { 4125, 3000, },
1513 { 4250, 3125, },
1514 { 4375, 3250, },
1515 { 4500, 3375, },
1516 { 4625, 3500, },
1517 { 4750, 3625, },
1518 { 4875, 3750, },
1519 { 5000, 3875, },
1520 { 5125, 4000, },
1521 { 5250, 4125, },
1522 { 5375, 4250, },
1523 { 5500, 4375, },
1524 { 5625, 4500, },
1525 { 5750, 4625, },
1526 { 5875, 4750, },
1527 { 6000, 4875, },
1528 { 6125, 5000, },
1529 { 6250, 5125, },
1530 { 6375, 5250, },
1531 { 6500, 5375, },
1532 { 6625, 5500, },
1533 { 6750, 5625, },
1534 { 6875, 5750, },
1535 { 7000, 5875, },
1536 { 7125, 6000, },
1537 { 7250, 6125, },
1538 { 7375, 6250, },
1539 { 7500, 6375, },
1540 { 7625, 6500, },
1541 { 7750, 6625, },
1542 { 7875, 6750, },
1543 { 8000, 6875, },
1544 { 8125, 7000, },
1545 { 8250, 7125, },
1546 { 8375, 7250, },
1547 { 8500, 7375, },
1548 { 8625, 7500, },
1549 { 8750, 7625, },
1550 { 8875, 7750, },
1551 { 9000, 7875, },
1552 { 9125, 8000, },
1553 { 9250, 8125, },
1554 { 9375, 8250, },
1555 { 9500, 8375, },
1556 { 9625, 8500, },
1557 { 9750, 8625, },
1558 { 9875, 8750, },
1559 { 10000, 8875, },
1560 { 10125, 9000, },
1561 { 10250, 9125, },
1562 { 10375, 9250, },
1563 { 10500, 9375, },
1564 { 10625, 9500, },
1565 { 10750, 9625, },
1566 { 10875, 9750, },
1567 { 11000, 9875, },
1568 { 11125, 10000, },
1569 { 11250, 10125, },
1570 { 11375, 10250, },
1571 { 11500, 10375, },
1572 { 11625, 10500, },
1573 { 11750, 10625, },
1574 { 11875, 10750, },
1575 { 12000, 10875, },
1576 { 12125, 11000, },
1577 { 12250, 11125, },
1578 { 12375, 11250, },
1579 { 12500, 11375, },
1580 { 12625, 11500, },
1581 { 12750, 11625, },
1582 { 12875, 11750, },
1583 { 13000, 11875, },
1584 { 13125, 12000, },
1585 { 13250, 12125, },
1586 { 13375, 12250, },
1587 { 13500, 12375, },
1588 { 13625, 12500, },
1589 { 13750, 12625, },
1590 { 13875, 12750, },
1591 { 14000, 12875, },
1592 { 14125, 13000, },
1593 { 14250, 13125, },
1594 { 14375, 13250, },
1595 { 14500, 13375, },
1596 { 14625, 13500, },
1597 { 14750, 13625, },
1598 { 14875, 13750, },
1599 { 15000, 13875, },
1600 { 15125, 14000, },
1601 { 15250, 14125, },
1602 { 15375, 14250, },
1603 { 15500, 14375, },
1604 { 15625, 14500, },
1605 { 15750, 14625, },
1606 { 15875, 14750, },
1607 { 16000, 14875, },
1608 { 16125, 15000, },
1609 };
1610 if (dev_priv->info->is_mobile)
1611 return v_table[pxvid].vm;
1612 else
1613 return v_table[pxvid].vd;
1654} 1614}
1655 1615
1656void i915_update_gfx_val(struct drm_i915_private *dev_priv) 1616void i915_update_gfx_val(struct drm_i915_private *dev_priv)
@@ -1884,9 +1844,9 @@ EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
1884int i915_driver_load(struct drm_device *dev, unsigned long flags) 1844int i915_driver_load(struct drm_device *dev, unsigned long flags)
1885{ 1845{
1886 struct drm_i915_private *dev_priv; 1846 struct drm_i915_private *dev_priv;
1887 resource_size_t base, size;
1888 int ret = 0, mmio_bar; 1847 int ret = 0, mmio_bar;
1889 uint32_t agp_size, prealloc_size; 1848 uint32_t agp_size;
1849
1890 /* i915 has 4 more counters */ 1850 /* i915 has 4 more counters */
1891 dev->counters += 4; 1851 dev->counters += 4;
1892 dev->types[6] = _DRM_STAT_IRQ; 1852 dev->types[6] = _DRM_STAT_IRQ;
@@ -1902,11 +1862,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1902 dev_priv->dev = dev; 1862 dev_priv->dev = dev;
1903 dev_priv->info = (struct intel_device_info *) flags; 1863 dev_priv->info = (struct intel_device_info *) flags;
1904 1864
1905 /* Add register map (needed for suspend/resume) */
1906 mmio_bar = IS_GEN2(dev) ? 1 : 0;
1907 base = pci_resource_start(dev->pdev, mmio_bar);
1908 size = pci_resource_len(dev->pdev, mmio_bar);
1909
1910 if (i915_get_bridge_dev(dev)) { 1865 if (i915_get_bridge_dev(dev)) {
1911 ret = -EIO; 1866 ret = -EIO;
1912 goto free_priv; 1867 goto free_priv;
@@ -1916,16 +1871,25 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1916 if (IS_GEN2(dev)) 1871 if (IS_GEN2(dev))
1917 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30)); 1872 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
1918 1873
1919 dev_priv->regs = ioremap(base, size); 1874 mmio_bar = IS_GEN2(dev) ? 1 : 0;
1875 dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0);
1920 if (!dev_priv->regs) { 1876 if (!dev_priv->regs) {
1921 DRM_ERROR("failed to map registers\n"); 1877 DRM_ERROR("failed to map registers\n");
1922 ret = -EIO; 1878 ret = -EIO;
1923 goto put_bridge; 1879 goto put_bridge;
1924 } 1880 }
1925 1881
1882 dev_priv->mm.gtt = intel_gtt_get();
1883 if (!dev_priv->mm.gtt) {
1884 DRM_ERROR("Failed to initialize GTT\n");
1885 ret = -ENODEV;
1886 goto out_iomapfree;
1887 }
1888
1889 agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
1890
1926 dev_priv->mm.gtt_mapping = 1891 dev_priv->mm.gtt_mapping =
1927 io_mapping_create_wc(dev->agp->base, 1892 io_mapping_create_wc(dev->agp->base, agp_size);
1928 dev->agp->agp_info.aper_size * 1024*1024);
1929 if (dev_priv->mm.gtt_mapping == NULL) { 1893 if (dev_priv->mm.gtt_mapping == NULL) {
1930 ret = -EIO; 1894 ret = -EIO;
1931 goto out_rmmap; 1895 goto out_rmmap;
@@ -1937,24 +1901,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1937 * MTRR if present. Even if a UC MTRR isn't present. 1901 * MTRR if present. Even if a UC MTRR isn't present.
1938 */ 1902 */
1939 dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base, 1903 dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base,
1940 dev->agp->agp_info.aper_size * 1904 agp_size,
1941 1024 * 1024,
1942 MTRR_TYPE_WRCOMB, 1); 1905 MTRR_TYPE_WRCOMB, 1);
1943 if (dev_priv->mm.gtt_mtrr < 0) { 1906 if (dev_priv->mm.gtt_mtrr < 0) {
1944 DRM_INFO("MTRR allocation failed. Graphics " 1907 DRM_INFO("MTRR allocation failed. Graphics "
1945 "performance may suffer.\n"); 1908 "performance may suffer.\n");
1946 } 1909 }
1947 1910
1948 dev_priv->mm.gtt = intel_gtt_get();
1949 if (!dev_priv->mm.gtt) {
1950 DRM_ERROR("Failed to initialize GTT\n");
1951 ret = -ENODEV;
1952 goto out_iomapfree;
1953 }
1954
1955 prealloc_size = dev_priv->mm.gtt->gtt_stolen_entries << PAGE_SHIFT;
1956 agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
1957
1958 /* The i915 workqueue is primarily used for batched retirement of 1911 /* The i915 workqueue is primarily used for batched retirement of
1959 * requests (and thus managing bo) once the task has been completed 1912 * requests (and thus managing bo) once the task has been completed
1960 * by the GPU. i915_gem_retire_requests() is called directly when we 1913 * by the GPU. i915_gem_retire_requests() is called directly when we
@@ -1962,7 +1915,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1962 * bo. 1915 * bo.
1963 * 1916 *
1964 * It is also used for periodic low-priority events, such as 1917 * It is also used for periodic low-priority events, such as
1965 * idle-timers and hangcheck. 1918 * idle-timers and recording error state.
1966 * 1919 *
1967 * All tasks on the workqueue are expected to acquire the dev mutex 1920 * All tasks on the workqueue are expected to acquire the dev mutex
1968 * so there is no point in running more than one instance of the 1921 * so there is no point in running more than one instance of the
@@ -1980,20 +1933,11 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1980 /* enable GEM by default */ 1933 /* enable GEM by default */
1981 dev_priv->has_gem = 1; 1934 dev_priv->has_gem = 1;
1982 1935
1983 if (prealloc_size > agp_size * 3 / 4) {
1984 DRM_ERROR("Detected broken video BIOS with %d/%dkB of video "
1985 "memory stolen.\n",
1986 prealloc_size / 1024, agp_size / 1024);
1987 DRM_ERROR("Disabling GEM. (try reducing stolen memory or "
1988 "updating the BIOS to fix).\n");
1989 dev_priv->has_gem = 0;
1990 }
1991
1992 if (dev_priv->has_gem == 0 && 1936 if (dev_priv->has_gem == 0 &&
1993 drm_core_check_feature(dev, DRIVER_MODESET)) { 1937 drm_core_check_feature(dev, DRIVER_MODESET)) {
1994 DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n"); 1938 DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n");
1995 ret = -ENODEV; 1939 ret = -ENODEV;
1996 goto out_iomapfree; 1940 goto out_workqueue_free;
1997 } 1941 }
1998 1942
1999 dev->driver->get_vblank_counter = i915_get_vblank_counter; 1943 dev->driver->get_vblank_counter = i915_get_vblank_counter;
@@ -2016,8 +1960,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2016 /* Init HWS */ 1960 /* Init HWS */
2017 if (!I915_NEED_GFX_HWS(dev)) { 1961 if (!I915_NEED_GFX_HWS(dev)) {
2018 ret = i915_init_phys_hws(dev); 1962 ret = i915_init_phys_hws(dev);
2019 if (ret != 0) 1963 if (ret)
2020 goto out_workqueue_free; 1964 goto out_gem_unload;
2021 } 1965 }
2022 1966
2023 if (IS_PINEVIEW(dev)) 1967 if (IS_PINEVIEW(dev))
@@ -2039,16 +1983,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2039 if (!IS_I945G(dev) && !IS_I945GM(dev)) 1983 if (!IS_I945G(dev) && !IS_I945GM(dev))
2040 pci_enable_msi(dev->pdev); 1984 pci_enable_msi(dev->pdev);
2041 1985
2042 spin_lock_init(&dev_priv->user_irq_lock); 1986 spin_lock_init(&dev_priv->irq_lock);
2043 spin_lock_init(&dev_priv->error_lock); 1987 spin_lock_init(&dev_priv->error_lock);
2044 dev_priv->trace_irq_seqno = 0; 1988 dev_priv->trace_irq_seqno = 0;
2045 1989
2046 ret = drm_vblank_init(dev, I915_NUM_PIPE); 1990 ret = drm_vblank_init(dev, I915_NUM_PIPE);
2047 1991 if (ret)
2048 if (ret) { 1992 goto out_gem_unload;
2049 (void) i915_driver_unload(dev);
2050 return ret;
2051 }
2052 1993
2053 /* Start out suspended */ 1994 /* Start out suspended */
2054 dev_priv->mm.suspended = 1; 1995 dev_priv->mm.suspended = 1;
@@ -2056,10 +1997,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2056 intel_detect_pch(dev); 1997 intel_detect_pch(dev);
2057 1998
2058 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1999 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2059 ret = i915_load_modeset_init(dev, prealloc_size, agp_size); 2000 ret = i915_load_modeset_init(dev);
2060 if (ret < 0) { 2001 if (ret < 0) {
2061 DRM_ERROR("failed to init modeset\n"); 2002 DRM_ERROR("failed to init modeset\n");
2062 goto out_workqueue_free; 2003 goto out_gem_unload;
2063 } 2004 }
2064 } 2005 }
2065 2006
@@ -2077,12 +2018,18 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2077 2018
2078 return 0; 2019 return 0;
2079 2020
2021out_gem_unload:
2022 if (dev->pdev->msi_enabled)
2023 pci_disable_msi(dev->pdev);
2024
2025 intel_teardown_gmbus(dev);
2026 intel_teardown_mchbar(dev);
2080out_workqueue_free: 2027out_workqueue_free:
2081 destroy_workqueue(dev_priv->wq); 2028 destroy_workqueue(dev_priv->wq);
2082out_iomapfree: 2029out_iomapfree:
2083 io_mapping_free(dev_priv->mm.gtt_mapping); 2030 io_mapping_free(dev_priv->mm.gtt_mapping);
2084out_rmmap: 2031out_rmmap:
2085 iounmap(dev_priv->regs); 2032 pci_iounmap(dev->pdev, dev_priv->regs);
2086put_bridge: 2033put_bridge:
2087 pci_dev_put(dev_priv->bridge_dev); 2034 pci_dev_put(dev_priv->bridge_dev);
2088free_priv: 2035free_priv:
@@ -2099,6 +2046,9 @@ int i915_driver_unload(struct drm_device *dev)
2099 i915_mch_dev = NULL; 2046 i915_mch_dev = NULL;
2100 spin_unlock(&mchdev_lock); 2047 spin_unlock(&mchdev_lock);
2101 2048
2049 if (dev_priv->mm.inactive_shrinker.shrink)
2050 unregister_shrinker(&dev_priv->mm.inactive_shrinker);
2051
2102 mutex_lock(&dev->struct_mutex); 2052 mutex_lock(&dev->struct_mutex);
2103 ret = i915_gpu_idle(dev); 2053 ret = i915_gpu_idle(dev);
2104 if (ret) 2054 if (ret)
@@ -2156,7 +2106,7 @@ int i915_driver_unload(struct drm_device *dev)
2156 mutex_unlock(&dev->struct_mutex); 2106 mutex_unlock(&dev->struct_mutex);
2157 if (I915_HAS_FBC(dev) && i915_powersave) 2107 if (I915_HAS_FBC(dev) && i915_powersave)
2158 i915_cleanup_compression(dev); 2108 i915_cleanup_compression(dev);
2159 drm_mm_takedown(&dev_priv->mm.vram); 2109 drm_mm_takedown(&dev_priv->mm.stolen);
2160 2110
2161 intel_cleanup_overlay(dev); 2111 intel_cleanup_overlay(dev);
2162 2112
@@ -2165,7 +2115,7 @@ int i915_driver_unload(struct drm_device *dev)
2165 } 2115 }
2166 2116
2167 if (dev_priv->regs != NULL) 2117 if (dev_priv->regs != NULL)
2168 iounmap(dev_priv->regs); 2118 pci_iounmap(dev->pdev, dev_priv->regs);
2169 2119
2170 intel_teardown_gmbus(dev); 2120 intel_teardown_gmbus(dev);
2171 intel_teardown_mchbar(dev); 2121 intel_teardown_mchbar(dev);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index f737960712e6..5f20cd988612 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -111,7 +111,7 @@ static const struct intel_device_info intel_i965g_info = {
111 111
112static const struct intel_device_info intel_i965gm_info = { 112static const struct intel_device_info intel_i965gm_info = {
113 .gen = 4, .is_crestline = 1, 113 .gen = 4, .is_crestline = 1,
114 .is_mobile = 1, .has_fbc = 1, .has_rc6 = 1, .has_hotplug = 1, 114 .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
115 .has_overlay = 1, 115 .has_overlay = 1,
116 .supports_tv = 1, 116 .supports_tv = 1,
117}; 117};
@@ -130,7 +130,7 @@ static const struct intel_device_info intel_g45_info = {
130 130
131static const struct intel_device_info intel_gm45_info = { 131static const struct intel_device_info intel_gm45_info = {
132 .gen = 4, .is_g4x = 1, 132 .gen = 4, .is_g4x = 1,
133 .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_rc6 = 1, 133 .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
134 .has_pipe_cxsr = 1, .has_hotplug = 1, 134 .has_pipe_cxsr = 1, .has_hotplug = 1,
135 .supports_tv = 1, 135 .supports_tv = 1,
136 .has_bsd_ring = 1, 136 .has_bsd_ring = 1,
@@ -150,7 +150,7 @@ static const struct intel_device_info intel_ironlake_d_info = {
150 150
151static const struct intel_device_info intel_ironlake_m_info = { 151static const struct intel_device_info intel_ironlake_m_info = {
152 .gen = 5, .is_mobile = 1, 152 .gen = 5, .is_mobile = 1,
153 .need_gfx_hws = 1, .has_rc6 = 1, .has_hotplug = 1, 153 .need_gfx_hws = 1, .has_hotplug = 1,
154 .has_fbc = 0, /* disabled due to buggy hardware */ 154 .has_fbc = 0, /* disabled due to buggy hardware */
155 .has_bsd_ring = 1, 155 .has_bsd_ring = 1,
156}; 156};
@@ -244,6 +244,28 @@ void intel_detect_pch (struct drm_device *dev)
244 } 244 }
245} 245}
246 246
247void __gen6_force_wake_get(struct drm_i915_private *dev_priv)
248{
249 int count;
250
251 count = 0;
252 while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1))
253 udelay(10);
254
255 I915_WRITE_NOTRACE(FORCEWAKE, 1);
256 POSTING_READ(FORCEWAKE);
257
258 count = 0;
259 while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0)
260 udelay(10);
261}
262
263void __gen6_force_wake_put(struct drm_i915_private *dev_priv)
264{
265 I915_WRITE_NOTRACE(FORCEWAKE, 0);
266 POSTING_READ(FORCEWAKE);
267}
268
247static int i915_drm_freeze(struct drm_device *dev) 269static int i915_drm_freeze(struct drm_device *dev)
248{ 270{
249 struct drm_i915_private *dev_priv = dev->dev_private; 271 struct drm_i915_private *dev_priv = dev->dev_private;
@@ -304,6 +326,12 @@ static int i915_drm_thaw(struct drm_device *dev)
304 struct drm_i915_private *dev_priv = dev->dev_private; 326 struct drm_i915_private *dev_priv = dev->dev_private;
305 int error = 0; 327 int error = 0;
306 328
329 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
330 mutex_lock(&dev->struct_mutex);
331 i915_gem_restore_gtt_mappings(dev);
332 mutex_unlock(&dev->struct_mutex);
333 }
334
307 i915_restore_state(dev); 335 i915_restore_state(dev);
308 intel_opregion_setup(dev); 336 intel_opregion_setup(dev);
309 337
@@ -405,6 +433,14 @@ static int ironlake_do_reset(struct drm_device *dev, u8 flags)
405 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); 433 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
406} 434}
407 435
436static int gen6_do_reset(struct drm_device *dev, u8 flags)
437{
438 struct drm_i915_private *dev_priv = dev->dev_private;
439
440 I915_WRITE(GEN6_GDRST, GEN6_GRDOM_FULL);
441 return wait_for((I915_READ(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
442}
443
408/** 444/**
409 * i965_reset - reset chip after a hang 445 * i965_reset - reset chip after a hang
410 * @dev: drm device to reset 446 * @dev: drm device to reset
@@ -431,7 +467,8 @@ int i915_reset(struct drm_device *dev, u8 flags)
431 bool need_display = true; 467 bool need_display = true;
432 int ret; 468 int ret;
433 469
434 mutex_lock(&dev->struct_mutex); 470 if (!mutex_trylock(&dev->struct_mutex))
471 return -EBUSY;
435 472
436 i915_gem_reset(dev); 473 i915_gem_reset(dev);
437 474
@@ -439,6 +476,9 @@ int i915_reset(struct drm_device *dev, u8 flags)
439 if (get_seconds() - dev_priv->last_gpu_reset < 5) { 476 if (get_seconds() - dev_priv->last_gpu_reset < 5) {
440 DRM_ERROR("GPU hanging too fast, declaring wedged!\n"); 477 DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
441 } else switch (INTEL_INFO(dev)->gen) { 478 } else switch (INTEL_INFO(dev)->gen) {
479 case 6:
480 ret = gen6_do_reset(dev, flags);
481 break;
442 case 5: 482 case 5:
443 ret = ironlake_do_reset(dev, flags); 483 ret = ironlake_do_reset(dev, flags);
444 break; 484 break;
@@ -472,9 +512,14 @@ int i915_reset(struct drm_device *dev, u8 flags)
472 */ 512 */
473 if (drm_core_check_feature(dev, DRIVER_MODESET) || 513 if (drm_core_check_feature(dev, DRIVER_MODESET) ||
474 !dev_priv->mm.suspended) { 514 !dev_priv->mm.suspended) {
475 struct intel_ring_buffer *ring = &dev_priv->render_ring;
476 dev_priv->mm.suspended = 0; 515 dev_priv->mm.suspended = 0;
477 ring->init(dev, ring); 516
517 dev_priv->ring[RCS].init(&dev_priv->ring[RCS]);
518 if (HAS_BSD(dev))
519 dev_priv->ring[VCS].init(&dev_priv->ring[VCS]);
520 if (HAS_BLT(dev))
521 dev_priv->ring[BCS].init(&dev_priv->ring[BCS]);
522
478 mutex_unlock(&dev->struct_mutex); 523 mutex_unlock(&dev->struct_mutex);
479 drm_irq_uninstall(dev); 524 drm_irq_uninstall(dev);
480 drm_irq_install(dev); 525 drm_irq_install(dev);
@@ -661,8 +706,6 @@ static int __init i915_init(void)
661 706
662 driver.num_ioctls = i915_max_ioctl; 707 driver.num_ioctls = i915_max_ioctl;
663 708
664 i915_gem_shrinker_init();
665
666 /* 709 /*
667 * If CONFIG_DRM_I915_KMS is set, default to KMS unless 710 * If CONFIG_DRM_I915_KMS is set, default to KMS unless
668 * explicitly disabled with the module pararmeter. 711 * explicitly disabled with the module pararmeter.
@@ -684,17 +727,11 @@ static int __init i915_init(void)
684 driver.driver_features &= ~DRIVER_MODESET; 727 driver.driver_features &= ~DRIVER_MODESET;
685#endif 728#endif
686 729
687 if (!(driver.driver_features & DRIVER_MODESET)) {
688 driver.suspend = i915_suspend;
689 driver.resume = i915_resume;
690 }
691
692 return drm_init(&driver); 730 return drm_init(&driver);
693} 731}
694 732
695static void __exit i915_exit(void) 733static void __exit i915_exit(void)
696{ 734{
697 i915_gem_shrinker_exit();
698 drm_exit(&driver); 735 drm_exit(&driver);
699} 736}
700 737
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 409826da3099..30780f2cab6f 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -89,7 +89,7 @@ struct drm_i915_gem_phys_object {
89 int id; 89 int id;
90 struct page **page_list; 90 struct page **page_list;
91 drm_dma_handle_t *handle; 91 drm_dma_handle_t *handle;
92 struct drm_gem_object *cur_obj; 92 struct drm_i915_gem_object *cur_obj;
93}; 93};
94 94
95struct mem_block { 95struct mem_block {
@@ -124,9 +124,9 @@ struct drm_i915_master_private {
124#define I915_FENCE_REG_NONE -1 124#define I915_FENCE_REG_NONE -1
125 125
126struct drm_i915_fence_reg { 126struct drm_i915_fence_reg {
127 struct drm_gem_object *obj;
128 struct list_head lru_list; 127 struct list_head lru_list;
129 bool gpu; 128 struct drm_i915_gem_object *obj;
129 uint32_t setup_seqno;
130}; 130};
131 131
132struct sdvo_device_mapping { 132struct sdvo_device_mapping {
@@ -139,6 +139,8 @@ struct sdvo_device_mapping {
139 u8 ddc_pin; 139 u8 ddc_pin;
140}; 140};
141 141
142struct intel_display_error_state;
143
142struct drm_i915_error_state { 144struct drm_i915_error_state {
143 u32 eir; 145 u32 eir;
144 u32 pgtbl_er; 146 u32 pgtbl_er;
@@ -148,11 +150,23 @@ struct drm_i915_error_state {
148 u32 ipehr; 150 u32 ipehr;
149 u32 instdone; 151 u32 instdone;
150 u32 acthd; 152 u32 acthd;
153 u32 error; /* gen6+ */
154 u32 bcs_acthd; /* gen6+ blt engine */
155 u32 bcs_ipehr;
156 u32 bcs_ipeir;
157 u32 bcs_instdone;
158 u32 bcs_seqno;
159 u32 vcs_acthd; /* gen6+ bsd engine */
160 u32 vcs_ipehr;
161 u32 vcs_ipeir;
162 u32 vcs_instdone;
163 u32 vcs_seqno;
151 u32 instpm; 164 u32 instpm;
152 u32 instps; 165 u32 instps;
153 u32 instdone1; 166 u32 instdone1;
154 u32 seqno; 167 u32 seqno;
155 u64 bbaddr; 168 u64 bbaddr;
169 u64 fence[16];
156 struct timeval time; 170 struct timeval time;
157 struct drm_i915_error_object { 171 struct drm_i915_error_object {
158 int page_count; 172 int page_count;
@@ -171,9 +185,11 @@ struct drm_i915_error_state {
171 u32 tiling:2; 185 u32 tiling:2;
172 u32 dirty:1; 186 u32 dirty:1;
173 u32 purgeable:1; 187 u32 purgeable:1;
174 } *active_bo; 188 u32 ring:4;
175 u32 active_bo_count; 189 } *active_bo, *pinned_bo;
190 u32 active_bo_count, pinned_bo_count;
176 struct intel_overlay_error_state *overlay; 191 struct intel_overlay_error_state *overlay;
192 struct intel_display_error_state *display;
177}; 193};
178 194
179struct drm_i915_display_funcs { 195struct drm_i915_display_funcs {
@@ -207,7 +223,6 @@ struct intel_device_info {
207 u8 is_broadwater : 1; 223 u8 is_broadwater : 1;
208 u8 is_crestline : 1; 224 u8 is_crestline : 1;
209 u8 has_fbc : 1; 225 u8 has_fbc : 1;
210 u8 has_rc6 : 1;
211 u8 has_pipe_cxsr : 1; 226 u8 has_pipe_cxsr : 1;
212 u8 has_hotplug : 1; 227 u8 has_hotplug : 1;
213 u8 cursor_needs_physical : 1; 228 u8 cursor_needs_physical : 1;
@@ -253,20 +268,15 @@ typedef struct drm_i915_private {
253 } *gmbus; 268 } *gmbus;
254 269
255 struct pci_dev *bridge_dev; 270 struct pci_dev *bridge_dev;
256 struct intel_ring_buffer render_ring; 271 struct intel_ring_buffer ring[I915_NUM_RINGS];
257 struct intel_ring_buffer bsd_ring;
258 struct intel_ring_buffer blt_ring;
259 uint32_t next_seqno; 272 uint32_t next_seqno;
260 273
261 drm_dma_handle_t *status_page_dmah; 274 drm_dma_handle_t *status_page_dmah;
262 void *seqno_page;
263 dma_addr_t dma_status_page; 275 dma_addr_t dma_status_page;
264 uint32_t counter; 276 uint32_t counter;
265 unsigned int seqno_gfx_addr;
266 drm_local_map_t hws_map; 277 drm_local_map_t hws_map;
267 struct drm_gem_object *seqno_obj; 278 struct drm_i915_gem_object *pwrctx;
268 struct drm_gem_object *pwrctx; 279 struct drm_i915_gem_object *renderctx;
269 struct drm_gem_object *renderctx;
270 280
271 struct resource mch_res; 281 struct resource mch_res;
272 282
@@ -275,25 +285,17 @@ typedef struct drm_i915_private {
275 int front_offset; 285 int front_offset;
276 int current_page; 286 int current_page;
277 int page_flipping; 287 int page_flipping;
278#define I915_DEBUG_READ (1<<0)
279#define I915_DEBUG_WRITE (1<<1)
280 unsigned long debug_flags;
281 288
282 wait_queue_head_t irq_queue;
283 atomic_t irq_received; 289 atomic_t irq_received;
284 /** Protects user_irq_refcount and irq_mask_reg */
285 spinlock_t user_irq_lock;
286 u32 trace_irq_seqno; 290 u32 trace_irq_seqno;
291
292 /* protects the irq masks */
293 spinlock_t irq_lock;
287 /** Cached value of IMR to avoid reads in updating the bitfield */ 294 /** Cached value of IMR to avoid reads in updating the bitfield */
288 u32 irq_mask_reg;
289 u32 pipestat[2]; 295 u32 pipestat[2];
290 /** splitted irq regs for graphics and display engine on Ironlake, 296 u32 irq_mask;
291 irq_mask_reg is still used for display irq. */ 297 u32 gt_irq_mask;
292 u32 gt_irq_mask_reg; 298 u32 pch_irq_mask;
293 u32 gt_irq_enable_reg;
294 u32 de_irq_enable_reg;
295 u32 pch_irq_mask_reg;
296 u32 pch_irq_enable_reg;
297 299
298 u32 hotplug_supported_mask; 300 u32 hotplug_supported_mask;
299 struct work_struct hotplug_work; 301 struct work_struct hotplug_work;
@@ -306,7 +308,7 @@ typedef struct drm_i915_private {
306 int num_pipe; 308 int num_pipe;
307 309
308 /* For hangcheck timer */ 310 /* For hangcheck timer */
309#define DRM_I915_HANGCHECK_PERIOD 250 /* in ms */ 311#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
310 struct timer_list hangcheck_timer; 312 struct timer_list hangcheck_timer;
311 int hangcheck_count; 313 int hangcheck_count;
312 uint32_t last_acthd; 314 uint32_t last_acthd;
@@ -530,23 +532,21 @@ typedef struct drm_i915_private {
530 532
531 struct { 533 struct {
532 /** Bridge to intel-gtt-ko */ 534 /** Bridge to intel-gtt-ko */
533 struct intel_gtt *gtt; 535 const struct intel_gtt *gtt;
534 /** Memory allocator for GTT stolen memory */ 536 /** Memory allocator for GTT stolen memory */
535 struct drm_mm vram; 537 struct drm_mm stolen;
536 /** Memory allocator for GTT */ 538 /** Memory allocator for GTT */
537 struct drm_mm gtt_space; 539 struct drm_mm gtt_space;
540 /** List of all objects in gtt_space. Used to restore gtt
541 * mappings on resume */
542 struct list_head gtt_list;
543 /** End of mappable part of GTT */
544 unsigned long gtt_mappable_end;
538 545
539 struct io_mapping *gtt_mapping; 546 struct io_mapping *gtt_mapping;
540 int gtt_mtrr; 547 int gtt_mtrr;
541 548
542 /** 549 struct shrinker inactive_shrinker;
543 * Membership on list of all loaded devices, used to evict
544 * inactive buffers under memory pressure.
545 *
546 * Modifications should only be done whilst holding the
547 * shrink_list_lock spinlock.
548 */
549 struct list_head shrink_list;
550 550
551 /** 551 /**
552 * List of objects currently involved in rendering. 552 * List of objects currently involved in rendering.
@@ -609,16 +609,6 @@ typedef struct drm_i915_private {
609 struct delayed_work retire_work; 609 struct delayed_work retire_work;
610 610
611 /** 611 /**
612 * Waiting sequence number, if any
613 */
614 uint32_t waiting_gem_seqno;
615
616 /**
617 * Last seq seen at irq time
618 */
619 uint32_t irq_gem_seqno;
620
621 /**
622 * Flag if the X Server, and thus DRM, is not currently in 612 * Flag if the X Server, and thus DRM, is not currently in
623 * control of the device. 613 * control of the device.
624 * 614 *
@@ -645,16 +635,11 @@ typedef struct drm_i915_private {
645 /* storage for physical objects */ 635 /* storage for physical objects */
646 struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT]; 636 struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
647 637
648 uint32_t flush_rings;
649
650 /* accounting, useful for userland debugging */ 638 /* accounting, useful for userland debugging */
651 size_t object_memory;
652 size_t pin_memory;
653 size_t gtt_memory;
654 size_t gtt_total; 639 size_t gtt_total;
640 size_t mappable_gtt_total;
641 size_t object_memory;
655 u32 object_count; 642 u32 object_count;
656 u32 pin_count;
657 u32 gtt_count;
658 } mm; 643 } mm;
659 struct sdvo_device_mapping sdvo_mappings[2]; 644 struct sdvo_device_mapping sdvo_mappings[2];
660 /* indicate whether the LVDS_BORDER should be enabled or not */ 645 /* indicate whether the LVDS_BORDER should be enabled or not */
@@ -688,14 +673,14 @@ typedef struct drm_i915_private {
688 u8 fmax; 673 u8 fmax;
689 u8 fstart; 674 u8 fstart;
690 675
691 u64 last_count1; 676 u64 last_count1;
692 unsigned long last_time1; 677 unsigned long last_time1;
693 u64 last_count2; 678 u64 last_count2;
694 struct timespec last_time2; 679 struct timespec last_time2;
695 unsigned long gfx_power; 680 unsigned long gfx_power;
696 int c_m; 681 int c_m;
697 int r_t; 682 int r_t;
698 u8 corr; 683 u8 corr;
699 spinlock_t *mchdev_lock; 684 spinlock_t *mchdev_lock;
700 685
701 enum no_fbc_reason no_fbc_reason; 686 enum no_fbc_reason no_fbc_reason;
@@ -709,20 +694,20 @@ typedef struct drm_i915_private {
709 struct intel_fbdev *fbdev; 694 struct intel_fbdev *fbdev;
710} drm_i915_private_t; 695} drm_i915_private_t;
711 696
712/** driver private structure attached to each drm_gem_object */
713struct drm_i915_gem_object { 697struct drm_i915_gem_object {
714 struct drm_gem_object base; 698 struct drm_gem_object base;
715 699
716 /** Current space allocated to this object in the GTT, if any. */ 700 /** Current space allocated to this object in the GTT, if any. */
717 struct drm_mm_node *gtt_space; 701 struct drm_mm_node *gtt_space;
702 struct list_head gtt_list;
718 703
719 /** This object's place on the active/flushing/inactive lists */ 704 /** This object's place on the active/flushing/inactive lists */
720 struct list_head ring_list; 705 struct list_head ring_list;
721 struct list_head mm_list; 706 struct list_head mm_list;
722 /** This object's place on GPU write list */ 707 /** This object's place on GPU write list */
723 struct list_head gpu_write_list; 708 struct list_head gpu_write_list;
724 /** This object's place on eviction list */ 709 /** This object's place in the batchbuffer or on the eviction list */
725 struct list_head evict_list; 710 struct list_head exec_list;
726 711
727 /** 712 /**
728 * This is set if the object is on the active or flushing lists 713 * This is set if the object is on the active or flushing lists
@@ -738,6 +723,12 @@ struct drm_i915_gem_object {
738 unsigned int dirty : 1; 723 unsigned int dirty : 1;
739 724
740 /** 725 /**
726 * This is set if the object has been written to since the last
727 * GPU flush.
728 */
729 unsigned int pending_gpu_write : 1;
730
731 /**
741 * Fence register bits (if any) for this object. Will be set 732 * Fence register bits (if any) for this object. Will be set
742 * as needed when mapped into the GTT. 733 * as needed when mapped into the GTT.
743 * Protected by dev->struct_mutex. 734 * Protected by dev->struct_mutex.
@@ -747,29 +738,15 @@ struct drm_i915_gem_object {
747 signed int fence_reg : 5; 738 signed int fence_reg : 5;
748 739
749 /** 740 /**
750 * Used for checking the object doesn't appear more than once
751 * in an execbuffer object list.
752 */
753 unsigned int in_execbuffer : 1;
754
755 /**
756 * Advice: are the backing pages purgeable? 741 * Advice: are the backing pages purgeable?
757 */ 742 */
758 unsigned int madv : 2; 743 unsigned int madv : 2;
759 744
760 /** 745 /**
761 * Refcount for the pages array. With the current locking scheme, there
762 * are at most two concurrent users: Binding a bo to the gtt and
763 * pwrite/pread using physical addresses. So two bits for a maximum
764 * of two users are enough.
765 */
766 unsigned int pages_refcount : 2;
767#define DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT 0x3
768
769 /**
770 * Current tiling mode for the object. 746 * Current tiling mode for the object.
771 */ 747 */
772 unsigned int tiling_mode : 2; 748 unsigned int tiling_mode : 2;
749 unsigned int tiling_changed : 1;
773 750
774 /** How many users have pinned this object in GTT space. The following 751 /** How many users have pinned this object in GTT space. The following
775 * users can each hold at most one reference: pwrite/pread, pin_ioctl 752 * users can each hold at most one reference: pwrite/pread, pin_ioctl
@@ -783,28 +760,54 @@ struct drm_i915_gem_object {
783 unsigned int pin_count : 4; 760 unsigned int pin_count : 4;
784#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf 761#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
785 762
786 /** AGP memory structure for our GTT binding. */ 763 /**
787 DRM_AGP_MEM *agp_mem; 764 * Is the object at the current location in the gtt mappable and
765 * fenceable? Used to avoid costly recalculations.
766 */
767 unsigned int map_and_fenceable : 1;
768
769 /**
770 * Whether the current gtt mapping needs to be mappable (and isn't just
771 * mappable by accident). Track pin and fault separate for a more
772 * accurate mappable working set.
773 */
774 unsigned int fault_mappable : 1;
775 unsigned int pin_mappable : 1;
776
777 /*
778 * Is the GPU currently using a fence to access this buffer,
779 */
780 unsigned int pending_fenced_gpu_access:1;
781 unsigned int fenced_gpu_access:1;
788 782
789 struct page **pages; 783 struct page **pages;
790 784
791 /** 785 /**
792 * Current offset of the object in GTT space. 786 * DMAR support
793 *
794 * This is the same as gtt_space->start
795 */ 787 */
796 uint32_t gtt_offset; 788 struct scatterlist *sg_list;
789 int num_sg;
797 790
798 /* Which ring is refering to is this object */ 791 /**
799 struct intel_ring_buffer *ring; 792 * Used for performing relocations during execbuffer insertion.
793 */
794 struct hlist_node exec_node;
795 unsigned long exec_handle;
800 796
801 /** 797 /**
802 * Fake offset for use by mmap(2) 798 * Current offset of the object in GTT space.
799 *
800 * This is the same as gtt_space->start
803 */ 801 */
804 uint64_t mmap_offset; 802 uint32_t gtt_offset;
805 803
806 /** Breadcrumb of last rendering to the buffer. */ 804 /** Breadcrumb of last rendering to the buffer. */
807 uint32_t last_rendering_seqno; 805 uint32_t last_rendering_seqno;
806 struct intel_ring_buffer *ring;
807
808 /** Breadcrumb of last fenced GPU access to the buffer. */
809 uint32_t last_fenced_seqno;
810 struct intel_ring_buffer *last_fenced_ring;
808 811
809 /** Current tiling stride for the object, if it's tiled. */ 812 /** Current tiling stride for the object, if it's tiled. */
810 uint32_t stride; 813 uint32_t stride;
@@ -880,6 +883,68 @@ enum intel_chip_family {
880 CHIP_I965 = 0x08, 883 CHIP_I965 = 0x08,
881}; 884};
882 885
886#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
887
888#define IS_I830(dev) ((dev)->pci_device == 0x3577)
889#define IS_845G(dev) ((dev)->pci_device == 0x2562)
890#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x)
891#define IS_I865G(dev) ((dev)->pci_device == 0x2572)
892#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g)
893#define IS_I915GM(dev) ((dev)->pci_device == 0x2592)
894#define IS_I945G(dev) ((dev)->pci_device == 0x2772)
895#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm)
896#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater)
897#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline)
898#define IS_GM45(dev) ((dev)->pci_device == 0x2A42)
899#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x)
900#define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001)
901#define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011)
902#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview)
903#define IS_G33(dev) (INTEL_INFO(dev)->is_g33)
904#define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042)
905#define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046)
906#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile)
907
908#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2)
909#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3)
910#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4)
911#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5)
912#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6)
913
914#define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring)
915#define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring)
916#define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws)
917
918#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay)
919#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical)
920
921/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
922 * rows, which changed the alignment requirements and fence programming.
923 */
924#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
925 IS_I915GM(dev)))
926#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
927#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev))
928#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev))
929#define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev))
930#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv)
931#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug)
932/* dsparb controlled by hw only */
933#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
934
935#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
936#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
937#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
938
939#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev))
940#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev))
941
942#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
943#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
944#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
945
946#include "i915_trace.h"
947
883extern struct drm_ioctl_desc i915_ioctls[]; 948extern struct drm_ioctl_desc i915_ioctls[];
884extern int i915_max_ioctl; 949extern int i915_max_ioctl;
885extern unsigned int i915_fbpercrtc; 950extern unsigned int i915_fbpercrtc;
@@ -907,8 +972,8 @@ extern int i915_driver_device_is_agp(struct drm_device * dev);
907extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, 972extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
908 unsigned long arg); 973 unsigned long arg);
909extern int i915_emit_box(struct drm_device *dev, 974extern int i915_emit_box(struct drm_device *dev,
910 struct drm_clip_rect *boxes, 975 struct drm_clip_rect *box,
911 int i, int DR1, int DR4); 976 int DR1, int DR4);
912extern int i915_reset(struct drm_device *dev, u8 flags); 977extern int i915_reset(struct drm_device *dev, u8 flags);
913extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); 978extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
914extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); 979extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
@@ -918,6 +983,7 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
918 983
919/* i915_irq.c */ 984/* i915_irq.c */
920void i915_hangcheck_elapsed(unsigned long data); 985void i915_hangcheck_elapsed(unsigned long data);
986void i915_handle_error(struct drm_device *dev, bool wedged);
921extern int i915_irq_emit(struct drm_device *dev, void *data, 987extern int i915_irq_emit(struct drm_device *dev, void *data,
922 struct drm_file *file_priv); 988 struct drm_file *file_priv);
923extern int i915_irq_wait(struct drm_device *dev, void *data, 989extern int i915_irq_wait(struct drm_device *dev, void *data,
@@ -1017,15 +1083,28 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
1017 struct drm_file *file_priv); 1083 struct drm_file *file_priv);
1018void i915_gem_load(struct drm_device *dev); 1084void i915_gem_load(struct drm_device *dev);
1019int i915_gem_init_object(struct drm_gem_object *obj); 1085int i915_gem_init_object(struct drm_gem_object *obj);
1020struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 1086void i915_gem_flush_ring(struct drm_device *dev,
1021 size_t size); 1087 struct intel_ring_buffer *ring,
1088 uint32_t invalidate_domains,
1089 uint32_t flush_domains);
1090struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
1091 size_t size);
1022void i915_gem_free_object(struct drm_gem_object *obj); 1092void i915_gem_free_object(struct drm_gem_object *obj);
1023int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment); 1093int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
1024void i915_gem_object_unpin(struct drm_gem_object *obj); 1094 uint32_t alignment,
1025int i915_gem_object_unbind(struct drm_gem_object *obj); 1095 bool map_and_fenceable);
1026void i915_gem_release_mmap(struct drm_gem_object *obj); 1096void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
1097int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
1098void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
1027void i915_gem_lastclose(struct drm_device *dev); 1099void i915_gem_lastclose(struct drm_device *dev);
1028 1100
1101int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
1102int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1103 bool interruptible);
1104void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1105 struct intel_ring_buffer *ring,
1106 u32 seqno);
1107
1029/** 1108/**
1030 * Returns true if seq1 is later than seq2. 1109 * Returns true if seq1 is later than seq2.
1031 */ 1110 */
@@ -1035,73 +1114,88 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1035 return (int32_t)(seq1 - seq2) >= 0; 1114 return (int32_t)(seq1 - seq2) >= 0;
1036} 1115}
1037 1116
1038int i915_gem_object_get_fence_reg(struct drm_gem_object *obj, 1117static inline u32
1039 bool interruptible); 1118i915_gem_next_request_seqno(struct drm_device *dev,
1040int i915_gem_object_put_fence_reg(struct drm_gem_object *obj, 1119 struct intel_ring_buffer *ring)
1041 bool interruptible); 1120{
1121 drm_i915_private_t *dev_priv = dev->dev_private;
1122 return ring->outstanding_lazy_request = dev_priv->next_seqno;
1123}
1124
1125int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1126 struct intel_ring_buffer *pipelined,
1127 bool interruptible);
1128int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
1129
1042void i915_gem_retire_requests(struct drm_device *dev); 1130void i915_gem_retire_requests(struct drm_device *dev);
1043void i915_gem_reset(struct drm_device *dev); 1131void i915_gem_reset(struct drm_device *dev);
1044void i915_gem_clflush_object(struct drm_gem_object *obj); 1132void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
1045int i915_gem_object_set_domain(struct drm_gem_object *obj, 1133int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
1046 uint32_t read_domains, 1134 uint32_t read_domains,
1047 uint32_t write_domain); 1135 uint32_t write_domain);
1048int i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj, 1136int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
1049 bool interruptible); 1137 bool interruptible);
1050int i915_gem_init_ringbuffer(struct drm_device *dev); 1138int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
1051void i915_gem_cleanup_ringbuffer(struct drm_device *dev); 1139void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
1052int i915_gem_do_init(struct drm_device *dev, unsigned long start, 1140void i915_gem_do_init(struct drm_device *dev,
1053 unsigned long end); 1141 unsigned long start,
1054int i915_gpu_idle(struct drm_device *dev); 1142 unsigned long mappable_end,
1055int i915_gem_idle(struct drm_device *dev); 1143 unsigned long end);
1056uint32_t i915_add_request(struct drm_device *dev, 1144int __must_check i915_gpu_idle(struct drm_device *dev);
1057 struct drm_file *file_priv, 1145int __must_check i915_gem_idle(struct drm_device *dev);
1058 struct drm_i915_gem_request *request, 1146int __must_check i915_add_request(struct drm_device *dev,
1059 struct intel_ring_buffer *ring); 1147 struct drm_file *file_priv,
1060int i915_do_wait_request(struct drm_device *dev, 1148 struct drm_i915_gem_request *request,
1061 uint32_t seqno, 1149 struct intel_ring_buffer *ring);
1062 bool interruptible, 1150int __must_check i915_do_wait_request(struct drm_device *dev,
1063 struct intel_ring_buffer *ring); 1151 uint32_t seqno,
1152 bool interruptible,
1153 struct intel_ring_buffer *ring);
1064int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); 1154int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
1065int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, 1155int __must_check
1066 int write); 1156i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
1067int i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, 1157 bool write);
1068 bool pipelined); 1158int __must_check
1159i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
1160 struct intel_ring_buffer *pipelined);
1069int i915_gem_attach_phys_object(struct drm_device *dev, 1161int i915_gem_attach_phys_object(struct drm_device *dev,
1070 struct drm_gem_object *obj, 1162 struct drm_i915_gem_object *obj,
1071 int id, 1163 int id,
1072 int align); 1164 int align);
1073void i915_gem_detach_phys_object(struct drm_device *dev, 1165void i915_gem_detach_phys_object(struct drm_device *dev,
1074 struct drm_gem_object *obj); 1166 struct drm_i915_gem_object *obj);
1075void i915_gem_free_all_phys_object(struct drm_device *dev); 1167void i915_gem_free_all_phys_object(struct drm_device *dev);
1076void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv); 1168void i915_gem_release(struct drm_device *dev, struct drm_file *file);
1077 1169
1078void i915_gem_shrinker_init(void); 1170/* i915_gem_gtt.c */
1079void i915_gem_shrinker_exit(void); 1171void i915_gem_restore_gtt_mappings(struct drm_device *dev);
1172int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
1173void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
1080 1174
1081/* i915_gem_evict.c */ 1175/* i915_gem_evict.c */
1082int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment); 1176int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size,
1083int i915_gem_evict_everything(struct drm_device *dev); 1177 unsigned alignment, bool mappable);
1084int i915_gem_evict_inactive(struct drm_device *dev); 1178int __must_check i915_gem_evict_everything(struct drm_device *dev,
1179 bool purgeable_only);
1180int __must_check i915_gem_evict_inactive(struct drm_device *dev,
1181 bool purgeable_only);
1085 1182
1086/* i915_gem_tiling.c */ 1183/* i915_gem_tiling.c */
1087void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); 1184void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
1088void i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj); 1185void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
1089void i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj); 1186void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
1090bool i915_tiling_ok(struct drm_device *dev, int stride, int size,
1091 int tiling_mode);
1092bool i915_gem_object_fence_offset_ok(struct drm_gem_object *obj,
1093 int tiling_mode);
1094 1187
1095/* i915_gem_debug.c */ 1188/* i915_gem_debug.c */
1096void i915_gem_dump_object(struct drm_gem_object *obj, int len, 1189void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
1097 const char *where, uint32_t mark); 1190 const char *where, uint32_t mark);
1098#if WATCH_LISTS 1191#if WATCH_LISTS
1099int i915_verify_lists(struct drm_device *dev); 1192int i915_verify_lists(struct drm_device *dev);
1100#else 1193#else
1101#define i915_verify_lists(dev) 0 1194#define i915_verify_lists(dev) 0
1102#endif 1195#endif
1103void i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle); 1196void i915_gem_object_check_coherency(struct drm_i915_gem_object *obj,
1104void i915_gem_dump_object(struct drm_gem_object *obj, int len, 1197 int handle);
1198void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
1105 const char *where, uint32_t mark); 1199 const char *where, uint32_t mark);
1106 1200
1107/* i915_debugfs.c */ 1201/* i915_debugfs.c */
@@ -1170,79 +1264,120 @@ extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
1170#ifdef CONFIG_DEBUG_FS 1264#ifdef CONFIG_DEBUG_FS
1171extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev); 1265extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
1172extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error); 1266extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error);
1267
1268extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
1269extern void intel_display_print_error_state(struct seq_file *m,
1270 struct drm_device *dev,
1271 struct intel_display_error_state *error);
1173#endif 1272#endif
1174 1273
1274#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
1275
1276#define BEGIN_LP_RING(n) \
1277 intel_ring_begin(LP_RING(dev_priv), (n))
1278
1279#define OUT_RING(x) \
1280 intel_ring_emit(LP_RING(dev_priv), x)
1281
1282#define ADVANCE_LP_RING() \
1283 intel_ring_advance(LP_RING(dev_priv))
1284
1175/** 1285/**
1176 * Lock test for when it's just for synchronization of ring access. 1286 * Lock test for when it's just for synchronization of ring access.
1177 * 1287 *
1178 * In that case, we don't need to do it when GEM is initialized as nobody else 1288 * In that case, we don't need to do it when GEM is initialized as nobody else
1179 * has access to the ring. 1289 * has access to the ring.
1180 */ 1290 */
1181#define RING_LOCK_TEST_WITH_RETURN(dev, file_priv) do { \ 1291#define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \
1182 if (((drm_i915_private_t *)dev->dev_private)->render_ring.gem_object \ 1292 if (LP_RING(dev->dev_private)->obj == NULL) \
1183 == NULL) \ 1293 LOCK_TEST_WITH_RETURN(dev, file); \
1184 LOCK_TEST_WITH_RETURN(dev, file_priv); \
1185} while (0) 1294} while (0)
1186 1295
1187static inline u32 i915_read(struct drm_i915_private *dev_priv, u32 reg) 1296
1297#define __i915_read(x, y) \
1298static inline u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
1299 u##x val = read##y(dev_priv->regs + reg); \
1300 trace_i915_reg_rw('R', reg, val, sizeof(val)); \
1301 return val; \
1302}
1303__i915_read(8, b)
1304__i915_read(16, w)
1305__i915_read(32, l)
1306__i915_read(64, q)
1307#undef __i915_read
1308
1309#define __i915_write(x, y) \
1310static inline void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
1311 trace_i915_reg_rw('W', reg, val, sizeof(val)); \
1312 write##y(val, dev_priv->regs + reg); \
1313}
1314__i915_write(8, b)
1315__i915_write(16, w)
1316__i915_write(32, l)
1317__i915_write(64, q)
1318#undef __i915_write
1319
1320#define I915_READ8(reg) i915_read8(dev_priv, (reg))
1321#define I915_WRITE8(reg, val) i915_write8(dev_priv, (reg), (val))
1322
1323#define I915_READ16(reg) i915_read16(dev_priv, (reg))
1324#define I915_WRITE16(reg, val) i915_write16(dev_priv, (reg), (val))
1325#define I915_READ16_NOTRACE(reg) readw(dev_priv->regs + (reg))
1326#define I915_WRITE16_NOTRACE(reg, val) writew(val, dev_priv->regs + (reg))
1327
1328#define I915_READ(reg) i915_read32(dev_priv, (reg))
1329#define I915_WRITE(reg, val) i915_write32(dev_priv, (reg), (val))
1330#define I915_READ_NOTRACE(reg) readl(dev_priv->regs + (reg))
1331#define I915_WRITE_NOTRACE(reg, val) writel(val, dev_priv->regs + (reg))
1332
1333#define I915_WRITE64(reg, val) i915_write64(dev_priv, (reg), (val))
1334#define I915_READ64(reg) i915_read64(dev_priv, (reg))
1335
1336#define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg)
1337#define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg)
1338
1339
1340/* On SNB platform, before reading ring registers forcewake bit
1341 * must be set to prevent GT core from power down and stale values being
1342 * returned.
1343 */
1344void __gen6_force_wake_get(struct drm_i915_private *dev_priv);
1345void __gen6_force_wake_put (struct drm_i915_private *dev_priv);
1346static inline u32 i915_safe_read(struct drm_i915_private *dev_priv, u32 reg)
1188{ 1347{
1189 u32 val; 1348 u32 val;
1190 1349
1191 val = readl(dev_priv->regs + reg); 1350 if (dev_priv->info->gen >= 6) {
1192 if (dev_priv->debug_flags & I915_DEBUG_READ) 1351 __gen6_force_wake_get(dev_priv);
1193 printk(KERN_ERR "read 0x%08x from 0x%08x\n", val, reg); 1352 val = I915_READ(reg);
1353 __gen6_force_wake_put(dev_priv);
1354 } else
1355 val = I915_READ(reg);
1356
1194 return val; 1357 return val;
1195} 1358}
1196 1359
1197static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg, 1360static inline void
1198 u32 val) 1361i915_write(struct drm_i915_private *dev_priv, u32 reg, u64 val, int len)
1199{ 1362{
1200 writel(val, dev_priv->regs + reg); 1363 /* Trace down the write operation before the real write */
1201 if (dev_priv->debug_flags & I915_DEBUG_WRITE) 1364 trace_i915_reg_rw('W', reg, val, len);
1202 printk(KERN_ERR "wrote 0x%08x to 0x%08x\n", val, reg); 1365 switch (len) {
1366 case 8:
1367 writeq(val, dev_priv->regs + reg);
1368 break;
1369 case 4:
1370 writel(val, dev_priv->regs + reg);
1371 break;
1372 case 2:
1373 writew(val, dev_priv->regs + reg);
1374 break;
1375 case 1:
1376 writeb(val, dev_priv->regs + reg);
1377 break;
1378 }
1203} 1379}
1204 1380
1205#define I915_READ(reg) i915_read(dev_priv, (reg))
1206#define I915_WRITE(reg, val) i915_write(dev_priv, (reg), (val))
1207#define I915_READ16(reg) readw(dev_priv->regs + (reg))
1208#define I915_WRITE16(reg, val) writel(val, dev_priv->regs + (reg))
1209#define I915_READ8(reg) readb(dev_priv->regs + (reg))
1210#define I915_WRITE8(reg, val) writeb(val, dev_priv->regs + (reg))
1211#define I915_WRITE64(reg, val) writeq(val, dev_priv->regs + (reg))
1212#define I915_READ64(reg) readq(dev_priv->regs + (reg))
1213#define POSTING_READ(reg) (void)I915_READ(reg)
1214#define POSTING_READ16(reg) (void)I915_READ16(reg)
1215
1216#define I915_DEBUG_ENABLE_IO() (dev_priv->debug_flags |= I915_DEBUG_READ | \
1217 I915_DEBUG_WRITE)
1218#define I915_DEBUG_DISABLE_IO() (dev_priv->debug_flags &= ~(I915_DEBUG_READ | \
1219 I915_DEBUG_WRITE))
1220
1221#define I915_VERBOSE 0
1222
1223#define BEGIN_LP_RING(n) do { \
1224 drm_i915_private_t *dev_priv__ = dev->dev_private; \
1225 if (I915_VERBOSE) \
1226 DRM_DEBUG(" BEGIN_LP_RING %x\n", (int)(n)); \
1227 intel_ring_begin(dev, &dev_priv__->render_ring, (n)); \
1228} while (0)
1229
1230
1231#define OUT_RING(x) do { \
1232 drm_i915_private_t *dev_priv__ = dev->dev_private; \
1233 if (I915_VERBOSE) \
1234 DRM_DEBUG(" OUT_RING %x\n", (int)(x)); \
1235 intel_ring_emit(dev, &dev_priv__->render_ring, x); \
1236} while (0)
1237
1238#define ADVANCE_LP_RING() do { \
1239 drm_i915_private_t *dev_priv__ = dev->dev_private; \
1240 if (I915_VERBOSE) \
1241 DRM_DEBUG("ADVANCE_LP_RING %x\n", \
1242 dev_priv__->render_ring.tail); \
1243 intel_ring_advance(dev, &dev_priv__->render_ring); \
1244} while(0)
1245
1246/** 1381/**
1247 * Reads a dword out of the status page, which is written to from the command 1382 * Reads a dword out of the status page, which is written to from the command
1248 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or 1383 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
@@ -1259,72 +1394,9 @@ static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg,
1259 * The area from dword 0x20 to 0x3ff is available for driver usage. 1394 * The area from dword 0x20 to 0x3ff is available for driver usage.
1260 */ 1395 */
1261#define READ_HWSP(dev_priv, reg) (((volatile u32 *)\ 1396#define READ_HWSP(dev_priv, reg) (((volatile u32 *)\
1262 (dev_priv->render_ring.status_page.page_addr))[reg]) 1397 (LP_RING(dev_priv)->status_page.page_addr))[reg])
1263#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX) 1398#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
1264#define I915_GEM_HWS_INDEX 0x20 1399#define I915_GEM_HWS_INDEX 0x20
1265#define I915_BREADCRUMB_INDEX 0x21 1400#define I915_BREADCRUMB_INDEX 0x21
1266 1401
1267#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
1268
1269#define IS_I830(dev) ((dev)->pci_device == 0x3577)
1270#define IS_845G(dev) ((dev)->pci_device == 0x2562)
1271#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x)
1272#define IS_I865G(dev) ((dev)->pci_device == 0x2572)
1273#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g)
1274#define IS_I915GM(dev) ((dev)->pci_device == 0x2592)
1275#define IS_I945G(dev) ((dev)->pci_device == 0x2772)
1276#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm)
1277#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater)
1278#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline)
1279#define IS_GM45(dev) ((dev)->pci_device == 0x2A42)
1280#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x)
1281#define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001)
1282#define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011)
1283#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview)
1284#define IS_G33(dev) (INTEL_INFO(dev)->is_g33)
1285#define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042)
1286#define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046)
1287#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile)
1288
1289#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2)
1290#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3)
1291#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4)
1292#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5)
1293#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6)
1294
1295#define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring)
1296#define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring)
1297#define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws)
1298
1299#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay)
1300#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical)
1301
1302/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
1303 * rows, which changed the alignment requirements and fence programming.
1304 */
1305#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
1306 IS_I915GM(dev)))
1307#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
1308#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev))
1309#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev))
1310#define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev))
1311#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv)
1312#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug)
1313/* dsparb controlled by hw only */
1314#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
1315
1316#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
1317#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
1318#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
1319#define I915_HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6)
1320
1321#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev))
1322#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev))
1323
1324#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
1325#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
1326#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
1327
1328#define PRIMARY_RINGBUFFER_SIZE (128*1024)
1329
1330#endif 1402#endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 275ec6ed43ae..726c2ccd674c 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -34,38 +34,31 @@
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/swap.h> 35#include <linux/swap.h>
36#include <linux/pci.h> 36#include <linux/pci.h>
37#include <linux/intel-gtt.h>
38 37
39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); 38static void i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40 39static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); 40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 41static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
43static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 42 bool write);
44static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, 43static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
45 int write);
46static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
47 uint64_t offset, 44 uint64_t offset,
48 uint64_t size); 45 uint64_t size);
49static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); 46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
50static int i915_gem_object_wait_rendering(struct drm_gem_object *obj, 47static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
51 bool interruptible); 48 unsigned alignment,
52static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, 49 bool map_and_fenceable);
53 unsigned alignment); 50static void i915_gem_clear_fence_reg(struct drm_device *dev,
54static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); 51 struct drm_i915_fence_reg *reg);
55static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 52static int i915_gem_phys_pwrite(struct drm_device *dev,
53 struct drm_i915_gem_object *obj,
56 struct drm_i915_gem_pwrite *args, 54 struct drm_i915_gem_pwrite *args,
57 struct drm_file *file_priv); 55 struct drm_file *file);
58static void i915_gem_free_object_tail(struct drm_gem_object *obj); 56static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
59 57
60static int 58static int i915_gem_inactive_shrink(struct shrinker *shrinker,
61i915_gem_object_get_pages(struct drm_gem_object *obj, 59 int nr_to_scan,
62 gfp_t gfpmask); 60 gfp_t gfp_mask);
63 61
64static void
65i915_gem_object_put_pages(struct drm_gem_object *obj);
66
67static LIST_HEAD(shrink_list);
68static DEFINE_SPINLOCK(shrink_list_lock);
69 62
70/* some bookkeeping */ 63/* some bookkeeping */
71static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 64static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
@@ -82,34 +75,6 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
82 dev_priv->mm.object_memory -= size; 75 dev_priv->mm.object_memory -= size;
83} 76}
84 77
85static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
86 size_t size)
87{
88 dev_priv->mm.gtt_count++;
89 dev_priv->mm.gtt_memory += size;
90}
91
92static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
93 size_t size)
94{
95 dev_priv->mm.gtt_count--;
96 dev_priv->mm.gtt_memory -= size;
97}
98
99static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
100 size_t size)
101{
102 dev_priv->mm.pin_count++;
103 dev_priv->mm.pin_memory += size;
104}
105
106static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
107 size_t size)
108{
109 dev_priv->mm.pin_count--;
110 dev_priv->mm.pin_memory -= size;
111}
112
113int 78int
114i915_gem_check_is_wedged(struct drm_device *dev) 79i915_gem_check_is_wedged(struct drm_device *dev)
115{ 80{
@@ -140,7 +105,7 @@ i915_gem_check_is_wedged(struct drm_device *dev)
140 return -EIO; 105 return -EIO;
141} 106}
142 107
143static int i915_mutex_lock_interruptible(struct drm_device *dev) 108int i915_mutex_lock_interruptible(struct drm_device *dev)
144{ 109{
145 struct drm_i915_private *dev_priv = dev->dev_private; 110 struct drm_i915_private *dev_priv = dev->dev_private;
146 int ret; 111 int ret;
@@ -163,75 +128,76 @@ static int i915_mutex_lock_interruptible(struct drm_device *dev)
163} 128}
164 129
165static inline bool 130static inline bool
166i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) 131i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
167{ 132{
168 return obj_priv->gtt_space && 133 return obj->gtt_space && !obj->active && obj->pin_count == 0;
169 !obj_priv->active &&
170 obj_priv->pin_count == 0;
171} 134}
172 135
173int i915_gem_do_init(struct drm_device *dev, 136void i915_gem_do_init(struct drm_device *dev,
174 unsigned long start, 137 unsigned long start,
175 unsigned long end) 138 unsigned long mappable_end,
139 unsigned long end)
176{ 140{
177 drm_i915_private_t *dev_priv = dev->dev_private; 141 drm_i915_private_t *dev_priv = dev->dev_private;
178 142
179 if (start >= end ||
180 (start & (PAGE_SIZE - 1)) != 0 ||
181 (end & (PAGE_SIZE - 1)) != 0) {
182 return -EINVAL;
183 }
184
185 drm_mm_init(&dev_priv->mm.gtt_space, start, 143 drm_mm_init(&dev_priv->mm.gtt_space, start,
186 end - start); 144 end - start);
187 145
188 dev_priv->mm.gtt_total = end - start; 146 dev_priv->mm.gtt_total = end - start;
189 147 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
190 return 0; 148 dev_priv->mm.gtt_mappable_end = mappable_end;
191} 149}
192 150
193int 151int
194i915_gem_init_ioctl(struct drm_device *dev, void *data, 152i915_gem_init_ioctl(struct drm_device *dev, void *data,
195 struct drm_file *file_priv) 153 struct drm_file *file)
196{ 154{
197 struct drm_i915_gem_init *args = data; 155 struct drm_i915_gem_init *args = data;
198 int ret; 156
157 if (args->gtt_start >= args->gtt_end ||
158 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
159 return -EINVAL;
199 160
200 mutex_lock(&dev->struct_mutex); 161 mutex_lock(&dev->struct_mutex);
201 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end); 162 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
202 mutex_unlock(&dev->struct_mutex); 163 mutex_unlock(&dev->struct_mutex);
203 164
204 return ret; 165 return 0;
205} 166}
206 167
207int 168int
208i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 169i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
209 struct drm_file *file_priv) 170 struct drm_file *file)
210{ 171{
211 struct drm_i915_private *dev_priv = dev->dev_private; 172 struct drm_i915_private *dev_priv = dev->dev_private;
212 struct drm_i915_gem_get_aperture *args = data; 173 struct drm_i915_gem_get_aperture *args = data;
174 struct drm_i915_gem_object *obj;
175 size_t pinned;
213 176
214 if (!(dev->driver->driver_features & DRIVER_GEM)) 177 if (!(dev->driver->driver_features & DRIVER_GEM))
215 return -ENODEV; 178 return -ENODEV;
216 179
180 pinned = 0;
217 mutex_lock(&dev->struct_mutex); 181 mutex_lock(&dev->struct_mutex);
218 args->aper_size = dev_priv->mm.gtt_total; 182 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
219 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory; 183 pinned += obj->gtt_space->size;
220 mutex_unlock(&dev->struct_mutex); 184 mutex_unlock(&dev->struct_mutex);
221 185
186 args->aper_size = dev_priv->mm.gtt_total;
187 args->aper_available_size = args->aper_size -pinned;
188
222 return 0; 189 return 0;
223} 190}
224 191
225
226/** 192/**
227 * Creates a new mm object and returns a handle to it. 193 * Creates a new mm object and returns a handle to it.
228 */ 194 */
229int 195int
230i915_gem_create_ioctl(struct drm_device *dev, void *data, 196i915_gem_create_ioctl(struct drm_device *dev, void *data,
231 struct drm_file *file_priv) 197 struct drm_file *file)
232{ 198{
233 struct drm_i915_gem_create *args = data; 199 struct drm_i915_gem_create *args = data;
234 struct drm_gem_object *obj; 200 struct drm_i915_gem_object *obj;
235 int ret; 201 int ret;
236 u32 handle; 202 u32 handle;
237 203
@@ -242,45 +208,28 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
242 if (obj == NULL) 208 if (obj == NULL)
243 return -ENOMEM; 209 return -ENOMEM;
244 210
245 ret = drm_gem_handle_create(file_priv, obj, &handle); 211 ret = drm_gem_handle_create(file, &obj->base, &handle);
246 if (ret) { 212 if (ret) {
247 drm_gem_object_release(obj); 213 drm_gem_object_release(&obj->base);
248 i915_gem_info_remove_obj(dev->dev_private, obj->size); 214 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
249 kfree(obj); 215 kfree(obj);
250 return ret; 216 return ret;
251 } 217 }
252 218
253 /* drop reference from allocate - handle holds it now */ 219 /* drop reference from allocate - handle holds it now */
254 drm_gem_object_unreference(obj); 220 drm_gem_object_unreference(&obj->base);
255 trace_i915_gem_object_create(obj); 221 trace_i915_gem_object_create(obj);
256 222
257 args->handle = handle; 223 args->handle = handle;
258 return 0; 224 return 0;
259} 225}
260 226
261static inline int 227static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
262fast_shmem_read(struct page **pages,
263 loff_t page_base, int page_offset,
264 char __user *data,
265 int length)
266{
267 char *vaddr;
268 int ret;
269
270 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
271 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
272 kunmap_atomic(vaddr);
273
274 return ret;
275}
276
277static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
278{ 228{
279 drm_i915_private_t *dev_priv = obj->dev->dev_private; 229 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
281 230
282 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
283 obj_priv->tiling_mode != I915_TILING_NONE; 232 obj->tiling_mode != I915_TILING_NONE;
284} 233}
285 234
286static inline void 235static inline void
@@ -356,38 +305,51 @@ slow_shmem_bit17_copy(struct page *gpu_page,
356 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). 305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
357 */ 306 */
358static int 307static int
359i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, 308i915_gem_shmem_pread_fast(struct drm_device *dev,
309 struct drm_i915_gem_object *obj,
360 struct drm_i915_gem_pread *args, 310 struct drm_i915_gem_pread *args,
361 struct drm_file *file_priv) 311 struct drm_file *file)
362{ 312{
363 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 313 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
364 ssize_t remain; 314 ssize_t remain;
365 loff_t offset, page_base; 315 loff_t offset;
366 char __user *user_data; 316 char __user *user_data;
367 int page_offset, page_length; 317 int page_offset, page_length;
368 318
369 user_data = (char __user *) (uintptr_t) args->data_ptr; 319 user_data = (char __user *) (uintptr_t) args->data_ptr;
370 remain = args->size; 320 remain = args->size;
371 321
372 obj_priv = to_intel_bo(obj);
373 offset = args->offset; 322 offset = args->offset;
374 323
375 while (remain > 0) { 324 while (remain > 0) {
325 struct page *page;
326 char *vaddr;
327 int ret;
328
376 /* Operation in this page 329 /* Operation in this page
377 * 330 *
378 * page_base = page offset within aperture
379 * page_offset = offset within page 331 * page_offset = offset within page
380 * page_length = bytes to copy for this page 332 * page_length = bytes to copy for this page
381 */ 333 */
382 page_base = (offset & ~(PAGE_SIZE-1));
383 page_offset = offset & (PAGE_SIZE-1); 334 page_offset = offset & (PAGE_SIZE-1);
384 page_length = remain; 335 page_length = remain;
385 if ((page_offset + remain) > PAGE_SIZE) 336 if ((page_offset + remain) > PAGE_SIZE)
386 page_length = PAGE_SIZE - page_offset; 337 page_length = PAGE_SIZE - page_offset;
387 338
388 if (fast_shmem_read(obj_priv->pages, 339 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
389 page_base, page_offset, 340 GFP_HIGHUSER | __GFP_RECLAIMABLE);
390 user_data, page_length)) 341 if (IS_ERR(page))
342 return PTR_ERR(page);
343
344 vaddr = kmap_atomic(page);
345 ret = __copy_to_user_inatomic(user_data,
346 vaddr + page_offset,
347 page_length);
348 kunmap_atomic(vaddr);
349
350 mark_page_accessed(page);
351 page_cache_release(page);
352 if (ret)
391 return -EFAULT; 353 return -EFAULT;
392 354
393 remain -= page_length; 355 remain -= page_length;
@@ -398,30 +360,6 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
398 return 0; 360 return 0;
399} 361}
400 362
401static int
402i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
403{
404 int ret;
405
406 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
407
408 /* If we've insufficient memory to map in the pages, attempt
409 * to make some space by throwing out some old buffers.
410 */
411 if (ret == -ENOMEM) {
412 struct drm_device *dev = obj->dev;
413
414 ret = i915_gem_evict_something(dev, obj->size,
415 i915_gem_get_gtt_alignment(obj));
416 if (ret)
417 return ret;
418
419 ret = i915_gem_object_get_pages(obj, 0);
420 }
421
422 return ret;
423}
424
425/** 363/**
426 * This is the fallback shmem pread path, which allocates temporary storage 364 * This is the fallback shmem pread path, which allocates temporary storage
427 * in kernel space to copy_to_user into outside of the struct_mutex, so we 365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
@@ -429,18 +367,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
429 * and not take page faults. 367 * and not take page faults.
430 */ 368 */
431static int 369static int
432i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, 370i915_gem_shmem_pread_slow(struct drm_device *dev,
371 struct drm_i915_gem_object *obj,
433 struct drm_i915_gem_pread *args, 372 struct drm_i915_gem_pread *args,
434 struct drm_file *file_priv) 373 struct drm_file *file)
435{ 374{
436 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 375 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
437 struct mm_struct *mm = current->mm; 376 struct mm_struct *mm = current->mm;
438 struct page **user_pages; 377 struct page **user_pages;
439 ssize_t remain; 378 ssize_t remain;
440 loff_t offset, pinned_pages, i; 379 loff_t offset, pinned_pages, i;
441 loff_t first_data_page, last_data_page, num_pages; 380 loff_t first_data_page, last_data_page, num_pages;
442 int shmem_page_index, shmem_page_offset; 381 int shmem_page_offset;
443 int data_page_index, data_page_offset; 382 int data_page_index, data_page_offset;
444 int page_length; 383 int page_length;
445 int ret; 384 int ret;
446 uint64_t data_ptr = args->data_ptr; 385 uint64_t data_ptr = args->data_ptr;
@@ -479,19 +418,18 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
479 418
480 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 419 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
481 420
482 obj_priv = to_intel_bo(obj);
483 offset = args->offset; 421 offset = args->offset;
484 422
485 while (remain > 0) { 423 while (remain > 0) {
424 struct page *page;
425
486 /* Operation in this page 426 /* Operation in this page
487 * 427 *
488 * shmem_page_index = page number within shmem file
489 * shmem_page_offset = offset within page in shmem file 428 * shmem_page_offset = offset within page in shmem file
490 * data_page_index = page number in get_user_pages return 429 * data_page_index = page number in get_user_pages return
491 * data_page_offset = offset with data_page_index page. 430 * data_page_offset = offset with data_page_index page.
492 * page_length = bytes to copy for this page 431 * page_length = bytes to copy for this page
493 */ 432 */
494 shmem_page_index = offset / PAGE_SIZE;
495 shmem_page_offset = offset & ~PAGE_MASK; 433 shmem_page_offset = offset & ~PAGE_MASK;
496 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 434 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
497 data_page_offset = data_ptr & ~PAGE_MASK; 435 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -502,8 +440,13 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
502 if ((data_page_offset + page_length) > PAGE_SIZE) 440 if ((data_page_offset + page_length) > PAGE_SIZE)
503 page_length = PAGE_SIZE - data_page_offset; 441 page_length = PAGE_SIZE - data_page_offset;
504 442
443 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
444 GFP_HIGHUSER | __GFP_RECLAIMABLE);
445 if (IS_ERR(page))
446 return PTR_ERR(page);
447
505 if (do_bit17_swizzling) { 448 if (do_bit17_swizzling) {
506 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 449 slow_shmem_bit17_copy(page,
507 shmem_page_offset, 450 shmem_page_offset,
508 user_pages[data_page_index], 451 user_pages[data_page_index],
509 data_page_offset, 452 data_page_offset,
@@ -512,11 +455,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
512 } else { 455 } else {
513 slow_shmem_copy(user_pages[data_page_index], 456 slow_shmem_copy(user_pages[data_page_index],
514 data_page_offset, 457 data_page_offset,
515 obj_priv->pages[shmem_page_index], 458 page,
516 shmem_page_offset, 459 shmem_page_offset,
517 page_length); 460 page_length);
518 } 461 }
519 462
463 mark_page_accessed(page);
464 page_cache_release(page);
465
520 remain -= page_length; 466 remain -= page_length;
521 data_ptr += page_length; 467 data_ptr += page_length;
522 offset += page_length; 468 offset += page_length;
@@ -525,6 +471,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
525out: 471out:
526 for (i = 0; i < pinned_pages; i++) { 472 for (i = 0; i < pinned_pages; i++) {
527 SetPageDirty(user_pages[i]); 473 SetPageDirty(user_pages[i]);
474 mark_page_accessed(user_pages[i]);
528 page_cache_release(user_pages[i]); 475 page_cache_release(user_pages[i]);
529 } 476 }
530 drm_free_large(user_pages); 477 drm_free_large(user_pages);
@@ -539,11 +486,10 @@ out:
539 */ 486 */
540int 487int
541i915_gem_pread_ioctl(struct drm_device *dev, void *data, 488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
542 struct drm_file *file_priv) 489 struct drm_file *file)
543{ 490{
544 struct drm_i915_gem_pread *args = data; 491 struct drm_i915_gem_pread *args = data;
545 struct drm_gem_object *obj; 492 struct drm_i915_gem_object *obj;
546 struct drm_i915_gem_object *obj_priv;
547 int ret = 0; 493 int ret = 0;
548 494
549 if (args->size == 0) 495 if (args->size == 0)
@@ -563,39 +509,33 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
563 if (ret) 509 if (ret)
564 return ret; 510 return ret;
565 511
566 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 512 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
567 if (obj == NULL) { 513 if (obj == NULL) {
568 ret = -ENOENT; 514 ret = -ENOENT;
569 goto unlock; 515 goto unlock;
570 } 516 }
571 obj_priv = to_intel_bo(obj);
572 517
573 /* Bounds check source. */ 518 /* Bounds check source. */
574 if (args->offset > obj->size || args->size > obj->size - args->offset) { 519 if (args->offset > obj->base.size ||
520 args->size > obj->base.size - args->offset) {
575 ret = -EINVAL; 521 ret = -EINVAL;
576 goto out; 522 goto out;
577 } 523 }
578 524
579 ret = i915_gem_object_get_pages_or_evict(obj);
580 if (ret)
581 goto out;
582
583 ret = i915_gem_object_set_cpu_read_domain_range(obj, 525 ret = i915_gem_object_set_cpu_read_domain_range(obj,
584 args->offset, 526 args->offset,
585 args->size); 527 args->size);
586 if (ret) 528 if (ret)
587 goto out_put; 529 goto out;
588 530
589 ret = -EFAULT; 531 ret = -EFAULT;
590 if (!i915_gem_object_needs_bit17_swizzle(obj)) 532 if (!i915_gem_object_needs_bit17_swizzle(obj))
591 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); 533 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
592 if (ret == -EFAULT) 534 if (ret == -EFAULT)
593 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); 535 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
594 536
595out_put:
596 i915_gem_object_put_pages(obj);
597out: 537out:
598 drm_gem_object_unreference(obj); 538 drm_gem_object_unreference(&obj->base);
599unlock: 539unlock:
600 mutex_unlock(&dev->struct_mutex); 540 mutex_unlock(&dev->struct_mutex);
601 return ret; 541 return ret;
@@ -645,32 +585,16 @@ slow_kernel_write(struct io_mapping *mapping,
645 io_mapping_unmap(dst_vaddr); 585 io_mapping_unmap(dst_vaddr);
646} 586}
647 587
648static inline int
649fast_shmem_write(struct page **pages,
650 loff_t page_base, int page_offset,
651 char __user *data,
652 int length)
653{
654 char *vaddr;
655 int ret;
656
657 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
658 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
659 kunmap_atomic(vaddr);
660
661 return ret;
662}
663
664/** 588/**
665 * This is the fast pwrite path, where we copy the data directly from the 589 * This is the fast pwrite path, where we copy the data directly from the
666 * user into the GTT, uncached. 590 * user into the GTT, uncached.
667 */ 591 */
668static int 592static int
669i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 593i915_gem_gtt_pwrite_fast(struct drm_device *dev,
594 struct drm_i915_gem_object *obj,
670 struct drm_i915_gem_pwrite *args, 595 struct drm_i915_gem_pwrite *args,
671 struct drm_file *file_priv) 596 struct drm_file *file)
672{ 597{
673 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
674 drm_i915_private_t *dev_priv = dev->dev_private; 598 drm_i915_private_t *dev_priv = dev->dev_private;
675 ssize_t remain; 599 ssize_t remain;
676 loff_t offset, page_base; 600 loff_t offset, page_base;
@@ -680,8 +604,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
680 user_data = (char __user *) (uintptr_t) args->data_ptr; 604 user_data = (char __user *) (uintptr_t) args->data_ptr;
681 remain = args->size; 605 remain = args->size;
682 606
683 obj_priv = to_intel_bo(obj); 607 offset = obj->gtt_offset + args->offset;
684 offset = obj_priv->gtt_offset + args->offset;
685 608
686 while (remain > 0) { 609 while (remain > 0) {
687 /* Operation in this page 610 /* Operation in this page
@@ -721,11 +644,11 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
721 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). 644 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
722 */ 645 */
723static int 646static int
724i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 647i915_gem_gtt_pwrite_slow(struct drm_device *dev,
648 struct drm_i915_gem_object *obj,
725 struct drm_i915_gem_pwrite *args, 649 struct drm_i915_gem_pwrite *args,
726 struct drm_file *file_priv) 650 struct drm_file *file)
727{ 651{
728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
729 drm_i915_private_t *dev_priv = dev->dev_private; 652 drm_i915_private_t *dev_priv = dev->dev_private;
730 ssize_t remain; 653 ssize_t remain;
731 loff_t gtt_page_base, offset; 654 loff_t gtt_page_base, offset;
@@ -762,12 +685,15 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
762 goto out_unpin_pages; 685 goto out_unpin_pages;
763 } 686 }
764 687
765 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 688 ret = i915_gem_object_set_to_gtt_domain(obj, true);
766 if (ret) 689 if (ret)
767 goto out_unpin_pages; 690 goto out_unpin_pages;
768 691
769 obj_priv = to_intel_bo(obj); 692 ret = i915_gem_object_put_fence(obj);
770 offset = obj_priv->gtt_offset + args->offset; 693 if (ret)
694 goto out_unpin_pages;
695
696 offset = obj->gtt_offset + args->offset;
771 697
772 while (remain > 0) { 698 while (remain > 0) {
773 /* Operation in this page 699 /* Operation in this page
@@ -813,39 +739,58 @@ out_unpin_pages:
813 * copy_from_user into the kmapped pages backing the object. 739 * copy_from_user into the kmapped pages backing the object.
814 */ 740 */
815static int 741static int
816i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 742i915_gem_shmem_pwrite_fast(struct drm_device *dev,
743 struct drm_i915_gem_object *obj,
817 struct drm_i915_gem_pwrite *args, 744 struct drm_i915_gem_pwrite *args,
818 struct drm_file *file_priv) 745 struct drm_file *file)
819{ 746{
820 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 747 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
821 ssize_t remain; 748 ssize_t remain;
822 loff_t offset, page_base; 749 loff_t offset;
823 char __user *user_data; 750 char __user *user_data;
824 int page_offset, page_length; 751 int page_offset, page_length;
825 752
826 user_data = (char __user *) (uintptr_t) args->data_ptr; 753 user_data = (char __user *) (uintptr_t) args->data_ptr;
827 remain = args->size; 754 remain = args->size;
828 755
829 obj_priv = to_intel_bo(obj);
830 offset = args->offset; 756 offset = args->offset;
831 obj_priv->dirty = 1; 757 obj->dirty = 1;
832 758
833 while (remain > 0) { 759 while (remain > 0) {
760 struct page *page;
761 char *vaddr;
762 int ret;
763
834 /* Operation in this page 764 /* Operation in this page
835 * 765 *
836 * page_base = page offset within aperture
837 * page_offset = offset within page 766 * page_offset = offset within page
838 * page_length = bytes to copy for this page 767 * page_length = bytes to copy for this page
839 */ 768 */
840 page_base = (offset & ~(PAGE_SIZE-1));
841 page_offset = offset & (PAGE_SIZE-1); 769 page_offset = offset & (PAGE_SIZE-1);
842 page_length = remain; 770 page_length = remain;
843 if ((page_offset + remain) > PAGE_SIZE) 771 if ((page_offset + remain) > PAGE_SIZE)
844 page_length = PAGE_SIZE - page_offset; 772 page_length = PAGE_SIZE - page_offset;
845 773
846 if (fast_shmem_write(obj_priv->pages, 774 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
847 page_base, page_offset, 775 GFP_HIGHUSER | __GFP_RECLAIMABLE);
848 user_data, page_length)) 776 if (IS_ERR(page))
777 return PTR_ERR(page);
778
779 vaddr = kmap_atomic(page, KM_USER0);
780 ret = __copy_from_user_inatomic(vaddr + page_offset,
781 user_data,
782 page_length);
783 kunmap_atomic(vaddr, KM_USER0);
784
785 set_page_dirty(page);
786 mark_page_accessed(page);
787 page_cache_release(page);
788
789 /* If we get a fault while copying data, then (presumably) our
790 * source page isn't available. Return the error and we'll
791 * retry in the slow path.
792 */
793 if (ret)
849 return -EFAULT; 794 return -EFAULT;
850 795
851 remain -= page_length; 796 remain -= page_length;
@@ -864,17 +809,18 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
864 * struct_mutex is held. 809 * struct_mutex is held.
865 */ 810 */
866static int 811static int
867i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 812i915_gem_shmem_pwrite_slow(struct drm_device *dev,
813 struct drm_i915_gem_object *obj,
868 struct drm_i915_gem_pwrite *args, 814 struct drm_i915_gem_pwrite *args,
869 struct drm_file *file_priv) 815 struct drm_file *file)
870{ 816{
871 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 817 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
872 struct mm_struct *mm = current->mm; 818 struct mm_struct *mm = current->mm;
873 struct page **user_pages; 819 struct page **user_pages;
874 ssize_t remain; 820 ssize_t remain;
875 loff_t offset, pinned_pages, i; 821 loff_t offset, pinned_pages, i;
876 loff_t first_data_page, last_data_page, num_pages; 822 loff_t first_data_page, last_data_page, num_pages;
877 int shmem_page_index, shmem_page_offset; 823 int shmem_page_offset;
878 int data_page_index, data_page_offset; 824 int data_page_index, data_page_offset;
879 int page_length; 825 int page_length;
880 int ret; 826 int ret;
@@ -912,20 +858,19 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
912 858
913 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 859 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
914 860
915 obj_priv = to_intel_bo(obj);
916 offset = args->offset; 861 offset = args->offset;
917 obj_priv->dirty = 1; 862 obj->dirty = 1;
918 863
919 while (remain > 0) { 864 while (remain > 0) {
865 struct page *page;
866
920 /* Operation in this page 867 /* Operation in this page
921 * 868 *
922 * shmem_page_index = page number within shmem file
923 * shmem_page_offset = offset within page in shmem file 869 * shmem_page_offset = offset within page in shmem file
924 * data_page_index = page number in get_user_pages return 870 * data_page_index = page number in get_user_pages return
925 * data_page_offset = offset with data_page_index page. 871 * data_page_offset = offset with data_page_index page.
926 * page_length = bytes to copy for this page 872 * page_length = bytes to copy for this page
927 */ 873 */
928 shmem_page_index = offset / PAGE_SIZE;
929 shmem_page_offset = offset & ~PAGE_MASK; 874 shmem_page_offset = offset & ~PAGE_MASK;
930 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 875 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
931 data_page_offset = data_ptr & ~PAGE_MASK; 876 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -936,21 +881,32 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
936 if ((data_page_offset + page_length) > PAGE_SIZE) 881 if ((data_page_offset + page_length) > PAGE_SIZE)
937 page_length = PAGE_SIZE - data_page_offset; 882 page_length = PAGE_SIZE - data_page_offset;
938 883
884 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
885 GFP_HIGHUSER | __GFP_RECLAIMABLE);
886 if (IS_ERR(page)) {
887 ret = PTR_ERR(page);
888 goto out;
889 }
890
939 if (do_bit17_swizzling) { 891 if (do_bit17_swizzling) {
940 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 892 slow_shmem_bit17_copy(page,
941 shmem_page_offset, 893 shmem_page_offset,
942 user_pages[data_page_index], 894 user_pages[data_page_index],
943 data_page_offset, 895 data_page_offset,
944 page_length, 896 page_length,
945 0); 897 0);
946 } else { 898 } else {
947 slow_shmem_copy(obj_priv->pages[shmem_page_index], 899 slow_shmem_copy(page,
948 shmem_page_offset, 900 shmem_page_offset,
949 user_pages[data_page_index], 901 user_pages[data_page_index],
950 data_page_offset, 902 data_page_offset,
951 page_length); 903 page_length);
952 } 904 }
953 905
906 set_page_dirty(page);
907 mark_page_accessed(page);
908 page_cache_release(page);
909
954 remain -= page_length; 910 remain -= page_length;
955 data_ptr += page_length; 911 data_ptr += page_length;
956 offset += page_length; 912 offset += page_length;
@@ -974,8 +930,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
974 struct drm_file *file) 930 struct drm_file *file)
975{ 931{
976 struct drm_i915_gem_pwrite *args = data; 932 struct drm_i915_gem_pwrite *args = data;
977 struct drm_gem_object *obj; 933 struct drm_i915_gem_object *obj;
978 struct drm_i915_gem_object *obj_priv;
979 int ret; 934 int ret;
980 935
981 if (args->size == 0) 936 if (args->size == 0)
@@ -995,15 +950,15 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
995 if (ret) 950 if (ret)
996 return ret; 951 return ret;
997 952
998 obj = drm_gem_object_lookup(dev, file, args->handle); 953 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
999 if (obj == NULL) { 954 if (obj == NULL) {
1000 ret = -ENOENT; 955 ret = -ENOENT;
1001 goto unlock; 956 goto unlock;
1002 } 957 }
1003 obj_priv = to_intel_bo(obj);
1004 958
1005 /* Bounds check destination. */ 959 /* Bounds check destination. */
1006 if (args->offset > obj->size || args->size > obj->size - args->offset) { 960 if (args->offset > obj->base.size ||
961 args->size > obj->base.size - args->offset) {
1007 ret = -EINVAL; 962 ret = -EINVAL;
1008 goto out; 963 goto out;
1009 } 964 }
@@ -1014,16 +969,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1014 * pread/pwrite currently are reading and writing from the CPU 969 * pread/pwrite currently are reading and writing from the CPU
1015 * perspective, requiring manual detiling by the client. 970 * perspective, requiring manual detiling by the client.
1016 */ 971 */
1017 if (obj_priv->phys_obj) 972 if (obj->phys_obj)
1018 ret = i915_gem_phys_pwrite(dev, obj, args, file); 973 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1019 else if (obj_priv->tiling_mode == I915_TILING_NONE && 974 else if (obj->gtt_space &&
1020 obj_priv->gtt_space && 975 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1021 obj->write_domain != I915_GEM_DOMAIN_CPU) { 976 ret = i915_gem_object_pin(obj, 0, true);
1022 ret = i915_gem_object_pin(obj, 0);
1023 if (ret) 977 if (ret)
1024 goto out; 978 goto out;
1025 979
1026 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 980 ret = i915_gem_object_set_to_gtt_domain(obj, true);
981 if (ret)
982 goto out_unpin;
983
984 ret = i915_gem_object_put_fence(obj);
1027 if (ret) 985 if (ret)
1028 goto out_unpin; 986 goto out_unpin;
1029 987
@@ -1034,26 +992,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1034out_unpin: 992out_unpin:
1035 i915_gem_object_unpin(obj); 993 i915_gem_object_unpin(obj);
1036 } else { 994 } else {
1037 ret = i915_gem_object_get_pages_or_evict(obj);
1038 if (ret)
1039 goto out;
1040
1041 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 995 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1042 if (ret) 996 if (ret)
1043 goto out_put; 997 goto out;
1044 998
1045 ret = -EFAULT; 999 ret = -EFAULT;
1046 if (!i915_gem_object_needs_bit17_swizzle(obj)) 1000 if (!i915_gem_object_needs_bit17_swizzle(obj))
1047 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file); 1001 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1048 if (ret == -EFAULT) 1002 if (ret == -EFAULT)
1049 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file); 1003 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1050
1051out_put:
1052 i915_gem_object_put_pages(obj);
1053 } 1004 }
1054 1005
1055out: 1006out:
1056 drm_gem_object_unreference(obj); 1007 drm_gem_object_unreference(&obj->base);
1057unlock: 1008unlock:
1058 mutex_unlock(&dev->struct_mutex); 1009 mutex_unlock(&dev->struct_mutex);
1059 return ret; 1010 return ret;
@@ -1065,12 +1016,10 @@ unlock:
1065 */ 1016 */
1066int 1017int
1067i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1018i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1068 struct drm_file *file_priv) 1019 struct drm_file *file)
1069{ 1020{
1070 struct drm_i915_private *dev_priv = dev->dev_private;
1071 struct drm_i915_gem_set_domain *args = data; 1021 struct drm_i915_gem_set_domain *args = data;
1072 struct drm_gem_object *obj; 1022 struct drm_i915_gem_object *obj;
1073 struct drm_i915_gem_object *obj_priv;
1074 uint32_t read_domains = args->read_domains; 1023 uint32_t read_domains = args->read_domains;
1075 uint32_t write_domain = args->write_domain; 1024 uint32_t write_domain = args->write_domain;
1076 int ret; 1025 int ret;
@@ -1095,28 +1044,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1095 if (ret) 1044 if (ret)
1096 return ret; 1045 return ret;
1097 1046
1098 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1047 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1099 if (obj == NULL) { 1048 if (obj == NULL) {
1100 ret = -ENOENT; 1049 ret = -ENOENT;
1101 goto unlock; 1050 goto unlock;
1102 } 1051 }
1103 obj_priv = to_intel_bo(obj);
1104
1105 intel_mark_busy(dev, obj);
1106 1052
1107 if (read_domains & I915_GEM_DOMAIN_GTT) { 1053 if (read_domains & I915_GEM_DOMAIN_GTT) {
1108 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1054 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1109 1055
1110 /* Update the LRU on the fence for the CPU access that's
1111 * about to occur.
1112 */
1113 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1114 struct drm_i915_fence_reg *reg =
1115 &dev_priv->fence_regs[obj_priv->fence_reg];
1116 list_move_tail(&reg->lru_list,
1117 &dev_priv->mm.fence_list);
1118 }
1119
1120 /* Silently promote "you're not bound, there was nothing to do" 1056 /* Silently promote "you're not bound, there was nothing to do"
1121 * to success, since the client was just asking us to 1057 * to success, since the client was just asking us to
1122 * make sure everything was done. 1058 * make sure everything was done.
@@ -1127,11 +1063,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1127 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1063 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1128 } 1064 }
1129 1065
1130 /* Maintain LRU order of "inactive" objects */ 1066 drm_gem_object_unreference(&obj->base);
1131 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1132 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1133
1134 drm_gem_object_unreference(obj);
1135unlock: 1067unlock:
1136 mutex_unlock(&dev->struct_mutex); 1068 mutex_unlock(&dev->struct_mutex);
1137 return ret; 1069 return ret;
@@ -1142,10 +1074,10 @@ unlock:
1142 */ 1074 */
1143int 1075int
1144i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1076i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1145 struct drm_file *file_priv) 1077 struct drm_file *file)
1146{ 1078{
1147 struct drm_i915_gem_sw_finish *args = data; 1079 struct drm_i915_gem_sw_finish *args = data;
1148 struct drm_gem_object *obj; 1080 struct drm_i915_gem_object *obj;
1149 int ret = 0; 1081 int ret = 0;
1150 1082
1151 if (!(dev->driver->driver_features & DRIVER_GEM)) 1083 if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1155,17 +1087,17 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1155 if (ret) 1087 if (ret)
1156 return ret; 1088 return ret;
1157 1089
1158 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1090 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1159 if (obj == NULL) { 1091 if (obj == NULL) {
1160 ret = -ENOENT; 1092 ret = -ENOENT;
1161 goto unlock; 1093 goto unlock;
1162 } 1094 }
1163 1095
1164 /* Pinned buffers may be scanout, so flush the cache */ 1096 /* Pinned buffers may be scanout, so flush the cache */
1165 if (to_intel_bo(obj)->pin_count) 1097 if (obj->pin_count)
1166 i915_gem_object_flush_cpu_write_domain(obj); 1098 i915_gem_object_flush_cpu_write_domain(obj);
1167 1099
1168 drm_gem_object_unreference(obj); 1100 drm_gem_object_unreference(&obj->base);
1169unlock: 1101unlock:
1170 mutex_unlock(&dev->struct_mutex); 1102 mutex_unlock(&dev->struct_mutex);
1171 return ret; 1103 return ret;
@@ -1180,8 +1112,9 @@ unlock:
1180 */ 1112 */
1181int 1113int
1182i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1114i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1183 struct drm_file *file_priv) 1115 struct drm_file *file)
1184{ 1116{
1117 struct drm_i915_private *dev_priv = dev->dev_private;
1185 struct drm_i915_gem_mmap *args = data; 1118 struct drm_i915_gem_mmap *args = data;
1186 struct drm_gem_object *obj; 1119 struct drm_gem_object *obj;
1187 loff_t offset; 1120 loff_t offset;
@@ -1190,10 +1123,15 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1190 if (!(dev->driver->driver_features & DRIVER_GEM)) 1123 if (!(dev->driver->driver_features & DRIVER_GEM))
1191 return -ENODEV; 1124 return -ENODEV;
1192 1125
1193 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1126 obj = drm_gem_object_lookup(dev, file, args->handle);
1194 if (obj == NULL) 1127 if (obj == NULL)
1195 return -ENOENT; 1128 return -ENOENT;
1196 1129
1130 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1131 drm_gem_object_unreference_unlocked(obj);
1132 return -E2BIG;
1133 }
1134
1197 offset = args->offset; 1135 offset = args->offset;
1198 1136
1199 down_write(&current->mm->mmap_sem); 1137 down_write(&current->mm->mmap_sem);
@@ -1228,10 +1166,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1228 */ 1166 */
1229int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1167int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1230{ 1168{
1231 struct drm_gem_object *obj = vma->vm_private_data; 1169 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1232 struct drm_device *dev = obj->dev; 1170 struct drm_device *dev = obj->base.dev;
1233 drm_i915_private_t *dev_priv = dev->dev_private; 1171 drm_i915_private_t *dev_priv = dev->dev_private;
1234 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1235 pgoff_t page_offset; 1172 pgoff_t page_offset;
1236 unsigned long pfn; 1173 unsigned long pfn;
1237 int ret = 0; 1174 int ret = 0;
@@ -1243,27 +1180,35 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1243 1180
1244 /* Now bind it into the GTT if needed */ 1181 /* Now bind it into the GTT if needed */
1245 mutex_lock(&dev->struct_mutex); 1182 mutex_lock(&dev->struct_mutex);
1246 if (!obj_priv->gtt_space) {
1247 ret = i915_gem_object_bind_to_gtt(obj, 0);
1248 if (ret)
1249 goto unlock;
1250 1183
1251 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1184 if (!obj->map_and_fenceable) {
1185 ret = i915_gem_object_unbind(obj);
1252 if (ret) 1186 if (ret)
1253 goto unlock; 1187 goto unlock;
1254 } 1188 }
1255 1189 if (!obj->gtt_space) {
1256 /* Need a new fence register? */ 1190 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1257 if (obj_priv->tiling_mode != I915_TILING_NONE) {
1258 ret = i915_gem_object_get_fence_reg(obj, true);
1259 if (ret) 1191 if (ret)
1260 goto unlock; 1192 goto unlock;
1261 } 1193 }
1262 1194
1263 if (i915_gem_object_is_inactive(obj_priv)) 1195 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1264 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); 1196 if (ret)
1197 goto unlock;
1198
1199 if (obj->tiling_mode == I915_TILING_NONE)
1200 ret = i915_gem_object_put_fence(obj);
1201 else
1202 ret = i915_gem_object_get_fence(obj, NULL, true);
1203 if (ret)
1204 goto unlock;
1205
1206 if (i915_gem_object_is_inactive(obj))
1207 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1208
1209 obj->fault_mappable = true;
1265 1210
1266 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1211 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1267 page_offset; 1212 page_offset;
1268 1213
1269 /* Finally, remap it using the new GTT offset */ 1214 /* Finally, remap it using the new GTT offset */
@@ -1272,11 +1217,12 @@ unlock:
1272 mutex_unlock(&dev->struct_mutex); 1217 mutex_unlock(&dev->struct_mutex);
1273 1218
1274 switch (ret) { 1219 switch (ret) {
1220 case -EAGAIN:
1221 set_need_resched();
1275 case 0: 1222 case 0:
1276 case -ERESTARTSYS: 1223 case -ERESTARTSYS:
1277 return VM_FAULT_NOPAGE; 1224 return VM_FAULT_NOPAGE;
1278 case -ENOMEM: 1225 case -ENOMEM:
1279 case -EAGAIN:
1280 return VM_FAULT_OOM; 1226 return VM_FAULT_OOM;
1281 default: 1227 default:
1282 return VM_FAULT_SIGBUS; 1228 return VM_FAULT_SIGBUS;
@@ -1295,37 +1241,39 @@ unlock:
1295 * This routine allocates and attaches a fake offset for @obj. 1241 * This routine allocates and attaches a fake offset for @obj.
1296 */ 1242 */
1297static int 1243static int
1298i915_gem_create_mmap_offset(struct drm_gem_object *obj) 1244i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1299{ 1245{
1300 struct drm_device *dev = obj->dev; 1246 struct drm_device *dev = obj->base.dev;
1301 struct drm_gem_mm *mm = dev->mm_private; 1247 struct drm_gem_mm *mm = dev->mm_private;
1302 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1303 struct drm_map_list *list; 1248 struct drm_map_list *list;
1304 struct drm_local_map *map; 1249 struct drm_local_map *map;
1305 int ret = 0; 1250 int ret = 0;
1306 1251
1307 /* Set the object up for mmap'ing */ 1252 /* Set the object up for mmap'ing */
1308 list = &obj->map_list; 1253 list = &obj->base.map_list;
1309 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL); 1254 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1310 if (!list->map) 1255 if (!list->map)
1311 return -ENOMEM; 1256 return -ENOMEM;
1312 1257
1313 map = list->map; 1258 map = list->map;
1314 map->type = _DRM_GEM; 1259 map->type = _DRM_GEM;
1315 map->size = obj->size; 1260 map->size = obj->base.size;
1316 map->handle = obj; 1261 map->handle = obj;
1317 1262
1318 /* Get a DRM GEM mmap offset allocated... */ 1263 /* Get a DRM GEM mmap offset allocated... */
1319 list->file_offset_node = drm_mm_search_free(&mm->offset_manager, 1264 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1320 obj->size / PAGE_SIZE, 0, 0); 1265 obj->base.size / PAGE_SIZE,
1266 0, 0);
1321 if (!list->file_offset_node) { 1267 if (!list->file_offset_node) {
1322 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); 1268 DRM_ERROR("failed to allocate offset for bo %d\n",
1269 obj->base.name);
1323 ret = -ENOSPC; 1270 ret = -ENOSPC;
1324 goto out_free_list; 1271 goto out_free_list;
1325 } 1272 }
1326 1273
1327 list->file_offset_node = drm_mm_get_block(list->file_offset_node, 1274 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1328 obj->size / PAGE_SIZE, 0); 1275 obj->base.size / PAGE_SIZE,
1276 0);
1329 if (!list->file_offset_node) { 1277 if (!list->file_offset_node) {
1330 ret = -ENOMEM; 1278 ret = -ENOMEM;
1331 goto out_free_list; 1279 goto out_free_list;
@@ -1338,16 +1286,13 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1338 goto out_free_mm; 1286 goto out_free_mm;
1339 } 1287 }
1340 1288
1341 /* By now we should be all set, any drm_mmap request on the offset
1342 * below will get to our mmap & fault handler */
1343 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1344
1345 return 0; 1289 return 0;
1346 1290
1347out_free_mm: 1291out_free_mm:
1348 drm_mm_put_block(list->file_offset_node); 1292 drm_mm_put_block(list->file_offset_node);
1349out_free_list: 1293out_free_list:
1350 kfree(list->map); 1294 kfree(list->map);
1295 list->map = NULL;
1351 1296
1352 return ret; 1297 return ret;
1353} 1298}
@@ -1367,38 +1312,51 @@ out_free_list:
1367 * fixup by i915_gem_fault(). 1312 * fixup by i915_gem_fault().
1368 */ 1313 */
1369void 1314void
1370i915_gem_release_mmap(struct drm_gem_object *obj) 1315i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1371{ 1316{
1372 struct drm_device *dev = obj->dev; 1317 if (!obj->fault_mappable)
1373 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1318 return;
1374 1319
1375 if (dev->dev_mapping) 1320 unmap_mapping_range(obj->base.dev->dev_mapping,
1376 unmap_mapping_range(dev->dev_mapping, 1321 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1377 obj_priv->mmap_offset, obj->size, 1); 1322 obj->base.size, 1);
1323
1324 obj->fault_mappable = false;
1378} 1325}
1379 1326
1380static void 1327static void
1381i915_gem_free_mmap_offset(struct drm_gem_object *obj) 1328i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1382{ 1329{
1383 struct drm_device *dev = obj->dev; 1330 struct drm_device *dev = obj->base.dev;
1384 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1385 struct drm_gem_mm *mm = dev->mm_private; 1331 struct drm_gem_mm *mm = dev->mm_private;
1386 struct drm_map_list *list; 1332 struct drm_map_list *list = &obj->base.map_list;
1387 1333
1388 list = &obj->map_list;
1389 drm_ht_remove_item(&mm->offset_hash, &list->hash); 1334 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1335 drm_mm_put_block(list->file_offset_node);
1336 kfree(list->map);
1337 list->map = NULL;
1338}
1390 1339
1391 if (list->file_offset_node) { 1340static uint32_t
1392 drm_mm_put_block(list->file_offset_node); 1341i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1393 list->file_offset_node = NULL; 1342{
1394 } 1343 struct drm_device *dev = obj->base.dev;
1344 uint32_t size;
1395 1345
1396 if (list->map) { 1346 if (INTEL_INFO(dev)->gen >= 4 ||
1397 kfree(list->map); 1347 obj->tiling_mode == I915_TILING_NONE)
1398 list->map = NULL; 1348 return obj->base.size;
1399 } 1349
1350 /* Previous chips need a power-of-two fence region when tiling */
1351 if (INTEL_INFO(dev)->gen == 3)
1352 size = 1024*1024;
1353 else
1354 size = 512*1024;
1355
1356 while (size < obj->base.size)
1357 size <<= 1;
1400 1358
1401 obj_priv->mmap_offset = 0; 1359 return size;
1402} 1360}
1403 1361
1404/** 1362/**
@@ -1406,42 +1364,68 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1406 * @obj: object to check 1364 * @obj: object to check
1407 * 1365 *
1408 * Return the required GTT alignment for an object, taking into account 1366 * Return the required GTT alignment for an object, taking into account
1409 * potential fence register mapping if needed. 1367 * potential fence register mapping.
1410 */ 1368 */
1411static uint32_t 1369static uint32_t
1412i915_gem_get_gtt_alignment(struct drm_gem_object *obj) 1370i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
1413{ 1371{
1414 struct drm_device *dev = obj->dev; 1372 struct drm_device *dev = obj->base.dev;
1415 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1416 int start, i;
1417 1373
1418 /* 1374 /*
1419 * Minimum alignment is 4k (GTT page size), but might be greater 1375 * Minimum alignment is 4k (GTT page size), but might be greater
1420 * if a fence register is needed for the object. 1376 * if a fence register is needed for the object.
1421 */ 1377 */
1422 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE) 1378 if (INTEL_INFO(dev)->gen >= 4 ||
1379 obj->tiling_mode == I915_TILING_NONE)
1423 return 4096; 1380 return 4096;
1424 1381
1425 /* 1382 /*
1426 * Previous chips need to be aligned to the size of the smallest 1383 * Previous chips need to be aligned to the size of the smallest
1427 * fence register that can contain the object. 1384 * fence register that can contain the object.
1428 */ 1385 */
1429 if (INTEL_INFO(dev)->gen == 3) 1386 return i915_gem_get_gtt_size(obj);
1430 start = 1024*1024; 1387}
1431 else 1388
1432 start = 512*1024; 1389/**
1390 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1391 * unfenced object
1392 * @obj: object to check
1393 *
1394 * Return the required GTT alignment for an object, only taking into account
1395 * unfenced tiled surface requirements.
1396 */
1397static uint32_t
1398i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
1399{
1400 struct drm_device *dev = obj->base.dev;
1401 int tile_height;
1433 1402
1434 for (i = start; i < obj->size; i <<= 1) 1403 /*
1435 ; 1404 * Minimum alignment is 4k (GTT page size) for sane hw.
1405 */
1406 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1407 obj->tiling_mode == I915_TILING_NONE)
1408 return 4096;
1436 1409
1437 return i; 1410 /*
1411 * Older chips need unfenced tiled buffers to be aligned to the left
1412 * edge of an even tile row (where tile rows are counted as if the bo is
1413 * placed in a fenced gtt region).
1414 */
1415 if (IS_GEN2(dev) ||
1416 (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
1417 tile_height = 32;
1418 else
1419 tile_height = 8;
1420
1421 return tile_height * obj->stride * 2;
1438} 1422}
1439 1423
1440/** 1424/**
1441 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1425 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1442 * @dev: DRM device 1426 * @dev: DRM device
1443 * @data: GTT mapping ioctl data 1427 * @data: GTT mapping ioctl data
1444 * @file_priv: GEM object info 1428 * @file: GEM object info
1445 * 1429 *
1446 * Simply returns the fake offset to userspace so it can mmap it. 1430 * Simply returns the fake offset to userspace so it can mmap it.
1447 * The mmap call will end up in drm_gem_mmap(), which will set things 1431 * The mmap call will end up in drm_gem_mmap(), which will set things
@@ -1454,11 +1438,11 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1454 */ 1438 */
1455int 1439int
1456i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1440i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1457 struct drm_file *file_priv) 1441 struct drm_file *file)
1458{ 1442{
1443 struct drm_i915_private *dev_priv = dev->dev_private;
1459 struct drm_i915_gem_mmap_gtt *args = data; 1444 struct drm_i915_gem_mmap_gtt *args = data;
1460 struct drm_gem_object *obj; 1445 struct drm_i915_gem_object *obj;
1461 struct drm_i915_gem_object *obj_priv;
1462 int ret; 1446 int ret;
1463 1447
1464 if (!(dev->driver->driver_features & DRIVER_GEM)) 1448 if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1468,130 +1452,196 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1468 if (ret) 1452 if (ret)
1469 return ret; 1453 return ret;
1470 1454
1471 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1455 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1472 if (obj == NULL) { 1456 if (obj == NULL) {
1473 ret = -ENOENT; 1457 ret = -ENOENT;
1474 goto unlock; 1458 goto unlock;
1475 } 1459 }
1476 obj_priv = to_intel_bo(obj);
1477 1460
1478 if (obj_priv->madv != I915_MADV_WILLNEED) { 1461 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1462 ret = -E2BIG;
1463 goto unlock;
1464 }
1465
1466 if (obj->madv != I915_MADV_WILLNEED) {
1479 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1467 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1480 ret = -EINVAL; 1468 ret = -EINVAL;
1481 goto out; 1469 goto out;
1482 } 1470 }
1483 1471
1484 if (!obj_priv->mmap_offset) { 1472 if (!obj->base.map_list.map) {
1485 ret = i915_gem_create_mmap_offset(obj); 1473 ret = i915_gem_create_mmap_offset(obj);
1486 if (ret) 1474 if (ret)
1487 goto out; 1475 goto out;
1488 } 1476 }
1489 1477
1490 args->offset = obj_priv->mmap_offset; 1478 args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1491
1492 /*
1493 * Pull it into the GTT so that we have a page list (makes the
1494 * initial fault faster and any subsequent flushing possible).
1495 */
1496 if (!obj_priv->agp_mem) {
1497 ret = i915_gem_object_bind_to_gtt(obj, 0);
1498 if (ret)
1499 goto out;
1500 }
1501 1479
1502out: 1480out:
1503 drm_gem_object_unreference(obj); 1481 drm_gem_object_unreference(&obj->base);
1504unlock: 1482unlock:
1505 mutex_unlock(&dev->struct_mutex); 1483 mutex_unlock(&dev->struct_mutex);
1506 return ret; 1484 return ret;
1507} 1485}
1508 1486
1487static int
1488i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1489 gfp_t gfpmask)
1490{
1491 int page_count, i;
1492 struct address_space *mapping;
1493 struct inode *inode;
1494 struct page *page;
1495
1496 /* Get the list of pages out of our struct file. They'll be pinned
1497 * at this point until we release them.
1498 */
1499 page_count = obj->base.size / PAGE_SIZE;
1500 BUG_ON(obj->pages != NULL);
1501 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1502 if (obj->pages == NULL)
1503 return -ENOMEM;
1504
1505 inode = obj->base.filp->f_path.dentry->d_inode;
1506 mapping = inode->i_mapping;
1507 for (i = 0; i < page_count; i++) {
1508 page = read_cache_page_gfp(mapping, i,
1509 GFP_HIGHUSER |
1510 __GFP_COLD |
1511 __GFP_RECLAIMABLE |
1512 gfpmask);
1513 if (IS_ERR(page))
1514 goto err_pages;
1515
1516 obj->pages[i] = page;
1517 }
1518
1519 if (obj->tiling_mode != I915_TILING_NONE)
1520 i915_gem_object_do_bit_17_swizzle(obj);
1521
1522 return 0;
1523
1524err_pages:
1525 while (i--)
1526 page_cache_release(obj->pages[i]);
1527
1528 drm_free_large(obj->pages);
1529 obj->pages = NULL;
1530 return PTR_ERR(page);
1531}
1532
1509static void 1533static void
1510i915_gem_object_put_pages(struct drm_gem_object *obj) 1534i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1511{ 1535{
1512 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1536 int page_count = obj->base.size / PAGE_SIZE;
1513 int page_count = obj->size / PAGE_SIZE;
1514 int i; 1537 int i;
1515 1538
1516 BUG_ON(obj_priv->pages_refcount == 0); 1539 BUG_ON(obj->madv == __I915_MADV_PURGED);
1517 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1518
1519 if (--obj_priv->pages_refcount != 0)
1520 return;
1521 1540
1522 if (obj_priv->tiling_mode != I915_TILING_NONE) 1541 if (obj->tiling_mode != I915_TILING_NONE)
1523 i915_gem_object_save_bit_17_swizzle(obj); 1542 i915_gem_object_save_bit_17_swizzle(obj);
1524 1543
1525 if (obj_priv->madv == I915_MADV_DONTNEED) 1544 if (obj->madv == I915_MADV_DONTNEED)
1526 obj_priv->dirty = 0; 1545 obj->dirty = 0;
1527 1546
1528 for (i = 0; i < page_count; i++) { 1547 for (i = 0; i < page_count; i++) {
1529 if (obj_priv->dirty) 1548 if (obj->dirty)
1530 set_page_dirty(obj_priv->pages[i]); 1549 set_page_dirty(obj->pages[i]);
1531 1550
1532 if (obj_priv->madv == I915_MADV_WILLNEED) 1551 if (obj->madv == I915_MADV_WILLNEED)
1533 mark_page_accessed(obj_priv->pages[i]); 1552 mark_page_accessed(obj->pages[i]);
1534 1553
1535 page_cache_release(obj_priv->pages[i]); 1554 page_cache_release(obj->pages[i]);
1536 } 1555 }
1537 obj_priv->dirty = 0; 1556 obj->dirty = 0;
1538 1557
1539 drm_free_large(obj_priv->pages); 1558 drm_free_large(obj->pages);
1540 obj_priv->pages = NULL; 1559 obj->pages = NULL;
1541} 1560}
1542 1561
1543static uint32_t 1562void
1544i915_gem_next_request_seqno(struct drm_device *dev, 1563i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1545 struct intel_ring_buffer *ring) 1564 struct intel_ring_buffer *ring,
1546{ 1565 u32 seqno)
1547 drm_i915_private_t *dev_priv = dev->dev_private;
1548
1549 ring->outstanding_lazy_request = true;
1550 return dev_priv->next_seqno;
1551}
1552
1553static void
1554i915_gem_object_move_to_active(struct drm_gem_object *obj,
1555 struct intel_ring_buffer *ring)
1556{ 1566{
1557 struct drm_device *dev = obj->dev; 1567 struct drm_device *dev = obj->base.dev;
1558 struct drm_i915_private *dev_priv = dev->dev_private; 1568 struct drm_i915_private *dev_priv = dev->dev_private;
1559 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1560 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1561 1569
1562 BUG_ON(ring == NULL); 1570 BUG_ON(ring == NULL);
1563 obj_priv->ring = ring; 1571 obj->ring = ring;
1564 1572
1565 /* Add a reference if we're newly entering the active list. */ 1573 /* Add a reference if we're newly entering the active list. */
1566 if (!obj_priv->active) { 1574 if (!obj->active) {
1567 drm_gem_object_reference(obj); 1575 drm_gem_object_reference(&obj->base);
1568 obj_priv->active = 1; 1576 obj->active = 1;
1569 } 1577 }
1570 1578
1571 /* Move from whatever list we were on to the tail of execution. */ 1579 /* Move from whatever list we were on to the tail of execution. */
1572 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list); 1580 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1573 list_move_tail(&obj_priv->ring_list, &ring->active_list); 1581 list_move_tail(&obj->ring_list, &ring->active_list);
1574 obj_priv->last_rendering_seqno = seqno; 1582
1583 obj->last_rendering_seqno = seqno;
1584 if (obj->fenced_gpu_access) {
1585 struct drm_i915_fence_reg *reg;
1586
1587 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1588
1589 obj->last_fenced_seqno = seqno;
1590 obj->last_fenced_ring = ring;
1591
1592 reg = &dev_priv->fence_regs[obj->fence_reg];
1593 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1594 }
1575} 1595}
1576 1596
1577static void 1597static void
1578i915_gem_object_move_to_flushing(struct drm_gem_object *obj) 1598i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1579{ 1599{
1580 struct drm_device *dev = obj->dev; 1600 list_del_init(&obj->ring_list);
1601 obj->last_rendering_seqno = 0;
1602}
1603
1604static void
1605i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1606{
1607 struct drm_device *dev = obj->base.dev;
1581 drm_i915_private_t *dev_priv = dev->dev_private; 1608 drm_i915_private_t *dev_priv = dev->dev_private;
1582 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1583 1609
1584 BUG_ON(!obj_priv->active); 1610 BUG_ON(!obj->active);
1585 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list); 1611 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1586 list_del_init(&obj_priv->ring_list); 1612
1587 obj_priv->last_rendering_seqno = 0; 1613 i915_gem_object_move_off_active(obj);
1614}
1615
1616static void
1617i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1618{
1619 struct drm_device *dev = obj->base.dev;
1620 struct drm_i915_private *dev_priv = dev->dev_private;
1621
1622 if (obj->pin_count != 0)
1623 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1624 else
1625 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1626
1627 BUG_ON(!list_empty(&obj->gpu_write_list));
1628 BUG_ON(!obj->active);
1629 obj->ring = NULL;
1630
1631 i915_gem_object_move_off_active(obj);
1632 obj->fenced_gpu_access = false;
1633
1634 obj->active = 0;
1635 obj->pending_gpu_write = false;
1636 drm_gem_object_unreference(&obj->base);
1637
1638 WARN_ON(i915_verify_lists(dev));
1588} 1639}
1589 1640
1590/* Immediately discard the backing storage */ 1641/* Immediately discard the backing storage */
1591static void 1642static void
1592i915_gem_object_truncate(struct drm_gem_object *obj) 1643i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1593{ 1644{
1594 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1595 struct inode *inode; 1645 struct inode *inode;
1596 1646
1597 /* Our goal here is to return as much of the memory as 1647 /* Our goal here is to return as much of the memory as
@@ -1600,42 +1650,18 @@ i915_gem_object_truncate(struct drm_gem_object *obj)
1600 * backing pages, *now*. Here we mirror the actions taken 1650 * backing pages, *now*. Here we mirror the actions taken
1601 * when by shmem_delete_inode() to release the backing store. 1651 * when by shmem_delete_inode() to release the backing store.
1602 */ 1652 */
1603 inode = obj->filp->f_path.dentry->d_inode; 1653 inode = obj->base.filp->f_path.dentry->d_inode;
1604 truncate_inode_pages(inode->i_mapping, 0); 1654 truncate_inode_pages(inode->i_mapping, 0);
1605 if (inode->i_op->truncate_range) 1655 if (inode->i_op->truncate_range)
1606 inode->i_op->truncate_range(inode, 0, (loff_t)-1); 1656 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1607 1657
1608 obj_priv->madv = __I915_MADV_PURGED; 1658 obj->madv = __I915_MADV_PURGED;
1609} 1659}
1610 1660
1611static inline int 1661static inline int
1612i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv) 1662i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1613{ 1663{
1614 return obj_priv->madv == I915_MADV_DONTNEED; 1664 return obj->madv == I915_MADV_DONTNEED;
1615}
1616
1617static void
1618i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1619{
1620 struct drm_device *dev = obj->dev;
1621 drm_i915_private_t *dev_priv = dev->dev_private;
1622 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1623
1624 if (obj_priv->pin_count != 0)
1625 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1626 else
1627 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1628 list_del_init(&obj_priv->ring_list);
1629
1630 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1631
1632 obj_priv->last_rendering_seqno = 0;
1633 obj_priv->ring = NULL;
1634 if (obj_priv->active) {
1635 obj_priv->active = 0;
1636 drm_gem_object_unreference(obj);
1637 }
1638 WARN_ON(i915_verify_lists(dev));
1639} 1665}
1640 1666
1641static void 1667static void
@@ -1643,37 +1669,27 @@ i915_gem_process_flushing_list(struct drm_device *dev,
1643 uint32_t flush_domains, 1669 uint32_t flush_domains,
1644 struct intel_ring_buffer *ring) 1670 struct intel_ring_buffer *ring)
1645{ 1671{
1646 drm_i915_private_t *dev_priv = dev->dev_private; 1672 struct drm_i915_gem_object *obj, *next;
1647 struct drm_i915_gem_object *obj_priv, *next;
1648 1673
1649 list_for_each_entry_safe(obj_priv, next, 1674 list_for_each_entry_safe(obj, next,
1650 &ring->gpu_write_list, 1675 &ring->gpu_write_list,
1651 gpu_write_list) { 1676 gpu_write_list) {
1652 struct drm_gem_object *obj = &obj_priv->base; 1677 if (obj->base.write_domain & flush_domains) {
1678 uint32_t old_write_domain = obj->base.write_domain;
1653 1679
1654 if (obj->write_domain & flush_domains) { 1680 obj->base.write_domain = 0;
1655 uint32_t old_write_domain = obj->write_domain; 1681 list_del_init(&obj->gpu_write_list);
1656 1682 i915_gem_object_move_to_active(obj, ring,
1657 obj->write_domain = 0; 1683 i915_gem_next_request_seqno(dev, ring));
1658 list_del_init(&obj_priv->gpu_write_list);
1659 i915_gem_object_move_to_active(obj, ring);
1660
1661 /* update the fence lru list */
1662 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1663 struct drm_i915_fence_reg *reg =
1664 &dev_priv->fence_regs[obj_priv->fence_reg];
1665 list_move_tail(&reg->lru_list,
1666 &dev_priv->mm.fence_list);
1667 }
1668 1684
1669 trace_i915_gem_object_change_domain(obj, 1685 trace_i915_gem_object_change_domain(obj,
1670 obj->read_domains, 1686 obj->base.read_domains,
1671 old_write_domain); 1687 old_write_domain);
1672 } 1688 }
1673 } 1689 }
1674} 1690}
1675 1691
1676uint32_t 1692int
1677i915_add_request(struct drm_device *dev, 1693i915_add_request(struct drm_device *dev,
1678 struct drm_file *file, 1694 struct drm_file *file,
1679 struct drm_i915_gem_request *request, 1695 struct drm_i915_gem_request *request,
@@ -1683,17 +1699,17 @@ i915_add_request(struct drm_device *dev,
1683 struct drm_i915_file_private *file_priv = NULL; 1699 struct drm_i915_file_private *file_priv = NULL;
1684 uint32_t seqno; 1700 uint32_t seqno;
1685 int was_empty; 1701 int was_empty;
1702 int ret;
1703
1704 BUG_ON(request == NULL);
1686 1705
1687 if (file != NULL) 1706 if (file != NULL)
1688 file_priv = file->driver_priv; 1707 file_priv = file->driver_priv;
1689 1708
1690 if (request == NULL) { 1709 ret = ring->add_request(ring, &seqno);
1691 request = kzalloc(sizeof(*request), GFP_KERNEL); 1710 if (ret)
1692 if (request == NULL) 1711 return ret;
1693 return 0;
1694 }
1695 1712
1696 seqno = ring->add_request(dev, ring, 0);
1697 ring->outstanding_lazy_request = false; 1713 ring->outstanding_lazy_request = false;
1698 1714
1699 request->seqno = seqno; 1715 request->seqno = seqno;
@@ -1717,26 +1733,7 @@ i915_add_request(struct drm_device *dev,
1717 queue_delayed_work(dev_priv->wq, 1733 queue_delayed_work(dev_priv->wq,
1718 &dev_priv->mm.retire_work, HZ); 1734 &dev_priv->mm.retire_work, HZ);
1719 } 1735 }
1720 return seqno; 1736 return 0;
1721}
1722
1723/**
1724 * Command execution barrier
1725 *
1726 * Ensures that all commands in the ring are finished
1727 * before signalling the CPU
1728 */
1729static void
1730i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1731{
1732 uint32_t flush_domains = 0;
1733
1734 /* The sampler always gets flushed on i965 (sigh) */
1735 if (INTEL_INFO(dev)->gen >= 4)
1736 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1737
1738 ring->flush(dev, ring,
1739 I915_GEM_DOMAIN_COMMAND, flush_domains);
1740} 1737}
1741 1738
1742static inline void 1739static inline void
@@ -1769,62 +1766,76 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1769 } 1766 }
1770 1767
1771 while (!list_empty(&ring->active_list)) { 1768 while (!list_empty(&ring->active_list)) {
1772 struct drm_i915_gem_object *obj_priv; 1769 struct drm_i915_gem_object *obj;
1773 1770
1774 obj_priv = list_first_entry(&ring->active_list, 1771 obj = list_first_entry(&ring->active_list,
1775 struct drm_i915_gem_object, 1772 struct drm_i915_gem_object,
1776 ring_list); 1773 ring_list);
1777 1774
1778 obj_priv->base.write_domain = 0; 1775 obj->base.write_domain = 0;
1779 list_del_init(&obj_priv->gpu_write_list); 1776 list_del_init(&obj->gpu_write_list);
1780 i915_gem_object_move_to_inactive(&obj_priv->base); 1777 i915_gem_object_move_to_inactive(obj);
1778 }
1779}
1780
1781static void i915_gem_reset_fences(struct drm_device *dev)
1782{
1783 struct drm_i915_private *dev_priv = dev->dev_private;
1784 int i;
1785
1786 for (i = 0; i < 16; i++) {
1787 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1788 struct drm_i915_gem_object *obj = reg->obj;
1789
1790 if (!obj)
1791 continue;
1792
1793 if (obj->tiling_mode)
1794 i915_gem_release_mmap(obj);
1795
1796 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1797 reg->obj->fenced_gpu_access = false;
1798 reg->obj->last_fenced_seqno = 0;
1799 reg->obj->last_fenced_ring = NULL;
1800 i915_gem_clear_fence_reg(dev, reg);
1781 } 1801 }
1782} 1802}
1783 1803
1784void i915_gem_reset(struct drm_device *dev) 1804void i915_gem_reset(struct drm_device *dev)
1785{ 1805{
1786 struct drm_i915_private *dev_priv = dev->dev_private; 1806 struct drm_i915_private *dev_priv = dev->dev_private;
1787 struct drm_i915_gem_object *obj_priv; 1807 struct drm_i915_gem_object *obj;
1788 int i; 1808 int i;
1789 1809
1790 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring); 1810 for (i = 0; i < I915_NUM_RINGS; i++)
1791 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring); 1811 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1792 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1793 1812
1794 /* Remove anything from the flushing lists. The GPU cache is likely 1813 /* Remove anything from the flushing lists. The GPU cache is likely
1795 * to be lost on reset along with the data, so simply move the 1814 * to be lost on reset along with the data, so simply move the
1796 * lost bo to the inactive list. 1815 * lost bo to the inactive list.
1797 */ 1816 */
1798 while (!list_empty(&dev_priv->mm.flushing_list)) { 1817 while (!list_empty(&dev_priv->mm.flushing_list)) {
1799 obj_priv = list_first_entry(&dev_priv->mm.flushing_list, 1818 obj= list_first_entry(&dev_priv->mm.flushing_list,
1800 struct drm_i915_gem_object, 1819 struct drm_i915_gem_object,
1801 mm_list); 1820 mm_list);
1802 1821
1803 obj_priv->base.write_domain = 0; 1822 obj->base.write_domain = 0;
1804 list_del_init(&obj_priv->gpu_write_list); 1823 list_del_init(&obj->gpu_write_list);
1805 i915_gem_object_move_to_inactive(&obj_priv->base); 1824 i915_gem_object_move_to_inactive(obj);
1806 } 1825 }
1807 1826
1808 /* Move everything out of the GPU domains to ensure we do any 1827 /* Move everything out of the GPU domains to ensure we do any
1809 * necessary invalidation upon reuse. 1828 * necessary invalidation upon reuse.
1810 */ 1829 */
1811 list_for_each_entry(obj_priv, 1830 list_for_each_entry(obj,
1812 &dev_priv->mm.inactive_list, 1831 &dev_priv->mm.inactive_list,
1813 mm_list) 1832 mm_list)
1814 { 1833 {
1815 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1834 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1816 } 1835 }
1817 1836
1818 /* The fence registers are invalidated so clear them out */ 1837 /* The fence registers are invalidated so clear them out */
1819 for (i = 0; i < 16; i++) { 1838 i915_gem_reset_fences(dev);
1820 struct drm_i915_fence_reg *reg;
1821
1822 reg = &dev_priv->fence_regs[i];
1823 if (!reg->obj)
1824 continue;
1825
1826 i915_gem_clear_fence_reg(reg->obj);
1827 }
1828} 1839}
1829 1840
1830/** 1841/**
@@ -1836,6 +1847,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1836{ 1847{
1837 drm_i915_private_t *dev_priv = dev->dev_private; 1848 drm_i915_private_t *dev_priv = dev->dev_private;
1838 uint32_t seqno; 1849 uint32_t seqno;
1850 int i;
1839 1851
1840 if (!ring->status_page.page_addr || 1852 if (!ring->status_page.page_addr ||
1841 list_empty(&ring->request_list)) 1853 list_empty(&ring->request_list))
@@ -1843,7 +1855,12 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1843 1855
1844 WARN_ON(i915_verify_lists(dev)); 1856 WARN_ON(i915_verify_lists(dev));
1845 1857
1846 seqno = ring->get_seqno(dev, ring); 1858 seqno = ring->get_seqno(ring);
1859
1860 for (i = 0; i < I915_NUM_RINGS; i++)
1861 if (seqno >= ring->sync_seqno[i])
1862 ring->sync_seqno[i] = 0;
1863
1847 while (!list_empty(&ring->request_list)) { 1864 while (!list_empty(&ring->request_list)) {
1848 struct drm_i915_gem_request *request; 1865 struct drm_i915_gem_request *request;
1849 1866
@@ -1865,18 +1882,16 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1865 * by the ringbuffer to the flushing/inactive lists as appropriate. 1882 * by the ringbuffer to the flushing/inactive lists as appropriate.
1866 */ 1883 */
1867 while (!list_empty(&ring->active_list)) { 1884 while (!list_empty(&ring->active_list)) {
1868 struct drm_gem_object *obj; 1885 struct drm_i915_gem_object *obj;
1869 struct drm_i915_gem_object *obj_priv;
1870 1886
1871 obj_priv = list_first_entry(&ring->active_list, 1887 obj= list_first_entry(&ring->active_list,
1872 struct drm_i915_gem_object, 1888 struct drm_i915_gem_object,
1873 ring_list); 1889 ring_list);
1874 1890
1875 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno)) 1891 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1876 break; 1892 break;
1877 1893
1878 obj = &obj_priv->base; 1894 if (obj->base.write_domain != 0)
1879 if (obj->write_domain != 0)
1880 i915_gem_object_move_to_flushing(obj); 1895 i915_gem_object_move_to_flushing(obj);
1881 else 1896 else
1882 i915_gem_object_move_to_inactive(obj); 1897 i915_gem_object_move_to_inactive(obj);
@@ -1884,7 +1899,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1884 1899
1885 if (unlikely (dev_priv->trace_irq_seqno && 1900 if (unlikely (dev_priv->trace_irq_seqno &&
1886 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) { 1901 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1887 ring->user_irq_put(dev, ring); 1902 ring->irq_put(ring);
1888 dev_priv->trace_irq_seqno = 0; 1903 dev_priv->trace_irq_seqno = 0;
1889 } 1904 }
1890 1905
@@ -1895,24 +1910,24 @@ void
1895i915_gem_retire_requests(struct drm_device *dev) 1910i915_gem_retire_requests(struct drm_device *dev)
1896{ 1911{
1897 drm_i915_private_t *dev_priv = dev->dev_private; 1912 drm_i915_private_t *dev_priv = dev->dev_private;
1913 int i;
1898 1914
1899 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 1915 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1900 struct drm_i915_gem_object *obj_priv, *tmp; 1916 struct drm_i915_gem_object *obj, *next;
1901 1917
1902 /* We must be careful that during unbind() we do not 1918 /* We must be careful that during unbind() we do not
1903 * accidentally infinitely recurse into retire requests. 1919 * accidentally infinitely recurse into retire requests.
1904 * Currently: 1920 * Currently:
1905 * retire -> free -> unbind -> wait -> retire_ring 1921 * retire -> free -> unbind -> wait -> retire_ring
1906 */ 1922 */
1907 list_for_each_entry_safe(obj_priv, tmp, 1923 list_for_each_entry_safe(obj, next,
1908 &dev_priv->mm.deferred_free_list, 1924 &dev_priv->mm.deferred_free_list,
1909 mm_list) 1925 mm_list)
1910 i915_gem_free_object_tail(&obj_priv->base); 1926 i915_gem_free_object_tail(obj);
1911 } 1927 }
1912 1928
1913 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring); 1929 for (i = 0; i < I915_NUM_RINGS; i++)
1914 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring); 1930 i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]);
1915 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
1916} 1931}
1917 1932
1918static void 1933static void
@@ -1934,9 +1949,9 @@ i915_gem_retire_work_handler(struct work_struct *work)
1934 i915_gem_retire_requests(dev); 1949 i915_gem_retire_requests(dev);
1935 1950
1936 if (!dev_priv->mm.suspended && 1951 if (!dev_priv->mm.suspended &&
1937 (!list_empty(&dev_priv->render_ring.request_list) || 1952 (!list_empty(&dev_priv->ring[RCS].request_list) ||
1938 !list_empty(&dev_priv->bsd_ring.request_list) || 1953 !list_empty(&dev_priv->ring[VCS].request_list) ||
1939 !list_empty(&dev_priv->blt_ring.request_list))) 1954 !list_empty(&dev_priv->ring[BCS].request_list)))
1940 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); 1955 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1941 mutex_unlock(&dev->struct_mutex); 1956 mutex_unlock(&dev->struct_mutex);
1942} 1957}
@@ -1954,14 +1969,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1954 if (atomic_read(&dev_priv->mm.wedged)) 1969 if (atomic_read(&dev_priv->mm.wedged))
1955 return -EAGAIN; 1970 return -EAGAIN;
1956 1971
1957 if (ring->outstanding_lazy_request) { 1972 if (seqno == ring->outstanding_lazy_request) {
1958 seqno = i915_add_request(dev, NULL, NULL, ring); 1973 struct drm_i915_gem_request *request;
1959 if (seqno == 0) 1974
1975 request = kzalloc(sizeof(*request), GFP_KERNEL);
1976 if (request == NULL)
1960 return -ENOMEM; 1977 return -ENOMEM;
1978
1979 ret = i915_add_request(dev, NULL, request, ring);
1980 if (ret) {
1981 kfree(request);
1982 return ret;
1983 }
1984
1985 seqno = request->seqno;
1961 } 1986 }
1962 BUG_ON(seqno == dev_priv->next_seqno);
1963 1987
1964 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { 1988 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
1965 if (HAS_PCH_SPLIT(dev)) 1989 if (HAS_PCH_SPLIT(dev))
1966 ier = I915_READ(DEIER) | I915_READ(GTIER); 1990 ier = I915_READ(DEIER) | I915_READ(GTIER);
1967 else 1991 else
@@ -1975,21 +1999,21 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1975 1999
1976 trace_i915_gem_request_wait_begin(dev, seqno); 2000 trace_i915_gem_request_wait_begin(dev, seqno);
1977 2001
1978 ring->waiting_gem_seqno = seqno; 2002 ring->waiting_seqno = seqno;
1979 ring->user_irq_get(dev, ring); 2003 ret = -ENODEV;
1980 if (interruptible) 2004 if (ring->irq_get(ring)) {
1981 ret = wait_event_interruptible(ring->irq_queue, 2005 if (interruptible)
1982 i915_seqno_passed( 2006 ret = wait_event_interruptible(ring->irq_queue,
1983 ring->get_seqno(dev, ring), seqno) 2007 i915_seqno_passed(ring->get_seqno(ring), seqno)
1984 || atomic_read(&dev_priv->mm.wedged)); 2008 || atomic_read(&dev_priv->mm.wedged));
1985 else 2009 else
1986 wait_event(ring->irq_queue, 2010 wait_event(ring->irq_queue,
1987 i915_seqno_passed( 2011 i915_seqno_passed(ring->get_seqno(ring), seqno)
1988 ring->get_seqno(dev, ring), seqno) 2012 || atomic_read(&dev_priv->mm.wedged));
1989 || atomic_read(&dev_priv->mm.wedged));
1990 2013
1991 ring->user_irq_put(dev, ring); 2014 ring->irq_put(ring);
1992 ring->waiting_gem_seqno = 0; 2015 }
2016 ring->waiting_seqno = 0;
1993 2017
1994 trace_i915_gem_request_wait_end(dev, seqno); 2018 trace_i915_gem_request_wait_end(dev, seqno);
1995 } 2019 }
@@ -1998,7 +2022,7 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1998 2022
1999 if (ret && ret != -ERESTARTSYS) 2023 if (ret && ret != -ERESTARTSYS)
2000 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n", 2024 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2001 __func__, ret, seqno, ring->get_seqno(dev, ring), 2025 __func__, ret, seqno, ring->get_seqno(ring),
2002 dev_priv->next_seqno); 2026 dev_priv->next_seqno);
2003 2027
2004 /* Directly dispatch request retiring. While we have the work queue 2028 /* Directly dispatch request retiring. While we have the work queue
@@ -2023,70 +2047,30 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno,
2023 return i915_do_wait_request(dev, seqno, 1, ring); 2047 return i915_do_wait_request(dev, seqno, 1, ring);
2024} 2048}
2025 2049
2026static void
2027i915_gem_flush_ring(struct drm_device *dev,
2028 struct drm_file *file_priv,
2029 struct intel_ring_buffer *ring,
2030 uint32_t invalidate_domains,
2031 uint32_t flush_domains)
2032{
2033 ring->flush(dev, ring, invalidate_domains, flush_domains);
2034 i915_gem_process_flushing_list(dev, flush_domains, ring);
2035}
2036
2037static void
2038i915_gem_flush(struct drm_device *dev,
2039 struct drm_file *file_priv,
2040 uint32_t invalidate_domains,
2041 uint32_t flush_domains,
2042 uint32_t flush_rings)
2043{
2044 drm_i915_private_t *dev_priv = dev->dev_private;
2045
2046 if (flush_domains & I915_GEM_DOMAIN_CPU)
2047 drm_agp_chipset_flush(dev);
2048
2049 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2050 if (flush_rings & RING_RENDER)
2051 i915_gem_flush_ring(dev, file_priv,
2052 &dev_priv->render_ring,
2053 invalidate_domains, flush_domains);
2054 if (flush_rings & RING_BSD)
2055 i915_gem_flush_ring(dev, file_priv,
2056 &dev_priv->bsd_ring,
2057 invalidate_domains, flush_domains);
2058 if (flush_rings & RING_BLT)
2059 i915_gem_flush_ring(dev, file_priv,
2060 &dev_priv->blt_ring,
2061 invalidate_domains, flush_domains);
2062 }
2063}
2064
2065/** 2050/**
2066 * Ensures that all rendering to the object has completed and the object is 2051 * Ensures that all rendering to the object has completed and the object is
2067 * safe to unbind from the GTT or access from the CPU. 2052 * safe to unbind from the GTT or access from the CPU.
2068 */ 2053 */
2069static int 2054int
2070i915_gem_object_wait_rendering(struct drm_gem_object *obj, 2055i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2071 bool interruptible) 2056 bool interruptible)
2072{ 2057{
2073 struct drm_device *dev = obj->dev; 2058 struct drm_device *dev = obj->base.dev;
2074 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2075 int ret; 2059 int ret;
2076 2060
2077 /* This function only exists to support waiting for existing rendering, 2061 /* This function only exists to support waiting for existing rendering,
2078 * not for emitting required flushes. 2062 * not for emitting required flushes.
2079 */ 2063 */
2080 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0); 2064 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2081 2065
2082 /* If there is rendering queued on the buffer being evicted, wait for 2066 /* If there is rendering queued on the buffer being evicted, wait for
2083 * it. 2067 * it.
2084 */ 2068 */
2085 if (obj_priv->active) { 2069 if (obj->active) {
2086 ret = i915_do_wait_request(dev, 2070 ret = i915_do_wait_request(dev,
2087 obj_priv->last_rendering_seqno, 2071 obj->last_rendering_seqno,
2088 interruptible, 2072 interruptible,
2089 obj_priv->ring); 2073 obj->ring);
2090 if (ret) 2074 if (ret)
2091 return ret; 2075 return ret;
2092 } 2076 }
@@ -2098,17 +2082,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2098 * Unbinds an object from the GTT aperture. 2082 * Unbinds an object from the GTT aperture.
2099 */ 2083 */
2100int 2084int
2101i915_gem_object_unbind(struct drm_gem_object *obj) 2085i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2102{ 2086{
2103 struct drm_device *dev = obj->dev;
2104 struct drm_i915_private *dev_priv = dev->dev_private;
2105 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2106 int ret = 0; 2087 int ret = 0;
2107 2088
2108 if (obj_priv->gtt_space == NULL) 2089 if (obj->gtt_space == NULL)
2109 return 0; 2090 return 0;
2110 2091
2111 if (obj_priv->pin_count != 0) { 2092 if (obj->pin_count != 0) {
2112 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2093 DRM_ERROR("Attempting to unbind pinned buffer\n");
2113 return -EINVAL; 2094 return -EINVAL;
2114 } 2095 }
@@ -2131,27 +2112,27 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2131 */ 2112 */
2132 if (ret) { 2113 if (ret) {
2133 i915_gem_clflush_object(obj); 2114 i915_gem_clflush_object(obj);
2134 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU; 2115 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2135 } 2116 }
2136 2117
2137 /* release the fence reg _after_ flushing */ 2118 /* release the fence reg _after_ flushing */
2138 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) 2119 ret = i915_gem_object_put_fence(obj);
2139 i915_gem_clear_fence_reg(obj); 2120 if (ret == -ERESTARTSYS)
2140 2121 return ret;
2141 drm_unbind_agp(obj_priv->agp_mem);
2142 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2143 2122
2144 i915_gem_object_put_pages(obj); 2123 i915_gem_gtt_unbind_object(obj);
2145 BUG_ON(obj_priv->pages_refcount); 2124 i915_gem_object_put_pages_gtt(obj);
2146 2125
2147 i915_gem_info_remove_gtt(dev_priv, obj->size); 2126 list_del_init(&obj->gtt_list);
2148 list_del_init(&obj_priv->mm_list); 2127 list_del_init(&obj->mm_list);
2128 /* Avoid an unnecessary call to unbind on rebind. */
2129 obj->map_and_fenceable = true;
2149 2130
2150 drm_mm_put_block(obj_priv->gtt_space); 2131 drm_mm_put_block(obj->gtt_space);
2151 obj_priv->gtt_space = NULL; 2132 obj->gtt_space = NULL;
2152 obj_priv->gtt_offset = 0; 2133 obj->gtt_offset = 0;
2153 2134
2154 if (i915_gem_object_is_purgeable(obj_priv)) 2135 if (i915_gem_object_is_purgeable(obj))
2155 i915_gem_object_truncate(obj); 2136 i915_gem_object_truncate(obj);
2156 2137
2157 trace_i915_gem_object_unbind(obj); 2138 trace_i915_gem_object_unbind(obj);
@@ -2159,14 +2140,25 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2159 return ret; 2140 return ret;
2160} 2141}
2161 2142
2143void
2144i915_gem_flush_ring(struct drm_device *dev,
2145 struct intel_ring_buffer *ring,
2146 uint32_t invalidate_domains,
2147 uint32_t flush_domains)
2148{
2149 ring->flush(ring, invalidate_domains, flush_domains);
2150 i915_gem_process_flushing_list(dev, flush_domains, ring);
2151}
2152
2162static int i915_ring_idle(struct drm_device *dev, 2153static int i915_ring_idle(struct drm_device *dev,
2163 struct intel_ring_buffer *ring) 2154 struct intel_ring_buffer *ring)
2164{ 2155{
2165 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2156 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2166 return 0; 2157 return 0;
2167 2158
2168 i915_gem_flush_ring(dev, NULL, ring, 2159 if (!list_empty(&ring->gpu_write_list))
2169 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 2160 i915_gem_flush_ring(dev, ring,
2161 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2170 return i915_wait_request(dev, 2162 return i915_wait_request(dev,
2171 i915_gem_next_request_seqno(dev, ring), 2163 i915_gem_next_request_seqno(dev, ring),
2172 ring); 2164 ring);
@@ -2177,7 +2169,7 @@ i915_gpu_idle(struct drm_device *dev)
2177{ 2169{
2178 drm_i915_private_t *dev_priv = dev->dev_private; 2170 drm_i915_private_t *dev_priv = dev->dev_private;
2179 bool lists_empty; 2171 bool lists_empty;
2180 int ret; 2172 int ret, i;
2181 2173
2182 lists_empty = (list_empty(&dev_priv->mm.flushing_list) && 2174 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2183 list_empty(&dev_priv->mm.active_list)); 2175 list_empty(&dev_priv->mm.active_list));
@@ -2185,258 +2177,296 @@ i915_gpu_idle(struct drm_device *dev)
2185 return 0; 2177 return 0;
2186 2178
2187 /* Flush everything onto the inactive list. */ 2179 /* Flush everything onto the inactive list. */
2188 ret = i915_ring_idle(dev, &dev_priv->render_ring); 2180 for (i = 0; i < I915_NUM_RINGS; i++) {
2189 if (ret) 2181 ret = i915_ring_idle(dev, &dev_priv->ring[i]);
2190 return ret; 2182 if (ret)
2191 2183 return ret;
2192 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2193 if (ret)
2194 return ret;
2195
2196 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2197 if (ret)
2198 return ret;
2199
2200 return 0;
2201}
2202
2203static int
2204i915_gem_object_get_pages(struct drm_gem_object *obj,
2205 gfp_t gfpmask)
2206{
2207 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2208 int page_count, i;
2209 struct address_space *mapping;
2210 struct inode *inode;
2211 struct page *page;
2212
2213 BUG_ON(obj_priv->pages_refcount
2214 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2215
2216 if (obj_priv->pages_refcount++ != 0)
2217 return 0;
2218
2219 /* Get the list of pages out of our struct file. They'll be pinned
2220 * at this point until we release them.
2221 */
2222 page_count = obj->size / PAGE_SIZE;
2223 BUG_ON(obj_priv->pages != NULL);
2224 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2225 if (obj_priv->pages == NULL) {
2226 obj_priv->pages_refcount--;
2227 return -ENOMEM;
2228 }
2229
2230 inode = obj->filp->f_path.dentry->d_inode;
2231 mapping = inode->i_mapping;
2232 for (i = 0; i < page_count; i++) {
2233 page = read_cache_page_gfp(mapping, i,
2234 GFP_HIGHUSER |
2235 __GFP_COLD |
2236 __GFP_RECLAIMABLE |
2237 gfpmask);
2238 if (IS_ERR(page))
2239 goto err_pages;
2240
2241 obj_priv->pages[i] = page;
2242 } 2184 }
2243 2185
2244 if (obj_priv->tiling_mode != I915_TILING_NONE)
2245 i915_gem_object_do_bit_17_swizzle(obj);
2246
2247 return 0; 2186 return 0;
2248
2249err_pages:
2250 while (i--)
2251 page_cache_release(obj_priv->pages[i]);
2252
2253 drm_free_large(obj_priv->pages);
2254 obj_priv->pages = NULL;
2255 obj_priv->pages_refcount--;
2256 return PTR_ERR(page);
2257} 2187}
2258 2188
2259static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg) 2189static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2190 struct intel_ring_buffer *pipelined)
2260{ 2191{
2261 struct drm_gem_object *obj = reg->obj; 2192 struct drm_device *dev = obj->base.dev;
2262 struct drm_device *dev = obj->dev;
2263 drm_i915_private_t *dev_priv = dev->dev_private; 2193 drm_i915_private_t *dev_priv = dev->dev_private;
2264 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2194 u32 size = obj->gtt_space->size;
2265 int regnum = obj_priv->fence_reg; 2195 int regnum = obj->fence_reg;
2266 uint64_t val; 2196 uint64_t val;
2267 2197
2268 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2198 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2269 0xfffff000) << 32; 2199 0xfffff000) << 32;
2270 val |= obj_priv->gtt_offset & 0xfffff000; 2200 val |= obj->gtt_offset & 0xfffff000;
2271 val |= (uint64_t)((obj_priv->stride / 128) - 1) << 2201 val |= (uint64_t)((obj->stride / 128) - 1) <<
2272 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2202 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2273 2203
2274 if (obj_priv->tiling_mode == I915_TILING_Y) 2204 if (obj->tiling_mode == I915_TILING_Y)
2275 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2205 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2276 val |= I965_FENCE_REG_VALID; 2206 val |= I965_FENCE_REG_VALID;
2277 2207
2278 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val); 2208 if (pipelined) {
2209 int ret = intel_ring_begin(pipelined, 6);
2210 if (ret)
2211 return ret;
2212
2213 intel_ring_emit(pipelined, MI_NOOP);
2214 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2215 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2216 intel_ring_emit(pipelined, (u32)val);
2217 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2218 intel_ring_emit(pipelined, (u32)(val >> 32));
2219 intel_ring_advance(pipelined);
2220 } else
2221 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2222
2223 return 0;
2279} 2224}
2280 2225
2281static void i965_write_fence_reg(struct drm_i915_fence_reg *reg) 2226static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2227 struct intel_ring_buffer *pipelined)
2282{ 2228{
2283 struct drm_gem_object *obj = reg->obj; 2229 struct drm_device *dev = obj->base.dev;
2284 struct drm_device *dev = obj->dev;
2285 drm_i915_private_t *dev_priv = dev->dev_private; 2230 drm_i915_private_t *dev_priv = dev->dev_private;
2286 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2231 u32 size = obj->gtt_space->size;
2287 int regnum = obj_priv->fence_reg; 2232 int regnum = obj->fence_reg;
2288 uint64_t val; 2233 uint64_t val;
2289 2234
2290 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2235 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2291 0xfffff000) << 32; 2236 0xfffff000) << 32;
2292 val |= obj_priv->gtt_offset & 0xfffff000; 2237 val |= obj->gtt_offset & 0xfffff000;
2293 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2238 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2294 if (obj_priv->tiling_mode == I915_TILING_Y) 2239 if (obj->tiling_mode == I915_TILING_Y)
2295 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2240 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2296 val |= I965_FENCE_REG_VALID; 2241 val |= I965_FENCE_REG_VALID;
2297 2242
2298 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); 2243 if (pipelined) {
2244 int ret = intel_ring_begin(pipelined, 6);
2245 if (ret)
2246 return ret;
2247
2248 intel_ring_emit(pipelined, MI_NOOP);
2249 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2250 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2251 intel_ring_emit(pipelined, (u32)val);
2252 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2253 intel_ring_emit(pipelined, (u32)(val >> 32));
2254 intel_ring_advance(pipelined);
2255 } else
2256 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2257
2258 return 0;
2299} 2259}
2300 2260
2301static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) 2261static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2262 struct intel_ring_buffer *pipelined)
2302{ 2263{
2303 struct drm_gem_object *obj = reg->obj; 2264 struct drm_device *dev = obj->base.dev;
2304 struct drm_device *dev = obj->dev;
2305 drm_i915_private_t *dev_priv = dev->dev_private; 2265 drm_i915_private_t *dev_priv = dev->dev_private;
2306 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2266 u32 size = obj->gtt_space->size;
2307 int regnum = obj_priv->fence_reg; 2267 u32 fence_reg, val, pitch_val;
2308 int tile_width; 2268 int tile_width;
2309 uint32_t fence_reg, val;
2310 uint32_t pitch_val;
2311 2269
2312 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || 2270 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2313 (obj_priv->gtt_offset & (obj->size - 1))) { 2271 (size & -size) != size ||
2314 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n", 2272 (obj->gtt_offset & (size - 1)),
2315 __func__, obj_priv->gtt_offset, obj->size); 2273 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2316 return; 2274 obj->gtt_offset, obj->map_and_fenceable, size))
2317 } 2275 return -EINVAL;
2318 2276
2319 if (obj_priv->tiling_mode == I915_TILING_Y && 2277 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2320 HAS_128_BYTE_Y_TILING(dev))
2321 tile_width = 128; 2278 tile_width = 128;
2322 else 2279 else
2323 tile_width = 512; 2280 tile_width = 512;
2324 2281
2325 /* Note: pitch better be a power of two tile widths */ 2282 /* Note: pitch better be a power of two tile widths */
2326 pitch_val = obj_priv->stride / tile_width; 2283 pitch_val = obj->stride / tile_width;
2327 pitch_val = ffs(pitch_val) - 1; 2284 pitch_val = ffs(pitch_val) - 1;
2328 2285
2329 if (obj_priv->tiling_mode == I915_TILING_Y && 2286 val = obj->gtt_offset;
2330 HAS_128_BYTE_Y_TILING(dev)) 2287 if (obj->tiling_mode == I915_TILING_Y)
2331 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2332 else
2333 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2334
2335 val = obj_priv->gtt_offset;
2336 if (obj_priv->tiling_mode == I915_TILING_Y)
2337 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2288 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2338 val |= I915_FENCE_SIZE_BITS(obj->size); 2289 val |= I915_FENCE_SIZE_BITS(size);
2339 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2290 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2340 val |= I830_FENCE_REG_VALID; 2291 val |= I830_FENCE_REG_VALID;
2341 2292
2342 if (regnum < 8) 2293 fence_reg = obj->fence_reg;
2343 fence_reg = FENCE_REG_830_0 + (regnum * 4); 2294 if (fence_reg < 8)
2295 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2344 else 2296 else
2345 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); 2297 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2346 I915_WRITE(fence_reg, val); 2298
2299 if (pipelined) {
2300 int ret = intel_ring_begin(pipelined, 4);
2301 if (ret)
2302 return ret;
2303
2304 intel_ring_emit(pipelined, MI_NOOP);
2305 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2306 intel_ring_emit(pipelined, fence_reg);
2307 intel_ring_emit(pipelined, val);
2308 intel_ring_advance(pipelined);
2309 } else
2310 I915_WRITE(fence_reg, val);
2311
2312 return 0;
2347} 2313}
2348 2314
2349static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) 2315static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2316 struct intel_ring_buffer *pipelined)
2350{ 2317{
2351 struct drm_gem_object *obj = reg->obj; 2318 struct drm_device *dev = obj->base.dev;
2352 struct drm_device *dev = obj->dev;
2353 drm_i915_private_t *dev_priv = dev->dev_private; 2319 drm_i915_private_t *dev_priv = dev->dev_private;
2354 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2320 u32 size = obj->gtt_space->size;
2355 int regnum = obj_priv->fence_reg; 2321 int regnum = obj->fence_reg;
2356 uint32_t val; 2322 uint32_t val;
2357 uint32_t pitch_val; 2323 uint32_t pitch_val;
2358 uint32_t fence_size_bits;
2359 2324
2360 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || 2325 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2361 (obj_priv->gtt_offset & (obj->size - 1))) { 2326 (size & -size) != size ||
2362 WARN(1, "%s: object 0x%08x not 512K or size aligned\n", 2327 (obj->gtt_offset & (size - 1)),
2363 __func__, obj_priv->gtt_offset); 2328 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2364 return; 2329 obj->gtt_offset, size))
2365 } 2330 return -EINVAL;
2366 2331
2367 pitch_val = obj_priv->stride / 128; 2332 pitch_val = obj->stride / 128;
2368 pitch_val = ffs(pitch_val) - 1; 2333 pitch_val = ffs(pitch_val) - 1;
2369 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2370 2334
2371 val = obj_priv->gtt_offset; 2335 val = obj->gtt_offset;
2372 if (obj_priv->tiling_mode == I915_TILING_Y) 2336 if (obj->tiling_mode == I915_TILING_Y)
2373 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2337 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2374 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size); 2338 val |= I830_FENCE_SIZE_BITS(size);
2375 WARN_ON(fence_size_bits & ~0x00000f00);
2376 val |= fence_size_bits;
2377 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2339 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2378 val |= I830_FENCE_REG_VALID; 2340 val |= I830_FENCE_REG_VALID;
2379 2341
2380 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); 2342 if (pipelined) {
2343 int ret = intel_ring_begin(pipelined, 4);
2344 if (ret)
2345 return ret;
2346
2347 intel_ring_emit(pipelined, MI_NOOP);
2348 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2349 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2350 intel_ring_emit(pipelined, val);
2351 intel_ring_advance(pipelined);
2352 } else
2353 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2354
2355 return 0;
2381} 2356}
2382 2357
2383static int i915_find_fence_reg(struct drm_device *dev, 2358static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2384 bool interruptible) 2359{
2360 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2361}
2362
2363static int
2364i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2365 struct intel_ring_buffer *pipelined,
2366 bool interruptible)
2367{
2368 int ret;
2369
2370 if (obj->fenced_gpu_access) {
2371 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2372 i915_gem_flush_ring(obj->base.dev,
2373 obj->last_fenced_ring,
2374 0, obj->base.write_domain);
2375
2376 obj->fenced_gpu_access = false;
2377 }
2378
2379 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2380 if (!ring_passed_seqno(obj->last_fenced_ring,
2381 obj->last_fenced_seqno)) {
2382 ret = i915_do_wait_request(obj->base.dev,
2383 obj->last_fenced_seqno,
2384 interruptible,
2385 obj->last_fenced_ring);
2386 if (ret)
2387 return ret;
2388 }
2389
2390 obj->last_fenced_seqno = 0;
2391 obj->last_fenced_ring = NULL;
2392 }
2393
2394 return 0;
2395}
2396
2397int
2398i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2399{
2400 int ret;
2401
2402 if (obj->tiling_mode)
2403 i915_gem_release_mmap(obj);
2404
2405 ret = i915_gem_object_flush_fence(obj, NULL, true);
2406 if (ret)
2407 return ret;
2408
2409 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2410 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2411 i915_gem_clear_fence_reg(obj->base.dev,
2412 &dev_priv->fence_regs[obj->fence_reg]);
2413
2414 obj->fence_reg = I915_FENCE_REG_NONE;
2415 }
2416
2417 return 0;
2418}
2419
2420static struct drm_i915_fence_reg *
2421i915_find_fence_reg(struct drm_device *dev,
2422 struct intel_ring_buffer *pipelined)
2385{ 2423{
2386 struct drm_i915_fence_reg *reg = NULL;
2387 struct drm_i915_gem_object *obj_priv = NULL;
2388 struct drm_i915_private *dev_priv = dev->dev_private; 2424 struct drm_i915_private *dev_priv = dev->dev_private;
2389 struct drm_gem_object *obj = NULL; 2425 struct drm_i915_fence_reg *reg, *first, *avail;
2390 int i, avail, ret; 2426 int i;
2391 2427
2392 /* First try to find a free reg */ 2428 /* First try to find a free reg */
2393 avail = 0; 2429 avail = NULL;
2394 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 2430 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2395 reg = &dev_priv->fence_regs[i]; 2431 reg = &dev_priv->fence_regs[i];
2396 if (!reg->obj) 2432 if (!reg->obj)
2397 return i; 2433 return reg;
2398 2434
2399 obj_priv = to_intel_bo(reg->obj); 2435 if (!reg->obj->pin_count)
2400 if (!obj_priv->pin_count) 2436 avail = reg;
2401 avail++;
2402 } 2437 }
2403 2438
2404 if (avail == 0) 2439 if (avail == NULL)
2405 return -ENOSPC; 2440 return NULL;
2406 2441
2407 /* None available, try to steal one or wait for a user to finish */ 2442 /* None available, try to steal one or wait for a user to finish */
2408 i = I915_FENCE_REG_NONE; 2443 avail = first = NULL;
2409 list_for_each_entry(reg, &dev_priv->mm.fence_list, 2444 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2410 lru_list) { 2445 if (reg->obj->pin_count)
2411 obj = reg->obj;
2412 obj_priv = to_intel_bo(obj);
2413
2414 if (obj_priv->pin_count)
2415 continue; 2446 continue;
2416 2447
2417 /* found one! */ 2448 if (first == NULL)
2418 i = obj_priv->fence_reg; 2449 first = reg;
2419 break; 2450
2451 if (!pipelined ||
2452 !reg->obj->last_fenced_ring ||
2453 reg->obj->last_fenced_ring == pipelined) {
2454 avail = reg;
2455 break;
2456 }
2420 } 2457 }
2421 2458
2422 BUG_ON(i == I915_FENCE_REG_NONE); 2459 if (avail == NULL)
2460 avail = first;
2423 2461
2424 /* We only have a reference on obj from the active list. put_fence_reg 2462 return avail;
2425 * might drop that one, causing a use-after-free in it. So hold a
2426 * private reference to obj like the other callers of put_fence_reg
2427 * (set_tiling ioctl) do. */
2428 drm_gem_object_reference(obj);
2429 ret = i915_gem_object_put_fence_reg(obj, interruptible);
2430 drm_gem_object_unreference(obj);
2431 if (ret != 0)
2432 return ret;
2433
2434 return i;
2435} 2463}
2436 2464
2437/** 2465/**
2438 * i915_gem_object_get_fence_reg - set up a fence reg for an object 2466 * i915_gem_object_get_fence - set up a fence reg for an object
2439 * @obj: object to map through a fence reg 2467 * @obj: object to map through a fence reg
2468 * @pipelined: ring on which to queue the change, or NULL for CPU access
2469 * @interruptible: must we wait uninterruptibly for the register to retire?
2440 * 2470 *
2441 * When mapping objects through the GTT, userspace wants to be able to write 2471 * When mapping objects through the GTT, userspace wants to be able to write
2442 * to them without having to worry about swizzling if the object is tiled. 2472 * to them without having to worry about swizzling if the object is tiled.
@@ -2448,72 +2478,138 @@ static int i915_find_fence_reg(struct drm_device *dev,
2448 * and tiling format. 2478 * and tiling format.
2449 */ 2479 */
2450int 2480int
2451i915_gem_object_get_fence_reg(struct drm_gem_object *obj, 2481i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2452 bool interruptible) 2482 struct intel_ring_buffer *pipelined,
2483 bool interruptible)
2453{ 2484{
2454 struct drm_device *dev = obj->dev; 2485 struct drm_device *dev = obj->base.dev;
2455 struct drm_i915_private *dev_priv = dev->dev_private; 2486 struct drm_i915_private *dev_priv = dev->dev_private;
2456 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2487 struct drm_i915_fence_reg *reg;
2457 struct drm_i915_fence_reg *reg = NULL;
2458 int ret; 2488 int ret;
2459 2489
2460 /* Just update our place in the LRU if our fence is getting used. */ 2490 /* XXX disable pipelining. There are bugs. Shocking. */
2461 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { 2491 pipelined = NULL;
2462 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2492
2493 /* Just update our place in the LRU if our fence is getting reused. */
2494 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2495 reg = &dev_priv->fence_regs[obj->fence_reg];
2463 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2496 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2497
2498 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2499 pipelined = NULL;
2500
2501 if (!pipelined) {
2502 if (reg->setup_seqno) {
2503 if (!ring_passed_seqno(obj->last_fenced_ring,
2504 reg->setup_seqno)) {
2505 ret = i915_do_wait_request(obj->base.dev,
2506 reg->setup_seqno,
2507 interruptible,
2508 obj->last_fenced_ring);
2509 if (ret)
2510 return ret;
2511 }
2512
2513 reg->setup_seqno = 0;
2514 }
2515 } else if (obj->last_fenced_ring &&
2516 obj->last_fenced_ring != pipelined) {
2517 ret = i915_gem_object_flush_fence(obj,
2518 pipelined,
2519 interruptible);
2520 if (ret)
2521 return ret;
2522 } else if (obj->tiling_changed) {
2523 if (obj->fenced_gpu_access) {
2524 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2525 i915_gem_flush_ring(obj->base.dev, obj->ring,
2526 0, obj->base.write_domain);
2527
2528 obj->fenced_gpu_access = false;
2529 }
2530 }
2531
2532 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2533 pipelined = NULL;
2534 BUG_ON(!pipelined && reg->setup_seqno);
2535
2536 if (obj->tiling_changed) {
2537 if (pipelined) {
2538 reg->setup_seqno =
2539 i915_gem_next_request_seqno(dev, pipelined);
2540 obj->last_fenced_seqno = reg->setup_seqno;
2541 obj->last_fenced_ring = pipelined;
2542 }
2543 goto update;
2544 }
2545
2464 return 0; 2546 return 0;
2465 } 2547 }
2466 2548
2467 switch (obj_priv->tiling_mode) { 2549 reg = i915_find_fence_reg(dev, pipelined);
2468 case I915_TILING_NONE: 2550 if (reg == NULL)
2469 WARN(1, "allocating a fence for non-tiled object?\n"); 2551 return -ENOSPC;
2470 break;
2471 case I915_TILING_X:
2472 if (!obj_priv->stride)
2473 return -EINVAL;
2474 WARN((obj_priv->stride & (512 - 1)),
2475 "object 0x%08x is X tiled but has non-512B pitch\n",
2476 obj_priv->gtt_offset);
2477 break;
2478 case I915_TILING_Y:
2479 if (!obj_priv->stride)
2480 return -EINVAL;
2481 WARN((obj_priv->stride & (128 - 1)),
2482 "object 0x%08x is Y tiled but has non-128B pitch\n",
2483 obj_priv->gtt_offset);
2484 break;
2485 }
2486 2552
2487 ret = i915_find_fence_reg(dev, interruptible); 2553 ret = i915_gem_object_flush_fence(obj, pipelined, interruptible);
2488 if (ret < 0) 2554 if (ret)
2489 return ret; 2555 return ret;
2490 2556
2491 obj_priv->fence_reg = ret; 2557 if (reg->obj) {
2492 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2558 struct drm_i915_gem_object *old = reg->obj;
2493 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2559
2560 drm_gem_object_reference(&old->base);
2561
2562 if (old->tiling_mode)
2563 i915_gem_release_mmap(old);
2564
2565 ret = i915_gem_object_flush_fence(old,
2566 pipelined,
2567 interruptible);
2568 if (ret) {
2569 drm_gem_object_unreference(&old->base);
2570 return ret;
2571 }
2572
2573 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2574 pipelined = NULL;
2575
2576 old->fence_reg = I915_FENCE_REG_NONE;
2577 old->last_fenced_ring = pipelined;
2578 old->last_fenced_seqno =
2579 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2580
2581 drm_gem_object_unreference(&old->base);
2582 } else if (obj->last_fenced_seqno == 0)
2583 pipelined = NULL;
2494 2584
2495 reg->obj = obj; 2585 reg->obj = obj;
2586 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2587 obj->fence_reg = reg - dev_priv->fence_regs;
2588 obj->last_fenced_ring = pipelined;
2589
2590 reg->setup_seqno =
2591 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2592 obj->last_fenced_seqno = reg->setup_seqno;
2496 2593
2594update:
2595 obj->tiling_changed = false;
2497 switch (INTEL_INFO(dev)->gen) { 2596 switch (INTEL_INFO(dev)->gen) {
2498 case 6: 2597 case 6:
2499 sandybridge_write_fence_reg(reg); 2598 ret = sandybridge_write_fence_reg(obj, pipelined);
2500 break; 2599 break;
2501 case 5: 2600 case 5:
2502 case 4: 2601 case 4:
2503 i965_write_fence_reg(reg); 2602 ret = i965_write_fence_reg(obj, pipelined);
2504 break; 2603 break;
2505 case 3: 2604 case 3:
2506 i915_write_fence_reg(reg); 2605 ret = i915_write_fence_reg(obj, pipelined);
2507 break; 2606 break;
2508 case 2: 2607 case 2:
2509 i830_write_fence_reg(reg); 2608 ret = i830_write_fence_reg(obj, pipelined);
2510 break; 2609 break;
2511 } 2610 }
2512 2611
2513 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg, 2612 return ret;
2514 obj_priv->tiling_mode);
2515
2516 return 0;
2517} 2613}
2518 2614
2519/** 2615/**
@@ -2521,154 +2617,127 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2521 * @obj: object to clear 2617 * @obj: object to clear
2522 * 2618 *
2523 * Zeroes out the fence register itself and clears out the associated 2619 * Zeroes out the fence register itself and clears out the associated
2524 * data structures in dev_priv and obj_priv. 2620 * data structures in dev_priv and obj.
2525 */ 2621 */
2526static void 2622static void
2527i915_gem_clear_fence_reg(struct drm_gem_object *obj) 2623i915_gem_clear_fence_reg(struct drm_device *dev,
2624 struct drm_i915_fence_reg *reg)
2528{ 2625{
2529 struct drm_device *dev = obj->dev;
2530 drm_i915_private_t *dev_priv = dev->dev_private; 2626 drm_i915_private_t *dev_priv = dev->dev_private;
2531 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2627 uint32_t fence_reg = reg - dev_priv->fence_regs;
2532 struct drm_i915_fence_reg *reg =
2533 &dev_priv->fence_regs[obj_priv->fence_reg];
2534 uint32_t fence_reg;
2535 2628
2536 switch (INTEL_INFO(dev)->gen) { 2629 switch (INTEL_INFO(dev)->gen) {
2537 case 6: 2630 case 6:
2538 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + 2631 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2539 (obj_priv->fence_reg * 8), 0);
2540 break; 2632 break;
2541 case 5: 2633 case 5:
2542 case 4: 2634 case 4:
2543 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); 2635 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2544 break; 2636 break;
2545 case 3: 2637 case 3:
2546 if (obj_priv->fence_reg >= 8) 2638 if (fence_reg >= 8)
2547 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4; 2639 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2548 else 2640 else
2549 case 2: 2641 case 2:
2550 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; 2642 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2551 2643
2552 I915_WRITE(fence_reg, 0); 2644 I915_WRITE(fence_reg, 0);
2553 break; 2645 break;
2554 } 2646 }
2555 2647
2556 reg->obj = NULL;
2557 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2558 list_del_init(&reg->lru_list); 2648 list_del_init(&reg->lru_list);
2559} 2649 reg->obj = NULL;
2560 2650 reg->setup_seqno = 0;
2561/**
2562 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2563 * to the buffer to finish, and then resets the fence register.
2564 * @obj: tiled object holding a fence register.
2565 * @bool: whether the wait upon the fence is interruptible
2566 *
2567 * Zeroes out the fence register itself and clears out the associated
2568 * data structures in dev_priv and obj_priv.
2569 */
2570int
2571i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2572 bool interruptible)
2573{
2574 struct drm_device *dev = obj->dev;
2575 struct drm_i915_private *dev_priv = dev->dev_private;
2576 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2577 struct drm_i915_fence_reg *reg;
2578
2579 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2580 return 0;
2581
2582 /* If we've changed tiling, GTT-mappings of the object
2583 * need to re-fault to ensure that the correct fence register
2584 * setup is in place.
2585 */
2586 i915_gem_release_mmap(obj);
2587
2588 /* On the i915, GPU access to tiled buffers is via a fence,
2589 * therefore we must wait for any outstanding access to complete
2590 * before clearing the fence.
2591 */
2592 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2593 if (reg->gpu) {
2594 int ret;
2595
2596 ret = i915_gem_object_flush_gpu_write_domain(obj);
2597 if (ret)
2598 return ret;
2599
2600 ret = i915_gem_object_wait_rendering(obj, interruptible);
2601 if (ret)
2602 return ret;
2603
2604 reg->gpu = false;
2605 }
2606
2607 i915_gem_object_flush_gtt_write_domain(obj);
2608 i915_gem_clear_fence_reg(obj);
2609
2610 return 0;
2611} 2651}
2612 2652
2613/** 2653/**
2614 * Finds free space in the GTT aperture and binds the object there. 2654 * Finds free space in the GTT aperture and binds the object there.
2615 */ 2655 */
2616static int 2656static int
2617i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) 2657i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2658 unsigned alignment,
2659 bool map_and_fenceable)
2618{ 2660{
2619 struct drm_device *dev = obj->dev; 2661 struct drm_device *dev = obj->base.dev;
2620 drm_i915_private_t *dev_priv = dev->dev_private; 2662 drm_i915_private_t *dev_priv = dev->dev_private;
2621 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2622 struct drm_mm_node *free_space; 2663 struct drm_mm_node *free_space;
2623 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN; 2664 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2665 u32 size, fence_size, fence_alignment, unfenced_alignment;
2666 bool mappable, fenceable;
2624 int ret; 2667 int ret;
2625 2668
2626 if (obj_priv->madv != I915_MADV_WILLNEED) { 2669 if (obj->madv != I915_MADV_WILLNEED) {
2627 DRM_ERROR("Attempting to bind a purgeable object\n"); 2670 DRM_ERROR("Attempting to bind a purgeable object\n");
2628 return -EINVAL; 2671 return -EINVAL;
2629 } 2672 }
2630 2673
2674 fence_size = i915_gem_get_gtt_size(obj);
2675 fence_alignment = i915_gem_get_gtt_alignment(obj);
2676 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
2677
2631 if (alignment == 0) 2678 if (alignment == 0)
2632 alignment = i915_gem_get_gtt_alignment(obj); 2679 alignment = map_and_fenceable ? fence_alignment :
2633 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { 2680 unfenced_alignment;
2681 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2634 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2682 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2635 return -EINVAL; 2683 return -EINVAL;
2636 } 2684 }
2637 2685
2686 size = map_and_fenceable ? fence_size : obj->base.size;
2687
2638 /* If the object is bigger than the entire aperture, reject it early 2688 /* If the object is bigger than the entire aperture, reject it early
2639 * before evicting everything in a vain attempt to find space. 2689 * before evicting everything in a vain attempt to find space.
2640 */ 2690 */
2641 if (obj->size > dev_priv->mm.gtt_total) { 2691 if (obj->base.size >
2692 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2642 DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 2693 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2643 return -E2BIG; 2694 return -E2BIG;
2644 } 2695 }
2645 2696
2646 search_free: 2697 search_free:
2647 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2698 if (map_and_fenceable)
2648 obj->size, alignment, 0); 2699 free_space =
2649 if (free_space != NULL) 2700 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2650 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, 2701 size, alignment, 0,
2651 alignment); 2702 dev_priv->mm.gtt_mappable_end,
2652 if (obj_priv->gtt_space == NULL) { 2703 0);
2704 else
2705 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2706 size, alignment, 0);
2707
2708 if (free_space != NULL) {
2709 if (map_and_fenceable)
2710 obj->gtt_space =
2711 drm_mm_get_block_range_generic(free_space,
2712 size, alignment, 0,
2713 dev_priv->mm.gtt_mappable_end,
2714 0);
2715 else
2716 obj->gtt_space =
2717 drm_mm_get_block(free_space, size, alignment);
2718 }
2719 if (obj->gtt_space == NULL) {
2653 /* If the gtt is empty and we're still having trouble 2720 /* If the gtt is empty and we're still having trouble
2654 * fitting our object in, we're out of memory. 2721 * fitting our object in, we're out of memory.
2655 */ 2722 */
2656 ret = i915_gem_evict_something(dev, obj->size, alignment); 2723 ret = i915_gem_evict_something(dev, size, alignment,
2724 map_and_fenceable);
2657 if (ret) 2725 if (ret)
2658 return ret; 2726 return ret;
2659 2727
2660 goto search_free; 2728 goto search_free;
2661 } 2729 }
2662 2730
2663 ret = i915_gem_object_get_pages(obj, gfpmask); 2731 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2664 if (ret) { 2732 if (ret) {
2665 drm_mm_put_block(obj_priv->gtt_space); 2733 drm_mm_put_block(obj->gtt_space);
2666 obj_priv->gtt_space = NULL; 2734 obj->gtt_space = NULL;
2667 2735
2668 if (ret == -ENOMEM) { 2736 if (ret == -ENOMEM) {
2669 /* first try to clear up some space from the GTT */ 2737 /* first try to clear up some space from the GTT */
2670 ret = i915_gem_evict_something(dev, obj->size, 2738 ret = i915_gem_evict_something(dev, size,
2671 alignment); 2739 alignment,
2740 map_and_fenceable);
2672 if (ret) { 2741 if (ret) {
2673 /* now try to shrink everyone else */ 2742 /* now try to shrink everyone else */
2674 if (gfpmask) { 2743 if (gfpmask) {
@@ -2685,122 +2754,113 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2685 return ret; 2754 return ret;
2686 } 2755 }
2687 2756
2688 /* Create an AGP memory structure pointing at our pages, and bind it 2757 ret = i915_gem_gtt_bind_object(obj);
2689 * into the GTT. 2758 if (ret) {
2690 */ 2759 i915_gem_object_put_pages_gtt(obj);
2691 obj_priv->agp_mem = drm_agp_bind_pages(dev, 2760 drm_mm_put_block(obj->gtt_space);
2692 obj_priv->pages, 2761 obj->gtt_space = NULL;
2693 obj->size >> PAGE_SHIFT, 2762
2694 obj_priv->gtt_space->start, 2763 ret = i915_gem_evict_something(dev, size,
2695 obj_priv->agp_type); 2764 alignment, map_and_fenceable);
2696 if (obj_priv->agp_mem == NULL) {
2697 i915_gem_object_put_pages(obj);
2698 drm_mm_put_block(obj_priv->gtt_space);
2699 obj_priv->gtt_space = NULL;
2700
2701 ret = i915_gem_evict_something(dev, obj->size, alignment);
2702 if (ret) 2765 if (ret)
2703 return ret; 2766 return ret;
2704 2767
2705 goto search_free; 2768 goto search_free;
2706 } 2769 }
2707 2770
2708 /* keep track of bounds object by adding it to the inactive list */ 2771 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2709 list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); 2772 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2710 i915_gem_info_add_gtt(dev_priv, obj->size);
2711 2773
2712 /* Assert that the object is not currently in any GPU domain. As it 2774 /* Assert that the object is not currently in any GPU domain. As it
2713 * wasn't in the GTT, there shouldn't be any way it could have been in 2775 * wasn't in the GTT, there shouldn't be any way it could have been in
2714 * a GPU cache 2776 * a GPU cache
2715 */ 2777 */
2716 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 2778 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2717 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 2779 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2718 2780
2719 obj_priv->gtt_offset = obj_priv->gtt_space->start; 2781 obj->gtt_offset = obj->gtt_space->start;
2720 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2721 2782
2783 fenceable =
2784 obj->gtt_space->size == fence_size &&
2785 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2786
2787 mappable =
2788 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2789
2790 obj->map_and_fenceable = mappable && fenceable;
2791
2792 trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
2722 return 0; 2793 return 0;
2723} 2794}
2724 2795
2725void 2796void
2726i915_gem_clflush_object(struct drm_gem_object *obj) 2797i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2727{ 2798{
2728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2729
2730 /* If we don't have a page list set up, then we're not pinned 2799 /* If we don't have a page list set up, then we're not pinned
2731 * to GPU, and we can ignore the cache flush because it'll happen 2800 * to GPU, and we can ignore the cache flush because it'll happen
2732 * again at bind time. 2801 * again at bind time.
2733 */ 2802 */
2734 if (obj_priv->pages == NULL) 2803 if (obj->pages == NULL)
2735 return; 2804 return;
2736 2805
2737 trace_i915_gem_object_clflush(obj); 2806 trace_i915_gem_object_clflush(obj);
2738 2807
2739 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE); 2808 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2740} 2809}
2741 2810
2742/** Flushes any GPU write domain for the object if it's dirty. */ 2811/** Flushes any GPU write domain for the object if it's dirty. */
2743static int 2812static void
2744i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) 2813i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2745{ 2814{
2746 struct drm_device *dev = obj->dev; 2815 struct drm_device *dev = obj->base.dev;
2747 uint32_t old_write_domain;
2748 2816
2749 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) 2817 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2750 return 0; 2818 return;
2751 2819
2752 /* Queue the GPU write cache flushing we need. */ 2820 /* Queue the GPU write cache flushing we need. */
2753 old_write_domain = obj->write_domain; 2821 i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
2754 i915_gem_flush_ring(dev, NULL, 2822 BUG_ON(obj->base.write_domain);
2755 to_intel_bo(obj)->ring,
2756 0, obj->write_domain);
2757 BUG_ON(obj->write_domain);
2758
2759 trace_i915_gem_object_change_domain(obj,
2760 obj->read_domains,
2761 old_write_domain);
2762
2763 return 0;
2764} 2823}
2765 2824
2766/** Flushes the GTT write domain for the object if it's dirty. */ 2825/** Flushes the GTT write domain for the object if it's dirty. */
2767static void 2826static void
2768i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) 2827i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2769{ 2828{
2770 uint32_t old_write_domain; 2829 uint32_t old_write_domain;
2771 2830
2772 if (obj->write_domain != I915_GEM_DOMAIN_GTT) 2831 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2773 return; 2832 return;
2774 2833
2775 /* No actual flushing is required for the GTT write domain. Writes 2834 /* No actual flushing is required for the GTT write domain. Writes
2776 * to it immediately go to main memory as far as we know, so there's 2835 * to it immediately go to main memory as far as we know, so there's
2777 * no chipset flush. It also doesn't land in render cache. 2836 * no chipset flush. It also doesn't land in render cache.
2778 */ 2837 */
2779 old_write_domain = obj->write_domain; 2838 i915_gem_release_mmap(obj);
2780 obj->write_domain = 0; 2839
2840 old_write_domain = obj->base.write_domain;
2841 obj->base.write_domain = 0;
2781 2842
2782 trace_i915_gem_object_change_domain(obj, 2843 trace_i915_gem_object_change_domain(obj,
2783 obj->read_domains, 2844 obj->base.read_domains,
2784 old_write_domain); 2845 old_write_domain);
2785} 2846}
2786 2847
2787/** Flushes the CPU write domain for the object if it's dirty. */ 2848/** Flushes the CPU write domain for the object if it's dirty. */
2788static void 2849static void
2789i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) 2850i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2790{ 2851{
2791 struct drm_device *dev = obj->dev;
2792 uint32_t old_write_domain; 2852 uint32_t old_write_domain;
2793 2853
2794 if (obj->write_domain != I915_GEM_DOMAIN_CPU) 2854 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2795 return; 2855 return;
2796 2856
2797 i915_gem_clflush_object(obj); 2857 i915_gem_clflush_object(obj);
2798 drm_agp_chipset_flush(dev); 2858 intel_gtt_chipset_flush();
2799 old_write_domain = obj->write_domain; 2859 old_write_domain = obj->base.write_domain;
2800 obj->write_domain = 0; 2860 obj->base.write_domain = 0;
2801 2861
2802 trace_i915_gem_object_change_domain(obj, 2862 trace_i915_gem_object_change_domain(obj,
2803 obj->read_domains, 2863 obj->base.read_domains,
2804 old_write_domain); 2864 old_write_domain);
2805} 2865}
2806 2866
@@ -2811,37 +2871,36 @@ i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2811 * flushes to occur. 2871 * flushes to occur.
2812 */ 2872 */
2813int 2873int
2814i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) 2874i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2815{ 2875{
2816 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2817 uint32_t old_write_domain, old_read_domains; 2876 uint32_t old_write_domain, old_read_domains;
2818 int ret; 2877 int ret;
2819 2878
2820 /* Not valid to be called on unbound objects. */ 2879 /* Not valid to be called on unbound objects. */
2821 if (obj_priv->gtt_space == NULL) 2880 if (obj->gtt_space == NULL)
2822 return -EINVAL; 2881 return -EINVAL;
2823 2882
2824 ret = i915_gem_object_flush_gpu_write_domain(obj); 2883 i915_gem_object_flush_gpu_write_domain(obj);
2825 if (ret != 0) 2884 if (obj->pending_gpu_write || write) {
2826 return ret; 2885 ret = i915_gem_object_wait_rendering(obj, true);
2827 ret = i915_gem_object_wait_rendering(obj, true); 2886 if (ret)
2828 if (ret) 2887 return ret;
2829 return ret; 2888 }
2830 2889
2831 i915_gem_object_flush_cpu_write_domain(obj); 2890 i915_gem_object_flush_cpu_write_domain(obj);
2832 2891
2833 old_write_domain = obj->write_domain; 2892 old_write_domain = obj->base.write_domain;
2834 old_read_domains = obj->read_domains; 2893 old_read_domains = obj->base.read_domains;
2835 2894
2836 /* It should now be out of any other write domains, and we can update 2895 /* It should now be out of any other write domains, and we can update
2837 * the domain values for our changes. 2896 * the domain values for our changes.
2838 */ 2897 */
2839 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 2898 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2840 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2899 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2841 if (write) { 2900 if (write) {
2842 obj->read_domains = I915_GEM_DOMAIN_GTT; 2901 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2843 obj->write_domain = I915_GEM_DOMAIN_GTT; 2902 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2844 obj_priv->dirty = 1; 2903 obj->dirty = 1;
2845 } 2904 }
2846 2905
2847 trace_i915_gem_object_change_domain(obj, 2906 trace_i915_gem_object_change_domain(obj,
@@ -2856,23 +2915,20 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2856 * wait, as in modesetting process we're not supposed to be interrupted. 2915 * wait, as in modesetting process we're not supposed to be interrupted.
2857 */ 2916 */
2858int 2917int
2859i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, 2918i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
2860 bool pipelined) 2919 struct intel_ring_buffer *pipelined)
2861{ 2920{
2862 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2863 uint32_t old_read_domains; 2921 uint32_t old_read_domains;
2864 int ret; 2922 int ret;
2865 2923
2866 /* Not valid to be called on unbound objects. */ 2924 /* Not valid to be called on unbound objects. */
2867 if (obj_priv->gtt_space == NULL) 2925 if (obj->gtt_space == NULL)
2868 return -EINVAL; 2926 return -EINVAL;
2869 2927
2870 ret = i915_gem_object_flush_gpu_write_domain(obj); 2928 i915_gem_object_flush_gpu_write_domain(obj);
2871 if (ret)
2872 return ret;
2873 2929
2874 /* Currently, we are always called from an non-interruptible context. */ 2930 /* Currently, we are always called from an non-interruptible context. */
2875 if (!pipelined) { 2931 if (pipelined != obj->ring) {
2876 ret = i915_gem_object_wait_rendering(obj, false); 2932 ret = i915_gem_object_wait_rendering(obj, false);
2877 if (ret) 2933 if (ret)
2878 return ret; 2934 return ret;
@@ -2880,12 +2936,12 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2880 2936
2881 i915_gem_object_flush_cpu_write_domain(obj); 2937 i915_gem_object_flush_cpu_write_domain(obj);
2882 2938
2883 old_read_domains = obj->read_domains; 2939 old_read_domains = obj->base.read_domains;
2884 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2940 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2885 2941
2886 trace_i915_gem_object_change_domain(obj, 2942 trace_i915_gem_object_change_domain(obj,
2887 old_read_domains, 2943 old_read_domains,
2888 obj->write_domain); 2944 obj->base.write_domain);
2889 2945
2890 return 0; 2946 return 0;
2891} 2947}
@@ -2898,10 +2954,10 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2898 return 0; 2954 return 0;
2899 2955
2900 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) 2956 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2901 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring, 2957 i915_gem_flush_ring(obj->base.dev, obj->ring,
2902 0, obj->base.write_domain); 2958 0, obj->base.write_domain);
2903 2959
2904 return i915_gem_object_wait_rendering(&obj->base, interruptible); 2960 return i915_gem_object_wait_rendering(obj, interruptible);
2905} 2961}
2906 2962
2907/** 2963/**
@@ -2911,14 +2967,12 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2911 * flushes to occur. 2967 * flushes to occur.
2912 */ 2968 */
2913static int 2969static int
2914i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) 2970i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2915{ 2971{
2916 uint32_t old_write_domain, old_read_domains; 2972 uint32_t old_write_domain, old_read_domains;
2917 int ret; 2973 int ret;
2918 2974
2919 ret = i915_gem_object_flush_gpu_write_domain(obj); 2975 i915_gem_object_flush_gpu_write_domain(obj);
2920 if (ret != 0)
2921 return ret;
2922 ret = i915_gem_object_wait_rendering(obj, true); 2976 ret = i915_gem_object_wait_rendering(obj, true);
2923 if (ret) 2977 if (ret)
2924 return ret; 2978 return ret;
@@ -2930,27 +2984,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2930 */ 2984 */
2931 i915_gem_object_set_to_full_cpu_read_domain(obj); 2985 i915_gem_object_set_to_full_cpu_read_domain(obj);
2932 2986
2933 old_write_domain = obj->write_domain; 2987 old_write_domain = obj->base.write_domain;
2934 old_read_domains = obj->read_domains; 2988 old_read_domains = obj->base.read_domains;
2935 2989
2936 /* Flush the CPU cache if it's still invalid. */ 2990 /* Flush the CPU cache if it's still invalid. */
2937 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2991 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2938 i915_gem_clflush_object(obj); 2992 i915_gem_clflush_object(obj);
2939 2993
2940 obj->read_domains |= I915_GEM_DOMAIN_CPU; 2994 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2941 } 2995 }
2942 2996
2943 /* It should now be out of any other write domains, and we can update 2997 /* It should now be out of any other write domains, and we can update
2944 * the domain values for our changes. 2998 * the domain values for our changes.
2945 */ 2999 */
2946 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3000 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2947 3001
2948 /* If we're writing through the CPU, then the GPU read domains will 3002 /* If we're writing through the CPU, then the GPU read domains will
2949 * need to be invalidated at next use. 3003 * need to be invalidated at next use.
2950 */ 3004 */
2951 if (write) { 3005 if (write) {
2952 obj->read_domains = I915_GEM_DOMAIN_CPU; 3006 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2953 obj->write_domain = I915_GEM_DOMAIN_CPU; 3007 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2954 } 3008 }
2955 3009
2956 trace_i915_gem_object_change_domain(obj, 3010 trace_i915_gem_object_change_domain(obj,
@@ -2960,184 +3014,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2960 return 0; 3014 return 0;
2961} 3015}
2962 3016
2963/*
2964 * Set the next domain for the specified object. This
2965 * may not actually perform the necessary flushing/invaliding though,
2966 * as that may want to be batched with other set_domain operations
2967 *
2968 * This is (we hope) the only really tricky part of gem. The goal
2969 * is fairly simple -- track which caches hold bits of the object
2970 * and make sure they remain coherent. A few concrete examples may
2971 * help to explain how it works. For shorthand, we use the notation
2972 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2973 * a pair of read and write domain masks.
2974 *
2975 * Case 1: the batch buffer
2976 *
2977 * 1. Allocated
2978 * 2. Written by CPU
2979 * 3. Mapped to GTT
2980 * 4. Read by GPU
2981 * 5. Unmapped from GTT
2982 * 6. Freed
2983 *
2984 * Let's take these a step at a time
2985 *
2986 * 1. Allocated
2987 * Pages allocated from the kernel may still have
2988 * cache contents, so we set them to (CPU, CPU) always.
2989 * 2. Written by CPU (using pwrite)
2990 * The pwrite function calls set_domain (CPU, CPU) and
2991 * this function does nothing (as nothing changes)
2992 * 3. Mapped by GTT
2993 * This function asserts that the object is not
2994 * currently in any GPU-based read or write domains
2995 * 4. Read by GPU
2996 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2997 * As write_domain is zero, this function adds in the
2998 * current read domains (CPU+COMMAND, 0).
2999 * flush_domains is set to CPU.
3000 * invalidate_domains is set to COMMAND
3001 * clflush is run to get data out of the CPU caches
3002 * then i915_dev_set_domain calls i915_gem_flush to
3003 * emit an MI_FLUSH and drm_agp_chipset_flush
3004 * 5. Unmapped from GTT
3005 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3006 * flush_domains and invalidate_domains end up both zero
3007 * so no flushing/invalidating happens
3008 * 6. Freed
3009 * yay, done
3010 *
3011 * Case 2: The shared render buffer
3012 *
3013 * 1. Allocated
3014 * 2. Mapped to GTT
3015 * 3. Read/written by GPU
3016 * 4. set_domain to (CPU,CPU)
3017 * 5. Read/written by CPU
3018 * 6. Read/written by GPU
3019 *
3020 * 1. Allocated
3021 * Same as last example, (CPU, CPU)
3022 * 2. Mapped to GTT
3023 * Nothing changes (assertions find that it is not in the GPU)
3024 * 3. Read/written by GPU
3025 * execbuffer calls set_domain (RENDER, RENDER)
3026 * flush_domains gets CPU
3027 * invalidate_domains gets GPU
3028 * clflush (obj)
3029 * MI_FLUSH and drm_agp_chipset_flush
3030 * 4. set_domain (CPU, CPU)
3031 * flush_domains gets GPU
3032 * invalidate_domains gets CPU
3033 * wait_rendering (obj) to make sure all drawing is complete.
3034 * This will include an MI_FLUSH to get the data from GPU
3035 * to memory
3036 * clflush (obj) to invalidate the CPU cache
3037 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3038 * 5. Read/written by CPU
3039 * cache lines are loaded and dirtied
3040 * 6. Read written by GPU
3041 * Same as last GPU access
3042 *
3043 * Case 3: The constant buffer
3044 *
3045 * 1. Allocated
3046 * 2. Written by CPU
3047 * 3. Read by GPU
3048 * 4. Updated (written) by CPU again
3049 * 5. Read by GPU
3050 *
3051 * 1. Allocated
3052 * (CPU, CPU)
3053 * 2. Written by CPU
3054 * (CPU, CPU)
3055 * 3. Read by GPU
3056 * (CPU+RENDER, 0)
3057 * flush_domains = CPU
3058 * invalidate_domains = RENDER
3059 * clflush (obj)
3060 * MI_FLUSH
3061 * drm_agp_chipset_flush
3062 * 4. Updated (written) by CPU again
3063 * (CPU, CPU)
3064 * flush_domains = 0 (no previous write domain)
3065 * invalidate_domains = 0 (no new read domains)
3066 * 5. Read by GPU
3067 * (CPU+RENDER, 0)
3068 * flush_domains = CPU
3069 * invalidate_domains = RENDER
3070 * clflush (obj)
3071 * MI_FLUSH
3072 * drm_agp_chipset_flush
3073 */
3074static void
3075i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3076 struct intel_ring_buffer *ring)
3077{
3078 struct drm_device *dev = obj->dev;
3079 struct drm_i915_private *dev_priv = dev->dev_private;
3080 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3081 uint32_t invalidate_domains = 0;
3082 uint32_t flush_domains = 0;
3083 uint32_t old_read_domains;
3084
3085 intel_mark_busy(dev, obj);
3086
3087 /*
3088 * If the object isn't moving to a new write domain,
3089 * let the object stay in multiple read domains
3090 */
3091 if (obj->pending_write_domain == 0)
3092 obj->pending_read_domains |= obj->read_domains;
3093 else
3094 obj_priv->dirty = 1;
3095
3096 /*
3097 * Flush the current write domain if
3098 * the new read domains don't match. Invalidate
3099 * any read domains which differ from the old
3100 * write domain
3101 */
3102 if (obj->write_domain &&
3103 (obj->write_domain != obj->pending_read_domains ||
3104 obj_priv->ring != ring)) {
3105 flush_domains |= obj->write_domain;
3106 invalidate_domains |=
3107 obj->pending_read_domains & ~obj->write_domain;
3108 }
3109 /*
3110 * Invalidate any read caches which may have
3111 * stale data. That is, any new read domains.
3112 */
3113 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3114 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3115 i915_gem_clflush_object(obj);
3116
3117 old_read_domains = obj->read_domains;
3118
3119 /* The actual obj->write_domain will be updated with
3120 * pending_write_domain after we emit the accumulated flush for all
3121 * of our domain changes in execbuffers (which clears objects'
3122 * write_domains). So if we have a current write domain that we
3123 * aren't changing, set pending_write_domain to that.
3124 */
3125 if (flush_domains == 0 && obj->pending_write_domain == 0)
3126 obj->pending_write_domain = obj->write_domain;
3127 obj->read_domains = obj->pending_read_domains;
3128
3129 dev->invalidate_domains |= invalidate_domains;
3130 dev->flush_domains |= flush_domains;
3131 if (flush_domains & I915_GEM_GPU_DOMAINS)
3132 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3133 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3134 dev_priv->mm.flush_rings |= ring->id;
3135
3136 trace_i915_gem_object_change_domain(obj,
3137 old_read_domains,
3138 obj->write_domain);
3139}
3140
3141/** 3017/**
3142 * Moves the object from a partially CPU read to a full one. 3018 * Moves the object from a partially CPU read to a full one.
3143 * 3019 *
@@ -3145,30 +3021,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3145 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). 3021 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3146 */ 3022 */
3147static void 3023static void
3148i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) 3024i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3149{ 3025{
3150 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3026 if (!obj->page_cpu_valid)
3151
3152 if (!obj_priv->page_cpu_valid)
3153 return; 3027 return;
3154 3028
3155 /* If we're partially in the CPU read domain, finish moving it in. 3029 /* If we're partially in the CPU read domain, finish moving it in.
3156 */ 3030 */
3157 if (obj->read_domains & I915_GEM_DOMAIN_CPU) { 3031 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3158 int i; 3032 int i;
3159 3033
3160 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { 3034 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3161 if (obj_priv->page_cpu_valid[i]) 3035 if (obj->page_cpu_valid[i])
3162 continue; 3036 continue;
3163 drm_clflush_pages(obj_priv->pages + i, 1); 3037 drm_clflush_pages(obj->pages + i, 1);
3164 } 3038 }
3165 } 3039 }
3166 3040
3167 /* Free the page_cpu_valid mappings which are now stale, whether 3041 /* Free the page_cpu_valid mappings which are now stale, whether
3168 * or not we've got I915_GEM_DOMAIN_CPU. 3042 * or not we've got I915_GEM_DOMAIN_CPU.
3169 */ 3043 */
3170 kfree(obj_priv->page_cpu_valid); 3044 kfree(obj->page_cpu_valid);
3171 obj_priv->page_cpu_valid = NULL; 3045 obj->page_cpu_valid = NULL;
3172} 3046}
3173 3047
3174/** 3048/**
@@ -3184,19 +3058,16 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3184 * flushes to occur. 3058 * flushes to occur.
3185 */ 3059 */
3186static int 3060static int
3187i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 3061i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3188 uint64_t offset, uint64_t size) 3062 uint64_t offset, uint64_t size)
3189{ 3063{
3190 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3191 uint32_t old_read_domains; 3064 uint32_t old_read_domains;
3192 int i, ret; 3065 int i, ret;
3193 3066
3194 if (offset == 0 && size == obj->size) 3067 if (offset == 0 && size == obj->base.size)
3195 return i915_gem_object_set_to_cpu_domain(obj, 0); 3068 return i915_gem_object_set_to_cpu_domain(obj, 0);
3196 3069
3197 ret = i915_gem_object_flush_gpu_write_domain(obj); 3070 i915_gem_object_flush_gpu_write_domain(obj);
3198 if (ret != 0)
3199 return ret;
3200 ret = i915_gem_object_wait_rendering(obj, true); 3071 ret = i915_gem_object_wait_rendering(obj, true);
3201 if (ret) 3072 if (ret)
3202 return ret; 3073 return ret;
@@ -3204,457 +3075,45 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3204 i915_gem_object_flush_gtt_write_domain(obj); 3075 i915_gem_object_flush_gtt_write_domain(obj);
3205 3076
3206 /* If we're already fully in the CPU read domain, we're done. */ 3077 /* If we're already fully in the CPU read domain, we're done. */
3207 if (obj_priv->page_cpu_valid == NULL && 3078 if (obj->page_cpu_valid == NULL &&
3208 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) 3079 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3209 return 0; 3080 return 0;
3210 3081
3211 /* Otherwise, create/clear the per-page CPU read domain flag if we're 3082 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3212 * newly adding I915_GEM_DOMAIN_CPU 3083 * newly adding I915_GEM_DOMAIN_CPU
3213 */ 3084 */
3214 if (obj_priv->page_cpu_valid == NULL) { 3085 if (obj->page_cpu_valid == NULL) {
3215 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE, 3086 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3216 GFP_KERNEL); 3087 GFP_KERNEL);
3217 if (obj_priv->page_cpu_valid == NULL) 3088 if (obj->page_cpu_valid == NULL)
3218 return -ENOMEM; 3089 return -ENOMEM;
3219 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) 3090 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3220 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); 3091 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3221 3092
3222 /* Flush the cache on any pages that are still invalid from the CPU's 3093 /* Flush the cache on any pages that are still invalid from the CPU's
3223 * perspective. 3094 * perspective.
3224 */ 3095 */
3225 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 3096 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3226 i++) { 3097 i++) {
3227 if (obj_priv->page_cpu_valid[i]) 3098 if (obj->page_cpu_valid[i])
3228 continue; 3099 continue;
3229 3100
3230 drm_clflush_pages(obj_priv->pages + i, 1); 3101 drm_clflush_pages(obj->pages + i, 1);
3231 3102
3232 obj_priv->page_cpu_valid[i] = 1; 3103 obj->page_cpu_valid[i] = 1;
3233 } 3104 }
3234 3105
3235 /* It should now be out of any other write domains, and we can update 3106 /* It should now be out of any other write domains, and we can update
3236 * the domain values for our changes. 3107 * the domain values for our changes.
3237 */ 3108 */
3238 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3109 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3239 3110
3240 old_read_domains = obj->read_domains; 3111 old_read_domains = obj->base.read_domains;
3241 obj->read_domains |= I915_GEM_DOMAIN_CPU; 3112 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3242 3113
3243 trace_i915_gem_object_change_domain(obj, 3114 trace_i915_gem_object_change_domain(obj,
3244 old_read_domains, 3115 old_read_domains,
3245 obj->write_domain); 3116 obj->base.write_domain);
3246
3247 return 0;
3248}
3249
3250static int
3251i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3252 struct drm_file *file_priv,
3253 struct drm_i915_gem_exec_object2 *entry,
3254 struct drm_i915_gem_relocation_entry *reloc)
3255{
3256 struct drm_device *dev = obj->base.dev;
3257 struct drm_gem_object *target_obj;
3258 uint32_t target_offset;
3259 int ret = -EINVAL;
3260
3261 target_obj = drm_gem_object_lookup(dev, file_priv,
3262 reloc->target_handle);
3263 if (target_obj == NULL)
3264 return -ENOENT;
3265
3266 target_offset = to_intel_bo(target_obj)->gtt_offset;
3267
3268#if WATCH_RELOC
3269 DRM_INFO("%s: obj %p offset %08x target %d "
3270 "read %08x write %08x gtt %08x "
3271 "presumed %08x delta %08x\n",
3272 __func__,
3273 obj,
3274 (int) reloc->offset,
3275 (int) reloc->target_handle,
3276 (int) reloc->read_domains,
3277 (int) reloc->write_domain,
3278 (int) target_offset,
3279 (int) reloc->presumed_offset,
3280 reloc->delta);
3281#endif
3282
3283 /* The target buffer should have appeared before us in the
3284 * exec_object list, so it should have a GTT space bound by now.
3285 */
3286 if (target_offset == 0) {
3287 DRM_ERROR("No GTT space found for object %d\n",
3288 reloc->target_handle);
3289 goto err;
3290 }
3291
3292 /* Validate that the target is in a valid r/w GPU domain */
3293 if (reloc->write_domain & (reloc->write_domain - 1)) {
3294 DRM_ERROR("reloc with multiple write domains: "
3295 "obj %p target %d offset %d "
3296 "read %08x write %08x",
3297 obj, reloc->target_handle,
3298 (int) reloc->offset,
3299 reloc->read_domains,
3300 reloc->write_domain);
3301 goto err;
3302 }
3303 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3304 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3305 DRM_ERROR("reloc with read/write CPU domains: "
3306 "obj %p target %d offset %d "
3307 "read %08x write %08x",
3308 obj, reloc->target_handle,
3309 (int) reloc->offset,
3310 reloc->read_domains,
3311 reloc->write_domain);
3312 goto err;
3313 }
3314 if (reloc->write_domain && target_obj->pending_write_domain &&
3315 reloc->write_domain != target_obj->pending_write_domain) {
3316 DRM_ERROR("Write domain conflict: "
3317 "obj %p target %d offset %d "
3318 "new %08x old %08x\n",
3319 obj, reloc->target_handle,
3320 (int) reloc->offset,
3321 reloc->write_domain,
3322 target_obj->pending_write_domain);
3323 goto err;
3324 }
3325
3326 target_obj->pending_read_domains |= reloc->read_domains;
3327 target_obj->pending_write_domain |= reloc->write_domain;
3328
3329 /* If the relocation already has the right value in it, no
3330 * more work needs to be done.
3331 */
3332 if (target_offset == reloc->presumed_offset)
3333 goto out;
3334
3335 /* Check that the relocation address is valid... */
3336 if (reloc->offset > obj->base.size - 4) {
3337 DRM_ERROR("Relocation beyond object bounds: "
3338 "obj %p target %d offset %d size %d.\n",
3339 obj, reloc->target_handle,
3340 (int) reloc->offset,
3341 (int) obj->base.size);
3342 goto err;
3343 }
3344 if (reloc->offset & 3) {
3345 DRM_ERROR("Relocation not 4-byte aligned: "
3346 "obj %p target %d offset %d.\n",
3347 obj, reloc->target_handle,
3348 (int) reloc->offset);
3349 goto err;
3350 }
3351
3352 /* and points to somewhere within the target object. */
3353 if (reloc->delta >= target_obj->size) {
3354 DRM_ERROR("Relocation beyond target object bounds: "
3355 "obj %p target %d delta %d size %d.\n",
3356 obj, reloc->target_handle,
3357 (int) reloc->delta,
3358 (int) target_obj->size);
3359 goto err;
3360 }
3361
3362 reloc->delta += target_offset;
3363 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3364 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3365 char *vaddr;
3366
3367 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3368 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3369 kunmap_atomic(vaddr);
3370 } else {
3371 struct drm_i915_private *dev_priv = dev->dev_private;
3372 uint32_t __iomem *reloc_entry;
3373 void __iomem *reloc_page;
3374
3375 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3376 if (ret)
3377 goto err;
3378
3379 /* Map the page containing the relocation we're going to perform. */
3380 reloc->offset += obj->gtt_offset;
3381 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3382 reloc->offset & PAGE_MASK);
3383 reloc_entry = (uint32_t __iomem *)
3384 (reloc_page + (reloc->offset & ~PAGE_MASK));
3385 iowrite32(reloc->delta, reloc_entry);
3386 io_mapping_unmap_atomic(reloc_page);
3387 }
3388
3389 /* and update the user's relocation entry */
3390 reloc->presumed_offset = target_offset;
3391
3392out:
3393 ret = 0;
3394err:
3395 drm_gem_object_unreference(target_obj);
3396 return ret;
3397}
3398
3399static int
3400i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3401 struct drm_file *file_priv,
3402 struct drm_i915_gem_exec_object2 *entry)
3403{
3404 struct drm_i915_gem_relocation_entry __user *user_relocs;
3405 int i, ret;
3406
3407 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3408 for (i = 0; i < entry->relocation_count; i++) {
3409 struct drm_i915_gem_relocation_entry reloc;
3410
3411 if (__copy_from_user_inatomic(&reloc,
3412 user_relocs+i,
3413 sizeof(reloc)))
3414 return -EFAULT;
3415
3416 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3417 if (ret)
3418 return ret;
3419
3420 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3421 &reloc.presumed_offset,
3422 sizeof(reloc.presumed_offset)))
3423 return -EFAULT;
3424 }
3425
3426 return 0;
3427}
3428
3429static int
3430i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3431 struct drm_file *file_priv,
3432 struct drm_i915_gem_exec_object2 *entry,
3433 struct drm_i915_gem_relocation_entry *relocs)
3434{
3435 int i, ret;
3436
3437 for (i = 0; i < entry->relocation_count; i++) {
3438 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3439 if (ret)
3440 return ret;
3441 }
3442
3443 return 0;
3444}
3445
3446static int
3447i915_gem_execbuffer_relocate(struct drm_device *dev,
3448 struct drm_file *file,
3449 struct drm_gem_object **object_list,
3450 struct drm_i915_gem_exec_object2 *exec_list,
3451 int count)
3452{
3453 int i, ret;
3454
3455 for (i = 0; i < count; i++) {
3456 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3457 obj->base.pending_read_domains = 0;
3458 obj->base.pending_write_domain = 0;
3459 ret = i915_gem_execbuffer_relocate_object(obj, file,
3460 &exec_list[i]);
3461 if (ret)
3462 return ret;
3463 }
3464
3465 return 0;
3466}
3467
3468static int
3469i915_gem_execbuffer_reserve(struct drm_device *dev,
3470 struct drm_file *file,
3471 struct drm_gem_object **object_list,
3472 struct drm_i915_gem_exec_object2 *exec_list,
3473 int count)
3474{
3475 struct drm_i915_private *dev_priv = dev->dev_private;
3476 int ret, i, retry;
3477
3478 /* attempt to pin all of the buffers into the GTT */
3479 for (retry = 0; retry < 2; retry++) {
3480 ret = 0;
3481 for (i = 0; i < count; i++) {
3482 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3483 struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3484 bool need_fence =
3485 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3486 obj->tiling_mode != I915_TILING_NONE;
3487
3488 /* Check fence reg constraints and rebind if necessary */
3489 if (need_fence &&
3490 !i915_gem_object_fence_offset_ok(&obj->base,
3491 obj->tiling_mode)) {
3492 ret = i915_gem_object_unbind(&obj->base);
3493 if (ret)
3494 break;
3495 }
3496
3497 ret = i915_gem_object_pin(&obj->base, entry->alignment);
3498 if (ret)
3499 break;
3500
3501 /*
3502 * Pre-965 chips need a fence register set up in order
3503 * to properly handle blits to/from tiled surfaces.
3504 */
3505 if (need_fence) {
3506 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3507 if (ret) {
3508 i915_gem_object_unpin(&obj->base);
3509 break;
3510 }
3511
3512 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3513 }
3514
3515 entry->offset = obj->gtt_offset;
3516 }
3517
3518 while (i--)
3519 i915_gem_object_unpin(object_list[i]);
3520
3521 if (ret == 0)
3522 break;
3523
3524 if (ret != -ENOSPC || retry)
3525 return ret;
3526
3527 ret = i915_gem_evict_everything(dev);
3528 if (ret)
3529 return ret;
3530 }
3531
3532 return 0;
3533}
3534
3535static int
3536i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3537 struct drm_file *file,
3538 struct drm_gem_object **object_list,
3539 struct drm_i915_gem_exec_object2 *exec_list,
3540 int count)
3541{
3542 struct drm_i915_gem_relocation_entry *reloc;
3543 int i, total, ret;
3544
3545 for (i = 0; i < count; i++) {
3546 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3547 obj->in_execbuffer = false;
3548 }
3549
3550 mutex_unlock(&dev->struct_mutex);
3551
3552 total = 0;
3553 for (i = 0; i < count; i++)
3554 total += exec_list[i].relocation_count;
3555
3556 reloc = drm_malloc_ab(total, sizeof(*reloc));
3557 if (reloc == NULL) {
3558 mutex_lock(&dev->struct_mutex);
3559 return -ENOMEM;
3560 }
3561
3562 total = 0;
3563 for (i = 0; i < count; i++) {
3564 struct drm_i915_gem_relocation_entry __user *user_relocs;
3565
3566 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3567
3568 if (copy_from_user(reloc+total, user_relocs,
3569 exec_list[i].relocation_count *
3570 sizeof(*reloc))) {
3571 ret = -EFAULT;
3572 mutex_lock(&dev->struct_mutex);
3573 goto err;
3574 }
3575
3576 total += exec_list[i].relocation_count;
3577 }
3578
3579 ret = i915_mutex_lock_interruptible(dev);
3580 if (ret) {
3581 mutex_lock(&dev->struct_mutex);
3582 goto err;
3583 }
3584
3585 ret = i915_gem_execbuffer_reserve(dev, file,
3586 object_list, exec_list,
3587 count);
3588 if (ret)
3589 goto err;
3590
3591 total = 0;
3592 for (i = 0; i < count; i++) {
3593 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3594 obj->base.pending_read_domains = 0;
3595 obj->base.pending_write_domain = 0;
3596 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3597 &exec_list[i],
3598 reloc + total);
3599 if (ret)
3600 goto err;
3601
3602 total += exec_list[i].relocation_count;
3603 }
3604
3605 /* Leave the user relocations as are, this is the painfully slow path,
3606 * and we want to avoid the complication of dropping the lock whilst
3607 * having buffers reserved in the aperture and so causing spurious
3608 * ENOSPC for random operations.
3609 */
3610
3611err:
3612 drm_free_large(reloc);
3613 return ret;
3614}
3615
3616static int
3617i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3618 struct drm_file *file,
3619 struct intel_ring_buffer *ring,
3620 struct drm_gem_object **objects,
3621 int count)
3622{
3623 struct drm_i915_private *dev_priv = dev->dev_private;
3624 int ret, i;
3625
3626 /* Zero the global flush/invalidate flags. These
3627 * will be modified as new domains are computed
3628 * for each object
3629 */
3630 dev->invalidate_domains = 0;
3631 dev->flush_domains = 0;
3632 dev_priv->mm.flush_rings = 0;
3633 for (i = 0; i < count; i++)
3634 i915_gem_object_set_to_gpu_domain(objects[i], ring);
3635
3636 if (dev->invalidate_domains | dev->flush_domains) {
3637#if WATCH_EXEC
3638 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3639 __func__,
3640 dev->invalidate_domains,
3641 dev->flush_domains);
3642#endif
3643 i915_gem_flush(dev, file,
3644 dev->invalidate_domains,
3645 dev->flush_domains,
3646 dev_priv->mm.flush_rings);
3647 }
3648
3649 for (i = 0; i < count; i++) {
3650 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3651 /* XXX replace with semaphores */
3652 if (obj->ring && ring != obj->ring) {
3653 ret = i915_gem_object_wait_rendering(&obj->base, true);
3654 if (ret)
3655 return ret;
3656 }
3657 }
3658 3117
3659 return 0; 3118 return 0;
3660} 3119}
@@ -3694,599 +3153,129 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3694 return 0; 3153 return 0;
3695 3154
3696 ret = 0; 3155 ret = 0;
3697 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { 3156 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3698 /* And wait for the seqno passing without holding any locks and 3157 /* And wait for the seqno passing without holding any locks and
3699 * causing extra latency for others. This is safe as the irq 3158 * causing extra latency for others. This is safe as the irq
3700 * generation is designed to be run atomically and so is 3159 * generation is designed to be run atomically and so is
3701 * lockless. 3160 * lockless.
3702 */ 3161 */
3703 ring->user_irq_get(dev, ring); 3162 if (ring->irq_get(ring)) {
3704 ret = wait_event_interruptible(ring->irq_queue, 3163 ret = wait_event_interruptible(ring->irq_queue,
3705 i915_seqno_passed(ring->get_seqno(dev, ring), seqno) 3164 i915_seqno_passed(ring->get_seqno(ring), seqno)
3706 || atomic_read(&dev_priv->mm.wedged)); 3165 || atomic_read(&dev_priv->mm.wedged));
3707 ring->user_irq_put(dev, ring); 3166 ring->irq_put(ring);
3708
3709 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3710 ret = -EIO;
3711 }
3712
3713 if (ret == 0)
3714 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3715
3716 return ret;
3717}
3718
3719static int
3720i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3721 uint64_t exec_offset)
3722{
3723 uint32_t exec_start, exec_len;
3724
3725 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3726 exec_len = (uint32_t) exec->batch_len;
3727
3728 if ((exec_start | exec_len) & 0x7)
3729 return -EINVAL;
3730
3731 if (!exec_start)
3732 return -EINVAL;
3733
3734 return 0;
3735}
3736
3737static int
3738validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3739 int count)
3740{
3741 int i;
3742
3743 for (i = 0; i < count; i++) {
3744 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3745 int length; /* limited by fault_in_pages_readable() */
3746
3747 /* First check for malicious input causing overflow */
3748 if (exec[i].relocation_count >
3749 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3750 return -EINVAL;
3751
3752 length = exec[i].relocation_count *
3753 sizeof(struct drm_i915_gem_relocation_entry);
3754 if (!access_ok(VERIFY_READ, ptr, length))
3755 return -EFAULT;
3756
3757 /* we may also need to update the presumed offsets */
3758 if (!access_ok(VERIFY_WRITE, ptr, length))
3759 return -EFAULT;
3760
3761 if (fault_in_pages_readable(ptr, length))
3762 return -EFAULT;
3763 }
3764
3765 return 0;
3766}
3767
3768static int
3769i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3770 struct drm_file *file,
3771 struct drm_i915_gem_execbuffer2 *args,
3772 struct drm_i915_gem_exec_object2 *exec_list)
3773{
3774 drm_i915_private_t *dev_priv = dev->dev_private;
3775 struct drm_gem_object **object_list = NULL;
3776 struct drm_gem_object *batch_obj;
3777 struct drm_i915_gem_object *obj_priv;
3778 struct drm_clip_rect *cliprects = NULL;
3779 struct drm_i915_gem_request *request = NULL;
3780 int ret, i, flips;
3781 uint64_t exec_offset;
3782
3783 struct intel_ring_buffer *ring = NULL;
3784
3785 ret = i915_gem_check_is_wedged(dev);
3786 if (ret)
3787 return ret;
3788
3789 ret = validate_exec_list(exec_list, args->buffer_count);
3790 if (ret)
3791 return ret;
3792
3793#if WATCH_EXEC
3794 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3795 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3796#endif
3797 switch (args->flags & I915_EXEC_RING_MASK) {
3798 case I915_EXEC_DEFAULT:
3799 case I915_EXEC_RENDER:
3800 ring = &dev_priv->render_ring;
3801 break;
3802 case I915_EXEC_BSD:
3803 if (!HAS_BSD(dev)) {
3804 DRM_ERROR("execbuf with invalid ring (BSD)\n");
3805 return -EINVAL;
3806 }
3807 ring = &dev_priv->bsd_ring;
3808 break;
3809 case I915_EXEC_BLT:
3810 if (!HAS_BLT(dev)) {
3811 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3812 return -EINVAL;
3813 }
3814 ring = &dev_priv->blt_ring;
3815 break;
3816 default:
3817 DRM_ERROR("execbuf with unknown ring: %d\n",
3818 (int)(args->flags & I915_EXEC_RING_MASK));
3819 return -EINVAL;
3820 }
3821
3822 if (args->buffer_count < 1) {
3823 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3824 return -EINVAL;
3825 }
3826 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3827 if (object_list == NULL) {
3828 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3829 args->buffer_count);
3830 ret = -ENOMEM;
3831 goto pre_mutex_err;
3832 }
3833
3834 if (args->num_cliprects != 0) {
3835 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3836 GFP_KERNEL);
3837 if (cliprects == NULL) {
3838 ret = -ENOMEM;
3839 goto pre_mutex_err;
3840 }
3841
3842 ret = copy_from_user(cliprects,
3843 (struct drm_clip_rect __user *)
3844 (uintptr_t) args->cliprects_ptr,
3845 sizeof(*cliprects) * args->num_cliprects);
3846 if (ret != 0) {
3847 DRM_ERROR("copy %d cliprects failed: %d\n",
3848 args->num_cliprects, ret);
3849 ret = -EFAULT;
3850 goto pre_mutex_err;
3851 }
3852 }
3853
3854 request = kzalloc(sizeof(*request), GFP_KERNEL);
3855 if (request == NULL) {
3856 ret = -ENOMEM;
3857 goto pre_mutex_err;
3858 }
3859
3860 ret = i915_mutex_lock_interruptible(dev);
3861 if (ret)
3862 goto pre_mutex_err;
3863
3864 if (dev_priv->mm.suspended) {
3865 mutex_unlock(&dev->struct_mutex);
3866 ret = -EBUSY;
3867 goto pre_mutex_err;
3868 }
3869
3870 /* Look up object handles */
3871 for (i = 0; i < args->buffer_count; i++) {
3872 object_list[i] = drm_gem_object_lookup(dev, file,
3873 exec_list[i].handle);
3874 if (object_list[i] == NULL) {
3875 DRM_ERROR("Invalid object handle %d at index %d\n",
3876 exec_list[i].handle, i);
3877 /* prevent error path from reading uninitialized data */
3878 args->buffer_count = i + 1;
3879 ret = -ENOENT;
3880 goto err;
3881 }
3882
3883 obj_priv = to_intel_bo(object_list[i]);
3884 if (obj_priv->in_execbuffer) {
3885 DRM_ERROR("Object %p appears more than once in object list\n",
3886 object_list[i]);
3887 /* prevent error path from reading uninitialized data */
3888 args->buffer_count = i + 1;
3889 ret = -EINVAL;
3890 goto err;
3891 }
3892 obj_priv->in_execbuffer = true;
3893 }
3894
3895 /* Move the objects en-masse into the GTT, evicting if necessary. */
3896 ret = i915_gem_execbuffer_reserve(dev, file,
3897 object_list, exec_list,
3898 args->buffer_count);
3899 if (ret)
3900 goto err;
3901
3902 /* The objects are in their final locations, apply the relocations. */
3903 ret = i915_gem_execbuffer_relocate(dev, file,
3904 object_list, exec_list,
3905 args->buffer_count);
3906 if (ret) {
3907 if (ret == -EFAULT) {
3908 ret = i915_gem_execbuffer_relocate_slow(dev, file,
3909 object_list,
3910 exec_list,
3911 args->buffer_count);
3912 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3913 }
3914 if (ret)
3915 goto err;
3916 }
3917
3918 /* Set the pending read domains for the batch buffer to COMMAND */
3919 batch_obj = object_list[args->buffer_count-1];
3920 if (batch_obj->pending_write_domain) {
3921 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3922 ret = -EINVAL;
3923 goto err;
3924 }
3925 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3926
3927 /* Sanity check the batch buffer */
3928 exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3929 ret = i915_gem_check_execbuffer(args, exec_offset);
3930 if (ret != 0) {
3931 DRM_ERROR("execbuf with invalid offset/length\n");
3932 goto err;
3933 }
3934
3935 ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3936 object_list, args->buffer_count);
3937 if (ret)
3938 goto err;
3939
3940 for (i = 0; i < args->buffer_count; i++) {
3941 struct drm_gem_object *obj = object_list[i];
3942 uint32_t old_write_domain = obj->write_domain;
3943 obj->write_domain = obj->pending_write_domain;
3944 trace_i915_gem_object_change_domain(obj,
3945 obj->read_domains,
3946 old_write_domain);
3947 }
3948
3949#if WATCH_COHERENCY
3950 for (i = 0; i < args->buffer_count; i++) {
3951 i915_gem_object_check_coherency(object_list[i],
3952 exec_list[i].handle);
3953 }
3954#endif
3955
3956#if WATCH_EXEC
3957 i915_gem_dump_object(batch_obj,
3958 args->batch_len,
3959 __func__,
3960 ~0);
3961#endif
3962
3963 /* Check for any pending flips. As we only maintain a flip queue depth
3964 * of 1, we can simply insert a WAIT for the next display flip prior
3965 * to executing the batch and avoid stalling the CPU.
3966 */
3967 flips = 0;
3968 for (i = 0; i < args->buffer_count; i++) {
3969 if (object_list[i]->write_domain)
3970 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3971 }
3972 if (flips) {
3973 int plane, flip_mask;
3974
3975 for (plane = 0; flips >> plane; plane++) {
3976 if (((flips >> plane) & 1) == 0)
3977 continue;
3978
3979 if (plane)
3980 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3981 else
3982 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3983
3984 intel_ring_begin(dev, ring, 2);
3985 intel_ring_emit(dev, ring,
3986 MI_WAIT_FOR_EVENT | flip_mask);
3987 intel_ring_emit(dev, ring, MI_NOOP);
3988 intel_ring_advance(dev, ring);
3989 }
3990 }
3991
3992 /* Exec the batchbuffer */
3993 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3994 cliprects, exec_offset);
3995 if (ret) {
3996 DRM_ERROR("dispatch failed %d\n", ret);
3997 goto err;
3998 }
3999
4000 /*
4001 * Ensure that the commands in the batch buffer are
4002 * finished before the interrupt fires
4003 */
4004 i915_retire_commands(dev, ring);
4005
4006 for (i = 0; i < args->buffer_count; i++) {
4007 struct drm_gem_object *obj = object_list[i];
4008
4009 i915_gem_object_move_to_active(obj, ring);
4010 if (obj->write_domain)
4011 list_move_tail(&to_intel_bo(obj)->gpu_write_list,
4012 &ring->gpu_write_list);
4013 }
4014
4015 i915_add_request(dev, file, request, ring);
4016 request = NULL;
4017
4018err:
4019 for (i = 0; i < args->buffer_count; i++) {
4020 if (object_list[i]) {
4021 obj_priv = to_intel_bo(object_list[i]);
4022 obj_priv->in_execbuffer = false;
4023 }
4024 drm_gem_object_unreference(object_list[i]);
4025 }
4026
4027 mutex_unlock(&dev->struct_mutex);
4028
4029pre_mutex_err:
4030 drm_free_large(object_list);
4031 kfree(cliprects);
4032 kfree(request);
4033
4034 return ret;
4035}
4036
4037/*
4038 * Legacy execbuffer just creates an exec2 list from the original exec object
4039 * list array and passes it to the real function.
4040 */
4041int
4042i915_gem_execbuffer(struct drm_device *dev, void *data,
4043 struct drm_file *file_priv)
4044{
4045 struct drm_i915_gem_execbuffer *args = data;
4046 struct drm_i915_gem_execbuffer2 exec2;
4047 struct drm_i915_gem_exec_object *exec_list = NULL;
4048 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4049 int ret, i;
4050
4051#if WATCH_EXEC
4052 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4053 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4054#endif
4055
4056 if (args->buffer_count < 1) {
4057 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4058 return -EINVAL;
4059 }
4060
4061 /* Copy in the exec list from userland */
4062 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4063 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4064 if (exec_list == NULL || exec2_list == NULL) {
4065 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4066 args->buffer_count);
4067 drm_free_large(exec_list);
4068 drm_free_large(exec2_list);
4069 return -ENOMEM;
4070 }
4071 ret = copy_from_user(exec_list,
4072 (struct drm_i915_relocation_entry __user *)
4073 (uintptr_t) args->buffers_ptr,
4074 sizeof(*exec_list) * args->buffer_count);
4075 if (ret != 0) {
4076 DRM_ERROR("copy %d exec entries failed %d\n",
4077 args->buffer_count, ret);
4078 drm_free_large(exec_list);
4079 drm_free_large(exec2_list);
4080 return -EFAULT;
4081 }
4082
4083 for (i = 0; i < args->buffer_count; i++) {
4084 exec2_list[i].handle = exec_list[i].handle;
4085 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4086 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4087 exec2_list[i].alignment = exec_list[i].alignment;
4088 exec2_list[i].offset = exec_list[i].offset;
4089 if (INTEL_INFO(dev)->gen < 4)
4090 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4091 else
4092 exec2_list[i].flags = 0;
4093 }
4094 3167
4095 exec2.buffers_ptr = args->buffers_ptr; 3168 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
4096 exec2.buffer_count = args->buffer_count; 3169 ret = -EIO;
4097 exec2.batch_start_offset = args->batch_start_offset;
4098 exec2.batch_len = args->batch_len;
4099 exec2.DR1 = args->DR1;
4100 exec2.DR4 = args->DR4;
4101 exec2.num_cliprects = args->num_cliprects;
4102 exec2.cliprects_ptr = args->cliprects_ptr;
4103 exec2.flags = I915_EXEC_RENDER;
4104
4105 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4106 if (!ret) {
4107 /* Copy the new buffer offsets back to the user's exec list. */
4108 for (i = 0; i < args->buffer_count; i++)
4109 exec_list[i].offset = exec2_list[i].offset;
4110 /* ... and back out to userspace */
4111 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4112 (uintptr_t) args->buffers_ptr,
4113 exec_list,
4114 sizeof(*exec_list) * args->buffer_count);
4115 if (ret) {
4116 ret = -EFAULT;
4117 DRM_ERROR("failed to copy %d exec entries "
4118 "back to user (%d)\n",
4119 args->buffer_count, ret);
4120 } 3170 }
4121 } 3171 }
4122 3172
4123 drm_free_large(exec_list); 3173 if (ret == 0)
4124 drm_free_large(exec2_list); 3174 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4125 return ret;
4126}
4127
4128int
4129i915_gem_execbuffer2(struct drm_device *dev, void *data,
4130 struct drm_file *file_priv)
4131{
4132 struct drm_i915_gem_execbuffer2 *args = data;
4133 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4134 int ret;
4135
4136#if WATCH_EXEC
4137 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4138 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4139#endif
4140
4141 if (args->buffer_count < 1) {
4142 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4143 return -EINVAL;
4144 }
4145
4146 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4147 if (exec2_list == NULL) {
4148 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4149 args->buffer_count);
4150 return -ENOMEM;
4151 }
4152 ret = copy_from_user(exec2_list,
4153 (struct drm_i915_relocation_entry __user *)
4154 (uintptr_t) args->buffers_ptr,
4155 sizeof(*exec2_list) * args->buffer_count);
4156 if (ret != 0) {
4157 DRM_ERROR("copy %d exec entries failed %d\n",
4158 args->buffer_count, ret);
4159 drm_free_large(exec2_list);
4160 return -EFAULT;
4161 }
4162
4163 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4164 if (!ret) {
4165 /* Copy the new buffer offsets back to the user's exec list. */
4166 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4167 (uintptr_t) args->buffers_ptr,
4168 exec2_list,
4169 sizeof(*exec2_list) * args->buffer_count);
4170 if (ret) {
4171 ret = -EFAULT;
4172 DRM_ERROR("failed to copy %d exec entries "
4173 "back to user (%d)\n",
4174 args->buffer_count, ret);
4175 }
4176 }
4177 3175
4178 drm_free_large(exec2_list);
4179 return ret; 3176 return ret;
4180} 3177}
4181 3178
4182int 3179int
4183i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) 3180i915_gem_object_pin(struct drm_i915_gem_object *obj,
3181 uint32_t alignment,
3182 bool map_and_fenceable)
4184{ 3183{
4185 struct drm_device *dev = obj->dev; 3184 struct drm_device *dev = obj->base.dev;
4186 struct drm_i915_private *dev_priv = dev->dev_private; 3185 struct drm_i915_private *dev_priv = dev->dev_private;
4187 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4188 int ret; 3186 int ret;
4189 3187
4190 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); 3188 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4191 WARN_ON(i915_verify_lists(dev)); 3189 WARN_ON(i915_verify_lists(dev));
4192 3190
4193 if (obj_priv->gtt_space != NULL) { 3191 if (obj->gtt_space != NULL) {
4194 if (alignment == 0) 3192 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
4195 alignment = i915_gem_get_gtt_alignment(obj); 3193 (map_and_fenceable && !obj->map_and_fenceable)) {
4196 if (obj_priv->gtt_offset & (alignment - 1)) { 3194 WARN(obj->pin_count,
4197 WARN(obj_priv->pin_count, 3195 "bo is already pinned with incorrect alignment:"
4198 "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n", 3196 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4199 obj_priv->gtt_offset, alignment); 3197 " obj->map_and_fenceable=%d\n",
3198 obj->gtt_offset, alignment,
3199 map_and_fenceable,
3200 obj->map_and_fenceable);
4200 ret = i915_gem_object_unbind(obj); 3201 ret = i915_gem_object_unbind(obj);
4201 if (ret) 3202 if (ret)
4202 return ret; 3203 return ret;
4203 } 3204 }
4204 } 3205 }
4205 3206
4206 if (obj_priv->gtt_space == NULL) { 3207 if (obj->gtt_space == NULL) {
4207 ret = i915_gem_object_bind_to_gtt(obj, alignment); 3208 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3209 map_and_fenceable);
4208 if (ret) 3210 if (ret)
4209 return ret; 3211 return ret;
4210 } 3212 }
4211 3213
4212 obj_priv->pin_count++; 3214 if (obj->pin_count++ == 0) {
4213 3215 if (!obj->active)
4214 /* If the object is not active and not pending a flush, 3216 list_move_tail(&obj->mm_list,
4215 * remove it from the inactive list
4216 */
4217 if (obj_priv->pin_count == 1) {
4218 i915_gem_info_add_pin(dev_priv, obj->size);
4219 if (!obj_priv->active)
4220 list_move_tail(&obj_priv->mm_list,
4221 &dev_priv->mm.pinned_list); 3217 &dev_priv->mm.pinned_list);
4222 } 3218 }
3219 obj->pin_mappable |= map_and_fenceable;
4223 3220
4224 WARN_ON(i915_verify_lists(dev)); 3221 WARN_ON(i915_verify_lists(dev));
4225 return 0; 3222 return 0;
4226} 3223}
4227 3224
4228void 3225void
4229i915_gem_object_unpin(struct drm_gem_object *obj) 3226i915_gem_object_unpin(struct drm_i915_gem_object *obj)
4230{ 3227{
4231 struct drm_device *dev = obj->dev; 3228 struct drm_device *dev = obj->base.dev;
4232 drm_i915_private_t *dev_priv = dev->dev_private; 3229 drm_i915_private_t *dev_priv = dev->dev_private;
4233 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4234 3230
4235 WARN_ON(i915_verify_lists(dev)); 3231 WARN_ON(i915_verify_lists(dev));
4236 obj_priv->pin_count--; 3232 BUG_ON(obj->pin_count == 0);
4237 BUG_ON(obj_priv->pin_count < 0); 3233 BUG_ON(obj->gtt_space == NULL);
4238 BUG_ON(obj_priv->gtt_space == NULL);
4239 3234
4240 /* If the object is no longer pinned, and is 3235 if (--obj->pin_count == 0) {
4241 * neither active nor being flushed, then stick it on 3236 if (!obj->active)
4242 * the inactive list 3237 list_move_tail(&obj->mm_list,
4243 */
4244 if (obj_priv->pin_count == 0) {
4245 if (!obj_priv->active)
4246 list_move_tail(&obj_priv->mm_list,
4247 &dev_priv->mm.inactive_list); 3238 &dev_priv->mm.inactive_list);
4248 i915_gem_info_remove_pin(dev_priv, obj->size); 3239 obj->pin_mappable = false;
4249 } 3240 }
4250 WARN_ON(i915_verify_lists(dev)); 3241 WARN_ON(i915_verify_lists(dev));
4251} 3242}
4252 3243
4253int 3244int
4254i915_gem_pin_ioctl(struct drm_device *dev, void *data, 3245i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4255 struct drm_file *file_priv) 3246 struct drm_file *file)
4256{ 3247{
4257 struct drm_i915_gem_pin *args = data; 3248 struct drm_i915_gem_pin *args = data;
4258 struct drm_gem_object *obj; 3249 struct drm_i915_gem_object *obj;
4259 struct drm_i915_gem_object *obj_priv;
4260 int ret; 3250 int ret;
4261 3251
4262 ret = i915_mutex_lock_interruptible(dev); 3252 ret = i915_mutex_lock_interruptible(dev);
4263 if (ret) 3253 if (ret)
4264 return ret; 3254 return ret;
4265 3255
4266 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3256 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4267 if (obj == NULL) { 3257 if (obj == NULL) {
4268 ret = -ENOENT; 3258 ret = -ENOENT;
4269 goto unlock; 3259 goto unlock;
4270 } 3260 }
4271 obj_priv = to_intel_bo(obj);
4272 3261
4273 if (obj_priv->madv != I915_MADV_WILLNEED) { 3262 if (obj->madv != I915_MADV_WILLNEED) {
4274 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3263 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4275 ret = -EINVAL; 3264 ret = -EINVAL;
4276 goto out; 3265 goto out;
4277 } 3266 }
4278 3267
4279 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { 3268 if (obj->pin_filp != NULL && obj->pin_filp != file) {
4280 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3269 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4281 args->handle); 3270 args->handle);
4282 ret = -EINVAL; 3271 ret = -EINVAL;
4283 goto out; 3272 goto out;
4284 } 3273 }
4285 3274
4286 obj_priv->user_pin_count++; 3275 obj->user_pin_count++;
4287 obj_priv->pin_filp = file_priv; 3276 obj->pin_filp = file;
4288 if (obj_priv->user_pin_count == 1) { 3277 if (obj->user_pin_count == 1) {
4289 ret = i915_gem_object_pin(obj, args->alignment); 3278 ret = i915_gem_object_pin(obj, args->alignment, true);
4290 if (ret) 3279 if (ret)
4291 goto out; 3280 goto out;
4292 } 3281 }
@@ -4295,9 +3284,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4295 * as the X server doesn't manage domains yet 3284 * as the X server doesn't manage domains yet
4296 */ 3285 */
4297 i915_gem_object_flush_cpu_write_domain(obj); 3286 i915_gem_object_flush_cpu_write_domain(obj);
4298 args->offset = obj_priv->gtt_offset; 3287 args->offset = obj->gtt_offset;
4299out: 3288out:
4300 drm_gem_object_unreference(obj); 3289 drm_gem_object_unreference(&obj->base);
4301unlock: 3290unlock:
4302 mutex_unlock(&dev->struct_mutex); 3291 mutex_unlock(&dev->struct_mutex);
4303 return ret; 3292 return ret;
@@ -4305,38 +3294,36 @@ unlock:
4305 3294
4306int 3295int
4307i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3296i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4308 struct drm_file *file_priv) 3297 struct drm_file *file)
4309{ 3298{
4310 struct drm_i915_gem_pin *args = data; 3299 struct drm_i915_gem_pin *args = data;
4311 struct drm_gem_object *obj; 3300 struct drm_i915_gem_object *obj;
4312 struct drm_i915_gem_object *obj_priv;
4313 int ret; 3301 int ret;
4314 3302
4315 ret = i915_mutex_lock_interruptible(dev); 3303 ret = i915_mutex_lock_interruptible(dev);
4316 if (ret) 3304 if (ret)
4317 return ret; 3305 return ret;
4318 3306
4319 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3307 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4320 if (obj == NULL) { 3308 if (obj == NULL) {
4321 ret = -ENOENT; 3309 ret = -ENOENT;
4322 goto unlock; 3310 goto unlock;
4323 } 3311 }
4324 obj_priv = to_intel_bo(obj);
4325 3312
4326 if (obj_priv->pin_filp != file_priv) { 3313 if (obj->pin_filp != file) {
4327 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3314 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4328 args->handle); 3315 args->handle);
4329 ret = -EINVAL; 3316 ret = -EINVAL;
4330 goto out; 3317 goto out;
4331 } 3318 }
4332 obj_priv->user_pin_count--; 3319 obj->user_pin_count--;
4333 if (obj_priv->user_pin_count == 0) { 3320 if (obj->user_pin_count == 0) {
4334 obj_priv->pin_filp = NULL; 3321 obj->pin_filp = NULL;
4335 i915_gem_object_unpin(obj); 3322 i915_gem_object_unpin(obj);
4336 } 3323 }
4337 3324
4338out: 3325out:
4339 drm_gem_object_unreference(obj); 3326 drm_gem_object_unreference(&obj->base);
4340unlock: 3327unlock:
4341 mutex_unlock(&dev->struct_mutex); 3328 mutex_unlock(&dev->struct_mutex);
4342 return ret; 3329 return ret;
@@ -4344,48 +3331,50 @@ unlock:
4344 3331
4345int 3332int
4346i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3333i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4347 struct drm_file *file_priv) 3334 struct drm_file *file)
4348{ 3335{
4349 struct drm_i915_gem_busy *args = data; 3336 struct drm_i915_gem_busy *args = data;
4350 struct drm_gem_object *obj; 3337 struct drm_i915_gem_object *obj;
4351 struct drm_i915_gem_object *obj_priv;
4352 int ret; 3338 int ret;
4353 3339
4354 ret = i915_mutex_lock_interruptible(dev); 3340 ret = i915_mutex_lock_interruptible(dev);
4355 if (ret) 3341 if (ret)
4356 return ret; 3342 return ret;
4357 3343
4358 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3344 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4359 if (obj == NULL) { 3345 if (obj == NULL) {
4360 ret = -ENOENT; 3346 ret = -ENOENT;
4361 goto unlock; 3347 goto unlock;
4362 } 3348 }
4363 obj_priv = to_intel_bo(obj);
4364 3349
4365 /* Count all active objects as busy, even if they are currently not used 3350 /* Count all active objects as busy, even if they are currently not used
4366 * by the gpu. Users of this interface expect objects to eventually 3351 * by the gpu. Users of this interface expect objects to eventually
4367 * become non-busy without any further actions, therefore emit any 3352 * become non-busy without any further actions, therefore emit any
4368 * necessary flushes here. 3353 * necessary flushes here.
4369 */ 3354 */
4370 args->busy = obj_priv->active; 3355 args->busy = obj->active;
4371 if (args->busy) { 3356 if (args->busy) {
4372 /* Unconditionally flush objects, even when the gpu still uses this 3357 /* Unconditionally flush objects, even when the gpu still uses this
4373 * object. Userspace calling this function indicates that it wants to 3358 * object. Userspace calling this function indicates that it wants to
4374 * use this buffer rather sooner than later, so issuing the required 3359 * use this buffer rather sooner than later, so issuing the required
4375 * flush earlier is beneficial. 3360 * flush earlier is beneficial.
4376 */ 3361 */
4377 if (obj->write_domain & I915_GEM_GPU_DOMAINS) { 3362 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
4378 i915_gem_flush_ring(dev, file_priv, 3363 i915_gem_flush_ring(dev, obj->ring,
4379 obj_priv->ring, 3364 0, obj->base.write_domain);
4380 0, obj->write_domain); 3365 } else if (obj->ring->outstanding_lazy_request ==
4381 } else if (obj_priv->ring->outstanding_lazy_request) { 3366 obj->last_rendering_seqno) {
3367 struct drm_i915_gem_request *request;
3368
4382 /* This ring is not being cleared by active usage, 3369 /* This ring is not being cleared by active usage,
4383 * so emit a request to do so. 3370 * so emit a request to do so.
4384 */ 3371 */
4385 u32 seqno = i915_add_request(dev, 3372 request = kzalloc(sizeof(*request), GFP_KERNEL);
4386 NULL, NULL, 3373 if (request)
4387 obj_priv->ring); 3374 ret = i915_add_request(dev,
4388 if (seqno == 0) 3375 NULL, request,
3376 obj->ring);
3377 else
4389 ret = -ENOMEM; 3378 ret = -ENOMEM;
4390 } 3379 }
4391 3380
@@ -4394,12 +3383,12 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4394 * are actually unmasked, and our working set ends up being 3383 * are actually unmasked, and our working set ends up being
4395 * larger than required. 3384 * larger than required.
4396 */ 3385 */
4397 i915_gem_retire_requests_ring(dev, obj_priv->ring); 3386 i915_gem_retire_requests_ring(dev, obj->ring);
4398 3387
4399 args->busy = obj_priv->active; 3388 args->busy = obj->active;
4400 } 3389 }
4401 3390
4402 drm_gem_object_unreference(obj); 3391 drm_gem_object_unreference(&obj->base);
4403unlock: 3392unlock:
4404 mutex_unlock(&dev->struct_mutex); 3393 mutex_unlock(&dev->struct_mutex);
4405 return ret; 3394 return ret;
@@ -4417,8 +3406,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4417 struct drm_file *file_priv) 3406 struct drm_file *file_priv)
4418{ 3407{
4419 struct drm_i915_gem_madvise *args = data; 3408 struct drm_i915_gem_madvise *args = data;
4420 struct drm_gem_object *obj; 3409 struct drm_i915_gem_object *obj;
4421 struct drm_i915_gem_object *obj_priv;
4422 int ret; 3410 int ret;
4423 3411
4424 switch (args->madv) { 3412 switch (args->madv) {
@@ -4433,37 +3421,36 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4433 if (ret) 3421 if (ret)
4434 return ret; 3422 return ret;
4435 3423
4436 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3424 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4437 if (obj == NULL) { 3425 if (obj == NULL) {
4438 ret = -ENOENT; 3426 ret = -ENOENT;
4439 goto unlock; 3427 goto unlock;
4440 } 3428 }
4441 obj_priv = to_intel_bo(obj);
4442 3429
4443 if (obj_priv->pin_count) { 3430 if (obj->pin_count) {
4444 ret = -EINVAL; 3431 ret = -EINVAL;
4445 goto out; 3432 goto out;
4446 } 3433 }
4447 3434
4448 if (obj_priv->madv != __I915_MADV_PURGED) 3435 if (obj->madv != __I915_MADV_PURGED)
4449 obj_priv->madv = args->madv; 3436 obj->madv = args->madv;
4450 3437
4451 /* if the object is no longer bound, discard its backing storage */ 3438 /* if the object is no longer bound, discard its backing storage */
4452 if (i915_gem_object_is_purgeable(obj_priv) && 3439 if (i915_gem_object_is_purgeable(obj) &&
4453 obj_priv->gtt_space == NULL) 3440 obj->gtt_space == NULL)
4454 i915_gem_object_truncate(obj); 3441 i915_gem_object_truncate(obj);
4455 3442
4456 args->retained = obj_priv->madv != __I915_MADV_PURGED; 3443 args->retained = obj->madv != __I915_MADV_PURGED;
4457 3444
4458out: 3445out:
4459 drm_gem_object_unreference(obj); 3446 drm_gem_object_unreference(&obj->base);
4460unlock: 3447unlock:
4461 mutex_unlock(&dev->struct_mutex); 3448 mutex_unlock(&dev->struct_mutex);
4462 return ret; 3449 return ret;
4463} 3450}
4464 3451
4465struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 3452struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4466 size_t size) 3453 size_t size)
4467{ 3454{
4468 struct drm_i915_private *dev_priv = dev->dev_private; 3455 struct drm_i915_private *dev_priv = dev->dev_private;
4469 struct drm_i915_gem_object *obj; 3456 struct drm_i915_gem_object *obj;
@@ -4486,11 +3473,15 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4486 obj->base.driver_private = NULL; 3473 obj->base.driver_private = NULL;
4487 obj->fence_reg = I915_FENCE_REG_NONE; 3474 obj->fence_reg = I915_FENCE_REG_NONE;
4488 INIT_LIST_HEAD(&obj->mm_list); 3475 INIT_LIST_HEAD(&obj->mm_list);
3476 INIT_LIST_HEAD(&obj->gtt_list);
4489 INIT_LIST_HEAD(&obj->ring_list); 3477 INIT_LIST_HEAD(&obj->ring_list);
3478 INIT_LIST_HEAD(&obj->exec_list);
4490 INIT_LIST_HEAD(&obj->gpu_write_list); 3479 INIT_LIST_HEAD(&obj->gpu_write_list);
4491 obj->madv = I915_MADV_WILLNEED; 3480 obj->madv = I915_MADV_WILLNEED;
3481 /* Avoid an unnecessary call to unbind on the first bind. */
3482 obj->map_and_fenceable = true;
4492 3483
4493 return &obj->base; 3484 return obj;
4494} 3485}
4495 3486
4496int i915_gem_init_object(struct drm_gem_object *obj) 3487int i915_gem_init_object(struct drm_gem_object *obj)
@@ -4500,42 +3491,41 @@ int i915_gem_init_object(struct drm_gem_object *obj)
4500 return 0; 3491 return 0;
4501} 3492}
4502 3493
4503static void i915_gem_free_object_tail(struct drm_gem_object *obj) 3494static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
4504{ 3495{
4505 struct drm_device *dev = obj->dev; 3496 struct drm_device *dev = obj->base.dev;
4506 drm_i915_private_t *dev_priv = dev->dev_private; 3497 drm_i915_private_t *dev_priv = dev->dev_private;
4507 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4508 int ret; 3498 int ret;
4509 3499
4510 ret = i915_gem_object_unbind(obj); 3500 ret = i915_gem_object_unbind(obj);
4511 if (ret == -ERESTARTSYS) { 3501 if (ret == -ERESTARTSYS) {
4512 list_move(&obj_priv->mm_list, 3502 list_move(&obj->mm_list,
4513 &dev_priv->mm.deferred_free_list); 3503 &dev_priv->mm.deferred_free_list);
4514 return; 3504 return;
4515 } 3505 }
4516 3506
4517 if (obj_priv->mmap_offset) 3507 if (obj->base.map_list.map)
4518 i915_gem_free_mmap_offset(obj); 3508 i915_gem_free_mmap_offset(obj);
4519 3509
4520 drm_gem_object_release(obj); 3510 drm_gem_object_release(&obj->base);
4521 i915_gem_info_remove_obj(dev_priv, obj->size); 3511 i915_gem_info_remove_obj(dev_priv, obj->base.size);
4522 3512
4523 kfree(obj_priv->page_cpu_valid); 3513 kfree(obj->page_cpu_valid);
4524 kfree(obj_priv->bit_17); 3514 kfree(obj->bit_17);
4525 kfree(obj_priv); 3515 kfree(obj);
4526} 3516}
4527 3517
4528void i915_gem_free_object(struct drm_gem_object *obj) 3518void i915_gem_free_object(struct drm_gem_object *gem_obj)
4529{ 3519{
4530 struct drm_device *dev = obj->dev; 3520 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4531 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3521 struct drm_device *dev = obj->base.dev;
4532 3522
4533 trace_i915_gem_object_destroy(obj); 3523 trace_i915_gem_object_destroy(obj);
4534 3524
4535 while (obj_priv->pin_count > 0) 3525 while (obj->pin_count > 0)
4536 i915_gem_object_unpin(obj); 3526 i915_gem_object_unpin(obj);
4537 3527
4538 if (obj_priv->phys_obj) 3528 if (obj->phys_obj)
4539 i915_gem_detach_phys_object(dev, obj); 3529 i915_gem_detach_phys_object(dev, obj);
4540 3530
4541 i915_gem_free_object_tail(obj); 3531 i915_gem_free_object_tail(obj);
@@ -4562,13 +3552,15 @@ i915_gem_idle(struct drm_device *dev)
4562 3552
4563 /* Under UMS, be paranoid and evict. */ 3553 /* Under UMS, be paranoid and evict. */
4564 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 3554 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4565 ret = i915_gem_evict_inactive(dev); 3555 ret = i915_gem_evict_inactive(dev, false);
4566 if (ret) { 3556 if (ret) {
4567 mutex_unlock(&dev->struct_mutex); 3557 mutex_unlock(&dev->struct_mutex);
4568 return ret; 3558 return ret;
4569 } 3559 }
4570 } 3560 }
4571 3561
3562 i915_gem_reset_fences(dev);
3563
4572 /* Hack! Don't let anybody do execbuf while we don't control the chip. 3564 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4573 * We need to replace this with a semaphore, or something. 3565 * We need to replace this with a semaphore, or something.
4574 * And not confound mm.suspended! 3566 * And not confound mm.suspended!
@@ -4587,82 +3579,15 @@ i915_gem_idle(struct drm_device *dev)
4587 return 0; 3579 return 0;
4588} 3580}
4589 3581
4590/*
4591 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4592 * over cache flushing.
4593 */
4594static int
4595i915_gem_init_pipe_control(struct drm_device *dev)
4596{
4597 drm_i915_private_t *dev_priv = dev->dev_private;
4598 struct drm_gem_object *obj;
4599 struct drm_i915_gem_object *obj_priv;
4600 int ret;
4601
4602 obj = i915_gem_alloc_object(dev, 4096);
4603 if (obj == NULL) {
4604 DRM_ERROR("Failed to allocate seqno page\n");
4605 ret = -ENOMEM;
4606 goto err;
4607 }
4608 obj_priv = to_intel_bo(obj);
4609 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4610
4611 ret = i915_gem_object_pin(obj, 4096);
4612 if (ret)
4613 goto err_unref;
4614
4615 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4616 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4617 if (dev_priv->seqno_page == NULL)
4618 goto err_unpin;
4619
4620 dev_priv->seqno_obj = obj;
4621 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4622
4623 return 0;
4624
4625err_unpin:
4626 i915_gem_object_unpin(obj);
4627err_unref:
4628 drm_gem_object_unreference(obj);
4629err:
4630 return ret;
4631}
4632
4633
4634static void
4635i915_gem_cleanup_pipe_control(struct drm_device *dev)
4636{
4637 drm_i915_private_t *dev_priv = dev->dev_private;
4638 struct drm_gem_object *obj;
4639 struct drm_i915_gem_object *obj_priv;
4640
4641 obj = dev_priv->seqno_obj;
4642 obj_priv = to_intel_bo(obj);
4643 kunmap(obj_priv->pages[0]);
4644 i915_gem_object_unpin(obj);
4645 drm_gem_object_unreference(obj);
4646 dev_priv->seqno_obj = NULL;
4647
4648 dev_priv->seqno_page = NULL;
4649}
4650
4651int 3582int
4652i915_gem_init_ringbuffer(struct drm_device *dev) 3583i915_gem_init_ringbuffer(struct drm_device *dev)
4653{ 3584{
4654 drm_i915_private_t *dev_priv = dev->dev_private; 3585 drm_i915_private_t *dev_priv = dev->dev_private;
4655 int ret; 3586 int ret;
4656 3587
4657 if (HAS_PIPE_CONTROL(dev)) {
4658 ret = i915_gem_init_pipe_control(dev);
4659 if (ret)
4660 return ret;
4661 }
4662
4663 ret = intel_init_render_ring_buffer(dev); 3588 ret = intel_init_render_ring_buffer(dev);
4664 if (ret) 3589 if (ret)
4665 goto cleanup_pipe_control; 3590 return ret;
4666 3591
4667 if (HAS_BSD(dev)) { 3592 if (HAS_BSD(dev)) {
4668 ret = intel_init_bsd_ring_buffer(dev); 3593 ret = intel_init_bsd_ring_buffer(dev);
@@ -4681,12 +3606,9 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
4681 return 0; 3606 return 0;
4682 3607
4683cleanup_bsd_ring: 3608cleanup_bsd_ring:
4684 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 3609 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4685cleanup_render_ring: 3610cleanup_render_ring:
4686 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3611 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4687cleanup_pipe_control:
4688 if (HAS_PIPE_CONTROL(dev))
4689 i915_gem_cleanup_pipe_control(dev);
4690 return ret; 3612 return ret;
4691} 3613}
4692 3614
@@ -4694,12 +3616,10 @@ void
4694i915_gem_cleanup_ringbuffer(struct drm_device *dev) 3616i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4695{ 3617{
4696 drm_i915_private_t *dev_priv = dev->dev_private; 3618 drm_i915_private_t *dev_priv = dev->dev_private;
3619 int i;
4697 3620
4698 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3621 for (i = 0; i < I915_NUM_RINGS; i++)
4699 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 3622 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
4700 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
4701 if (HAS_PIPE_CONTROL(dev))
4702 i915_gem_cleanup_pipe_control(dev);
4703} 3623}
4704 3624
4705int 3625int
@@ -4707,7 +3627,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4707 struct drm_file *file_priv) 3627 struct drm_file *file_priv)
4708{ 3628{
4709 drm_i915_private_t *dev_priv = dev->dev_private; 3629 drm_i915_private_t *dev_priv = dev->dev_private;
4710 int ret; 3630 int ret, i;
4711 3631
4712 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3632 if (drm_core_check_feature(dev, DRIVER_MODESET))
4713 return 0; 3633 return 0;
@@ -4727,14 +3647,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4727 } 3647 }
4728 3648
4729 BUG_ON(!list_empty(&dev_priv->mm.active_list)); 3649 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4730 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4731 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4732 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4733 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 3650 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4734 BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); 3651 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4735 BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); 3652 for (i = 0; i < I915_NUM_RINGS; i++) {
4736 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list)); 3653 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
4737 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list)); 3654 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3655 }
4738 mutex_unlock(&dev->struct_mutex); 3656 mutex_unlock(&dev->struct_mutex);
4739 3657
4740 ret = drm_irq_install(dev); 3658 ret = drm_irq_install(dev);
@@ -4796,17 +3714,14 @@ i915_gem_load(struct drm_device *dev)
4796 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 3714 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4797 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3715 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4798 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 3716 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4799 init_ring_lists(&dev_priv->render_ring); 3717 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
4800 init_ring_lists(&dev_priv->bsd_ring); 3718 for (i = 0; i < I915_NUM_RINGS; i++)
4801 init_ring_lists(&dev_priv->blt_ring); 3719 init_ring_lists(&dev_priv->ring[i]);
4802 for (i = 0; i < 16; i++) 3720 for (i = 0; i < 16; i++)
4803 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 3721 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4804 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 3722 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4805 i915_gem_retire_work_handler); 3723 i915_gem_retire_work_handler);
4806 init_completion(&dev_priv->error_completion); 3724 init_completion(&dev_priv->error_completion);
4807 spin_lock(&shrink_list_lock);
4808 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4809 spin_unlock(&shrink_list_lock);
4810 3725
4811 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 3726 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4812 if (IS_GEN3(dev)) { 3727 if (IS_GEN3(dev)) {
@@ -4849,6 +3764,10 @@ i915_gem_load(struct drm_device *dev)
4849 } 3764 }
4850 i915_gem_detect_bit_6_swizzle(dev); 3765 i915_gem_detect_bit_6_swizzle(dev);
4851 init_waitqueue_head(&dev_priv->pending_flip_queue); 3766 init_waitqueue_head(&dev_priv->pending_flip_queue);
3767
3768 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3769 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3770 register_shrinker(&dev_priv->mm.inactive_shrinker);
4852} 3771}
4853 3772
4854/* 3773/*
@@ -4918,47 +3837,47 @@ void i915_gem_free_all_phys_object(struct drm_device *dev)
4918} 3837}
4919 3838
4920void i915_gem_detach_phys_object(struct drm_device *dev, 3839void i915_gem_detach_phys_object(struct drm_device *dev,
4921 struct drm_gem_object *obj) 3840 struct drm_i915_gem_object *obj)
4922{ 3841{
4923 struct drm_i915_gem_object *obj_priv; 3842 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3843 char *vaddr;
4924 int i; 3844 int i;
4925 int ret;
4926 int page_count; 3845 int page_count;
4927 3846
4928 obj_priv = to_intel_bo(obj); 3847 if (!obj->phys_obj)
4929 if (!obj_priv->phys_obj)
4930 return; 3848 return;
3849 vaddr = obj->phys_obj->handle->vaddr;
4931 3850
4932 ret = i915_gem_object_get_pages(obj, 0); 3851 page_count = obj->base.size / PAGE_SIZE;
4933 if (ret)
4934 goto out;
4935
4936 page_count = obj->size / PAGE_SIZE;
4937
4938 for (i = 0; i < page_count; i++) { 3852 for (i = 0; i < page_count; i++) {
4939 char *dst = kmap_atomic(obj_priv->pages[i]); 3853 struct page *page = read_cache_page_gfp(mapping, i,
4940 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3854 GFP_HIGHUSER | __GFP_RECLAIMABLE);
4941 3855 if (!IS_ERR(page)) {
4942 memcpy(dst, src, PAGE_SIZE); 3856 char *dst = kmap_atomic(page);
4943 kunmap_atomic(dst); 3857 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3858 kunmap_atomic(dst);
3859
3860 drm_clflush_pages(&page, 1);
3861
3862 set_page_dirty(page);
3863 mark_page_accessed(page);
3864 page_cache_release(page);
3865 }
4944 } 3866 }
4945 drm_clflush_pages(obj_priv->pages, page_count); 3867 intel_gtt_chipset_flush();
4946 drm_agp_chipset_flush(dev);
4947 3868
4948 i915_gem_object_put_pages(obj); 3869 obj->phys_obj->cur_obj = NULL;
4949out: 3870 obj->phys_obj = NULL;
4950 obj_priv->phys_obj->cur_obj = NULL;
4951 obj_priv->phys_obj = NULL;
4952} 3871}
4953 3872
4954int 3873int
4955i915_gem_attach_phys_object(struct drm_device *dev, 3874i915_gem_attach_phys_object(struct drm_device *dev,
4956 struct drm_gem_object *obj, 3875 struct drm_i915_gem_object *obj,
4957 int id, 3876 int id,
4958 int align) 3877 int align)
4959{ 3878{
3879 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4960 drm_i915_private_t *dev_priv = dev->dev_private; 3880 drm_i915_private_t *dev_priv = dev->dev_private;
4961 struct drm_i915_gem_object *obj_priv;
4962 int ret = 0; 3881 int ret = 0;
4963 int page_count; 3882 int page_count;
4964 int i; 3883 int i;
@@ -4966,10 +3885,8 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4966 if (id > I915_MAX_PHYS_OBJECT) 3885 if (id > I915_MAX_PHYS_OBJECT)
4967 return -EINVAL; 3886 return -EINVAL;
4968 3887
4969 obj_priv = to_intel_bo(obj); 3888 if (obj->phys_obj) {
4970 3889 if (obj->phys_obj->id == id)
4971 if (obj_priv->phys_obj) {
4972 if (obj_priv->phys_obj->id == id)
4973 return 0; 3890 return 0;
4974 i915_gem_detach_phys_object(dev, obj); 3891 i915_gem_detach_phys_object(dev, obj);
4975 } 3892 }
@@ -4977,51 +3894,50 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4977 /* create a new object */ 3894 /* create a new object */
4978 if (!dev_priv->mm.phys_objs[id - 1]) { 3895 if (!dev_priv->mm.phys_objs[id - 1]) {
4979 ret = i915_gem_init_phys_object(dev, id, 3896 ret = i915_gem_init_phys_object(dev, id,
4980 obj->size, align); 3897 obj->base.size, align);
4981 if (ret) { 3898 if (ret) {
4982 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); 3899 DRM_ERROR("failed to init phys object %d size: %zu\n",
4983 goto out; 3900 id, obj->base.size);
3901 return ret;
4984 } 3902 }
4985 } 3903 }
4986 3904
4987 /* bind to the object */ 3905 /* bind to the object */
4988 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3906 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4989 obj_priv->phys_obj->cur_obj = obj; 3907 obj->phys_obj->cur_obj = obj;
4990 3908
4991 ret = i915_gem_object_get_pages(obj, 0); 3909 page_count = obj->base.size / PAGE_SIZE;
4992 if (ret) {
4993 DRM_ERROR("failed to get page list\n");
4994 goto out;
4995 }
4996
4997 page_count = obj->size / PAGE_SIZE;
4998 3910
4999 for (i = 0; i < page_count; i++) { 3911 for (i = 0; i < page_count; i++) {
5000 char *src = kmap_atomic(obj_priv->pages[i]); 3912 struct page *page;
5001 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3913 char *dst, *src;
5002 3914
3915 page = read_cache_page_gfp(mapping, i,
3916 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3917 if (IS_ERR(page))
3918 return PTR_ERR(page);
3919
3920 src = kmap_atomic(page);
3921 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5003 memcpy(dst, src, PAGE_SIZE); 3922 memcpy(dst, src, PAGE_SIZE);
5004 kunmap_atomic(src); 3923 kunmap_atomic(src);
5005 }
5006 3924
5007 i915_gem_object_put_pages(obj); 3925 mark_page_accessed(page);
3926 page_cache_release(page);
3927 }
5008 3928
5009 return 0; 3929 return 0;
5010out:
5011 return ret;
5012} 3930}
5013 3931
5014static int 3932static int
5015i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 3933i915_gem_phys_pwrite(struct drm_device *dev,
3934 struct drm_i915_gem_object *obj,
5016 struct drm_i915_gem_pwrite *args, 3935 struct drm_i915_gem_pwrite *args,
5017 struct drm_file *file_priv) 3936 struct drm_file *file_priv)
5018{ 3937{
5019 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3938 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
5020 void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
5021 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr; 3939 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
5022 3940
5023 DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
5024
5025 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 3941 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5026 unsigned long unwritten; 3942 unsigned long unwritten;
5027 3943
@@ -5036,7 +3952,7 @@ i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5036 return -EFAULT; 3952 return -EFAULT;
5037 } 3953 }
5038 3954
5039 drm_agp_chipset_flush(dev); 3955 intel_gtt_chipset_flush();
5040 return 0; 3956 return 0;
5041} 3957}
5042 3958
@@ -5074,144 +3990,68 @@ i915_gpu_is_active(struct drm_device *dev)
5074} 3990}
5075 3991
5076static int 3992static int
5077i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask) 3993i915_gem_inactive_shrink(struct shrinker *shrinker,
5078{ 3994 int nr_to_scan,
5079 drm_i915_private_t *dev_priv, *next_dev; 3995 gfp_t gfp_mask)
5080 struct drm_i915_gem_object *obj_priv, *next_obj; 3996{
5081 int cnt = 0; 3997 struct drm_i915_private *dev_priv =
5082 int would_deadlock = 1; 3998 container_of(shrinker,
3999 struct drm_i915_private,
4000 mm.inactive_shrinker);
4001 struct drm_device *dev = dev_priv->dev;
4002 struct drm_i915_gem_object *obj, *next;
4003 int cnt;
4004
4005 if (!mutex_trylock(&dev->struct_mutex))
4006 return 0;
5083 4007
5084 /* "fast-path" to count number of available objects */ 4008 /* "fast-path" to count number of available objects */
5085 if (nr_to_scan == 0) { 4009 if (nr_to_scan == 0) {
5086 spin_lock(&shrink_list_lock); 4010 cnt = 0;
5087 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4011 list_for_each_entry(obj,
5088 struct drm_device *dev = dev_priv->dev; 4012 &dev_priv->mm.inactive_list,
5089 4013 mm_list)
5090 if (mutex_trylock(&dev->struct_mutex)) { 4014 cnt++;
5091 list_for_each_entry(obj_priv, 4015 mutex_unlock(&dev->struct_mutex);
5092 &dev_priv->mm.inactive_list, 4016 return cnt / 100 * sysctl_vfs_cache_pressure;
5093 mm_list)
5094 cnt++;
5095 mutex_unlock(&dev->struct_mutex);
5096 }
5097 }
5098 spin_unlock(&shrink_list_lock);
5099
5100 return (cnt / 100) * sysctl_vfs_cache_pressure;
5101 } 4017 }
5102 4018
5103 spin_lock(&shrink_list_lock);
5104
5105rescan: 4019rescan:
5106 /* first scan for clean buffers */ 4020 /* first scan for clean buffers */
5107 list_for_each_entry_safe(dev_priv, next_dev, 4021 i915_gem_retire_requests(dev);
5108 &shrink_list, mm.shrink_list) {
5109 struct drm_device *dev = dev_priv->dev;
5110
5111 if (! mutex_trylock(&dev->struct_mutex))
5112 continue;
5113
5114 spin_unlock(&shrink_list_lock);
5115 i915_gem_retire_requests(dev);
5116 4022
5117 list_for_each_entry_safe(obj_priv, next_obj, 4023 list_for_each_entry_safe(obj, next,
5118 &dev_priv->mm.inactive_list, 4024 &dev_priv->mm.inactive_list,
5119 mm_list) { 4025 mm_list) {
5120 if (i915_gem_object_is_purgeable(obj_priv)) { 4026 if (i915_gem_object_is_purgeable(obj)) {
5121 i915_gem_object_unbind(&obj_priv->base); 4027 if (i915_gem_object_unbind(obj) == 0 &&
5122 if (--nr_to_scan <= 0) 4028 --nr_to_scan == 0)
5123 break; 4029 break;
5124 }
5125 } 4030 }
5126
5127 spin_lock(&shrink_list_lock);
5128 mutex_unlock(&dev->struct_mutex);
5129
5130 would_deadlock = 0;
5131
5132 if (nr_to_scan <= 0)
5133 break;
5134 } 4031 }
5135 4032
5136 /* second pass, evict/count anything still on the inactive list */ 4033 /* second pass, evict/count anything still on the inactive list */
5137 list_for_each_entry_safe(dev_priv, next_dev, 4034 cnt = 0;
5138 &shrink_list, mm.shrink_list) { 4035 list_for_each_entry_safe(obj, next,
5139 struct drm_device *dev = dev_priv->dev; 4036 &dev_priv->mm.inactive_list,
5140 4037 mm_list) {
5141 if (! mutex_trylock(&dev->struct_mutex)) 4038 if (nr_to_scan &&
5142 continue; 4039 i915_gem_object_unbind(obj) == 0)
5143 4040 nr_to_scan--;
5144 spin_unlock(&shrink_list_lock); 4041 else
5145 4042 cnt++;
5146 list_for_each_entry_safe(obj_priv, next_obj,
5147 &dev_priv->mm.inactive_list,
5148 mm_list) {
5149 if (nr_to_scan > 0) {
5150 i915_gem_object_unbind(&obj_priv->base);
5151 nr_to_scan--;
5152 } else
5153 cnt++;
5154 }
5155
5156 spin_lock(&shrink_list_lock);
5157 mutex_unlock(&dev->struct_mutex);
5158
5159 would_deadlock = 0;
5160 } 4043 }
5161 4044
5162 if (nr_to_scan) { 4045 if (nr_to_scan && i915_gpu_is_active(dev)) {
5163 int active = 0;
5164
5165 /* 4046 /*
5166 * We are desperate for pages, so as a last resort, wait 4047 * We are desperate for pages, so as a last resort, wait
5167 * for the GPU to finish and discard whatever we can. 4048 * for the GPU to finish and discard whatever we can.
5168 * This has a dramatic impact to reduce the number of 4049 * This has a dramatic impact to reduce the number of
5169 * OOM-killer events whilst running the GPU aggressively. 4050 * OOM-killer events whilst running the GPU aggressively.
5170 */ 4051 */
5171 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4052 if (i915_gpu_idle(dev) == 0)
5172 struct drm_device *dev = dev_priv->dev;
5173
5174 if (!mutex_trylock(&dev->struct_mutex))
5175 continue;
5176
5177 spin_unlock(&shrink_list_lock);
5178
5179 if (i915_gpu_is_active(dev)) {
5180 i915_gpu_idle(dev);
5181 active++;
5182 }
5183
5184 spin_lock(&shrink_list_lock);
5185 mutex_unlock(&dev->struct_mutex);
5186 }
5187
5188 if (active)
5189 goto rescan; 4053 goto rescan;
5190 } 4054 }
5191 4055 mutex_unlock(&dev->struct_mutex);
5192 spin_unlock(&shrink_list_lock); 4056 return cnt / 100 * sysctl_vfs_cache_pressure;
5193
5194 if (would_deadlock)
5195 return -1;
5196 else if (cnt > 0)
5197 return (cnt / 100) * sysctl_vfs_cache_pressure;
5198 else
5199 return 0;
5200}
5201
5202static struct shrinker shrinker = {
5203 .shrink = i915_gem_shrink,
5204 .seeks = DEFAULT_SEEKS,
5205};
5206
5207__init void
5208i915_gem_shrinker_init(void)
5209{
5210 register_shrinker(&shrinker);
5211}
5212
5213__exit void
5214i915_gem_shrinker_exit(void)
5215{
5216 unregister_shrinker(&shrinker);
5217} 4057}
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
index 48644b840a8d..29d014c48ca2 100644
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -152,13 +152,12 @@ i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end,
152} 152}
153 153
154void 154void
155i915_gem_dump_object(struct drm_gem_object *obj, int len, 155i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
156 const char *where, uint32_t mark) 156 const char *where, uint32_t mark)
157{ 157{
158 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
159 int page; 158 int page;
160 159
161 DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset); 160 DRM_INFO("%s: object at offset %08x\n", where, obj->gtt_offset);
162 for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) { 161 for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) {
163 int page_len, chunk, chunk_len; 162 int page_len, chunk, chunk_len;
164 163
@@ -170,9 +169,9 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len,
170 chunk_len = page_len - chunk; 169 chunk_len = page_len - chunk;
171 if (chunk_len > 128) 170 if (chunk_len > 128)
172 chunk_len = 128; 171 chunk_len = 128;
173 i915_gem_dump_page(obj_priv->pages[page], 172 i915_gem_dump_page(obj->pages[page],
174 chunk, chunk + chunk_len, 173 chunk, chunk + chunk_len,
175 obj_priv->gtt_offset + 174 obj->gtt_offset +
176 page * PAGE_SIZE, 175 page * PAGE_SIZE,
177 mark); 176 mark);
178 } 177 }
@@ -182,21 +181,19 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len,
182 181
183#if WATCH_COHERENCY 182#if WATCH_COHERENCY
184void 183void
185i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle) 184i915_gem_object_check_coherency(struct drm_i915_gem_object *obj, int handle)
186{ 185{
187 struct drm_device *dev = obj->dev; 186 struct drm_device *dev = obj->base.dev;
188 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
189 int page; 187 int page;
190 uint32_t *gtt_mapping; 188 uint32_t *gtt_mapping;
191 uint32_t *backing_map = NULL; 189 uint32_t *backing_map = NULL;
192 int bad_count = 0; 190 int bad_count = 0;
193 191
194 DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n", 192 DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n",
195 __func__, obj, obj_priv->gtt_offset, handle, 193 __func__, obj, obj->gtt_offset, handle,
196 obj->size / 1024); 194 obj->size / 1024);
197 195
198 gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset, 196 gtt_mapping = ioremap(dev->agp->base + obj->gtt_offset, obj->base.size);
199 obj->size);
200 if (gtt_mapping == NULL) { 197 if (gtt_mapping == NULL) {
201 DRM_ERROR("failed to map GTT space\n"); 198 DRM_ERROR("failed to map GTT space\n");
202 return; 199 return;
@@ -205,7 +202,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
205 for (page = 0; page < obj->size / PAGE_SIZE; page++) { 202 for (page = 0; page < obj->size / PAGE_SIZE; page++) {
206 int i; 203 int i;
207 204
208 backing_map = kmap_atomic(obj_priv->pages[page], KM_USER0); 205 backing_map = kmap_atomic(obj->pages[page], KM_USER0);
209 206
210 if (backing_map == NULL) { 207 if (backing_map == NULL) {
211 DRM_ERROR("failed to map backing page\n"); 208 DRM_ERROR("failed to map backing page\n");
@@ -220,7 +217,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
220 if (cpuval != gttval) { 217 if (cpuval != gttval) {
221 DRM_INFO("incoherent CPU vs GPU at 0x%08x: " 218 DRM_INFO("incoherent CPU vs GPU at 0x%08x: "
222 "0x%08x vs 0x%08x\n", 219 "0x%08x vs 0x%08x\n",
223 (int)(obj_priv->gtt_offset + 220 (int)(obj->gtt_offset +
224 page * PAGE_SIZE + i * 4), 221 page * PAGE_SIZE + i * 4),
225 cpuval, gttval); 222 cpuval, gttval);
226 if (bad_count++ >= 8) { 223 if (bad_count++ >= 8) {
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index d8ae7d1d0cc6..78b8cf90c922 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -32,28 +32,36 @@
32#include "i915_drm.h" 32#include "i915_drm.h"
33 33
34static bool 34static bool
35mark_free(struct drm_i915_gem_object *obj_priv, 35mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
36 struct list_head *unwind)
37{ 36{
38 list_add(&obj_priv->evict_list, unwind); 37 list_add(&obj->exec_list, unwind);
39 drm_gem_object_reference(&obj_priv->base); 38 drm_gem_object_reference(&obj->base);
40 return drm_mm_scan_add_block(obj_priv->gtt_space); 39 return drm_mm_scan_add_block(obj->gtt_space);
41} 40}
42 41
43int 42int
44i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment) 43i915_gem_evict_something(struct drm_device *dev, int min_size,
44 unsigned alignment, bool mappable)
45{ 45{
46 drm_i915_private_t *dev_priv = dev->dev_private; 46 drm_i915_private_t *dev_priv = dev->dev_private;
47 struct list_head eviction_list, unwind_list; 47 struct list_head eviction_list, unwind_list;
48 struct drm_i915_gem_object *obj_priv; 48 struct drm_i915_gem_object *obj;
49 int ret = 0; 49 int ret = 0;
50 50
51 i915_gem_retire_requests(dev); 51 i915_gem_retire_requests(dev);
52 52
53 /* Re-check for free space after retiring requests */ 53 /* Re-check for free space after retiring requests */
54 if (drm_mm_search_free(&dev_priv->mm.gtt_space, 54 if (mappable) {
55 min_size, alignment, 0)) 55 if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
56 return 0; 56 min_size, alignment, 0,
57 dev_priv->mm.gtt_mappable_end,
58 0))
59 return 0;
60 } else {
61 if (drm_mm_search_free(&dev_priv->mm.gtt_space,
62 min_size, alignment, 0))
63 return 0;
64 }
57 65
58 /* 66 /*
59 * The goal is to evict objects and amalgamate space in LRU order. 67 * The goal is to evict objects and amalgamate space in LRU order.
@@ -79,45 +87,50 @@ i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignmen
79 */ 87 */
80 88
81 INIT_LIST_HEAD(&unwind_list); 89 INIT_LIST_HEAD(&unwind_list);
82 drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment); 90 if (mappable)
91 drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size,
92 alignment, 0,
93 dev_priv->mm.gtt_mappable_end);
94 else
95 drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
83 96
84 /* First see if there is a large enough contiguous idle region... */ 97 /* First see if there is a large enough contiguous idle region... */
85 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) { 98 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
86 if (mark_free(obj_priv, &unwind_list)) 99 if (mark_free(obj, &unwind_list))
87 goto found; 100 goto found;
88 } 101 }
89 102
90 /* Now merge in the soon-to-be-expired objects... */ 103 /* Now merge in the soon-to-be-expired objects... */
91 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 104 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
92 /* Does the object require an outstanding flush? */ 105 /* Does the object require an outstanding flush? */
93 if (obj_priv->base.write_domain || obj_priv->pin_count) 106 if (obj->base.write_domain || obj->pin_count)
94 continue; 107 continue;
95 108
96 if (mark_free(obj_priv, &unwind_list)) 109 if (mark_free(obj, &unwind_list))
97 goto found; 110 goto found;
98 } 111 }
99 112
100 /* Finally add anything with a pending flush (in order of retirement) */ 113 /* Finally add anything with a pending flush (in order of retirement) */
101 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) { 114 list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
102 if (obj_priv->pin_count) 115 if (obj->pin_count)
103 continue; 116 continue;
104 117
105 if (mark_free(obj_priv, &unwind_list)) 118 if (mark_free(obj, &unwind_list))
106 goto found; 119 goto found;
107 } 120 }
108 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 121 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
109 if (! obj_priv->base.write_domain || obj_priv->pin_count) 122 if (! obj->base.write_domain || obj->pin_count)
110 continue; 123 continue;
111 124
112 if (mark_free(obj_priv, &unwind_list)) 125 if (mark_free(obj, &unwind_list))
113 goto found; 126 goto found;
114 } 127 }
115 128
116 /* Nothing found, clean up and bail out! */ 129 /* Nothing found, clean up and bail out! */
117 list_for_each_entry(obj_priv, &unwind_list, evict_list) { 130 list_for_each_entry(obj, &unwind_list, exec_list) {
118 ret = drm_mm_scan_remove_block(obj_priv->gtt_space); 131 ret = drm_mm_scan_remove_block(obj->gtt_space);
119 BUG_ON(ret); 132 BUG_ON(ret);
120 drm_gem_object_unreference(&obj_priv->base); 133 drm_gem_object_unreference(&obj->base);
121 } 134 }
122 135
123 /* We expect the caller to unpin, evict all and try again, or give up. 136 /* We expect the caller to unpin, evict all and try again, or give up.
@@ -131,33 +144,33 @@ found:
131 * temporary list. */ 144 * temporary list. */
132 INIT_LIST_HEAD(&eviction_list); 145 INIT_LIST_HEAD(&eviction_list);
133 while (!list_empty(&unwind_list)) { 146 while (!list_empty(&unwind_list)) {
134 obj_priv = list_first_entry(&unwind_list, 147 obj = list_first_entry(&unwind_list,
135 struct drm_i915_gem_object, 148 struct drm_i915_gem_object,
136 evict_list); 149 exec_list);
137 if (drm_mm_scan_remove_block(obj_priv->gtt_space)) { 150 if (drm_mm_scan_remove_block(obj->gtt_space)) {
138 list_move(&obj_priv->evict_list, &eviction_list); 151 list_move(&obj->exec_list, &eviction_list);
139 continue; 152 continue;
140 } 153 }
141 list_del(&obj_priv->evict_list); 154 list_del_init(&obj->exec_list);
142 drm_gem_object_unreference(&obj_priv->base); 155 drm_gem_object_unreference(&obj->base);
143 } 156 }
144 157
145 /* Unbinding will emit any required flushes */ 158 /* Unbinding will emit any required flushes */
146 while (!list_empty(&eviction_list)) { 159 while (!list_empty(&eviction_list)) {
147 obj_priv = list_first_entry(&eviction_list, 160 obj = list_first_entry(&eviction_list,
148 struct drm_i915_gem_object, 161 struct drm_i915_gem_object,
149 evict_list); 162 exec_list);
150 if (ret == 0) 163 if (ret == 0)
151 ret = i915_gem_object_unbind(&obj_priv->base); 164 ret = i915_gem_object_unbind(obj);
152 list_del(&obj_priv->evict_list); 165 list_del_init(&obj->exec_list);
153 drm_gem_object_unreference(&obj_priv->base); 166 drm_gem_object_unreference(&obj->base);
154 } 167 }
155 168
156 return ret; 169 return ret;
157} 170}
158 171
159int 172int
160i915_gem_evict_everything(struct drm_device *dev) 173i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
161{ 174{
162 drm_i915_private_t *dev_priv = dev->dev_private; 175 drm_i915_private_t *dev_priv = dev->dev_private;
163 int ret; 176 int ret;
@@ -176,36 +189,22 @@ i915_gem_evict_everything(struct drm_device *dev)
176 189
177 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 190 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
178 191
179 ret = i915_gem_evict_inactive(dev); 192 return i915_gem_evict_inactive(dev, purgeable_only);
180 if (ret)
181 return ret;
182
183 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
184 list_empty(&dev_priv->mm.flushing_list) &&
185 list_empty(&dev_priv->mm.active_list));
186 BUG_ON(!lists_empty);
187
188 return 0;
189} 193}
190 194
191/** Unbinds all inactive objects. */ 195/** Unbinds all inactive objects. */
192int 196int
193i915_gem_evict_inactive(struct drm_device *dev) 197i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
194{ 198{
195 drm_i915_private_t *dev_priv = dev->dev_private; 199 drm_i915_private_t *dev_priv = dev->dev_private;
196 200 struct drm_i915_gem_object *obj, *next;
197 while (!list_empty(&dev_priv->mm.inactive_list)) { 201
198 struct drm_gem_object *obj; 202 list_for_each_entry_safe(obj, next,
199 int ret; 203 &dev_priv->mm.inactive_list, mm_list) {
200 204 if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
201 obj = &list_first_entry(&dev_priv->mm.inactive_list, 205 int ret = i915_gem_object_unbind(obj);
202 struct drm_i915_gem_object, 206 if (ret)
203 mm_list)->base; 207 return ret;
204
205 ret = i915_gem_object_unbind(obj);
206 if (ret != 0) {
207 DRM_ERROR("Error unbinding object: %d\n", ret);
208 return ret;
209 } 208 }
210 } 209 }
211 210
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
new file mode 100644
index 000000000000..fda0dc858a1f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -0,0 +1,1310 @@
1/*
2 * Copyright © 2008,2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Chris Wilson <chris@chris-wilson.co.uk>
26 *
27 */
28
29#include "drmP.h"
30#include "drm.h"
31#include "i915_drm.h"
32#include "i915_drv.h"
33#include "i915_trace.h"
34#include "intel_drv.h"
35
36struct change_domains {
37 uint32_t invalidate_domains;
38 uint32_t flush_domains;
39 uint32_t flush_rings;
40};
41
42/*
43 * Set the next domain for the specified object. This
44 * may not actually perform the necessary flushing/invaliding though,
45 * as that may want to be batched with other set_domain operations
46 *
47 * This is (we hope) the only really tricky part of gem. The goal
48 * is fairly simple -- track which caches hold bits of the object
49 * and make sure they remain coherent. A few concrete examples may
50 * help to explain how it works. For shorthand, we use the notation
51 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
52 * a pair of read and write domain masks.
53 *
54 * Case 1: the batch buffer
55 *
56 * 1. Allocated
57 * 2. Written by CPU
58 * 3. Mapped to GTT
59 * 4. Read by GPU
60 * 5. Unmapped from GTT
61 * 6. Freed
62 *
63 * Let's take these a step at a time
64 *
65 * 1. Allocated
66 * Pages allocated from the kernel may still have
67 * cache contents, so we set them to (CPU, CPU) always.
68 * 2. Written by CPU (using pwrite)
69 * The pwrite function calls set_domain (CPU, CPU) and
70 * this function does nothing (as nothing changes)
71 * 3. Mapped by GTT
72 * This function asserts that the object is not
73 * currently in any GPU-based read or write domains
74 * 4. Read by GPU
75 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
76 * As write_domain is zero, this function adds in the
77 * current read domains (CPU+COMMAND, 0).
78 * flush_domains is set to CPU.
79 * invalidate_domains is set to COMMAND
80 * clflush is run to get data out of the CPU caches
81 * then i915_dev_set_domain calls i915_gem_flush to
82 * emit an MI_FLUSH and drm_agp_chipset_flush
83 * 5. Unmapped from GTT
84 * i915_gem_object_unbind calls set_domain (CPU, CPU)
85 * flush_domains and invalidate_domains end up both zero
86 * so no flushing/invalidating happens
87 * 6. Freed
88 * yay, done
89 *
90 * Case 2: The shared render buffer
91 *
92 * 1. Allocated
93 * 2. Mapped to GTT
94 * 3. Read/written by GPU
95 * 4. set_domain to (CPU,CPU)
96 * 5. Read/written by CPU
97 * 6. Read/written by GPU
98 *
99 * 1. Allocated
100 * Same as last example, (CPU, CPU)
101 * 2. Mapped to GTT
102 * Nothing changes (assertions find that it is not in the GPU)
103 * 3. Read/written by GPU
104 * execbuffer calls set_domain (RENDER, RENDER)
105 * flush_domains gets CPU
106 * invalidate_domains gets GPU
107 * clflush (obj)
108 * MI_FLUSH and drm_agp_chipset_flush
109 * 4. set_domain (CPU, CPU)
110 * flush_domains gets GPU
111 * invalidate_domains gets CPU
112 * wait_rendering (obj) to make sure all drawing is complete.
113 * This will include an MI_FLUSH to get the data from GPU
114 * to memory
115 * clflush (obj) to invalidate the CPU cache
116 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
117 * 5. Read/written by CPU
118 * cache lines are loaded and dirtied
119 * 6. Read written by GPU
120 * Same as last GPU access
121 *
122 * Case 3: The constant buffer
123 *
124 * 1. Allocated
125 * 2. Written by CPU
126 * 3. Read by GPU
127 * 4. Updated (written) by CPU again
128 * 5. Read by GPU
129 *
130 * 1. Allocated
131 * (CPU, CPU)
132 * 2. Written by CPU
133 * (CPU, CPU)
134 * 3. Read by GPU
135 * (CPU+RENDER, 0)
136 * flush_domains = CPU
137 * invalidate_domains = RENDER
138 * clflush (obj)
139 * MI_FLUSH
140 * drm_agp_chipset_flush
141 * 4. Updated (written) by CPU again
142 * (CPU, CPU)
143 * flush_domains = 0 (no previous write domain)
144 * invalidate_domains = 0 (no new read domains)
145 * 5. Read by GPU
146 * (CPU+RENDER, 0)
147 * flush_domains = CPU
148 * invalidate_domains = RENDER
149 * clflush (obj)
150 * MI_FLUSH
151 * drm_agp_chipset_flush
152 */
153static void
154i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
155 struct intel_ring_buffer *ring,
156 struct change_domains *cd)
157{
158 uint32_t invalidate_domains = 0, flush_domains = 0;
159
160 /*
161 * If the object isn't moving to a new write domain,
162 * let the object stay in multiple read domains
163 */
164 if (obj->base.pending_write_domain == 0)
165 obj->base.pending_read_domains |= obj->base.read_domains;
166
167 /*
168 * Flush the current write domain if
169 * the new read domains don't match. Invalidate
170 * any read domains which differ from the old
171 * write domain
172 */
173 if (obj->base.write_domain &&
174 (((obj->base.write_domain != obj->base.pending_read_domains ||
175 obj->ring != ring)) ||
176 (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
177 flush_domains |= obj->base.write_domain;
178 invalidate_domains |=
179 obj->base.pending_read_domains & ~obj->base.write_domain;
180 }
181 /*
182 * Invalidate any read caches which may have
183 * stale data. That is, any new read domains.
184 */
185 invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
186 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
187 i915_gem_clflush_object(obj);
188
189 /* blow away mappings if mapped through GTT */
190 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
191 i915_gem_release_mmap(obj);
192
193 /* The actual obj->write_domain will be updated with
194 * pending_write_domain after we emit the accumulated flush for all
195 * of our domain changes in execbuffers (which clears objects'
196 * write_domains). So if we have a current write domain that we
197 * aren't changing, set pending_write_domain to that.
198 */
199 if (flush_domains == 0 && obj->base.pending_write_domain == 0)
200 obj->base.pending_write_domain = obj->base.write_domain;
201
202 cd->invalidate_domains |= invalidate_domains;
203 cd->flush_domains |= flush_domains;
204 if (flush_domains & I915_GEM_GPU_DOMAINS)
205 cd->flush_rings |= obj->ring->id;
206 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
207 cd->flush_rings |= ring->id;
208}
209
210struct eb_objects {
211 int and;
212 struct hlist_head buckets[0];
213};
214
215static struct eb_objects *
216eb_create(int size)
217{
218 struct eb_objects *eb;
219 int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
220 while (count > size)
221 count >>= 1;
222 eb = kzalloc(count*sizeof(struct hlist_head) +
223 sizeof(struct eb_objects),
224 GFP_KERNEL);
225 if (eb == NULL)
226 return eb;
227
228 eb->and = count - 1;
229 return eb;
230}
231
232static void
233eb_reset(struct eb_objects *eb)
234{
235 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
236}
237
238static void
239eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
240{
241 hlist_add_head(&obj->exec_node,
242 &eb->buckets[obj->exec_handle & eb->and]);
243}
244
245static struct drm_i915_gem_object *
246eb_get_object(struct eb_objects *eb, unsigned long handle)
247{
248 struct hlist_head *head;
249 struct hlist_node *node;
250 struct drm_i915_gem_object *obj;
251
252 head = &eb->buckets[handle & eb->and];
253 hlist_for_each(node, head) {
254 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
255 if (obj->exec_handle == handle)
256 return obj;
257 }
258
259 return NULL;
260}
261
262static void
263eb_destroy(struct eb_objects *eb)
264{
265 kfree(eb);
266}
267
268static int
269i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
270 struct eb_objects *eb,
271 struct drm_i915_gem_exec_object2 *entry,
272 struct drm_i915_gem_relocation_entry *reloc)
273{
274 struct drm_device *dev = obj->base.dev;
275 struct drm_gem_object *target_obj;
276 uint32_t target_offset;
277 int ret = -EINVAL;
278
279 /* we've already hold a reference to all valid objects */
280 target_obj = &eb_get_object(eb, reloc->target_handle)->base;
281 if (unlikely(target_obj == NULL))
282 return -ENOENT;
283
284 target_offset = to_intel_bo(target_obj)->gtt_offset;
285
286#if WATCH_RELOC
287 DRM_INFO("%s: obj %p offset %08x target %d "
288 "read %08x write %08x gtt %08x "
289 "presumed %08x delta %08x\n",
290 __func__,
291 obj,
292 (int) reloc->offset,
293 (int) reloc->target_handle,
294 (int) reloc->read_domains,
295 (int) reloc->write_domain,
296 (int) target_offset,
297 (int) reloc->presumed_offset,
298 reloc->delta);
299#endif
300
301 /* The target buffer should have appeared before us in the
302 * exec_object list, so it should have a GTT space bound by now.
303 */
304 if (unlikely(target_offset == 0)) {
305 DRM_ERROR("No GTT space found for object %d\n",
306 reloc->target_handle);
307 return ret;
308 }
309
310 /* Validate that the target is in a valid r/w GPU domain */
311 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
312 DRM_ERROR("reloc with multiple write domains: "
313 "obj %p target %d offset %d "
314 "read %08x write %08x",
315 obj, reloc->target_handle,
316 (int) reloc->offset,
317 reloc->read_domains,
318 reloc->write_domain);
319 return ret;
320 }
321 if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
322 DRM_ERROR("reloc with read/write CPU domains: "
323 "obj %p target %d offset %d "
324 "read %08x write %08x",
325 obj, reloc->target_handle,
326 (int) reloc->offset,
327 reloc->read_domains,
328 reloc->write_domain);
329 return ret;
330 }
331 if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
332 reloc->write_domain != target_obj->pending_write_domain)) {
333 DRM_ERROR("Write domain conflict: "
334 "obj %p target %d offset %d "
335 "new %08x old %08x\n",
336 obj, reloc->target_handle,
337 (int) reloc->offset,
338 reloc->write_domain,
339 target_obj->pending_write_domain);
340 return ret;
341 }
342
343 target_obj->pending_read_domains |= reloc->read_domains;
344 target_obj->pending_write_domain |= reloc->write_domain;
345
346 /* If the relocation already has the right value in it, no
347 * more work needs to be done.
348 */
349 if (target_offset == reloc->presumed_offset)
350 return 0;
351
352 /* Check that the relocation address is valid... */
353 if (unlikely(reloc->offset > obj->base.size - 4)) {
354 DRM_ERROR("Relocation beyond object bounds: "
355 "obj %p target %d offset %d size %d.\n",
356 obj, reloc->target_handle,
357 (int) reloc->offset,
358 (int) obj->base.size);
359 return ret;
360 }
361 if (unlikely(reloc->offset & 3)) {
362 DRM_ERROR("Relocation not 4-byte aligned: "
363 "obj %p target %d offset %d.\n",
364 obj, reloc->target_handle,
365 (int) reloc->offset);
366 return ret;
367 }
368
369 /* and points to somewhere within the target object. */
370 if (unlikely(reloc->delta >= target_obj->size)) {
371 DRM_ERROR("Relocation beyond target object bounds: "
372 "obj %p target %d delta %d size %d.\n",
373 obj, reloc->target_handle,
374 (int) reloc->delta,
375 (int) target_obj->size);
376 return ret;
377 }
378
379 reloc->delta += target_offset;
380 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
381 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
382 char *vaddr;
383
384 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
385 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
386 kunmap_atomic(vaddr);
387 } else {
388 struct drm_i915_private *dev_priv = dev->dev_private;
389 uint32_t __iomem *reloc_entry;
390 void __iomem *reloc_page;
391
392 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
393 if (ret)
394 return ret;
395
396 /* Map the page containing the relocation we're going to perform. */
397 reloc->offset += obj->gtt_offset;
398 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
399 reloc->offset & PAGE_MASK);
400 reloc_entry = (uint32_t __iomem *)
401 (reloc_page + (reloc->offset & ~PAGE_MASK));
402 iowrite32(reloc->delta, reloc_entry);
403 io_mapping_unmap_atomic(reloc_page);
404 }
405
406 /* and update the user's relocation entry */
407 reloc->presumed_offset = target_offset;
408
409 return 0;
410}
411
412static int
413i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
414 struct eb_objects *eb,
415 struct drm_i915_gem_exec_object2 *entry)
416{
417 struct drm_i915_gem_relocation_entry __user *user_relocs;
418 int i, ret;
419
420 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
421 for (i = 0; i < entry->relocation_count; i++) {
422 struct drm_i915_gem_relocation_entry reloc;
423
424 if (__copy_from_user_inatomic(&reloc,
425 user_relocs+i,
426 sizeof(reloc)))
427 return -EFAULT;
428
429 ret = i915_gem_execbuffer_relocate_entry(obj, eb, entry, &reloc);
430 if (ret)
431 return ret;
432
433 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
434 &reloc.presumed_offset,
435 sizeof(reloc.presumed_offset)))
436 return -EFAULT;
437 }
438
439 return 0;
440}
441
442static int
443i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
444 struct eb_objects *eb,
445 struct drm_i915_gem_exec_object2 *entry,
446 struct drm_i915_gem_relocation_entry *relocs)
447{
448 int i, ret;
449
450 for (i = 0; i < entry->relocation_count; i++) {
451 ret = i915_gem_execbuffer_relocate_entry(obj, eb, entry, &relocs[i]);
452 if (ret)
453 return ret;
454 }
455
456 return 0;
457}
458
459static int
460i915_gem_execbuffer_relocate(struct drm_device *dev,
461 struct eb_objects *eb,
462 struct list_head *objects,
463 struct drm_i915_gem_exec_object2 *exec)
464{
465 struct drm_i915_gem_object *obj;
466 int ret;
467
468 list_for_each_entry(obj, objects, exec_list) {
469 obj->base.pending_read_domains = 0;
470 obj->base.pending_write_domain = 0;
471 ret = i915_gem_execbuffer_relocate_object(obj, eb, exec++);
472 if (ret)
473 return ret;
474 }
475
476 return 0;
477}
478
479static int
480i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
481 struct drm_file *file,
482 struct list_head *objects,
483 struct drm_i915_gem_exec_object2 *exec)
484{
485 struct drm_i915_gem_object *obj;
486 struct drm_i915_gem_exec_object2 *entry;
487 int ret, retry;
488 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
489
490 /* Attempt to pin all of the buffers into the GTT.
491 * This is done in 3 phases:
492 *
493 * 1a. Unbind all objects that do not match the GTT constraints for
494 * the execbuffer (fenceable, mappable, alignment etc).
495 * 1b. Increment pin count for already bound objects.
496 * 2. Bind new objects.
497 * 3. Decrement pin count.
498 *
499 * This avoid unnecessary unbinding of later objects in order to makr
500 * room for the earlier objects *unless* we need to defragment.
501 */
502 retry = 0;
503 do {
504 ret = 0;
505
506 /* Unbind any ill-fitting objects or pin. */
507 entry = exec;
508 list_for_each_entry(obj, objects, exec_list) {
509 bool need_fence, need_mappable;
510
511 if (!obj->gtt_space) {
512 entry++;
513 continue;
514 }
515
516 need_fence =
517 has_fenced_gpu_access &&
518 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
519 obj->tiling_mode != I915_TILING_NONE;
520 need_mappable =
521 entry->relocation_count ? true : need_fence;
522
523 if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
524 (need_mappable && !obj->map_and_fenceable))
525 ret = i915_gem_object_unbind(obj);
526 else
527 ret = i915_gem_object_pin(obj,
528 entry->alignment,
529 need_mappable);
530 if (ret)
531 goto err;
532
533 entry++;
534 }
535
536 /* Bind fresh objects */
537 entry = exec;
538 list_for_each_entry(obj, objects, exec_list) {
539 bool need_fence;
540
541 need_fence =
542 has_fenced_gpu_access &&
543 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
544 obj->tiling_mode != I915_TILING_NONE;
545
546 if (!obj->gtt_space) {
547 bool need_mappable =
548 entry->relocation_count ? true : need_fence;
549
550 ret = i915_gem_object_pin(obj,
551 entry->alignment,
552 need_mappable);
553 if (ret)
554 break;
555 }
556
557 if (has_fenced_gpu_access) {
558 if (need_fence) {
559 ret = i915_gem_object_get_fence(obj, ring, 1);
560 if (ret)
561 break;
562 } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
563 obj->tiling_mode == I915_TILING_NONE) {
564 /* XXX pipelined! */
565 ret = i915_gem_object_put_fence(obj);
566 if (ret)
567 break;
568 }
569 obj->pending_fenced_gpu_access = need_fence;
570 }
571
572 entry->offset = obj->gtt_offset;
573 entry++;
574 }
575
576 /* Decrement pin count for bound objects */
577 list_for_each_entry(obj, objects, exec_list) {
578 if (obj->gtt_space)
579 i915_gem_object_unpin(obj);
580 }
581
582 if (ret != -ENOSPC || retry > 1)
583 return ret;
584
585 /* First attempt, just clear anything that is purgeable.
586 * Second attempt, clear the entire GTT.
587 */
588 ret = i915_gem_evict_everything(ring->dev, retry == 0);
589 if (ret)
590 return ret;
591
592 retry++;
593 } while (1);
594
595err:
596 obj = list_entry(obj->exec_list.prev,
597 struct drm_i915_gem_object,
598 exec_list);
599 while (objects != &obj->exec_list) {
600 if (obj->gtt_space)
601 i915_gem_object_unpin(obj);
602
603 obj = list_entry(obj->exec_list.prev,
604 struct drm_i915_gem_object,
605 exec_list);
606 }
607
608 return ret;
609}
610
611static int
612i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
613 struct drm_file *file,
614 struct intel_ring_buffer *ring,
615 struct list_head *objects,
616 struct eb_objects *eb,
617 struct drm_i915_gem_exec_object2 *exec,
618 int count)
619{
620 struct drm_i915_gem_relocation_entry *reloc;
621 struct drm_i915_gem_object *obj;
622 int i, total, ret;
623
624 /* We may process another execbuffer during the unlock... */
625 while (list_empty(objects)) {
626 obj = list_first_entry(objects,
627 struct drm_i915_gem_object,
628 exec_list);
629 list_del_init(&obj->exec_list);
630 drm_gem_object_unreference(&obj->base);
631 }
632
633 mutex_unlock(&dev->struct_mutex);
634
635 total = 0;
636 for (i = 0; i < count; i++)
637 total += exec[i].relocation_count;
638
639 reloc = drm_malloc_ab(total, sizeof(*reloc));
640 if (reloc == NULL) {
641 mutex_lock(&dev->struct_mutex);
642 return -ENOMEM;
643 }
644
645 total = 0;
646 for (i = 0; i < count; i++) {
647 struct drm_i915_gem_relocation_entry __user *user_relocs;
648
649 user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
650
651 if (copy_from_user(reloc+total, user_relocs,
652 exec[i].relocation_count * sizeof(*reloc))) {
653 ret = -EFAULT;
654 mutex_lock(&dev->struct_mutex);
655 goto err;
656 }
657
658 total += exec[i].relocation_count;
659 }
660
661 ret = i915_mutex_lock_interruptible(dev);
662 if (ret) {
663 mutex_lock(&dev->struct_mutex);
664 goto err;
665 }
666
667 /* reacquire the objects */
668 INIT_LIST_HEAD(objects);
669 eb_reset(eb);
670 for (i = 0; i < count; i++) {
671 struct drm_i915_gem_object *obj;
672
673 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
674 exec[i].handle));
675 if (obj == NULL) {
676 DRM_ERROR("Invalid object handle %d at index %d\n",
677 exec[i].handle, i);
678 ret = -ENOENT;
679 goto err;
680 }
681
682 list_add_tail(&obj->exec_list, objects);
683 obj->exec_handle = exec[i].handle;
684 eb_add_object(eb, obj);
685 }
686
687 ret = i915_gem_execbuffer_reserve(ring, file, objects, exec);
688 if (ret)
689 goto err;
690
691 total = 0;
692 list_for_each_entry(obj, objects, exec_list) {
693 obj->base.pending_read_domains = 0;
694 obj->base.pending_write_domain = 0;
695 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
696 exec,
697 reloc + total);
698 if (ret)
699 goto err;
700
701 total += exec->relocation_count;
702 exec++;
703 }
704
705 /* Leave the user relocations as are, this is the painfully slow path,
706 * and we want to avoid the complication of dropping the lock whilst
707 * having buffers reserved in the aperture and so causing spurious
708 * ENOSPC for random operations.
709 */
710
711err:
712 drm_free_large(reloc);
713 return ret;
714}
715
716static void
717i915_gem_execbuffer_flush(struct drm_device *dev,
718 uint32_t invalidate_domains,
719 uint32_t flush_domains,
720 uint32_t flush_rings)
721{
722 drm_i915_private_t *dev_priv = dev->dev_private;
723 int i;
724
725 if (flush_domains & I915_GEM_DOMAIN_CPU)
726 intel_gtt_chipset_flush();
727
728 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
729 for (i = 0; i < I915_NUM_RINGS; i++)
730 if (flush_rings & (1 << i))
731 i915_gem_flush_ring(dev, &dev_priv->ring[i],
732 invalidate_domains,
733 flush_domains);
734 }
735}
736
737static int
738i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
739 struct intel_ring_buffer *to)
740{
741 struct intel_ring_buffer *from = obj->ring;
742 u32 seqno;
743 int ret, idx;
744
745 if (from == NULL || to == from)
746 return 0;
747
748 if (INTEL_INFO(obj->base.dev)->gen < 6)
749 return i915_gem_object_wait_rendering(obj, true);
750
751 idx = intel_ring_sync_index(from, to);
752
753 seqno = obj->last_rendering_seqno;
754 if (seqno <= from->sync_seqno[idx])
755 return 0;
756
757 if (seqno == from->outstanding_lazy_request) {
758 struct drm_i915_gem_request *request;
759
760 request = kzalloc(sizeof(*request), GFP_KERNEL);
761 if (request == NULL)
762 return -ENOMEM;
763
764 ret = i915_add_request(obj->base.dev, NULL, request, from);
765 if (ret) {
766 kfree(request);
767 return ret;
768 }
769
770 seqno = request->seqno;
771 }
772
773 from->sync_seqno[idx] = seqno;
774 return intel_ring_sync(to, from, seqno - 1);
775}
776
777static int
778i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
779 struct list_head *objects)
780{
781 struct drm_i915_gem_object *obj;
782 struct change_domains cd;
783 int ret;
784
785 cd.invalidate_domains = 0;
786 cd.flush_domains = 0;
787 cd.flush_rings = 0;
788 list_for_each_entry(obj, objects, exec_list)
789 i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
790
791 if (cd.invalidate_domains | cd.flush_domains) {
792#if WATCH_EXEC
793 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
794 __func__,
795 cd.invalidate_domains,
796 cd.flush_domains);
797#endif
798 i915_gem_execbuffer_flush(ring->dev,
799 cd.invalidate_domains,
800 cd.flush_domains,
801 cd.flush_rings);
802 }
803
804 list_for_each_entry(obj, objects, exec_list) {
805 ret = i915_gem_execbuffer_sync_rings(obj, ring);
806 if (ret)
807 return ret;
808 }
809
810 return 0;
811}
812
813static bool
814i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
815{
816 return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
817}
818
819static int
820validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
821 int count)
822{
823 int i;
824
825 for (i = 0; i < count; i++) {
826 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
827 int length; /* limited by fault_in_pages_readable() */
828
829 /* First check for malicious input causing overflow */
830 if (exec[i].relocation_count >
831 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
832 return -EINVAL;
833
834 length = exec[i].relocation_count *
835 sizeof(struct drm_i915_gem_relocation_entry);
836 if (!access_ok(VERIFY_READ, ptr, length))
837 return -EFAULT;
838
839 /* we may also need to update the presumed offsets */
840 if (!access_ok(VERIFY_WRITE, ptr, length))
841 return -EFAULT;
842
843 if (fault_in_pages_readable(ptr, length))
844 return -EFAULT;
845 }
846
847 return 0;
848}
849
850static int
851i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring,
852 struct list_head *objects)
853{
854 struct drm_i915_gem_object *obj;
855 int flips;
856
857 /* Check for any pending flips. As we only maintain a flip queue depth
858 * of 1, we can simply insert a WAIT for the next display flip prior
859 * to executing the batch and avoid stalling the CPU.
860 */
861 flips = 0;
862 list_for_each_entry(obj, objects, exec_list) {
863 if (obj->base.write_domain)
864 flips |= atomic_read(&obj->pending_flip);
865 }
866 if (flips) {
867 int plane, flip_mask, ret;
868
869 for (plane = 0; flips >> plane; plane++) {
870 if (((flips >> plane) & 1) == 0)
871 continue;
872
873 if (plane)
874 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
875 else
876 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
877
878 ret = intel_ring_begin(ring, 2);
879 if (ret)
880 return ret;
881
882 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
883 intel_ring_emit(ring, MI_NOOP);
884 intel_ring_advance(ring);
885 }
886 }
887
888 return 0;
889}
890
891static void
892i915_gem_execbuffer_move_to_active(struct list_head *objects,
893 struct intel_ring_buffer *ring,
894 u32 seqno)
895{
896 struct drm_i915_gem_object *obj;
897
898 list_for_each_entry(obj, objects, exec_list) {
899 obj->base.read_domains = obj->base.pending_read_domains;
900 obj->base.write_domain = obj->base.pending_write_domain;
901 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
902
903 i915_gem_object_move_to_active(obj, ring, seqno);
904 if (obj->base.write_domain) {
905 obj->dirty = 1;
906 obj->pending_gpu_write = true;
907 list_move_tail(&obj->gpu_write_list,
908 &ring->gpu_write_list);
909 intel_mark_busy(ring->dev, obj);
910 }
911
912 trace_i915_gem_object_change_domain(obj,
913 obj->base.read_domains,
914 obj->base.write_domain);
915 }
916}
917
918static void
919i915_gem_execbuffer_retire_commands(struct drm_device *dev,
920 struct drm_file *file,
921 struct intel_ring_buffer *ring)
922{
923 struct drm_i915_gem_request *request;
924 u32 flush_domains;
925
926 /*
927 * Ensure that the commands in the batch buffer are
928 * finished before the interrupt fires.
929 *
930 * The sampler always gets flushed on i965 (sigh).
931 */
932 flush_domains = 0;
933 if (INTEL_INFO(dev)->gen >= 4)
934 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
935
936 ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
937
938 /* Add a breadcrumb for the completion of the batch buffer */
939 request = kzalloc(sizeof(*request), GFP_KERNEL);
940 if (request == NULL || i915_add_request(dev, file, request, ring)) {
941 i915_gem_next_request_seqno(dev, ring);
942 kfree(request);
943 }
944}
945
946static int
947i915_gem_do_execbuffer(struct drm_device *dev, void *data,
948 struct drm_file *file,
949 struct drm_i915_gem_execbuffer2 *args,
950 struct drm_i915_gem_exec_object2 *exec)
951{
952 drm_i915_private_t *dev_priv = dev->dev_private;
953 struct list_head objects;
954 struct eb_objects *eb;
955 struct drm_i915_gem_object *batch_obj;
956 struct drm_clip_rect *cliprects = NULL;
957 struct intel_ring_buffer *ring;
958 u32 exec_start, exec_len;
959 u32 seqno;
960 int ret, i;
961
962 if (!i915_gem_check_execbuffer(args)) {
963 DRM_ERROR("execbuf with invalid offset/length\n");
964 return -EINVAL;
965 }
966
967 ret = validate_exec_list(exec, args->buffer_count);
968 if (ret)
969 return ret;
970
971#if WATCH_EXEC
972 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
973 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
974#endif
975 switch (args->flags & I915_EXEC_RING_MASK) {
976 case I915_EXEC_DEFAULT:
977 case I915_EXEC_RENDER:
978 ring = &dev_priv->ring[RCS];
979 break;
980 case I915_EXEC_BSD:
981 if (!HAS_BSD(dev)) {
982 DRM_ERROR("execbuf with invalid ring (BSD)\n");
983 return -EINVAL;
984 }
985 ring = &dev_priv->ring[VCS];
986 break;
987 case I915_EXEC_BLT:
988 if (!HAS_BLT(dev)) {
989 DRM_ERROR("execbuf with invalid ring (BLT)\n");
990 return -EINVAL;
991 }
992 ring = &dev_priv->ring[BCS];
993 break;
994 default:
995 DRM_ERROR("execbuf with unknown ring: %d\n",
996 (int)(args->flags & I915_EXEC_RING_MASK));
997 return -EINVAL;
998 }
999
1000 if (args->buffer_count < 1) {
1001 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1002 return -EINVAL;
1003 }
1004
1005 if (args->num_cliprects != 0) {
1006 if (ring != &dev_priv->ring[RCS]) {
1007 DRM_ERROR("clip rectangles are only valid with the render ring\n");
1008 return -EINVAL;
1009 }
1010
1011 cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
1012 GFP_KERNEL);
1013 if (cliprects == NULL) {
1014 ret = -ENOMEM;
1015 goto pre_mutex_err;
1016 }
1017
1018 if (copy_from_user(cliprects,
1019 (struct drm_clip_rect __user *)(uintptr_t)
1020 args->cliprects_ptr,
1021 sizeof(*cliprects)*args->num_cliprects)) {
1022 ret = -EFAULT;
1023 goto pre_mutex_err;
1024 }
1025 }
1026
1027 ret = i915_mutex_lock_interruptible(dev);
1028 if (ret)
1029 goto pre_mutex_err;
1030
1031 if (dev_priv->mm.suspended) {
1032 mutex_unlock(&dev->struct_mutex);
1033 ret = -EBUSY;
1034 goto pre_mutex_err;
1035 }
1036
1037 eb = eb_create(args->buffer_count);
1038 if (eb == NULL) {
1039 mutex_unlock(&dev->struct_mutex);
1040 ret = -ENOMEM;
1041 goto pre_mutex_err;
1042 }
1043
1044 /* Look up object handles */
1045 INIT_LIST_HEAD(&objects);
1046 for (i = 0; i < args->buffer_count; i++) {
1047 struct drm_i915_gem_object *obj;
1048
1049 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
1050 exec[i].handle));
1051 if (obj == NULL) {
1052 DRM_ERROR("Invalid object handle %d at index %d\n",
1053 exec[i].handle, i);
1054 /* prevent error path from reading uninitialized data */
1055 ret = -ENOENT;
1056 goto err;
1057 }
1058
1059 if (!list_empty(&obj->exec_list)) {
1060 DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
1061 obj, exec[i].handle, i);
1062 ret = -EINVAL;
1063 goto err;
1064 }
1065
1066 list_add_tail(&obj->exec_list, &objects);
1067 obj->exec_handle = exec[i].handle;
1068 eb_add_object(eb, obj);
1069 }
1070
1071 /* Move the objects en-masse into the GTT, evicting if necessary. */
1072 ret = i915_gem_execbuffer_reserve(ring, file, &objects, exec);
1073 if (ret)
1074 goto err;
1075
1076 /* The objects are in their final locations, apply the relocations. */
1077 ret = i915_gem_execbuffer_relocate(dev, eb, &objects, exec);
1078 if (ret) {
1079 if (ret == -EFAULT) {
1080 ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
1081 &objects, eb,
1082 exec,
1083 args->buffer_count);
1084 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1085 }
1086 if (ret)
1087 goto err;
1088 }
1089
1090 /* Set the pending read domains for the batch buffer to COMMAND */
1091 batch_obj = list_entry(objects.prev,
1092 struct drm_i915_gem_object,
1093 exec_list);
1094 if (batch_obj->base.pending_write_domain) {
1095 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
1096 ret = -EINVAL;
1097 goto err;
1098 }
1099 batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1100
1101 ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1102 if (ret)
1103 goto err;
1104
1105 ret = i915_gem_execbuffer_wait_for_flips(ring, &objects);
1106 if (ret)
1107 goto err;
1108
1109 seqno = i915_gem_next_request_seqno(dev, ring);
1110 for (i = 0; i < I915_NUM_RINGS-1; i++) {
1111 if (seqno < ring->sync_seqno[i]) {
1112 /* The GPU can not handle its semaphore value wrapping,
1113 * so every billion or so execbuffers, we need to stall
1114 * the GPU in order to reset the counters.
1115 */
1116 ret = i915_gpu_idle(dev);
1117 if (ret)
1118 goto err;
1119
1120 BUG_ON(ring->sync_seqno[i]);
1121 }
1122 }
1123
1124 exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1125 exec_len = args->batch_len;
1126 if (cliprects) {
1127 for (i = 0; i < args->num_cliprects; i++) {
1128 ret = i915_emit_box(dev, &cliprects[i],
1129 args->DR1, args->DR4);
1130 if (ret)
1131 goto err;
1132
1133 ret = ring->dispatch_execbuffer(ring,
1134 exec_start, exec_len);
1135 if (ret)
1136 goto err;
1137 }
1138 } else {
1139 ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
1140 if (ret)
1141 goto err;
1142 }
1143
1144 i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
1145 i915_gem_execbuffer_retire_commands(dev, file, ring);
1146
1147err:
1148 eb_destroy(eb);
1149 while (!list_empty(&objects)) {
1150 struct drm_i915_gem_object *obj;
1151
1152 obj = list_first_entry(&objects,
1153 struct drm_i915_gem_object,
1154 exec_list);
1155 list_del_init(&obj->exec_list);
1156 drm_gem_object_unreference(&obj->base);
1157 }
1158
1159 mutex_unlock(&dev->struct_mutex);
1160
1161pre_mutex_err:
1162 kfree(cliprects);
1163 return ret;
1164}
1165
1166/*
1167 * Legacy execbuffer just creates an exec2 list from the original exec object
1168 * list array and passes it to the real function.
1169 */
1170int
1171i915_gem_execbuffer(struct drm_device *dev, void *data,
1172 struct drm_file *file)
1173{
1174 struct drm_i915_gem_execbuffer *args = data;
1175 struct drm_i915_gem_execbuffer2 exec2;
1176 struct drm_i915_gem_exec_object *exec_list = NULL;
1177 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1178 int ret, i;
1179
1180#if WATCH_EXEC
1181 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1182 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1183#endif
1184
1185 if (args->buffer_count < 1) {
1186 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1187 return -EINVAL;
1188 }
1189
1190 /* Copy in the exec list from userland */
1191 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1192 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1193 if (exec_list == NULL || exec2_list == NULL) {
1194 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1195 args->buffer_count);
1196 drm_free_large(exec_list);
1197 drm_free_large(exec2_list);
1198 return -ENOMEM;
1199 }
1200 ret = copy_from_user(exec_list,
1201 (struct drm_i915_relocation_entry __user *)
1202 (uintptr_t) args->buffers_ptr,
1203 sizeof(*exec_list) * args->buffer_count);
1204 if (ret != 0) {
1205 DRM_ERROR("copy %d exec entries failed %d\n",
1206 args->buffer_count, ret);
1207 drm_free_large(exec_list);
1208 drm_free_large(exec2_list);
1209 return -EFAULT;
1210 }
1211
1212 for (i = 0; i < args->buffer_count; i++) {
1213 exec2_list[i].handle = exec_list[i].handle;
1214 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1215 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1216 exec2_list[i].alignment = exec_list[i].alignment;
1217 exec2_list[i].offset = exec_list[i].offset;
1218 if (INTEL_INFO(dev)->gen < 4)
1219 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1220 else
1221 exec2_list[i].flags = 0;
1222 }
1223
1224 exec2.buffers_ptr = args->buffers_ptr;
1225 exec2.buffer_count = args->buffer_count;
1226 exec2.batch_start_offset = args->batch_start_offset;
1227 exec2.batch_len = args->batch_len;
1228 exec2.DR1 = args->DR1;
1229 exec2.DR4 = args->DR4;
1230 exec2.num_cliprects = args->num_cliprects;
1231 exec2.cliprects_ptr = args->cliprects_ptr;
1232 exec2.flags = I915_EXEC_RENDER;
1233
1234 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1235 if (!ret) {
1236 /* Copy the new buffer offsets back to the user's exec list. */
1237 for (i = 0; i < args->buffer_count; i++)
1238 exec_list[i].offset = exec2_list[i].offset;
1239 /* ... and back out to userspace */
1240 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1241 (uintptr_t) args->buffers_ptr,
1242 exec_list,
1243 sizeof(*exec_list) * args->buffer_count);
1244 if (ret) {
1245 ret = -EFAULT;
1246 DRM_ERROR("failed to copy %d exec entries "
1247 "back to user (%d)\n",
1248 args->buffer_count, ret);
1249 }
1250 }
1251
1252 drm_free_large(exec_list);
1253 drm_free_large(exec2_list);
1254 return ret;
1255}
1256
1257int
1258i915_gem_execbuffer2(struct drm_device *dev, void *data,
1259 struct drm_file *file)
1260{
1261 struct drm_i915_gem_execbuffer2 *args = data;
1262 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1263 int ret;
1264
1265#if WATCH_EXEC
1266 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1267 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1268#endif
1269
1270 if (args->buffer_count < 1) {
1271 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
1272 return -EINVAL;
1273 }
1274
1275 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1276 if (exec2_list == NULL) {
1277 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1278 args->buffer_count);
1279 return -ENOMEM;
1280 }
1281 ret = copy_from_user(exec2_list,
1282 (struct drm_i915_relocation_entry __user *)
1283 (uintptr_t) args->buffers_ptr,
1284 sizeof(*exec2_list) * args->buffer_count);
1285 if (ret != 0) {
1286 DRM_ERROR("copy %d exec entries failed %d\n",
1287 args->buffer_count, ret);
1288 drm_free_large(exec2_list);
1289 return -EFAULT;
1290 }
1291
1292 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1293 if (!ret) {
1294 /* Copy the new buffer offsets back to the user's exec list. */
1295 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1296 (uintptr_t) args->buffers_ptr,
1297 exec2_list,
1298 sizeof(*exec2_list) * args->buffer_count);
1299 if (ret) {
1300 ret = -EFAULT;
1301 DRM_ERROR("failed to copy %d exec entries "
1302 "back to user (%d)\n",
1303 args->buffer_count, ret);
1304 }
1305 }
1306
1307 drm_free_large(exec2_list);
1308 return ret;
1309}
1310
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
new file mode 100644
index 000000000000..86673e77d7cb
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -0,0 +1,99 @@
1/*
2 * Copyright © 2010 Daniel Vetter
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include "drmP.h"
26#include "drm.h"
27#include "i915_drm.h"
28#include "i915_drv.h"
29#include "i915_trace.h"
30#include "intel_drv.h"
31
32void i915_gem_restore_gtt_mappings(struct drm_device *dev)
33{
34 struct drm_i915_private *dev_priv = dev->dev_private;
35 struct drm_i915_gem_object *obj;
36
37 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
38 i915_gem_clflush_object(obj);
39
40 if (dev_priv->mm.gtt->needs_dmar) {
41 BUG_ON(!obj->sg_list);
42
43 intel_gtt_insert_sg_entries(obj->sg_list,
44 obj->num_sg,
45 obj->gtt_space->start
46 >> PAGE_SHIFT,
47 obj->agp_type);
48 } else
49 intel_gtt_insert_pages(obj->gtt_space->start
50 >> PAGE_SHIFT,
51 obj->base.size >> PAGE_SHIFT,
52 obj->pages,
53 obj->agp_type);
54 }
55
56 intel_gtt_chipset_flush();
57}
58
59int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
60{
61 struct drm_device *dev = obj->base.dev;
62 struct drm_i915_private *dev_priv = dev->dev_private;
63 int ret;
64
65 if (dev_priv->mm.gtt->needs_dmar) {
66 ret = intel_gtt_map_memory(obj->pages,
67 obj->base.size >> PAGE_SHIFT,
68 &obj->sg_list,
69 &obj->num_sg);
70 if (ret != 0)
71 return ret;
72
73 intel_gtt_insert_sg_entries(obj->sg_list,
74 obj->num_sg,
75 obj->gtt_space->start >> PAGE_SHIFT,
76 obj->agp_type);
77 } else
78 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
79 obj->base.size >> PAGE_SHIFT,
80 obj->pages,
81 obj->agp_type);
82
83 return 0;
84}
85
86void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
87{
88 struct drm_device *dev = obj->base.dev;
89 struct drm_i915_private *dev_priv = dev->dev_private;
90
91 if (dev_priv->mm.gtt->needs_dmar) {
92 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
93 obj->sg_list = NULL;
94 obj->num_sg = 0;
95 }
96
97 intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
98 obj->base.size >> PAGE_SHIFT);
99}
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index af352de70be1..22a32b9932c5 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -181,7 +181,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
181} 181}
182 182
183/* Check pitch constriants for all chips & tiling formats */ 183/* Check pitch constriants for all chips & tiling formats */
184bool 184static bool
185i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) 185i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
186{ 186{
187 int tile_width; 187 int tile_width;
@@ -232,32 +232,44 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
232 return true; 232 return true;
233} 233}
234 234
235bool 235/* Is the current GTT allocation valid for the change in tiling? */
236i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode) 236static bool
237i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
237{ 238{
238 struct drm_device *dev = obj->dev; 239 u32 size;
239 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
240
241 if (obj_priv->gtt_space == NULL)
242 return true;
243 240
244 if (tiling_mode == I915_TILING_NONE) 241 if (tiling_mode == I915_TILING_NONE)
245 return true; 242 return true;
246 243
247 if (INTEL_INFO(dev)->gen >= 4) 244 if (INTEL_INFO(obj->base.dev)->gen >= 4)
248 return true; 245 return true;
249 246
250 if (obj_priv->gtt_offset & (obj->size - 1)) 247 if (INTEL_INFO(obj->base.dev)->gen == 3) {
251 return false; 248 if (obj->gtt_offset & ~I915_FENCE_START_MASK)
252
253 if (IS_GEN3(dev)) {
254 if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK)
255 return false; 249 return false;
256 } else { 250 } else {
257 if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK) 251 if (obj->gtt_offset & ~I830_FENCE_START_MASK)
258 return false; 252 return false;
259 } 253 }
260 254
255 /*
256 * Previous chips need to be aligned to the size of the smallest
257 * fence register that can contain the object.
258 */
259 if (INTEL_INFO(obj->base.dev)->gen == 3)
260 size = 1024*1024;
261 else
262 size = 512*1024;
263
264 while (size < obj->base.size)
265 size <<= 1;
266
267 if (obj->gtt_space->size != size)
268 return false;
269
270 if (obj->gtt_offset & (size - 1))
271 return false;
272
261 return true; 273 return true;
262} 274}
263 275
@@ -267,30 +279,29 @@ i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode)
267 */ 279 */
268int 280int
269i915_gem_set_tiling(struct drm_device *dev, void *data, 281i915_gem_set_tiling(struct drm_device *dev, void *data,
270 struct drm_file *file_priv) 282 struct drm_file *file)
271{ 283{
272 struct drm_i915_gem_set_tiling *args = data; 284 struct drm_i915_gem_set_tiling *args = data;
273 drm_i915_private_t *dev_priv = dev->dev_private; 285 drm_i915_private_t *dev_priv = dev->dev_private;
274 struct drm_gem_object *obj; 286 struct drm_i915_gem_object *obj;
275 struct drm_i915_gem_object *obj_priv;
276 int ret; 287 int ret;
277 288
278 ret = i915_gem_check_is_wedged(dev); 289 ret = i915_gem_check_is_wedged(dev);
279 if (ret) 290 if (ret)
280 return ret; 291 return ret;
281 292
282 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 293 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
283 if (obj == NULL) 294 if (obj == NULL)
284 return -ENOENT; 295 return -ENOENT;
285 obj_priv = to_intel_bo(obj);
286 296
287 if (!i915_tiling_ok(dev, args->stride, obj->size, args->tiling_mode)) { 297 if (!i915_tiling_ok(dev,
288 drm_gem_object_unreference_unlocked(obj); 298 args->stride, obj->base.size, args->tiling_mode)) {
299 drm_gem_object_unreference_unlocked(&obj->base);
289 return -EINVAL; 300 return -EINVAL;
290 } 301 }
291 302
292 if (obj_priv->pin_count) { 303 if (obj->pin_count) {
293 drm_gem_object_unreference_unlocked(obj); 304 drm_gem_object_unreference_unlocked(&obj->base);
294 return -EBUSY; 305 return -EBUSY;
295 } 306 }
296 307
@@ -324,34 +335,28 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
324 } 335 }
325 336
326 mutex_lock(&dev->struct_mutex); 337 mutex_lock(&dev->struct_mutex);
327 if (args->tiling_mode != obj_priv->tiling_mode || 338 if (args->tiling_mode != obj->tiling_mode ||
328 args->stride != obj_priv->stride) { 339 args->stride != obj->stride) {
329 /* We need to rebind the object if its current allocation 340 /* We need to rebind the object if its current allocation
330 * no longer meets the alignment restrictions for its new 341 * no longer meets the alignment restrictions for its new
331 * tiling mode. Otherwise we can just leave it alone, but 342 * tiling mode. Otherwise we can just leave it alone, but
332 * need to ensure that any fence register is cleared. 343 * need to ensure that any fence register is cleared.
333 */ 344 */
334 if (!i915_gem_object_fence_offset_ok(obj, args->tiling_mode)) 345 i915_gem_release_mmap(obj);
335 ret = i915_gem_object_unbind(obj);
336 else if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
337 ret = i915_gem_object_put_fence_reg(obj, true);
338 else
339 i915_gem_release_mmap(obj);
340 346
341 if (ret != 0) { 347 obj->map_and_fenceable =
342 args->tiling_mode = obj_priv->tiling_mode; 348 obj->gtt_space == NULL ||
343 args->stride = obj_priv->stride; 349 (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end &&
344 goto err; 350 i915_gem_object_fence_ok(obj, args->tiling_mode));
345 }
346 351
347 obj_priv->tiling_mode = args->tiling_mode; 352 obj->tiling_changed = true;
348 obj_priv->stride = args->stride; 353 obj->tiling_mode = args->tiling_mode;
354 obj->stride = args->stride;
349 } 355 }
350err: 356 drm_gem_object_unreference(&obj->base);
351 drm_gem_object_unreference(obj);
352 mutex_unlock(&dev->struct_mutex); 357 mutex_unlock(&dev->struct_mutex);
353 358
354 return ret; 359 return 0;
355} 360}
356 361
357/** 362/**
@@ -359,22 +364,20 @@ err:
359 */ 364 */
360int 365int
361i915_gem_get_tiling(struct drm_device *dev, void *data, 366i915_gem_get_tiling(struct drm_device *dev, void *data,
362 struct drm_file *file_priv) 367 struct drm_file *file)
363{ 368{
364 struct drm_i915_gem_get_tiling *args = data; 369 struct drm_i915_gem_get_tiling *args = data;
365 drm_i915_private_t *dev_priv = dev->dev_private; 370 drm_i915_private_t *dev_priv = dev->dev_private;
366 struct drm_gem_object *obj; 371 struct drm_i915_gem_object *obj;
367 struct drm_i915_gem_object *obj_priv;
368 372
369 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 373 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
370 if (obj == NULL) 374 if (obj == NULL)
371 return -ENOENT; 375 return -ENOENT;
372 obj_priv = to_intel_bo(obj);
373 376
374 mutex_lock(&dev->struct_mutex); 377 mutex_lock(&dev->struct_mutex);
375 378
376 args->tiling_mode = obj_priv->tiling_mode; 379 args->tiling_mode = obj->tiling_mode;
377 switch (obj_priv->tiling_mode) { 380 switch (obj->tiling_mode) {
378 case I915_TILING_X: 381 case I915_TILING_X:
379 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; 382 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
380 break; 383 break;
@@ -394,7 +397,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
394 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 397 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
395 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 398 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
396 399
397 drm_gem_object_unreference(obj); 400 drm_gem_object_unreference(&obj->base);
398 mutex_unlock(&dev->struct_mutex); 401 mutex_unlock(&dev->struct_mutex);
399 402
400 return 0; 403 return 0;
@@ -424,46 +427,44 @@ i915_gem_swizzle_page(struct page *page)
424} 427}
425 428
426void 429void
427i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj) 430i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
428{ 431{
429 struct drm_device *dev = obj->dev; 432 struct drm_device *dev = obj->base.dev;
430 drm_i915_private_t *dev_priv = dev->dev_private; 433 drm_i915_private_t *dev_priv = dev->dev_private;
431 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 434 int page_count = obj->base.size >> PAGE_SHIFT;
432 int page_count = obj->size >> PAGE_SHIFT;
433 int i; 435 int i;
434 436
435 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) 437 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
436 return; 438 return;
437 439
438 if (obj_priv->bit_17 == NULL) 440 if (obj->bit_17 == NULL)
439 return; 441 return;
440 442
441 for (i = 0; i < page_count; i++) { 443 for (i = 0; i < page_count; i++) {
442 char new_bit_17 = page_to_phys(obj_priv->pages[i]) >> 17; 444 char new_bit_17 = page_to_phys(obj->pages[i]) >> 17;
443 if ((new_bit_17 & 0x1) != 445 if ((new_bit_17 & 0x1) !=
444 (test_bit(i, obj_priv->bit_17) != 0)) { 446 (test_bit(i, obj->bit_17) != 0)) {
445 i915_gem_swizzle_page(obj_priv->pages[i]); 447 i915_gem_swizzle_page(obj->pages[i]);
446 set_page_dirty(obj_priv->pages[i]); 448 set_page_dirty(obj->pages[i]);
447 } 449 }
448 } 450 }
449} 451}
450 452
451void 453void
452i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj) 454i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
453{ 455{
454 struct drm_device *dev = obj->dev; 456 struct drm_device *dev = obj->base.dev;
455 drm_i915_private_t *dev_priv = dev->dev_private; 457 drm_i915_private_t *dev_priv = dev->dev_private;
456 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 458 int page_count = obj->base.size >> PAGE_SHIFT;
457 int page_count = obj->size >> PAGE_SHIFT;
458 int i; 459 int i;
459 460
460 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) 461 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
461 return; 462 return;
462 463
463 if (obj_priv->bit_17 == NULL) { 464 if (obj->bit_17 == NULL) {
464 obj_priv->bit_17 = kmalloc(BITS_TO_LONGS(page_count) * 465 obj->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
465 sizeof(long), GFP_KERNEL); 466 sizeof(long), GFP_KERNEL);
466 if (obj_priv->bit_17 == NULL) { 467 if (obj->bit_17 == NULL) {
467 DRM_ERROR("Failed to allocate memory for bit 17 " 468 DRM_ERROR("Failed to allocate memory for bit 17 "
468 "record\n"); 469 "record\n");
469 return; 470 return;
@@ -471,9 +472,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
471 } 472 }
472 473
473 for (i = 0; i < page_count; i++) { 474 for (i = 0; i < page_count; i++) {
474 if (page_to_phys(obj_priv->pages[i]) & (1 << 17)) 475 if (page_to_phys(obj->pages[i]) & (1 << 17))
475 __set_bit(i, obj_priv->bit_17); 476 __set_bit(i, obj->bit_17);
476 else 477 else
477 __clear_bit(i, obj_priv->bit_17); 478 __clear_bit(i, obj->bit_17);
478 } 479 }
479} 480}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 729fd0c91d7b..2ddb98b5c90f 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -67,20 +67,20 @@
67void 67void
68ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask) 68ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
69{ 69{
70 if ((dev_priv->gt_irq_mask_reg & mask) != 0) { 70 if ((dev_priv->gt_irq_mask & mask) != 0) {
71 dev_priv->gt_irq_mask_reg &= ~mask; 71 dev_priv->gt_irq_mask &= ~mask;
72 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); 72 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
73 (void) I915_READ(GTIMR); 73 POSTING_READ(GTIMR);
74 } 74 }
75} 75}
76 76
77void 77void
78ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask) 78ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
79{ 79{
80 if ((dev_priv->gt_irq_mask_reg & mask) != mask) { 80 if ((dev_priv->gt_irq_mask & mask) != mask) {
81 dev_priv->gt_irq_mask_reg |= mask; 81 dev_priv->gt_irq_mask |= mask;
82 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); 82 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
83 (void) I915_READ(GTIMR); 83 POSTING_READ(GTIMR);
84 } 84 }
85} 85}
86 86
@@ -88,40 +88,40 @@ ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
88static void 88static void
89ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) 89ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
90{ 90{
91 if ((dev_priv->irq_mask_reg & mask) != 0) { 91 if ((dev_priv->irq_mask & mask) != 0) {
92 dev_priv->irq_mask_reg &= ~mask; 92 dev_priv->irq_mask &= ~mask;
93 I915_WRITE(DEIMR, dev_priv->irq_mask_reg); 93 I915_WRITE(DEIMR, dev_priv->irq_mask);
94 (void) I915_READ(DEIMR); 94 POSTING_READ(DEIMR);
95 } 95 }
96} 96}
97 97
98static inline void 98static inline void
99ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask) 99ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
100{ 100{
101 if ((dev_priv->irq_mask_reg & mask) != mask) { 101 if ((dev_priv->irq_mask & mask) != mask) {
102 dev_priv->irq_mask_reg |= mask; 102 dev_priv->irq_mask |= mask;
103 I915_WRITE(DEIMR, dev_priv->irq_mask_reg); 103 I915_WRITE(DEIMR, dev_priv->irq_mask);
104 (void) I915_READ(DEIMR); 104 POSTING_READ(DEIMR);
105 } 105 }
106} 106}
107 107
108void 108void
109i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask) 109i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
110{ 110{
111 if ((dev_priv->irq_mask_reg & mask) != 0) { 111 if ((dev_priv->irq_mask & mask) != 0) {
112 dev_priv->irq_mask_reg &= ~mask; 112 dev_priv->irq_mask &= ~mask;
113 I915_WRITE(IMR, dev_priv->irq_mask_reg); 113 I915_WRITE(IMR, dev_priv->irq_mask);
114 (void) I915_READ(IMR); 114 POSTING_READ(IMR);
115 } 115 }
116} 116}
117 117
118void 118void
119i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask) 119i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
120{ 120{
121 if ((dev_priv->irq_mask_reg & mask) != mask) { 121 if ((dev_priv->irq_mask & mask) != mask) {
122 dev_priv->irq_mask_reg |= mask; 122 dev_priv->irq_mask |= mask;
123 I915_WRITE(IMR, dev_priv->irq_mask_reg); 123 I915_WRITE(IMR, dev_priv->irq_mask);
124 (void) I915_READ(IMR); 124 POSTING_READ(IMR);
125 } 125 }
126} 126}
127 127
@@ -144,7 +144,7 @@ i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
144 dev_priv->pipestat[pipe] |= mask; 144 dev_priv->pipestat[pipe] |= mask;
145 /* Enable the interrupt, clear any pending status */ 145 /* Enable the interrupt, clear any pending status */
146 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16)); 146 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
147 (void) I915_READ(reg); 147 POSTING_READ(reg);
148 } 148 }
149} 149}
150 150
@@ -156,16 +156,19 @@ i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
156 156
157 dev_priv->pipestat[pipe] &= ~mask; 157 dev_priv->pipestat[pipe] &= ~mask;
158 I915_WRITE(reg, dev_priv->pipestat[pipe]); 158 I915_WRITE(reg, dev_priv->pipestat[pipe]);
159 (void) I915_READ(reg); 159 POSTING_READ(reg);
160 } 160 }
161} 161}
162 162
163/** 163/**
164 * intel_enable_asle - enable ASLE interrupt for OpRegion 164 * intel_enable_asle - enable ASLE interrupt for OpRegion
165 */ 165 */
166void intel_enable_asle (struct drm_device *dev) 166void intel_enable_asle(struct drm_device *dev)
167{ 167{
168 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 168 drm_i915_private_t *dev_priv = dev->dev_private;
169 unsigned long irqflags;
170
171 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
169 172
170 if (HAS_PCH_SPLIT(dev)) 173 if (HAS_PCH_SPLIT(dev))
171 ironlake_enable_display_irq(dev_priv, DE_GSE); 174 ironlake_enable_display_irq(dev_priv, DE_GSE);
@@ -176,6 +179,8 @@ void intel_enable_asle (struct drm_device *dev)
176 i915_enable_pipestat(dev_priv, 0, 179 i915_enable_pipestat(dev_priv, 0,
177 PIPE_LEGACY_BLC_EVENT_ENABLE); 180 PIPE_LEGACY_BLC_EVENT_ENABLE);
178 } 181 }
182
183 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
179} 184}
180 185
181/** 186/**
@@ -297,8 +302,8 @@ static void notify_ring(struct drm_device *dev,
297 struct intel_ring_buffer *ring) 302 struct intel_ring_buffer *ring)
298{ 303{
299 struct drm_i915_private *dev_priv = dev->dev_private; 304 struct drm_i915_private *dev_priv = dev->dev_private;
300 u32 seqno = ring->get_seqno(dev, ring); 305 u32 seqno = ring->get_seqno(ring);
301 ring->irq_gem_seqno = seqno; 306 ring->irq_seqno = seqno;
302 trace_i915_gem_request_complete(dev, seqno); 307 trace_i915_gem_request_complete(dev, seqno);
303 wake_up_all(&ring->irq_queue); 308 wake_up_all(&ring->irq_queue);
304 dev_priv->hangcheck_count = 0; 309 dev_priv->hangcheck_count = 0;
@@ -321,7 +326,7 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
321 /* disable master interrupt before clearing iir */ 326 /* disable master interrupt before clearing iir */
322 de_ier = I915_READ(DEIER); 327 de_ier = I915_READ(DEIER);
323 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); 328 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
324 (void)I915_READ(DEIER); 329 POSTING_READ(DEIER);
325 330
326 de_iir = I915_READ(DEIIR); 331 de_iir = I915_READ(DEIIR);
327 gt_iir = I915_READ(GTIIR); 332 gt_iir = I915_READ(GTIIR);
@@ -344,12 +349,12 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
344 READ_BREADCRUMB(dev_priv); 349 READ_BREADCRUMB(dev_priv);
345 } 350 }
346 351
347 if (gt_iir & GT_PIPE_NOTIFY) 352 if (gt_iir & GT_USER_INTERRUPT)
348 notify_ring(dev, &dev_priv->render_ring); 353 notify_ring(dev, &dev_priv->ring[RCS]);
349 if (gt_iir & bsd_usr_interrupt) 354 if (gt_iir & bsd_usr_interrupt)
350 notify_ring(dev, &dev_priv->bsd_ring); 355 notify_ring(dev, &dev_priv->ring[VCS]);
351 if (HAS_BLT(dev) && gt_iir & GT_BLT_USER_INTERRUPT) 356 if (gt_iir & GT_BLT_USER_INTERRUPT)
352 notify_ring(dev, &dev_priv->blt_ring); 357 notify_ring(dev, &dev_priv->ring[BCS]);
353 358
354 if (de_iir & DE_GSE) 359 if (de_iir & DE_GSE)
355 intel_opregion_gse_intr(dev); 360 intel_opregion_gse_intr(dev);
@@ -386,7 +391,7 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
386 391
387done: 392done:
388 I915_WRITE(DEIER, de_ier); 393 I915_WRITE(DEIER, de_ier);
389 (void)I915_READ(DEIER); 394 POSTING_READ(DEIER);
390 395
391 return ret; 396 return ret;
392} 397}
@@ -423,28 +428,23 @@ static void i915_error_work_func(struct work_struct *work)
423#ifdef CONFIG_DEBUG_FS 428#ifdef CONFIG_DEBUG_FS
424static struct drm_i915_error_object * 429static struct drm_i915_error_object *
425i915_error_object_create(struct drm_device *dev, 430i915_error_object_create(struct drm_device *dev,
426 struct drm_gem_object *src) 431 struct drm_i915_gem_object *src)
427{ 432{
428 drm_i915_private_t *dev_priv = dev->dev_private; 433 drm_i915_private_t *dev_priv = dev->dev_private;
429 struct drm_i915_error_object *dst; 434 struct drm_i915_error_object *dst;
430 struct drm_i915_gem_object *src_priv;
431 int page, page_count; 435 int page, page_count;
432 u32 reloc_offset; 436 u32 reloc_offset;
433 437
434 if (src == NULL) 438 if (src == NULL || src->pages == NULL)
435 return NULL;
436
437 src_priv = to_intel_bo(src);
438 if (src_priv->pages == NULL)
439 return NULL; 439 return NULL;
440 440
441 page_count = src->size / PAGE_SIZE; 441 page_count = src->base.size / PAGE_SIZE;
442 442
443 dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC); 443 dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC);
444 if (dst == NULL) 444 if (dst == NULL)
445 return NULL; 445 return NULL;
446 446
447 reloc_offset = src_priv->gtt_offset; 447 reloc_offset = src->gtt_offset;
448 for (page = 0; page < page_count; page++) { 448 for (page = 0; page < page_count; page++) {
449 unsigned long flags; 449 unsigned long flags;
450 void __iomem *s; 450 void __iomem *s;
@@ -466,7 +466,7 @@ i915_error_object_create(struct drm_device *dev,
466 reloc_offset += PAGE_SIZE; 466 reloc_offset += PAGE_SIZE;
467 } 467 }
468 dst->page_count = page_count; 468 dst->page_count = page_count;
469 dst->gtt_offset = src_priv->gtt_offset; 469 dst->gtt_offset = src->gtt_offset;
470 470
471 return dst; 471 return dst;
472 472
@@ -520,36 +520,96 @@ i915_get_bbaddr(struct drm_device *dev, u32 *ring)
520} 520}
521 521
522static u32 522static u32
523i915_ringbuffer_last_batch(struct drm_device *dev) 523i915_ringbuffer_last_batch(struct drm_device *dev,
524 struct intel_ring_buffer *ring)
524{ 525{
525 struct drm_i915_private *dev_priv = dev->dev_private; 526 struct drm_i915_private *dev_priv = dev->dev_private;
526 u32 head, bbaddr; 527 u32 head, bbaddr;
527 u32 *ring; 528 u32 *val;
528 529
529 /* Locate the current position in the ringbuffer and walk back 530 /* Locate the current position in the ringbuffer and walk back
530 * to find the most recently dispatched batch buffer. 531 * to find the most recently dispatched batch buffer.
531 */ 532 */
532 bbaddr = 0; 533 head = I915_READ_HEAD(ring) & HEAD_ADDR;
533 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
534 ring = (u32 *)(dev_priv->render_ring.virtual_start + head);
535 534
536 while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) { 535 val = (u32 *)(ring->virtual_start + head);
537 bbaddr = i915_get_bbaddr(dev, ring); 536 while (--val >= (u32 *)ring->virtual_start) {
537 bbaddr = i915_get_bbaddr(dev, val);
538 if (bbaddr) 538 if (bbaddr)
539 break; 539 return bbaddr;
540 } 540 }
541 541
542 if (bbaddr == 0) { 542 val = (u32 *)(ring->virtual_start + ring->size);
543 ring = (u32 *)(dev_priv->render_ring.virtual_start 543 while (--val >= (u32 *)ring->virtual_start) {
544 + dev_priv->render_ring.size); 544 bbaddr = i915_get_bbaddr(dev, val);
545 while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) { 545 if (bbaddr)
546 bbaddr = i915_get_bbaddr(dev, ring); 546 return bbaddr;
547 if (bbaddr)
548 break;
549 }
550 } 547 }
551 548
552 return bbaddr; 549 return 0;
550}
551
552static u32 capture_bo_list(struct drm_i915_error_buffer *err,
553 int count,
554 struct list_head *head)
555{
556 struct drm_i915_gem_object *obj;
557 int i = 0;
558
559 list_for_each_entry(obj, head, mm_list) {
560 err->size = obj->base.size;
561 err->name = obj->base.name;
562 err->seqno = obj->last_rendering_seqno;
563 err->gtt_offset = obj->gtt_offset;
564 err->read_domains = obj->base.read_domains;
565 err->write_domain = obj->base.write_domain;
566 err->fence_reg = obj->fence_reg;
567 err->pinned = 0;
568 if (obj->pin_count > 0)
569 err->pinned = 1;
570 if (obj->user_pin_count > 0)
571 err->pinned = -1;
572 err->tiling = obj->tiling_mode;
573 err->dirty = obj->dirty;
574 err->purgeable = obj->madv != I915_MADV_WILLNEED;
575 err->ring = obj->ring ? obj->ring->id : 0;
576
577 if (++i == count)
578 break;
579
580 err++;
581 }
582
583 return i;
584}
585
586static void i915_gem_record_fences(struct drm_device *dev,
587 struct drm_i915_error_state *error)
588{
589 struct drm_i915_private *dev_priv = dev->dev_private;
590 int i;
591
592 /* Fences */
593 switch (INTEL_INFO(dev)->gen) {
594 case 6:
595 for (i = 0; i < 16; i++)
596 error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
597 break;
598 case 5:
599 case 4:
600 for (i = 0; i < 16; i++)
601 error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
602 break;
603 case 3:
604 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
605 for (i = 0; i < 8; i++)
606 error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
607 case 2:
608 for (i = 0; i < 8; i++)
609 error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
610 break;
611
612 }
553} 613}
554 614
555/** 615/**
@@ -564,9 +624,9 @@ i915_ringbuffer_last_batch(struct drm_device *dev)
564static void i915_capture_error_state(struct drm_device *dev) 624static void i915_capture_error_state(struct drm_device *dev)
565{ 625{
566 struct drm_i915_private *dev_priv = dev->dev_private; 626 struct drm_i915_private *dev_priv = dev->dev_private;
567 struct drm_i915_gem_object *obj_priv; 627 struct drm_i915_gem_object *obj;
568 struct drm_i915_error_state *error; 628 struct drm_i915_error_state *error;
569 struct drm_gem_object *batchbuffer[2]; 629 struct drm_i915_gem_object *batchbuffer[2];
570 unsigned long flags; 630 unsigned long flags;
571 u32 bbaddr; 631 u32 bbaddr;
572 int count; 632 int count;
@@ -585,20 +645,33 @@ static void i915_capture_error_state(struct drm_device *dev)
585 645
586 DRM_DEBUG_DRIVER("generating error event\n"); 646 DRM_DEBUG_DRIVER("generating error event\n");
587 647
588 error->seqno = 648 error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]);
589 dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring);
590 error->eir = I915_READ(EIR); 649 error->eir = I915_READ(EIR);
591 error->pgtbl_er = I915_READ(PGTBL_ER); 650 error->pgtbl_er = I915_READ(PGTBL_ER);
592 error->pipeastat = I915_READ(PIPEASTAT); 651 error->pipeastat = I915_READ(PIPEASTAT);
593 error->pipebstat = I915_READ(PIPEBSTAT); 652 error->pipebstat = I915_READ(PIPEBSTAT);
594 error->instpm = I915_READ(INSTPM); 653 error->instpm = I915_READ(INSTPM);
595 if (INTEL_INFO(dev)->gen < 4) { 654 error->error = 0;
596 error->ipeir = I915_READ(IPEIR); 655 if (INTEL_INFO(dev)->gen >= 6) {
597 error->ipehr = I915_READ(IPEHR); 656 error->error = I915_READ(ERROR_GEN6);
598 error->instdone = I915_READ(INSTDONE); 657
599 error->acthd = I915_READ(ACTHD); 658 error->bcs_acthd = I915_READ(BCS_ACTHD);
600 error->bbaddr = 0; 659 error->bcs_ipehr = I915_READ(BCS_IPEHR);
601 } else { 660 error->bcs_ipeir = I915_READ(BCS_IPEIR);
661 error->bcs_instdone = I915_READ(BCS_INSTDONE);
662 error->bcs_seqno = 0;
663 if (dev_priv->ring[BCS].get_seqno)
664 error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]);
665
666 error->vcs_acthd = I915_READ(VCS_ACTHD);
667 error->vcs_ipehr = I915_READ(VCS_IPEHR);
668 error->vcs_ipeir = I915_READ(VCS_IPEIR);
669 error->vcs_instdone = I915_READ(VCS_INSTDONE);
670 error->vcs_seqno = 0;
671 if (dev_priv->ring[VCS].get_seqno)
672 error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]);
673 }
674 if (INTEL_INFO(dev)->gen >= 4) {
602 error->ipeir = I915_READ(IPEIR_I965); 675 error->ipeir = I915_READ(IPEIR_I965);
603 error->ipehr = I915_READ(IPEHR_I965); 676 error->ipehr = I915_READ(IPEHR_I965);
604 error->instdone = I915_READ(INSTDONE_I965); 677 error->instdone = I915_READ(INSTDONE_I965);
@@ -606,42 +679,45 @@ static void i915_capture_error_state(struct drm_device *dev)
606 error->instdone1 = I915_READ(INSTDONE1); 679 error->instdone1 = I915_READ(INSTDONE1);
607 error->acthd = I915_READ(ACTHD_I965); 680 error->acthd = I915_READ(ACTHD_I965);
608 error->bbaddr = I915_READ64(BB_ADDR); 681 error->bbaddr = I915_READ64(BB_ADDR);
682 } else {
683 error->ipeir = I915_READ(IPEIR);
684 error->ipehr = I915_READ(IPEHR);
685 error->instdone = I915_READ(INSTDONE);
686 error->acthd = I915_READ(ACTHD);
687 error->bbaddr = 0;
609 } 688 }
689 i915_gem_record_fences(dev, error);
610 690
611 bbaddr = i915_ringbuffer_last_batch(dev); 691 bbaddr = i915_ringbuffer_last_batch(dev, &dev_priv->ring[RCS]);
612 692
613 /* Grab the current batchbuffer, most likely to have crashed. */ 693 /* Grab the current batchbuffer, most likely to have crashed. */
614 batchbuffer[0] = NULL; 694 batchbuffer[0] = NULL;
615 batchbuffer[1] = NULL; 695 batchbuffer[1] = NULL;
616 count = 0; 696 count = 0;
617 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 697 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
618 struct drm_gem_object *obj = &obj_priv->base;
619
620 if (batchbuffer[0] == NULL && 698 if (batchbuffer[0] == NULL &&
621 bbaddr >= obj_priv->gtt_offset && 699 bbaddr >= obj->gtt_offset &&
622 bbaddr < obj_priv->gtt_offset + obj->size) 700 bbaddr < obj->gtt_offset + obj->base.size)
623 batchbuffer[0] = obj; 701 batchbuffer[0] = obj;
624 702
625 if (batchbuffer[1] == NULL && 703 if (batchbuffer[1] == NULL &&
626 error->acthd >= obj_priv->gtt_offset && 704 error->acthd >= obj->gtt_offset &&
627 error->acthd < obj_priv->gtt_offset + obj->size) 705 error->acthd < obj->gtt_offset + obj->base.size)
628 batchbuffer[1] = obj; 706 batchbuffer[1] = obj;
629 707
630 count++; 708 count++;
631 } 709 }
632 /* Scan the other lists for completeness for those bizarre errors. */ 710 /* Scan the other lists for completeness for those bizarre errors. */
633 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) { 711 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
634 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) { 712 list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
635 struct drm_gem_object *obj = &obj_priv->base;
636
637 if (batchbuffer[0] == NULL && 713 if (batchbuffer[0] == NULL &&
638 bbaddr >= obj_priv->gtt_offset && 714 bbaddr >= obj->gtt_offset &&
639 bbaddr < obj_priv->gtt_offset + obj->size) 715 bbaddr < obj->gtt_offset + obj->base.size)
640 batchbuffer[0] = obj; 716 batchbuffer[0] = obj;
641 717
642 if (batchbuffer[1] == NULL && 718 if (batchbuffer[1] == NULL &&
643 error->acthd >= obj_priv->gtt_offset && 719 error->acthd >= obj->gtt_offset &&
644 error->acthd < obj_priv->gtt_offset + obj->size) 720 error->acthd < obj->gtt_offset + obj->base.size)
645 batchbuffer[1] = obj; 721 batchbuffer[1] = obj;
646 722
647 if (batchbuffer[0] && batchbuffer[1]) 723 if (batchbuffer[0] && batchbuffer[1])
@@ -649,17 +725,15 @@ static void i915_capture_error_state(struct drm_device *dev)
649 } 725 }
650 } 726 }
651 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) { 727 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
652 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) { 728 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
653 struct drm_gem_object *obj = &obj_priv->base;
654
655 if (batchbuffer[0] == NULL && 729 if (batchbuffer[0] == NULL &&
656 bbaddr >= obj_priv->gtt_offset && 730 bbaddr >= obj->gtt_offset &&
657 bbaddr < obj_priv->gtt_offset + obj->size) 731 bbaddr < obj->gtt_offset + obj->base.size)
658 batchbuffer[0] = obj; 732 batchbuffer[0] = obj;
659 733
660 if (batchbuffer[1] == NULL && 734 if (batchbuffer[1] == NULL &&
661 error->acthd >= obj_priv->gtt_offset && 735 error->acthd >= obj->gtt_offset &&
662 error->acthd < obj_priv->gtt_offset + obj->size) 736 error->acthd < obj->gtt_offset + obj->base.size)
663 batchbuffer[1] = obj; 737 batchbuffer[1] = obj;
664 738
665 if (batchbuffer[0] && batchbuffer[1]) 739 if (batchbuffer[0] && batchbuffer[1])
@@ -678,46 +752,41 @@ static void i915_capture_error_state(struct drm_device *dev)
678 752
679 /* Record the ringbuffer */ 753 /* Record the ringbuffer */
680 error->ringbuffer = i915_error_object_create(dev, 754 error->ringbuffer = i915_error_object_create(dev,
681 dev_priv->render_ring.gem_object); 755 dev_priv->ring[RCS].obj);
682 756
683 /* Record buffers on the active list. */ 757 /* Record buffers on the active and pinned lists. */
684 error->active_bo = NULL; 758 error->active_bo = NULL;
685 error->active_bo_count = 0; 759 error->pinned_bo = NULL;
686 760
687 if (count) 761 error->active_bo_count = count;
762 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
763 count++;
764 error->pinned_bo_count = count - error->active_bo_count;
765
766 if (count) {
688 error->active_bo = kmalloc(sizeof(*error->active_bo)*count, 767 error->active_bo = kmalloc(sizeof(*error->active_bo)*count,
689 GFP_ATOMIC); 768 GFP_ATOMIC);
690 769 if (error->active_bo)
691 if (error->active_bo) { 770 error->pinned_bo =
692 int i = 0; 771 error->active_bo + error->active_bo_count;
693 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
694 struct drm_gem_object *obj = &obj_priv->base;
695
696 error->active_bo[i].size = obj->size;
697 error->active_bo[i].name = obj->name;
698 error->active_bo[i].seqno = obj_priv->last_rendering_seqno;
699 error->active_bo[i].gtt_offset = obj_priv->gtt_offset;
700 error->active_bo[i].read_domains = obj->read_domains;
701 error->active_bo[i].write_domain = obj->write_domain;
702 error->active_bo[i].fence_reg = obj_priv->fence_reg;
703 error->active_bo[i].pinned = 0;
704 if (obj_priv->pin_count > 0)
705 error->active_bo[i].pinned = 1;
706 if (obj_priv->user_pin_count > 0)
707 error->active_bo[i].pinned = -1;
708 error->active_bo[i].tiling = obj_priv->tiling_mode;
709 error->active_bo[i].dirty = obj_priv->dirty;
710 error->active_bo[i].purgeable = obj_priv->madv != I915_MADV_WILLNEED;
711
712 if (++i == count)
713 break;
714 }
715 error->active_bo_count = i;
716 } 772 }
717 773
774 if (error->active_bo)
775 error->active_bo_count =
776 capture_bo_list(error->active_bo,
777 error->active_bo_count,
778 &dev_priv->mm.active_list);
779
780 if (error->pinned_bo)
781 error->pinned_bo_count =
782 capture_bo_list(error->pinned_bo,
783 error->pinned_bo_count,
784 &dev_priv->mm.pinned_list);
785
718 do_gettimeofday(&error->time); 786 do_gettimeofday(&error->time);
719 787
720 error->overlay = intel_overlay_capture_error_state(dev); 788 error->overlay = intel_overlay_capture_error_state(dev);
789 error->display = intel_display_capture_error_state(dev);
721 790
722 spin_lock_irqsave(&dev_priv->error_lock, flags); 791 spin_lock_irqsave(&dev_priv->error_lock, flags);
723 if (dev_priv->first_error == NULL) { 792 if (dev_priv->first_error == NULL) {
@@ -775,7 +844,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
775 printk(KERN_ERR " ACTHD: 0x%08x\n", 844 printk(KERN_ERR " ACTHD: 0x%08x\n",
776 I915_READ(ACTHD_I965)); 845 I915_READ(ACTHD_I965));
777 I915_WRITE(IPEIR_I965, ipeir); 846 I915_WRITE(IPEIR_I965, ipeir);
778 (void)I915_READ(IPEIR_I965); 847 POSTING_READ(IPEIR_I965);
779 } 848 }
780 if (eir & GM45_ERROR_PAGE_TABLE) { 849 if (eir & GM45_ERROR_PAGE_TABLE) {
781 u32 pgtbl_err = I915_READ(PGTBL_ER); 850 u32 pgtbl_err = I915_READ(PGTBL_ER);
@@ -783,7 +852,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
783 printk(KERN_ERR " PGTBL_ER: 0x%08x\n", 852 printk(KERN_ERR " PGTBL_ER: 0x%08x\n",
784 pgtbl_err); 853 pgtbl_err);
785 I915_WRITE(PGTBL_ER, pgtbl_err); 854 I915_WRITE(PGTBL_ER, pgtbl_err);
786 (void)I915_READ(PGTBL_ER); 855 POSTING_READ(PGTBL_ER);
787 } 856 }
788 } 857 }
789 858
@@ -794,7 +863,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
794 printk(KERN_ERR " PGTBL_ER: 0x%08x\n", 863 printk(KERN_ERR " PGTBL_ER: 0x%08x\n",
795 pgtbl_err); 864 pgtbl_err);
796 I915_WRITE(PGTBL_ER, pgtbl_err); 865 I915_WRITE(PGTBL_ER, pgtbl_err);
797 (void)I915_READ(PGTBL_ER); 866 POSTING_READ(PGTBL_ER);
798 } 867 }
799 } 868 }
800 869
@@ -825,7 +894,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
825 printk(KERN_ERR " ACTHD: 0x%08x\n", 894 printk(KERN_ERR " ACTHD: 0x%08x\n",
826 I915_READ(ACTHD)); 895 I915_READ(ACTHD));
827 I915_WRITE(IPEIR, ipeir); 896 I915_WRITE(IPEIR, ipeir);
828 (void)I915_READ(IPEIR); 897 POSTING_READ(IPEIR);
829 } else { 898 } else {
830 u32 ipeir = I915_READ(IPEIR_I965); 899 u32 ipeir = I915_READ(IPEIR_I965);
831 900
@@ -842,12 +911,12 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
842 printk(KERN_ERR " ACTHD: 0x%08x\n", 911 printk(KERN_ERR " ACTHD: 0x%08x\n",
843 I915_READ(ACTHD_I965)); 912 I915_READ(ACTHD_I965));
844 I915_WRITE(IPEIR_I965, ipeir); 913 I915_WRITE(IPEIR_I965, ipeir);
845 (void)I915_READ(IPEIR_I965); 914 POSTING_READ(IPEIR_I965);
846 } 915 }
847 } 916 }
848 917
849 I915_WRITE(EIR, eir); 918 I915_WRITE(EIR, eir);
850 (void)I915_READ(EIR); 919 POSTING_READ(EIR);
851 eir = I915_READ(EIR); 920 eir = I915_READ(EIR);
852 if (eir) { 921 if (eir) {
853 /* 922 /*
@@ -870,7 +939,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
870 * so userspace knows something bad happened (should trigger collection 939 * so userspace knows something bad happened (should trigger collection
871 * of a ring dump etc.). 940 * of a ring dump etc.).
872 */ 941 */
873static void i915_handle_error(struct drm_device *dev, bool wedged) 942void i915_handle_error(struct drm_device *dev, bool wedged)
874{ 943{
875 struct drm_i915_private *dev_priv = dev->dev_private; 944 struct drm_i915_private *dev_priv = dev->dev_private;
876 945
@@ -884,11 +953,11 @@ static void i915_handle_error(struct drm_device *dev, bool wedged)
884 /* 953 /*
885 * Wakeup waiting processes so they don't hang 954 * Wakeup waiting processes so they don't hang
886 */ 955 */
887 wake_up_all(&dev_priv->render_ring.irq_queue); 956 wake_up_all(&dev_priv->ring[RCS].irq_queue);
888 if (HAS_BSD(dev)) 957 if (HAS_BSD(dev))
889 wake_up_all(&dev_priv->bsd_ring.irq_queue); 958 wake_up_all(&dev_priv->ring[VCS].irq_queue);
890 if (HAS_BLT(dev)) 959 if (HAS_BLT(dev))
891 wake_up_all(&dev_priv->blt_ring.irq_queue); 960 wake_up_all(&dev_priv->ring[BCS].irq_queue);
892 } 961 }
893 962
894 queue_work(dev_priv->wq, &dev_priv->error_work); 963 queue_work(dev_priv->wq, &dev_priv->error_work);
@@ -899,7 +968,7 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
899 drm_i915_private_t *dev_priv = dev->dev_private; 968 drm_i915_private_t *dev_priv = dev->dev_private;
900 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; 969 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
901 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 970 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
902 struct drm_i915_gem_object *obj_priv; 971 struct drm_i915_gem_object *obj;
903 struct intel_unpin_work *work; 972 struct intel_unpin_work *work;
904 unsigned long flags; 973 unsigned long flags;
905 bool stall_detected; 974 bool stall_detected;
@@ -918,13 +987,13 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
918 } 987 }
919 988
920 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */ 989 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
921 obj_priv = to_intel_bo(work->pending_flip_obj); 990 obj = work->pending_flip_obj;
922 if (INTEL_INFO(dev)->gen >= 4) { 991 if (INTEL_INFO(dev)->gen >= 4) {
923 int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF; 992 int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF;
924 stall_detected = I915_READ(dspsurf) == obj_priv->gtt_offset; 993 stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
925 } else { 994 } else {
926 int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR; 995 int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR;
927 stall_detected = I915_READ(dspaddr) == (obj_priv->gtt_offset + 996 stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
928 crtc->y * crtc->fb->pitch + 997 crtc->y * crtc->fb->pitch +
929 crtc->x * crtc->fb->bits_per_pixel/8); 998 crtc->x * crtc->fb->bits_per_pixel/8);
930 } 999 }
@@ -970,7 +1039,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
970 * It doesn't set the bit in iir again, but it still produces 1039 * It doesn't set the bit in iir again, but it still produces
971 * interrupts (for non-MSI). 1040 * interrupts (for non-MSI).
972 */ 1041 */
973 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1042 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
974 pipea_stats = I915_READ(PIPEASTAT); 1043 pipea_stats = I915_READ(PIPEASTAT);
975 pipeb_stats = I915_READ(PIPEBSTAT); 1044 pipeb_stats = I915_READ(PIPEBSTAT);
976 1045
@@ -993,7 +1062,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
993 I915_WRITE(PIPEBSTAT, pipeb_stats); 1062 I915_WRITE(PIPEBSTAT, pipeb_stats);
994 irq_received = 1; 1063 irq_received = 1;
995 } 1064 }
996 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1065 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
997 1066
998 if (!irq_received) 1067 if (!irq_received)
999 break; 1068 break;
@@ -1026,9 +1095,9 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
1026 } 1095 }
1027 1096
1028 if (iir & I915_USER_INTERRUPT) 1097 if (iir & I915_USER_INTERRUPT)
1029 notify_ring(dev, &dev_priv->render_ring); 1098 notify_ring(dev, &dev_priv->ring[RCS]);
1030 if (HAS_BSD(dev) && (iir & I915_BSD_USER_INTERRUPT)) 1099 if (iir & I915_BSD_USER_INTERRUPT)
1031 notify_ring(dev, &dev_priv->bsd_ring); 1100 notify_ring(dev, &dev_priv->ring[VCS]);
1032 1101
1033 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) { 1102 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
1034 intel_prepare_page_flip(dev, 0); 1103 intel_prepare_page_flip(dev, 0);
@@ -1101,12 +1170,13 @@ static int i915_emit_irq(struct drm_device * dev)
1101 if (master_priv->sarea_priv) 1170 if (master_priv->sarea_priv)
1102 master_priv->sarea_priv->last_enqueue = dev_priv->counter; 1171 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
1103 1172
1104 BEGIN_LP_RING(4); 1173 if (BEGIN_LP_RING(4) == 0) {
1105 OUT_RING(MI_STORE_DWORD_INDEX); 1174 OUT_RING(MI_STORE_DWORD_INDEX);
1106 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1175 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1107 OUT_RING(dev_priv->counter); 1176 OUT_RING(dev_priv->counter);
1108 OUT_RING(MI_USER_INTERRUPT); 1177 OUT_RING(MI_USER_INTERRUPT);
1109 ADVANCE_LP_RING(); 1178 ADVANCE_LP_RING();
1179 }
1110 1180
1111 return dev_priv->counter; 1181 return dev_priv->counter;
1112} 1182}
@@ -1114,12 +1184,11 @@ static int i915_emit_irq(struct drm_device * dev)
1114void i915_trace_irq_get(struct drm_device *dev, u32 seqno) 1184void i915_trace_irq_get(struct drm_device *dev, u32 seqno)
1115{ 1185{
1116 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1186 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1117 struct intel_ring_buffer *render_ring = &dev_priv->render_ring; 1187 struct intel_ring_buffer *ring = LP_RING(dev_priv);
1118 1188
1119 if (dev_priv->trace_irq_seqno == 0) 1189 if (dev_priv->trace_irq_seqno == 0 &&
1120 render_ring->user_irq_get(dev, render_ring); 1190 ring->irq_get(ring))
1121 1191 dev_priv->trace_irq_seqno = seqno;
1122 dev_priv->trace_irq_seqno = seqno;
1123} 1192}
1124 1193
1125static int i915_wait_irq(struct drm_device * dev, int irq_nr) 1194static int i915_wait_irq(struct drm_device * dev, int irq_nr)
@@ -1127,7 +1196,7 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
1127 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1196 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1128 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; 1197 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1129 int ret = 0; 1198 int ret = 0;
1130 struct intel_ring_buffer *render_ring = &dev_priv->render_ring; 1199 struct intel_ring_buffer *ring = LP_RING(dev_priv);
1131 1200
1132 DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr, 1201 DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
1133 READ_BREADCRUMB(dev_priv)); 1202 READ_BREADCRUMB(dev_priv));
@@ -1141,10 +1210,12 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
1141 if (master_priv->sarea_priv) 1210 if (master_priv->sarea_priv)
1142 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; 1211 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1143 1212
1144 render_ring->user_irq_get(dev, render_ring); 1213 ret = -ENODEV;
1145 DRM_WAIT_ON(ret, dev_priv->render_ring.irq_queue, 3 * DRM_HZ, 1214 if (ring->irq_get(ring)) {
1146 READ_BREADCRUMB(dev_priv) >= irq_nr); 1215 DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ,
1147 render_ring->user_irq_put(dev, render_ring); 1216 READ_BREADCRUMB(dev_priv) >= irq_nr);
1217 ring->irq_put(ring);
1218 }
1148 1219
1149 if (ret == -EBUSY) { 1220 if (ret == -EBUSY) {
1150 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", 1221 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
@@ -1163,7 +1234,7 @@ int i915_irq_emit(struct drm_device *dev, void *data,
1163 drm_i915_irq_emit_t *emit = data; 1234 drm_i915_irq_emit_t *emit = data;
1164 int result; 1235 int result;
1165 1236
1166 if (!dev_priv || !dev_priv->render_ring.virtual_start) { 1237 if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
1167 DRM_ERROR("called with no initialization\n"); 1238 DRM_ERROR("called with no initialization\n");
1168 return -EINVAL; 1239 return -EINVAL;
1169 } 1240 }
@@ -1209,9 +1280,9 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
1209 if (!i915_pipe_enabled(dev, pipe)) 1280 if (!i915_pipe_enabled(dev, pipe))
1210 return -EINVAL; 1281 return -EINVAL;
1211 1282
1212 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1283 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1213 if (HAS_PCH_SPLIT(dev)) 1284 if (HAS_PCH_SPLIT(dev))
1214 ironlake_enable_display_irq(dev_priv, (pipe == 0) ? 1285 ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
1215 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK); 1286 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1216 else if (INTEL_INFO(dev)->gen >= 4) 1287 else if (INTEL_INFO(dev)->gen >= 4)
1217 i915_enable_pipestat(dev_priv, pipe, 1288 i915_enable_pipestat(dev_priv, pipe,
@@ -1219,7 +1290,7 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
1219 else 1290 else
1220 i915_enable_pipestat(dev_priv, pipe, 1291 i915_enable_pipestat(dev_priv, pipe,
1221 PIPE_VBLANK_INTERRUPT_ENABLE); 1292 PIPE_VBLANK_INTERRUPT_ENABLE);
1222 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1293 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1223 return 0; 1294 return 0;
1224} 1295}
1225 1296
@@ -1231,15 +1302,15 @@ void i915_disable_vblank(struct drm_device *dev, int pipe)
1231 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1302 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1232 unsigned long irqflags; 1303 unsigned long irqflags;
1233 1304
1234 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1305 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1235 if (HAS_PCH_SPLIT(dev)) 1306 if (HAS_PCH_SPLIT(dev))
1236 ironlake_disable_display_irq(dev_priv, (pipe == 0) ? 1307 ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
1237 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK); 1308 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1238 else 1309 else
1239 i915_disable_pipestat(dev_priv, pipe, 1310 i915_disable_pipestat(dev_priv, pipe,
1240 PIPE_VBLANK_INTERRUPT_ENABLE | 1311 PIPE_VBLANK_INTERRUPT_ENABLE |
1241 PIPE_START_VBLANK_INTERRUPT_ENABLE); 1312 PIPE_START_VBLANK_INTERRUPT_ENABLE);
1242 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1313 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1243} 1314}
1244 1315
1245void i915_enable_interrupt (struct drm_device *dev) 1316void i915_enable_interrupt (struct drm_device *dev)
@@ -1306,12 +1377,50 @@ int i915_vblank_swap(struct drm_device *dev, void *data,
1306 return -EINVAL; 1377 return -EINVAL;
1307} 1378}
1308 1379
1309static struct drm_i915_gem_request * 1380static u32
1310i915_get_tail_request(struct drm_device *dev) 1381ring_last_seqno(struct intel_ring_buffer *ring)
1311{ 1382{
1312 drm_i915_private_t *dev_priv = dev->dev_private; 1383 return list_entry(ring->request_list.prev,
1313 return list_entry(dev_priv->render_ring.request_list.prev, 1384 struct drm_i915_gem_request, list)->seqno;
1314 struct drm_i915_gem_request, list); 1385}
1386
1387static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
1388{
1389 if (list_empty(&ring->request_list) ||
1390 i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
1391 /* Issue a wake-up to catch stuck h/w. */
1392 if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) {
1393 DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
1394 ring->name,
1395 ring->waiting_seqno,
1396 ring->get_seqno(ring));
1397 wake_up_all(&ring->irq_queue);
1398 *err = true;
1399 }
1400 return true;
1401 }
1402 return false;
1403}
1404
1405static bool kick_ring(struct intel_ring_buffer *ring)
1406{
1407 struct drm_device *dev = ring->dev;
1408 struct drm_i915_private *dev_priv = dev->dev_private;
1409 u32 tmp = I915_READ_CTL(ring);
1410 if (tmp & RING_WAIT) {
1411 DRM_ERROR("Kicking stuck wait on %s\n",
1412 ring->name);
1413 I915_WRITE_CTL(ring, tmp);
1414 return true;
1415 }
1416 if (IS_GEN6(dev) &&
1417 (tmp & RING_WAIT_SEMAPHORE)) {
1418 DRM_ERROR("Kicking stuck semaphore on %s\n",
1419 ring->name);
1420 I915_WRITE_CTL(ring, tmp);
1421 return true;
1422 }
1423 return false;
1315} 1424}
1316 1425
1317/** 1426/**
@@ -1325,6 +1434,17 @@ void i915_hangcheck_elapsed(unsigned long data)
1325 struct drm_device *dev = (struct drm_device *)data; 1434 struct drm_device *dev = (struct drm_device *)data;
1326 drm_i915_private_t *dev_priv = dev->dev_private; 1435 drm_i915_private_t *dev_priv = dev->dev_private;
1327 uint32_t acthd, instdone, instdone1; 1436 uint32_t acthd, instdone, instdone1;
1437 bool err = false;
1438
1439 /* If all work is done then ACTHD clearly hasn't advanced. */
1440 if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
1441 i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
1442 i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) {
1443 dev_priv->hangcheck_count = 0;
1444 if (err)
1445 goto repeat;
1446 return;
1447 }
1328 1448
1329 if (INTEL_INFO(dev)->gen < 4) { 1449 if (INTEL_INFO(dev)->gen < 4) {
1330 acthd = I915_READ(ACTHD); 1450 acthd = I915_READ(ACTHD);
@@ -1336,38 +1456,6 @@ void i915_hangcheck_elapsed(unsigned long data)
1336 instdone1 = I915_READ(INSTDONE1); 1456 instdone1 = I915_READ(INSTDONE1);
1337 } 1457 }
1338 1458
1339 /* If all work is done then ACTHD clearly hasn't advanced. */
1340 if (list_empty(&dev_priv->render_ring.request_list) ||
1341 i915_seqno_passed(dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring),
1342 i915_get_tail_request(dev)->seqno)) {
1343 bool missed_wakeup = false;
1344
1345 dev_priv->hangcheck_count = 0;
1346
1347 /* Issue a wake-up to catch stuck h/w. */
1348 if (dev_priv->render_ring.waiting_gem_seqno &&
1349 waitqueue_active(&dev_priv->render_ring.irq_queue)) {
1350 wake_up_all(&dev_priv->render_ring.irq_queue);
1351 missed_wakeup = true;
1352 }
1353
1354 if (dev_priv->bsd_ring.waiting_gem_seqno &&
1355 waitqueue_active(&dev_priv->bsd_ring.irq_queue)) {
1356 wake_up_all(&dev_priv->bsd_ring.irq_queue);
1357 missed_wakeup = true;
1358 }
1359
1360 if (dev_priv->blt_ring.waiting_gem_seqno &&
1361 waitqueue_active(&dev_priv->blt_ring.irq_queue)) {
1362 wake_up_all(&dev_priv->blt_ring.irq_queue);
1363 missed_wakeup = true;
1364 }
1365
1366 if (missed_wakeup)
1367 DRM_ERROR("Hangcheck timer elapsed... GPU idle, missed IRQ.\n");
1368 return;
1369 }
1370
1371 if (dev_priv->last_acthd == acthd && 1459 if (dev_priv->last_acthd == acthd &&
1372 dev_priv->last_instdone == instdone && 1460 dev_priv->last_instdone == instdone &&
1373 dev_priv->last_instdone1 == instdone1) { 1461 dev_priv->last_instdone1 == instdone1) {
@@ -1380,12 +1468,17 @@ void i915_hangcheck_elapsed(unsigned long data)
1380 * and break the hang. This should work on 1468 * and break the hang. This should work on
1381 * all but the second generation chipsets. 1469 * all but the second generation chipsets.
1382 */ 1470 */
1383 u32 tmp = I915_READ(PRB0_CTL); 1471
1384 if (tmp & RING_WAIT) { 1472 if (kick_ring(&dev_priv->ring[RCS]))
1385 I915_WRITE(PRB0_CTL, tmp); 1473 goto repeat;
1386 POSTING_READ(PRB0_CTL); 1474
1387 goto out; 1475 if (HAS_BSD(dev) &&
1388 } 1476 kick_ring(&dev_priv->ring[VCS]))
1477 goto repeat;
1478
1479 if (HAS_BLT(dev) &&
1480 kick_ring(&dev_priv->ring[BCS]))
1481 goto repeat;
1389 } 1482 }
1390 1483
1391 i915_handle_error(dev, true); 1484 i915_handle_error(dev, true);
@@ -1399,7 +1492,7 @@ void i915_hangcheck_elapsed(unsigned long data)
1399 dev_priv->last_instdone1 = instdone1; 1492 dev_priv->last_instdone1 = instdone1;
1400 } 1493 }
1401 1494
1402out: 1495repeat:
1403 /* Reset timer case chip hangs without another request being added */ 1496 /* Reset timer case chip hangs without another request being added */
1404 mod_timer(&dev_priv->hangcheck_timer, 1497 mod_timer(&dev_priv->hangcheck_timer,
1405 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); 1498 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
@@ -1417,17 +1510,17 @@ static void ironlake_irq_preinstall(struct drm_device *dev)
1417 1510
1418 I915_WRITE(DEIMR, 0xffffffff); 1511 I915_WRITE(DEIMR, 0xffffffff);
1419 I915_WRITE(DEIER, 0x0); 1512 I915_WRITE(DEIER, 0x0);
1420 (void) I915_READ(DEIER); 1513 POSTING_READ(DEIER);
1421 1514
1422 /* and GT */ 1515 /* and GT */
1423 I915_WRITE(GTIMR, 0xffffffff); 1516 I915_WRITE(GTIMR, 0xffffffff);
1424 I915_WRITE(GTIER, 0x0); 1517 I915_WRITE(GTIER, 0x0);
1425 (void) I915_READ(GTIER); 1518 POSTING_READ(GTIER);
1426 1519
1427 /* south display irq */ 1520 /* south display irq */
1428 I915_WRITE(SDEIMR, 0xffffffff); 1521 I915_WRITE(SDEIMR, 0xffffffff);
1429 I915_WRITE(SDEIER, 0x0); 1522 I915_WRITE(SDEIER, 0x0);
1430 (void) I915_READ(SDEIER); 1523 POSTING_READ(SDEIER);
1431} 1524}
1432 1525
1433static int ironlake_irq_postinstall(struct drm_device *dev) 1526static int ironlake_irq_postinstall(struct drm_device *dev)
@@ -1436,38 +1529,38 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
1436 /* enable kind of interrupts always enabled */ 1529 /* enable kind of interrupts always enabled */
1437 u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | 1530 u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
1438 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE; 1531 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
1439 u32 render_mask = GT_PIPE_NOTIFY | GT_BSD_USER_INTERRUPT; 1532 u32 render_irqs;
1440 u32 hotplug_mask; 1533 u32 hotplug_mask;
1441 1534
1442 dev_priv->irq_mask_reg = ~display_mask; 1535 dev_priv->irq_mask = ~display_mask;
1443 dev_priv->de_irq_enable_reg = display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK;
1444 1536
1445 /* should always can generate irq */ 1537 /* should always can generate irq */
1446 I915_WRITE(DEIIR, I915_READ(DEIIR)); 1538 I915_WRITE(DEIIR, I915_READ(DEIIR));
1447 I915_WRITE(DEIMR, dev_priv->irq_mask_reg); 1539 I915_WRITE(DEIMR, dev_priv->irq_mask);
1448 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg); 1540 I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
1449 (void) I915_READ(DEIER); 1541 POSTING_READ(DEIER);
1450
1451 if (IS_GEN6(dev)) {
1452 render_mask =
1453 GT_PIPE_NOTIFY |
1454 GT_GEN6_BSD_USER_INTERRUPT |
1455 GT_BLT_USER_INTERRUPT;
1456 }
1457 1542
1458 dev_priv->gt_irq_mask_reg = ~render_mask; 1543 dev_priv->gt_irq_mask = ~0;
1459 dev_priv->gt_irq_enable_reg = render_mask;
1460 1544
1461 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1545 I915_WRITE(GTIIR, I915_READ(GTIIR));
1462 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); 1546 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1463 if (IS_GEN6(dev)) { 1547 if (IS_GEN6(dev)) {
1464 I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT); 1548 I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_USER_INTERRUPT);
1465 I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_IMR_USER_INTERRUPT); 1549 I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_USER_INTERRUPT);
1466 I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT); 1550 I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT);
1467 } 1551 }
1468 1552
1469 I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg); 1553 if (IS_GEN6(dev))
1470 (void) I915_READ(GTIER); 1554 render_irqs =
1555 GT_USER_INTERRUPT |
1556 GT_GEN6_BSD_USER_INTERRUPT |
1557 GT_BLT_USER_INTERRUPT;
1558 else
1559 render_irqs =
1560 GT_USER_INTERRUPT |
1561 GT_BSD_USER_INTERRUPT;
1562 I915_WRITE(GTIER, render_irqs);
1563 POSTING_READ(GTIER);
1471 1564
1472 if (HAS_PCH_CPT(dev)) { 1565 if (HAS_PCH_CPT(dev)) {
1473 hotplug_mask = SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT | 1566 hotplug_mask = SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT |
@@ -1477,13 +1570,12 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
1477 SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG; 1570 SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG;
1478 } 1571 }
1479 1572
1480 dev_priv->pch_irq_mask_reg = ~hotplug_mask; 1573 dev_priv->pch_irq_mask = ~hotplug_mask;
1481 dev_priv->pch_irq_enable_reg = hotplug_mask;
1482 1574
1483 I915_WRITE(SDEIIR, I915_READ(SDEIIR)); 1575 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
1484 I915_WRITE(SDEIMR, dev_priv->pch_irq_mask_reg); 1576 I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
1485 I915_WRITE(SDEIER, dev_priv->pch_irq_enable_reg); 1577 I915_WRITE(SDEIER, hotplug_mask);
1486 (void) I915_READ(SDEIER); 1578 POSTING_READ(SDEIER);
1487 1579
1488 if (IS_IRONLAKE_M(dev)) { 1580 if (IS_IRONLAKE_M(dev)) {
1489 /* Clear & enable PCU event interrupts */ 1581 /* Clear & enable PCU event interrupts */
@@ -1519,7 +1611,7 @@ void i915_driver_irq_preinstall(struct drm_device * dev)
1519 I915_WRITE(PIPEBSTAT, 0); 1611 I915_WRITE(PIPEBSTAT, 0);
1520 I915_WRITE(IMR, 0xffffffff); 1612 I915_WRITE(IMR, 0xffffffff);
1521 I915_WRITE(IER, 0x0); 1613 I915_WRITE(IER, 0x0);
1522 (void) I915_READ(IER); 1614 POSTING_READ(IER);
1523} 1615}
1524 1616
1525/* 1617/*
@@ -1532,11 +1624,11 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1532 u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR; 1624 u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
1533 u32 error_mask; 1625 u32 error_mask;
1534 1626
1535 DRM_INIT_WAITQUEUE(&dev_priv->render_ring.irq_queue); 1627 DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
1536 if (HAS_BSD(dev)) 1628 if (HAS_BSD(dev))
1537 DRM_INIT_WAITQUEUE(&dev_priv->bsd_ring.irq_queue); 1629 DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
1538 if (HAS_BLT(dev)) 1630 if (HAS_BLT(dev))
1539 DRM_INIT_WAITQUEUE(&dev_priv->blt_ring.irq_queue); 1631 DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);
1540 1632
1541 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; 1633 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
1542 1634
@@ -1544,7 +1636,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1544 return ironlake_irq_postinstall(dev); 1636 return ironlake_irq_postinstall(dev);
1545 1637
1546 /* Unmask the interrupts that we always want on. */ 1638 /* Unmask the interrupts that we always want on. */
1547 dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX; 1639 dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
1548 1640
1549 dev_priv->pipestat[0] = 0; 1641 dev_priv->pipestat[0] = 0;
1550 dev_priv->pipestat[1] = 0; 1642 dev_priv->pipestat[1] = 0;
@@ -1553,7 +1645,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1553 /* Enable in IER... */ 1645 /* Enable in IER... */
1554 enable_mask |= I915_DISPLAY_PORT_INTERRUPT; 1646 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
1555 /* and unmask in IMR */ 1647 /* and unmask in IMR */
1556 dev_priv->irq_mask_reg &= ~I915_DISPLAY_PORT_INTERRUPT; 1648 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1557 } 1649 }
1558 1650
1559 /* 1651 /*
@@ -1571,9 +1663,9 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1571 } 1663 }
1572 I915_WRITE(EMR, error_mask); 1664 I915_WRITE(EMR, error_mask);
1573 1665
1574 I915_WRITE(IMR, dev_priv->irq_mask_reg); 1666 I915_WRITE(IMR, dev_priv->irq_mask);
1575 I915_WRITE(IER, enable_mask); 1667 I915_WRITE(IER, enable_mask);
1576 (void) I915_READ(IER); 1668 POSTING_READ(IER);
1577 1669
1578 if (I915_HAS_HOTPLUG(dev)) { 1670 if (I915_HAS_HOTPLUG(dev)) {
1579 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); 1671 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 878fc766a12c..b284c13fd632 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -78,6 +78,12 @@
78#define GRDOM_RENDER (1<<2) 78#define GRDOM_RENDER (1<<2)
79#define GRDOM_MEDIA (3<<2) 79#define GRDOM_MEDIA (3<<2)
80 80
81#define GEN6_GDRST 0x941c
82#define GEN6_GRDOM_FULL (1 << 0)
83#define GEN6_GRDOM_RENDER (1 << 1)
84#define GEN6_GRDOM_MEDIA (1 << 2)
85#define GEN6_GRDOM_BLT (1 << 3)
86
81/* VGA stuff */ 87/* VGA stuff */
82 88
83#define VGA_ST01_MDA 0x3ba 89#define VGA_ST01_MDA 0x3ba
@@ -158,12 +164,23 @@
158#define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */ 164#define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */
159#define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) 165#define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1)
160#define MI_STORE_DWORD_INDEX_SHIFT 2 166#define MI_STORE_DWORD_INDEX_SHIFT 2
161#define MI_LOAD_REGISTER_IMM MI_INSTR(0x22, 1) 167/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
168 * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
169 * simply ignores the register load under certain conditions.
170 * - One can actually load arbitrary many arbitrary registers: Simply issue x
171 * address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
172 */
173#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1)
162#define MI_FLUSH_DW MI_INSTR(0x26, 2) /* for GEN6 */ 174#define MI_FLUSH_DW MI_INSTR(0x26, 2) /* for GEN6 */
163#define MI_BATCH_BUFFER MI_INSTR(0x30, 1) 175#define MI_BATCH_BUFFER MI_INSTR(0x30, 1)
164#define MI_BATCH_NON_SECURE (1) 176#define MI_BATCH_NON_SECURE (1)
165#define MI_BATCH_NON_SECURE_I965 (1<<8) 177#define MI_BATCH_NON_SECURE_I965 (1<<8)
166#define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0) 178#define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0)
179#define MI_SEMAPHORE_MBOX MI_INSTR(0x16, 1) /* gen6+ */
180#define MI_SEMAPHORE_GLOBAL_GTT (1<<22)
181#define MI_SEMAPHORE_UPDATE (1<<21)
182#define MI_SEMAPHORE_COMPARE (1<<20)
183#define MI_SEMAPHORE_REGISTER (1<<18)
167/* 184/*
168 * 3D instructions used by the kernel 185 * 3D instructions used by the kernel
169 */ 186 */
@@ -256,10 +273,6 @@
256 * Instruction and interrupt control regs 273 * Instruction and interrupt control regs
257 */ 274 */
258#define PGTBL_ER 0x02024 275#define PGTBL_ER 0x02024
259#define PRB0_TAIL 0x02030
260#define PRB0_HEAD 0x02034
261#define PRB0_START 0x02038
262#define PRB0_CTL 0x0203c
263#define RENDER_RING_BASE 0x02000 276#define RENDER_RING_BASE 0x02000
264#define BSD_RING_BASE 0x04000 277#define BSD_RING_BASE 0x04000
265#define GEN6_BSD_RING_BASE 0x12000 278#define GEN6_BSD_RING_BASE 0x12000
@@ -268,9 +281,13 @@
268#define RING_HEAD(base) ((base)+0x34) 281#define RING_HEAD(base) ((base)+0x34)
269#define RING_START(base) ((base)+0x38) 282#define RING_START(base) ((base)+0x38)
270#define RING_CTL(base) ((base)+0x3c) 283#define RING_CTL(base) ((base)+0x3c)
284#define RING_SYNC_0(base) ((base)+0x40)
285#define RING_SYNC_1(base) ((base)+0x44)
286#define RING_MAX_IDLE(base) ((base)+0x54)
271#define RING_HWS_PGA(base) ((base)+0x80) 287#define RING_HWS_PGA(base) ((base)+0x80)
272#define RING_HWS_PGA_GEN6(base) ((base)+0x2080) 288#define RING_HWS_PGA_GEN6(base) ((base)+0x2080)
273#define RING_ACTHD(base) ((base)+0x74) 289#define RING_ACTHD(base) ((base)+0x74)
290#define RING_NOPID(base) ((base)+0x94)
274#define TAIL_ADDR 0x001FFFF8 291#define TAIL_ADDR 0x001FFFF8
275#define HEAD_WRAP_COUNT 0xFFE00000 292#define HEAD_WRAP_COUNT 0xFFE00000
276#define HEAD_WRAP_ONE 0x00200000 293#define HEAD_WRAP_ONE 0x00200000
@@ -285,10 +302,17 @@
285#define RING_INVALID 0x00000000 302#define RING_INVALID 0x00000000
286#define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */ 303#define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */
287#define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */ 304#define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */
305#define RING_WAIT_SEMAPHORE (1<<10) /* gen6+ */
306#if 0
307#define PRB0_TAIL 0x02030
308#define PRB0_HEAD 0x02034
309#define PRB0_START 0x02038
310#define PRB0_CTL 0x0203c
288#define PRB1_TAIL 0x02040 /* 915+ only */ 311#define PRB1_TAIL 0x02040 /* 915+ only */
289#define PRB1_HEAD 0x02044 /* 915+ only */ 312#define PRB1_HEAD 0x02044 /* 915+ only */
290#define PRB1_START 0x02048 /* 915+ only */ 313#define PRB1_START 0x02048 /* 915+ only */
291#define PRB1_CTL 0x0204c /* 915+ only */ 314#define PRB1_CTL 0x0204c /* 915+ only */
315#endif
292#define IPEIR_I965 0x02064 316#define IPEIR_I965 0x02064
293#define IPEHR_I965 0x02068 317#define IPEHR_I965 0x02068
294#define INSTDONE_I965 0x0206c 318#define INSTDONE_I965 0x0206c
@@ -305,11 +329,42 @@
305#define INSTDONE 0x02090 329#define INSTDONE 0x02090
306#define NOPID 0x02094 330#define NOPID 0x02094
307#define HWSTAM 0x02098 331#define HWSTAM 0x02098
332#define VCS_INSTDONE 0x1206C
333#define VCS_IPEIR 0x12064
334#define VCS_IPEHR 0x12068
335#define VCS_ACTHD 0x12074
336#define BCS_INSTDONE 0x2206C
337#define BCS_IPEIR 0x22064
338#define BCS_IPEHR 0x22068
339#define BCS_ACTHD 0x22074
340
341#define ERROR_GEN6 0x040a0
342
343/* GM45+ chicken bits -- debug workaround bits that may be required
344 * for various sorts of correct behavior. The top 16 bits of each are
345 * the enables for writing to the corresponding low bit.
346 */
347#define _3D_CHICKEN 0x02084
348#define _3D_CHICKEN2 0x0208c
349/* Disables pipelining of read flushes past the SF-WIZ interface.
350 * Required on all Ironlake steppings according to the B-Spec, but the
351 * particular danger of not doing so is not specified.
352 */
353# define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14)
354#define _3D_CHICKEN3 0x02090
308 355
309#define MI_MODE 0x0209c 356#define MI_MODE 0x0209c
310# define VS_TIMER_DISPATCH (1 << 6) 357# define VS_TIMER_DISPATCH (1 << 6)
311# define MI_FLUSH_ENABLE (1 << 11) 358# define MI_FLUSH_ENABLE (1 << 11)
312 359
360#define GFX_MODE 0x02520
361#define GFX_RUN_LIST_ENABLE (1<<15)
362#define GFX_TLB_INVALIDATE_ALWAYS (1<<13)
363#define GFX_SURFACE_FAULT_ENABLE (1<<12)
364#define GFX_REPLAY_MODE (1<<11)
365#define GFX_PSMI_GRANULARITY (1<<10)
366#define GFX_PPGTT_ENABLE (1<<9)
367
313#define SCPD0 0x0209c /* 915+ only */ 368#define SCPD0 0x0209c /* 915+ only */
314#define IER 0x020a0 369#define IER 0x020a0
315#define IIR 0x020a4 370#define IIR 0x020a4
@@ -461,7 +516,7 @@
461#define GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR (1 << 3) 516#define GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR (1 << 3)
462 517
463#define GEN6_BSD_IMR 0x120a8 518#define GEN6_BSD_IMR 0x120a8
464#define GEN6_BSD_IMR_USER_INTERRUPT (1 << 12) 519#define GEN6_BSD_USER_INTERRUPT (1 << 12)
465 520
466#define GEN6_BSD_RNCID 0x12198 521#define GEN6_BSD_RNCID 0x12198
467 522
@@ -1168,7 +1223,6 @@
1168#define VTOTAL(pipe) _PIPE(pipe, VTOTAL_A, VTOTAL_B) 1223#define VTOTAL(pipe) _PIPE(pipe, VTOTAL_A, VTOTAL_B)
1169#define VBLANK(pipe) _PIPE(pipe, VBLANK_A, VBLANK_B) 1224#define VBLANK(pipe) _PIPE(pipe, VBLANK_A, VBLANK_B)
1170#define VSYNC(pipe) _PIPE(pipe, VSYNC_A, VSYNC_B) 1225#define VSYNC(pipe) _PIPE(pipe, VSYNC_A, VSYNC_B)
1171#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC)
1172#define BCLRPAT(pipe) _PIPE(pipe, BCLRPAT_A, BCLRPAT_B) 1226#define BCLRPAT(pipe) _PIPE(pipe, BCLRPAT_A, BCLRPAT_B)
1173 1227
1174/* VGA port control */ 1228/* VGA port control */
@@ -2182,6 +2236,7 @@
2182#define PIPE_6BPC (2 << 5) 2236#define PIPE_6BPC (2 << 5)
2183#define PIPE_12BPC (3 << 5) 2237#define PIPE_12BPC (3 << 5)
2184 2238
2239#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC)
2185#define PIPECONF(pipe) _PIPE(pipe, PIPEACONF, PIPEBCONF) 2240#define PIPECONF(pipe) _PIPE(pipe, PIPEACONF, PIPEBCONF)
2186#define PIPEDSL(pipe) _PIPE(pipe, PIPEADSL, PIPEBDSL) 2241#define PIPEDSL(pipe) _PIPE(pipe, PIPEADSL, PIPEBDSL)
2187 2242
@@ -2351,6 +2406,10 @@
2351#define CURBBASE 0x700c4 2406#define CURBBASE 0x700c4
2352#define CURBPOS 0x700c8 2407#define CURBPOS 0x700c8
2353 2408
2409#define CURCNTR(pipe) _PIPE(pipe, CURACNTR, CURBCNTR)
2410#define CURBASE(pipe) _PIPE(pipe, CURABASE, CURBBASE)
2411#define CURPOS(pipe) _PIPE(pipe, CURAPOS, CURBPOS)
2412
2354/* Display A control */ 2413/* Display A control */
2355#define DSPACNTR 0x70180 2414#define DSPACNTR 0x70180
2356#define DISPLAY_PLANE_ENABLE (1<<31) 2415#define DISPLAY_PLANE_ENABLE (1<<31)
@@ -2586,6 +2645,8 @@
2586#define GTIER 0x4401c 2645#define GTIER 0x4401c
2587 2646
2588#define ILK_DISPLAY_CHICKEN2 0x42004 2647#define ILK_DISPLAY_CHICKEN2 0x42004
2648/* Required on all Ironlake and Sandybridge according to the B-Spec. */
2649#define ILK_ELPIN_409_SELECT (1 << 25)
2589#define ILK_DPARB_GATE (1<<22) 2650#define ILK_DPARB_GATE (1<<22)
2590#define ILK_VSDPFD_FULL (1<<21) 2651#define ILK_VSDPFD_FULL (1<<21)
2591#define ILK_DSPCLK_GATE 0x42020 2652#define ILK_DSPCLK_GATE 0x42020
@@ -2669,6 +2730,7 @@
2669#define PCH_DPLL(pipe) _PIPE(pipe, PCH_DPLL_A, PCH_DPLL_B) 2730#define PCH_DPLL(pipe) _PIPE(pipe, PCH_DPLL_A, PCH_DPLL_B)
2670 2731
2671#define PCH_FPA0 0xc6040 2732#define PCH_FPA0 0xc6040
2733#define FP_CB_TUNE (0x3<<22)
2672#define PCH_FPA1 0xc6044 2734#define PCH_FPA1 0xc6044
2673#define PCH_FPB0 0xc6048 2735#define PCH_FPB0 0xc6048
2674#define PCH_FPB1 0xc604c 2736#define PCH_FPB1 0xc604c
@@ -3053,4 +3115,64 @@
3053#define EDP_LINK_TRAIN_800MV_0DB_SNB_B (0x38<<22) 3115#define EDP_LINK_TRAIN_800MV_0DB_SNB_B (0x38<<22)
3054#define EDP_LINK_TRAIN_VOL_EMP_MASK_SNB (0x3f<<22) 3116#define EDP_LINK_TRAIN_VOL_EMP_MASK_SNB (0x3f<<22)
3055 3117
3118#define FORCEWAKE 0xA18C
3119#define FORCEWAKE_ACK 0x130090
3120
3121#define GEN6_RC_NORMAL_FREQ 0xA008
3122#define GEN6_TURBO_DISABLE (1<<31)
3123#define GEN6_FREQUENCY(x) ((x)<<25)
3124#define GEN6_OFFSET(x) ((x)<<19)
3125#define GEN6_AGGRESSIVE_TURBO (0<<15)
3126#define GEN6_RC_VIDEO_FREQ 0xA00C
3127#define GEN6_RC_CONTROL 0xA090
3128#define GEN6_RC_CTL_RC6pp_ENABLE (1<<16)
3129#define GEN6_RC_CTL_RC6p_ENABLE (1<<17)
3130#define GEN6_RC_CTL_RC6_ENABLE (1<<18)
3131#define GEN6_RC_CTL_RC1e_ENABLE (1<<20)
3132#define GEN6_RC_CTL_RC7_ENABLE (1<<22)
3133#define GEN6_RC_CTL_EI_MODE(x) ((x)<<27)
3134#define GEN6_RC_CTL_HW_ENABLE (1<<31)
3135#define GEN6_RP_DOWN_TIMEOUT 0xA010
3136#define GEN6_RP_INTERRUPT_LIMITS 0xA014
3137#define GEN6_RP_CONTROL 0xA024
3138#define GEN6_RP_MEDIA_TURBO (1<<11)
3139#define GEN6_RP_USE_NORMAL_FREQ (1<<9)
3140#define GEN6_RP_MEDIA_IS_GFX (1<<8)
3141#define GEN6_RP_ENABLE (1<<7)
3142#define GEN6_RP_UP_BUSY_MAX (0x2<<3)
3143#define GEN6_RP_DOWN_BUSY_MIN (0x2<<0)
3144#define GEN6_RP_UP_THRESHOLD 0xA02C
3145#define GEN6_RP_DOWN_THRESHOLD 0xA030
3146#define GEN6_RP_UP_EI 0xA068
3147#define GEN6_RP_DOWN_EI 0xA06C
3148#define GEN6_RP_IDLE_HYSTERSIS 0xA070
3149#define GEN6_RC_STATE 0xA094
3150#define GEN6_RC1_WAKE_RATE_LIMIT 0xA098
3151#define GEN6_RC6_WAKE_RATE_LIMIT 0xA09C
3152#define GEN6_RC6pp_WAKE_RATE_LIMIT 0xA0A0
3153#define GEN6_RC_EVALUATION_INTERVAL 0xA0A8
3154#define GEN6_RC_IDLE_HYSTERSIS 0xA0AC
3155#define GEN6_RC_SLEEP 0xA0B0
3156#define GEN6_RC1e_THRESHOLD 0xA0B4
3157#define GEN6_RC6_THRESHOLD 0xA0B8
3158#define GEN6_RC6p_THRESHOLD 0xA0BC
3159#define GEN6_RC6pp_THRESHOLD 0xA0C0
3160
3161#define GEN6_PMISR 0x44020
3162#define GEN6_PMIMR 0x44024
3163#define GEN6_PMIIR 0x44028
3164#define GEN6_PMIER 0x4402C
3165#define GEN6_PM_MBOX_EVENT (1<<25)
3166#define GEN6_PM_THERMAL_EVENT (1<<24)
3167#define GEN6_PM_RP_DOWN_TIMEOUT (1<<6)
3168#define GEN6_PM_RP_UP_THRESHOLD (1<<5)
3169#define GEN6_PM_RP_DOWN_THRESHOLD (1<<4)
3170#define GEN6_PM_RP_UP_EI_EXPIRED (1<<2)
3171#define GEN6_PM_RP_DOWN_EI_EXPIRED (1<<1)
3172
3173#define GEN6_PCODE_MAILBOX 0x138124
3174#define GEN6_PCODE_READY (1<<31)
3175#define GEN6_PCODE_WRITE_MIN_FREQ_TABLE 0x9
3176#define GEN6_PCODE_DATA 0x138128
3177
3056#endif /* _I915_REG_H_ */ 3178#endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 42729d25da58..a311809f3c80 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -235,6 +235,7 @@ static void i915_restore_vga(struct drm_device *dev)
235static void i915_save_modeset_reg(struct drm_device *dev) 235static void i915_save_modeset_reg(struct drm_device *dev)
236{ 236{
237 struct drm_i915_private *dev_priv = dev->dev_private; 237 struct drm_i915_private *dev_priv = dev->dev_private;
238 int i;
238 239
239 if (drm_core_check_feature(dev, DRIVER_MODESET)) 240 if (drm_core_check_feature(dev, DRIVER_MODESET))
240 return; 241 return;
@@ -367,6 +368,28 @@ static void i915_save_modeset_reg(struct drm_device *dev)
367 } 368 }
368 i915_save_palette(dev, PIPE_B); 369 i915_save_palette(dev, PIPE_B);
369 dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT); 370 dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
371
372 /* Fences */
373 switch (INTEL_INFO(dev)->gen) {
374 case 6:
375 for (i = 0; i < 16; i++)
376 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
377 break;
378 case 5:
379 case 4:
380 for (i = 0; i < 16; i++)
381 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
382 break;
383 case 3:
384 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
385 for (i = 0; i < 8; i++)
386 dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
387 case 2:
388 for (i = 0; i < 8; i++)
389 dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
390 break;
391 }
392
370 return; 393 return;
371} 394}
372 395
@@ -375,10 +398,33 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
375 struct drm_i915_private *dev_priv = dev->dev_private; 398 struct drm_i915_private *dev_priv = dev->dev_private;
376 int dpll_a_reg, fpa0_reg, fpa1_reg; 399 int dpll_a_reg, fpa0_reg, fpa1_reg;
377 int dpll_b_reg, fpb0_reg, fpb1_reg; 400 int dpll_b_reg, fpb0_reg, fpb1_reg;
401 int i;
378 402
379 if (drm_core_check_feature(dev, DRIVER_MODESET)) 403 if (drm_core_check_feature(dev, DRIVER_MODESET))
380 return; 404 return;
381 405
406 /* Fences */
407 switch (INTEL_INFO(dev)->gen) {
408 case 6:
409 for (i = 0; i < 16; i++)
410 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
411 break;
412 case 5:
413 case 4:
414 for (i = 0; i < 16; i++)
415 I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
416 break;
417 case 3:
418 case 2:
419 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
420 for (i = 0; i < 8; i++)
421 I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
422 for (i = 0; i < 8; i++)
423 I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
424 break;
425 }
426
427
382 if (HAS_PCH_SPLIT(dev)) { 428 if (HAS_PCH_SPLIT(dev)) {
383 dpll_a_reg = PCH_DPLL_A; 429 dpll_a_reg = PCH_DPLL_A;
384 dpll_b_reg = PCH_DPLL_B; 430 dpll_b_reg = PCH_DPLL_B;
@@ -774,6 +820,8 @@ int i915_save_state(struct drm_device *dev)
774 if (HAS_PCH_SPLIT(dev)) 820 if (HAS_PCH_SPLIT(dev))
775 ironlake_disable_drps(dev); 821 ironlake_disable_drps(dev);
776 822
823 intel_disable_clock_gating(dev);
824
777 /* Cache mode state */ 825 /* Cache mode state */
778 dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); 826 dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
779 827
@@ -788,28 +836,6 @@ int i915_save_state(struct drm_device *dev)
788 for (i = 0; i < 3; i++) 836 for (i = 0; i < 3; i++)
789 dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2)); 837 dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
790 838
791 /* Fences */
792 switch (INTEL_INFO(dev)->gen) {
793 case 6:
794 for (i = 0; i < 16; i++)
795 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
796 break;
797 case 5:
798 case 4:
799 for (i = 0; i < 16; i++)
800 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
801 break;
802 case 3:
803 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
804 for (i = 0; i < 8; i++)
805 dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
806 case 2:
807 for (i = 0; i < 8; i++)
808 dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
809 break;
810
811 }
812
813 return 0; 839 return 0;
814} 840}
815 841
@@ -823,27 +849,6 @@ int i915_restore_state(struct drm_device *dev)
823 /* Hardware status page */ 849 /* Hardware status page */
824 I915_WRITE(HWS_PGA, dev_priv->saveHWS); 850 I915_WRITE(HWS_PGA, dev_priv->saveHWS);
825 851
826 /* Fences */
827 switch (INTEL_INFO(dev)->gen) {
828 case 6:
829 for (i = 0; i < 16; i++)
830 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
831 break;
832 case 5:
833 case 4:
834 for (i = 0; i < 16; i++)
835 I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
836 break;
837 case 3:
838 case 2:
839 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
840 for (i = 0; i < 8; i++)
841 I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
842 for (i = 0; i < 8; i++)
843 I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
844 break;
845 }
846
847 i915_restore_display(dev); 852 i915_restore_display(dev);
848 853
849 /* Interrupt state */ 854 /* Interrupt state */
@@ -860,7 +865,7 @@ int i915_restore_state(struct drm_device *dev)
860 } 865 }
861 866
862 /* Clock gating state */ 867 /* Clock gating state */
863 intel_init_clock_gating(dev); 868 intel_enable_clock_gating(dev);
864 869
865 if (HAS_PCH_SPLIT(dev)) { 870 if (HAS_PCH_SPLIT(dev)) {
866 ironlake_enable_drps(dev); 871 ironlake_enable_drps(dev);
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index fea97a21cc14..7f0fc3ed61aa 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -6,6 +6,7 @@
6#include <linux/tracepoint.h> 6#include <linux/tracepoint.h>
7 7
8#include <drm/drmP.h> 8#include <drm/drmP.h>
9#include "i915_drv.h"
9 10
10#undef TRACE_SYSTEM 11#undef TRACE_SYSTEM
11#define TRACE_SYSTEM i915 12#define TRACE_SYSTEM i915
@@ -16,18 +17,18 @@
16 17
17TRACE_EVENT(i915_gem_object_create, 18TRACE_EVENT(i915_gem_object_create,
18 19
19 TP_PROTO(struct drm_gem_object *obj), 20 TP_PROTO(struct drm_i915_gem_object *obj),
20 21
21 TP_ARGS(obj), 22 TP_ARGS(obj),
22 23
23 TP_STRUCT__entry( 24 TP_STRUCT__entry(
24 __field(struct drm_gem_object *, obj) 25 __field(struct drm_i915_gem_object *, obj)
25 __field(u32, size) 26 __field(u32, size)
26 ), 27 ),
27 28
28 TP_fast_assign( 29 TP_fast_assign(
29 __entry->obj = obj; 30 __entry->obj = obj;
30 __entry->size = obj->size; 31 __entry->size = obj->base.size;
31 ), 32 ),
32 33
33 TP_printk("obj=%p, size=%u", __entry->obj, __entry->size) 34 TP_printk("obj=%p, size=%u", __entry->obj, __entry->size)
@@ -35,40 +36,43 @@ TRACE_EVENT(i915_gem_object_create,
35 36
36TRACE_EVENT(i915_gem_object_bind, 37TRACE_EVENT(i915_gem_object_bind,
37 38
38 TP_PROTO(struct drm_gem_object *obj, u32 gtt_offset), 39 TP_PROTO(struct drm_i915_gem_object *obj, u32 gtt_offset, bool mappable),
39 40
40 TP_ARGS(obj, gtt_offset), 41 TP_ARGS(obj, gtt_offset, mappable),
41 42
42 TP_STRUCT__entry( 43 TP_STRUCT__entry(
43 __field(struct drm_gem_object *, obj) 44 __field(struct drm_i915_gem_object *, obj)
44 __field(u32, gtt_offset) 45 __field(u32, gtt_offset)
46 __field(bool, mappable)
45 ), 47 ),
46 48
47 TP_fast_assign( 49 TP_fast_assign(
48 __entry->obj = obj; 50 __entry->obj = obj;
49 __entry->gtt_offset = gtt_offset; 51 __entry->gtt_offset = gtt_offset;
52 __entry->mappable = mappable;
50 ), 53 ),
51 54
52 TP_printk("obj=%p, gtt_offset=%08x", 55 TP_printk("obj=%p, gtt_offset=%08x%s",
53 __entry->obj, __entry->gtt_offset) 56 __entry->obj, __entry->gtt_offset,
57 __entry->mappable ? ", mappable" : "")
54); 58);
55 59
56TRACE_EVENT(i915_gem_object_change_domain, 60TRACE_EVENT(i915_gem_object_change_domain,
57 61
58 TP_PROTO(struct drm_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain), 62 TP_PROTO(struct drm_i915_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain),
59 63
60 TP_ARGS(obj, old_read_domains, old_write_domain), 64 TP_ARGS(obj, old_read_domains, old_write_domain),
61 65
62 TP_STRUCT__entry( 66 TP_STRUCT__entry(
63 __field(struct drm_gem_object *, obj) 67 __field(struct drm_i915_gem_object *, obj)
64 __field(u32, read_domains) 68 __field(u32, read_domains)
65 __field(u32, write_domain) 69 __field(u32, write_domain)
66 ), 70 ),
67 71
68 TP_fast_assign( 72 TP_fast_assign(
69 __entry->obj = obj; 73 __entry->obj = obj;
70 __entry->read_domains = obj->read_domains | (old_read_domains << 16); 74 __entry->read_domains = obj->base.read_domains | (old_read_domains << 16);
71 __entry->write_domain = obj->write_domain | (old_write_domain << 16); 75 __entry->write_domain = obj->base.write_domain | (old_write_domain << 16);
72 ), 76 ),
73 77
74 TP_printk("obj=%p, read=%04x, write=%04x", 78 TP_printk("obj=%p, read=%04x, write=%04x",
@@ -76,36 +80,14 @@ TRACE_EVENT(i915_gem_object_change_domain,
76 __entry->read_domains, __entry->write_domain) 80 __entry->read_domains, __entry->write_domain)
77); 81);
78 82
79TRACE_EVENT(i915_gem_object_get_fence,
80
81 TP_PROTO(struct drm_gem_object *obj, int fence, int tiling_mode),
82
83 TP_ARGS(obj, fence, tiling_mode),
84
85 TP_STRUCT__entry(
86 __field(struct drm_gem_object *, obj)
87 __field(int, fence)
88 __field(int, tiling_mode)
89 ),
90
91 TP_fast_assign(
92 __entry->obj = obj;
93 __entry->fence = fence;
94 __entry->tiling_mode = tiling_mode;
95 ),
96
97 TP_printk("obj=%p, fence=%d, tiling=%d",
98 __entry->obj, __entry->fence, __entry->tiling_mode)
99);
100
101DECLARE_EVENT_CLASS(i915_gem_object, 83DECLARE_EVENT_CLASS(i915_gem_object,
102 84
103 TP_PROTO(struct drm_gem_object *obj), 85 TP_PROTO(struct drm_i915_gem_object *obj),
104 86
105 TP_ARGS(obj), 87 TP_ARGS(obj),
106 88
107 TP_STRUCT__entry( 89 TP_STRUCT__entry(
108 __field(struct drm_gem_object *, obj) 90 __field(struct drm_i915_gem_object *, obj)
109 ), 91 ),
110 92
111 TP_fast_assign( 93 TP_fast_assign(
@@ -117,21 +99,21 @@ DECLARE_EVENT_CLASS(i915_gem_object,
117 99
118DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush, 100DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush,
119 101
120 TP_PROTO(struct drm_gem_object *obj), 102 TP_PROTO(struct drm_i915_gem_object *obj),
121 103
122 TP_ARGS(obj) 104 TP_ARGS(obj)
123); 105);
124 106
125DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind, 107DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind,
126 108
127 TP_PROTO(struct drm_gem_object *obj), 109 TP_PROTO(struct drm_i915_gem_object *obj),
128 110
129 TP_ARGS(obj) 111 TP_ARGS(obj)
130); 112);
131 113
132DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy, 114DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
133 115
134 TP_PROTO(struct drm_gem_object *obj), 116 TP_PROTO(struct drm_i915_gem_object *obj),
135 117
136 TP_ARGS(obj) 118 TP_ARGS(obj)
137); 119);
@@ -263,13 +245,13 @@ DEFINE_EVENT(i915_ring, i915_ring_wait_end,
263); 245);
264 246
265TRACE_EVENT(i915_flip_request, 247TRACE_EVENT(i915_flip_request,
266 TP_PROTO(int plane, struct drm_gem_object *obj), 248 TP_PROTO(int plane, struct drm_i915_gem_object *obj),
267 249
268 TP_ARGS(plane, obj), 250 TP_ARGS(plane, obj),
269 251
270 TP_STRUCT__entry( 252 TP_STRUCT__entry(
271 __field(int, plane) 253 __field(int, plane)
272 __field(struct drm_gem_object *, obj) 254 __field(struct drm_i915_gem_object *, obj)
273 ), 255 ),
274 256
275 TP_fast_assign( 257 TP_fast_assign(
@@ -281,13 +263,13 @@ TRACE_EVENT(i915_flip_request,
281); 263);
282 264
283TRACE_EVENT(i915_flip_complete, 265TRACE_EVENT(i915_flip_complete,
284 TP_PROTO(int plane, struct drm_gem_object *obj), 266 TP_PROTO(int plane, struct drm_i915_gem_object *obj),
285 267
286 TP_ARGS(plane, obj), 268 TP_ARGS(plane, obj),
287 269
288 TP_STRUCT__entry( 270 TP_STRUCT__entry(
289 __field(int, plane) 271 __field(int, plane)
290 __field(struct drm_gem_object *, obj) 272 __field(struct drm_i915_gem_object *, obj)
291 ), 273 ),
292 274
293 TP_fast_assign( 275 TP_fast_assign(
@@ -298,6 +280,29 @@ TRACE_EVENT(i915_flip_complete,
298 TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj) 280 TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
299); 281);
300 282
283TRACE_EVENT(i915_reg_rw,
284 TP_PROTO(int cmd, uint32_t reg, uint64_t val, int len),
285
286 TP_ARGS(cmd, reg, val, len),
287
288 TP_STRUCT__entry(
289 __field(int, cmd)
290 __field(uint32_t, reg)
291 __field(uint64_t, val)
292 __field(int, len)
293 ),
294
295 TP_fast_assign(
296 __entry->cmd = cmd;
297 __entry->reg = reg;
298 __entry->val = (uint64_t)val;
299 __entry->len = len;
300 ),
301
302 TP_printk("cmd=%c, reg=0x%x, val=0x%llx, len=%d",
303 __entry->cmd, __entry->reg, __entry->val, __entry->len)
304);
305
301#endif /* _I915_TRACE_H_ */ 306#endif /* _I915_TRACE_H_ */
302 307
303/* This part must be outside protection */ 308/* This part must be outside protection */
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index d9b7092439ef..17c213fef0ec 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -1066,13 +1066,13 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1066 struct drm_i915_private *dev_priv = dev->dev_private; 1066 struct drm_i915_private *dev_priv = dev->dev_private;
1067 struct drm_framebuffer *fb = crtc->fb; 1067 struct drm_framebuffer *fb = crtc->fb;
1068 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 1068 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1069 struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); 1069 struct drm_i915_gem_object *obj = intel_fb->obj;
1070 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1070 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1071 int plane, i; 1071 int plane, i;
1072 u32 fbc_ctl, fbc_ctl2; 1072 u32 fbc_ctl, fbc_ctl2;
1073 1073
1074 if (fb->pitch == dev_priv->cfb_pitch && 1074 if (fb->pitch == dev_priv->cfb_pitch &&
1075 obj_priv->fence_reg == dev_priv->cfb_fence && 1075 obj->fence_reg == dev_priv->cfb_fence &&
1076 intel_crtc->plane == dev_priv->cfb_plane && 1076 intel_crtc->plane == dev_priv->cfb_plane &&
1077 I915_READ(FBC_CONTROL) & FBC_CTL_EN) 1077 I915_READ(FBC_CONTROL) & FBC_CTL_EN)
1078 return; 1078 return;
@@ -1086,7 +1086,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1086 1086
1087 /* FBC_CTL wants 64B units */ 1087 /* FBC_CTL wants 64B units */
1088 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; 1088 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
1089 dev_priv->cfb_fence = obj_priv->fence_reg; 1089 dev_priv->cfb_fence = obj->fence_reg;
1090 dev_priv->cfb_plane = intel_crtc->plane; 1090 dev_priv->cfb_plane = intel_crtc->plane;
1091 plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB; 1091 plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
1092 1092
@@ -1096,7 +1096,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1096 1096
1097 /* Set it up... */ 1097 /* Set it up... */
1098 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane; 1098 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
1099 if (obj_priv->tiling_mode != I915_TILING_NONE) 1099 if (obj->tiling_mode != I915_TILING_NONE)
1100 fbc_ctl2 |= FBC_CTL_CPU_FENCE; 1100 fbc_ctl2 |= FBC_CTL_CPU_FENCE;
1101 I915_WRITE(FBC_CONTROL2, fbc_ctl2); 1101 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
1102 I915_WRITE(FBC_FENCE_OFF, crtc->y); 1102 I915_WRITE(FBC_FENCE_OFF, crtc->y);
@@ -1107,7 +1107,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1107 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */ 1107 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
1108 fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT; 1108 fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
1109 fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT; 1109 fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
1110 if (obj_priv->tiling_mode != I915_TILING_NONE) 1110 if (obj->tiling_mode != I915_TILING_NONE)
1111 fbc_ctl |= dev_priv->cfb_fence; 1111 fbc_ctl |= dev_priv->cfb_fence;
1112 I915_WRITE(FBC_CONTROL, fbc_ctl); 1112 I915_WRITE(FBC_CONTROL, fbc_ctl);
1113 1113
@@ -1150,7 +1150,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1150 struct drm_i915_private *dev_priv = dev->dev_private; 1150 struct drm_i915_private *dev_priv = dev->dev_private;
1151 struct drm_framebuffer *fb = crtc->fb; 1151 struct drm_framebuffer *fb = crtc->fb;
1152 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 1152 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1153 struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); 1153 struct drm_i915_gem_object *obj = intel_fb->obj;
1154 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1154 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1155 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; 1155 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1156 unsigned long stall_watermark = 200; 1156 unsigned long stall_watermark = 200;
@@ -1159,7 +1159,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1159 dpfc_ctl = I915_READ(DPFC_CONTROL); 1159 dpfc_ctl = I915_READ(DPFC_CONTROL);
1160 if (dpfc_ctl & DPFC_CTL_EN) { 1160 if (dpfc_ctl & DPFC_CTL_EN) {
1161 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 && 1161 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
1162 dev_priv->cfb_fence == obj_priv->fence_reg && 1162 dev_priv->cfb_fence == obj->fence_reg &&
1163 dev_priv->cfb_plane == intel_crtc->plane && 1163 dev_priv->cfb_plane == intel_crtc->plane &&
1164 dev_priv->cfb_y == crtc->y) 1164 dev_priv->cfb_y == crtc->y)
1165 return; 1165 return;
@@ -1170,12 +1170,12 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1170 } 1170 }
1171 1171
1172 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; 1172 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
1173 dev_priv->cfb_fence = obj_priv->fence_reg; 1173 dev_priv->cfb_fence = obj->fence_reg;
1174 dev_priv->cfb_plane = intel_crtc->plane; 1174 dev_priv->cfb_plane = intel_crtc->plane;
1175 dev_priv->cfb_y = crtc->y; 1175 dev_priv->cfb_y = crtc->y;
1176 1176
1177 dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X; 1177 dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
1178 if (obj_priv->tiling_mode != I915_TILING_NONE) { 1178 if (obj->tiling_mode != I915_TILING_NONE) {
1179 dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence; 1179 dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
1180 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY); 1180 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
1181 } else { 1181 } else {
@@ -1221,7 +1221,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1221 struct drm_i915_private *dev_priv = dev->dev_private; 1221 struct drm_i915_private *dev_priv = dev->dev_private;
1222 struct drm_framebuffer *fb = crtc->fb; 1222 struct drm_framebuffer *fb = crtc->fb;
1223 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 1223 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1224 struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); 1224 struct drm_i915_gem_object *obj = intel_fb->obj;
1225 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1225 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1226 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; 1226 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1227 unsigned long stall_watermark = 200; 1227 unsigned long stall_watermark = 200;
@@ -1230,9 +1230,9 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1230 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); 1230 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
1231 if (dpfc_ctl & DPFC_CTL_EN) { 1231 if (dpfc_ctl & DPFC_CTL_EN) {
1232 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 && 1232 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
1233 dev_priv->cfb_fence == obj_priv->fence_reg && 1233 dev_priv->cfb_fence == obj->fence_reg &&
1234 dev_priv->cfb_plane == intel_crtc->plane && 1234 dev_priv->cfb_plane == intel_crtc->plane &&
1235 dev_priv->cfb_offset == obj_priv->gtt_offset && 1235 dev_priv->cfb_offset == obj->gtt_offset &&
1236 dev_priv->cfb_y == crtc->y) 1236 dev_priv->cfb_y == crtc->y)
1237 return; 1237 return;
1238 1238
@@ -1242,14 +1242,14 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1242 } 1242 }
1243 1243
1244 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; 1244 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
1245 dev_priv->cfb_fence = obj_priv->fence_reg; 1245 dev_priv->cfb_fence = obj->fence_reg;
1246 dev_priv->cfb_plane = intel_crtc->plane; 1246 dev_priv->cfb_plane = intel_crtc->plane;
1247 dev_priv->cfb_offset = obj_priv->gtt_offset; 1247 dev_priv->cfb_offset = obj->gtt_offset;
1248 dev_priv->cfb_y = crtc->y; 1248 dev_priv->cfb_y = crtc->y;
1249 1249
1250 dpfc_ctl &= DPFC_RESERVED; 1250 dpfc_ctl &= DPFC_RESERVED;
1251 dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X); 1251 dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
1252 if (obj_priv->tiling_mode != I915_TILING_NONE) { 1252 if (obj->tiling_mode != I915_TILING_NONE) {
1253 dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence); 1253 dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence);
1254 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY); 1254 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
1255 } else { 1255 } else {
@@ -1260,7 +1260,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1260 (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) | 1260 (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
1261 (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT)); 1261 (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
1262 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); 1262 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
1263 I915_WRITE(ILK_FBC_RT_BASE, obj_priv->gtt_offset | ILK_FBC_RT_VALID); 1263 I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
1264 /* enable it... */ 1264 /* enable it... */
1265 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); 1265 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
1266 1266
@@ -1345,7 +1345,7 @@ static void intel_update_fbc(struct drm_device *dev)
1345 struct intel_crtc *intel_crtc; 1345 struct intel_crtc *intel_crtc;
1346 struct drm_framebuffer *fb; 1346 struct drm_framebuffer *fb;
1347 struct intel_framebuffer *intel_fb; 1347 struct intel_framebuffer *intel_fb;
1348 struct drm_i915_gem_object *obj_priv; 1348 struct drm_i915_gem_object *obj;
1349 1349
1350 DRM_DEBUG_KMS("\n"); 1350 DRM_DEBUG_KMS("\n");
1351 1351
@@ -1384,9 +1384,9 @@ static void intel_update_fbc(struct drm_device *dev)
1384 intel_crtc = to_intel_crtc(crtc); 1384 intel_crtc = to_intel_crtc(crtc);
1385 fb = crtc->fb; 1385 fb = crtc->fb;
1386 intel_fb = to_intel_framebuffer(fb); 1386 intel_fb = to_intel_framebuffer(fb);
1387 obj_priv = to_intel_bo(intel_fb->obj); 1387 obj = intel_fb->obj;
1388 1388
1389 if (intel_fb->obj->size > dev_priv->cfb_size) { 1389 if (intel_fb->obj->base.size > dev_priv->cfb_size) {
1390 DRM_DEBUG_KMS("framebuffer too large, disabling " 1390 DRM_DEBUG_KMS("framebuffer too large, disabling "
1391 "compression\n"); 1391 "compression\n");
1392 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; 1392 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
@@ -1410,7 +1410,7 @@ static void intel_update_fbc(struct drm_device *dev)
1410 dev_priv->no_fbc_reason = FBC_BAD_PLANE; 1410 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
1411 goto out_disable; 1411 goto out_disable;
1412 } 1412 }
1413 if (obj_priv->tiling_mode != I915_TILING_X) { 1413 if (obj->tiling_mode != I915_TILING_X) {
1414 DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n"); 1414 DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n");
1415 dev_priv->no_fbc_reason = FBC_NOT_TILED; 1415 dev_priv->no_fbc_reason = FBC_NOT_TILED;
1416 goto out_disable; 1416 goto out_disable;
@@ -1433,14 +1433,13 @@ out_disable:
1433 1433
1434int 1434int
1435intel_pin_and_fence_fb_obj(struct drm_device *dev, 1435intel_pin_and_fence_fb_obj(struct drm_device *dev,
1436 struct drm_gem_object *obj, 1436 struct drm_i915_gem_object *obj,
1437 bool pipelined) 1437 struct intel_ring_buffer *pipelined)
1438{ 1438{
1439 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1440 u32 alignment; 1439 u32 alignment;
1441 int ret; 1440 int ret;
1442 1441
1443 switch (obj_priv->tiling_mode) { 1442 switch (obj->tiling_mode) {
1444 case I915_TILING_NONE: 1443 case I915_TILING_NONE:
1445 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) 1444 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
1446 alignment = 128 * 1024; 1445 alignment = 128 * 1024;
@@ -1461,7 +1460,7 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
1461 BUG(); 1460 BUG();
1462 } 1461 }
1463 1462
1464 ret = i915_gem_object_pin(obj, alignment); 1463 ret = i915_gem_object_pin(obj, alignment, true);
1465 if (ret) 1464 if (ret)
1466 return ret; 1465 return ret;
1467 1466
@@ -1474,9 +1473,8 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
1474 * framebuffer compression. For simplicity, we always install 1473 * framebuffer compression. For simplicity, we always install
1475 * a fence as the cost is not that onerous. 1474 * a fence as the cost is not that onerous.
1476 */ 1475 */
1477 if (obj_priv->fence_reg == I915_FENCE_REG_NONE && 1476 if (obj->tiling_mode != I915_TILING_NONE) {
1478 obj_priv->tiling_mode != I915_TILING_NONE) { 1477 ret = i915_gem_object_get_fence(obj, pipelined, false);
1479 ret = i915_gem_object_get_fence_reg(obj, false);
1480 if (ret) 1478 if (ret)
1481 goto err_unpin; 1479 goto err_unpin;
1482 } 1480 }
@@ -1497,8 +1495,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1497 struct drm_i915_private *dev_priv = dev->dev_private; 1495 struct drm_i915_private *dev_priv = dev->dev_private;
1498 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1496 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1499 struct intel_framebuffer *intel_fb; 1497 struct intel_framebuffer *intel_fb;
1500 struct drm_i915_gem_object *obj_priv; 1498 struct drm_i915_gem_object *obj;
1501 struct drm_gem_object *obj;
1502 int plane = intel_crtc->plane; 1499 int plane = intel_crtc->plane;
1503 unsigned long Start, Offset; 1500 unsigned long Start, Offset;
1504 u32 dspcntr; 1501 u32 dspcntr;
@@ -1515,7 +1512,6 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1515 1512
1516 intel_fb = to_intel_framebuffer(fb); 1513 intel_fb = to_intel_framebuffer(fb);
1517 obj = intel_fb->obj; 1514 obj = intel_fb->obj;
1518 obj_priv = to_intel_bo(obj);
1519 1515
1520 reg = DSPCNTR(plane); 1516 reg = DSPCNTR(plane);
1521 dspcntr = I915_READ(reg); 1517 dspcntr = I915_READ(reg);
@@ -1540,7 +1536,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1540 return -EINVAL; 1536 return -EINVAL;
1541 } 1537 }
1542 if (INTEL_INFO(dev)->gen >= 4) { 1538 if (INTEL_INFO(dev)->gen >= 4) {
1543 if (obj_priv->tiling_mode != I915_TILING_NONE) 1539 if (obj->tiling_mode != I915_TILING_NONE)
1544 dspcntr |= DISPPLANE_TILED; 1540 dspcntr |= DISPPLANE_TILED;
1545 else 1541 else
1546 dspcntr &= ~DISPPLANE_TILED; 1542 dspcntr &= ~DISPPLANE_TILED;
@@ -1552,7 +1548,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1552 1548
1553 I915_WRITE(reg, dspcntr); 1549 I915_WRITE(reg, dspcntr);
1554 1550
1555 Start = obj_priv->gtt_offset; 1551 Start = obj->gtt_offset;
1556 Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8); 1552 Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
1557 1553
1558 DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", 1554 DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
@@ -1598,7 +1594,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
1598 mutex_lock(&dev->struct_mutex); 1594 mutex_lock(&dev->struct_mutex);
1599 ret = intel_pin_and_fence_fb_obj(dev, 1595 ret = intel_pin_and_fence_fb_obj(dev,
1600 to_intel_framebuffer(crtc->fb)->obj, 1596 to_intel_framebuffer(crtc->fb)->obj,
1601 false); 1597 NULL);
1602 if (ret != 0) { 1598 if (ret != 0) {
1603 mutex_unlock(&dev->struct_mutex); 1599 mutex_unlock(&dev->struct_mutex);
1604 return ret; 1600 return ret;
@@ -1606,18 +1602,17 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
1606 1602
1607 if (old_fb) { 1603 if (old_fb) {
1608 struct drm_i915_private *dev_priv = dev->dev_private; 1604 struct drm_i915_private *dev_priv = dev->dev_private;
1609 struct drm_gem_object *obj = to_intel_framebuffer(old_fb)->obj; 1605 struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
1610 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1611 1606
1612 wait_event(dev_priv->pending_flip_queue, 1607 wait_event(dev_priv->pending_flip_queue,
1613 atomic_read(&obj_priv->pending_flip) == 0); 1608 atomic_read(&obj->pending_flip) == 0);
1614 1609
1615 /* Big Hammer, we also need to ensure that any pending 1610 /* Big Hammer, we also need to ensure that any pending
1616 * MI_WAIT_FOR_EVENT inside a user batch buffer on the 1611 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
1617 * current scanout is retired before unpinning the old 1612 * current scanout is retired before unpinning the old
1618 * framebuffer. 1613 * framebuffer.
1619 */ 1614 */
1620 ret = i915_gem_object_flush_gpu(obj_priv, false); 1615 ret = i915_gem_object_flush_gpu(obj, false);
1621 if (ret) { 1616 if (ret) {
1622 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj); 1617 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
1623 mutex_unlock(&dev->struct_mutex); 1618 mutex_unlock(&dev->struct_mutex);
@@ -1996,31 +1991,31 @@ static void intel_flush_display_plane(struct drm_device *dev,
1996static void intel_clear_scanline_wait(struct drm_device *dev) 1991static void intel_clear_scanline_wait(struct drm_device *dev)
1997{ 1992{
1998 struct drm_i915_private *dev_priv = dev->dev_private; 1993 struct drm_i915_private *dev_priv = dev->dev_private;
1994 struct intel_ring_buffer *ring;
1999 u32 tmp; 1995 u32 tmp;
2000 1996
2001 if (IS_GEN2(dev)) 1997 if (IS_GEN2(dev))
2002 /* Can't break the hang on i8xx */ 1998 /* Can't break the hang on i8xx */
2003 return; 1999 return;
2004 2000
2005 tmp = I915_READ(PRB0_CTL); 2001 ring = LP_RING(dev_priv);
2006 if (tmp & RING_WAIT) { 2002 tmp = I915_READ_CTL(ring);
2007 I915_WRITE(PRB0_CTL, tmp); 2003 if (tmp & RING_WAIT)
2008 POSTING_READ(PRB0_CTL); 2004 I915_WRITE_CTL(ring, tmp);
2009 }
2010} 2005}
2011 2006
2012static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) 2007static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
2013{ 2008{
2014 struct drm_i915_gem_object *obj_priv; 2009 struct drm_i915_gem_object *obj;
2015 struct drm_i915_private *dev_priv; 2010 struct drm_i915_private *dev_priv;
2016 2011
2017 if (crtc->fb == NULL) 2012 if (crtc->fb == NULL)
2018 return; 2013 return;
2019 2014
2020 obj_priv = to_intel_bo(to_intel_framebuffer(crtc->fb)->obj); 2015 obj = to_intel_framebuffer(crtc->fb)->obj;
2021 dev_priv = crtc->dev->dev_private; 2016 dev_priv = crtc->dev->dev_private;
2022 wait_event(dev_priv->pending_flip_queue, 2017 wait_event(dev_priv->pending_flip_queue,
2023 atomic_read(&obj_priv->pending_flip) == 0); 2018 atomic_read(&obj->pending_flip) == 0);
2024} 2019}
2025 2020
2026static void ironlake_crtc_enable(struct drm_crtc *crtc) 2021static void ironlake_crtc_enable(struct drm_crtc *crtc)
@@ -3453,7 +3448,7 @@ static void ironlake_update_wm(struct drm_device *dev,
3453 * display plane is used. 3448 * display plane is used.
3454 */ 3449 */
3455 tmp = 0; 3450 tmp = 0;
3456 if (enabled == 1 && /* XXX disabled due to buggy implmentation? */ 0) { 3451 if (enabled == 1) {
3457 unsigned long line_time_us; 3452 unsigned long line_time_us;
3458 int small, large, plane_fbc; 3453 int small, large, plane_fbc;
3459 int sr_clock, entries; 3454 int sr_clock, entries;
@@ -3857,6 +3852,22 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
3857 reduced_clock.m2; 3852 reduced_clock.m2;
3858 } 3853 }
3859 3854
3855 /* Enable autotuning of the PLL clock (if permissible) */
3856 if (HAS_PCH_SPLIT(dev)) {
3857 int factor = 21;
3858
3859 if (is_lvds) {
3860 if ((dev_priv->lvds_use_ssc &&
3861 dev_priv->lvds_ssc_freq == 100) ||
3862 (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
3863 factor = 25;
3864 } else if (is_sdvo && is_tv)
3865 factor = 20;
3866
3867 if (clock.m1 < factor * clock.n)
3868 fp |= FP_CB_TUNE;
3869 }
3870
3860 dpll = 0; 3871 dpll = 0;
3861 if (!HAS_PCH_SPLIT(dev)) 3872 if (!HAS_PCH_SPLIT(dev))
3862 dpll = DPLL_VGA_MODE_DIS; 3873 dpll = DPLL_VGA_MODE_DIS;
@@ -4071,7 +4082,6 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
4071 } 4082 }
4072 4083
4073 if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) { 4084 if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
4074 I915_WRITE(fp_reg, fp);
4075 I915_WRITE(dpll_reg, dpll); 4085 I915_WRITE(dpll_reg, dpll);
4076 4086
4077 /* Wait for the clocks to stabilize. */ 4087 /* Wait for the clocks to stabilize. */
@@ -4089,13 +4099,13 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
4089 } 4099 }
4090 I915_WRITE(DPLL_MD(pipe), temp); 4100 I915_WRITE(DPLL_MD(pipe), temp);
4091 } else { 4101 } else {
4092 /* write it again -- the BIOS does, after all */ 4102 /* The pixel multiplier can only be updated once the
4103 * DPLL is enabled and the clocks are stable.
4104 *
4105 * So write it again.
4106 */
4093 I915_WRITE(dpll_reg, dpll); 4107 I915_WRITE(dpll_reg, dpll);
4094 } 4108 }
4095
4096 /* Wait for the clocks to stabilize. */
4097 POSTING_READ(dpll_reg);
4098 udelay(150);
4099 } 4109 }
4100 4110
4101 intel_crtc->lowfreq_avail = false; 4111 intel_crtc->lowfreq_avail = false;
@@ -4331,15 +4341,14 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
4331} 4341}
4332 4342
4333static int intel_crtc_cursor_set(struct drm_crtc *crtc, 4343static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4334 struct drm_file *file_priv, 4344 struct drm_file *file,
4335 uint32_t handle, 4345 uint32_t handle,
4336 uint32_t width, uint32_t height) 4346 uint32_t width, uint32_t height)
4337{ 4347{
4338 struct drm_device *dev = crtc->dev; 4348 struct drm_device *dev = crtc->dev;
4339 struct drm_i915_private *dev_priv = dev->dev_private; 4349 struct drm_i915_private *dev_priv = dev->dev_private;
4340 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 4350 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4341 struct drm_gem_object *bo; 4351 struct drm_i915_gem_object *obj;
4342 struct drm_i915_gem_object *obj_priv;
4343 uint32_t addr; 4352 uint32_t addr;
4344 int ret; 4353 int ret;
4345 4354
@@ -4349,7 +4358,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4349 if (!handle) { 4358 if (!handle) {
4350 DRM_DEBUG_KMS("cursor off\n"); 4359 DRM_DEBUG_KMS("cursor off\n");
4351 addr = 0; 4360 addr = 0;
4352 bo = NULL; 4361 obj = NULL;
4353 mutex_lock(&dev->struct_mutex); 4362 mutex_lock(&dev->struct_mutex);
4354 goto finish; 4363 goto finish;
4355 } 4364 }
@@ -4360,13 +4369,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4360 return -EINVAL; 4369 return -EINVAL;
4361 } 4370 }
4362 4371
4363 bo = drm_gem_object_lookup(dev, file_priv, handle); 4372 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
4364 if (!bo) 4373 if (!obj)
4365 return -ENOENT; 4374 return -ENOENT;
4366 4375
4367 obj_priv = to_intel_bo(bo); 4376 if (obj->base.size < width * height * 4) {
4368
4369 if (bo->size < width * height * 4) {
4370 DRM_ERROR("buffer is to small\n"); 4377 DRM_ERROR("buffer is to small\n");
4371 ret = -ENOMEM; 4378 ret = -ENOMEM;
4372 goto fail; 4379 goto fail;
@@ -4375,29 +4382,41 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4375 /* we only need to pin inside GTT if cursor is non-phy */ 4382 /* we only need to pin inside GTT if cursor is non-phy */
4376 mutex_lock(&dev->struct_mutex); 4383 mutex_lock(&dev->struct_mutex);
4377 if (!dev_priv->info->cursor_needs_physical) { 4384 if (!dev_priv->info->cursor_needs_physical) {
4378 ret = i915_gem_object_pin(bo, PAGE_SIZE); 4385 if (obj->tiling_mode) {
4386 DRM_ERROR("cursor cannot be tiled\n");
4387 ret = -EINVAL;
4388 goto fail_locked;
4389 }
4390
4391 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
4379 if (ret) { 4392 if (ret) {
4380 DRM_ERROR("failed to pin cursor bo\n"); 4393 DRM_ERROR("failed to pin cursor bo\n");
4381 goto fail_locked; 4394 goto fail_locked;
4382 } 4395 }
4383 4396
4384 ret = i915_gem_object_set_to_gtt_domain(bo, 0); 4397 ret = i915_gem_object_set_to_gtt_domain(obj, 0);
4398 if (ret) {
4399 DRM_ERROR("failed to move cursor bo into the GTT\n");
4400 goto fail_unpin;
4401 }
4402
4403 ret = i915_gem_object_put_fence(obj);
4385 if (ret) { 4404 if (ret) {
4386 DRM_ERROR("failed to move cursor bo into the GTT\n"); 4405 DRM_ERROR("failed to move cursor bo into the GTT\n");
4387 goto fail_unpin; 4406 goto fail_unpin;
4388 } 4407 }
4389 4408
4390 addr = obj_priv->gtt_offset; 4409 addr = obj->gtt_offset;
4391 } else { 4410 } else {
4392 int align = IS_I830(dev) ? 16 * 1024 : 256; 4411 int align = IS_I830(dev) ? 16 * 1024 : 256;
4393 ret = i915_gem_attach_phys_object(dev, bo, 4412 ret = i915_gem_attach_phys_object(dev, obj,
4394 (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1, 4413 (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
4395 align); 4414 align);
4396 if (ret) { 4415 if (ret) {
4397 DRM_ERROR("failed to attach phys object\n"); 4416 DRM_ERROR("failed to attach phys object\n");
4398 goto fail_locked; 4417 goto fail_locked;
4399 } 4418 }
4400 addr = obj_priv->phys_obj->handle->busaddr; 4419 addr = obj->phys_obj->handle->busaddr;
4401 } 4420 }
4402 4421
4403 if (IS_GEN2(dev)) 4422 if (IS_GEN2(dev))
@@ -4406,17 +4425,17 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4406 finish: 4425 finish:
4407 if (intel_crtc->cursor_bo) { 4426 if (intel_crtc->cursor_bo) {
4408 if (dev_priv->info->cursor_needs_physical) { 4427 if (dev_priv->info->cursor_needs_physical) {
4409 if (intel_crtc->cursor_bo != bo) 4428 if (intel_crtc->cursor_bo != obj)
4410 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo); 4429 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
4411 } else 4430 } else
4412 i915_gem_object_unpin(intel_crtc->cursor_bo); 4431 i915_gem_object_unpin(intel_crtc->cursor_bo);
4413 drm_gem_object_unreference(intel_crtc->cursor_bo); 4432 drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
4414 } 4433 }
4415 4434
4416 mutex_unlock(&dev->struct_mutex); 4435 mutex_unlock(&dev->struct_mutex);
4417 4436
4418 intel_crtc->cursor_addr = addr; 4437 intel_crtc->cursor_addr = addr;
4419 intel_crtc->cursor_bo = bo; 4438 intel_crtc->cursor_bo = obj;
4420 intel_crtc->cursor_width = width; 4439 intel_crtc->cursor_width = width;
4421 intel_crtc->cursor_height = height; 4440 intel_crtc->cursor_height = height;
4422 4441
@@ -4424,11 +4443,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4424 4443
4425 return 0; 4444 return 0;
4426fail_unpin: 4445fail_unpin:
4427 i915_gem_object_unpin(bo); 4446 i915_gem_object_unpin(obj);
4428fail_locked: 4447fail_locked:
4429 mutex_unlock(&dev->struct_mutex); 4448 mutex_unlock(&dev->struct_mutex);
4430fail: 4449fail:
4431 drm_gem_object_unreference_unlocked(bo); 4450 drm_gem_object_unreference_unlocked(&obj->base);
4432 return ret; 4451 return ret;
4433} 4452}
4434 4453
@@ -4739,8 +4758,14 @@ static void intel_gpu_idle_timer(unsigned long arg)
4739 struct drm_device *dev = (struct drm_device *)arg; 4758 struct drm_device *dev = (struct drm_device *)arg;
4740 drm_i915_private_t *dev_priv = dev->dev_private; 4759 drm_i915_private_t *dev_priv = dev->dev_private;
4741 4760
4742 dev_priv->busy = false; 4761 if (!list_empty(&dev_priv->mm.active_list)) {
4762 /* Still processing requests, so just re-arm the timer. */
4763 mod_timer(&dev_priv->idle_timer, jiffies +
4764 msecs_to_jiffies(GPU_IDLE_TIMEOUT));
4765 return;
4766 }
4743 4767
4768 dev_priv->busy = false;
4744 queue_work(dev_priv->wq, &dev_priv->idle_work); 4769 queue_work(dev_priv->wq, &dev_priv->idle_work);
4745} 4770}
4746 4771
@@ -4751,9 +4776,17 @@ static void intel_crtc_idle_timer(unsigned long arg)
4751 struct intel_crtc *intel_crtc = (struct intel_crtc *)arg; 4776 struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
4752 struct drm_crtc *crtc = &intel_crtc->base; 4777 struct drm_crtc *crtc = &intel_crtc->base;
4753 drm_i915_private_t *dev_priv = crtc->dev->dev_private; 4778 drm_i915_private_t *dev_priv = crtc->dev->dev_private;
4779 struct intel_framebuffer *intel_fb;
4754 4780
4755 intel_crtc->busy = false; 4781 intel_fb = to_intel_framebuffer(crtc->fb);
4782 if (intel_fb && intel_fb->obj->active) {
4783 /* The framebuffer is still being accessed by the GPU. */
4784 mod_timer(&intel_crtc->idle_timer, jiffies +
4785 msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
4786 return;
4787 }
4756 4788
4789 intel_crtc->busy = false;
4757 queue_work(dev_priv->wq, &dev_priv->idle_work); 4790 queue_work(dev_priv->wq, &dev_priv->idle_work);
4758} 4791}
4759 4792
@@ -4888,7 +4921,7 @@ static void intel_idle_update(struct work_struct *work)
4888 * buffer), we'll also mark the display as busy, so we know to increase its 4921 * buffer), we'll also mark the display as busy, so we know to increase its
4889 * clock frequency. 4922 * clock frequency.
4890 */ 4923 */
4891void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj) 4924void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
4892{ 4925{
4893 drm_i915_private_t *dev_priv = dev->dev_private; 4926 drm_i915_private_t *dev_priv = dev->dev_private;
4894 struct drm_crtc *crtc = NULL; 4927 struct drm_crtc *crtc = NULL;
@@ -4969,8 +5002,9 @@ static void intel_unpin_work_fn(struct work_struct *__work)
4969 5002
4970 mutex_lock(&work->dev->struct_mutex); 5003 mutex_lock(&work->dev->struct_mutex);
4971 i915_gem_object_unpin(work->old_fb_obj); 5004 i915_gem_object_unpin(work->old_fb_obj);
4972 drm_gem_object_unreference(work->pending_flip_obj); 5005 drm_gem_object_unreference(&work->pending_flip_obj->base);
4973 drm_gem_object_unreference(work->old_fb_obj); 5006 drm_gem_object_unreference(&work->old_fb_obj->base);
5007
4974 mutex_unlock(&work->dev->struct_mutex); 5008 mutex_unlock(&work->dev->struct_mutex);
4975 kfree(work); 5009 kfree(work);
4976} 5010}
@@ -4981,7 +5015,7 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
4981 drm_i915_private_t *dev_priv = dev->dev_private; 5015 drm_i915_private_t *dev_priv = dev->dev_private;
4982 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5016 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4983 struct intel_unpin_work *work; 5017 struct intel_unpin_work *work;
4984 struct drm_i915_gem_object *obj_priv; 5018 struct drm_i915_gem_object *obj;
4985 struct drm_pending_vblank_event *e; 5019 struct drm_pending_vblank_event *e;
4986 struct timeval now; 5020 struct timeval now;
4987 unsigned long flags; 5021 unsigned long flags;
@@ -5013,11 +5047,13 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
5013 5047
5014 spin_unlock_irqrestore(&dev->event_lock, flags); 5048 spin_unlock_irqrestore(&dev->event_lock, flags);
5015 5049
5016 obj_priv = to_intel_bo(work->old_fb_obj); 5050 obj = work->old_fb_obj;
5051
5017 atomic_clear_mask(1 << intel_crtc->plane, 5052 atomic_clear_mask(1 << intel_crtc->plane,
5018 &obj_priv->pending_flip.counter); 5053 &obj->pending_flip.counter);
5019 if (atomic_read(&obj_priv->pending_flip) == 0) 5054 if (atomic_read(&obj->pending_flip) == 0)
5020 wake_up(&dev_priv->pending_flip_queue); 5055 wake_up(&dev_priv->pending_flip_queue);
5056
5021 schedule_work(&work->work); 5057 schedule_work(&work->work);
5022 5058
5023 trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); 5059 trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
@@ -5063,8 +5099,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5063 struct drm_device *dev = crtc->dev; 5099 struct drm_device *dev = crtc->dev;
5064 struct drm_i915_private *dev_priv = dev->dev_private; 5100 struct drm_i915_private *dev_priv = dev->dev_private;
5065 struct intel_framebuffer *intel_fb; 5101 struct intel_framebuffer *intel_fb;
5066 struct drm_i915_gem_object *obj_priv; 5102 struct drm_i915_gem_object *obj;
5067 struct drm_gem_object *obj;
5068 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5103 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5069 struct intel_unpin_work *work; 5104 struct intel_unpin_work *work;
5070 unsigned long flags, offset; 5105 unsigned long flags, offset;
@@ -5098,13 +5133,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5098 obj = intel_fb->obj; 5133 obj = intel_fb->obj;
5099 5134
5100 mutex_lock(&dev->struct_mutex); 5135 mutex_lock(&dev->struct_mutex);
5101 ret = intel_pin_and_fence_fb_obj(dev, obj, true); 5136 ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
5102 if (ret) 5137 if (ret)
5103 goto cleanup_work; 5138 goto cleanup_work;
5104 5139
5105 /* Reference the objects for the scheduled work. */ 5140 /* Reference the objects for the scheduled work. */
5106 drm_gem_object_reference(work->old_fb_obj); 5141 drm_gem_object_reference(&work->old_fb_obj->base);
5107 drm_gem_object_reference(obj); 5142 drm_gem_object_reference(&obj->base);
5108 5143
5109 crtc->fb = fb; 5144 crtc->fb = fb;
5110 5145
@@ -5112,22 +5147,16 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5112 if (ret) 5147 if (ret)
5113 goto cleanup_objs; 5148 goto cleanup_objs;
5114 5149
5115 /* Block clients from rendering to the new back buffer until
5116 * the flip occurs and the object is no longer visible.
5117 */
5118 atomic_add(1 << intel_crtc->plane,
5119 &to_intel_bo(work->old_fb_obj)->pending_flip);
5120
5121 work->pending_flip_obj = obj;
5122 obj_priv = to_intel_bo(obj);
5123
5124 if (IS_GEN3(dev) || IS_GEN2(dev)) { 5150 if (IS_GEN3(dev) || IS_GEN2(dev)) {
5125 u32 flip_mask; 5151 u32 flip_mask;
5126 5152
5127 /* Can't queue multiple flips, so wait for the previous 5153 /* Can't queue multiple flips, so wait for the previous
5128 * one to finish before executing the next. 5154 * one to finish before executing the next.
5129 */ 5155 */
5130 BEGIN_LP_RING(2); 5156 ret = BEGIN_LP_RING(2);
5157 if (ret)
5158 goto cleanup_objs;
5159
5131 if (intel_crtc->plane) 5160 if (intel_crtc->plane)
5132 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; 5161 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
5133 else 5162 else
@@ -5137,18 +5166,28 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5137 ADVANCE_LP_RING(); 5166 ADVANCE_LP_RING();
5138 } 5167 }
5139 5168
5169 work->pending_flip_obj = obj;
5170
5140 work->enable_stall_check = true; 5171 work->enable_stall_check = true;
5141 5172
5142 /* Offset into the new buffer for cases of shared fbs between CRTCs */ 5173 /* Offset into the new buffer for cases of shared fbs between CRTCs */
5143 offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8; 5174 offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
5144 5175
5145 BEGIN_LP_RING(4); 5176 ret = BEGIN_LP_RING(4);
5146 switch(INTEL_INFO(dev)->gen) { 5177 if (ret)
5178 goto cleanup_objs;
5179
5180 /* Block clients from rendering to the new back buffer until
5181 * the flip occurs and the object is no longer visible.
5182 */
5183 atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
5184
5185 switch (INTEL_INFO(dev)->gen) {
5147 case 2: 5186 case 2:
5148 OUT_RING(MI_DISPLAY_FLIP | 5187 OUT_RING(MI_DISPLAY_FLIP |
5149 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5188 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5150 OUT_RING(fb->pitch); 5189 OUT_RING(fb->pitch);
5151 OUT_RING(obj_priv->gtt_offset + offset); 5190 OUT_RING(obj->gtt_offset + offset);
5152 OUT_RING(MI_NOOP); 5191 OUT_RING(MI_NOOP);
5153 break; 5192 break;
5154 5193
@@ -5156,7 +5195,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5156 OUT_RING(MI_DISPLAY_FLIP_I915 | 5195 OUT_RING(MI_DISPLAY_FLIP_I915 |
5157 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5196 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5158 OUT_RING(fb->pitch); 5197 OUT_RING(fb->pitch);
5159 OUT_RING(obj_priv->gtt_offset + offset); 5198 OUT_RING(obj->gtt_offset + offset);
5160 OUT_RING(MI_NOOP); 5199 OUT_RING(MI_NOOP);
5161 break; 5200 break;
5162 5201
@@ -5169,7 +5208,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5169 OUT_RING(MI_DISPLAY_FLIP | 5208 OUT_RING(MI_DISPLAY_FLIP |
5170 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5209 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5171 OUT_RING(fb->pitch); 5210 OUT_RING(fb->pitch);
5172 OUT_RING(obj_priv->gtt_offset | obj_priv->tiling_mode); 5211 OUT_RING(obj->gtt_offset | obj->tiling_mode);
5173 5212
5174 /* XXX Enabling the panel-fitter across page-flip is so far 5213 /* XXX Enabling the panel-fitter across page-flip is so far
5175 * untested on non-native modes, so ignore it for now. 5214 * untested on non-native modes, so ignore it for now.
@@ -5183,8 +5222,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5183 case 6: 5222 case 6:
5184 OUT_RING(MI_DISPLAY_FLIP | 5223 OUT_RING(MI_DISPLAY_FLIP |
5185 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5224 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5186 OUT_RING(fb->pitch | obj_priv->tiling_mode); 5225 OUT_RING(fb->pitch | obj->tiling_mode);
5187 OUT_RING(obj_priv->gtt_offset); 5226 OUT_RING(obj->gtt_offset);
5188 5227
5189 pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE; 5228 pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
5190 pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff; 5229 pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff;
@@ -5200,8 +5239,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5200 return 0; 5239 return 0;
5201 5240
5202cleanup_objs: 5241cleanup_objs:
5203 drm_gem_object_unreference(work->old_fb_obj); 5242 drm_gem_object_unreference(&work->old_fb_obj->base);
5204 drm_gem_object_unreference(obj); 5243 drm_gem_object_unreference(&obj->base);
5205cleanup_work: 5244cleanup_work:
5206 mutex_unlock(&dev->struct_mutex); 5245 mutex_unlock(&dev->struct_mutex);
5207 5246
@@ -5338,7 +5377,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
5338} 5377}
5339 5378
5340int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, 5379int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
5341 struct drm_file *file_priv) 5380 struct drm_file *file)
5342{ 5381{
5343 drm_i915_private_t *dev_priv = dev->dev_private; 5382 drm_i915_private_t *dev_priv = dev->dev_private;
5344 struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data; 5383 struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
@@ -5488,19 +5527,19 @@ static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
5488 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 5527 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
5489 5528
5490 drm_framebuffer_cleanup(fb); 5529 drm_framebuffer_cleanup(fb);
5491 drm_gem_object_unreference_unlocked(intel_fb->obj); 5530 drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
5492 5531
5493 kfree(intel_fb); 5532 kfree(intel_fb);
5494} 5533}
5495 5534
5496static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, 5535static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
5497 struct drm_file *file_priv, 5536 struct drm_file *file,
5498 unsigned int *handle) 5537 unsigned int *handle)
5499{ 5538{
5500 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 5539 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
5501 struct drm_gem_object *object = intel_fb->obj; 5540 struct drm_i915_gem_object *obj = intel_fb->obj;
5502 5541
5503 return drm_gem_handle_create(file_priv, object, handle); 5542 return drm_gem_handle_create(file, &obj->base, handle);
5504} 5543}
5505 5544
5506static const struct drm_framebuffer_funcs intel_fb_funcs = { 5545static const struct drm_framebuffer_funcs intel_fb_funcs = {
@@ -5511,12 +5550,11 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
5511int intel_framebuffer_init(struct drm_device *dev, 5550int intel_framebuffer_init(struct drm_device *dev,
5512 struct intel_framebuffer *intel_fb, 5551 struct intel_framebuffer *intel_fb,
5513 struct drm_mode_fb_cmd *mode_cmd, 5552 struct drm_mode_fb_cmd *mode_cmd,
5514 struct drm_gem_object *obj) 5553 struct drm_i915_gem_object *obj)
5515{ 5554{
5516 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
5517 int ret; 5555 int ret;
5518 5556
5519 if (obj_priv->tiling_mode == I915_TILING_Y) 5557 if (obj->tiling_mode == I915_TILING_Y)
5520 return -EINVAL; 5558 return -EINVAL;
5521 5559
5522 if (mode_cmd->pitch & 63) 5560 if (mode_cmd->pitch & 63)
@@ -5548,11 +5586,11 @@ intel_user_framebuffer_create(struct drm_device *dev,
5548 struct drm_file *filp, 5586 struct drm_file *filp,
5549 struct drm_mode_fb_cmd *mode_cmd) 5587 struct drm_mode_fb_cmd *mode_cmd)
5550{ 5588{
5551 struct drm_gem_object *obj; 5589 struct drm_i915_gem_object *obj;
5552 struct intel_framebuffer *intel_fb; 5590 struct intel_framebuffer *intel_fb;
5553 int ret; 5591 int ret;
5554 5592
5555 obj = drm_gem_object_lookup(dev, filp, mode_cmd->handle); 5593 obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handle));
5556 if (!obj) 5594 if (!obj)
5557 return ERR_PTR(-ENOENT); 5595 return ERR_PTR(-ENOENT);
5558 5596
@@ -5560,10 +5598,9 @@ intel_user_framebuffer_create(struct drm_device *dev,
5560 if (!intel_fb) 5598 if (!intel_fb)
5561 return ERR_PTR(-ENOMEM); 5599 return ERR_PTR(-ENOMEM);
5562 5600
5563 ret = intel_framebuffer_init(dev, intel_fb, 5601 ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
5564 mode_cmd, obj);
5565 if (ret) { 5602 if (ret) {
5566 drm_gem_object_unreference_unlocked(obj); 5603 drm_gem_object_unreference_unlocked(&obj->base);
5567 kfree(intel_fb); 5604 kfree(intel_fb);
5568 return ERR_PTR(ret); 5605 return ERR_PTR(ret);
5569 } 5606 }
@@ -5576,10 +5613,10 @@ static const struct drm_mode_config_funcs intel_mode_funcs = {
5576 .output_poll_changed = intel_fb_output_poll_changed, 5613 .output_poll_changed = intel_fb_output_poll_changed,
5577}; 5614};
5578 5615
5579static struct drm_gem_object * 5616static struct drm_i915_gem_object *
5580intel_alloc_context_page(struct drm_device *dev) 5617intel_alloc_context_page(struct drm_device *dev)
5581{ 5618{
5582 struct drm_gem_object *ctx; 5619 struct drm_i915_gem_object *ctx;
5583 int ret; 5620 int ret;
5584 5621
5585 ctx = i915_gem_alloc_object(dev, 4096); 5622 ctx = i915_gem_alloc_object(dev, 4096);
@@ -5589,7 +5626,7 @@ intel_alloc_context_page(struct drm_device *dev)
5589 } 5626 }
5590 5627
5591 mutex_lock(&dev->struct_mutex); 5628 mutex_lock(&dev->struct_mutex);
5592 ret = i915_gem_object_pin(ctx, 4096); 5629 ret = i915_gem_object_pin(ctx, 4096, true);
5593 if (ret) { 5630 if (ret) {
5594 DRM_ERROR("failed to pin power context: %d\n", ret); 5631 DRM_ERROR("failed to pin power context: %d\n", ret);
5595 goto err_unref; 5632 goto err_unref;
@@ -5607,7 +5644,7 @@ intel_alloc_context_page(struct drm_device *dev)
5607err_unpin: 5644err_unpin:
5608 i915_gem_object_unpin(ctx); 5645 i915_gem_object_unpin(ctx);
5609err_unref: 5646err_unref:
5610 drm_gem_object_unreference(ctx); 5647 drm_gem_object_unreference(&ctx->base);
5611 mutex_unlock(&dev->struct_mutex); 5648 mutex_unlock(&dev->struct_mutex);
5612 return NULL; 5649 return NULL;
5613} 5650}
@@ -5805,7 +5842,92 @@ void intel_init_emon(struct drm_device *dev)
5805 dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK); 5842 dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
5806} 5843}
5807 5844
5808void intel_init_clock_gating(struct drm_device *dev) 5845static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
5846{
5847 int i;
5848
5849 /* Here begins a magic sequence of register writes to enable
5850 * auto-downclocking.
5851 *
5852 * Perhaps there might be some value in exposing these to
5853 * userspace...
5854 */
5855 I915_WRITE(GEN6_RC_STATE, 0);
5856 __gen6_force_wake_get(dev_priv);
5857
5858 /* disable the counters and set determistic thresholds */
5859 I915_WRITE(GEN6_RC_CONTROL, 0);
5860
5861 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
5862 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
5863 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
5864 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5865 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5866
5867 for (i = 0; i < I915_NUM_RINGS; i++)
5868 I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10);
5869
5870 I915_WRITE(GEN6_RC_SLEEP, 0);
5871 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
5872 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
5873 I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
5874 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
5875
5876 I915_WRITE(GEN6_RC_CONTROL,
5877 GEN6_RC_CTL_RC6p_ENABLE |
5878 GEN6_RC_CTL_RC6_ENABLE |
5879 GEN6_RC_CTL_HW_ENABLE);
5880
5881 I915_WRITE(GEN6_RC_NORMAL_FREQ,
5882 GEN6_FREQUENCY(10) |
5883 GEN6_OFFSET(0) |
5884 GEN6_AGGRESSIVE_TURBO);
5885 I915_WRITE(GEN6_RC_VIDEO_FREQ,
5886 GEN6_FREQUENCY(12));
5887
5888 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5889 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5890 18 << 24 |
5891 6 << 16);
5892 I915_WRITE(GEN6_RP_UP_THRESHOLD, 90000);
5893 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 100000);
5894 I915_WRITE(GEN6_RP_UP_EI, 100000);
5895 I915_WRITE(GEN6_RP_DOWN_EI, 300000);
5896 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5897 I915_WRITE(GEN6_RP_CONTROL,
5898 GEN6_RP_MEDIA_TURBO |
5899 GEN6_RP_USE_NORMAL_FREQ |
5900 GEN6_RP_MEDIA_IS_GFX |
5901 GEN6_RP_ENABLE |
5902 GEN6_RP_UP_BUSY_MAX |
5903 GEN6_RP_DOWN_BUSY_MIN);
5904
5905 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
5906 500))
5907 DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
5908
5909 I915_WRITE(GEN6_PCODE_DATA, 0);
5910 I915_WRITE(GEN6_PCODE_MAILBOX,
5911 GEN6_PCODE_READY |
5912 GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
5913 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
5914 500))
5915 DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
5916
5917 /* requires MSI enabled */
5918 I915_WRITE(GEN6_PMIER,
5919 GEN6_PM_MBOX_EVENT |
5920 GEN6_PM_THERMAL_EVENT |
5921 GEN6_PM_RP_DOWN_TIMEOUT |
5922 GEN6_PM_RP_UP_THRESHOLD |
5923 GEN6_PM_RP_DOWN_THRESHOLD |
5924 GEN6_PM_RP_UP_EI_EXPIRED |
5925 GEN6_PM_RP_DOWN_EI_EXPIRED);
5926
5927 __gen6_force_wake_put(dev_priv);
5928}
5929
5930void intel_enable_clock_gating(struct drm_device *dev)
5809{ 5931{
5810 struct drm_i915_private *dev_priv = dev->dev_private; 5932 struct drm_i915_private *dev_priv = dev->dev_private;
5811 5933
@@ -5877,7 +5999,17 @@ void intel_init_clock_gating(struct drm_device *dev)
5877 ILK_DPFC_DIS2 | 5999 ILK_DPFC_DIS2 |
5878 ILK_CLK_FBC); 6000 ILK_CLK_FBC);
5879 } 6001 }
5880 return; 6002
6003 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6004 I915_READ(ILK_DISPLAY_CHICKEN2) |
6005 ILK_ELPIN_409_SELECT);
6006
6007 if (IS_GEN5(dev)) {
6008 I915_WRITE(_3D_CHICKEN2,
6009 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
6010 _3D_CHICKEN2_WM_READ_PIPELINED);
6011 }
6012
5881 } else if (IS_G4X(dev)) { 6013 } else if (IS_G4X(dev)) {
5882 uint32_t dspclk_gate; 6014 uint32_t dspclk_gate;
5883 I915_WRITE(RENCLK_GATE_D1, 0); 6015 I915_WRITE(RENCLK_GATE_D1, 0);
@@ -5920,20 +6052,18 @@ void intel_init_clock_gating(struct drm_device *dev)
5920 * GPU can automatically power down the render unit if given a page 6052 * GPU can automatically power down the render unit if given a page
5921 * to save state. 6053 * to save state.
5922 */ 6054 */
5923 if (IS_IRONLAKE_M(dev)) { 6055 if (IS_IRONLAKE_M(dev) && 0) { /* XXX causes a failure during suspend */
5924 if (dev_priv->renderctx == NULL) 6056 if (dev_priv->renderctx == NULL)
5925 dev_priv->renderctx = intel_alloc_context_page(dev); 6057 dev_priv->renderctx = intel_alloc_context_page(dev);
5926 if (dev_priv->renderctx) { 6058 if (dev_priv->renderctx) {
5927 struct drm_i915_gem_object *obj_priv; 6059 struct drm_i915_gem_object *obj = dev_priv->renderctx;
5928 obj_priv = to_intel_bo(dev_priv->renderctx); 6060 if (BEGIN_LP_RING(4) == 0) {
5929 if (obj_priv) {
5930 BEGIN_LP_RING(4);
5931 OUT_RING(MI_SET_CONTEXT); 6061 OUT_RING(MI_SET_CONTEXT);
5932 OUT_RING(obj_priv->gtt_offset | 6062 OUT_RING(obj->gtt_offset |
5933 MI_MM_SPACE_GTT | 6063 MI_MM_SPACE_GTT |
5934 MI_SAVE_EXT_STATE_EN | 6064 MI_SAVE_EXT_STATE_EN |
5935 MI_RESTORE_EXT_STATE_EN | 6065 MI_RESTORE_EXT_STATE_EN |
5936 MI_RESTORE_INHIBIT); 6066 MI_RESTORE_INHIBIT);
5937 OUT_RING(MI_NOOP); 6067 OUT_RING(MI_NOOP);
5938 OUT_RING(MI_FLUSH); 6068 OUT_RING(MI_FLUSH);
5939 ADVANCE_LP_RING(); 6069 ADVANCE_LP_RING();
@@ -5943,27 +6073,46 @@ void intel_init_clock_gating(struct drm_device *dev)
5943 "Disable RC6\n"); 6073 "Disable RC6\n");
5944 } 6074 }
5945 6075
5946 if (I915_HAS_RC6(dev) && drm_core_check_feature(dev, DRIVER_MODESET)) { 6076 if (IS_GEN4(dev) && IS_MOBILE(dev)) {
5947 struct drm_i915_gem_object *obj_priv = NULL; 6077 if (dev_priv->pwrctx == NULL)
5948 6078 dev_priv->pwrctx = intel_alloc_context_page(dev);
5949 if (dev_priv->pwrctx) { 6079 if (dev_priv->pwrctx) {
5950 obj_priv = to_intel_bo(dev_priv->pwrctx); 6080 struct drm_i915_gem_object *obj = dev_priv->pwrctx;
5951 } else { 6081 I915_WRITE(PWRCTXA, obj->gtt_offset | PWRCTX_EN);
5952 struct drm_gem_object *pwrctx;
5953
5954 pwrctx = intel_alloc_context_page(dev);
5955 if (pwrctx) {
5956 dev_priv->pwrctx = pwrctx;
5957 obj_priv = to_intel_bo(pwrctx);
5958 }
5959 }
5960
5961 if (obj_priv) {
5962 I915_WRITE(PWRCTXA, obj_priv->gtt_offset | PWRCTX_EN);
5963 I915_WRITE(MCHBAR_RENDER_STANDBY, 6082 I915_WRITE(MCHBAR_RENDER_STANDBY,
5964 I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT); 6083 I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT);
5965 } 6084 }
5966 } 6085 }
6086
6087 if (IS_GEN6(dev))
6088 gen6_enable_rc6(dev_priv);
6089}
6090
6091void intel_disable_clock_gating(struct drm_device *dev)
6092{
6093 struct drm_i915_private *dev_priv = dev->dev_private;
6094
6095 if (dev_priv->renderctx) {
6096 struct drm_i915_gem_object *obj = dev_priv->renderctx;
6097
6098 I915_WRITE(CCID, 0);
6099 POSTING_READ(CCID);
6100
6101 i915_gem_object_unpin(obj);
6102 drm_gem_object_unreference(&obj->base);
6103 dev_priv->renderctx = NULL;
6104 }
6105
6106 if (dev_priv->pwrctx) {
6107 struct drm_i915_gem_object *obj = dev_priv->pwrctx;
6108
6109 I915_WRITE(PWRCTXA, 0);
6110 POSTING_READ(PWRCTXA);
6111
6112 i915_gem_object_unpin(obj);
6113 drm_gem_object_unreference(&obj->base);
6114 dev_priv->pwrctx = NULL;
6115 }
5967} 6116}
5968 6117
5969/* Set up chip specific display functions */ 6118/* Set up chip specific display functions */
@@ -6192,7 +6341,7 @@ void intel_modeset_init(struct drm_device *dev)
6192 6341
6193 intel_setup_outputs(dev); 6342 intel_setup_outputs(dev);
6194 6343
6195 intel_init_clock_gating(dev); 6344 intel_enable_clock_gating(dev);
6196 6345
6197 /* Just disable it once at startup */ 6346 /* Just disable it once at startup */
6198 i915_disable_vga(dev); 6347 i915_disable_vga(dev);
@@ -6233,29 +6382,11 @@ void intel_modeset_cleanup(struct drm_device *dev)
6233 if (dev_priv->display.disable_fbc) 6382 if (dev_priv->display.disable_fbc)
6234 dev_priv->display.disable_fbc(dev); 6383 dev_priv->display.disable_fbc(dev);
6235 6384
6236 if (dev_priv->renderctx) {
6237 struct drm_i915_gem_object *obj_priv;
6238
6239 obj_priv = to_intel_bo(dev_priv->renderctx);
6240 I915_WRITE(CCID, obj_priv->gtt_offset &~ CCID_EN);
6241 I915_READ(CCID);
6242 i915_gem_object_unpin(dev_priv->renderctx);
6243 drm_gem_object_unreference(dev_priv->renderctx);
6244 }
6245
6246 if (dev_priv->pwrctx) {
6247 struct drm_i915_gem_object *obj_priv;
6248
6249 obj_priv = to_intel_bo(dev_priv->pwrctx);
6250 I915_WRITE(PWRCTXA, obj_priv->gtt_offset &~ PWRCTX_EN);
6251 I915_READ(PWRCTXA);
6252 i915_gem_object_unpin(dev_priv->pwrctx);
6253 drm_gem_object_unreference(dev_priv->pwrctx);
6254 }
6255
6256 if (IS_IRONLAKE_M(dev)) 6385 if (IS_IRONLAKE_M(dev))
6257 ironlake_disable_drps(dev); 6386 ironlake_disable_drps(dev);
6258 6387
6388 intel_disable_clock_gating(dev);
6389
6259 mutex_unlock(&dev->struct_mutex); 6390 mutex_unlock(&dev->struct_mutex);
6260 6391
6261 /* Disable the irq before mode object teardown, for the irq might 6392 /* Disable the irq before mode object teardown, for the irq might
@@ -6306,3 +6437,113 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
6306 pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl); 6437 pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl);
6307 return 0; 6438 return 0;
6308} 6439}
6440
6441#ifdef CONFIG_DEBUG_FS
6442#include <linux/seq_file.h>
6443
6444struct intel_display_error_state {
6445 struct intel_cursor_error_state {
6446 u32 control;
6447 u32 position;
6448 u32 base;
6449 u32 size;
6450 } cursor[2];
6451
6452 struct intel_pipe_error_state {
6453 u32 conf;
6454 u32 source;
6455
6456 u32 htotal;
6457 u32 hblank;
6458 u32 hsync;
6459 u32 vtotal;
6460 u32 vblank;
6461 u32 vsync;
6462 } pipe[2];
6463
6464 struct intel_plane_error_state {
6465 u32 control;
6466 u32 stride;
6467 u32 size;
6468 u32 pos;
6469 u32 addr;
6470 u32 surface;
6471 u32 tile_offset;
6472 } plane[2];
6473};
6474
6475struct intel_display_error_state *
6476intel_display_capture_error_state(struct drm_device *dev)
6477{
6478 drm_i915_private_t *dev_priv = dev->dev_private;
6479 struct intel_display_error_state *error;
6480 int i;
6481
6482 error = kmalloc(sizeof(*error), GFP_ATOMIC);
6483 if (error == NULL)
6484 return NULL;
6485
6486 for (i = 0; i < 2; i++) {
6487 error->cursor[i].control = I915_READ(CURCNTR(i));
6488 error->cursor[i].position = I915_READ(CURPOS(i));
6489 error->cursor[i].base = I915_READ(CURBASE(i));
6490
6491 error->plane[i].control = I915_READ(DSPCNTR(i));
6492 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
6493 error->plane[i].size = I915_READ(DSPSIZE(i));
6494 error->plane[i].pos= I915_READ(DSPPOS(i));
6495 error->plane[i].addr = I915_READ(DSPADDR(i));
6496 if (INTEL_INFO(dev)->gen >= 4) {
6497 error->plane[i].surface = I915_READ(DSPSURF(i));
6498 error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
6499 }
6500
6501 error->pipe[i].conf = I915_READ(PIPECONF(i));
6502 error->pipe[i].source = I915_READ(PIPESRC(i));
6503 error->pipe[i].htotal = I915_READ(HTOTAL(i));
6504 error->pipe[i].hblank = I915_READ(HBLANK(i));
6505 error->pipe[i].hsync = I915_READ(HSYNC(i));
6506 error->pipe[i].vtotal = I915_READ(VTOTAL(i));
6507 error->pipe[i].vblank = I915_READ(VBLANK(i));
6508 error->pipe[i].vsync = I915_READ(VSYNC(i));
6509 }
6510
6511 return error;
6512}
6513
6514void
6515intel_display_print_error_state(struct seq_file *m,
6516 struct drm_device *dev,
6517 struct intel_display_error_state *error)
6518{
6519 int i;
6520
6521 for (i = 0; i < 2; i++) {
6522 seq_printf(m, "Pipe [%d]:\n", i);
6523 seq_printf(m, " CONF: %08x\n", error->pipe[i].conf);
6524 seq_printf(m, " SRC: %08x\n", error->pipe[i].source);
6525 seq_printf(m, " HTOTAL: %08x\n", error->pipe[i].htotal);
6526 seq_printf(m, " HBLANK: %08x\n", error->pipe[i].hblank);
6527 seq_printf(m, " HSYNC: %08x\n", error->pipe[i].hsync);
6528 seq_printf(m, " VTOTAL: %08x\n", error->pipe[i].vtotal);
6529 seq_printf(m, " VBLANK: %08x\n", error->pipe[i].vblank);
6530 seq_printf(m, " VSYNC: %08x\n", error->pipe[i].vsync);
6531
6532 seq_printf(m, "Plane [%d]:\n", i);
6533 seq_printf(m, " CNTR: %08x\n", error->plane[i].control);
6534 seq_printf(m, " STRIDE: %08x\n", error->plane[i].stride);
6535 seq_printf(m, " SIZE: %08x\n", error->plane[i].size);
6536 seq_printf(m, " POS: %08x\n", error->plane[i].pos);
6537 seq_printf(m, " ADDR: %08x\n", error->plane[i].addr);
6538 if (INTEL_INFO(dev)->gen >= 4) {
6539 seq_printf(m, " SURF: %08x\n", error->plane[i].surface);
6540 seq_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset);
6541 }
6542
6543 seq_printf(m, "Cursor [%d]:\n", i);
6544 seq_printf(m, " CNTR: %08x\n", error->cursor[i].control);
6545 seq_printf(m, " POS: %08x\n", error->cursor[i].position);
6546 seq_printf(m, " BASE: %08x\n", error->cursor[i].base);
6547 }
6548}
6549#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 864417cffe9a..1dc60408d5b8 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -1442,8 +1442,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
1442 /* Changes to enable or select take place the vblank 1442 /* Changes to enable or select take place the vblank
1443 * after being written. 1443 * after being written.
1444 */ 1444 */
1445 intel_wait_for_vblank(intel_dp->base.base.dev, 1445 intel_wait_for_vblank(dev, intel_crtc->pipe);
1446 intel_crtc->pipe);
1447 } 1446 }
1448 1447
1449 I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN); 1448 I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN);
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index e52c6125bb1f..acdea6549ec4 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -127,7 +127,7 @@ intel_mode_get_pixel_multiplier(const struct drm_display_mode *mode)
127 127
128struct intel_framebuffer { 128struct intel_framebuffer {
129 struct drm_framebuffer base; 129 struct drm_framebuffer base;
130 struct drm_gem_object *obj; 130 struct drm_i915_gem_object *obj;
131}; 131};
132 132
133struct intel_fbdev { 133struct intel_fbdev {
@@ -166,7 +166,7 @@ struct intel_crtc {
166 struct intel_unpin_work *unpin_work; 166 struct intel_unpin_work *unpin_work;
167 int fdi_lanes; 167 int fdi_lanes;
168 168
169 struct drm_gem_object *cursor_bo; 169 struct drm_i915_gem_object *cursor_bo;
170 uint32_t cursor_addr; 170 uint32_t cursor_addr;
171 int16_t cursor_x, cursor_y; 171 int16_t cursor_x, cursor_y;
172 int16_t cursor_width, cursor_height; 172 int16_t cursor_width, cursor_height;
@@ -220,8 +220,8 @@ intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
220struct intel_unpin_work { 220struct intel_unpin_work {
221 struct work_struct work; 221 struct work_struct work;
222 struct drm_device *dev; 222 struct drm_device *dev;
223 struct drm_gem_object *old_fb_obj; 223 struct drm_i915_gem_object *old_fb_obj;
224 struct drm_gem_object *pending_flip_obj; 224 struct drm_i915_gem_object *pending_flip_obj;
225 struct drm_pending_vblank_event *event; 225 struct drm_pending_vblank_event *event;
226 int pending; 226 int pending;
227 bool enable_stall_check; 227 bool enable_stall_check;
@@ -236,7 +236,8 @@ void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
236extern bool intel_sdvo_init(struct drm_device *dev, int output_device); 236extern bool intel_sdvo_init(struct drm_device *dev, int output_device);
237extern void intel_dvo_init(struct drm_device *dev); 237extern void intel_dvo_init(struct drm_device *dev);
238extern void intel_tv_init(struct drm_device *dev); 238extern void intel_tv_init(struct drm_device *dev);
239extern void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj); 239extern void intel_mark_busy(struct drm_device *dev,
240 struct drm_i915_gem_object *obj);
240extern bool intel_lvds_init(struct drm_device *dev); 241extern bool intel_lvds_init(struct drm_device *dev);
241extern void intel_dp_init(struct drm_device *dev, int dp_reg); 242extern void intel_dp_init(struct drm_device *dev, int dp_reg);
242void 243void
@@ -293,19 +294,20 @@ extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
293 u16 blue, int regno); 294 u16 blue, int regno);
294extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green, 295extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
295 u16 *blue, int regno); 296 u16 *blue, int regno);
296extern void intel_init_clock_gating(struct drm_device *dev); 297extern void intel_enable_clock_gating(struct drm_device *dev);
298extern void intel_disable_clock_gating(struct drm_device *dev);
297extern void ironlake_enable_drps(struct drm_device *dev); 299extern void ironlake_enable_drps(struct drm_device *dev);
298extern void ironlake_disable_drps(struct drm_device *dev); 300extern void ironlake_disable_drps(struct drm_device *dev);
299extern void intel_init_emon(struct drm_device *dev); 301extern void intel_init_emon(struct drm_device *dev);
300 302
301extern int intel_pin_and_fence_fb_obj(struct drm_device *dev, 303extern int intel_pin_and_fence_fb_obj(struct drm_device *dev,
302 struct drm_gem_object *obj, 304 struct drm_i915_gem_object *obj,
303 bool pipelined); 305 struct intel_ring_buffer *pipelined);
304 306
305extern int intel_framebuffer_init(struct drm_device *dev, 307extern int intel_framebuffer_init(struct drm_device *dev,
306 struct intel_framebuffer *ifb, 308 struct intel_framebuffer *ifb,
307 struct drm_mode_fb_cmd *mode_cmd, 309 struct drm_mode_fb_cmd *mode_cmd,
308 struct drm_gem_object *obj); 310 struct drm_i915_gem_object *obj);
309extern int intel_fbdev_init(struct drm_device *dev); 311extern int intel_fbdev_init(struct drm_device *dev);
310extern void intel_fbdev_fini(struct drm_device *dev); 312extern void intel_fbdev_fini(struct drm_device *dev);
311 313
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index af2a1dddc28e..c2cffeb4fe89 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -65,8 +65,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
65 struct fb_info *info; 65 struct fb_info *info;
66 struct drm_framebuffer *fb; 66 struct drm_framebuffer *fb;
67 struct drm_mode_fb_cmd mode_cmd; 67 struct drm_mode_fb_cmd mode_cmd;
68 struct drm_gem_object *fbo = NULL; 68 struct drm_i915_gem_object *obj;
69 struct drm_i915_gem_object *obj_priv;
70 struct device *device = &dev->pdev->dev; 69 struct device *device = &dev->pdev->dev;
71 int size, ret, mmio_bar = IS_GEN2(dev) ? 1 : 0; 70 int size, ret, mmio_bar = IS_GEN2(dev) ? 1 : 0;
72 71
@@ -83,18 +82,17 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
83 82
84 size = mode_cmd.pitch * mode_cmd.height; 83 size = mode_cmd.pitch * mode_cmd.height;
85 size = ALIGN(size, PAGE_SIZE); 84 size = ALIGN(size, PAGE_SIZE);
86 fbo = i915_gem_alloc_object(dev, size); 85 obj = i915_gem_alloc_object(dev, size);
87 if (!fbo) { 86 if (!obj) {
88 DRM_ERROR("failed to allocate framebuffer\n"); 87 DRM_ERROR("failed to allocate framebuffer\n");
89 ret = -ENOMEM; 88 ret = -ENOMEM;
90 goto out; 89 goto out;
91 } 90 }
92 obj_priv = to_intel_bo(fbo);
93 91
94 mutex_lock(&dev->struct_mutex); 92 mutex_lock(&dev->struct_mutex);
95 93
96 /* Flush everything out, we'll be doing GTT only from now on */ 94 /* Flush everything out, we'll be doing GTT only from now on */
97 ret = intel_pin_and_fence_fb_obj(dev, fbo, false); 95 ret = intel_pin_and_fence_fb_obj(dev, obj, false);
98 if (ret) { 96 if (ret) {
99 DRM_ERROR("failed to pin fb: %d\n", ret); 97 DRM_ERROR("failed to pin fb: %d\n", ret);
100 goto out_unref; 98 goto out_unref;
@@ -108,7 +106,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
108 106
109 info->par = ifbdev; 107 info->par = ifbdev;
110 108
111 ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, fbo); 109 ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj);
112 if (ret) 110 if (ret)
113 goto out_unpin; 111 goto out_unpin;
114 112
@@ -134,11 +132,10 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
134 else 132 else
135 info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0); 133 info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0);
136 134
137 info->fix.smem_start = dev->mode_config.fb_base + obj_priv->gtt_offset; 135 info->fix.smem_start = dev->mode_config.fb_base + obj->gtt_offset;
138 info->fix.smem_len = size; 136 info->fix.smem_len = size;
139 137
140 info->screen_base = ioremap_wc(dev->agp->base + obj_priv->gtt_offset, 138 info->screen_base = ioremap_wc(dev->agp->base + obj->gtt_offset, size);
141 size);
142 if (!info->screen_base) { 139 if (!info->screen_base) {
143 ret = -ENOSPC; 140 ret = -ENOSPC;
144 goto out_unpin; 141 goto out_unpin;
@@ -168,7 +165,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
168 165
169 DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n", 166 DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n",
170 fb->width, fb->height, 167 fb->width, fb->height,
171 obj_priv->gtt_offset, fbo); 168 obj->gtt_offset, obj);
172 169
173 170
174 mutex_unlock(&dev->struct_mutex); 171 mutex_unlock(&dev->struct_mutex);
@@ -176,9 +173,9 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
176 return 0; 173 return 0;
177 174
178out_unpin: 175out_unpin:
179 i915_gem_object_unpin(fbo); 176 i915_gem_object_unpin(obj);
180out_unref: 177out_unref:
181 drm_gem_object_unreference(fbo); 178 drm_gem_object_unreference(&obj->base);
182 mutex_unlock(&dev->struct_mutex); 179 mutex_unlock(&dev->struct_mutex);
183out: 180out:
184 return ret; 181 return ret;
@@ -225,7 +222,7 @@ static void intel_fbdev_destroy(struct drm_device *dev,
225 222
226 drm_framebuffer_cleanup(&ifb->base); 223 drm_framebuffer_cleanup(&ifb->base);
227 if (ifb->obj) { 224 if (ifb->obj) {
228 drm_gem_object_unreference_unlocked(ifb->obj); 225 drm_gem_object_unreference_unlocked(&ifb->obj->base);
229 ifb->obj = NULL; 226 ifb->obj = NULL;
230 } 227 }
231} 228}
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 3dba086e7eea..58040f68ed7a 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -85,8 +85,9 @@ static u32 get_reserved(struct intel_gpio *gpio)
85 85
86 /* On most chips, these bits must be preserved in software. */ 86 /* On most chips, these bits must be preserved in software. */
87 if (!IS_I830(dev) && !IS_845G(dev)) 87 if (!IS_I830(dev) && !IS_845G(dev))
88 reserved = I915_READ(gpio->reg) & (GPIO_DATA_PULLUP_DISABLE | 88 reserved = I915_READ_NOTRACE(gpio->reg) &
89 GPIO_CLOCK_PULLUP_DISABLE); 89 (GPIO_DATA_PULLUP_DISABLE |
90 GPIO_CLOCK_PULLUP_DISABLE);
90 91
91 return reserved; 92 return reserved;
92} 93}
@@ -96,9 +97,9 @@ static int get_clock(void *data)
96 struct intel_gpio *gpio = data; 97 struct intel_gpio *gpio = data;
97 struct drm_i915_private *dev_priv = gpio->dev_priv; 98 struct drm_i915_private *dev_priv = gpio->dev_priv;
98 u32 reserved = get_reserved(gpio); 99 u32 reserved = get_reserved(gpio);
99 I915_WRITE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK); 100 I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
100 I915_WRITE(gpio->reg, reserved); 101 I915_WRITE_NOTRACE(gpio->reg, reserved);
101 return (I915_READ(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0; 102 return (I915_READ_NOTRACE(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
102} 103}
103 104
104static int get_data(void *data) 105static int get_data(void *data)
@@ -106,9 +107,9 @@ static int get_data(void *data)
106 struct intel_gpio *gpio = data; 107 struct intel_gpio *gpio = data;
107 struct drm_i915_private *dev_priv = gpio->dev_priv; 108 struct drm_i915_private *dev_priv = gpio->dev_priv;
108 u32 reserved = get_reserved(gpio); 109 u32 reserved = get_reserved(gpio);
109 I915_WRITE(gpio->reg, reserved | GPIO_DATA_DIR_MASK); 110 I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
110 I915_WRITE(gpio->reg, reserved); 111 I915_WRITE_NOTRACE(gpio->reg, reserved);
111 return (I915_READ(gpio->reg) & GPIO_DATA_VAL_IN) != 0; 112 return (I915_READ_NOTRACE(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
112} 113}
113 114
114static void set_clock(void *data, int state_high) 115static void set_clock(void *data, int state_high)
@@ -124,7 +125,7 @@ static void set_clock(void *data, int state_high)
124 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK | 125 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
125 GPIO_CLOCK_VAL_MASK; 126 GPIO_CLOCK_VAL_MASK;
126 127
127 I915_WRITE(gpio->reg, reserved | clock_bits); 128 I915_WRITE_NOTRACE(gpio->reg, reserved | clock_bits);
128 POSTING_READ(gpio->reg); 129 POSTING_READ(gpio->reg);
129} 130}
130 131
@@ -141,7 +142,7 @@ static void set_data(void *data, int state_high)
141 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK | 142 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
142 GPIO_DATA_VAL_MASK; 143 GPIO_DATA_VAL_MASK;
143 144
144 I915_WRITE(gpio->reg, reserved | data_bits); 145 I915_WRITE_NOTRACE(gpio->reg, reserved | data_bits);
145 POSTING_READ(gpio->reg); 146 POSTING_READ(gpio->reg);
146} 147}
147 148
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 25bcedf386fd..aa2307080be2 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -304,14 +304,13 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
304 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay; 304 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
305 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay; 305 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
306 306
307 pfit_control |= PFIT_ENABLE;
308 /* 965+ is easy, it does everything in hw */ 307 /* 965+ is easy, it does everything in hw */
309 if (scaled_width > scaled_height) 308 if (scaled_width > scaled_height)
310 pfit_control |= PFIT_SCALING_PILLAR; 309 pfit_control |= PFIT_ENABLE | PFIT_SCALING_PILLAR;
311 else if (scaled_width < scaled_height) 310 else if (scaled_width < scaled_height)
312 pfit_control |= PFIT_SCALING_LETTER; 311 pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER;
313 else 312 else if (adjusted_mode->hdisplay != mode->hdisplay)
314 pfit_control |= PFIT_SCALING_AUTO; 313 pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
315 } else { 314 } else {
316 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay; 315 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
317 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay; 316 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
@@ -358,13 +357,17 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
358 * Full scaling, even if it changes the aspect ratio. 357 * Full scaling, even if it changes the aspect ratio.
359 * Fortunately this is all done for us in hw. 358 * Fortunately this is all done for us in hw.
360 */ 359 */
361 pfit_control |= PFIT_ENABLE; 360 if (mode->vdisplay != adjusted_mode->vdisplay ||
362 if (INTEL_INFO(dev)->gen >= 4) 361 mode->hdisplay != adjusted_mode->hdisplay) {
363 pfit_control |= PFIT_SCALING_AUTO; 362 pfit_control |= PFIT_ENABLE;
364 else 363 if (INTEL_INFO(dev)->gen >= 4)
365 pfit_control |= (VERT_AUTO_SCALE | HORIZ_AUTO_SCALE | 364 pfit_control |= PFIT_SCALING_AUTO;
366 VERT_INTERP_BILINEAR | 365 else
367 HORIZ_INTERP_BILINEAR); 366 pfit_control |= (VERT_AUTO_SCALE |
367 VERT_INTERP_BILINEAR |
368 HORIZ_AUTO_SCALE |
369 HORIZ_INTERP_BILINEAR);
370 }
368 break; 371 break;
369 372
370 default: 373 default:
@@ -914,6 +917,8 @@ bool intel_lvds_init(struct drm_device *dev)
914 917
915 intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT); 918 intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT);
916 intel_encoder->crtc_mask = (1 << 1); 919 intel_encoder->crtc_mask = (1 << 1);
920 if (INTEL_INFO(dev)->gen >= 5)
921 intel_encoder->crtc_mask |= (1 << 0);
917 drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs); 922 drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs);
918 drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs); 923 drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs);
919 connector->display_info.subpixel_order = SubPixelHorizontalRGB; 924 connector->display_info.subpixel_order = SubPixelHorizontalRGB;
@@ -1019,10 +1024,18 @@ bool intel_lvds_init(struct drm_device *dev)
1019out: 1024out:
1020 if (HAS_PCH_SPLIT(dev)) { 1025 if (HAS_PCH_SPLIT(dev)) {
1021 u32 pwm; 1026 u32 pwm;
1022 /* make sure PWM is enabled */ 1027
1028 pipe = (I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT) ? 1 : 0;
1029
1030 /* make sure PWM is enabled and locked to the LVDS pipe */
1023 pwm = I915_READ(BLC_PWM_CPU_CTL2); 1031 pwm = I915_READ(BLC_PWM_CPU_CTL2);
1024 pwm |= (PWM_ENABLE | PWM_PIPE_B); 1032 if (pipe == 0 && (pwm & PWM_PIPE_B))
1025 I915_WRITE(BLC_PWM_CPU_CTL2, pwm); 1033 I915_WRITE(BLC_PWM_CPU_CTL2, pwm & ~PWM_ENABLE);
1034 if (pipe)
1035 pwm |= PWM_PIPE_B;
1036 else
1037 pwm &= ~PWM_PIPE_B;
1038 I915_WRITE(BLC_PWM_CPU_CTL2, pwm | PWM_ENABLE);
1026 1039
1027 pwm = I915_READ(BLC_PWM_PCH_CTL1); 1040 pwm = I915_READ(BLC_PWM_PCH_CTL1);
1028 pwm |= PWM_PCH_ENABLE; 1041 pwm |= PWM_PCH_ENABLE;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 9b0d9a867aea..f295a7aaadf9 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -273,14 +273,8 @@ void intel_opregion_enable_asle(struct drm_device *dev)
273 struct opregion_asle *asle = dev_priv->opregion.asle; 273 struct opregion_asle *asle = dev_priv->opregion.asle;
274 274
275 if (asle) { 275 if (asle) {
276 if (IS_MOBILE(dev)) { 276 if (IS_MOBILE(dev))
277 unsigned long irqflags;
278
279 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
280 intel_enable_asle(dev); 277 intel_enable_asle(dev);
281 spin_unlock_irqrestore(&dev_priv->user_irq_lock,
282 irqflags);
283 }
284 278
285 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN | 279 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN |
286 ASLE_PFMB_EN; 280 ASLE_PFMB_EN;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 02ff0a481f47..3fbb98b948d6 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -221,15 +221,16 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
221 int ret; 221 int ret;
222 222
223 BUG_ON(overlay->last_flip_req); 223 BUG_ON(overlay->last_flip_req);
224 overlay->last_flip_req = 224 ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv));
225 i915_add_request(dev, NULL, request, &dev_priv->render_ring); 225 if (ret) {
226 if (overlay->last_flip_req == 0) 226 kfree(request);
227 return -ENOMEM; 227 return ret;
228 228 }
229 overlay->last_flip_req = request->seqno;
229 overlay->flip_tail = tail; 230 overlay->flip_tail = tail;
230 ret = i915_do_wait_request(dev, 231 ret = i915_do_wait_request(dev,
231 overlay->last_flip_req, true, 232 overlay->last_flip_req, true,
232 &dev_priv->render_ring); 233 LP_RING(dev_priv));
233 if (ret) 234 if (ret)
234 return ret; 235 return ret;
235 236
@@ -289,6 +290,7 @@ i830_deactivate_pipe_a(struct drm_device *dev)
289static int intel_overlay_on(struct intel_overlay *overlay) 290static int intel_overlay_on(struct intel_overlay *overlay)
290{ 291{
291 struct drm_device *dev = overlay->dev; 292 struct drm_device *dev = overlay->dev;
293 struct drm_i915_private *dev_priv = dev->dev_private;
292 struct drm_i915_gem_request *request; 294 struct drm_i915_gem_request *request;
293 int pipe_a_quirk = 0; 295 int pipe_a_quirk = 0;
294 int ret; 296 int ret;
@@ -308,7 +310,12 @@ static int intel_overlay_on(struct intel_overlay *overlay)
308 goto out; 310 goto out;
309 } 311 }
310 312
311 BEGIN_LP_RING(4); 313 ret = BEGIN_LP_RING(4);
314 if (ret) {
315 kfree(request);
316 goto out;
317 }
318
312 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON); 319 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON);
313 OUT_RING(overlay->flip_addr | OFC_UPDATE); 320 OUT_RING(overlay->flip_addr | OFC_UPDATE);
314 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); 321 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
@@ -332,6 +339,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
332 struct drm_i915_gem_request *request; 339 struct drm_i915_gem_request *request;
333 u32 flip_addr = overlay->flip_addr; 340 u32 flip_addr = overlay->flip_addr;
334 u32 tmp; 341 u32 tmp;
342 int ret;
335 343
336 BUG_ON(!overlay->active); 344 BUG_ON(!overlay->active);
337 345
@@ -347,36 +355,44 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
347 if (tmp & (1 << 17)) 355 if (tmp & (1 << 17))
348 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); 356 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
349 357
350 BEGIN_LP_RING(2); 358 ret = BEGIN_LP_RING(2);
359 if (ret) {
360 kfree(request);
361 return ret;
362 }
351 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); 363 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
352 OUT_RING(flip_addr); 364 OUT_RING(flip_addr);
353 ADVANCE_LP_RING(); 365 ADVANCE_LP_RING();
354 366
355 overlay->last_flip_req = 367 ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv));
356 i915_add_request(dev, NULL, request, &dev_priv->render_ring); 368 if (ret) {
369 kfree(request);
370 return ret;
371 }
372
373 overlay->last_flip_req = request->seqno;
357 return 0; 374 return 0;
358} 375}
359 376
360static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay) 377static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
361{ 378{
362 struct drm_gem_object *obj = &overlay->old_vid_bo->base; 379 struct drm_i915_gem_object *obj = overlay->old_vid_bo;
363 380
364 i915_gem_object_unpin(obj); 381 i915_gem_object_unpin(obj);
365 drm_gem_object_unreference(obj); 382 drm_gem_object_unreference(&obj->base);
366 383
367 overlay->old_vid_bo = NULL; 384 overlay->old_vid_bo = NULL;
368} 385}
369 386
370static void intel_overlay_off_tail(struct intel_overlay *overlay) 387static void intel_overlay_off_tail(struct intel_overlay *overlay)
371{ 388{
372 struct drm_gem_object *obj; 389 struct drm_i915_gem_object *obj = overlay->vid_bo;
373 390
374 /* never have the overlay hw on without showing a frame */ 391 /* never have the overlay hw on without showing a frame */
375 BUG_ON(!overlay->vid_bo); 392 BUG_ON(!overlay->vid_bo);
376 obj = &overlay->vid_bo->base;
377 393
378 i915_gem_object_unpin(obj); 394 i915_gem_object_unpin(obj);
379 drm_gem_object_unreference(obj); 395 drm_gem_object_unreference(&obj->base);
380 overlay->vid_bo = NULL; 396 overlay->vid_bo = NULL;
381 397
382 overlay->crtc->overlay = NULL; 398 overlay->crtc->overlay = NULL;
@@ -389,8 +405,10 @@ static int intel_overlay_off(struct intel_overlay *overlay,
389 bool interruptible) 405 bool interruptible)
390{ 406{
391 struct drm_device *dev = overlay->dev; 407 struct drm_device *dev = overlay->dev;
408 struct drm_i915_private *dev_priv = dev->dev_private;
392 u32 flip_addr = overlay->flip_addr; 409 u32 flip_addr = overlay->flip_addr;
393 struct drm_i915_gem_request *request; 410 struct drm_i915_gem_request *request;
411 int ret;
394 412
395 BUG_ON(!overlay->active); 413 BUG_ON(!overlay->active);
396 414
@@ -404,7 +422,11 @@ static int intel_overlay_off(struct intel_overlay *overlay,
404 * of the hw. Do it in both cases */ 422 * of the hw. Do it in both cases */
405 flip_addr |= OFC_UPDATE; 423 flip_addr |= OFC_UPDATE;
406 424
407 BEGIN_LP_RING(6); 425 ret = BEGIN_LP_RING(6);
426 if (ret) {
427 kfree(request);
428 return ret;
429 }
408 /* wait for overlay to go idle */ 430 /* wait for overlay to go idle */
409 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); 431 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
410 OUT_RING(flip_addr); 432 OUT_RING(flip_addr);
@@ -432,7 +454,7 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay,
432 return 0; 454 return 0;
433 455
434 ret = i915_do_wait_request(dev, overlay->last_flip_req, 456 ret = i915_do_wait_request(dev, overlay->last_flip_req,
435 interruptible, &dev_priv->render_ring); 457 interruptible, LP_RING(dev_priv));
436 if (ret) 458 if (ret)
437 return ret; 459 return ret;
438 460
@@ -467,7 +489,12 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
467 if (request == NULL) 489 if (request == NULL)
468 return -ENOMEM; 490 return -ENOMEM;
469 491
470 BEGIN_LP_RING(2); 492 ret = BEGIN_LP_RING(2);
493 if (ret) {
494 kfree(request);
495 return ret;
496 }
497
471 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); 498 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
472 OUT_RING(MI_NOOP); 499 OUT_RING(MI_NOOP);
473 ADVANCE_LP_RING(); 500 ADVANCE_LP_RING();
@@ -736,13 +763,12 @@ static u32 overlay_cmd_reg(struct put_image_params *params)
736} 763}
737 764
738static int intel_overlay_do_put_image(struct intel_overlay *overlay, 765static int intel_overlay_do_put_image(struct intel_overlay *overlay,
739 struct drm_gem_object *new_bo, 766 struct drm_i915_gem_object *new_bo,
740 struct put_image_params *params) 767 struct put_image_params *params)
741{ 768{
742 int ret, tmp_width; 769 int ret, tmp_width;
743 struct overlay_registers *regs; 770 struct overlay_registers *regs;
744 bool scale_changed = false; 771 bool scale_changed = false;
745 struct drm_i915_gem_object *bo_priv = to_intel_bo(new_bo);
746 struct drm_device *dev = overlay->dev; 772 struct drm_device *dev = overlay->dev;
747 773
748 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 774 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -753,7 +779,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
753 if (ret != 0) 779 if (ret != 0)
754 return ret; 780 return ret;
755 781
756 ret = i915_gem_object_pin(new_bo, PAGE_SIZE); 782 ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true);
757 if (ret != 0) 783 if (ret != 0)
758 return ret; 784 return ret;
759 785
@@ -761,6 +787,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
761 if (ret != 0) 787 if (ret != 0)
762 goto out_unpin; 788 goto out_unpin;
763 789
790 ret = i915_gem_object_put_fence(new_bo);
791 if (ret)
792 goto out_unpin;
793
764 if (!overlay->active) { 794 if (!overlay->active) {
765 regs = intel_overlay_map_regs(overlay); 795 regs = intel_overlay_map_regs(overlay);
766 if (!regs) { 796 if (!regs) {
@@ -797,7 +827,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
797 regs->SWIDTHSW = calc_swidthsw(overlay->dev, 827 regs->SWIDTHSW = calc_swidthsw(overlay->dev,
798 params->offset_Y, tmp_width); 828 params->offset_Y, tmp_width);
799 regs->SHEIGHT = params->src_h; 829 regs->SHEIGHT = params->src_h;
800 regs->OBUF_0Y = bo_priv->gtt_offset + params-> offset_Y; 830 regs->OBUF_0Y = new_bo->gtt_offset + params-> offset_Y;
801 regs->OSTRIDE = params->stride_Y; 831 regs->OSTRIDE = params->stride_Y;
802 832
803 if (params->format & I915_OVERLAY_YUV_PLANAR) { 833 if (params->format & I915_OVERLAY_YUV_PLANAR) {
@@ -811,8 +841,8 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
811 params->src_w/uv_hscale); 841 params->src_w/uv_hscale);
812 regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16; 842 regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16;
813 regs->SHEIGHT |= (params->src_h/uv_vscale) << 16; 843 regs->SHEIGHT |= (params->src_h/uv_vscale) << 16;
814 regs->OBUF_0U = bo_priv->gtt_offset + params->offset_U; 844 regs->OBUF_0U = new_bo->gtt_offset + params->offset_U;
815 regs->OBUF_0V = bo_priv->gtt_offset + params->offset_V; 845 regs->OBUF_0V = new_bo->gtt_offset + params->offset_V;
816 regs->OSTRIDE |= params->stride_UV << 16; 846 regs->OSTRIDE |= params->stride_UV << 16;
817 } 847 }
818 848
@@ -829,7 +859,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
829 goto out_unpin; 859 goto out_unpin;
830 860
831 overlay->old_vid_bo = overlay->vid_bo; 861 overlay->old_vid_bo = overlay->vid_bo;
832 overlay->vid_bo = to_intel_bo(new_bo); 862 overlay->vid_bo = new_bo;
833 863
834 return 0; 864 return 0;
835 865
@@ -942,7 +972,7 @@ static int check_overlay_scaling(struct put_image_params *rec)
942 972
943static int check_overlay_src(struct drm_device *dev, 973static int check_overlay_src(struct drm_device *dev,
944 struct drm_intel_overlay_put_image *rec, 974 struct drm_intel_overlay_put_image *rec,
945 struct drm_gem_object *new_bo) 975 struct drm_i915_gem_object *new_bo)
946{ 976{
947 int uv_hscale = uv_hsubsampling(rec->flags); 977 int uv_hscale = uv_hsubsampling(rec->flags);
948 int uv_vscale = uv_vsubsampling(rec->flags); 978 int uv_vscale = uv_vsubsampling(rec->flags);
@@ -1027,7 +1057,7 @@ static int check_overlay_src(struct drm_device *dev,
1027 return -EINVAL; 1057 return -EINVAL;
1028 1058
1029 tmp = rec->stride_Y*rec->src_height; 1059 tmp = rec->stride_Y*rec->src_height;
1030 if (rec->offset_Y + tmp > new_bo->size) 1060 if (rec->offset_Y + tmp > new_bo->base.size)
1031 return -EINVAL; 1061 return -EINVAL;
1032 break; 1062 break;
1033 1063
@@ -1038,12 +1068,12 @@ static int check_overlay_src(struct drm_device *dev,
1038 return -EINVAL; 1068 return -EINVAL;
1039 1069
1040 tmp = rec->stride_Y * rec->src_height; 1070 tmp = rec->stride_Y * rec->src_height;
1041 if (rec->offset_Y + tmp > new_bo->size) 1071 if (rec->offset_Y + tmp > new_bo->base.size)
1042 return -EINVAL; 1072 return -EINVAL;
1043 1073
1044 tmp = rec->stride_UV * (rec->src_height / uv_vscale); 1074 tmp = rec->stride_UV * (rec->src_height / uv_vscale);
1045 if (rec->offset_U + tmp > new_bo->size || 1075 if (rec->offset_U + tmp > new_bo->base.size ||
1046 rec->offset_V + tmp > new_bo->size) 1076 rec->offset_V + tmp > new_bo->base.size)
1047 return -EINVAL; 1077 return -EINVAL;
1048 break; 1078 break;
1049 } 1079 }
@@ -1086,7 +1116,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1086 struct intel_overlay *overlay; 1116 struct intel_overlay *overlay;
1087 struct drm_mode_object *drmmode_obj; 1117 struct drm_mode_object *drmmode_obj;
1088 struct intel_crtc *crtc; 1118 struct intel_crtc *crtc;
1089 struct drm_gem_object *new_bo; 1119 struct drm_i915_gem_object *new_bo;
1090 struct put_image_params *params; 1120 struct put_image_params *params;
1091 int ret; 1121 int ret;
1092 1122
@@ -1125,8 +1155,8 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1125 } 1155 }
1126 crtc = to_intel_crtc(obj_to_crtc(drmmode_obj)); 1156 crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
1127 1157
1128 new_bo = drm_gem_object_lookup(dev, file_priv, 1158 new_bo = to_intel_bo(drm_gem_object_lookup(dev, file_priv,
1129 put_image_rec->bo_handle); 1159 put_image_rec->bo_handle));
1130 if (!new_bo) { 1160 if (!new_bo) {
1131 ret = -ENOENT; 1161 ret = -ENOENT;
1132 goto out_free; 1162 goto out_free;
@@ -1135,6 +1165,12 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1135 mutex_lock(&dev->mode_config.mutex); 1165 mutex_lock(&dev->mode_config.mutex);
1136 mutex_lock(&dev->struct_mutex); 1166 mutex_lock(&dev->struct_mutex);
1137 1167
1168 if (new_bo->tiling_mode) {
1169 DRM_ERROR("buffer used for overlay image can not be tiled\n");
1170 ret = -EINVAL;
1171 goto out_unlock;
1172 }
1173
1138 ret = intel_overlay_recover_from_interrupt(overlay, true); 1174 ret = intel_overlay_recover_from_interrupt(overlay, true);
1139 if (ret != 0) 1175 if (ret != 0)
1140 goto out_unlock; 1176 goto out_unlock;
@@ -1217,7 +1253,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1217out_unlock: 1253out_unlock:
1218 mutex_unlock(&dev->struct_mutex); 1254 mutex_unlock(&dev->struct_mutex);
1219 mutex_unlock(&dev->mode_config.mutex); 1255 mutex_unlock(&dev->mode_config.mutex);
1220 drm_gem_object_unreference_unlocked(new_bo); 1256 drm_gem_object_unreference_unlocked(&new_bo->base);
1221out_free: 1257out_free:
1222 kfree(params); 1258 kfree(params);
1223 1259
@@ -1370,7 +1406,7 @@ void intel_setup_overlay(struct drm_device *dev)
1370{ 1406{
1371 drm_i915_private_t *dev_priv = dev->dev_private; 1407 drm_i915_private_t *dev_priv = dev->dev_private;
1372 struct intel_overlay *overlay; 1408 struct intel_overlay *overlay;
1373 struct drm_gem_object *reg_bo; 1409 struct drm_i915_gem_object *reg_bo;
1374 struct overlay_registers *regs; 1410 struct overlay_registers *regs;
1375 int ret; 1411 int ret;
1376 1412
@@ -1385,7 +1421,7 @@ void intel_setup_overlay(struct drm_device *dev)
1385 reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE); 1421 reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE);
1386 if (!reg_bo) 1422 if (!reg_bo)
1387 goto out_free; 1423 goto out_free;
1388 overlay->reg_bo = to_intel_bo(reg_bo); 1424 overlay->reg_bo = reg_bo;
1389 1425
1390 if (OVERLAY_NEEDS_PHYSICAL(dev)) { 1426 if (OVERLAY_NEEDS_PHYSICAL(dev)) {
1391 ret = i915_gem_attach_phys_object(dev, reg_bo, 1427 ret = i915_gem_attach_phys_object(dev, reg_bo,
@@ -1395,14 +1431,14 @@ void intel_setup_overlay(struct drm_device *dev)
1395 DRM_ERROR("failed to attach phys overlay regs\n"); 1431 DRM_ERROR("failed to attach phys overlay regs\n");
1396 goto out_free_bo; 1432 goto out_free_bo;
1397 } 1433 }
1398 overlay->flip_addr = overlay->reg_bo->phys_obj->handle->busaddr; 1434 overlay->flip_addr = reg_bo->phys_obj->handle->busaddr;
1399 } else { 1435 } else {
1400 ret = i915_gem_object_pin(reg_bo, PAGE_SIZE); 1436 ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true);
1401 if (ret) { 1437 if (ret) {
1402 DRM_ERROR("failed to pin overlay register bo\n"); 1438 DRM_ERROR("failed to pin overlay register bo\n");
1403 goto out_free_bo; 1439 goto out_free_bo;
1404 } 1440 }
1405 overlay->flip_addr = overlay->reg_bo->gtt_offset; 1441 overlay->flip_addr = reg_bo->gtt_offset;
1406 1442
1407 ret = i915_gem_object_set_to_gtt_domain(reg_bo, true); 1443 ret = i915_gem_object_set_to_gtt_domain(reg_bo, true);
1408 if (ret) { 1444 if (ret) {
@@ -1434,7 +1470,7 @@ void intel_setup_overlay(struct drm_device *dev)
1434out_unpin_bo: 1470out_unpin_bo:
1435 i915_gem_object_unpin(reg_bo); 1471 i915_gem_object_unpin(reg_bo);
1436out_free_bo: 1472out_free_bo:
1437 drm_gem_object_unreference(reg_bo); 1473 drm_gem_object_unreference(&reg_bo->base);
1438out_free: 1474out_free:
1439 kfree(overlay); 1475 kfree(overlay);
1440 return; 1476 return;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 92ff8f385278..7350ec2515c6 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -125,15 +125,55 @@ static int is_backlight_combination_mode(struct drm_device *dev)
125 return 0; 125 return 0;
126} 126}
127 127
128static u32 i915_read_blc_pwm_ctl(struct drm_i915_private *dev_priv)
129{
130 u32 val;
131
132 /* Restore the CTL value if it lost, e.g. GPU reset */
133
134 if (HAS_PCH_SPLIT(dev_priv->dev)) {
135 val = I915_READ(BLC_PWM_PCH_CTL2);
136 if (dev_priv->saveBLC_PWM_CTL2 == 0) {
137 dev_priv->saveBLC_PWM_CTL2 = val;
138 } else if (val == 0) {
139 I915_WRITE(BLC_PWM_PCH_CTL2,
140 dev_priv->saveBLC_PWM_CTL);
141 val = dev_priv->saveBLC_PWM_CTL;
142 }
143 } else {
144 val = I915_READ(BLC_PWM_CTL);
145 if (dev_priv->saveBLC_PWM_CTL == 0) {
146 dev_priv->saveBLC_PWM_CTL = val;
147 dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
148 } else if (val == 0) {
149 I915_WRITE(BLC_PWM_CTL,
150 dev_priv->saveBLC_PWM_CTL);
151 I915_WRITE(BLC_PWM_CTL2,
152 dev_priv->saveBLC_PWM_CTL2);
153 val = dev_priv->saveBLC_PWM_CTL;
154 }
155 }
156
157 return val;
158}
159
128u32 intel_panel_get_max_backlight(struct drm_device *dev) 160u32 intel_panel_get_max_backlight(struct drm_device *dev)
129{ 161{
130 struct drm_i915_private *dev_priv = dev->dev_private; 162 struct drm_i915_private *dev_priv = dev->dev_private;
131 u32 max; 163 u32 max;
132 164
165 max = i915_read_blc_pwm_ctl(dev_priv);
166 if (max == 0) {
167 /* XXX add code here to query mode clock or hardware clock
168 * and program max PWM appropriately.
169 */
170 printk_once(KERN_WARNING "fixme: max PWM is zero.\n");
171 return 1;
172 }
173
133 if (HAS_PCH_SPLIT(dev)) { 174 if (HAS_PCH_SPLIT(dev)) {
134 max = I915_READ(BLC_PWM_PCH_CTL2) >> 16; 175 max >>= 16;
135 } else { 176 } else {
136 max = I915_READ(BLC_PWM_CTL);
137 if (IS_PINEVIEW(dev)) { 177 if (IS_PINEVIEW(dev)) {
138 max >>= 17; 178 max >>= 17;
139 } else { 179 } else {
@@ -146,14 +186,6 @@ u32 intel_panel_get_max_backlight(struct drm_device *dev)
146 max *= 0xff; 186 max *= 0xff;
147 } 187 }
148 188
149 if (max == 0) {
150 /* XXX add code here to query mode clock or hardware clock
151 * and program max PWM appropriately.
152 */
153 DRM_ERROR("fixme: max PWM is zero.\n");
154 max = 1;
155 }
156
157 DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max); 189 DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max);
158 return max; 190 return max;
159} 191}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 31cd7e33e820..a3fd993e0de0 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -49,11 +49,11 @@ static u32 i915_gem_get_seqno(struct drm_device *dev)
49} 49}
50 50
51static void 51static void
52render_ring_flush(struct drm_device *dev, 52render_ring_flush(struct intel_ring_buffer *ring,
53 struct intel_ring_buffer *ring,
54 u32 invalidate_domains, 53 u32 invalidate_domains,
55 u32 flush_domains) 54 u32 flush_domains)
56{ 55{
56 struct drm_device *dev = ring->dev;
57 drm_i915_private_t *dev_priv = dev->dev_private; 57 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd; 58 u32 cmd;
59 59
@@ -109,49 +109,50 @@ render_ring_flush(struct drm_device *dev,
109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) 109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110 cmd |= MI_EXE_FLUSH; 110 cmd |= MI_EXE_FLUSH;
111 111
112 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
113 (IS_G4X(dev) || IS_GEN5(dev)))
114 cmd |= MI_INVALIDATE_ISP;
115
112#if WATCH_EXEC 116#if WATCH_EXEC
113 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd); 117 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114#endif 118#endif
115 intel_ring_begin(dev, ring, 2); 119 if (intel_ring_begin(ring, 2) == 0) {
116 intel_ring_emit(dev, ring, cmd); 120 intel_ring_emit(ring, cmd);
117 intel_ring_emit(dev, ring, MI_NOOP); 121 intel_ring_emit(ring, MI_NOOP);
118 intel_ring_advance(dev, ring); 122 intel_ring_advance(ring);
123 }
119 } 124 }
120} 125}
121 126
122static void ring_write_tail(struct drm_device *dev, 127static void ring_write_tail(struct intel_ring_buffer *ring,
123 struct intel_ring_buffer *ring,
124 u32 value) 128 u32 value)
125{ 129{
126 drm_i915_private_t *dev_priv = dev->dev_private; 130 drm_i915_private_t *dev_priv = ring->dev->dev_private;
127 I915_WRITE_TAIL(ring, value); 131 I915_WRITE_TAIL(ring, value);
128} 132}
129 133
130u32 intel_ring_get_active_head(struct drm_device *dev, 134u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
131 struct intel_ring_buffer *ring)
132{ 135{
133 drm_i915_private_t *dev_priv = dev->dev_private; 136 drm_i915_private_t *dev_priv = ring->dev->dev_private;
134 u32 acthd_reg = INTEL_INFO(dev)->gen >= 4 ? 137 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
135 RING_ACTHD(ring->mmio_base) : ACTHD; 138 RING_ACTHD(ring->mmio_base) : ACTHD;
136 139
137 return I915_READ(acthd_reg); 140 return I915_READ(acthd_reg);
138} 141}
139 142
140static int init_ring_common(struct drm_device *dev, 143static int init_ring_common(struct intel_ring_buffer *ring)
141 struct intel_ring_buffer *ring)
142{ 144{
145 drm_i915_private_t *dev_priv = ring->dev->dev_private;
146 struct drm_i915_gem_object *obj = ring->obj;
143 u32 head; 147 u32 head;
144 drm_i915_private_t *dev_priv = dev->dev_private;
145 struct drm_i915_gem_object *obj_priv;
146 obj_priv = to_intel_bo(ring->gem_object);
147 148
148 /* Stop the ring if it's running. */ 149 /* Stop the ring if it's running. */
149 I915_WRITE_CTL(ring, 0); 150 I915_WRITE_CTL(ring, 0);
150 I915_WRITE_HEAD(ring, 0); 151 I915_WRITE_HEAD(ring, 0);
151 ring->write_tail(dev, ring, 0); 152 ring->write_tail(ring, 0);
152 153
153 /* Initialize the ring. */ 154 /* Initialize the ring. */
154 I915_WRITE_START(ring, obj_priv->gtt_offset); 155 I915_WRITE_START(ring, obj->gtt_offset);
155 head = I915_READ_HEAD(ring) & HEAD_ADDR; 156 head = I915_READ_HEAD(ring) & HEAD_ADDR;
156 157
157 /* G45 ring initialization fails to reset head to zero */ 158 /* G45 ring initialization fails to reset head to zero */
@@ -178,12 +179,13 @@ static int init_ring_common(struct drm_device *dev,
178 } 179 }
179 180
180 I915_WRITE_CTL(ring, 181 I915_WRITE_CTL(ring,
181 ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES) 182 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
182 | RING_REPORT_64K | RING_VALID); 183 | RING_REPORT_64K | RING_VALID);
183 184
184 head = I915_READ_HEAD(ring) & HEAD_ADDR;
185 /* If the head is still not zero, the ring is dead */ 185 /* If the head is still not zero, the ring is dead */
186 if (head != 0) { 186 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
187 I915_READ_START(ring) != obj->gtt_offset ||
188 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
187 DRM_ERROR("%s initialization failed " 189 DRM_ERROR("%s initialization failed "
188 "ctl %08x head %08x tail %08x start %08x\n", 190 "ctl %08x head %08x tail %08x start %08x\n",
189 ring->name, 191 ring->name,
@@ -194,8 +196,8 @@ static int init_ring_common(struct drm_device *dev,
194 return -EIO; 196 return -EIO;
195 } 197 }
196 198
197 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 199 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
198 i915_kernel_lost_context(dev); 200 i915_kernel_lost_context(ring->dev);
199 else { 201 else {
200 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; 202 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
201 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; 203 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
@@ -203,335 +205,346 @@ static int init_ring_common(struct drm_device *dev,
203 if (ring->space < 0) 205 if (ring->space < 0)
204 ring->space += ring->size; 206 ring->space += ring->size;
205 } 207 }
208
206 return 0; 209 return 0;
207} 210}
208 211
209static int init_render_ring(struct drm_device *dev, 212static int init_render_ring(struct intel_ring_buffer *ring)
210 struct intel_ring_buffer *ring)
211{ 213{
212 drm_i915_private_t *dev_priv = dev->dev_private; 214 struct drm_device *dev = ring->dev;
213 int ret = init_ring_common(dev, ring); 215 struct drm_i915_private *dev_priv = dev->dev_private;
214 int mode; 216 int ret = init_ring_common(ring);
215 217
216 if (INTEL_INFO(dev)->gen > 3) { 218 if (INTEL_INFO(dev)->gen > 3) {
217 mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; 219 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
218 if (IS_GEN6(dev)) 220 if (IS_GEN6(dev))
219 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE; 221 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
220 I915_WRITE(MI_MODE, mode); 222 I915_WRITE(MI_MODE, mode);
221 } 223 }
224
222 return ret; 225 return ret;
223} 226}
224 227
225#define PIPE_CONTROL_FLUSH(addr) \ 228static void
226do { \ 229update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
227 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ 230{
228 PIPE_CONTROL_DEPTH_STALL | 2); \ 231 struct drm_device *dev = ring->dev;
229 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \ 232 struct drm_i915_private *dev_priv = dev->dev_private;
230 OUT_RING(0); \ 233 int id;
231 OUT_RING(0); \ 234
232} while (0) 235 /*
233 236 * cs -> 1 = vcs, 0 = bcs
234/** 237 * vcs -> 1 = bcs, 0 = cs,
235 * Creates a new sequence number, emitting a write of it to the status page 238 * bcs -> 1 = cs, 0 = vcs.
236 * plus an interrupt, which will trigger i915_user_interrupt_handler. 239 */
237 * 240 id = ring - dev_priv->ring;
238 * Must be called with struct_lock held. 241 id += 2 - i;
239 * 242 id %= 3;
240 * Returned sequence numbers are nonzero on success. 243
241 */ 244 intel_ring_emit(ring,
242static u32 245 MI_SEMAPHORE_MBOX |
243render_ring_add_request(struct drm_device *dev, 246 MI_SEMAPHORE_REGISTER |
244 struct intel_ring_buffer *ring, 247 MI_SEMAPHORE_UPDATE);
245 u32 flush_domains) 248 intel_ring_emit(ring, seqno);
249 intel_ring_emit(ring,
250 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
251}
252
253static int
254gen6_add_request(struct intel_ring_buffer *ring,
255 u32 *result)
246{ 256{
247 drm_i915_private_t *dev_priv = dev->dev_private;
248 u32 seqno; 257 u32 seqno;
258 int ret;
249 259
250 seqno = i915_gem_get_seqno(dev); 260 ret = intel_ring_begin(ring, 10);
251 261 if (ret)
252 if (IS_GEN6(dev)) { 262 return ret;
253 BEGIN_LP_RING(6);
254 OUT_RING(GFX_OP_PIPE_CONTROL | 3);
255 OUT_RING(PIPE_CONTROL_QW_WRITE |
256 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
257 PIPE_CONTROL_NOTIFY);
258 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
259 OUT_RING(seqno);
260 OUT_RING(0);
261 OUT_RING(0);
262 ADVANCE_LP_RING();
263 } else if (HAS_PIPE_CONTROL(dev)) {
264 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
265 263
266 /* 264 seqno = i915_gem_get_seqno(ring->dev);
267 * Workaround qword write incoherence by flushing the 265 update_semaphore(ring, 0, seqno);
268 * PIPE_NOTIFY buffers out to memory before requesting 266 update_semaphore(ring, 1, seqno);
269 * an interrupt.
270 */
271 BEGIN_LP_RING(32);
272 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
273 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
274 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
275 OUT_RING(seqno);
276 OUT_RING(0);
277 PIPE_CONTROL_FLUSH(scratch_addr);
278 scratch_addr += 128; /* write to separate cachelines */
279 PIPE_CONTROL_FLUSH(scratch_addr);
280 scratch_addr += 128;
281 PIPE_CONTROL_FLUSH(scratch_addr);
282 scratch_addr += 128;
283 PIPE_CONTROL_FLUSH(scratch_addr);
284 scratch_addr += 128;
285 PIPE_CONTROL_FLUSH(scratch_addr);
286 scratch_addr += 128;
287 PIPE_CONTROL_FLUSH(scratch_addr);
288 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
289 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
290 PIPE_CONTROL_NOTIFY);
291 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
292 OUT_RING(seqno);
293 OUT_RING(0);
294 ADVANCE_LP_RING();
295 } else {
296 BEGIN_LP_RING(4);
297 OUT_RING(MI_STORE_DWORD_INDEX);
298 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
299 OUT_RING(seqno);
300 267
301 OUT_RING(MI_USER_INTERRUPT); 268 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
302 ADVANCE_LP_RING(); 269 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
303 } 270 intel_ring_emit(ring, seqno);
304 return seqno; 271 intel_ring_emit(ring, MI_USER_INTERRUPT);
272 intel_ring_advance(ring);
273
274 *result = seqno;
275 return 0;
276}
277
278int
279intel_ring_sync(struct intel_ring_buffer *ring,
280 struct intel_ring_buffer *to,
281 u32 seqno)
282{
283 int ret;
284
285 ret = intel_ring_begin(ring, 4);
286 if (ret)
287 return ret;
288
289 intel_ring_emit(ring,
290 MI_SEMAPHORE_MBOX |
291 MI_SEMAPHORE_REGISTER |
292 intel_ring_sync_index(ring, to) << 17 |
293 MI_SEMAPHORE_COMPARE);
294 intel_ring_emit(ring, seqno);
295 intel_ring_emit(ring, 0);
296 intel_ring_emit(ring, MI_NOOP);
297 intel_ring_advance(ring);
298
299 return 0;
300}
301
302static int
303render_ring_add_request(struct intel_ring_buffer *ring,
304 u32 *result)
305{
306 struct drm_device *dev = ring->dev;
307 u32 seqno = i915_gem_get_seqno(dev);
308 int ret;
309
310 ret = intel_ring_begin(ring, 4);
311 if (ret)
312 return ret;
313
314 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
315 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
316 intel_ring_emit(ring, seqno);
317 intel_ring_emit(ring, MI_USER_INTERRUPT);
318 intel_ring_advance(ring);
319
320 *result = seqno;
321 return 0;
305} 322}
306 323
307static u32 324static u32
308render_ring_get_seqno(struct drm_device *dev, 325ring_get_seqno(struct intel_ring_buffer *ring)
309 struct intel_ring_buffer *ring)
310{ 326{
311 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 327 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
312 if (HAS_PIPE_CONTROL(dev))
313 return ((volatile u32 *)(dev_priv->seqno_page))[0];
314 else
315 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
316} 328}
317 329
318static void 330static bool
319render_ring_get_user_irq(struct drm_device *dev, 331render_ring_get_irq(struct intel_ring_buffer *ring)
320 struct intel_ring_buffer *ring)
321{ 332{
322 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 333 struct drm_device *dev = ring->dev;
323 unsigned long irqflags; 334
335 if (!dev->irq_enabled)
336 return false;
337
338 if (atomic_inc_return(&ring->irq_refcount) == 1) {
339 drm_i915_private_t *dev_priv = dev->dev_private;
340 unsigned long irqflags;
324 341
325 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 342 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
326 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
327 if (HAS_PCH_SPLIT(dev)) 343 if (HAS_PCH_SPLIT(dev))
328 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 344 ironlake_enable_graphics_irq(dev_priv,
345 GT_USER_INTERRUPT);
329 else 346 else
330 i915_enable_irq(dev_priv, I915_USER_INTERRUPT); 347 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
348 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
331 } 349 }
332 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 350
351 return true;
333} 352}
334 353
335static void 354static void
336render_ring_put_user_irq(struct drm_device *dev, 355render_ring_put_irq(struct intel_ring_buffer *ring)
337 struct intel_ring_buffer *ring)
338{ 356{
339 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 357 struct drm_device *dev = ring->dev;
340 unsigned long irqflags; 358
359 if (atomic_dec_and_test(&ring->irq_refcount)) {
360 drm_i915_private_t *dev_priv = dev->dev_private;
361 unsigned long irqflags;
341 362
342 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 363 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
343 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
344 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
345 if (HAS_PCH_SPLIT(dev)) 364 if (HAS_PCH_SPLIT(dev))
346 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 365 ironlake_disable_graphics_irq(dev_priv,
366 GT_USER_INTERRUPT);
347 else 367 else
348 i915_disable_irq(dev_priv, I915_USER_INTERRUPT); 368 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
369 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
349 } 370 }
350 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
351} 371}
352 372
353void intel_ring_setup_status_page(struct drm_device *dev, 373void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
354 struct intel_ring_buffer *ring)
355{ 374{
356 drm_i915_private_t *dev_priv = dev->dev_private; 375 drm_i915_private_t *dev_priv = ring->dev->dev_private;
357 if (IS_GEN6(dev)) { 376 u32 mmio = IS_GEN6(ring->dev) ?
358 I915_WRITE(RING_HWS_PGA_GEN6(ring->mmio_base), 377 RING_HWS_PGA_GEN6(ring->mmio_base) :
359 ring->status_page.gfx_addr); 378 RING_HWS_PGA(ring->mmio_base);
360 I915_READ(RING_HWS_PGA_GEN6(ring->mmio_base)); /* posting read */ 379 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
361 } else { 380 POSTING_READ(mmio);
362 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
363 ring->status_page.gfx_addr);
364 I915_READ(RING_HWS_PGA(ring->mmio_base)); /* posting read */
365 }
366
367} 381}
368 382
369static void 383static void
370bsd_ring_flush(struct drm_device *dev, 384bsd_ring_flush(struct intel_ring_buffer *ring,
371 struct intel_ring_buffer *ring, 385 u32 invalidate_domains,
372 u32 invalidate_domains, 386 u32 flush_domains)
373 u32 flush_domains)
374{ 387{
375 intel_ring_begin(dev, ring, 2); 388 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
376 intel_ring_emit(dev, ring, MI_FLUSH); 389 return;
377 intel_ring_emit(dev, ring, MI_NOOP);
378 intel_ring_advance(dev, ring);
379}
380 390
381static int init_bsd_ring(struct drm_device *dev, 391 if (intel_ring_begin(ring, 2) == 0) {
382 struct intel_ring_buffer *ring) 392 intel_ring_emit(ring, MI_FLUSH);
383{ 393 intel_ring_emit(ring, MI_NOOP);
384 return init_ring_common(dev, ring); 394 intel_ring_advance(ring);
395 }
385} 396}
386 397
387static u32 398static int
388ring_add_request(struct drm_device *dev, 399ring_add_request(struct intel_ring_buffer *ring,
389 struct intel_ring_buffer *ring, 400 u32 *result)
390 u32 flush_domains)
391{ 401{
392 u32 seqno; 402 u32 seqno;
403 int ret;
393 404
394 seqno = i915_gem_get_seqno(dev); 405 ret = intel_ring_begin(ring, 4);
406 if (ret)
407 return ret;
395 408
396 intel_ring_begin(dev, ring, 4); 409 seqno = i915_gem_get_seqno(ring->dev);
397 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
398 intel_ring_emit(dev, ring,
399 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
400 intel_ring_emit(dev, ring, seqno);
401 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
402 intel_ring_advance(dev, ring);
403 410
404 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); 411 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
412 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
413 intel_ring_emit(ring, seqno);
414 intel_ring_emit(ring, MI_USER_INTERRUPT);
415 intel_ring_advance(ring);
405 416
406 return seqno; 417 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
418 *result = seqno;
419 return 0;
407} 420}
408 421
409static void 422static bool
410bsd_ring_get_user_irq(struct drm_device *dev, 423ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
411 struct intel_ring_buffer *ring)
412{ 424{
413 /* do nothing */ 425 struct drm_device *dev = ring->dev;
426
427 if (!dev->irq_enabled)
428 return false;
429
430 if (atomic_inc_return(&ring->irq_refcount) == 1) {
431 drm_i915_private_t *dev_priv = dev->dev_private;
432 unsigned long irqflags;
433
434 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
435 ironlake_enable_graphics_irq(dev_priv, flag);
436 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
437 }
438
439 return true;
414} 440}
441
415static void 442static void
416bsd_ring_put_user_irq(struct drm_device *dev, 443ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
417 struct intel_ring_buffer *ring)
418{ 444{
419 /* do nothing */ 445 struct drm_device *dev = ring->dev;
446
447 if (atomic_dec_and_test(&ring->irq_refcount)) {
448 drm_i915_private_t *dev_priv = dev->dev_private;
449 unsigned long irqflags;
450
451 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
452 ironlake_disable_graphics_irq(dev_priv, flag);
453 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
454 }
420} 455}
421 456
422static u32 457static bool
423ring_status_page_get_seqno(struct drm_device *dev, 458bsd_ring_get_irq(struct intel_ring_buffer *ring)
424 struct intel_ring_buffer *ring)
425{ 459{
426 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 460 return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
461}
462static void
463bsd_ring_put_irq(struct intel_ring_buffer *ring)
464{
465 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
427} 466}
428 467
429static int 468static int
430ring_dispatch_gem_execbuffer(struct drm_device *dev, 469ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
431 struct intel_ring_buffer *ring,
432 struct drm_i915_gem_execbuffer2 *exec,
433 struct drm_clip_rect *cliprects,
434 uint64_t exec_offset)
435{ 470{
436 uint32_t exec_start; 471 int ret;
437 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 472
438 intel_ring_begin(dev, ring, 2); 473 ret = intel_ring_begin(ring, 2);
439 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START | 474 if (ret)
440 (2 << 6) | MI_BATCH_NON_SECURE_I965); 475 return ret;
441 intel_ring_emit(dev, ring, exec_start); 476
442 intel_ring_advance(dev, ring); 477 intel_ring_emit(ring,
478 MI_BATCH_BUFFER_START | (2 << 6) |
479 MI_BATCH_NON_SECURE_I965);
480 intel_ring_emit(ring, offset);
481 intel_ring_advance(ring);
482
443 return 0; 483 return 0;
444} 484}
445 485
446static int 486static int
447render_ring_dispatch_gem_execbuffer(struct drm_device *dev, 487render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
448 struct intel_ring_buffer *ring, 488 u32 offset, u32 len)
449 struct drm_i915_gem_execbuffer2 *exec,
450 struct drm_clip_rect *cliprects,
451 uint64_t exec_offset)
452{ 489{
490 struct drm_device *dev = ring->dev;
453 drm_i915_private_t *dev_priv = dev->dev_private; 491 drm_i915_private_t *dev_priv = dev->dev_private;
454 int nbox = exec->num_cliprects; 492 int ret;
455 int i = 0, count;
456 uint32_t exec_start, exec_len;
457 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
458 exec_len = (uint32_t) exec->batch_len;
459 493
460 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1); 494 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
461 495
462 count = nbox ? nbox : 1; 496 if (IS_I830(dev) || IS_845G(dev)) {
497 ret = intel_ring_begin(ring, 4);
498 if (ret)
499 return ret;
463 500
464 for (i = 0; i < count; i++) { 501 intel_ring_emit(ring, MI_BATCH_BUFFER);
465 if (i < nbox) { 502 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
466 int ret = i915_emit_box(dev, cliprects, i, 503 intel_ring_emit(ring, offset + len - 8);
467 exec->DR1, exec->DR4); 504 intel_ring_emit(ring, 0);
468 if (ret) 505 } else {
469 return ret; 506 ret = intel_ring_begin(ring, 2);
470 } 507 if (ret)
508 return ret;
471 509
472 if (IS_I830(dev) || IS_845G(dev)) { 510 if (INTEL_INFO(dev)->gen >= 4) {
473 intel_ring_begin(dev, ring, 4); 511 intel_ring_emit(ring,
474 intel_ring_emit(dev, ring, MI_BATCH_BUFFER); 512 MI_BATCH_BUFFER_START | (2 << 6) |
475 intel_ring_emit(dev, ring, 513 MI_BATCH_NON_SECURE_I965);
476 exec_start | MI_BATCH_NON_SECURE); 514 intel_ring_emit(ring, offset);
477 intel_ring_emit(dev, ring, exec_start + exec_len - 4);
478 intel_ring_emit(dev, ring, 0);
479 } else { 515 } else {
480 intel_ring_begin(dev, ring, 2); 516 intel_ring_emit(ring,
481 if (INTEL_INFO(dev)->gen >= 4) { 517 MI_BATCH_BUFFER_START | (2 << 6));
482 intel_ring_emit(dev, ring, 518 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
483 MI_BATCH_BUFFER_START | (2 << 6)
484 | MI_BATCH_NON_SECURE_I965);
485 intel_ring_emit(dev, ring, exec_start);
486 } else {
487 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
488 | (2 << 6));
489 intel_ring_emit(dev, ring, exec_start |
490 MI_BATCH_NON_SECURE);
491 }
492 } 519 }
493 intel_ring_advance(dev, ring);
494 }
495
496 if (IS_G4X(dev) || IS_GEN5(dev)) {
497 intel_ring_begin(dev, ring, 2);
498 intel_ring_emit(dev, ring, MI_FLUSH |
499 MI_NO_WRITE_FLUSH |
500 MI_INVALIDATE_ISP );
501 intel_ring_emit(dev, ring, MI_NOOP);
502 intel_ring_advance(dev, ring);
503 } 520 }
504 /* XXX breadcrumb */ 521 intel_ring_advance(ring);
505 522
506 return 0; 523 return 0;
507} 524}
508 525
509static void cleanup_status_page(struct drm_device *dev, 526static void cleanup_status_page(struct intel_ring_buffer *ring)
510 struct intel_ring_buffer *ring)
511{ 527{
512 drm_i915_private_t *dev_priv = dev->dev_private; 528 drm_i915_private_t *dev_priv = ring->dev->dev_private;
513 struct drm_gem_object *obj; 529 struct drm_i915_gem_object *obj;
514 struct drm_i915_gem_object *obj_priv;
515 530
516 obj = ring->status_page.obj; 531 obj = ring->status_page.obj;
517 if (obj == NULL) 532 if (obj == NULL)
518 return; 533 return;
519 obj_priv = to_intel_bo(obj);
520 534
521 kunmap(obj_priv->pages[0]); 535 kunmap(obj->pages[0]);
522 i915_gem_object_unpin(obj); 536 i915_gem_object_unpin(obj);
523 drm_gem_object_unreference(obj); 537 drm_gem_object_unreference(&obj->base);
524 ring->status_page.obj = NULL; 538 ring->status_page.obj = NULL;
525 539
526 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 540 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
527} 541}
528 542
529static int init_status_page(struct drm_device *dev, 543static int init_status_page(struct intel_ring_buffer *ring)
530 struct intel_ring_buffer *ring)
531{ 544{
545 struct drm_device *dev = ring->dev;
532 drm_i915_private_t *dev_priv = dev->dev_private; 546 drm_i915_private_t *dev_priv = dev->dev_private;
533 struct drm_gem_object *obj; 547 struct drm_i915_gem_object *obj;
534 struct drm_i915_gem_object *obj_priv;
535 int ret; 548 int ret;
536 549
537 obj = i915_gem_alloc_object(dev, 4096); 550 obj = i915_gem_alloc_object(dev, 4096);
@@ -540,16 +553,15 @@ static int init_status_page(struct drm_device *dev,
540 ret = -ENOMEM; 553 ret = -ENOMEM;
541 goto err; 554 goto err;
542 } 555 }
543 obj_priv = to_intel_bo(obj); 556 obj->agp_type = AGP_USER_CACHED_MEMORY;
544 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
545 557
546 ret = i915_gem_object_pin(obj, 4096); 558 ret = i915_gem_object_pin(obj, 4096, true);
547 if (ret != 0) { 559 if (ret != 0) {
548 goto err_unref; 560 goto err_unref;
549 } 561 }
550 562
551 ring->status_page.gfx_addr = obj_priv->gtt_offset; 563 ring->status_page.gfx_addr = obj->gtt_offset;
552 ring->status_page.page_addr = kmap(obj_priv->pages[0]); 564 ring->status_page.page_addr = kmap(obj->pages[0]);
553 if (ring->status_page.page_addr == NULL) { 565 if (ring->status_page.page_addr == NULL) {
554 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 566 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
555 goto err_unpin; 567 goto err_unpin;
@@ -557,7 +569,7 @@ static int init_status_page(struct drm_device *dev,
557 ring->status_page.obj = obj; 569 ring->status_page.obj = obj;
558 memset(ring->status_page.page_addr, 0, PAGE_SIZE); 570 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
559 571
560 intel_ring_setup_status_page(dev, ring); 572 intel_ring_setup_status_page(ring);
561 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", 573 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
562 ring->name, ring->status_page.gfx_addr); 574 ring->name, ring->status_page.gfx_addr);
563 575
@@ -566,7 +578,7 @@ static int init_status_page(struct drm_device *dev,
566err_unpin: 578err_unpin:
567 i915_gem_object_unpin(obj); 579 i915_gem_object_unpin(obj);
568err_unref: 580err_unref:
569 drm_gem_object_unreference(obj); 581 drm_gem_object_unreference(&obj->base);
570err: 582err:
571 return ret; 583 return ret;
572} 584}
@@ -574,9 +586,7 @@ err:
574int intel_init_ring_buffer(struct drm_device *dev, 586int intel_init_ring_buffer(struct drm_device *dev,
575 struct intel_ring_buffer *ring) 587 struct intel_ring_buffer *ring)
576{ 588{
577 struct drm_i915_private *dev_priv = dev->dev_private; 589 struct drm_i915_gem_object *obj;
578 struct drm_i915_gem_object *obj_priv;
579 struct drm_gem_object *obj;
580 int ret; 590 int ret;
581 591
582 ring->dev = dev; 592 ring->dev = dev;
@@ -585,7 +595,7 @@ int intel_init_ring_buffer(struct drm_device *dev,
585 INIT_LIST_HEAD(&ring->gpu_write_list); 595 INIT_LIST_HEAD(&ring->gpu_write_list);
586 596
587 if (I915_NEED_GFX_HWS(dev)) { 597 if (I915_NEED_GFX_HWS(dev)) {
588 ret = init_status_page(dev, ring); 598 ret = init_status_page(ring);
589 if (ret) 599 if (ret)
590 return ret; 600 return ret;
591 } 601 }
@@ -597,15 +607,14 @@ int intel_init_ring_buffer(struct drm_device *dev,
597 goto err_hws; 607 goto err_hws;
598 } 608 }
599 609
600 ring->gem_object = obj; 610 ring->obj = obj;
601 611
602 ret = i915_gem_object_pin(obj, PAGE_SIZE); 612 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
603 if (ret) 613 if (ret)
604 goto err_unref; 614 goto err_unref;
605 615
606 obj_priv = to_intel_bo(obj);
607 ring->map.size = ring->size; 616 ring->map.size = ring->size;
608 ring->map.offset = dev->agp->base + obj_priv->gtt_offset; 617 ring->map.offset = dev->agp->base + obj->gtt_offset;
609 ring->map.type = 0; 618 ring->map.type = 0;
610 ring->map.flags = 0; 619 ring->map.flags = 0;
611 ring->map.mtrr = 0; 620 ring->map.mtrr = 0;
@@ -618,60 +627,57 @@ int intel_init_ring_buffer(struct drm_device *dev,
618 } 627 }
619 628
620 ring->virtual_start = ring->map.handle; 629 ring->virtual_start = ring->map.handle;
621 ret = ring->init(dev, ring); 630 ret = ring->init(ring);
622 if (ret) 631 if (ret)
623 goto err_unmap; 632 goto err_unmap;
624 633
625 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 634 return 0;
626 i915_kernel_lost_context(dev);
627 else {
628 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
629 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
630 ring->space = ring->head - (ring->tail + 8);
631 if (ring->space < 0)
632 ring->space += ring->size;
633 }
634 return ret;
635 635
636err_unmap: 636err_unmap:
637 drm_core_ioremapfree(&ring->map, dev); 637 drm_core_ioremapfree(&ring->map, dev);
638err_unpin: 638err_unpin:
639 i915_gem_object_unpin(obj); 639 i915_gem_object_unpin(obj);
640err_unref: 640err_unref:
641 drm_gem_object_unreference(obj); 641 drm_gem_object_unreference(&obj->base);
642 ring->gem_object = NULL; 642 ring->obj = NULL;
643err_hws: 643err_hws:
644 cleanup_status_page(dev, ring); 644 cleanup_status_page(ring);
645 return ret; 645 return ret;
646} 646}
647 647
648void intel_cleanup_ring_buffer(struct drm_device *dev, 648void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
649 struct intel_ring_buffer *ring)
650{ 649{
651 if (ring->gem_object == NULL) 650 struct drm_i915_private *dev_priv;
651 int ret;
652
653 if (ring->obj == NULL)
652 return; 654 return;
653 655
654 drm_core_ioremapfree(&ring->map, dev); 656 /* Disable the ring buffer. The ring must be idle at this point */
657 dev_priv = ring->dev->dev_private;
658 ret = intel_wait_ring_buffer(ring, ring->size - 8);
659 I915_WRITE_CTL(ring, 0);
655 660
656 i915_gem_object_unpin(ring->gem_object); 661 drm_core_ioremapfree(&ring->map, ring->dev);
657 drm_gem_object_unreference(ring->gem_object); 662
658 ring->gem_object = NULL; 663 i915_gem_object_unpin(ring->obj);
664 drm_gem_object_unreference(&ring->obj->base);
665 ring->obj = NULL;
659 666
660 if (ring->cleanup) 667 if (ring->cleanup)
661 ring->cleanup(ring); 668 ring->cleanup(ring);
662 669
663 cleanup_status_page(dev, ring); 670 cleanup_status_page(ring);
664} 671}
665 672
666static int intel_wrap_ring_buffer(struct drm_device *dev, 673static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
667 struct intel_ring_buffer *ring)
668{ 674{
669 unsigned int *virt; 675 unsigned int *virt;
670 int rem; 676 int rem;
671 rem = ring->size - ring->tail; 677 rem = ring->size - ring->tail;
672 678
673 if (ring->space < rem) { 679 if (ring->space < rem) {
674 int ret = intel_wait_ring_buffer(dev, ring, rem); 680 int ret = intel_wait_ring_buffer(ring, rem);
675 if (ret) 681 if (ret)
676 return ret; 682 return ret;
677 } 683 }
@@ -689,11 +695,11 @@ static int intel_wrap_ring_buffer(struct drm_device *dev,
689 return 0; 695 return 0;
690} 696}
691 697
692int intel_wait_ring_buffer(struct drm_device *dev, 698int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
693 struct intel_ring_buffer *ring, int n)
694{ 699{
700 struct drm_device *dev = ring->dev;
701 struct drm_i915_private *dev_priv = dev->dev_private;
695 unsigned long end; 702 unsigned long end;
696 drm_i915_private_t *dev_priv = dev->dev_private;
697 u32 head; 703 u32 head;
698 704
699 trace_i915_ring_wait_begin (dev); 705 trace_i915_ring_wait_begin (dev);
@@ -711,7 +717,7 @@ int intel_wait_ring_buffer(struct drm_device *dev,
711 if (ring->space < 0) 717 if (ring->space < 0)
712 ring->space += ring->size; 718 ring->space += ring->size;
713 if (ring->space >= n) { 719 if (ring->space >= n) {
714 trace_i915_ring_wait_end (dev); 720 trace_i915_ring_wait_end(dev);
715 return 0; 721 return 0;
716 } 722 }
717 723
@@ -722,29 +728,39 @@ int intel_wait_ring_buffer(struct drm_device *dev,
722 } 728 }
723 729
724 msleep(1); 730 msleep(1);
731 if (atomic_read(&dev_priv->mm.wedged))
732 return -EAGAIN;
725 } while (!time_after(jiffies, end)); 733 } while (!time_after(jiffies, end));
726 trace_i915_ring_wait_end (dev); 734 trace_i915_ring_wait_end (dev);
727 return -EBUSY; 735 return -EBUSY;
728} 736}
729 737
730void intel_ring_begin(struct drm_device *dev, 738int intel_ring_begin(struct intel_ring_buffer *ring,
731 struct intel_ring_buffer *ring, 739 int num_dwords)
732 int num_dwords)
733{ 740{
734 int n = 4*num_dwords; 741 int n = 4*num_dwords;
735 if (unlikely(ring->tail + n > ring->size)) 742 int ret;
736 intel_wrap_ring_buffer(dev, ring); 743
737 if (unlikely(ring->space < n)) 744 if (unlikely(ring->tail + n > ring->size)) {
738 intel_wait_ring_buffer(dev, ring, n); 745 ret = intel_wrap_ring_buffer(ring);
746 if (unlikely(ret))
747 return ret;
748 }
749
750 if (unlikely(ring->space < n)) {
751 ret = intel_wait_ring_buffer(ring, n);
752 if (unlikely(ret))
753 return ret;
754 }
739 755
740 ring->space -= n; 756 ring->space -= n;
757 return 0;
741} 758}
742 759
743void intel_ring_advance(struct drm_device *dev, 760void intel_ring_advance(struct intel_ring_buffer *ring)
744 struct intel_ring_buffer *ring)
745{ 761{
746 ring->tail &= ring->size - 1; 762 ring->tail &= ring->size - 1;
747 ring->write_tail(dev, ring, ring->tail); 763 ring->write_tail(ring, ring->tail);
748} 764}
749 765
750static const struct intel_ring_buffer render_ring = { 766static const struct intel_ring_buffer render_ring = {
@@ -756,10 +772,10 @@ static const struct intel_ring_buffer render_ring = {
756 .write_tail = ring_write_tail, 772 .write_tail = ring_write_tail,
757 .flush = render_ring_flush, 773 .flush = render_ring_flush,
758 .add_request = render_ring_add_request, 774 .add_request = render_ring_add_request,
759 .get_seqno = render_ring_get_seqno, 775 .get_seqno = ring_get_seqno,
760 .user_irq_get = render_ring_get_user_irq, 776 .irq_get = render_ring_get_irq,
761 .user_irq_put = render_ring_put_user_irq, 777 .irq_put = render_ring_put_irq,
762 .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer, 778 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
763}; 779};
764 780
765/* ring buffer for bit-stream decoder */ 781/* ring buffer for bit-stream decoder */
@@ -769,22 +785,21 @@ static const struct intel_ring_buffer bsd_ring = {
769 .id = RING_BSD, 785 .id = RING_BSD,
770 .mmio_base = BSD_RING_BASE, 786 .mmio_base = BSD_RING_BASE,
771 .size = 32 * PAGE_SIZE, 787 .size = 32 * PAGE_SIZE,
772 .init = init_bsd_ring, 788 .init = init_ring_common,
773 .write_tail = ring_write_tail, 789 .write_tail = ring_write_tail,
774 .flush = bsd_ring_flush, 790 .flush = bsd_ring_flush,
775 .add_request = ring_add_request, 791 .add_request = ring_add_request,
776 .get_seqno = ring_status_page_get_seqno, 792 .get_seqno = ring_get_seqno,
777 .user_irq_get = bsd_ring_get_user_irq, 793 .irq_get = bsd_ring_get_irq,
778 .user_irq_put = bsd_ring_put_user_irq, 794 .irq_put = bsd_ring_put_irq,
779 .dispatch_gem_execbuffer = ring_dispatch_gem_execbuffer, 795 .dispatch_execbuffer = ring_dispatch_execbuffer,
780}; 796};
781 797
782 798
783static void gen6_bsd_ring_write_tail(struct drm_device *dev, 799static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
784 struct intel_ring_buffer *ring,
785 u32 value) 800 u32 value)
786{ 801{
787 drm_i915_private_t *dev_priv = dev->dev_private; 802 drm_i915_private_t *dev_priv = ring->dev->dev_private;
788 803
789 /* Every tail move must follow the sequence below */ 804 /* Every tail move must follow the sequence below */
790 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 805 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
@@ -803,69 +818,80 @@ static void gen6_bsd_ring_write_tail(struct drm_device *dev,
803 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE); 818 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
804} 819}
805 820
806static void gen6_ring_flush(struct drm_device *dev, 821static void gen6_ring_flush(struct intel_ring_buffer *ring,
807 struct intel_ring_buffer *ring,
808 u32 invalidate_domains, 822 u32 invalidate_domains,
809 u32 flush_domains) 823 u32 flush_domains)
810{ 824{
811 intel_ring_begin(dev, ring, 4); 825 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
812 intel_ring_emit(dev, ring, MI_FLUSH_DW); 826 return;
813 intel_ring_emit(dev, ring, 0); 827
814 intel_ring_emit(dev, ring, 0); 828 if (intel_ring_begin(ring, 4) == 0) {
815 intel_ring_emit(dev, ring, 0); 829 intel_ring_emit(ring, MI_FLUSH_DW);
816 intel_ring_advance(dev, ring); 830 intel_ring_emit(ring, 0);
831 intel_ring_emit(ring, 0);
832 intel_ring_emit(ring, 0);
833 intel_ring_advance(ring);
834 }
817} 835}
818 836
819static int 837static int
820gen6_ring_dispatch_gem_execbuffer(struct drm_device *dev, 838gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
821 struct intel_ring_buffer *ring, 839 u32 offset, u32 len)
822 struct drm_i915_gem_execbuffer2 *exec,
823 struct drm_clip_rect *cliprects,
824 uint64_t exec_offset)
825{ 840{
826 uint32_t exec_start; 841 int ret;
827 842
828 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 843 ret = intel_ring_begin(ring, 2);
844 if (ret)
845 return ret;
829 846
830 intel_ring_begin(dev, ring, 2); 847 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
831 intel_ring_emit(dev, ring,
832 MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
833 /* bit0-7 is the length on GEN6+ */ 848 /* bit0-7 is the length on GEN6+ */
834 intel_ring_emit(dev, ring, exec_start); 849 intel_ring_emit(ring, offset);
835 intel_ring_advance(dev, ring); 850 intel_ring_advance(ring);
836 851
837 return 0; 852 return 0;
838} 853}
839 854
855static bool
856gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
857{
858 return ring_get_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
859}
860
861static void
862gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
863{
864 ring_put_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
865}
866
840/* ring buffer for Video Codec for Gen6+ */ 867/* ring buffer for Video Codec for Gen6+ */
841static const struct intel_ring_buffer gen6_bsd_ring = { 868static const struct intel_ring_buffer gen6_bsd_ring = {
842 .name = "gen6 bsd ring", 869 .name = "gen6 bsd ring",
843 .id = RING_BSD, 870 .id = RING_BSD,
844 .mmio_base = GEN6_BSD_RING_BASE, 871 .mmio_base = GEN6_BSD_RING_BASE,
845 .size = 32 * PAGE_SIZE, 872 .size = 32 * PAGE_SIZE,
846 .init = init_bsd_ring, 873 .init = init_ring_common,
847 .write_tail = gen6_bsd_ring_write_tail, 874 .write_tail = gen6_bsd_ring_write_tail,
848 .flush = gen6_ring_flush, 875 .flush = gen6_ring_flush,
849 .add_request = ring_add_request, 876 .add_request = gen6_add_request,
850 .get_seqno = ring_status_page_get_seqno, 877 .get_seqno = ring_get_seqno,
851 .user_irq_get = bsd_ring_get_user_irq, 878 .irq_get = gen6_bsd_ring_get_irq,
852 .user_irq_put = bsd_ring_put_user_irq, 879 .irq_put = gen6_bsd_ring_put_irq,
853 .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, 880 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
854}; 881};
855 882
856/* Blitter support (SandyBridge+) */ 883/* Blitter support (SandyBridge+) */
857 884
858static void 885static bool
859blt_ring_get_user_irq(struct drm_device *dev, 886blt_ring_get_irq(struct intel_ring_buffer *ring)
860 struct intel_ring_buffer *ring)
861{ 887{
862 /* do nothing */ 888 return ring_get_irq(ring, GT_BLT_USER_INTERRUPT);
863} 889}
890
864static void 891static void
865blt_ring_put_user_irq(struct drm_device *dev, 892blt_ring_put_irq(struct intel_ring_buffer *ring)
866 struct intel_ring_buffer *ring)
867{ 893{
868 /* do nothing */ 894 ring_put_irq(ring, GT_BLT_USER_INTERRUPT);
869} 895}
870 896
871 897
@@ -883,32 +909,31 @@ to_blt_workaround(struct intel_ring_buffer *ring)
883 return ring->private; 909 return ring->private;
884} 910}
885 911
886static int blt_ring_init(struct drm_device *dev, 912static int blt_ring_init(struct intel_ring_buffer *ring)
887 struct intel_ring_buffer *ring)
888{ 913{
889 if (NEED_BLT_WORKAROUND(dev)) { 914 if (NEED_BLT_WORKAROUND(ring->dev)) {
890 struct drm_i915_gem_object *obj; 915 struct drm_i915_gem_object *obj;
891 u32 __iomem *ptr; 916 u32 *ptr;
892 int ret; 917 int ret;
893 918
894 obj = to_intel_bo(i915_gem_alloc_object(dev, 4096)); 919 obj = i915_gem_alloc_object(ring->dev, 4096);
895 if (obj == NULL) 920 if (obj == NULL)
896 return -ENOMEM; 921 return -ENOMEM;
897 922
898 ret = i915_gem_object_pin(&obj->base, 4096); 923 ret = i915_gem_object_pin(obj, 4096, true);
899 if (ret) { 924 if (ret) {
900 drm_gem_object_unreference(&obj->base); 925 drm_gem_object_unreference(&obj->base);
901 return ret; 926 return ret;
902 } 927 }
903 928
904 ptr = kmap(obj->pages[0]); 929 ptr = kmap(obj->pages[0]);
905 iowrite32(MI_BATCH_BUFFER_END, ptr); 930 *ptr++ = MI_BATCH_BUFFER_END;
906 iowrite32(MI_NOOP, ptr+1); 931 *ptr++ = MI_NOOP;
907 kunmap(obj->pages[0]); 932 kunmap(obj->pages[0]);
908 933
909 ret = i915_gem_object_set_to_gtt_domain(&obj->base, false); 934 ret = i915_gem_object_set_to_gtt_domain(obj, false);
910 if (ret) { 935 if (ret) {
911 i915_gem_object_unpin(&obj->base); 936 i915_gem_object_unpin(obj);
912 drm_gem_object_unreference(&obj->base); 937 drm_gem_object_unreference(&obj->base);
913 return ret; 938 return ret;
914 } 939 }
@@ -916,51 +941,39 @@ static int blt_ring_init(struct drm_device *dev,
916 ring->private = obj; 941 ring->private = obj;
917 } 942 }
918 943
919 return init_ring_common(dev, ring); 944 return init_ring_common(ring);
920} 945}
921 946
922static void blt_ring_begin(struct drm_device *dev, 947static int blt_ring_begin(struct intel_ring_buffer *ring,
923 struct intel_ring_buffer *ring,
924 int num_dwords) 948 int num_dwords)
925{ 949{
926 if (ring->private) { 950 if (ring->private) {
927 intel_ring_begin(dev, ring, num_dwords+2); 951 int ret = intel_ring_begin(ring, num_dwords+2);
928 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START); 952 if (ret)
929 intel_ring_emit(dev, ring, to_blt_workaround(ring)->gtt_offset); 953 return ret;
954
955 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
956 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
957
958 return 0;
930 } else 959 } else
931 intel_ring_begin(dev, ring, 4); 960 return intel_ring_begin(ring, 4);
932} 961}
933 962
934static void blt_ring_flush(struct drm_device *dev, 963static void blt_ring_flush(struct intel_ring_buffer *ring,
935 struct intel_ring_buffer *ring,
936 u32 invalidate_domains, 964 u32 invalidate_domains,
937 u32 flush_domains) 965 u32 flush_domains)
938{ 966{
939 blt_ring_begin(dev, ring, 4); 967 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
940 intel_ring_emit(dev, ring, MI_FLUSH_DW); 968 return;
941 intel_ring_emit(dev, ring, 0);
942 intel_ring_emit(dev, ring, 0);
943 intel_ring_emit(dev, ring, 0);
944 intel_ring_advance(dev, ring);
945}
946
947static u32
948blt_ring_add_request(struct drm_device *dev,
949 struct intel_ring_buffer *ring,
950 u32 flush_domains)
951{
952 u32 seqno = i915_gem_get_seqno(dev);
953
954 blt_ring_begin(dev, ring, 4);
955 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
956 intel_ring_emit(dev, ring,
957 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
958 intel_ring_emit(dev, ring, seqno);
959 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
960 intel_ring_advance(dev, ring);
961 969
962 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); 970 if (blt_ring_begin(ring, 4) == 0) {
963 return seqno; 971 intel_ring_emit(ring, MI_FLUSH_DW);
972 intel_ring_emit(ring, 0);
973 intel_ring_emit(ring, 0);
974 intel_ring_emit(ring, 0);
975 intel_ring_advance(ring);
976 }
964} 977}
965 978
966static void blt_ring_cleanup(struct intel_ring_buffer *ring) 979static void blt_ring_cleanup(struct intel_ring_buffer *ring)
@@ -981,47 +994,51 @@ static const struct intel_ring_buffer gen6_blt_ring = {
981 .init = blt_ring_init, 994 .init = blt_ring_init,
982 .write_tail = ring_write_tail, 995 .write_tail = ring_write_tail,
983 .flush = blt_ring_flush, 996 .flush = blt_ring_flush,
984 .add_request = blt_ring_add_request, 997 .add_request = gen6_add_request,
985 .get_seqno = ring_status_page_get_seqno, 998 .get_seqno = ring_get_seqno,
986 .user_irq_get = blt_ring_get_user_irq, 999 .irq_get = blt_ring_get_irq,
987 .user_irq_put = blt_ring_put_user_irq, 1000 .irq_put = blt_ring_put_irq,
988 .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, 1001 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
989 .cleanup = blt_ring_cleanup, 1002 .cleanup = blt_ring_cleanup,
990}; 1003};
991 1004
992int intel_init_render_ring_buffer(struct drm_device *dev) 1005int intel_init_render_ring_buffer(struct drm_device *dev)
993{ 1006{
994 drm_i915_private_t *dev_priv = dev->dev_private; 1007 drm_i915_private_t *dev_priv = dev->dev_private;
1008 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
995 1009
996 dev_priv->render_ring = render_ring; 1010 *ring = render_ring;
1011 if (INTEL_INFO(dev)->gen >= 6) {
1012 ring->add_request = gen6_add_request;
1013 }
997 1014
998 if (!I915_NEED_GFX_HWS(dev)) { 1015 if (!I915_NEED_GFX_HWS(dev)) {
999 dev_priv->render_ring.status_page.page_addr 1016 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1000 = dev_priv->status_page_dmah->vaddr; 1017 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1001 memset(dev_priv->render_ring.status_page.page_addr,
1002 0, PAGE_SIZE);
1003 } 1018 }
1004 1019
1005 return intel_init_ring_buffer(dev, &dev_priv->render_ring); 1020 return intel_init_ring_buffer(dev, ring);
1006} 1021}
1007 1022
1008int intel_init_bsd_ring_buffer(struct drm_device *dev) 1023int intel_init_bsd_ring_buffer(struct drm_device *dev)
1009{ 1024{
1010 drm_i915_private_t *dev_priv = dev->dev_private; 1025 drm_i915_private_t *dev_priv = dev->dev_private;
1026 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1011 1027
1012 if (IS_GEN6(dev)) 1028 if (IS_GEN6(dev))
1013 dev_priv->bsd_ring = gen6_bsd_ring; 1029 *ring = gen6_bsd_ring;
1014 else 1030 else
1015 dev_priv->bsd_ring = bsd_ring; 1031 *ring = bsd_ring;
1016 1032
1017 return intel_init_ring_buffer(dev, &dev_priv->bsd_ring); 1033 return intel_init_ring_buffer(dev, ring);
1018} 1034}
1019 1035
1020int intel_init_blt_ring_buffer(struct drm_device *dev) 1036int intel_init_blt_ring_buffer(struct drm_device *dev)
1021{ 1037{
1022 drm_i915_private_t *dev_priv = dev->dev_private; 1038 drm_i915_private_t *dev_priv = dev->dev_private;
1039 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1023 1040
1024 dev_priv->blt_ring = gen6_blt_ring; 1041 *ring = gen6_blt_ring;
1025 1042
1026 return intel_init_ring_buffer(dev, &dev_priv->blt_ring); 1043 return intel_init_ring_buffer(dev, ring);
1027} 1044}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index d2cd0f1efeed..8e2e357ad6ee 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -1,22 +1,37 @@
1#ifndef _INTEL_RINGBUFFER_H_ 1#ifndef _INTEL_RINGBUFFER_H_
2#define _INTEL_RINGBUFFER_H_ 2#define _INTEL_RINGBUFFER_H_
3 3
4enum {
5 RCS = 0x0,
6 VCS,
7 BCS,
8 I915_NUM_RINGS,
9};
10
4struct intel_hw_status_page { 11struct intel_hw_status_page {
5 void *page_addr; 12 u32 __iomem *page_addr;
6 unsigned int gfx_addr; 13 unsigned int gfx_addr;
7 struct drm_gem_object *obj; 14 struct drm_i915_gem_object *obj;
8}; 15};
9 16
10#define I915_READ_TAIL(ring) I915_READ(RING_TAIL(ring->mmio_base)) 17#define I915_RING_READ(reg) i915_safe_read(dev_priv, reg)
18
19#define I915_READ_TAIL(ring) I915_RING_READ(RING_TAIL(ring->mmio_base))
11#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL(ring->mmio_base), val) 20#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL(ring->mmio_base), val)
12#define I915_READ_START(ring) I915_READ(RING_START(ring->mmio_base)) 21
22#define I915_READ_START(ring) I915_RING_READ(RING_START(ring->mmio_base))
13#define I915_WRITE_START(ring, val) I915_WRITE(RING_START(ring->mmio_base), val) 23#define I915_WRITE_START(ring, val) I915_WRITE(RING_START(ring->mmio_base), val)
14#define I915_READ_HEAD(ring) I915_READ(RING_HEAD(ring->mmio_base)) 24
25#define I915_READ_HEAD(ring) I915_RING_READ(RING_HEAD(ring->mmio_base))
15#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD(ring->mmio_base), val) 26#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD(ring->mmio_base), val)
16#define I915_READ_CTL(ring) I915_READ(RING_CTL(ring->mmio_base)) 27
28#define I915_READ_CTL(ring) I915_RING_READ(RING_CTL(ring->mmio_base))
17#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val) 29#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val)
18 30
19struct drm_i915_gem_execbuffer2; 31#define I915_READ_NOPID(ring) I915_RING_READ(RING_NOPID(ring->mmio_base))
32#define I915_READ_SYNC_0(ring) I915_RING_READ(RING_SYNC_0(ring->mmio_base))
33#define I915_READ_SYNC_1(ring) I915_RING_READ(RING_SYNC_1(ring->mmio_base))
34
20struct intel_ring_buffer { 35struct intel_ring_buffer {
21 const char *name; 36 const char *name;
22 enum intel_ring_id { 37 enum intel_ring_id {
@@ -25,45 +40,36 @@ struct intel_ring_buffer {
25 RING_BLT = 0x4, 40 RING_BLT = 0x4,
26 } id; 41 } id;
27 u32 mmio_base; 42 u32 mmio_base;
28 unsigned long size;
29 void *virtual_start; 43 void *virtual_start;
30 struct drm_device *dev; 44 struct drm_device *dev;
31 struct drm_gem_object *gem_object; 45 struct drm_i915_gem_object *obj;
32 46
33 u32 actual_head; 47 u32 actual_head;
34 u32 head; 48 u32 head;
35 u32 tail; 49 u32 tail;
36 int space; 50 int space;
51 int size;
37 struct intel_hw_status_page status_page; 52 struct intel_hw_status_page status_page;
38 53
39 u32 irq_gem_seqno; /* last seq seem at irq time */ 54 u32 irq_seqno; /* last seq seem at irq time */
40 u32 waiting_gem_seqno; 55 u32 waiting_seqno;
41 int user_irq_refcount; 56 u32 sync_seqno[I915_NUM_RINGS-1];
42 void (*user_irq_get)(struct drm_device *dev, 57 atomic_t irq_refcount;
43 struct intel_ring_buffer *ring); 58 bool __must_check (*irq_get)(struct intel_ring_buffer *ring);
44 void (*user_irq_put)(struct drm_device *dev, 59 void (*irq_put)(struct intel_ring_buffer *ring);
45 struct intel_ring_buffer *ring);
46 60
47 int (*init)(struct drm_device *dev, 61 int (*init)(struct intel_ring_buffer *ring);
48 struct intel_ring_buffer *ring);
49 62
50 void (*write_tail)(struct drm_device *dev, 63 void (*write_tail)(struct intel_ring_buffer *ring,
51 struct intel_ring_buffer *ring,
52 u32 value); 64 u32 value);
53 void (*flush)(struct drm_device *dev, 65 void (*flush)(struct intel_ring_buffer *ring,
54 struct intel_ring_buffer *ring, 66 u32 invalidate_domains,
55 u32 invalidate_domains, 67 u32 flush_domains);
56 u32 flush_domains); 68 int (*add_request)(struct intel_ring_buffer *ring,
57 u32 (*add_request)(struct drm_device *dev, 69 u32 *seqno);
58 struct intel_ring_buffer *ring, 70 u32 (*get_seqno)(struct intel_ring_buffer *ring);
59 u32 flush_domains); 71 int (*dispatch_execbuffer)(struct intel_ring_buffer *ring,
60 u32 (*get_seqno)(struct drm_device *dev, 72 u32 offset, u32 length);
61 struct intel_ring_buffer *ring);
62 int (*dispatch_gem_execbuffer)(struct drm_device *dev,
63 struct intel_ring_buffer *ring,
64 struct drm_i915_gem_execbuffer2 *exec,
65 struct drm_clip_rect *cliprects,
66 uint64_t exec_offset);
67 void (*cleanup)(struct intel_ring_buffer *ring); 73 void (*cleanup)(struct intel_ring_buffer *ring);
68 74
69 /** 75 /**
@@ -96,7 +102,7 @@ struct intel_ring_buffer {
96 /** 102 /**
97 * Do we have some not yet emitted requests outstanding? 103 * Do we have some not yet emitted requests outstanding?
98 */ 104 */
99 bool outstanding_lazy_request; 105 u32 outstanding_lazy_request;
100 106
101 wait_queue_head_t irq_queue; 107 wait_queue_head_t irq_queue;
102 drm_local_map_t map; 108 drm_local_map_t map;
@@ -105,44 +111,54 @@ struct intel_ring_buffer {
105}; 111};
106 112
107static inline u32 113static inline u32
114intel_ring_sync_index(struct intel_ring_buffer *ring,
115 struct intel_ring_buffer *other)
116{
117 int idx;
118
119 /*
120 * cs -> 0 = vcs, 1 = bcs
121 * vcs -> 0 = bcs, 1 = cs,
122 * bcs -> 0 = cs, 1 = vcs.
123 */
124
125 idx = (other - ring) - 1;
126 if (idx < 0)
127 idx += I915_NUM_RINGS;
128
129 return idx;
130}
131
132static inline u32
108intel_read_status_page(struct intel_ring_buffer *ring, 133intel_read_status_page(struct intel_ring_buffer *ring,
109 int reg) 134 int reg)
110{ 135{
111 u32 *regs = ring->status_page.page_addr; 136 return ioread32(ring->status_page.page_addr + reg);
112 return regs[reg];
113} 137}
114 138
115int intel_init_ring_buffer(struct drm_device *dev, 139void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring);
116 struct intel_ring_buffer *ring); 140int __must_check intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n);
117void intel_cleanup_ring_buffer(struct drm_device *dev, 141int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n);
118 struct intel_ring_buffer *ring); 142
119int intel_wait_ring_buffer(struct drm_device *dev, 143static inline void intel_ring_emit(struct intel_ring_buffer *ring,
120 struct intel_ring_buffer *ring, int n); 144 u32 data)
121void intel_ring_begin(struct drm_device *dev,
122 struct intel_ring_buffer *ring, int n);
123
124static inline void intel_ring_emit(struct drm_device *dev,
125 struct intel_ring_buffer *ring,
126 unsigned int data)
127{ 145{
128 unsigned int *virt = ring->virtual_start + ring->tail; 146 iowrite32(data, ring->virtual_start + ring->tail);
129 *virt = data;
130 ring->tail += 4; 147 ring->tail += 4;
131} 148}
132 149
133void intel_ring_advance(struct drm_device *dev, 150void intel_ring_advance(struct intel_ring_buffer *ring);
134 struct intel_ring_buffer *ring);
135 151
136u32 intel_ring_get_seqno(struct drm_device *dev, 152u32 intel_ring_get_seqno(struct intel_ring_buffer *ring);
137 struct intel_ring_buffer *ring); 153int intel_ring_sync(struct intel_ring_buffer *ring,
154 struct intel_ring_buffer *to,
155 u32 seqno);
138 156
139int intel_init_render_ring_buffer(struct drm_device *dev); 157int intel_init_render_ring_buffer(struct drm_device *dev);
140int intel_init_bsd_ring_buffer(struct drm_device *dev); 158int intel_init_bsd_ring_buffer(struct drm_device *dev);
141int intel_init_blt_ring_buffer(struct drm_device *dev); 159int intel_init_blt_ring_buffer(struct drm_device *dev);
142 160
143u32 intel_ring_get_active_head(struct drm_device *dev, 161u32 intel_ring_get_active_head(struct intel_ring_buffer *ring);
144 struct intel_ring_buffer *ring); 162void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
145void intel_ring_setup_status_page(struct drm_device *dev,
146 struct intel_ring_buffer *ring);
147 163
148#endif /* _INTEL_RINGBUFFER_H_ */ 164#endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 2f7681989316..93206e4eaa6f 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1245,10 +1245,11 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
1245 int type; 1245 int type;
1246 1246
1247 /* Disable TV interrupts around load detect or we'll recurse */ 1247 /* Disable TV interrupts around load detect or we'll recurse */
1248 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1248 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1249 i915_disable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE | 1249 i915_disable_pipestat(dev_priv, 0,
1250 PIPE_HOTPLUG_INTERRUPT_ENABLE |
1250 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); 1251 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
1251 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1252 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1252 1253
1253 save_tv_dac = tv_dac = I915_READ(TV_DAC); 1254 save_tv_dac = tv_dac = I915_READ(TV_DAC);
1254 save_tv_ctl = tv_ctl = I915_READ(TV_CTL); 1255 save_tv_ctl = tv_ctl = I915_READ(TV_CTL);
@@ -1301,10 +1302,11 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
1301 I915_WRITE(TV_CTL, save_tv_ctl); 1302 I915_WRITE(TV_CTL, save_tv_ctl);
1302 1303
1303 /* Restore interrupt config */ 1304 /* Restore interrupt config */
1304 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1305 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1305 i915_enable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE | 1306 i915_enable_pipestat(dev_priv, 0,
1307 PIPE_HOTPLUG_INTERRUPT_ENABLE |
1306 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); 1308 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
1307 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1309 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1308 1310
1309 return type; 1311 return type;
1310} 1312}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-11 10:36:12 -0500