diff options
Diffstat (limited to 'drivers/dma/ioatdma.c')
-rw-r--r-- | drivers/dma/ioatdma.c | 369 |
1 files changed, 178 insertions, 191 deletions
diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioatdma.c index 850014139556..5fbe56b5cea0 100644 --- a/drivers/dma/ioatdma.c +++ b/drivers/dma/ioatdma.c | |||
@@ -32,16 +32,17 @@ | |||
32 | #include <linux/delay.h> | 32 | #include <linux/delay.h> |
33 | #include <linux/dma-mapping.h> | 33 | #include <linux/dma-mapping.h> |
34 | #include "ioatdma.h" | 34 | #include "ioatdma.h" |
35 | #include "ioatdma_io.h" | ||
36 | #include "ioatdma_registers.h" | 35 | #include "ioatdma_registers.h" |
37 | #include "ioatdma_hw.h" | 36 | #include "ioatdma_hw.h" |
38 | 37 | ||
39 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) | 38 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) |
40 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) | 39 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) |
41 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) | 40 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) |
41 | #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx) | ||
42 | 42 | ||
43 | /* internal functions */ | 43 | /* internal functions */ |
44 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); | 44 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); |
45 | static void ioat_shutdown(struct pci_dev *pdev); | ||
45 | static void __devexit ioat_remove(struct pci_dev *pdev); | 46 | static void __devexit ioat_remove(struct pci_dev *pdev); |
46 | 47 | ||
47 | static int enumerate_dma_channels(struct ioat_device *device) | 48 | static int enumerate_dma_channels(struct ioat_device *device) |
@@ -51,8 +52,8 @@ static int enumerate_dma_channels(struct ioat_device *device) | |||
51 | int i; | 52 | int i; |
52 | struct ioat_dma_chan *ioat_chan; | 53 | struct ioat_dma_chan *ioat_chan; |
53 | 54 | ||
54 | device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET); | 55 | device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); |
55 | xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET); | 56 | xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET); |
56 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); | 57 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); |
57 | 58 | ||
58 | for (i = 0; i < device->common.chancnt; i++) { | 59 | for (i = 0; i < device->common.chancnt; i++) { |
@@ -71,13 +72,79 @@ static int enumerate_dma_channels(struct ioat_device *device) | |||
71 | INIT_LIST_HEAD(&ioat_chan->used_desc); | 72 | INIT_LIST_HEAD(&ioat_chan->used_desc); |
72 | /* This should be made common somewhere in dmaengine.c */ | 73 | /* This should be made common somewhere in dmaengine.c */ |
73 | ioat_chan->common.device = &device->common; | 74 | ioat_chan->common.device = &device->common; |
74 | ioat_chan->common.client = NULL; | ||
75 | list_add_tail(&ioat_chan->common.device_node, | 75 | list_add_tail(&ioat_chan->common.device_node, |
76 | &device->common.channels); | 76 | &device->common.channels); |
77 | } | 77 | } |
78 | return device->common.chancnt; | 78 | return device->common.chancnt; |
79 | } | 79 | } |
80 | 80 | ||
81 | static void | ||
82 | ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) | ||
83 | { | ||
84 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | ||
85 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
86 | |||
87 | pci_unmap_addr_set(desc, src, addr); | ||
88 | |||
89 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | ||
90 | iter->hw->src_addr = addr; | ||
91 | addr += ioat_chan->xfercap; | ||
92 | } | ||
93 | |||
94 | } | ||
95 | |||
96 | static void | ||
97 | ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) | ||
98 | { | ||
99 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | ||
100 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
101 | |||
102 | pci_unmap_addr_set(desc, dst, addr); | ||
103 | |||
104 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | ||
105 | iter->hw->dst_addr = addr; | ||
106 | addr += ioat_chan->xfercap; | ||
107 | } | ||
108 | } | ||
109 | |||
110 | static dma_cookie_t | ||
111 | ioat_tx_submit(struct dma_async_tx_descriptor *tx) | ||
112 | { | ||
113 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
114 | struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); | ||
115 | int append = 0; | ||
116 | dma_cookie_t cookie; | ||
117 | struct ioat_desc_sw *group_start; | ||
118 | |||
119 | group_start = list_entry(desc->async_tx.tx_list.next, | ||
120 | struct ioat_desc_sw, node); | ||
121 | spin_lock_bh(&ioat_chan->desc_lock); | ||
122 | /* cookie incr and addition to used_list must be atomic */ | ||
123 | cookie = ioat_chan->common.cookie; | ||
124 | cookie++; | ||
125 | if (cookie < 0) | ||
126 | cookie = 1; | ||
127 | ioat_chan->common.cookie = desc->async_tx.cookie = cookie; | ||
128 | |||
129 | /* write address into NextDescriptor field of last desc in chain */ | ||
130 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = | ||
131 | group_start->async_tx.phys; | ||
132 | list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev); | ||
133 | |||
134 | ioat_chan->pending += desc->tx_cnt; | ||
135 | if (ioat_chan->pending >= 4) { | ||
136 | append = 1; | ||
137 | ioat_chan->pending = 0; | ||
138 | } | ||
139 | spin_unlock_bh(&ioat_chan->desc_lock); | ||
140 | |||
141 | if (append) | ||
142 | writeb(IOAT_CHANCMD_APPEND, | ||
143 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | ||
144 | |||
145 | return cookie; | ||
146 | } | ||
147 | |||
81 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | 148 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( |
82 | struct ioat_dma_chan *ioat_chan, | 149 | struct ioat_dma_chan *ioat_chan, |
83 | gfp_t flags) | 150 | gfp_t flags) |
@@ -99,8 +166,13 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | |||
99 | } | 166 | } |
100 | 167 | ||
101 | memset(desc, 0, sizeof(*desc)); | 168 | memset(desc, 0, sizeof(*desc)); |
169 | dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common); | ||
170 | desc_sw->async_tx.tx_set_src = ioat_set_src; | ||
171 | desc_sw->async_tx.tx_set_dest = ioat_set_dest; | ||
172 | desc_sw->async_tx.tx_submit = ioat_tx_submit; | ||
173 | INIT_LIST_HEAD(&desc_sw->async_tx.tx_list); | ||
102 | desc_sw->hw = desc; | 174 | desc_sw->hw = desc; |
103 | desc_sw->phys = phys; | 175 | desc_sw->async_tx.phys = phys; |
104 | 176 | ||
105 | return desc_sw; | 177 | return desc_sw; |
106 | } | 178 | } |
@@ -123,7 +195,7 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |||
123 | * In-use bit automatically set by reading chanctrl | 195 | * In-use bit automatically set by reading chanctrl |
124 | * If 0, we got it, if 1, someone else did | 196 | * If 0, we got it, if 1, someone else did |
125 | */ | 197 | */ |
126 | chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); | 198 | chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
127 | if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) | 199 | if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) |
128 | return -EBUSY; | 200 | return -EBUSY; |
129 | 201 | ||
@@ -132,12 +204,12 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |||
132 | IOAT_CHANCTRL_ERR_INT_EN | | 204 | IOAT_CHANCTRL_ERR_INT_EN | |
133 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | | 205 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | |
134 | IOAT_CHANCTRL_ERR_COMPLETION_EN; | 206 | IOAT_CHANCTRL_ERR_COMPLETION_EN; |
135 | ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); | 207 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
136 | 208 | ||
137 | chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET); | 209 | chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
138 | if (chanerr) { | 210 | if (chanerr) { |
139 | printk("IOAT: CHANERR = %x, clearing\n", chanerr); | 211 | printk("IOAT: CHANERR = %x, clearing\n", chanerr); |
140 | ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr); | 212 | writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
141 | } | 213 | } |
142 | 214 | ||
143 | /* Allocate descriptors */ | 215 | /* Allocate descriptors */ |
@@ -161,10 +233,10 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |||
161 | &ioat_chan->completion_addr); | 233 | &ioat_chan->completion_addr); |
162 | memset(ioat_chan->completion_virt, 0, | 234 | memset(ioat_chan->completion_virt, 0, |
163 | sizeof(*ioat_chan->completion_virt)); | 235 | sizeof(*ioat_chan->completion_virt)); |
164 | ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW, | 236 | writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF, |
165 | ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF); | 237 | ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); |
166 | ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH, | 238 | writel(((u64) ioat_chan->completion_addr) >> 32, |
167 | ((u64) ioat_chan->completion_addr) >> 32); | 239 | ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); |
168 | 240 | ||
169 | ioat_start_null_desc(ioat_chan); | 241 | ioat_start_null_desc(ioat_chan); |
170 | return i; | 242 | return i; |
@@ -182,18 +254,20 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |||
182 | 254 | ||
183 | ioat_dma_memcpy_cleanup(ioat_chan); | 255 | ioat_dma_memcpy_cleanup(ioat_chan); |
184 | 256 | ||
185 | ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); | 257 | writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
186 | 258 | ||
187 | spin_lock_bh(&ioat_chan->desc_lock); | 259 | spin_lock_bh(&ioat_chan->desc_lock); |
188 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { | 260 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { |
189 | in_use_descs++; | 261 | in_use_descs++; |
190 | list_del(&desc->node); | 262 | list_del(&desc->node); |
191 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | 263 | pci_pool_free(ioat_device->dma_pool, desc->hw, |
264 | desc->async_tx.phys); | ||
192 | kfree(desc); | 265 | kfree(desc); |
193 | } | 266 | } |
194 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { | 267 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { |
195 | list_del(&desc->node); | 268 | list_del(&desc->node); |
196 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | 269 | pci_pool_free(ioat_device->dma_pool, desc->hw, |
270 | desc->async_tx.phys); | ||
197 | kfree(desc); | 271 | kfree(desc); |
198 | } | 272 | } |
199 | spin_unlock_bh(&ioat_chan->desc_lock); | 273 | spin_unlock_bh(&ioat_chan->desc_lock); |
@@ -210,50 +284,30 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |||
210 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; | 284 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; |
211 | 285 | ||
212 | /* Tell hw the chan is free */ | 286 | /* Tell hw the chan is free */ |
213 | chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); | 287 | chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
214 | chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; | 288 | chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; |
215 | ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); | 289 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
216 | } | 290 | } |
217 | 291 | ||
218 | /** | 292 | static struct dma_async_tx_descriptor * |
219 | * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction | 293 | ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en) |
220 | * @ioat_chan: IOAT DMA channel handle | ||
221 | * @dest: DMA destination address | ||
222 | * @src: DMA source address | ||
223 | * @len: transaction length in bytes | ||
224 | */ | ||
225 | |||
226 | static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | ||
227 | dma_addr_t dest, | ||
228 | dma_addr_t src, | ||
229 | size_t len) | ||
230 | { | 294 | { |
231 | struct ioat_desc_sw *first; | 295 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); |
232 | struct ioat_desc_sw *prev; | 296 | struct ioat_desc_sw *first, *prev, *new; |
233 | struct ioat_desc_sw *new; | ||
234 | dma_cookie_t cookie; | ||
235 | LIST_HEAD(new_chain); | 297 | LIST_HEAD(new_chain); |
236 | u32 copy; | 298 | u32 copy; |
237 | size_t orig_len; | 299 | size_t orig_len; |
238 | dma_addr_t orig_src, orig_dst; | 300 | int desc_count = 0; |
239 | unsigned int desc_count = 0; | ||
240 | unsigned int append = 0; | ||
241 | |||
242 | if (!ioat_chan || !dest || !src) | ||
243 | return -EFAULT; | ||
244 | 301 | ||
245 | if (!len) | 302 | if (!len) |
246 | return ioat_chan->common.cookie; | 303 | return NULL; |
247 | 304 | ||
248 | orig_len = len; | 305 | orig_len = len; |
249 | orig_src = src; | ||
250 | orig_dst = dest; | ||
251 | 306 | ||
252 | first = NULL; | 307 | first = NULL; |
253 | prev = NULL; | 308 | prev = NULL; |
254 | 309 | ||
255 | spin_lock_bh(&ioat_chan->desc_lock); | 310 | spin_lock_bh(&ioat_chan->desc_lock); |
256 | |||
257 | while (len) { | 311 | while (len) { |
258 | if (!list_empty(&ioat_chan->free_desc)) { | 312 | if (!list_empty(&ioat_chan->free_desc)) { |
259 | new = to_ioat_desc(ioat_chan->free_desc.next); | 313 | new = to_ioat_desc(ioat_chan->free_desc.next); |
@@ -270,141 +324,36 @@ static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | |||
270 | 324 | ||
271 | new->hw->size = copy; | 325 | new->hw->size = copy; |
272 | new->hw->ctl = 0; | 326 | new->hw->ctl = 0; |
273 | new->hw->src_addr = src; | 327 | new->async_tx.cookie = 0; |
274 | new->hw->dst_addr = dest; | 328 | new->async_tx.ack = 1; |
275 | new->cookie = 0; | ||
276 | 329 | ||
277 | /* chain together the physical address list for the HW */ | 330 | /* chain together the physical address list for the HW */ |
278 | if (!first) | 331 | if (!first) |
279 | first = new; | 332 | first = new; |
280 | else | 333 | else |
281 | prev->hw->next = (u64) new->phys; | 334 | prev->hw->next = (u64) new->async_tx.phys; |
282 | 335 | ||
283 | prev = new; | 336 | prev = new; |
284 | |||
285 | len -= copy; | 337 | len -= copy; |
286 | dest += copy; | ||
287 | src += copy; | ||
288 | |||
289 | list_add_tail(&new->node, &new_chain); | 338 | list_add_tail(&new->node, &new_chain); |
290 | desc_count++; | 339 | desc_count++; |
291 | } | 340 | } |
292 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | ||
293 | new->hw->next = 0; | ||
294 | 341 | ||
295 | /* cookie incr and addition to used_list must be atomic */ | 342 | list_splice(&new_chain, &new->async_tx.tx_list); |
296 | 343 | ||
297 | cookie = ioat_chan->common.cookie; | 344 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; |
298 | cookie++; | 345 | new->hw->next = 0; |
299 | if (cookie < 0) | 346 | new->tx_cnt = desc_count; |
300 | cookie = 1; | 347 | new->async_tx.ack = 0; /* client is in control of this ack */ |
301 | ioat_chan->common.cookie = new->cookie = cookie; | 348 | new->async_tx.cookie = -EBUSY; |
302 | 349 | ||
303 | pci_unmap_addr_set(new, src, orig_src); | ||
304 | pci_unmap_addr_set(new, dst, orig_dst); | ||
305 | pci_unmap_len_set(new, src_len, orig_len); | 350 | pci_unmap_len_set(new, src_len, orig_len); |
306 | pci_unmap_len_set(new, dst_len, orig_len); | 351 | pci_unmap_len_set(new, dst_len, orig_len); |
307 | |||
308 | /* write address into NextDescriptor field of last desc in chain */ | ||
309 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys; | ||
310 | list_splice_init(&new_chain, ioat_chan->used_desc.prev); | ||
311 | |||
312 | ioat_chan->pending += desc_count; | ||
313 | if (ioat_chan->pending >= 20) { | ||
314 | append = 1; | ||
315 | ioat_chan->pending = 0; | ||
316 | } | ||
317 | |||
318 | spin_unlock_bh(&ioat_chan->desc_lock); | 352 | spin_unlock_bh(&ioat_chan->desc_lock); |
319 | 353 | ||
320 | if (append) | 354 | return new ? &new->async_tx : NULL; |
321 | ioatdma_chan_write8(ioat_chan, | ||
322 | IOAT_CHANCMD_OFFSET, | ||
323 | IOAT_CHANCMD_APPEND); | ||
324 | return cookie; | ||
325 | } | ||
326 | |||
327 | /** | ||
328 | * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs | ||
329 | * @chan: IOAT DMA channel handle | ||
330 | * @dest: DMA destination address | ||
331 | * @src: DMA source address | ||
332 | * @len: transaction length in bytes | ||
333 | */ | ||
334 | |||
335 | static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan, | ||
336 | void *dest, | ||
337 | void *src, | ||
338 | size_t len) | ||
339 | { | ||
340 | dma_addr_t dest_addr; | ||
341 | dma_addr_t src_addr; | ||
342 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
343 | |||
344 | dest_addr = pci_map_single(ioat_chan->device->pdev, | ||
345 | dest, len, PCI_DMA_FROMDEVICE); | ||
346 | src_addr = pci_map_single(ioat_chan->device->pdev, | ||
347 | src, len, PCI_DMA_TODEVICE); | ||
348 | |||
349 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | ||
350 | } | 355 | } |
351 | 356 | ||
352 | /** | ||
353 | * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page | ||
354 | * @chan: IOAT DMA channel handle | ||
355 | * @page: pointer to the page to copy to | ||
356 | * @offset: offset into that page | ||
357 | * @src: DMA source address | ||
358 | * @len: transaction length in bytes | ||
359 | */ | ||
360 | |||
361 | static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan, | ||
362 | struct page *page, | ||
363 | unsigned int offset, | ||
364 | void *src, | ||
365 | size_t len) | ||
366 | { | ||
367 | dma_addr_t dest_addr; | ||
368 | dma_addr_t src_addr; | ||
369 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
370 | |||
371 | dest_addr = pci_map_page(ioat_chan->device->pdev, | ||
372 | page, offset, len, PCI_DMA_FROMDEVICE); | ||
373 | src_addr = pci_map_single(ioat_chan->device->pdev, | ||
374 | src, len, PCI_DMA_TODEVICE); | ||
375 | |||
376 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | ||
377 | } | ||
378 | |||
379 | /** | ||
380 | * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages | ||
381 | * @chan: IOAT DMA channel handle | ||
382 | * @dest_pg: pointer to the page to copy to | ||
383 | * @dest_off: offset into that page | ||
384 | * @src_pg: pointer to the page to copy from | ||
385 | * @src_off: offset into that page | ||
386 | * @len: transaction length in bytes. This is guaranteed not to make a copy | ||
387 | * across a page boundary. | ||
388 | */ | ||
389 | |||
390 | static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan, | ||
391 | struct page *dest_pg, | ||
392 | unsigned int dest_off, | ||
393 | struct page *src_pg, | ||
394 | unsigned int src_off, | ||
395 | size_t len) | ||
396 | { | ||
397 | dma_addr_t dest_addr; | ||
398 | dma_addr_t src_addr; | ||
399 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
400 | |||
401 | dest_addr = pci_map_page(ioat_chan->device->pdev, | ||
402 | dest_pg, dest_off, len, PCI_DMA_FROMDEVICE); | ||
403 | src_addr = pci_map_page(ioat_chan->device->pdev, | ||
404 | src_pg, src_off, len, PCI_DMA_TODEVICE); | ||
405 | |||
406 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | ||
407 | } | ||
408 | 357 | ||
409 | /** | 358 | /** |
410 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw | 359 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw |
@@ -417,9 +366,8 @@ static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) | |||
417 | 366 | ||
418 | if (ioat_chan->pending != 0) { | 367 | if (ioat_chan->pending != 0) { |
419 | ioat_chan->pending = 0; | 368 | ioat_chan->pending = 0; |
420 | ioatdma_chan_write8(ioat_chan, | 369 | writeb(IOAT_CHANCMD_APPEND, |
421 | IOAT_CHANCMD_OFFSET, | 370 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
422 | IOAT_CHANCMD_APPEND); | ||
423 | } | 371 | } |
424 | } | 372 | } |
425 | 373 | ||
@@ -449,7 +397,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
449 | if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | 397 | if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == |
450 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { | 398 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { |
451 | printk("IOAT: Channel halted, chanerr = %x\n", | 399 | printk("IOAT: Channel halted, chanerr = %x\n", |
452 | ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET)); | 400 | readl(chan->reg_base + IOAT_CHANERR_OFFSET)); |
453 | 401 | ||
454 | /* TODO do something to salvage the situation */ | 402 | /* TODO do something to salvage the situation */ |
455 | } | 403 | } |
@@ -467,8 +415,8 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
467 | * exceeding xfercap, perhaps. If so, only the last one will | 415 | * exceeding xfercap, perhaps. If so, only the last one will |
468 | * have a cookie, and require unmapping. | 416 | * have a cookie, and require unmapping. |
469 | */ | 417 | */ |
470 | if (desc->cookie) { | 418 | if (desc->async_tx.cookie) { |
471 | cookie = desc->cookie; | 419 | cookie = desc->async_tx.cookie; |
472 | 420 | ||
473 | /* yes we are unmapping both _page and _single alloc'd | 421 | /* yes we are unmapping both _page and _single alloc'd |
474 | regions with unmap_page. Is this *really* that bad? | 422 | regions with unmap_page. Is this *really* that bad? |
@@ -483,14 +431,19 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
483 | PCI_DMA_TODEVICE); | 431 | PCI_DMA_TODEVICE); |
484 | } | 432 | } |
485 | 433 | ||
486 | if (desc->phys != phys_complete) { | 434 | if (desc->async_tx.phys != phys_complete) { |
487 | /* a completed entry, but not the last, so cleanup */ | 435 | /* a completed entry, but not the last, so cleanup |
488 | list_del(&desc->node); | 436 | * if the client is done with the descriptor |
489 | list_add_tail(&desc->node, &chan->free_desc); | 437 | */ |
438 | if (desc->async_tx.ack) { | ||
439 | list_del(&desc->node); | ||
440 | list_add_tail(&desc->node, &chan->free_desc); | ||
441 | } else | ||
442 | desc->async_tx.cookie = 0; | ||
490 | } else { | 443 | } else { |
491 | /* last used desc. Do not remove, so we can append from | 444 | /* last used desc. Do not remove, so we can append from |
492 | it, but don't look at it next time, either */ | 445 | it, but don't look at it next time, either */ |
493 | desc->cookie = 0; | 446 | desc->async_tx.cookie = 0; |
494 | 447 | ||
495 | /* TODO check status bits? */ | 448 | /* TODO check status bits? */ |
496 | break; | 449 | break; |
@@ -506,6 +459,17 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
506 | spin_unlock(&chan->cleanup_lock); | 459 | spin_unlock(&chan->cleanup_lock); |
507 | } | 460 | } |
508 | 461 | ||
462 | static void ioat_dma_dependency_added(struct dma_chan *chan) | ||
463 | { | ||
464 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
465 | spin_lock_bh(&ioat_chan->desc_lock); | ||
466 | if (ioat_chan->pending == 0) { | ||
467 | spin_unlock_bh(&ioat_chan->desc_lock); | ||
468 | ioat_dma_memcpy_cleanup(ioat_chan); | ||
469 | } else | ||
470 | spin_unlock_bh(&ioat_chan->desc_lock); | ||
471 | } | ||
472 | |||
509 | /** | 473 | /** |
510 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | 474 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction |
511 | * @chan: IOAT DMA channel handle | 475 | * @chan: IOAT DMA channel handle |
@@ -553,6 +517,8 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, | |||
553 | 517 | ||
554 | static struct pci_device_id ioat_pci_tbl[] = { | 518 | static struct pci_device_id ioat_pci_tbl[] = { |
555 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, | 519 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, |
520 | { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, | ||
521 | PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) }, | ||
556 | { 0, } | 522 | { 0, } |
557 | }; | 523 | }; |
558 | 524 | ||
@@ -560,6 +526,7 @@ static struct pci_driver ioat_pci_driver = { | |||
560 | .name = "ioatdma", | 526 | .name = "ioatdma", |
561 | .id_table = ioat_pci_tbl, | 527 | .id_table = ioat_pci_tbl, |
562 | .probe = ioat_probe, | 528 | .probe = ioat_probe, |
529 | .shutdown = ioat_shutdown, | ||
563 | .remove = __devexit_p(ioat_remove), | 530 | .remove = __devexit_p(ioat_remove), |
564 | }; | 531 | }; |
565 | 532 | ||
@@ -569,21 +536,21 @@ static irqreturn_t ioat_do_interrupt(int irq, void *data) | |||
569 | unsigned long attnstatus; | 536 | unsigned long attnstatus; |
570 | u8 intrctrl; | 537 | u8 intrctrl; |
571 | 538 | ||
572 | intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET); | 539 | intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); |
573 | 540 | ||
574 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) | 541 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) |
575 | return IRQ_NONE; | 542 | return IRQ_NONE; |
576 | 543 | ||
577 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { | 544 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { |
578 | ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); | 545 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
579 | return IRQ_NONE; | 546 | return IRQ_NONE; |
580 | } | 547 | } |
581 | 548 | ||
582 | attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET); | 549 | attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); |
583 | 550 | ||
584 | printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); | 551 | printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); |
585 | 552 | ||
586 | ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); | 553 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
587 | return IRQ_HANDLED; | 554 | return IRQ_HANDLED; |
588 | } | 555 | } |
589 | 556 | ||
@@ -607,19 +574,17 @@ static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) | |||
607 | 574 | ||
608 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | 575 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; |
609 | desc->hw->next = 0; | 576 | desc->hw->next = 0; |
577 | desc->async_tx.ack = 1; | ||
610 | 578 | ||
611 | list_add_tail(&desc->node, &ioat_chan->used_desc); | 579 | list_add_tail(&desc->node, &ioat_chan->used_desc); |
612 | spin_unlock_bh(&ioat_chan->desc_lock); | 580 | spin_unlock_bh(&ioat_chan->desc_lock); |
613 | 581 | ||
614 | #if (BITS_PER_LONG == 64) | 582 | writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF, |
615 | ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys); | 583 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW); |
616 | #else | 584 | writel(((u64) desc->async_tx.phys) >> 32, |
617 | ioatdma_chan_write32(ioat_chan, | 585 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH); |
618 | IOAT_CHAINADDR_OFFSET_LOW, | 586 | |
619 | (u32) desc->phys); | 587 | writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
620 | ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0); | ||
621 | #endif | ||
622 | ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START); | ||
623 | } | 588 | } |
624 | 589 | ||
625 | /* | 590 | /* |
@@ -633,6 +598,8 @@ static int ioat_self_test(struct ioat_device *device) | |||
633 | u8 *src; | 598 | u8 *src; |
634 | u8 *dest; | 599 | u8 *dest; |
635 | struct dma_chan *dma_chan; | 600 | struct dma_chan *dma_chan; |
601 | struct dma_async_tx_descriptor *tx; | ||
602 | dma_addr_t addr; | ||
636 | dma_cookie_t cookie; | 603 | dma_cookie_t cookie; |
637 | int err = 0; | 604 | int err = 0; |
638 | 605 | ||
@@ -658,7 +625,15 @@ static int ioat_self_test(struct ioat_device *device) | |||
658 | goto out; | 625 | goto out; |
659 | } | 626 | } |
660 | 627 | ||
661 | cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); | 628 | tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0); |
629 | async_tx_ack(tx); | ||
630 | addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE, | ||
631 | DMA_TO_DEVICE); | ||
632 | ioat_set_src(addr, tx, 0); | ||
633 | addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE, | ||
634 | DMA_FROM_DEVICE); | ||
635 | ioat_set_dest(addr, tx, 0); | ||
636 | cookie = ioat_tx_submit(tx); | ||
662 | ioat_dma_memcpy_issue_pending(dma_chan); | 637 | ioat_dma_memcpy_issue_pending(dma_chan); |
663 | msleep(1); | 638 | msleep(1); |
664 | 639 | ||
@@ -748,19 +723,20 @@ static int __devinit ioat_probe(struct pci_dev *pdev, | |||
748 | 723 | ||
749 | device->reg_base = reg_base; | 724 | device->reg_base = reg_base; |
750 | 725 | ||
751 | ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN); | 726 | writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET); |
752 | pci_set_master(pdev); | 727 | pci_set_master(pdev); |
753 | 728 | ||
754 | INIT_LIST_HEAD(&device->common.channels); | 729 | INIT_LIST_HEAD(&device->common.channels); |
755 | enumerate_dma_channels(device); | 730 | enumerate_dma_channels(device); |
756 | 731 | ||
732 | dma_cap_set(DMA_MEMCPY, device->common.cap_mask); | ||
757 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; | 733 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; |
758 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; | 734 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; |
759 | device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; | 735 | device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy; |
760 | device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; | 736 | device->common.device_is_tx_complete = ioat_dma_is_complete; |
761 | device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; | 737 | device->common.device_issue_pending = ioat_dma_memcpy_issue_pending; |
762 | device->common.device_memcpy_complete = ioat_dma_is_complete; | 738 | device->common.device_dependency_added = ioat_dma_dependency_added; |
763 | device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; | 739 | device->common.dev = &pdev->dev; |
764 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", | 740 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", |
765 | device->common.chancnt); | 741 | device->common.chancnt); |
766 | 742 | ||
@@ -787,9 +763,20 @@ err_request_regions: | |||
787 | err_set_dma_mask: | 763 | err_set_dma_mask: |
788 | pci_disable_device(pdev); | 764 | pci_disable_device(pdev); |
789 | err_enable_device: | 765 | err_enable_device: |
766 | |||
767 | printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n"); | ||
768 | |||
790 | return err; | 769 | return err; |
791 | } | 770 | } |
792 | 771 | ||
772 | static void ioat_shutdown(struct pci_dev *pdev) | ||
773 | { | ||
774 | struct ioat_device *device; | ||
775 | device = pci_get_drvdata(pdev); | ||
776 | |||
777 | dma_async_device_unregister(&device->common); | ||
778 | } | ||
779 | |||
793 | static void __devexit ioat_remove(struct pci_dev *pdev) | 780 | static void __devexit ioat_remove(struct pci_dev *pdev) |
794 | { | 781 | { |
795 | struct ioat_device *device; | 782 | struct ioat_device *device; |
@@ -818,7 +805,7 @@ static void __devexit ioat_remove(struct pci_dev *pdev) | |||
818 | } | 805 | } |
819 | 806 | ||
820 | /* MODULE API */ | 807 | /* MODULE API */ |
821 | MODULE_VERSION("1.7"); | 808 | MODULE_VERSION("1.9"); |
822 | MODULE_LICENSE("GPL"); | 809 | MODULE_LICENSE("GPL"); |
823 | MODULE_AUTHOR("Intel Corporation"); | 810 | MODULE_AUTHOR("Intel Corporation"); |
824 | 811 | ||