diff options
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/dma/ioat_dma.c | 173 | ||||
-rw-r--r-- | drivers/dma/ioatdma.h | 6 |
2 files changed, 95 insertions, 84 deletions
diff --git a/drivers/dma/ioat_dma.c b/drivers/dma/ioat_dma.c index c44f5517edbd..15906027c8de 100644 --- a/drivers/dma/ioat_dma.c +++ b/drivers/dma/ioat_dma.c | |||
@@ -46,9 +46,12 @@ | |||
46 | /* internal functions */ | 46 | /* internal functions */ |
47 | static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan); | 47 | static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan); |
48 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); | 48 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); |
49 | static struct ioat_desc_sw * | ||
50 | ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan); | ||
49 | 51 | ||
50 | static struct ioat_dma_chan *ioat_lookup_chan_by_index(struct ioatdma_device *device, | 52 | static inline struct ioat_dma_chan *ioat_lookup_chan_by_index( |
51 | int index) | 53 | struct ioatdma_device *device, |
54 | int index) | ||
52 | { | 55 | { |
53 | return device->idx[index]; | 56 | return device->idx[index]; |
54 | } | 57 | } |
@@ -148,57 +151,90 @@ static void ioat_set_src(dma_addr_t addr, | |||
148 | struct dma_async_tx_descriptor *tx, | 151 | struct dma_async_tx_descriptor *tx, |
149 | int index) | 152 | int index) |
150 | { | 153 | { |
151 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | 154 | tx_to_ioat_desc(tx)->src = addr; |
152 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
153 | |||
154 | pci_unmap_addr_set(desc, src, addr); | ||
155 | |||
156 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | ||
157 | iter->hw->src_addr = addr; | ||
158 | addr += ioat_chan->xfercap; | ||
159 | } | ||
160 | |||
161 | } | 155 | } |
162 | 156 | ||
163 | static void ioat_set_dest(dma_addr_t addr, | 157 | static void ioat_set_dest(dma_addr_t addr, |
164 | struct dma_async_tx_descriptor *tx, | 158 | struct dma_async_tx_descriptor *tx, |
165 | int index) | 159 | int index) |
166 | { | 160 | { |
167 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | 161 | tx_to_ioat_desc(tx)->dst = addr; |
168 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
169 | |||
170 | pci_unmap_addr_set(desc, dst, addr); | ||
171 | |||
172 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | ||
173 | iter->hw->dst_addr = addr; | ||
174 | addr += ioat_chan->xfercap; | ||
175 | } | ||
176 | } | 162 | } |
177 | 163 | ||
178 | static dma_cookie_t ioat_tx_submit(struct dma_async_tx_descriptor *tx) | 164 | static dma_cookie_t ioat_tx_submit(struct dma_async_tx_descriptor *tx) |
179 | { | 165 | { |
180 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | 166 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); |
181 | struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); | 167 | struct ioat_desc_sw *first = tx_to_ioat_desc(tx); |
168 | struct ioat_desc_sw *prev, *new; | ||
169 | struct ioat_dma_descriptor *hw; | ||
182 | int append = 0; | 170 | int append = 0; |
183 | dma_cookie_t cookie; | 171 | dma_cookie_t cookie; |
184 | struct ioat_desc_sw *group_start; | 172 | LIST_HEAD(new_chain); |
173 | u32 copy; | ||
174 | size_t len; | ||
175 | dma_addr_t src, dst; | ||
176 | int orig_ack; | ||
177 | unsigned int desc_count = 0; | ||
178 | |||
179 | /* src and dest and len are stored in the initial descriptor */ | ||
180 | len = first->len; | ||
181 | src = first->src; | ||
182 | dst = first->dst; | ||
183 | orig_ack = first->async_tx.ack; | ||
184 | new = first; | ||
185 | 185 | ||
186 | group_start = list_entry(desc->async_tx.tx_list.next, | ||
187 | struct ioat_desc_sw, node); | ||
188 | spin_lock_bh(&ioat_chan->desc_lock); | 186 | spin_lock_bh(&ioat_chan->desc_lock); |
187 | prev = to_ioat_desc(ioat_chan->used_desc.prev); | ||
188 | prefetch(prev->hw); | ||
189 | do { | ||
190 | copy = min((u32) len, ioat_chan->xfercap); | ||
191 | |||
192 | new->async_tx.ack = 1; | ||
193 | |||
194 | hw = new->hw; | ||
195 | hw->size = copy; | ||
196 | hw->ctl = 0; | ||
197 | hw->src_addr = src; | ||
198 | hw->dst_addr = dst; | ||
199 | hw->next = 0; | ||
200 | |||
201 | /* chain together the physical address list for the HW */ | ||
202 | wmb(); | ||
203 | prev->hw->next = (u64) new->async_tx.phys; | ||
204 | |||
205 | len -= copy; | ||
206 | dst += copy; | ||
207 | src += copy; | ||
208 | |||
209 | list_add_tail(&new->node, &new_chain); | ||
210 | desc_count++; | ||
211 | prev = new; | ||
212 | } while (len && (new = ioat_dma_get_next_descriptor(ioat_chan))); | ||
213 | |||
214 | hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | ||
215 | new->tx_cnt = desc_count; | ||
216 | new->async_tx.ack = orig_ack; /* client is in control of this ack */ | ||
217 | |||
218 | /* store the original values for use in later cleanup */ | ||
219 | if (new != first) { | ||
220 | new->src = first->src; | ||
221 | new->dst = first->dst; | ||
222 | new->len = first->len; | ||
223 | } | ||
224 | |||
189 | /* cookie incr and addition to used_list must be atomic */ | 225 | /* cookie incr and addition to used_list must be atomic */ |
190 | cookie = ioat_chan->common.cookie; | 226 | cookie = ioat_chan->common.cookie; |
191 | cookie++; | 227 | cookie++; |
192 | if (cookie < 0) | 228 | if (cookie < 0) |
193 | cookie = 1; | 229 | cookie = 1; |
194 | ioat_chan->common.cookie = desc->async_tx.cookie = cookie; | 230 | ioat_chan->common.cookie = new->async_tx.cookie = cookie; |
195 | 231 | ||
196 | /* write address into NextDescriptor field of last desc in chain */ | 232 | /* write address into NextDescriptor field of last desc in chain */ |
197 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = | 233 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = |
198 | group_start->async_tx.phys; | 234 | first->async_tx.phys; |
199 | list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev); | 235 | __list_splice(&new_chain, ioat_chan->used_desc.prev); |
200 | 236 | ||
201 | ioat_chan->pending += desc->tx_cnt; | 237 | ioat_chan->pending += desc_count; |
202 | if (ioat_chan->pending >= 4) { | 238 | if (ioat_chan->pending >= 4) { |
203 | append = 1; | 239 | append = 1; |
204 | ioat_chan->pending = 0; | 240 | ioat_chan->pending = 0; |
@@ -348,6 +384,7 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |||
348 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; | 384 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; |
349 | ioat_chan->pending = 0; | 385 | ioat_chan->pending = 0; |
350 | } | 386 | } |
387 | |||
351 | /** | 388 | /** |
352 | * ioat_dma_get_next_descriptor - return the next available descriptor | 389 | * ioat_dma_get_next_descriptor - return the next available descriptor |
353 | * @ioat_chan: IOAT DMA channel handle | 390 | * @ioat_chan: IOAT DMA channel handle |
@@ -356,8 +393,8 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |||
356 | * channel's desc_lock held. Allocates more descriptors if the channel | 393 | * channel's desc_lock held. Allocates more descriptors if the channel |
357 | * has run out. | 394 | * has run out. |
358 | */ | 395 | */ |
359 | static struct ioat_desc_sw *ioat_dma_get_next_descriptor( | 396 | static struct ioat_desc_sw * |
360 | struct ioat_dma_chan *ioat_chan) | 397 | ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan) |
361 | { | 398 | { |
362 | struct ioat_desc_sw *new = NULL; | 399 | struct ioat_desc_sw *new = NULL; |
363 | 400 | ||
@@ -382,51 +419,11 @@ static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy( | |||
382 | int int_en) | 419 | int int_en) |
383 | { | 420 | { |
384 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | 421 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); |
385 | struct ioat_desc_sw *first, *prev, *new; | 422 | struct ioat_desc_sw *new; |
386 | LIST_HEAD(new_chain); | ||
387 | u32 copy; | ||
388 | size_t orig_len; | ||
389 | int desc_count = 0; | ||
390 | |||
391 | if (!len) | ||
392 | return NULL; | ||
393 | |||
394 | orig_len = len; | ||
395 | |||
396 | first = NULL; | ||
397 | prev = NULL; | ||
398 | 423 | ||
399 | spin_lock_bh(&ioat_chan->desc_lock); | 424 | spin_lock_bh(&ioat_chan->desc_lock); |
400 | while (len) { | 425 | new = ioat_dma_get_next_descriptor(ioat_chan); |
401 | new = ioat_dma_get_next_descriptor(ioat_chan); | 426 | new->len = len; |
402 | copy = min((u32) len, ioat_chan->xfercap); | ||
403 | |||
404 | new->hw->size = copy; | ||
405 | new->hw->ctl = 0; | ||
406 | new->async_tx.cookie = 0; | ||
407 | new->async_tx.ack = 1; | ||
408 | |||
409 | /* chain together the physical address list for the HW */ | ||
410 | if (!first) | ||
411 | first = new; | ||
412 | else | ||
413 | prev->hw->next = (u64) new->async_tx.phys; | ||
414 | |||
415 | prev = new; | ||
416 | len -= copy; | ||
417 | list_add_tail(&new->node, &new_chain); | ||
418 | desc_count++; | ||
419 | } | ||
420 | |||
421 | list_splice(&new_chain, &new->async_tx.tx_list); | ||
422 | |||
423 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | ||
424 | new->hw->next = 0; | ||
425 | new->tx_cnt = desc_count; | ||
426 | new->async_tx.ack = 0; /* client is in control of this ack */ | ||
427 | new->async_tx.cookie = -EBUSY; | ||
428 | |||
429 | pci_unmap_len_set(new, len, orig_len); | ||
430 | spin_unlock_bh(&ioat_chan->desc_lock); | 427 | spin_unlock_bh(&ioat_chan->desc_lock); |
431 | 428 | ||
432 | return new ? &new->async_tx : NULL; | 429 | return new ? &new->async_tx : NULL; |
@@ -464,7 +461,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan) | |||
464 | 461 | ||
465 | prefetch(ioat_chan->completion_virt); | 462 | prefetch(ioat_chan->completion_virt); |
466 | 463 | ||
467 | if (!spin_trylock(&ioat_chan->cleanup_lock)) | 464 | if (!spin_trylock_bh(&ioat_chan->cleanup_lock)) |
468 | return; | 465 | return; |
469 | 466 | ||
470 | /* The completion writeback can happen at any time, | 467 | /* The completion writeback can happen at any time, |
@@ -474,12 +471,15 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan) | |||
474 | 471 | ||
475 | #if (BITS_PER_LONG == 64) | 472 | #if (BITS_PER_LONG == 64) |
476 | phys_complete = | 473 | phys_complete = |
477 | ioat_chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | 474 | ioat_chan->completion_virt->full |
475 | & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | ||
478 | #else | 476 | #else |
479 | phys_complete = ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; | 477 | phys_complete = |
478 | ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; | ||
480 | #endif | 479 | #endif |
481 | 480 | ||
482 | if ((ioat_chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | 481 | if ((ioat_chan->completion_virt->full |
482 | & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | ||
483 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { | 483 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { |
484 | dev_err(&ioat_chan->device->pdev->dev, | 484 | dev_err(&ioat_chan->device->pdev->dev, |
485 | "Channel halted, chanerr = %x\n", | 485 | "Channel halted, chanerr = %x\n", |
@@ -489,7 +489,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan) | |||
489 | } | 489 | } |
490 | 490 | ||
491 | if (phys_complete == ioat_chan->last_completion) { | 491 | if (phys_complete == ioat_chan->last_completion) { |
492 | spin_unlock(&ioat_chan->cleanup_lock); | 492 | spin_unlock_bh(&ioat_chan->cleanup_lock); |
493 | return; | 493 | return; |
494 | } | 494 | } |
495 | 495 | ||
@@ -548,7 +548,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan) | |||
548 | if (cookie != 0) | 548 | if (cookie != 0) |
549 | ioat_chan->completed_cookie = cookie; | 549 | ioat_chan->completed_cookie = cookie; |
550 | 550 | ||
551 | spin_unlock(&ioat_chan->cleanup_lock); | 551 | spin_unlock_bh(&ioat_chan->cleanup_lock); |
552 | } | 552 | } |
553 | 553 | ||
554 | static void ioat_dma_dependency_added(struct dma_chan *chan) | 554 | static void ioat_dma_dependency_added(struct dma_chan *chan) |
@@ -613,8 +613,13 @@ static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan) | |||
613 | spin_lock_bh(&ioat_chan->desc_lock); | 613 | spin_lock_bh(&ioat_chan->desc_lock); |
614 | 614 | ||
615 | desc = ioat_dma_get_next_descriptor(ioat_chan); | 615 | desc = ioat_dma_get_next_descriptor(ioat_chan); |
616 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | 616 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL |
617 | | IOAT_DMA_DESCRIPTOR_CTL_INT_GN | ||
618 | | IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | ||
617 | desc->hw->next = 0; | 619 | desc->hw->next = 0; |
620 | desc->hw->size = 0; | ||
621 | desc->hw->src_addr = 0; | ||
622 | desc->hw->dst_addr = 0; | ||
618 | desc->async_tx.ack = 1; | 623 | desc->async_tx.ack = 1; |
619 | 624 | ||
620 | list_add_tail(&desc->node, &ioat_chan->used_desc); | 625 | list_add_tail(&desc->node, &ioat_chan->used_desc); |
@@ -688,6 +693,12 @@ static int ioat_dma_self_test(struct ioatdma_device *device) | |||
688 | DMA_FROM_DEVICE); | 693 | DMA_FROM_DEVICE); |
689 | ioat_set_dest(addr, tx, 0); | 694 | ioat_set_dest(addr, tx, 0); |
690 | cookie = ioat_tx_submit(tx); | 695 | cookie = ioat_tx_submit(tx); |
696 | if (cookie < 0) { | ||
697 | dev_err(&device->pdev->dev, | ||
698 | "Self-test setup failed, disabling\n"); | ||
699 | err = -ENODEV; | ||
700 | goto free_resources; | ||
701 | } | ||
691 | ioat_dma_memcpy_issue_pending(dma_chan); | 702 | ioat_dma_memcpy_issue_pending(dma_chan); |
692 | msleep(1); | 703 | msleep(1); |
693 | 704 | ||
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h index d3643f264507..5f9881e7b0ed 100644 --- a/drivers/dma/ioatdma.h +++ b/drivers/dma/ioatdma.h | |||
@@ -124,9 +124,9 @@ struct ioat_desc_sw { | |||
124 | struct ioat_dma_descriptor *hw; | 124 | struct ioat_dma_descriptor *hw; |
125 | struct list_head node; | 125 | struct list_head node; |
126 | int tx_cnt; | 126 | int tx_cnt; |
127 | DECLARE_PCI_UNMAP_LEN(len) | 127 | size_t len; |
128 | DECLARE_PCI_UNMAP_ADDR(src) | 128 | dma_addr_t src; |
129 | DECLARE_PCI_UNMAP_ADDR(dst) | 129 | dma_addr_t dst; |
130 | struct dma_async_tx_descriptor async_tx; | 130 | struct dma_async_tx_descriptor async_tx; |
131 | }; | 131 | }; |
132 | 132 | ||