aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/usb/host/xhci-ring.c
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
committerGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
commitc71c03bda1e86c9d5198c5d83f712e695c4f2a1e (patch)
treeecb166cb3e2b7e2adb3b5e292245fefd23381ac8 /drivers/usb/host/xhci-ring.c
parentea53c912f8a86a8567697115b6a0d8152beee5c8 (diff)
parent6a00f206debf8a5c8899055726ad127dbeeed098 (diff)
Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp
Conflicts: litmus/sched_cedf.c
Diffstat (limited to 'drivers/usb/host/xhci-ring.c')
-rw-r--r--drivers/usb/host/xhci-ring.c1197
1 files changed, 806 insertions, 391 deletions
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 48e60d166ff0..70cacbbe7fb9 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -68,6 +68,10 @@
68#include <linux/slab.h> 68#include <linux/slab.h>
69#include "xhci.h" 69#include "xhci.h"
70 70
71static int handle_cmd_in_cmd_wait_list(struct xhci_hcd *xhci,
72 struct xhci_virt_device *virt_dev,
73 struct xhci_event_cmd *event);
74
71/* 75/*
72 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA 76 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
73 * address of the TRB. 77 * address of the TRB.
@@ -89,33 +93,35 @@ dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
89/* Does this link TRB point to the first segment in a ring, 93/* Does this link TRB point to the first segment in a ring,
90 * or was the previous TRB the last TRB on the last segment in the ERST? 94 * or was the previous TRB the last TRB on the last segment in the ERST?
91 */ 95 */
92static inline bool last_trb_on_last_seg(struct xhci_hcd *xhci, struct xhci_ring *ring, 96static bool last_trb_on_last_seg(struct xhci_hcd *xhci, struct xhci_ring *ring,
93 struct xhci_segment *seg, union xhci_trb *trb) 97 struct xhci_segment *seg, union xhci_trb *trb)
94{ 98{
95 if (ring == xhci->event_ring) 99 if (ring == xhci->event_ring)
96 return (trb == &seg->trbs[TRBS_PER_SEGMENT]) && 100 return (trb == &seg->trbs[TRBS_PER_SEGMENT]) &&
97 (seg->next == xhci->event_ring->first_seg); 101 (seg->next == xhci->event_ring->first_seg);
98 else 102 else
99 return trb->link.control & LINK_TOGGLE; 103 return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
100} 104}
101 105
102/* Is this TRB a link TRB or was the last TRB the last TRB in this event ring 106/* Is this TRB a link TRB or was the last TRB the last TRB in this event ring
103 * segment? I.e. would the updated event TRB pointer step off the end of the 107 * segment? I.e. would the updated event TRB pointer step off the end of the
104 * event seg? 108 * event seg?
105 */ 109 */
106static inline int last_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, 110static int last_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
107 struct xhci_segment *seg, union xhci_trb *trb) 111 struct xhci_segment *seg, union xhci_trb *trb)
108{ 112{
109 if (ring == xhci->event_ring) 113 if (ring == xhci->event_ring)
110 return trb == &seg->trbs[TRBS_PER_SEGMENT]; 114 return trb == &seg->trbs[TRBS_PER_SEGMENT];
111 else 115 else
112 return (trb->link.control & TRB_TYPE_BITMASK) == TRB_TYPE(TRB_LINK); 116 return (le32_to_cpu(trb->link.control) & TRB_TYPE_BITMASK)
117 == TRB_TYPE(TRB_LINK);
113} 118}
114 119
115static inline int enqueue_is_link_trb(struct xhci_ring *ring) 120static int enqueue_is_link_trb(struct xhci_ring *ring)
116{ 121{
117 struct xhci_link_trb *link = &ring->enqueue->link; 122 struct xhci_link_trb *link = &ring->enqueue->link;
118 return ((link->control & TRB_TYPE_BITMASK) == TRB_TYPE(TRB_LINK)); 123 return ((le32_to_cpu(link->control) & TRB_TYPE_BITMASK) ==
124 TRB_TYPE(TRB_LINK));
119} 125}
120 126
121/* Updates trb to point to the next TRB in the ring, and updates seg if the next 127/* Updates trb to point to the next TRB in the ring, and updates seg if the next
@@ -161,12 +167,6 @@ static void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring, bool consumer
161 next = ring->dequeue; 167 next = ring->dequeue;
162 } 168 }
163 addr = (unsigned long long) xhci_trb_virt_to_dma(ring->deq_seg, ring->dequeue); 169 addr = (unsigned long long) xhci_trb_virt_to_dma(ring->deq_seg, ring->dequeue);
164 if (ring == xhci->event_ring)
165 xhci_dbg(xhci, "Event ring deq = 0x%llx (DMA)\n", addr);
166 else if (ring == xhci->cmd_ring)
167 xhci_dbg(xhci, "Command ring deq = 0x%llx (DMA)\n", addr);
168 else
169 xhci_dbg(xhci, "Ring deq = 0x%llx (DMA)\n", addr);
170} 170}
171 171
172/* 172/*
@@ -193,7 +193,7 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
193 union xhci_trb *next; 193 union xhci_trb *next;
194 unsigned long long addr; 194 unsigned long long addr;
195 195
196 chain = ring->enqueue->generic.field[3] & TRB_CHAIN; 196 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
197 next = ++(ring->enqueue); 197 next = ++(ring->enqueue);
198 198
199 ring->enq_updates++; 199 ring->enq_updates++;
@@ -219,12 +219,14 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
219 * (which may mean the chain bit is cleared). 219 * (which may mean the chain bit is cleared).
220 */ 220 */
221 if (!xhci_link_trb_quirk(xhci)) { 221 if (!xhci_link_trb_quirk(xhci)) {
222 next->link.control &= ~TRB_CHAIN; 222 next->link.control &=
223 next->link.control |= chain; 223 cpu_to_le32(~TRB_CHAIN);
224 next->link.control |=
225 cpu_to_le32(chain);
224 } 226 }
225 /* Give this link TRB to the hardware */ 227 /* Give this link TRB to the hardware */
226 wmb(); 228 wmb();
227 next->link.control ^= TRB_CYCLE; 229 next->link.control ^= cpu_to_le32(TRB_CYCLE);
228 } 230 }
229 /* Toggle the cycle bit after the last ring segment. */ 231 /* Toggle the cycle bit after the last ring segment. */
230 if (last_trb_on_last_seg(xhci, ring, ring->enq_seg, next)) { 232 if (last_trb_on_last_seg(xhci, ring, ring->enq_seg, next)) {
@@ -240,12 +242,6 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
240 next = ring->enqueue; 242 next = ring->enqueue;
241 } 243 }
242 addr = (unsigned long long) xhci_trb_virt_to_dma(ring->enq_seg, ring->enqueue); 244 addr = (unsigned long long) xhci_trb_virt_to_dma(ring->enq_seg, ring->enqueue);
243 if (ring == xhci->event_ring)
244 xhci_dbg(xhci, "Event ring enq = 0x%llx (DMA)\n", addr);
245 else if (ring == xhci->cmd_ring)
246 xhci_dbg(xhci, "Command ring enq = 0x%llx (DMA)\n", addr);
247 else
248 xhci_dbg(xhci, "Ring enq = 0x%llx (DMA)\n", addr);
249} 245}
250 246
251/* 247/*
@@ -304,40 +300,35 @@ static int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
304/* Ring the host controller doorbell after placing a command on the ring */ 300/* Ring the host controller doorbell after placing a command on the ring */
305void xhci_ring_cmd_db(struct xhci_hcd *xhci) 301void xhci_ring_cmd_db(struct xhci_hcd *xhci)
306{ 302{
307 u32 temp;
308
309 xhci_dbg(xhci, "// Ding dong!\n"); 303 xhci_dbg(xhci, "// Ding dong!\n");
310 temp = xhci_readl(xhci, &xhci->dba->doorbell[0]) & DB_MASK; 304 xhci_writel(xhci, DB_VALUE_HOST, &xhci->dba->doorbell[0]);
311 xhci_writel(xhci, temp | DB_TARGET_HOST, &xhci->dba->doorbell[0]);
312 /* Flush PCI posted writes */ 305 /* Flush PCI posted writes */
313 xhci_readl(xhci, &xhci->dba->doorbell[0]); 306 xhci_readl(xhci, &xhci->dba->doorbell[0]);
314} 307}
315 308
316static void ring_ep_doorbell(struct xhci_hcd *xhci, 309void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
317 unsigned int slot_id, 310 unsigned int slot_id,
318 unsigned int ep_index, 311 unsigned int ep_index,
319 unsigned int stream_id) 312 unsigned int stream_id)
320{ 313{
321 struct xhci_virt_ep *ep; 314 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
322 unsigned int ep_state; 315 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
323 u32 field; 316 unsigned int ep_state = ep->ep_state;
324 __u32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
325 317
326 ep = &xhci->devs[slot_id]->eps[ep_index];
327 ep_state = ep->ep_state;
328 /* Don't ring the doorbell for this endpoint if there are pending 318 /* Don't ring the doorbell for this endpoint if there are pending
329 * cancellations because the we don't want to interrupt processing. 319 * cancellations because we don't want to interrupt processing.
330 * We don't want to restart any stream rings if there's a set dequeue 320 * We don't want to restart any stream rings if there's a set dequeue
331 * pointer command pending because the device can choose to start any 321 * pointer command pending because the device can choose to start any
332 * stream once the endpoint is on the HW schedule. 322 * stream once the endpoint is on the HW schedule.
333 * FIXME - check all the stream rings for pending cancellations. 323 * FIXME - check all the stream rings for pending cancellations.
334 */ 324 */
335 if (!(ep_state & EP_HALT_PENDING) && !(ep_state & SET_DEQ_PENDING) 325 if ((ep_state & EP_HALT_PENDING) || (ep_state & SET_DEQ_PENDING) ||
336 && !(ep_state & EP_HALTED)) { 326 (ep_state & EP_HALTED))
337 field = xhci_readl(xhci, db_addr) & DB_MASK; 327 return;
338 field |= EPI_TO_DB(ep_index) | STREAM_ID_TO_DB(stream_id); 328 xhci_writel(xhci, DB_VALUE(ep_index, stream_id), db_addr);
339 xhci_writel(xhci, field, db_addr); 329 /* The CPU has better things to do at this point than wait for a
340 } 330 * write-posting flush. It'll get there soon enough.
331 */
341} 332}
342 333
343/* Ring the doorbell for any rings with pending URBs */ 334/* Ring the doorbell for any rings with pending URBs */
@@ -353,7 +344,7 @@ static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
353 /* A ring has pending URBs if its TD list is not empty */ 344 /* A ring has pending URBs if its TD list is not empty */
354 if (!(ep->ep_state & EP_HAS_STREAMS)) { 345 if (!(ep->ep_state & EP_HAS_STREAMS)) {
355 if (!(list_empty(&ep->ring->td_list))) 346 if (!(list_empty(&ep->ring->td_list)))
356 ring_ep_doorbell(xhci, slot_id, ep_index, 0); 347 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
357 return; 348 return;
358 } 349 }
359 350
@@ -361,7 +352,8 @@ static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
361 stream_id++) { 352 stream_id++) {
362 struct xhci_stream_info *stream_info = ep->stream_info; 353 struct xhci_stream_info *stream_info = ep->stream_info;
363 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) 354 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list))
364 ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); 355 xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
356 stream_id);
365 } 357 }
366} 358}
367 359
@@ -380,10 +372,8 @@ static struct xhci_segment *find_trb_seg(
380 while (cur_seg->trbs > trb || 372 while (cur_seg->trbs > trb ||
381 &cur_seg->trbs[TRBS_PER_SEGMENT - 1] < trb) { 373 &cur_seg->trbs[TRBS_PER_SEGMENT - 1] < trb) {
382 generic_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1].generic; 374 generic_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1].generic;
383 if ((generic_trb->field[3] & TRB_TYPE_BITMASK) == 375 if (le32_to_cpu(generic_trb->field[3]) & LINK_TOGGLE)
384 TRB_TYPE(TRB_LINK) && 376 *cycle_state ^= 0x1;
385 (generic_trb->field[3] & LINK_TOGGLE))
386 *cycle_state = ~(*cycle_state) & 0x1;
387 cur_seg = cur_seg->next; 377 cur_seg = cur_seg->next;
388 if (cur_seg == start_seg) 378 if (cur_seg == start_seg)
389 /* Looped over the entire list. Oops! */ 379 /* Looped over the entire list. Oops! */
@@ -449,6 +439,10 @@ static struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,
449 * any link TRBs with the toggle cycle bit set. 439 * any link TRBs with the toggle cycle bit set.
450 * - Finally we move the dequeue state one TRB further, toggling the cycle bit 440 * - Finally we move the dequeue state one TRB further, toggling the cycle bit
451 * if we've moved it past a link TRB with the toggle cycle bit set. 441 * if we've moved it past a link TRB with the toggle cycle bit set.
442 *
443 * Some of the uses of xhci_generic_trb are grotty, but if they're done
444 * with correct __le32 accesses they should work fine. Only users of this are
445 * in here.
452 */ 446 */
453void xhci_find_new_dequeue_state(struct xhci_hcd *xhci, 447void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
454 unsigned int slot_id, unsigned int ep_index, 448 unsigned int slot_id, unsigned int ep_index,
@@ -474,36 +468,52 @@ void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
474 state->new_deq_seg = find_trb_seg(cur_td->start_seg, 468 state->new_deq_seg = find_trb_seg(cur_td->start_seg,
475 dev->eps[ep_index].stopped_trb, 469 dev->eps[ep_index].stopped_trb,
476 &state->new_cycle_state); 470 &state->new_cycle_state);
477 if (!state->new_deq_seg) 471 if (!state->new_deq_seg) {
478 BUG(); 472 WARN_ON(1);
473 return;
474 }
475
479 /* Dig out the cycle state saved by the xHC during the stop ep cmd */ 476 /* Dig out the cycle state saved by the xHC during the stop ep cmd */
480 xhci_dbg(xhci, "Finding endpoint context\n"); 477 xhci_dbg(xhci, "Finding endpoint context\n");
481 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index); 478 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
482 state->new_cycle_state = 0x1 & ep_ctx->deq; 479 state->new_cycle_state = 0x1 & le64_to_cpu(ep_ctx->deq);
483 480
484 state->new_deq_ptr = cur_td->last_trb; 481 state->new_deq_ptr = cur_td->last_trb;
485 xhci_dbg(xhci, "Finding segment containing last TRB in TD.\n"); 482 xhci_dbg(xhci, "Finding segment containing last TRB in TD.\n");
486 state->new_deq_seg = find_trb_seg(state->new_deq_seg, 483 state->new_deq_seg = find_trb_seg(state->new_deq_seg,
487 state->new_deq_ptr, 484 state->new_deq_ptr,
488 &state->new_cycle_state); 485 &state->new_cycle_state);
489 if (!state->new_deq_seg) 486 if (!state->new_deq_seg) {
490 BUG(); 487 WARN_ON(1);
488 return;
489 }
491 490
492 trb = &state->new_deq_ptr->generic; 491 trb = &state->new_deq_ptr->generic;
493 if ((trb->field[3] & TRB_TYPE_BITMASK) == TRB_TYPE(TRB_LINK) && 492 if ((le32_to_cpu(trb->field[3]) & TRB_TYPE_BITMASK) ==
494 (trb->field[3] & LINK_TOGGLE)) 493 TRB_TYPE(TRB_LINK) && (le32_to_cpu(trb->field[3]) & LINK_TOGGLE))
495 state->new_cycle_state = ~(state->new_cycle_state) & 0x1; 494 state->new_cycle_state ^= 0x1;
496 next_trb(xhci, ep_ring, &state->new_deq_seg, &state->new_deq_ptr); 495 next_trb(xhci, ep_ring, &state->new_deq_seg, &state->new_deq_ptr);
497 496
497 /*
498 * If there is only one segment in a ring, find_trb_seg()'s while loop
499 * will not run, and it will return before it has a chance to see if it
500 * needs to toggle the cycle bit. It can't tell if the stalled transfer
501 * ended just before the link TRB on a one-segment ring, or if the TD
502 * wrapped around the top of the ring, because it doesn't have the TD in
503 * question. Look for the one-segment case where stalled TRB's address
504 * is greater than the new dequeue pointer address.
505 */
506 if (ep_ring->first_seg == ep_ring->first_seg->next &&
507 state->new_deq_ptr < dev->eps[ep_index].stopped_trb)
508 state->new_cycle_state ^= 0x1;
509 xhci_dbg(xhci, "Cycle state = 0x%x\n", state->new_cycle_state);
510
498 /* Don't update the ring cycle state for the producer (us). */ 511 /* Don't update the ring cycle state for the producer (us). */
499 xhci_dbg(xhci, "New dequeue segment = %p (virtual)\n", 512 xhci_dbg(xhci, "New dequeue segment = %p (virtual)\n",
500 state->new_deq_seg); 513 state->new_deq_seg);
501 addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr); 514 addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
502 xhci_dbg(xhci, "New dequeue pointer = 0x%llx (DMA)\n", 515 xhci_dbg(xhci, "New dequeue pointer = 0x%llx (DMA)\n",
503 (unsigned long long) addr); 516 (unsigned long long) addr);
504 xhci_dbg(xhci, "Setting dequeue pointer in internal ring state.\n");
505 ep_ring->dequeue = state->new_deq_ptr;
506 ep_ring->deq_seg = state->new_deq_seg;
507} 517}
508 518
509static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, 519static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
@@ -515,12 +525,12 @@ static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
515 for (cur_seg = cur_td->start_seg, cur_trb = cur_td->first_trb; 525 for (cur_seg = cur_td->start_seg, cur_trb = cur_td->first_trb;
516 true; 526 true;
517 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) { 527 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
518 if ((cur_trb->generic.field[3] & TRB_TYPE_BITMASK) == 528 if ((le32_to_cpu(cur_trb->generic.field[3]) & TRB_TYPE_BITMASK)
519 TRB_TYPE(TRB_LINK)) { 529 == TRB_TYPE(TRB_LINK)) {
520 /* Unchain any chained Link TRBs, but 530 /* Unchain any chained Link TRBs, but
521 * leave the pointers intact. 531 * leave the pointers intact.
522 */ 532 */
523 cur_trb->generic.field[3] &= ~TRB_CHAIN; 533 cur_trb->generic.field[3] &= cpu_to_le32(~TRB_CHAIN);
524 xhci_dbg(xhci, "Cancel (unchain) link TRB\n"); 534 xhci_dbg(xhci, "Cancel (unchain) link TRB\n");
525 xhci_dbg(xhci, "Address = %p (0x%llx dma); " 535 xhci_dbg(xhci, "Address = %p (0x%llx dma); "
526 "in seg %p (0x%llx dma)\n", 536 "in seg %p (0x%llx dma)\n",
@@ -533,8 +543,9 @@ static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
533 cur_trb->generic.field[1] = 0; 543 cur_trb->generic.field[1] = 0;
534 cur_trb->generic.field[2] = 0; 544 cur_trb->generic.field[2] = 0;
535 /* Preserve only the cycle bit of this TRB */ 545 /* Preserve only the cycle bit of this TRB */
536 cur_trb->generic.field[3] &= TRB_CYCLE; 546 cur_trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
537 cur_trb->generic.field[3] |= TRB_TYPE(TRB_TR_NOOP); 547 cur_trb->generic.field[3] |= cpu_to_le32(
548 TRB_TYPE(TRB_TR_NOOP));
538 xhci_dbg(xhci, "Cancel TRB %p (0x%llx dma) " 549 xhci_dbg(xhci, "Cancel TRB %p (0x%llx dma) "
539 "in seg %p (0x%llx dma)\n", 550 "in seg %p (0x%llx dma)\n",
540 cur_trb, 551 cur_trb,
@@ -578,7 +589,7 @@ void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
578 ep->ep_state |= SET_DEQ_PENDING; 589 ep->ep_state |= SET_DEQ_PENDING;
579} 590}
580 591
581static inline void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci, 592static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
582 struct xhci_virt_ep *ep) 593 struct xhci_virt_ep *ep)
583{ 594{
584 ep->ep_state &= ~EP_HALT_PENDING; 595 ep->ep_state &= ~EP_HALT_PENDING;
@@ -594,24 +605,30 @@ static inline void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
594static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, 605static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
595 struct xhci_td *cur_td, int status, char *adjective) 606 struct xhci_td *cur_td, int status, char *adjective)
596{ 607{
597 struct usb_hcd *hcd = xhci_to_hcd(xhci); 608 struct usb_hcd *hcd;
598 struct urb *urb; 609 struct urb *urb;
599 struct urb_priv *urb_priv; 610 struct urb_priv *urb_priv;
600 611
601 urb = cur_td->urb; 612 urb = cur_td->urb;
602 urb_priv = urb->hcpriv; 613 urb_priv = urb->hcpriv;
603 urb_priv->td_cnt++; 614 urb_priv->td_cnt++;
615 hcd = bus_to_hcd(urb->dev->bus);
604 616
605 /* Only giveback urb when this is the last td in urb */ 617 /* Only giveback urb when this is the last td in urb */
606 if (urb_priv->td_cnt == urb_priv->length) { 618 if (urb_priv->td_cnt == urb_priv->length) {
619 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
620 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
621 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
622 if (xhci->quirks & XHCI_AMD_PLL_FIX)
623 usb_amd_quirk_pll_enable();
624 }
625 }
607 usb_hcd_unlink_urb_from_ep(hcd, urb); 626 usb_hcd_unlink_urb_from_ep(hcd, urb);
608 xhci_dbg(xhci, "Giveback %s URB %p\n", adjective, urb);
609 627
610 spin_unlock(&xhci->lock); 628 spin_unlock(&xhci->lock);
611 usb_hcd_giveback_urb(hcd, urb, status); 629 usb_hcd_giveback_urb(hcd, urb, status);
612 xhci_urb_free_priv(xhci, urb_priv); 630 xhci_urb_free_priv(xhci, urb_priv);
613 spin_lock(&xhci->lock); 631 spin_lock(&xhci->lock);
614 xhci_dbg(xhci, "%s URB given back\n", adjective);
615 } 632 }
616} 633}
617 634
@@ -626,10 +643,11 @@ static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
626 * bit cleared) so that the HW will skip over them. 643 * bit cleared) so that the HW will skip over them.
627 */ 644 */
628static void handle_stopped_endpoint(struct xhci_hcd *xhci, 645static void handle_stopped_endpoint(struct xhci_hcd *xhci,
629 union xhci_trb *trb) 646 union xhci_trb *trb, struct xhci_event_cmd *event)
630{ 647{
631 unsigned int slot_id; 648 unsigned int slot_id;
632 unsigned int ep_index; 649 unsigned int ep_index;
650 struct xhci_virt_device *virt_dev;
633 struct xhci_ring *ep_ring; 651 struct xhci_ring *ep_ring;
634 struct xhci_virt_ep *ep; 652 struct xhci_virt_ep *ep;
635 struct list_head *entry; 653 struct list_head *entry;
@@ -638,13 +656,30 @@ static void handle_stopped_endpoint(struct xhci_hcd *xhci,
638 656
639 struct xhci_dequeue_state deq_state; 657 struct xhci_dequeue_state deq_state;
640 658
659 if (unlikely(TRB_TO_SUSPEND_PORT(
660 le32_to_cpu(xhci->cmd_ring->dequeue->generic.field[3])))) {
661 slot_id = TRB_TO_SLOT_ID(
662 le32_to_cpu(xhci->cmd_ring->dequeue->generic.field[3]));
663 virt_dev = xhci->devs[slot_id];
664 if (virt_dev)
665 handle_cmd_in_cmd_wait_list(xhci, virt_dev,
666 event);
667 else
668 xhci_warn(xhci, "Stop endpoint command "
669 "completion for disabled slot %u\n",
670 slot_id);
671 return;
672 }
673
641 memset(&deq_state, 0, sizeof(deq_state)); 674 memset(&deq_state, 0, sizeof(deq_state));
642 slot_id = TRB_TO_SLOT_ID(trb->generic.field[3]); 675 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(trb->generic.field[3]));
643 ep_index = TRB_TO_EP_INDEX(trb->generic.field[3]); 676 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
644 ep = &xhci->devs[slot_id]->eps[ep_index]; 677 ep = &xhci->devs[slot_id]->eps[ep_index];
645 678
646 if (list_empty(&ep->cancelled_td_list)) { 679 if (list_empty(&ep->cancelled_td_list)) {
647 xhci_stop_watchdog_timer_in_irq(xhci, ep); 680 xhci_stop_watchdog_timer_in_irq(xhci, ep);
681 ep->stopped_td = NULL;
682 ep->stopped_trb = NULL;
648 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 683 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
649 return; 684 return;
650 } 685 }
@@ -803,8 +838,7 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
803 if (ret < 0) { 838 if (ret < 0) {
804 /* This is bad; the host is not responding to commands and it's 839 /* This is bad; the host is not responding to commands and it's
805 * not allowing itself to be halted. At least interrupts are 840 * not allowing itself to be halted. At least interrupts are
806 * disabled, so we can set HC_STATE_HALT and notify the 841 * disabled. If we call usb_hc_died(), it will attempt to
807 * USB core. But if we call usb_hc_died(), it will attempt to
808 * disconnect all device drivers under this host. Those 842 * disconnect all device drivers under this host. Those
809 * disconnect() methods will wait for all URBs to be unlinked, 843 * disconnect() methods will wait for all URBs to be unlinked,
810 * so we must complete them. 844 * so we must complete them.
@@ -849,9 +883,8 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
849 } 883 }
850 } 884 }
851 spin_unlock(&xhci->lock); 885 spin_unlock(&xhci->lock);
852 xhci_to_hcd(xhci)->state = HC_STATE_HALT;
853 xhci_dbg(xhci, "Calling usb_hc_died()\n"); 886 xhci_dbg(xhci, "Calling usb_hc_died()\n");
854 usb_hc_died(xhci_to_hcd(xhci)); 887 usb_hc_died(xhci_to_hcd(xhci)->primary_hcd);
855 xhci_dbg(xhci, "xHCI host controller is dead.\n"); 888 xhci_dbg(xhci, "xHCI host controller is dead.\n");
856} 889}
857 890
@@ -874,9 +907,9 @@ static void handle_set_deq_completion(struct xhci_hcd *xhci,
874 struct xhci_ep_ctx *ep_ctx; 907 struct xhci_ep_ctx *ep_ctx;
875 struct xhci_slot_ctx *slot_ctx; 908 struct xhci_slot_ctx *slot_ctx;
876 909
877 slot_id = TRB_TO_SLOT_ID(trb->generic.field[3]); 910 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(trb->generic.field[3]));
878 ep_index = TRB_TO_EP_INDEX(trb->generic.field[3]); 911 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
879 stream_id = TRB_TO_STREAM_ID(trb->generic.field[2]); 912 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
880 dev = xhci->devs[slot_id]; 913 dev = xhci->devs[slot_id];
881 914
882 ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id); 915 ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
@@ -892,11 +925,11 @@ static void handle_set_deq_completion(struct xhci_hcd *xhci,
892 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index); 925 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
893 slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx); 926 slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
894 927
895 if (GET_COMP_CODE(event->status) != COMP_SUCCESS) { 928 if (GET_COMP_CODE(le32_to_cpu(event->status)) != COMP_SUCCESS) {
896 unsigned int ep_state; 929 unsigned int ep_state;
897 unsigned int slot_state; 930 unsigned int slot_state;
898 931
899 switch (GET_COMP_CODE(event->status)) { 932 switch (GET_COMP_CODE(le32_to_cpu(event->status))) {
900 case COMP_TRB_ERR: 933 case COMP_TRB_ERR:
901 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because " 934 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because "
902 "of stream ID configuration\n"); 935 "of stream ID configuration\n");
@@ -904,9 +937,9 @@ static void handle_set_deq_completion(struct xhci_hcd *xhci,
904 case COMP_CTX_STATE: 937 case COMP_CTX_STATE:
905 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due " 938 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due "
906 "to incorrect slot or ep state.\n"); 939 "to incorrect slot or ep state.\n");
907 ep_state = ep_ctx->ep_info; 940 ep_state = le32_to_cpu(ep_ctx->ep_info);
908 ep_state &= EP_STATE_MASK; 941 ep_state &= EP_STATE_MASK;
909 slot_state = slot_ctx->dev_state; 942 slot_state = le32_to_cpu(slot_ctx->dev_state);
910 slot_state = GET_SLOT_STATE(slot_state); 943 slot_state = GET_SLOT_STATE(slot_state);
911 xhci_dbg(xhci, "Slot state = %u, EP state = %u\n", 944 xhci_dbg(xhci, "Slot state = %u, EP state = %u\n",
912 slot_state, ep_state); 945 slot_state, ep_state);
@@ -918,7 +951,7 @@ static void handle_set_deq_completion(struct xhci_hcd *xhci,
918 default: 951 default:
919 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown " 952 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown "
920 "completion code of %u.\n", 953 "completion code of %u.\n",
921 GET_COMP_CODE(event->status)); 954 GET_COMP_CODE(le32_to_cpu(event->status)));
922 break; 955 break;
923 } 956 }
924 /* OK what do we do now? The endpoint state is hosed, and we 957 /* OK what do we do now? The endpoint state is hosed, and we
@@ -929,10 +962,27 @@ static void handle_set_deq_completion(struct xhci_hcd *xhci,
929 */ 962 */
930 } else { 963 } else {
931 xhci_dbg(xhci, "Successful Set TR Deq Ptr cmd, deq = @%08llx\n", 964 xhci_dbg(xhci, "Successful Set TR Deq Ptr cmd, deq = @%08llx\n",
932 ep_ctx->deq); 965 le64_to_cpu(ep_ctx->deq));
966 if (xhci_trb_virt_to_dma(dev->eps[ep_index].queued_deq_seg,
967 dev->eps[ep_index].queued_deq_ptr) ==
968 (le64_to_cpu(ep_ctx->deq) & ~(EP_CTX_CYCLE_MASK))) {
969 /* Update the ring's dequeue segment and dequeue pointer
970 * to reflect the new position.
971 */
972 ep_ring->deq_seg = dev->eps[ep_index].queued_deq_seg;
973 ep_ring->dequeue = dev->eps[ep_index].queued_deq_ptr;
974 } else {
975 xhci_warn(xhci, "Mismatch between completed Set TR Deq "
976 "Ptr command & xHCI internal state.\n");
977 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
978 dev->eps[ep_index].queued_deq_seg,
979 dev->eps[ep_index].queued_deq_ptr);
980 }
933 } 981 }
934 982
935 dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING; 983 dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING;
984 dev->eps[ep_index].queued_deq_seg = NULL;
985 dev->eps[ep_index].queued_deq_ptr = NULL;
936 /* Restart any rings with pending URBs */ 986 /* Restart any rings with pending URBs */
937 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 987 ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
938} 988}
@@ -944,13 +994,13 @@ static void handle_reset_ep_completion(struct xhci_hcd *xhci,
944 int slot_id; 994 int slot_id;
945 unsigned int ep_index; 995 unsigned int ep_index;
946 996
947 slot_id = TRB_TO_SLOT_ID(trb->generic.field[3]); 997 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(trb->generic.field[3]));
948 ep_index = TRB_TO_EP_INDEX(trb->generic.field[3]); 998 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
949 /* This command will only fail if the endpoint wasn't halted, 999 /* This command will only fail if the endpoint wasn't halted,
950 * but we don't care. 1000 * but we don't care.
951 */ 1001 */
952 xhci_dbg(xhci, "Ignoring reset ep completion code of %u\n", 1002 xhci_dbg(xhci, "Ignoring reset ep completion code of %u\n",
953 (unsigned int) GET_COMP_CODE(event->status)); 1003 (unsigned int) GET_COMP_CODE(le32_to_cpu(event->status)));
954 1004
955 /* HW with the reset endpoint quirk needs to have a configure endpoint 1005 /* HW with the reset endpoint quirk needs to have a configure endpoint
956 * command complete before the endpoint can be used. Queue that here 1006 * command complete before the endpoint can be used. Queue that here
@@ -987,8 +1037,7 @@ static int handle_cmd_in_cmd_wait_list(struct xhci_hcd *xhci,
987 if (xhci->cmd_ring->dequeue != command->command_trb) 1037 if (xhci->cmd_ring->dequeue != command->command_trb)
988 return 0; 1038 return 0;
989 1039
990 command->status = 1040 command->status = GET_COMP_CODE(le32_to_cpu(event->status));
991 GET_COMP_CODE(event->status);
992 list_del(&command->cmd_list); 1041 list_del(&command->cmd_list);
993 if (command->completion) 1042 if (command->completion)
994 complete(command->completion); 1043 complete(command->completion);
@@ -1000,7 +1049,7 @@ static int handle_cmd_in_cmd_wait_list(struct xhci_hcd *xhci,
1000static void handle_cmd_completion(struct xhci_hcd *xhci, 1049static void handle_cmd_completion(struct xhci_hcd *xhci,
1001 struct xhci_event_cmd *event) 1050 struct xhci_event_cmd *event)
1002{ 1051{
1003 int slot_id = TRB_TO_SLOT_ID(event->flags); 1052 int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1004 u64 cmd_dma; 1053 u64 cmd_dma;
1005 dma_addr_t cmd_dequeue_dma; 1054 dma_addr_t cmd_dequeue_dma;
1006 struct xhci_input_control_ctx *ctrl_ctx; 1055 struct xhci_input_control_ctx *ctrl_ctx;
@@ -1009,7 +1058,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
1009 struct xhci_ring *ep_ring; 1058 struct xhci_ring *ep_ring;
1010 unsigned int ep_state; 1059 unsigned int ep_state;
1011 1060
1012 cmd_dma = event->cmd_trb; 1061 cmd_dma = le64_to_cpu(event->cmd_trb);
1013 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, 1062 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1014 xhci->cmd_ring->dequeue); 1063 xhci->cmd_ring->dequeue);
1015 /* Is the command ring deq ptr out of sync with the deq seg ptr? */ 1064 /* Is the command ring deq ptr out of sync with the deq seg ptr? */
@@ -1022,17 +1071,23 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
1022 xhci->error_bitmask |= 1 << 5; 1071 xhci->error_bitmask |= 1 << 5;
1023 return; 1072 return;
1024 } 1073 }
1025 switch (xhci->cmd_ring->dequeue->generic.field[3] & TRB_TYPE_BITMASK) { 1074 switch (le32_to_cpu(xhci->cmd_ring->dequeue->generic.field[3])
1075 & TRB_TYPE_BITMASK) {
1026 case TRB_TYPE(TRB_ENABLE_SLOT): 1076 case TRB_TYPE(TRB_ENABLE_SLOT):
1027 if (GET_COMP_CODE(event->status) == COMP_SUCCESS) 1077 if (GET_COMP_CODE(le32_to_cpu(event->status)) == COMP_SUCCESS)
1028 xhci->slot_id = slot_id; 1078 xhci->slot_id = slot_id;
1029 else 1079 else
1030 xhci->slot_id = 0; 1080 xhci->slot_id = 0;
1031 complete(&xhci->addr_dev); 1081 complete(&xhci->addr_dev);
1032 break; 1082 break;
1033 case TRB_TYPE(TRB_DISABLE_SLOT): 1083 case TRB_TYPE(TRB_DISABLE_SLOT):
1034 if (xhci->devs[slot_id]) 1084 if (xhci->devs[slot_id]) {
1085 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1086 /* Delete default control endpoint resources */
1087 xhci_free_device_endpoint_resources(xhci,
1088 xhci->devs[slot_id], true);
1035 xhci_free_virt_device(xhci, slot_id); 1089 xhci_free_virt_device(xhci, slot_id);
1090 }
1036 break; 1091 break;
1037 case TRB_TYPE(TRB_CONFIG_EP): 1092 case TRB_TYPE(TRB_CONFIG_EP):
1038 virt_dev = xhci->devs[slot_id]; 1093 virt_dev = xhci->devs[slot_id];
@@ -1049,7 +1104,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
1049 ctrl_ctx = xhci_get_input_control_ctx(xhci, 1104 ctrl_ctx = xhci_get_input_control_ctx(xhci,
1050 virt_dev->in_ctx); 1105 virt_dev->in_ctx);
1051 /* Input ctx add_flags are the endpoint index plus one */ 1106 /* Input ctx add_flags are the endpoint index plus one */
1052 ep_index = xhci_last_valid_endpoint(ctrl_ctx->add_flags) - 1; 1107 ep_index = xhci_last_valid_endpoint(le32_to_cpu(ctrl_ctx->add_flags)) - 1;
1053 /* A usb_set_interface() call directly after clearing a halted 1108 /* A usb_set_interface() call directly after clearing a halted
1054 * condition may race on this quirky hardware. Not worth 1109 * condition may race on this quirky hardware. Not worth
1055 * worrying about, since this is prototype hardware. Not sure 1110 * worrying about, since this is prototype hardware. Not sure
@@ -1058,8 +1113,8 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
1058 */ 1113 */
1059 if (xhci->quirks & XHCI_RESET_EP_QUIRK && 1114 if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
1060 ep_index != (unsigned int) -1 && 1115 ep_index != (unsigned int) -1 &&
1061 ctrl_ctx->add_flags - SLOT_FLAG == 1116 le32_to_cpu(ctrl_ctx->add_flags) - SLOT_FLAG ==
1062 ctrl_ctx->drop_flags) { 1117 le32_to_cpu(ctrl_ctx->drop_flags)) {
1063 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; 1118 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
1064 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state; 1119 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
1065 if (!(ep_state & EP_HALTED)) 1120 if (!(ep_state & EP_HALTED))
@@ -1076,28 +1131,27 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
1076bandwidth_change: 1131bandwidth_change:
1077 xhci_dbg(xhci, "Completed config ep cmd\n"); 1132 xhci_dbg(xhci, "Completed config ep cmd\n");
1078 xhci->devs[slot_id]->cmd_status = 1133 xhci->devs[slot_id]->cmd_status =
1079 GET_COMP_CODE(event->status); 1134 GET_COMP_CODE(le32_to_cpu(event->status));
1080 complete(&xhci->devs[slot_id]->cmd_completion); 1135 complete(&xhci->devs[slot_id]->cmd_completion);
1081 break; 1136 break;
1082 case TRB_TYPE(TRB_EVAL_CONTEXT): 1137 case TRB_TYPE(TRB_EVAL_CONTEXT):
1083 virt_dev = xhci->devs[slot_id]; 1138 virt_dev = xhci->devs[slot_id];
1084 if (handle_cmd_in_cmd_wait_list(xhci, virt_dev, event)) 1139 if (handle_cmd_in_cmd_wait_list(xhci, virt_dev, event))
1085 break; 1140 break;
1086 xhci->devs[slot_id]->cmd_status = GET_COMP_CODE(event->status); 1141 xhci->devs[slot_id]->cmd_status = GET_COMP_CODE(le32_to_cpu(event->status));
1087 complete(&xhci->devs[slot_id]->cmd_completion); 1142 complete(&xhci->devs[slot_id]->cmd_completion);
1088 break; 1143 break;
1089 case TRB_TYPE(TRB_ADDR_DEV): 1144 case TRB_TYPE(TRB_ADDR_DEV):
1090 xhci->devs[slot_id]->cmd_status = GET_COMP_CODE(event->status); 1145 xhci->devs[slot_id]->cmd_status = GET_COMP_CODE(le32_to_cpu(event->status));
1091 complete(&xhci->addr_dev); 1146 complete(&xhci->addr_dev);
1092 break; 1147 break;
1093 case TRB_TYPE(TRB_STOP_RING): 1148 case TRB_TYPE(TRB_STOP_RING):
1094 handle_stopped_endpoint(xhci, xhci->cmd_ring->dequeue); 1149 handle_stopped_endpoint(xhci, xhci->cmd_ring->dequeue, event);
1095 break; 1150 break;
1096 case TRB_TYPE(TRB_SET_DEQ): 1151 case TRB_TYPE(TRB_SET_DEQ):
1097 handle_set_deq_completion(xhci, event, xhci->cmd_ring->dequeue); 1152 handle_set_deq_completion(xhci, event, xhci->cmd_ring->dequeue);
1098 break; 1153 break;
1099 case TRB_TYPE(TRB_CMD_NOOP): 1154 case TRB_TYPE(TRB_CMD_NOOP):
1100 ++xhci->noops_handled;
1101 break; 1155 break;
1102 case TRB_TYPE(TRB_RESET_EP): 1156 case TRB_TYPE(TRB_RESET_EP):
1103 handle_reset_ep_completion(xhci, event, xhci->cmd_ring->dequeue); 1157 handle_reset_ep_completion(xhci, event, xhci->cmd_ring->dequeue);
@@ -1105,7 +1159,7 @@ bandwidth_change:
1105 case TRB_TYPE(TRB_RESET_DEV): 1159 case TRB_TYPE(TRB_RESET_DEV):
1106 xhci_dbg(xhci, "Completed reset device command.\n"); 1160 xhci_dbg(xhci, "Completed reset device command.\n");
1107 slot_id = TRB_TO_SLOT_ID( 1161 slot_id = TRB_TO_SLOT_ID(
1108 xhci->cmd_ring->dequeue->generic.field[3]); 1162 le32_to_cpu(xhci->cmd_ring->dequeue->generic.field[3]));
1109 virt_dev = xhci->devs[slot_id]; 1163 virt_dev = xhci->devs[slot_id];
1110 if (virt_dev) 1164 if (virt_dev)
1111 handle_cmd_in_cmd_wait_list(xhci, virt_dev, event); 1165 handle_cmd_in_cmd_wait_list(xhci, virt_dev, event);
@@ -1119,8 +1173,8 @@ bandwidth_change:
1119 break; 1173 break;
1120 } 1174 }
1121 xhci_dbg(xhci, "NEC firmware version %2x.%02x\n", 1175 xhci_dbg(xhci, "NEC firmware version %2x.%02x\n",
1122 NEC_FW_MAJOR(event->status), 1176 NEC_FW_MAJOR(le32_to_cpu(event->status)),
1123 NEC_FW_MINOR(event->status)); 1177 NEC_FW_MINOR(le32_to_cpu(event->status)));
1124 break; 1178 break;
1125 default: 1179 default:
1126 /* Skip over unknown commands on the event ring */ 1180 /* Skip over unknown commands on the event ring */
@@ -1135,32 +1189,175 @@ static void handle_vendor_event(struct xhci_hcd *xhci,
1135{ 1189{
1136 u32 trb_type; 1190 u32 trb_type;
1137 1191
1138 trb_type = TRB_FIELD_TO_TYPE(event->generic.field[3]); 1192 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1139 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); 1193 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1140 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) 1194 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1141 handle_cmd_completion(xhci, &event->event_cmd); 1195 handle_cmd_completion(xhci, &event->event_cmd);
1142} 1196}
1143 1197
1198/* @port_id: the one-based port ID from the hardware (indexed from array of all
1199 * port registers -- USB 3.0 and USB 2.0).
1200 *
1201 * Returns a zero-based port number, which is suitable for indexing into each of
1202 * the split roothubs' port arrays and bus state arrays.
1203 */
1204static unsigned int find_faked_portnum_from_hw_portnum(struct usb_hcd *hcd,
1205 struct xhci_hcd *xhci, u32 port_id)
1206{
1207 unsigned int i;
1208 unsigned int num_similar_speed_ports = 0;
1209
1210 /* port_id from the hardware is 1-based, but port_array[], usb3_ports[],
1211 * and usb2_ports are 0-based indexes. Count the number of similar
1212 * speed ports, up to 1 port before this port.
1213 */
1214 for (i = 0; i < (port_id - 1); i++) {
1215 u8 port_speed = xhci->port_array[i];
1216
1217 /*
1218 * Skip ports that don't have known speeds, or have duplicate
1219 * Extended Capabilities port speed entries.
1220 */
1221 if (port_speed == 0 || port_speed == DUPLICATE_ENTRY)
1222 continue;
1223
1224 /*
1225 * USB 3.0 ports are always under a USB 3.0 hub. USB 2.0 and
1226 * 1.1 ports are under the USB 2.0 hub. If the port speed
1227 * matches the device speed, it's a similar speed port.
1228 */
1229 if ((port_speed == 0x03) == (hcd->speed == HCD_USB3))
1230 num_similar_speed_ports++;
1231 }
1232 return num_similar_speed_ports;
1233}
1234
1144static void handle_port_status(struct xhci_hcd *xhci, 1235static void handle_port_status(struct xhci_hcd *xhci,
1145 union xhci_trb *event) 1236 union xhci_trb *event)
1146{ 1237{
1238 struct usb_hcd *hcd;
1147 u32 port_id; 1239 u32 port_id;
1240 u32 temp, temp1;
1241 int max_ports;
1242 int slot_id;
1243 unsigned int faked_port_index;
1244 u8 major_revision;
1245 struct xhci_bus_state *bus_state;
1246 __le32 __iomem **port_array;
1247 bool bogus_port_status = false;
1148 1248
1149 /* Port status change events always have a successful completion code */ 1249 /* Port status change events always have a successful completion code */
1150 if (GET_COMP_CODE(event->generic.field[2]) != COMP_SUCCESS) { 1250 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) {
1151 xhci_warn(xhci, "WARN: xHC returned failed port status event\n"); 1251 xhci_warn(xhci, "WARN: xHC returned failed port status event\n");
1152 xhci->error_bitmask |= 1 << 8; 1252 xhci->error_bitmask |= 1 << 8;
1153 } 1253 }
1154 /* FIXME: core doesn't care about all port link state changes yet */ 1254 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1155 port_id = GET_PORT_ID(event->generic.field[0]);
1156 xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id); 1255 xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
1157 1256
1257 max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1258 if ((port_id <= 0) || (port_id > max_ports)) {
1259 xhci_warn(xhci, "Invalid port id %d\n", port_id);
1260 bogus_port_status = true;
1261 goto cleanup;
1262 }
1263
1264 /* Figure out which usb_hcd this port is attached to:
1265 * is it a USB 3.0 port or a USB 2.0/1.1 port?
1266 */
1267 major_revision = xhci->port_array[port_id - 1];
1268 if (major_revision == 0) {
1269 xhci_warn(xhci, "Event for port %u not in "
1270 "Extended Capabilities, ignoring.\n",
1271 port_id);
1272 bogus_port_status = true;
1273 goto cleanup;
1274 }
1275 if (major_revision == DUPLICATE_ENTRY) {
1276 xhci_warn(xhci, "Event for port %u duplicated in"
1277 "Extended Capabilities, ignoring.\n",
1278 port_id);
1279 bogus_port_status = true;
1280 goto cleanup;
1281 }
1282
1283 /*
1284 * Hardware port IDs reported by a Port Status Change Event include USB
1285 * 3.0 and USB 2.0 ports. We want to check if the port has reported a
1286 * resume event, but we first need to translate the hardware port ID
1287 * into the index into the ports on the correct split roothub, and the
1288 * correct bus_state structure.
1289 */
1290 /* Find the right roothub. */
1291 hcd = xhci_to_hcd(xhci);
1292 if ((major_revision == 0x03) != (hcd->speed == HCD_USB3))
1293 hcd = xhci->shared_hcd;
1294 bus_state = &xhci->bus_state[hcd_index(hcd)];
1295 if (hcd->speed == HCD_USB3)
1296 port_array = xhci->usb3_ports;
1297 else
1298 port_array = xhci->usb2_ports;
1299 /* Find the faked port hub number */
1300 faked_port_index = find_faked_portnum_from_hw_portnum(hcd, xhci,
1301 port_id);
1302
1303 temp = xhci_readl(xhci, port_array[faked_port_index]);
1304 if (hcd->state == HC_STATE_SUSPENDED) {
1305 xhci_dbg(xhci, "resume root hub\n");
1306 usb_hcd_resume_root_hub(hcd);
1307 }
1308
1309 if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_RESUME) {
1310 xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1311
1312 temp1 = xhci_readl(xhci, &xhci->op_regs->command);
1313 if (!(temp1 & CMD_RUN)) {
1314 xhci_warn(xhci, "xHC is not running.\n");
1315 goto cleanup;
1316 }
1317
1318 if (DEV_SUPERSPEED(temp)) {
1319 xhci_dbg(xhci, "resume SS port %d\n", port_id);
1320 temp = xhci_port_state_to_neutral(temp);
1321 temp &= ~PORT_PLS_MASK;
1322 temp |= PORT_LINK_STROBE | XDEV_U0;
1323 xhci_writel(xhci, temp, port_array[faked_port_index]);
1324 slot_id = xhci_find_slot_id_by_port(hcd, xhci,
1325 faked_port_index);
1326 if (!slot_id) {
1327 xhci_dbg(xhci, "slot_id is zero\n");
1328 goto cleanup;
1329 }
1330 xhci_ring_device(xhci, slot_id);
1331 xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1332 /* Clear PORT_PLC */
1333 temp = xhci_readl(xhci, port_array[faked_port_index]);
1334 temp = xhci_port_state_to_neutral(temp);
1335 temp |= PORT_PLC;
1336 xhci_writel(xhci, temp, port_array[faked_port_index]);
1337 } else {
1338 xhci_dbg(xhci, "resume HS port %d\n", port_id);
1339 bus_state->resume_done[faked_port_index] = jiffies +
1340 msecs_to_jiffies(20);
1341 mod_timer(&hcd->rh_timer,
1342 bus_state->resume_done[faked_port_index]);
1343 /* Do the rest in GetPortStatus */
1344 }
1345 }
1346
1347cleanup:
1158 /* Update event ring dequeue pointer before dropping the lock */ 1348 /* Update event ring dequeue pointer before dropping the lock */
1159 inc_deq(xhci, xhci->event_ring, true); 1349 inc_deq(xhci, xhci->event_ring, true);
1160 1350
1351 /* Don't make the USB core poll the roothub if we got a bad port status
1352 * change event. Besides, at that point we can't tell which roothub
1353 * (USB 2.0 or USB 3.0) to kick.
1354 */
1355 if (bogus_port_status)
1356 return;
1357
1161 spin_unlock(&xhci->lock); 1358 spin_unlock(&xhci->lock);
1162 /* Pass this up to the core */ 1359 /* Pass this up to the core */
1163 usb_hcd_poll_rh_status(xhci_to_hcd(xhci)); 1360 usb_hcd_poll_rh_status(hcd);
1164 spin_lock(&xhci->lock); 1361 spin_lock(&xhci->lock);
1165} 1362}
1166 1363
@@ -1261,7 +1458,7 @@ static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
1261 * endpoint anyway. Check if a babble halted the 1458 * endpoint anyway. Check if a babble halted the
1262 * endpoint. 1459 * endpoint.
1263 */ 1460 */
1264 if ((ep_ctx->ep_info & EP_STATE_MASK) == EP_STATE_HALTED) 1461 if ((le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK) == EP_STATE_HALTED)
1265 return 1; 1462 return 1;
1266 1463
1267 return 0; 1464 return 0;
@@ -1299,12 +1496,12 @@ static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
1299 struct urb_priv *urb_priv; 1496 struct urb_priv *urb_priv;
1300 u32 trb_comp_code; 1497 u32 trb_comp_code;
1301 1498
1302 slot_id = TRB_TO_SLOT_ID(event->flags); 1499 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1303 xdev = xhci->devs[slot_id]; 1500 xdev = xhci->devs[slot_id];
1304 ep_index = TRB_TO_EP_ID(event->flags) - 1; 1501 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1305 ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); 1502 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1306 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); 1503 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1307 trb_comp_code = GET_COMP_CODE(event->transfer_len); 1504 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1308 1505
1309 if (skip) 1506 if (skip)
1310 goto td_cleanup; 1507 goto td_cleanup;
@@ -1377,8 +1574,17 @@ td_cleanup:
1377 1574
1378 urb_priv->td_cnt++; 1575 urb_priv->td_cnt++;
1379 /* Giveback the urb when all the tds are completed */ 1576 /* Giveback the urb when all the tds are completed */
1380 if (urb_priv->td_cnt == urb_priv->length) 1577 if (urb_priv->td_cnt == urb_priv->length) {
1381 ret = 1; 1578 ret = 1;
1579 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
1580 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
1581 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs
1582 == 0) {
1583 if (xhci->quirks & XHCI_AMD_PLL_FIX)
1584 usb_amd_quirk_pll_enable();
1585 }
1586 }
1587 }
1382 } 1588 }
1383 1589
1384 return ret; 1590 return ret;
@@ -1398,12 +1604,12 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1398 struct xhci_ep_ctx *ep_ctx; 1604 struct xhci_ep_ctx *ep_ctx;
1399 u32 trb_comp_code; 1605 u32 trb_comp_code;
1400 1606
1401 slot_id = TRB_TO_SLOT_ID(event->flags); 1607 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1402 xdev = xhci->devs[slot_id]; 1608 xdev = xhci->devs[slot_id];
1403 ep_index = TRB_TO_EP_ID(event->flags) - 1; 1609 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1404 ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); 1610 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1405 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); 1611 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1406 trb_comp_code = GET_COMP_CODE(event->transfer_len); 1612 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1407 1613
1408 xhci_debug_trb(xhci, xhci->event_ring->dequeue); 1614 xhci_debug_trb(xhci, xhci->event_ring->dequeue);
1409 switch (trb_comp_code) { 1615 switch (trb_comp_code) {
@@ -1417,7 +1623,6 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1417 "without IOC set??\n"); 1623 "without IOC set??\n");
1418 *status = -ESHUTDOWN; 1624 *status = -ESHUTDOWN;
1419 } else { 1625 } else {
1420 xhci_dbg(xhci, "Successful control transfer!\n");
1421 *status = 0; 1626 *status = 0;
1422 } 1627 }
1423 break; 1628 break;
@@ -1428,6 +1633,9 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1428 else 1633 else
1429 *status = 0; 1634 *status = 0;
1430 break; 1635 break;
1636 case COMP_STOP_INVAL:
1637 case COMP_STOP:
1638 return finish_td(xhci, td, event_trb, event, ep, status, false);
1431 default: 1639 default:
1432 if (!xhci_requires_manual_halt_cleanup(xhci, 1640 if (!xhci_requires_manual_halt_cleanup(xhci,
1433 ep_ctx, trb_comp_code)) 1641 ep_ctx, trb_comp_code))
@@ -1442,7 +1650,7 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1442 event_trb != td->last_trb) 1650 event_trb != td->last_trb)
1443 td->urb->actual_length = 1651 td->urb->actual_length =
1444 td->urb->transfer_buffer_length 1652 td->urb->transfer_buffer_length
1445 - TRB_LEN(event->transfer_len); 1653 - TRB_LEN(le32_to_cpu(event->transfer_len));
1446 else 1654 else
1447 td->urb->actual_length = 0; 1655 td->urb->actual_length = 0;
1448 1656
@@ -1472,15 +1680,12 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1472 } 1680 }
1473 } else { 1681 } else {
1474 /* Maybe the event was for the data stage? */ 1682 /* Maybe the event was for the data stage? */
1475 if (trb_comp_code != COMP_STOP_INVAL) { 1683 td->urb->actual_length =
1476 /* We didn't stop on a link TRB in the middle */ 1684 td->urb->transfer_buffer_length -
1477 td->urb->actual_length = 1685 TRB_LEN(le32_to_cpu(event->transfer_len));
1478 td->urb->transfer_buffer_length - 1686 xhci_dbg(xhci, "Waiting for status "
1479 TRB_LEN(event->transfer_len); 1687 "stage event\n");
1480 xhci_dbg(xhci, "Waiting for status " 1688 return 0;
1481 "stage event\n");
1482 return 0;
1483 }
1484 } 1689 }
1485 } 1690 }
1486 1691
@@ -1498,97 +1703,102 @@ static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
1498 struct urb_priv *urb_priv; 1703 struct urb_priv *urb_priv;
1499 int idx; 1704 int idx;
1500 int len = 0; 1705 int len = 0;
1501 int skip_td = 0;
1502 union xhci_trb *cur_trb; 1706 union xhci_trb *cur_trb;
1503 struct xhci_segment *cur_seg; 1707 struct xhci_segment *cur_seg;
1708 struct usb_iso_packet_descriptor *frame;
1504 u32 trb_comp_code; 1709 u32 trb_comp_code;
1710 bool skip_td = false;
1505 1711
1506 ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); 1712 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1507 trb_comp_code = GET_COMP_CODE(event->transfer_len); 1713 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1508 urb_priv = td->urb->hcpriv; 1714 urb_priv = td->urb->hcpriv;
1509 idx = urb_priv->td_cnt; 1715 idx = urb_priv->td_cnt;
1716 frame = &td->urb->iso_frame_desc[idx];
1510 1717
1511 if (ep->skip) { 1718 /* handle completion code */
1512 /* The transfer is partly done */ 1719 switch (trb_comp_code) {
1513 *status = -EXDEV; 1720 case COMP_SUCCESS:
1514 td->urb->iso_frame_desc[idx].status = -EXDEV; 1721 frame->status = 0;
1515 } else { 1722 break;
1516 /* handle completion code */ 1723 case COMP_SHORT_TX:
1517 switch (trb_comp_code) { 1724 frame->status = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
1518 case COMP_SUCCESS: 1725 -EREMOTEIO : 0;
1519 td->urb->iso_frame_desc[idx].status = 0; 1726 break;
1520 xhci_dbg(xhci, "Successful isoc transfer!\n"); 1727 case COMP_BW_OVER:
1521 break; 1728 frame->status = -ECOMM;
1522 case COMP_SHORT_TX: 1729 skip_td = true;
1523 if (td->urb->transfer_flags & URB_SHORT_NOT_OK) 1730 break;
1524 td->urb->iso_frame_desc[idx].status = 1731 case COMP_BUFF_OVER:
1525 -EREMOTEIO; 1732 case COMP_BABBLE:
1526 else 1733 frame->status = -EOVERFLOW;
1527 td->urb->iso_frame_desc[idx].status = 0; 1734 skip_td = true;
1528 break; 1735 break;
1529 case COMP_BW_OVER: 1736 case COMP_DEV_ERR:
1530 td->urb->iso_frame_desc[idx].status = -ECOMM; 1737 case COMP_STALL:
1531 skip_td = 1; 1738 frame->status = -EPROTO;
1532 break; 1739 skip_td = true;
1533 case COMP_BUFF_OVER: 1740 break;
1534 case COMP_BABBLE: 1741 case COMP_STOP:
1535 td->urb->iso_frame_desc[idx].status = -EOVERFLOW; 1742 case COMP_STOP_INVAL:
1536 skip_td = 1; 1743 break;
1537 break; 1744 default:
1538 case COMP_STALL: 1745 frame->status = -1;
1539 td->urb->iso_frame_desc[idx].status = -EPROTO; 1746 break;
1540 skip_td = 1;
1541 break;
1542 case COMP_STOP:
1543 case COMP_STOP_INVAL:
1544 break;
1545 default:
1546 td->urb->iso_frame_desc[idx].status = -1;
1547 break;
1548 }
1549 }
1550
1551 /* calc actual length */
1552 if (ep->skip) {
1553 td->urb->iso_frame_desc[idx].actual_length = 0;
1554 /* Update ring dequeue pointer */
1555 while (ep_ring->dequeue != td->last_trb)
1556 inc_deq(xhci, ep_ring, false);
1557 inc_deq(xhci, ep_ring, false);
1558 return finish_td(xhci, td, event_trb, event, ep, status, true);
1559 } 1747 }
1560 1748
1561 if (trb_comp_code == COMP_SUCCESS || skip_td == 1) { 1749 if (trb_comp_code == COMP_SUCCESS || skip_td) {
1562 td->urb->iso_frame_desc[idx].actual_length = 1750 frame->actual_length = frame->length;
1563 td->urb->iso_frame_desc[idx].length; 1751 td->urb->actual_length += frame->length;
1564 td->urb->actual_length +=
1565 td->urb->iso_frame_desc[idx].length;
1566 } else { 1752 } else {
1567 for (cur_trb = ep_ring->dequeue, 1753 for (cur_trb = ep_ring->dequeue,
1568 cur_seg = ep_ring->deq_seg; cur_trb != event_trb; 1754 cur_seg = ep_ring->deq_seg; cur_trb != event_trb;
1569 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) { 1755 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
1570 if ((cur_trb->generic.field[3] & 1756 if ((le32_to_cpu(cur_trb->generic.field[3]) &
1571 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) && 1757 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) &&
1572 (cur_trb->generic.field[3] & 1758 (le32_to_cpu(cur_trb->generic.field[3]) &
1573 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK)) 1759 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK))
1574 len += 1760 len += TRB_LEN(le32_to_cpu(cur_trb->generic.field[2]));
1575 TRB_LEN(cur_trb->generic.field[2]);
1576 } 1761 }
1577 len += TRB_LEN(cur_trb->generic.field[2]) - 1762 len += TRB_LEN(le32_to_cpu(cur_trb->generic.field[2])) -
1578 TRB_LEN(event->transfer_len); 1763 TRB_LEN(le32_to_cpu(event->transfer_len));
1579 1764
1580 if (trb_comp_code != COMP_STOP_INVAL) { 1765 if (trb_comp_code != COMP_STOP_INVAL) {
1581 td->urb->iso_frame_desc[idx].actual_length = len; 1766 frame->actual_length = len;
1582 td->urb->actual_length += len; 1767 td->urb->actual_length += len;
1583 } 1768 }
1584 } 1769 }
1585 1770
1586 if ((idx == urb_priv->length - 1) && *status == -EINPROGRESS)
1587 *status = 0;
1588
1589 return finish_td(xhci, td, event_trb, event, ep, status, false); 1771 return finish_td(xhci, td, event_trb, event, ep, status, false);
1590} 1772}
1591 1773
1774static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
1775 struct xhci_transfer_event *event,
1776 struct xhci_virt_ep *ep, int *status)
1777{
1778 struct xhci_ring *ep_ring;
1779 struct urb_priv *urb_priv;
1780 struct usb_iso_packet_descriptor *frame;
1781 int idx;
1782
1783 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1784 urb_priv = td->urb->hcpriv;
1785 idx = urb_priv->td_cnt;
1786 frame = &td->urb->iso_frame_desc[idx];
1787
1788 /* The transfer is partly done. */
1789 frame->status = -EXDEV;
1790
1791 /* calc actual length */
1792 frame->actual_length = 0;
1793
1794 /* Update ring dequeue pointer */
1795 while (ep_ring->dequeue != td->last_trb)
1796 inc_deq(xhci, ep_ring, false);
1797 inc_deq(xhci, ep_ring, false);
1798
1799 return finish_td(xhci, td, NULL, event, ep, status, true);
1800}
1801
1592/* 1802/*
1593 * Process bulk and interrupt tds, update urb status and actual_length. 1803 * Process bulk and interrupt tds, update urb status and actual_length.
1594 */ 1804 */
@@ -1601,8 +1811,8 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
1601 struct xhci_segment *cur_seg; 1811 struct xhci_segment *cur_seg;
1602 u32 trb_comp_code; 1812 u32 trb_comp_code;
1603 1813
1604 ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); 1814 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1605 trb_comp_code = GET_COMP_CODE(event->transfer_len); 1815 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1606 1816
1607 switch (trb_comp_code) { 1817 switch (trb_comp_code) {
1608 case COMP_SUCCESS: 1818 case COMP_SUCCESS:
@@ -1615,12 +1825,6 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
1615 else 1825 else
1616 *status = 0; 1826 *status = 0;
1617 } else { 1827 } else {
1618 if (usb_endpoint_xfer_bulk(&td->urb->ep->desc))
1619 xhci_dbg(xhci, "Successful bulk "
1620 "transfer!\n");
1621 else
1622 xhci_dbg(xhci, "Successful interrupt "
1623 "transfer!\n");
1624 *status = 0; 1828 *status = 0;
1625 } 1829 }
1626 break; 1830 break;
@@ -1634,23 +1838,23 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
1634 /* Others already handled above */ 1838 /* Others already handled above */
1635 break; 1839 break;
1636 } 1840 }
1637 dev_dbg(&td->urb->dev->dev, 1841 if (trb_comp_code == COMP_SHORT_TX)
1638 "ep %#x - asked for %d bytes, " 1842 xhci_dbg(xhci, "ep %#x - asked for %d bytes, "
1639 "%d bytes untransferred\n", 1843 "%d bytes untransferred\n",
1640 td->urb->ep->desc.bEndpointAddress, 1844 td->urb->ep->desc.bEndpointAddress,
1641 td->urb->transfer_buffer_length, 1845 td->urb->transfer_buffer_length,
1642 TRB_LEN(event->transfer_len)); 1846 TRB_LEN(le32_to_cpu(event->transfer_len)));
1643 /* Fast path - was this the last TRB in the TD for this URB? */ 1847 /* Fast path - was this the last TRB in the TD for this URB? */
1644 if (event_trb == td->last_trb) { 1848 if (event_trb == td->last_trb) {
1645 if (TRB_LEN(event->transfer_len) != 0) { 1849 if (TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) {
1646 td->urb->actual_length = 1850 td->urb->actual_length =
1647 td->urb->transfer_buffer_length - 1851 td->urb->transfer_buffer_length -
1648 TRB_LEN(event->transfer_len); 1852 TRB_LEN(le32_to_cpu(event->transfer_len));
1649 if (td->urb->transfer_buffer_length < 1853 if (td->urb->transfer_buffer_length <
1650 td->urb->actual_length) { 1854 td->urb->actual_length) {
1651 xhci_warn(xhci, "HC gave bad length " 1855 xhci_warn(xhci, "HC gave bad length "
1652 "of %d bytes left\n", 1856 "of %d bytes left\n",
1653 TRB_LEN(event->transfer_len)); 1857 TRB_LEN(le32_to_cpu(event->transfer_len)));
1654 td->urb->actual_length = 0; 1858 td->urb->actual_length = 0;
1655 if (td->urb->transfer_flags & URB_SHORT_NOT_OK) 1859 if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
1656 *status = -EREMOTEIO; 1860 *status = -EREMOTEIO;
@@ -1681,20 +1885,20 @@ static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
1681 for (cur_trb = ep_ring->dequeue, cur_seg = ep_ring->deq_seg; 1885 for (cur_trb = ep_ring->dequeue, cur_seg = ep_ring->deq_seg;
1682 cur_trb != event_trb; 1886 cur_trb != event_trb;
1683 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) { 1887 next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
1684 if ((cur_trb->generic.field[3] & 1888 if ((le32_to_cpu(cur_trb->generic.field[3]) &
1685 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) && 1889 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) &&
1686 (cur_trb->generic.field[3] & 1890 (le32_to_cpu(cur_trb->generic.field[3]) &
1687 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK)) 1891 TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK))
1688 td->urb->actual_length += 1892 td->urb->actual_length +=
1689 TRB_LEN(cur_trb->generic.field[2]); 1893 TRB_LEN(le32_to_cpu(cur_trb->generic.field[2]));
1690 } 1894 }
1691 /* If the ring didn't stop on a Link or No-op TRB, add 1895 /* If the ring didn't stop on a Link or No-op TRB, add
1692 * in the actual bytes transferred from the Normal TRB 1896 * in the actual bytes transferred from the Normal TRB
1693 */ 1897 */
1694 if (trb_comp_code != COMP_STOP_INVAL) 1898 if (trb_comp_code != COMP_STOP_INVAL)
1695 td->urb->actual_length += 1899 td->urb->actual_length +=
1696 TRB_LEN(cur_trb->generic.field[2]) - 1900 TRB_LEN(le32_to_cpu(cur_trb->generic.field[2])) -
1697 TRB_LEN(event->transfer_len); 1901 TRB_LEN(le32_to_cpu(event->transfer_len));
1698 } 1902 }
1699 1903
1700 return finish_td(xhci, td, event_trb, event, ep, status, false); 1904 return finish_td(xhci, td, event_trb, event, ep, status, false);
@@ -1724,7 +1928,7 @@ static int handle_tx_event(struct xhci_hcd *xhci,
1724 u32 trb_comp_code; 1928 u32 trb_comp_code;
1725 int ret = 0; 1929 int ret = 0;
1726 1930
1727 slot_id = TRB_TO_SLOT_ID(event->flags); 1931 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1728 xdev = xhci->devs[slot_id]; 1932 xdev = xhci->devs[slot_id];
1729 if (!xdev) { 1933 if (!xdev) {
1730 xhci_err(xhci, "ERROR Transfer event pointed to bad slot\n"); 1934 xhci_err(xhci, "ERROR Transfer event pointed to bad slot\n");
@@ -1732,20 +1936,20 @@ static int handle_tx_event(struct xhci_hcd *xhci,
1732 } 1936 }
1733 1937
1734 /* Endpoint ID is 1 based, our index is zero based */ 1938 /* Endpoint ID is 1 based, our index is zero based */
1735 ep_index = TRB_TO_EP_ID(event->flags) - 1; 1939 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1736 xhci_dbg(xhci, "%s - ep index = %d\n", __func__, ep_index);
1737 ep = &xdev->eps[ep_index]; 1940 ep = &xdev->eps[ep_index];
1738 ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); 1941 ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1739 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); 1942 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1740 if (!ep_ring || 1943 if (!ep_ring ||
1741 (ep_ctx->ep_info & EP_STATE_MASK) == EP_STATE_DISABLED) { 1944 (le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK) ==
1945 EP_STATE_DISABLED) {
1742 xhci_err(xhci, "ERROR Transfer event for disabled endpoint " 1946 xhci_err(xhci, "ERROR Transfer event for disabled endpoint "
1743 "or incorrect stream ring\n"); 1947 "or incorrect stream ring\n");
1744 return -ENODEV; 1948 return -ENODEV;
1745 } 1949 }
1746 1950
1747 event_dma = event->buffer; 1951 event_dma = le64_to_cpu(event->buffer);
1748 trb_comp_code = GET_COMP_CODE(event->transfer_len); 1952 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1749 /* Look for common error cases */ 1953 /* Look for common error cases */
1750 switch (trb_comp_code) { 1954 switch (trb_comp_code) {
1751 /* Skip codes that require special handling depending on 1955 /* Skip codes that require special handling depending on
@@ -1798,15 +2002,21 @@ static int handle_tx_event(struct xhci_hcd *xhci,
1798 if (!list_empty(&ep_ring->td_list)) 2002 if (!list_empty(&ep_ring->td_list))
1799 xhci_dbg(xhci, "Underrun Event for slot %d ep %d " 2003 xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
1800 "still with TDs queued?\n", 2004 "still with TDs queued?\n",
1801 TRB_TO_SLOT_ID(event->flags), ep_index); 2005 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2006 ep_index);
1802 goto cleanup; 2007 goto cleanup;
1803 case COMP_OVERRUN: 2008 case COMP_OVERRUN:
1804 xhci_dbg(xhci, "overrun event on endpoint\n"); 2009 xhci_dbg(xhci, "overrun event on endpoint\n");
1805 if (!list_empty(&ep_ring->td_list)) 2010 if (!list_empty(&ep_ring->td_list))
1806 xhci_dbg(xhci, "Overrun Event for slot %d ep %d " 2011 xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
1807 "still with TDs queued?\n", 2012 "still with TDs queued?\n",
1808 TRB_TO_SLOT_ID(event->flags), ep_index); 2013 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2014 ep_index);
1809 goto cleanup; 2015 goto cleanup;
2016 case COMP_DEV_ERR:
2017 xhci_warn(xhci, "WARN: detect an incompatible device");
2018 status = -EPROTO;
2019 break;
1810 case COMP_MISSED_INT: 2020 case COMP_MISSED_INT:
1811 /* 2021 /*
1812 * When encounter missed service error, one or more isoc tds 2022 * When encounter missed service error, one or more isoc tds
@@ -1834,9 +2044,11 @@ static int handle_tx_event(struct xhci_hcd *xhci,
1834 if (list_empty(&ep_ring->td_list)) { 2044 if (list_empty(&ep_ring->td_list)) {
1835 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d " 2045 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d "
1836 "with no TDs queued?\n", 2046 "with no TDs queued?\n",
1837 TRB_TO_SLOT_ID(event->flags), ep_index); 2047 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2048 ep_index);
1838 xhci_dbg(xhci, "Event TRB with TRB type ID %u\n", 2049 xhci_dbg(xhci, "Event TRB with TRB type ID %u\n",
1839 (unsigned int) (event->flags & TRB_TYPE_BITMASK)>>10); 2050 (unsigned int) (le32_to_cpu(event->flags)
2051 & TRB_TYPE_BITMASK)>>10);
1840 xhci_print_trb_offsets(xhci, (union xhci_trb *) event); 2052 xhci_print_trb_offsets(xhci, (union xhci_trb *) event);
1841 if (ep->skip) { 2053 if (ep->skip) {
1842 ep->skip = false; 2054 ep->skip = false;
@@ -1848,36 +2060,71 @@ static int handle_tx_event(struct xhci_hcd *xhci,
1848 } 2060 }
1849 2061
1850 td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list); 2062 td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list);
2063
1851 /* Is this a TRB in the currently executing TD? */ 2064 /* Is this a TRB in the currently executing TD? */
1852 event_seg = trb_in_td(ep_ring->deq_seg, ep_ring->dequeue, 2065 event_seg = trb_in_td(ep_ring->deq_seg, ep_ring->dequeue,
1853 td->last_trb, event_dma); 2066 td->last_trb, event_dma);
1854 if (event_seg && ep->skip) { 2067
1855 xhci_dbg(xhci, "Found td. Clear skip flag.\n"); 2068 /*
1856 ep->skip = false; 2069 * Skip the Force Stopped Event. The event_trb(event_dma) of FSE
2070 * is not in the current TD pointed by ep_ring->dequeue because
2071 * that the hardware dequeue pointer still at the previous TRB
2072 * of the current TD. The previous TRB maybe a Link TD or the
2073 * last TRB of the previous TD. The command completion handle
2074 * will take care the rest.
2075 */
2076 if (!event_seg && trb_comp_code == COMP_STOP_INVAL) {
2077 ret = 0;
2078 goto cleanup;
1857 } 2079 }
1858 if (!event_seg && 2080
1859 (!ep->skip || !usb_endpoint_xfer_isoc(&td->urb->ep->desc))) { 2081 if (!event_seg) {
1860 /* HC is busted, give up! */ 2082 if (!ep->skip ||
1861 xhci_err(xhci, "ERROR Transfer event TRB DMA ptr not " 2083 !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
2084 /* Some host controllers give a spurious
2085 * successful event after a short transfer.
2086 * Ignore it.
2087 */
2088 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2089 ep_ring->last_td_was_short) {
2090 ep_ring->last_td_was_short = false;
2091 ret = 0;
2092 goto cleanup;
2093 }
2094 /* HC is busted, give up! */
2095 xhci_err(xhci,
2096 "ERROR Transfer event TRB DMA ptr not "
1862 "part of current TD\n"); 2097 "part of current TD\n");
1863 return -ESHUTDOWN; 2098 return -ESHUTDOWN;
2099 }
2100
2101 ret = skip_isoc_td(xhci, td, event, ep, &status);
2102 goto cleanup;
1864 } 2103 }
2104 if (trb_comp_code == COMP_SHORT_TX)
2105 ep_ring->last_td_was_short = true;
2106 else
2107 ep_ring->last_td_was_short = false;
1865 2108
1866 if (event_seg) { 2109 if (ep->skip) {
1867 event_trb = &event_seg->trbs[(event_dma - 2110 xhci_dbg(xhci, "Found td. Clear skip flag.\n");
1868 event_seg->dma) / sizeof(*event_trb)]; 2111 ep->skip = false;
1869 /* 2112 }
1870 * No-op TRB should not trigger interrupts. 2113
1871 * If event_trb is a no-op TRB, it means the 2114 event_trb = &event_seg->trbs[(event_dma - event_seg->dma) /
1872 * corresponding TD has been cancelled. Just ignore 2115 sizeof(*event_trb)];
1873 * the TD. 2116 /*
1874 */ 2117 * No-op TRB should not trigger interrupts.
1875 if ((event_trb->generic.field[3] & TRB_TYPE_BITMASK) 2118 * If event_trb is a no-op TRB, it means the
1876 == TRB_TYPE(TRB_TR_NOOP)) { 2119 * corresponding TD has been cancelled. Just ignore
1877 xhci_dbg(xhci, "event_trb is a no-op TRB. " 2120 * the TD.
1878 "Skip it\n"); 2121 */
1879 goto cleanup; 2122 if ((le32_to_cpu(event_trb->generic.field[3])
1880 } 2123 & TRB_TYPE_BITMASK)
2124 == TRB_TYPE(TRB_TR_NOOP)) {
2125 xhci_dbg(xhci,
2126 "event_trb is a no-op TRB. Skip it\n");
2127 goto cleanup;
1881 } 2128 }
1882 2129
1883 /* Now update the urb's actual_length and give back to 2130 /* Now update the urb's actual_length and give back to
@@ -1915,12 +2162,23 @@ cleanup:
1915 trb_comp_code != COMP_BABBLE)) 2162 trb_comp_code != COMP_BABBLE))
1916 xhci_urb_free_priv(xhci, urb_priv); 2163 xhci_urb_free_priv(xhci, urb_priv);
1917 2164
1918 usb_hcd_unlink_urb_from_ep(xhci_to_hcd(xhci), urb); 2165 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
1919 xhci_dbg(xhci, "Giveback URB %p, len = %d, " 2166 if ((urb->actual_length != urb->transfer_buffer_length &&
1920 "status = %d\n", 2167 (urb->transfer_flags &
1921 urb, urb->actual_length, status); 2168 URB_SHORT_NOT_OK)) ||
2169 status != 0)
2170 xhci_dbg(xhci, "Giveback URB %p, len = %d, "
2171 "expected = %x, status = %d\n",
2172 urb, urb->actual_length,
2173 urb->transfer_buffer_length,
2174 status);
1922 spin_unlock(&xhci->lock); 2175 spin_unlock(&xhci->lock);
1923 usb_hcd_giveback_urb(xhci_to_hcd(xhci), urb, status); 2176 /* EHCI, UHCI, and OHCI always unconditionally set the
2177 * urb->status of an isochronous endpoint to 0.
2178 */
2179 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
2180 status = 0;
2181 usb_hcd_giveback_urb(bus_to_hcd(urb->dev->bus), urb, status);
1924 spin_lock(&xhci->lock); 2182 spin_lock(&xhci->lock);
1925 } 2183 }
1926 2184
@@ -1938,52 +2196,52 @@ cleanup:
1938/* 2196/*
1939 * This function handles all OS-owned events on the event ring. It may drop 2197 * This function handles all OS-owned events on the event ring. It may drop
1940 * xhci->lock between event processing (e.g. to pass up port status changes). 2198 * xhci->lock between event processing (e.g. to pass up port status changes).
2199 * Returns >0 for "possibly more events to process" (caller should call again),
2200 * otherwise 0 if done. In future, <0 returns should indicate error code.
1941 */ 2201 */
1942static void xhci_handle_event(struct xhci_hcd *xhci) 2202static int xhci_handle_event(struct xhci_hcd *xhci)
1943{ 2203{
1944 union xhci_trb *event; 2204 union xhci_trb *event;
1945 int update_ptrs = 1; 2205 int update_ptrs = 1;
1946 int ret; 2206 int ret;
1947 2207
1948 xhci_dbg(xhci, "In %s\n", __func__);
1949 if (!xhci->event_ring || !xhci->event_ring->dequeue) { 2208 if (!xhci->event_ring || !xhci->event_ring->dequeue) {
1950 xhci->error_bitmask |= 1 << 1; 2209 xhci->error_bitmask |= 1 << 1;
1951 return; 2210 return 0;
1952 } 2211 }
1953 2212
1954 event = xhci->event_ring->dequeue; 2213 event = xhci->event_ring->dequeue;
1955 /* Does the HC or OS own the TRB? */ 2214 /* Does the HC or OS own the TRB? */
1956 if ((event->event_cmd.flags & TRB_CYCLE) != 2215 if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
1957 xhci->event_ring->cycle_state) { 2216 xhci->event_ring->cycle_state) {
1958 xhci->error_bitmask |= 1 << 2; 2217 xhci->error_bitmask |= 1 << 2;
1959 return; 2218 return 0;
1960 } 2219 }
1961 xhci_dbg(xhci, "%s - OS owns TRB\n", __func__);
1962 2220
2221 /*
2222 * Barrier between reading the TRB_CYCLE (valid) flag above and any
2223 * speculative reads of the event's flags/data below.
2224 */
2225 rmb();
1963 /* FIXME: Handle more event types. */ 2226 /* FIXME: Handle more event types. */
1964 switch ((event->event_cmd.flags & TRB_TYPE_BITMASK)) { 2227 switch ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK)) {
1965 case TRB_TYPE(TRB_COMPLETION): 2228 case TRB_TYPE(TRB_COMPLETION):
1966 xhci_dbg(xhci, "%s - calling handle_cmd_completion\n", __func__);
1967 handle_cmd_completion(xhci, &event->event_cmd); 2229 handle_cmd_completion(xhci, &event->event_cmd);
1968 xhci_dbg(xhci, "%s - returned from handle_cmd_completion\n", __func__);
1969 break; 2230 break;
1970 case TRB_TYPE(TRB_PORT_STATUS): 2231 case TRB_TYPE(TRB_PORT_STATUS):
1971 xhci_dbg(xhci, "%s - calling handle_port_status\n", __func__);
1972 handle_port_status(xhci, event); 2232 handle_port_status(xhci, event);
1973 xhci_dbg(xhci, "%s - returned from handle_port_status\n", __func__);
1974 update_ptrs = 0; 2233 update_ptrs = 0;
1975 break; 2234 break;
1976 case TRB_TYPE(TRB_TRANSFER): 2235 case TRB_TYPE(TRB_TRANSFER):
1977 xhci_dbg(xhci, "%s - calling handle_tx_event\n", __func__);
1978 ret = handle_tx_event(xhci, &event->trans_event); 2236 ret = handle_tx_event(xhci, &event->trans_event);
1979 xhci_dbg(xhci, "%s - returned from handle_tx_event\n", __func__);
1980 if (ret < 0) 2237 if (ret < 0)
1981 xhci->error_bitmask |= 1 << 9; 2238 xhci->error_bitmask |= 1 << 9;
1982 else 2239 else
1983 update_ptrs = 0; 2240 update_ptrs = 0;
1984 break; 2241 break;
1985 default: 2242 default:
1986 if ((event->event_cmd.flags & TRB_TYPE_BITMASK) >= TRB_TYPE(48)) 2243 if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2244 TRB_TYPE(48))
1987 handle_vendor_event(xhci, event); 2245 handle_vendor_event(xhci, event);
1988 else 2246 else
1989 xhci->error_bitmask |= 1 << 3; 2247 xhci->error_bitmask |= 1 << 3;
@@ -1994,15 +2252,17 @@ static void xhci_handle_event(struct xhci_hcd *xhci)
1994 if (xhci->xhc_state & XHCI_STATE_DYING) { 2252 if (xhci->xhc_state & XHCI_STATE_DYING) {
1995 xhci_dbg(xhci, "xHCI host dying, returning from " 2253 xhci_dbg(xhci, "xHCI host dying, returning from "
1996 "event handler.\n"); 2254 "event handler.\n");
1997 return; 2255 return 0;
1998 } 2256 }
1999 2257
2000 if (update_ptrs) 2258 if (update_ptrs)
2001 /* Update SW event ring dequeue pointer */ 2259 /* Update SW event ring dequeue pointer */
2002 inc_deq(xhci, xhci->event_ring, true); 2260 inc_deq(xhci, xhci->event_ring, true);
2003 2261
2004 /* Are there more items on the event ring? */ 2262 /* Are there more items on the event ring? Caller will call us again to
2005 xhci_handle_event(xhci); 2263 * check.
2264 */
2265 return 1;
2006} 2266}
2007 2267
2008/* 2268/*
@@ -2028,24 +2288,12 @@ irqreturn_t xhci_irq(struct usb_hcd *hcd)
2028 2288
2029 if (!(status & STS_EINT)) { 2289 if (!(status & STS_EINT)) {
2030 spin_unlock(&xhci->lock); 2290 spin_unlock(&xhci->lock);
2031 xhci_warn(xhci, "Spurious interrupt.\n");
2032 return IRQ_NONE; 2291 return IRQ_NONE;
2033 } 2292 }
2034 xhci_dbg(xhci, "op reg status = %08x\n", status);
2035 xhci_dbg(xhci, "Event ring dequeue ptr:\n");
2036 xhci_dbg(xhci, "@%llx %08x %08x %08x %08x\n",
2037 (unsigned long long)
2038 xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, trb),
2039 lower_32_bits(trb->link.segment_ptr),
2040 upper_32_bits(trb->link.segment_ptr),
2041 (unsigned int) trb->link.intr_target,
2042 (unsigned int) trb->link.control);
2043
2044 if (status & STS_FATAL) { 2293 if (status & STS_FATAL) {
2045 xhci_warn(xhci, "WARNING: Host System Error\n"); 2294 xhci_warn(xhci, "WARNING: Host System Error\n");
2046 xhci_halt(xhci); 2295 xhci_halt(xhci);
2047hw_died: 2296hw_died:
2048 xhci_to_hcd(xhci)->state = HC_STATE_HALT;
2049 spin_unlock(&xhci->lock); 2297 spin_unlock(&xhci->lock);
2050 return -ESHUTDOWN; 2298 return -ESHUTDOWN;
2051 } 2299 }
@@ -2086,7 +2334,7 @@ hw_died:
2086 /* FIXME this should be a delayed service routine 2334 /* FIXME this should be a delayed service routine
2087 * that clears the EHB. 2335 * that clears the EHB.
2088 */ 2336 */
2089 xhci_handle_event(xhci); 2337 while (xhci_handle_event(xhci) > 0) {}
2090 2338
2091 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); 2339 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2092 /* If necessary, update the HW's version of the event ring deq ptr. */ 2340 /* If necessary, update the HW's version of the event ring deq ptr. */
@@ -2113,8 +2361,12 @@ hw_died:
2113irqreturn_t xhci_msi_irq(int irq, struct usb_hcd *hcd) 2361irqreturn_t xhci_msi_irq(int irq, struct usb_hcd *hcd)
2114{ 2362{
2115 irqreturn_t ret; 2363 irqreturn_t ret;
2364 struct xhci_hcd *xhci;
2116 2365
2366 xhci = hcd_to_xhci(hcd);
2117 set_bit(HCD_FLAG_SAW_IRQ, &hcd->flags); 2367 set_bit(HCD_FLAG_SAW_IRQ, &hcd->flags);
2368 if (xhci->shared_hcd)
2369 set_bit(HCD_FLAG_SAW_IRQ, &xhci->shared_hcd->flags);
2118 2370
2119 ret = xhci_irq(hcd); 2371 ret = xhci_irq(hcd);
2120 2372
@@ -2137,10 +2389,10 @@ static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
2137 struct xhci_generic_trb *trb; 2389 struct xhci_generic_trb *trb;
2138 2390
2139 trb = &ring->enqueue->generic; 2391 trb = &ring->enqueue->generic;
2140 trb->field[0] = field1; 2392 trb->field[0] = cpu_to_le32(field1);
2141 trb->field[1] = field2; 2393 trb->field[1] = cpu_to_le32(field2);
2142 trb->field[2] = field3; 2394 trb->field[2] = cpu_to_le32(field3);
2143 trb->field[3] = field4; 2395 trb->field[3] = cpu_to_le32(field4);
2144 inc_enq(xhci, ring, consumer, more_trbs_coming); 2396 inc_enq(xhci, ring, consumer, more_trbs_coming);
2145} 2397}
2146 2398
@@ -2152,7 +2404,6 @@ static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
2152 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags) 2404 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2153{ 2405{
2154 /* Make sure the endpoint has been added to xHC schedule */ 2406 /* Make sure the endpoint has been added to xHC schedule */
2155 xhci_dbg(xhci, "Endpoint state = 0x%x\n", ep_state);
2156 switch (ep_state) { 2407 switch (ep_state) {
2157 case EP_STATE_DISABLED: 2408 case EP_STATE_DISABLED:
2158 /* 2409 /*
@@ -2189,21 +2440,19 @@ static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
2189 struct xhci_ring *ring = ep_ring; 2440 struct xhci_ring *ring = ep_ring;
2190 union xhci_trb *next; 2441 union xhci_trb *next;
2191 2442
2192 xhci_dbg(xhci, "prepare_ring: pointing to link trb\n");
2193 next = ring->enqueue; 2443 next = ring->enqueue;
2194 2444
2195 while (last_trb(xhci, ring, ring->enq_seg, next)) { 2445 while (last_trb(xhci, ring, ring->enq_seg, next)) {
2196
2197 /* If we're not dealing with 0.95 hardware, 2446 /* If we're not dealing with 0.95 hardware,
2198 * clear the chain bit. 2447 * clear the chain bit.
2199 */ 2448 */
2200 if (!xhci_link_trb_quirk(xhci)) 2449 if (!xhci_link_trb_quirk(xhci))
2201 next->link.control &= ~TRB_CHAIN; 2450 next->link.control &= cpu_to_le32(~TRB_CHAIN);
2202 else 2451 else
2203 next->link.control |= TRB_CHAIN; 2452 next->link.control |= cpu_to_le32(TRB_CHAIN);
2204 2453
2205 wmb(); 2454 wmb();
2206 next->link.control ^= (u32) TRB_CYCLE; 2455 next->link.control ^= cpu_to_le32((u32) TRB_CYCLE);
2207 2456
2208 /* Toggle the cycle bit after the last ring segment. */ 2457 /* Toggle the cycle bit after the last ring segment. */
2209 if (last_trb_on_last_seg(xhci, ring, ring->enq_seg, next)) { 2458 if (last_trb_on_last_seg(xhci, ring, ring->enq_seg, next)) {
@@ -2246,8 +2495,8 @@ static int prepare_transfer(struct xhci_hcd *xhci,
2246 } 2495 }
2247 2496
2248 ret = prepare_ring(xhci, ep_ring, 2497 ret = prepare_ring(xhci, ep_ring,
2249 ep_ctx->ep_info & EP_STATE_MASK, 2498 le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
2250 num_trbs, mem_flags); 2499 num_trbs, mem_flags);
2251 if (ret) 2500 if (ret)
2252 return ret; 2501 return ret;
2253 2502
@@ -2258,7 +2507,7 @@ static int prepare_transfer(struct xhci_hcd *xhci,
2258 INIT_LIST_HEAD(&td->cancelled_td_list); 2507 INIT_LIST_HEAD(&td->cancelled_td_list);
2259 2508
2260 if (td_index == 0) { 2509 if (td_index == 0) {
2261 ret = usb_hcd_link_urb_to_ep(xhci_to_hcd(xhci), urb); 2510 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);
2262 if (unlikely(ret)) { 2511 if (unlikely(ret)) {
2263 xhci_urb_free_priv(xhci, urb_priv); 2512 xhci_urb_free_priv(xhci, urb_priv);
2264 urb->hcpriv = NULL; 2513 urb->hcpriv = NULL;
@@ -2294,12 +2543,13 @@ static unsigned int count_sg_trbs_needed(struct xhci_hcd *xhci, struct urb *urb)
2294 2543
2295 /* Scatter gather list entries may cross 64KB boundaries */ 2544 /* Scatter gather list entries may cross 64KB boundaries */
2296 running_total = TRB_MAX_BUFF_SIZE - 2545 running_total = TRB_MAX_BUFF_SIZE -
2297 (sg_dma_address(sg) & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); 2546 (sg_dma_address(sg) & (TRB_MAX_BUFF_SIZE - 1));
2547 running_total &= TRB_MAX_BUFF_SIZE - 1;
2298 if (running_total != 0) 2548 if (running_total != 0)
2299 num_trbs++; 2549 num_trbs++;
2300 2550
2301 /* How many more 64KB chunks to transfer, how many more TRBs? */ 2551 /* How many more 64KB chunks to transfer, how many more TRBs? */
2302 while (running_total < sg_dma_len(sg)) { 2552 while (running_total < sg_dma_len(sg) && running_total < temp) {
2303 num_trbs++; 2553 num_trbs++;
2304 running_total += TRB_MAX_BUFF_SIZE; 2554 running_total += TRB_MAX_BUFF_SIZE;
2305 } 2555 }
@@ -2314,7 +2564,8 @@ static unsigned int count_sg_trbs_needed(struct xhci_hcd *xhci, struct urb *urb)
2314 } 2564 }
2315 xhci_dbg(xhci, "\n"); 2565 xhci_dbg(xhci, "\n");
2316 if (!in_interrupt()) 2566 if (!in_interrupt())
2317 dev_dbg(&urb->dev->dev, "ep %#x - urb len = %d, sglist used, num_trbs = %d\n", 2567 xhci_dbg(xhci, "ep %#x - urb len = %d, sglist used, "
2568 "num_trbs = %d\n",
2318 urb->ep->desc.bEndpointAddress, 2569 urb->ep->desc.bEndpointAddress,
2319 urb->transfer_buffer_length, 2570 urb->transfer_buffer_length,
2320 num_trbs); 2571 num_trbs);
@@ -2324,11 +2575,11 @@ static unsigned int count_sg_trbs_needed(struct xhci_hcd *xhci, struct urb *urb)
2324static void check_trb_math(struct urb *urb, int num_trbs, int running_total) 2575static void check_trb_math(struct urb *urb, int num_trbs, int running_total)
2325{ 2576{
2326 if (num_trbs != 0) 2577 if (num_trbs != 0)
2327 dev_dbg(&urb->dev->dev, "%s - ep %#x - Miscalculated number of " 2578 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated number of "
2328 "TRBs, %d left\n", __func__, 2579 "TRBs, %d left\n", __func__,
2329 urb->ep->desc.bEndpointAddress, num_trbs); 2580 urb->ep->desc.bEndpointAddress, num_trbs);
2330 if (running_total != urb->transfer_buffer_length) 2581 if (running_total != urb->transfer_buffer_length)
2331 dev_dbg(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, " 2582 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "
2332 "queued %#x (%d), asked for %#x (%d)\n", 2583 "queued %#x (%d), asked for %#x (%d)\n",
2333 __func__, 2584 __func__,
2334 urb->ep->desc.bEndpointAddress, 2585 urb->ep->desc.bEndpointAddress,
@@ -2339,15 +2590,18 @@ static void check_trb_math(struct urb *urb, int num_trbs, int running_total)
2339 2590
2340static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, 2591static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
2341 unsigned int ep_index, unsigned int stream_id, int start_cycle, 2592 unsigned int ep_index, unsigned int stream_id, int start_cycle,
2342 struct xhci_generic_trb *start_trb, struct xhci_td *td) 2593 struct xhci_generic_trb *start_trb)
2343{ 2594{
2344 /* 2595 /*
2345 * Pass all the TRBs to the hardware at once and make sure this write 2596 * Pass all the TRBs to the hardware at once and make sure this write
2346 * isn't reordered. 2597 * isn't reordered.
2347 */ 2598 */
2348 wmb(); 2599 wmb();
2349 start_trb->field[3] |= start_cycle; 2600 if (start_cycle)
2350 ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); 2601 start_trb->field[3] |= cpu_to_le32(start_cycle);
2602 else
2603 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
2604 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
2351} 2605}
2352 2606
2353/* 2607/*
@@ -2364,7 +2618,7 @@ int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2364 int xhci_interval; 2618 int xhci_interval;
2365 int ep_interval; 2619 int ep_interval;
2366 2620
2367 xhci_interval = EP_INTERVAL_TO_UFRAMES(ep_ctx->ep_info); 2621 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
2368 ep_interval = urb->interval; 2622 ep_interval = urb->interval;
2369 /* Convert to microframes */ 2623 /* Convert to microframes */
2370 if (urb->dev->speed == USB_SPEED_LOW || 2624 if (urb->dev->speed == USB_SPEED_LOW ||
@@ -2374,7 +2628,7 @@ int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2374 * to set the polling interval (once the API is added). 2628 * to set the polling interval (once the API is added).
2375 */ 2629 */
2376 if (xhci_interval != ep_interval) { 2630 if (xhci_interval != ep_interval) {
2377 if (!printk_ratelimit()) 2631 if (printk_ratelimit())
2378 dev_dbg(&urb->dev->dev, "Driver uses different interval" 2632 dev_dbg(&urb->dev->dev, "Driver uses different interval"
2379 " (%d microframe%s) than xHCI " 2633 " (%d microframe%s) than xHCI "
2380 "(%d microframe%s)\n", 2634 "(%d microframe%s)\n",
@@ -2406,6 +2660,35 @@ static u32 xhci_td_remainder(unsigned int remainder)
2406 return (remainder >> 10) << 17; 2660 return (remainder >> 10) << 17;
2407} 2661}
2408 2662
2663/*
2664 * For xHCI 1.0 host controllers, TD size is the number of packets remaining in
2665 * the TD (*not* including this TRB).
2666 *
2667 * Total TD packet count = total_packet_count =
2668 * roundup(TD size in bytes / wMaxPacketSize)
2669 *
2670 * Packets transferred up to and including this TRB = packets_transferred =
2671 * rounddown(total bytes transferred including this TRB / wMaxPacketSize)
2672 *
2673 * TD size = total_packet_count - packets_transferred
2674 *
2675 * It must fit in bits 21:17, so it can't be bigger than 31.
2676 */
2677
2678static u32 xhci_v1_0_td_remainder(int running_total, int trb_buff_len,
2679 unsigned int total_packet_count, struct urb *urb)
2680{
2681 int packets_transferred;
2682
2683 /* All the TRB queueing functions don't count the current TRB in
2684 * running_total.
2685 */
2686 packets_transferred = (running_total + trb_buff_len) /
2687 le16_to_cpu(urb->ep->desc.wMaxPacketSize);
2688
2689 return xhci_td_remainder(total_packet_count - packets_transferred);
2690}
2691
2409static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags, 2692static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2410 struct urb *urb, int slot_id, unsigned int ep_index) 2693 struct urb *urb, int slot_id, unsigned int ep_index)
2411{ 2694{
@@ -2416,6 +2699,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2416 struct scatterlist *sg; 2699 struct scatterlist *sg;
2417 int num_sgs; 2700 int num_sgs;
2418 int trb_buff_len, this_sg_len, running_total; 2701 int trb_buff_len, this_sg_len, running_total;
2702 unsigned int total_packet_count;
2419 bool first_trb; 2703 bool first_trb;
2420 u64 addr; 2704 u64 addr;
2421 bool more_trbs_coming; 2705 bool more_trbs_coming;
@@ -2429,6 +2713,8 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2429 2713
2430 num_trbs = count_sg_trbs_needed(xhci, urb); 2714 num_trbs = count_sg_trbs_needed(xhci, urb);
2431 num_sgs = urb->num_sgs; 2715 num_sgs = urb->num_sgs;
2716 total_packet_count = roundup(urb->transfer_buffer_length,
2717 le16_to_cpu(urb->ep->desc.wMaxPacketSize));
2432 2718
2433 trb_buff_len = prepare_transfer(xhci, xhci->devs[slot_id], 2719 trb_buff_len = prepare_transfer(xhci, xhci->devs[slot_id],
2434 ep_index, urb->stream_id, 2720 ep_index, urb->stream_id,
@@ -2460,8 +2746,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2460 sg = urb->sg; 2746 sg = urb->sg;
2461 addr = (u64) sg_dma_address(sg); 2747 addr = (u64) sg_dma_address(sg);
2462 this_sg_len = sg_dma_len(sg); 2748 this_sg_len = sg_dma_len(sg);
2463 trb_buff_len = TRB_MAX_BUFF_SIZE - 2749 trb_buff_len = TRB_MAX_BUFF_SIZE - (addr & (TRB_MAX_BUFF_SIZE - 1));
2464 (addr & ((1 << TRB_MAX_BUFF_SHIFT) - 1));
2465 trb_buff_len = min_t(int, trb_buff_len, this_sg_len); 2750 trb_buff_len = min_t(int, trb_buff_len, this_sg_len);
2466 if (trb_buff_len > urb->transfer_buffer_length) 2751 if (trb_buff_len > urb->transfer_buffer_length)
2467 trb_buff_len = urb->transfer_buffer_length; 2752 trb_buff_len = urb->transfer_buffer_length;
@@ -2476,9 +2761,11 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2476 u32 remainder = 0; 2761 u32 remainder = 0;
2477 2762
2478 /* Don't change the cycle bit of the first TRB until later */ 2763 /* Don't change the cycle bit of the first TRB until later */
2479 if (first_trb) 2764 if (first_trb) {
2480 first_trb = false; 2765 first_trb = false;
2481 else 2766 if (start_cycle == 0)
2767 field |= 0x1;
2768 } else
2482 field |= ep_ring->cycle_state; 2769 field |= ep_ring->cycle_state;
2483 2770
2484 /* Chain all the TRBs together; clear the chain bit in the last 2771 /* Chain all the TRBs together; clear the chain bit in the last
@@ -2491,23 +2778,37 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2491 td->last_trb = ep_ring->enqueue; 2778 td->last_trb = ep_ring->enqueue;
2492 field |= TRB_IOC; 2779 field |= TRB_IOC;
2493 } 2780 }
2781
2782 /* Only set interrupt on short packet for IN endpoints */
2783 if (usb_urb_dir_in(urb))
2784 field |= TRB_ISP;
2785
2494 xhci_dbg(xhci, " sg entry: dma = %#x, len = %#x (%d), " 2786 xhci_dbg(xhci, " sg entry: dma = %#x, len = %#x (%d), "
2495 "64KB boundary at %#x, end dma = %#x\n", 2787 "64KB boundary at %#x, end dma = %#x\n",
2496 (unsigned int) addr, trb_buff_len, trb_buff_len, 2788 (unsigned int) addr, trb_buff_len, trb_buff_len,
2497 (unsigned int) (addr + TRB_MAX_BUFF_SIZE) & ~(TRB_MAX_BUFF_SIZE - 1), 2789 (unsigned int) (addr + TRB_MAX_BUFF_SIZE) & ~(TRB_MAX_BUFF_SIZE - 1),
2498 (unsigned int) addr + trb_buff_len); 2790 (unsigned int) addr + trb_buff_len);
2499 if (TRB_MAX_BUFF_SIZE - 2791 if (TRB_MAX_BUFF_SIZE -
2500 (addr & ((1 << TRB_MAX_BUFF_SHIFT) - 1)) < trb_buff_len) { 2792 (addr & (TRB_MAX_BUFF_SIZE - 1)) < trb_buff_len) {
2501 xhci_warn(xhci, "WARN: sg dma xfer crosses 64KB boundaries!\n"); 2793 xhci_warn(xhci, "WARN: sg dma xfer crosses 64KB boundaries!\n");
2502 xhci_dbg(xhci, "Next boundary at %#x, end dma = %#x\n", 2794 xhci_dbg(xhci, "Next boundary at %#x, end dma = %#x\n",
2503 (unsigned int) (addr + TRB_MAX_BUFF_SIZE) & ~(TRB_MAX_BUFF_SIZE - 1), 2795 (unsigned int) (addr + TRB_MAX_BUFF_SIZE) & ~(TRB_MAX_BUFF_SIZE - 1),
2504 (unsigned int) addr + trb_buff_len); 2796 (unsigned int) addr + trb_buff_len);
2505 } 2797 }
2506 remainder = xhci_td_remainder(urb->transfer_buffer_length - 2798
2507 running_total) ; 2799 /* Set the TRB length, TD size, and interrupter fields. */
2800 if (xhci->hci_version < 0x100) {
2801 remainder = xhci_td_remainder(
2802 urb->transfer_buffer_length -
2803 running_total);
2804 } else {
2805 remainder = xhci_v1_0_td_remainder(running_total,
2806 trb_buff_len, total_packet_count, urb);
2807 }
2508 length_field = TRB_LEN(trb_buff_len) | 2808 length_field = TRB_LEN(trb_buff_len) |
2509 remainder | 2809 remainder |
2510 TRB_INTR_TARGET(0); 2810 TRB_INTR_TARGET(0);
2811
2511 if (num_trbs > 1) 2812 if (num_trbs > 1)
2512 more_trbs_coming = true; 2813 more_trbs_coming = true;
2513 else 2814 else
@@ -2516,12 +2817,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2516 lower_32_bits(addr), 2817 lower_32_bits(addr),
2517 upper_32_bits(addr), 2818 upper_32_bits(addr),
2518 length_field, 2819 length_field,
2519 /* We always want to know if the TRB was short, 2820 field | TRB_TYPE(TRB_NORMAL));
2520 * or we won't get an event when it completes.
2521 * (Unless we use event data TRBs, which are a
2522 * waste of space and HC resources.)
2523 */
2524 field | TRB_ISP | TRB_TYPE(TRB_NORMAL));
2525 --num_trbs; 2821 --num_trbs;
2526 running_total += trb_buff_len; 2822 running_total += trb_buff_len;
2527 2823
@@ -2541,7 +2837,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2541 } 2837 }
2542 2838
2543 trb_buff_len = TRB_MAX_BUFF_SIZE - 2839 trb_buff_len = TRB_MAX_BUFF_SIZE -
2544 (addr & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); 2840 (addr & (TRB_MAX_BUFF_SIZE - 1));
2545 trb_buff_len = min_t(int, trb_buff_len, this_sg_len); 2841 trb_buff_len = min_t(int, trb_buff_len, this_sg_len);
2546 if (running_total + trb_buff_len > urb->transfer_buffer_length) 2842 if (running_total + trb_buff_len > urb->transfer_buffer_length)
2547 trb_buff_len = 2843 trb_buff_len =
@@ -2550,7 +2846,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2550 2846
2551 check_trb_math(urb, num_trbs, running_total); 2847 check_trb_math(urb, num_trbs, running_total);
2552 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, 2848 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
2553 start_cycle, start_trb, td); 2849 start_cycle, start_trb);
2554 return 0; 2850 return 0;
2555} 2851}
2556 2852
@@ -2569,6 +2865,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2569 u32 field, length_field; 2865 u32 field, length_field;
2570 2866
2571 int running_total, trb_buff_len, ret; 2867 int running_total, trb_buff_len, ret;
2868 unsigned int total_packet_count;
2572 u64 addr; 2869 u64 addr;
2573 2870
2574 if (urb->num_sgs) 2871 if (urb->num_sgs)
@@ -2581,7 +2878,8 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2581 num_trbs = 0; 2878 num_trbs = 0;
2582 /* How much data is (potentially) left before the 64KB boundary? */ 2879 /* How much data is (potentially) left before the 64KB boundary? */
2583 running_total = TRB_MAX_BUFF_SIZE - 2880 running_total = TRB_MAX_BUFF_SIZE -
2584 (urb->transfer_dma & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); 2881 (urb->transfer_dma & (TRB_MAX_BUFF_SIZE - 1));
2882 running_total &= TRB_MAX_BUFF_SIZE - 1;
2585 2883
2586 /* If there's some data on this 64KB chunk, or we have to send a 2884 /* If there's some data on this 64KB chunk, or we have to send a
2587 * zero-length transfer, we need at least one TRB 2885 * zero-length transfer, we need at least one TRB
@@ -2596,7 +2894,8 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2596 /* FIXME: this doesn't deal with URB_ZERO_PACKET - need one more */ 2894 /* FIXME: this doesn't deal with URB_ZERO_PACKET - need one more */
2597 2895
2598 if (!in_interrupt()) 2896 if (!in_interrupt())
2599 dev_dbg(&urb->dev->dev, "ep %#x - urb len = %#x (%d), addr = %#llx, num_trbs = %d\n", 2897 xhci_dbg(xhci, "ep %#x - urb len = %#x (%d), "
2898 "addr = %#llx, num_trbs = %d\n",
2600 urb->ep->desc.bEndpointAddress, 2899 urb->ep->desc.bEndpointAddress,
2601 urb->transfer_buffer_length, 2900 urb->transfer_buffer_length,
2602 urb->transfer_buffer_length, 2901 urb->transfer_buffer_length,
@@ -2621,11 +2920,13 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2621 start_cycle = ep_ring->cycle_state; 2920 start_cycle = ep_ring->cycle_state;
2622 2921
2623 running_total = 0; 2922 running_total = 0;
2923 total_packet_count = roundup(urb->transfer_buffer_length,
2924 le16_to_cpu(urb->ep->desc.wMaxPacketSize));
2624 /* How much data is in the first TRB? */ 2925 /* How much data is in the first TRB? */
2625 addr = (u64) urb->transfer_dma; 2926 addr = (u64) urb->transfer_dma;
2626 trb_buff_len = TRB_MAX_BUFF_SIZE - 2927 trb_buff_len = TRB_MAX_BUFF_SIZE -
2627 (urb->transfer_dma & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); 2928 (urb->transfer_dma & (TRB_MAX_BUFF_SIZE - 1));
2628 if (urb->transfer_buffer_length < trb_buff_len) 2929 if (trb_buff_len > urb->transfer_buffer_length)
2629 trb_buff_len = urb->transfer_buffer_length; 2930 trb_buff_len = urb->transfer_buffer_length;
2630 2931
2631 first_trb = true; 2932 first_trb = true;
@@ -2636,9 +2937,11 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2636 field = 0; 2937 field = 0;
2637 2938
2638 /* Don't change the cycle bit of the first TRB until later */ 2939 /* Don't change the cycle bit of the first TRB until later */
2639 if (first_trb) 2940 if (first_trb) {
2640 first_trb = false; 2941 first_trb = false;
2641 else 2942 if (start_cycle == 0)
2943 field |= 0x1;
2944 } else
2642 field |= ep_ring->cycle_state; 2945 field |= ep_ring->cycle_state;
2643 2946
2644 /* Chain all the TRBs together; clear the chain bit in the last 2947 /* Chain all the TRBs together; clear the chain bit in the last
@@ -2651,11 +2954,24 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2651 td->last_trb = ep_ring->enqueue; 2954 td->last_trb = ep_ring->enqueue;
2652 field |= TRB_IOC; 2955 field |= TRB_IOC;
2653 } 2956 }
2654 remainder = xhci_td_remainder(urb->transfer_buffer_length - 2957
2655 running_total); 2958 /* Only set interrupt on short packet for IN endpoints */
2959 if (usb_urb_dir_in(urb))
2960 field |= TRB_ISP;
2961
2962 /* Set the TRB length, TD size, and interrupter fields. */
2963 if (xhci->hci_version < 0x100) {
2964 remainder = xhci_td_remainder(
2965 urb->transfer_buffer_length -
2966 running_total);
2967 } else {
2968 remainder = xhci_v1_0_td_remainder(running_total,
2969 trb_buff_len, total_packet_count, urb);
2970 }
2656 length_field = TRB_LEN(trb_buff_len) | 2971 length_field = TRB_LEN(trb_buff_len) |
2657 remainder | 2972 remainder |
2658 TRB_INTR_TARGET(0); 2973 TRB_INTR_TARGET(0);
2974
2659 if (num_trbs > 1) 2975 if (num_trbs > 1)
2660 more_trbs_coming = true; 2976 more_trbs_coming = true;
2661 else 2977 else
@@ -2664,12 +2980,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2664 lower_32_bits(addr), 2980 lower_32_bits(addr),
2665 upper_32_bits(addr), 2981 upper_32_bits(addr),
2666 length_field, 2982 length_field,
2667 /* We always want to know if the TRB was short, 2983 field | TRB_TYPE(TRB_NORMAL));
2668 * or we won't get an event when it completes.
2669 * (Unless we use event data TRBs, which are a
2670 * waste of space and HC resources.)
2671 */
2672 field | TRB_ISP | TRB_TYPE(TRB_NORMAL));
2673 --num_trbs; 2984 --num_trbs;
2674 running_total += trb_buff_len; 2985 running_total += trb_buff_len;
2675 2986
@@ -2682,7 +2993,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2682 2993
2683 check_trb_math(urb, num_trbs, running_total); 2994 check_trb_math(urb, num_trbs, running_total);
2684 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, 2995 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
2685 start_cycle, start_trb, td); 2996 start_cycle, start_trb);
2686 return 0; 2997 return 0;
2687} 2998}
2688 2999
@@ -2743,16 +3054,35 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2743 /* Queue setup TRB - see section 6.4.1.2.1 */ 3054 /* Queue setup TRB - see section 6.4.1.2.1 */
2744 /* FIXME better way to translate setup_packet into two u32 fields? */ 3055 /* FIXME better way to translate setup_packet into two u32 fields? */
2745 setup = (struct usb_ctrlrequest *) urb->setup_packet; 3056 setup = (struct usb_ctrlrequest *) urb->setup_packet;
3057 field = 0;
3058 field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
3059 if (start_cycle == 0)
3060 field |= 0x1;
3061
3062 /* xHCI 1.0 6.4.1.2.1: Transfer Type field */
3063 if (xhci->hci_version == 0x100) {
3064 if (urb->transfer_buffer_length > 0) {
3065 if (setup->bRequestType & USB_DIR_IN)
3066 field |= TRB_TX_TYPE(TRB_DATA_IN);
3067 else
3068 field |= TRB_TX_TYPE(TRB_DATA_OUT);
3069 }
3070 }
3071
2746 queue_trb(xhci, ep_ring, false, true, 3072 queue_trb(xhci, ep_ring, false, true,
2747 /* FIXME endianness is probably going to bite my ass here. */ 3073 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
2748 setup->bRequestType | setup->bRequest << 8 | setup->wValue << 16, 3074 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
2749 setup->wIndex | setup->wLength << 16, 3075 TRB_LEN(8) | TRB_INTR_TARGET(0),
2750 TRB_LEN(8) | TRB_INTR_TARGET(0), 3076 /* Immediate data in pointer */
2751 /* Immediate data in pointer */ 3077 field);
2752 TRB_IDT | TRB_TYPE(TRB_SETUP));
2753 3078
2754 /* If there's data, queue data TRBs */ 3079 /* If there's data, queue data TRBs */
2755 field = 0; 3080 /* Only set interrupt on short packet for IN endpoints */
3081 if (usb_urb_dir_in(urb))
3082 field = TRB_ISP | TRB_TYPE(TRB_DATA);
3083 else
3084 field = TRB_TYPE(TRB_DATA);
3085
2756 length_field = TRB_LEN(urb->transfer_buffer_length) | 3086 length_field = TRB_LEN(urb->transfer_buffer_length) |
2757 xhci_td_remainder(urb->transfer_buffer_length) | 3087 xhci_td_remainder(urb->transfer_buffer_length) |
2758 TRB_INTR_TARGET(0); 3088 TRB_INTR_TARGET(0);
@@ -2763,8 +3093,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2763 lower_32_bits(urb->transfer_dma), 3093 lower_32_bits(urb->transfer_dma),
2764 upper_32_bits(urb->transfer_dma), 3094 upper_32_bits(urb->transfer_dma),
2765 length_field, 3095 length_field,
2766 /* Event on short tx */ 3096 field | ep_ring->cycle_state);
2767 field | TRB_ISP | TRB_TYPE(TRB_DATA) | ep_ring->cycle_state);
2768 } 3097 }
2769 3098
2770 /* Save the DMA address of the last TRB in the TD */ 3099 /* Save the DMA address of the last TRB in the TD */
@@ -2784,7 +3113,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2784 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state); 3113 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);
2785 3114
2786 giveback_first_trb(xhci, slot_id, ep_index, 0, 3115 giveback_first_trb(xhci, slot_id, ep_index, 0,
2787 start_cycle, start_trb, td); 3116 start_cycle, start_trb);
2788 return 0; 3117 return 0;
2789} 3118}
2790 3119
@@ -2797,8 +3126,8 @@ static int count_isoc_trbs_needed(struct xhci_hcd *xhci,
2797 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); 3126 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
2798 td_len = urb->iso_frame_desc[i].length; 3127 td_len = urb->iso_frame_desc[i].length;
2799 3128
2800 running_total = TRB_MAX_BUFF_SIZE - 3129 running_total = TRB_MAX_BUFF_SIZE - (addr & (TRB_MAX_BUFF_SIZE - 1));
2801 (addr & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); 3130 running_total &= TRB_MAX_BUFF_SIZE - 1;
2802 if (running_total != 0) 3131 if (running_total != 0)
2803 num_trbs++; 3132 num_trbs++;
2804 3133
@@ -2810,6 +3139,63 @@ static int count_isoc_trbs_needed(struct xhci_hcd *xhci,
2810 return num_trbs; 3139 return num_trbs;
2811} 3140}
2812 3141
3142/*
3143 * The transfer burst count field of the isochronous TRB defines the number of
3144 * bursts that are required to move all packets in this TD. Only SuperSpeed
3145 * devices can burst up to bMaxBurst number of packets per service interval.
3146 * This field is zero based, meaning a value of zero in the field means one
3147 * burst. Basically, for everything but SuperSpeed devices, this field will be
3148 * zero. Only xHCI 1.0 host controllers support this field.
3149 */
3150static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3151 struct usb_device *udev,
3152 struct urb *urb, unsigned int total_packet_count)
3153{
3154 unsigned int max_burst;
3155
3156 if (xhci->hci_version < 0x100 || udev->speed != USB_SPEED_SUPER)
3157 return 0;
3158
3159 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3160 return roundup(total_packet_count, max_burst + 1) - 1;
3161}
3162
3163/*
3164 * Returns the number of packets in the last "burst" of packets. This field is
3165 * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so
3166 * the last burst packet count is equal to the total number of packets in the
3167 * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst
3168 * must contain (bMaxBurst + 1) number of packets, but the last burst can
3169 * contain 1 to (bMaxBurst + 1) packets.
3170 */
3171static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3172 struct usb_device *udev,
3173 struct urb *urb, unsigned int total_packet_count)
3174{
3175 unsigned int max_burst;
3176 unsigned int residue;
3177
3178 if (xhci->hci_version < 0x100)
3179 return 0;
3180
3181 switch (udev->speed) {
3182 case USB_SPEED_SUPER:
3183 /* bMaxBurst is zero based: 0 means 1 packet per burst */
3184 max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3185 residue = total_packet_count % (max_burst + 1);
3186 /* If residue is zero, the last burst contains (max_burst + 1)
3187 * number of packets, but the TLBPC field is zero-based.
3188 */
3189 if (residue == 0)
3190 return max_burst;
3191 return residue - 1;
3192 default:
3193 if (total_packet_count == 0)
3194 return 0;
3195 return total_packet_count - 1;
3196 }
3197}
3198
2813/* This is for isoc transfer */ 3199/* This is for isoc transfer */
2814static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, 3200static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2815 struct urb *urb, int slot_id, unsigned int ep_index) 3201 struct urb *urb, int slot_id, unsigned int ep_index)
@@ -2825,6 +3211,7 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2825 int running_total, trb_buff_len, td_len, td_remain_len, ret; 3211 int running_total, trb_buff_len, td_len, td_remain_len, ret;
2826 u64 start_addr, addr; 3212 u64 start_addr, addr;
2827 int i, j; 3213 int i, j;
3214 bool more_trbs_coming;
2828 3215
2829 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; 3216 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
2830 3217
@@ -2835,7 +3222,7 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2835 } 3222 }
2836 3223
2837 if (!in_interrupt()) 3224 if (!in_interrupt())
2838 dev_dbg(&urb->dev->dev, "ep %#x - urb len = %#x (%d)," 3225 xhci_dbg(xhci, "ep %#x - urb len = %#x (%d),"
2839 " addr = %#llx, num_tds = %d\n", 3226 " addr = %#llx, num_tds = %d\n",
2840 urb->ep->desc.bEndpointAddress, 3227 urb->ep->desc.bEndpointAddress,
2841 urb->transfer_buffer_length, 3228 urb->transfer_buffer_length,
@@ -2849,12 +3236,22 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2849 3236
2850 /* Queue the first TRB, even if it's zero-length */ 3237 /* Queue the first TRB, even if it's zero-length */
2851 for (i = 0; i < num_tds; i++) { 3238 for (i = 0; i < num_tds; i++) {
2852 first_trb = true; 3239 unsigned int total_packet_count;
3240 unsigned int burst_count;
3241 unsigned int residue;
2853 3242
3243 first_trb = true;
2854 running_total = 0; 3244 running_total = 0;
2855 addr = start_addr + urb->iso_frame_desc[i].offset; 3245 addr = start_addr + urb->iso_frame_desc[i].offset;
2856 td_len = urb->iso_frame_desc[i].length; 3246 td_len = urb->iso_frame_desc[i].length;
2857 td_remain_len = td_len; 3247 td_remain_len = td_len;
3248 /* FIXME: Ignoring zero-length packets, can those happen? */
3249 total_packet_count = roundup(td_len,
3250 le16_to_cpu(urb->ep->desc.wMaxPacketSize));
3251 burst_count = xhci_get_burst_count(xhci, urb->dev, urb,
3252 total_packet_count);
3253 residue = xhci_get_last_burst_packet_count(xhci,
3254 urb->dev, urb, total_packet_count);
2858 3255
2859 trbs_per_td = count_isoc_trbs_needed(xhci, urb, i); 3256 trbs_per_td = count_isoc_trbs_needed(xhci, urb, i);
2860 3257
@@ -2868,14 +3265,17 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2868 3265
2869 for (j = 0; j < trbs_per_td; j++) { 3266 for (j = 0; j < trbs_per_td; j++) {
2870 u32 remainder = 0; 3267 u32 remainder = 0;
2871 field = 0; 3268 field = TRB_TBC(burst_count) | TRB_TLBPC(residue);
2872 3269
2873 if (first_trb) { 3270 if (first_trb) {
2874 /* Queue the isoc TRB */ 3271 /* Queue the isoc TRB */
2875 field |= TRB_TYPE(TRB_ISOC); 3272 field |= TRB_TYPE(TRB_ISOC);
2876 /* Assume URB_ISO_ASAP is set */ 3273 /* Assume URB_ISO_ASAP is set */
2877 field |= TRB_SIA; 3274 field |= TRB_SIA;
2878 if (i > 0) 3275 if (i == 0) {
3276 if (start_cycle == 0)
3277 field |= 0x1;
3278 } else
2879 field |= ep_ring->cycle_state; 3279 field |= ep_ring->cycle_state;
2880 first_trb = false; 3280 first_trb = false;
2881 } else { 3281 } else {
@@ -2884,15 +3284,26 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2884 field |= ep_ring->cycle_state; 3284 field |= ep_ring->cycle_state;
2885 } 3285 }
2886 3286
3287 /* Only set interrupt on short packet for IN EPs */
3288 if (usb_urb_dir_in(urb))
3289 field |= TRB_ISP;
3290
2887 /* Chain all the TRBs together; clear the chain bit in 3291 /* Chain all the TRBs together; clear the chain bit in
2888 * the last TRB to indicate it's the last TRB in the 3292 * the last TRB to indicate it's the last TRB in the
2889 * chain. 3293 * chain.
2890 */ 3294 */
2891 if (j < trbs_per_td - 1) { 3295 if (j < trbs_per_td - 1) {
2892 field |= TRB_CHAIN; 3296 field |= TRB_CHAIN;
3297 more_trbs_coming = true;
2893 } else { 3298 } else {
2894 td->last_trb = ep_ring->enqueue; 3299 td->last_trb = ep_ring->enqueue;
2895 field |= TRB_IOC; 3300 field |= TRB_IOC;
3301 if (xhci->hci_version == 0x100) {
3302 /* Set BEI bit except for the last td */
3303 if (i < num_tds - 1)
3304 field |= TRB_BEI;
3305 }
3306 more_trbs_coming = false;
2896 } 3307 }
2897 3308
2898 /* Calculate TRB length */ 3309 /* Calculate TRB length */
@@ -2901,20 +3312,24 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2901 if (trb_buff_len > td_remain_len) 3312 if (trb_buff_len > td_remain_len)
2902 trb_buff_len = td_remain_len; 3313 trb_buff_len = td_remain_len;
2903 3314
2904 remainder = xhci_td_remainder(td_len - running_total); 3315 /* Set the TRB length, TD size, & interrupter fields. */
3316 if (xhci->hci_version < 0x100) {
3317 remainder = xhci_td_remainder(
3318 td_len - running_total);
3319 } else {
3320 remainder = xhci_v1_0_td_remainder(
3321 running_total, trb_buff_len,
3322 total_packet_count, urb);
3323 }
2905 length_field = TRB_LEN(trb_buff_len) | 3324 length_field = TRB_LEN(trb_buff_len) |
2906 remainder | 3325 remainder |
2907 TRB_INTR_TARGET(0); 3326 TRB_INTR_TARGET(0);
2908 queue_trb(xhci, ep_ring, false, false, 3327
3328 queue_trb(xhci, ep_ring, false, more_trbs_coming,
2909 lower_32_bits(addr), 3329 lower_32_bits(addr),
2910 upper_32_bits(addr), 3330 upper_32_bits(addr),
2911 length_field, 3331 length_field,
2912 /* We always want to know if the TRB was short, 3332 field);
2913 * or we won't get an event when it completes.
2914 * (Unless we use event data TRBs, which are a
2915 * waste of space and HC resources.)
2916 */
2917 field | TRB_ISP);
2918 running_total += trb_buff_len; 3333 running_total += trb_buff_len;
2919 3334
2920 addr += trb_buff_len; 3335 addr += trb_buff_len;
@@ -2928,10 +3343,14 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
2928 } 3343 }
2929 } 3344 }
2930 3345
2931 wmb(); 3346 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
2932 start_trb->field[3] |= start_cycle; 3347 if (xhci->quirks & XHCI_AMD_PLL_FIX)
3348 usb_amd_quirk_pll_disable();
3349 }
3350 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
2933 3351
2934 ring_ep_doorbell(xhci, slot_id, ep_index, urb->stream_id); 3352 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3353 start_cycle, start_trb);
2935 return 0; 3354 return 0;
2936} 3355}
2937 3356
@@ -2966,8 +3385,8 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
2966 /* Check the ring to guarantee there is enough room for the whole urb. 3385 /* Check the ring to guarantee there is enough room for the whole urb.
2967 * Do not insert any td of the urb to the ring if the check failed. 3386 * Do not insert any td of the urb to the ring if the check failed.
2968 */ 3387 */
2969 ret = prepare_ring(xhci, ep_ring, ep_ctx->ep_info & EP_STATE_MASK, 3388 ret = prepare_ring(xhci, ep_ring, le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
2970 num_trbs, mem_flags); 3389 num_trbs, mem_flags);
2971 if (ret) 3390 if (ret)
2972 return ret; 3391 return ret;
2973 3392
@@ -2979,7 +3398,7 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
2979 urb->dev->speed == USB_SPEED_FULL) 3398 urb->dev->speed == USB_SPEED_FULL)
2980 urb->start_frame >>= 3; 3399 urb->start_frame >>= 3;
2981 3400
2982 xhci_interval = EP_INTERVAL_TO_UFRAMES(ep_ctx->ep_info); 3401 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
2983 ep_interval = urb->interval; 3402 ep_interval = urb->interval;
2984 /* Convert to microframes */ 3403 /* Convert to microframes */
2985 if (urb->dev->speed == USB_SPEED_LOW || 3404 if (urb->dev->speed == USB_SPEED_LOW ||
@@ -2989,7 +3408,7 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
2989 * to set the polling interval (once the API is added). 3408 * to set the polling interval (once the API is added).
2990 */ 3409 */
2991 if (xhci_interval != ep_interval) { 3410 if (xhci_interval != ep_interval) {
2992 if (!printk_ratelimit()) 3411 if (printk_ratelimit())
2993 dev_dbg(&urb->dev->dev, "Driver uses different interval" 3412 dev_dbg(&urb->dev->dev, "Driver uses different interval"
2994 " (%d microframe%s) than xHCI " 3413 " (%d microframe%s) than xHCI "
2995 "(%d microframe%s)\n", 3414 "(%d microframe%s)\n",
@@ -3039,24 +3458,6 @@ static int queue_command(struct xhci_hcd *xhci, u32 field1, u32 field2,
3039 return 0; 3458 return 0;
3040} 3459}
3041 3460
3042/* Queue a no-op command on the command ring */
3043static int queue_cmd_noop(struct xhci_hcd *xhci)
3044{
3045 return queue_command(xhci, 0, 0, 0, TRB_TYPE(TRB_CMD_NOOP), false);
3046}
3047
3048/*
3049 * Place a no-op command on the command ring to test the command and
3050 * event ring.
3051 */
3052void *xhci_setup_one_noop(struct xhci_hcd *xhci)
3053{
3054 if (queue_cmd_noop(xhci) < 0)
3055 return NULL;
3056 xhci->noops_submitted++;
3057 return xhci_ring_cmd_db;
3058}
3059
3060/* Queue a slot enable or disable request on the command ring */ 3461/* Queue a slot enable or disable request on the command ring */
3061int xhci_queue_slot_control(struct xhci_hcd *xhci, u32 trb_type, u32 slot_id) 3462int xhci_queue_slot_control(struct xhci_hcd *xhci, u32 trb_type, u32 slot_id)
3062{ 3463{
@@ -3108,15 +3509,20 @@ int xhci_queue_evaluate_context(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr,
3108 false); 3509 false);
3109} 3510}
3110 3511
3512/*
3513 * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
3514 * activity on an endpoint that is about to be suspended.
3515 */
3111int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, int slot_id, 3516int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, int slot_id,
3112 unsigned int ep_index) 3517 unsigned int ep_index, int suspend)
3113{ 3518{
3114 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id); 3519 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
3115 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index); 3520 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
3116 u32 type = TRB_TYPE(TRB_STOP_RING); 3521 u32 type = TRB_TYPE(TRB_STOP_RING);
3522 u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend);
3117 3523
3118 return queue_command(xhci, 0, 0, 0, 3524 return queue_command(xhci, 0, 0, 0,
3119 trb_slot_id | trb_ep_index | type, false); 3525 trb_slot_id | trb_ep_index | type | trb_suspend, false);
3120} 3526}
3121 3527
3122/* Set Transfer Ring Dequeue Pointer command. 3528/* Set Transfer Ring Dequeue Pointer command.
@@ -3132,6 +3538,7 @@ static int queue_set_tr_deq(struct xhci_hcd *xhci, int slot_id,
3132 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index); 3538 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
3133 u32 trb_stream_id = STREAM_ID_FOR_TRB(stream_id); 3539 u32 trb_stream_id = STREAM_ID_FOR_TRB(stream_id);
3134 u32 type = TRB_TYPE(TRB_SET_DEQ); 3540 u32 type = TRB_TYPE(TRB_SET_DEQ);
3541 struct xhci_virt_ep *ep;
3135 3542
3136 addr = xhci_trb_virt_to_dma(deq_seg, deq_ptr); 3543 addr = xhci_trb_virt_to_dma(deq_seg, deq_ptr);
3137 if (addr == 0) { 3544 if (addr == 0) {
@@ -3140,6 +3547,14 @@ static int queue_set_tr_deq(struct xhci_hcd *xhci, int slot_id,
3140 deq_seg, deq_ptr); 3547 deq_seg, deq_ptr);
3141 return 0; 3548 return 0;
3142 } 3549 }
3550 ep = &xhci->devs[slot_id]->eps[ep_index];
3551 if ((ep->ep_state & SET_DEQ_PENDING)) {
3552 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
3553 xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
3554 return 0;
3555 }
3556 ep->queued_deq_seg = deq_seg;
3557 ep->queued_deq_ptr = deq_ptr;
3143 return queue_command(xhci, lower_32_bits(addr) | cycle_state, 3558 return queue_command(xhci, lower_32_bits(addr) | cycle_state,
3144 upper_32_bits(addr), trb_stream_id, 3559 upper_32_bits(addr), trb_stream_id,
3145 trb_slot_id | trb_ep_index | type, false); 3560 trb_slot_id | trb_ep_index | type, false);
generated by cgit v1.2.2 (git 2.25.0) at 2025-07-27 11:09:55 -0400