1 files changed, 349 insertions, 24 deletions

diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
index b8fd270a8b0d..d64f5724bfc4 100644
--- a/drivers/usb/host/xhci-mem.c
+++ b/drivers/usb/host/xhci-mem.c
@@ -22,6 +22,7 @@
 
 #include <linux/usb.h>
 #include <linux/pci.h>
+#include <linux/slab.h>
 #include <linux/dmapool.h>
 
 #include "xhci.h"
@@ -125,6 +126,23 @@ void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
         kfree(ring);
 }
 
+static void xhci_initialize_ring_info(struct xhci_ring *ring)
+{
+        /* The ring is empty, so the enqueue pointer == dequeue pointer */
+        ring->enqueue = ring->first_seg->trbs;
+        ring->enq_seg = ring->first_seg;
+        ring->dequeue = ring->enqueue;
+        ring->deq_seg = ring->first_seg;
+        /* The ring is initialized to 0. The producer must write 1 to the cycle
+         * bit to handover ownership of the TRB, so PCS = 1.  The consumer must
+         * compare CCS to the cycle bit to check ownership, so CCS = 1.
+         */
+        ring->cycle_state = 1;
+        /* Not necessary for new rings, but needed for re-initialized rings */
+        ring->enq_updates = 0;
+        ring->deq_updates = 0;
+}
+
 /**
  * Create a new ring with zero or more segments.
  *
@@ -173,17 +191,7 @@ static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
                                 " segment %p (virtual), 0x%llx (DMA)\n",
                                 prev, (unsigned long long)prev->dma);
         }
-        /* The ring is empty, so the enqueue pointer == dequeue pointer */
-        ring->enqueue = ring->first_seg->trbs;
-        ring->enq_seg = ring->first_seg;
-        ring->dequeue = ring->enqueue;
-        ring->deq_seg = ring->first_seg;
-        /* The ring is initialized to 0. The producer must write 1 to the cycle
-         * bit to handover ownership of the TRB, so PCS = 1.  The consumer must
-         * compare CCS to the cycle bit to check ownership, so CCS = 1.
-         */
-        ring->cycle_state = 1;
-
+        xhci_initialize_ring_info(ring);
         return ring;
 
 fail:
@@ -191,6 +199,52 @@ fail:
         return 0;
 }
 
+void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci,
+                struct xhci_virt_device *virt_dev,
+                unsigned int ep_index)
+{
+        int rings_cached;
+
+        rings_cached = virt_dev->num_rings_cached;
+        if (rings_cached < XHCI_MAX_RINGS_CACHED) {
+                virt_dev->num_rings_cached++;
+                rings_cached = virt_dev->num_rings_cached;
+                virt_dev->ring_cache[rings_cached] =
+                        virt_dev->eps[ep_index].ring;
+                xhci_dbg(xhci, "Cached old ring, "
+                                "%d ring%s cached\n",
+                                rings_cached,
+                                (rings_cached > 1) ? "s" : "");
+        } else {
+                xhci_ring_free(xhci, virt_dev->eps[ep_index].ring);
+                xhci_dbg(xhci, "Ring cache full (%d rings), "
+                                "freeing ring\n",
+                                virt_dev->num_rings_cached);
+        }
+        virt_dev->eps[ep_index].ring = NULL;
+}
+
+/* Zero an endpoint ring (except for link TRBs) and move the enqueue and dequeue
+ * pointers to the beginning of the ring.
+ */
+static void xhci_reinit_cached_ring(struct xhci_hcd *xhci,
+                struct xhci_ring *ring)
+{
+        struct xhci_segment     *seg = ring->first_seg;
+        do {
+                memset(seg->trbs, 0,
+                                sizeof(union xhci_trb)*TRBS_PER_SEGMENT);
+                /* All endpoint rings have link TRBs */
+                xhci_link_segments(xhci, seg, seg->next, 1);
+                seg = seg->next;
+        } while (seg != ring->first_seg);
+        xhci_initialize_ring_info(ring);
+        /* td list should be empty since all URBs have been cancelled,
+         * but just in case...
+         */
+        INIT_LIST_HEAD(&ring->td_list);
+}
+
 #define CTX_SIZE(_hcc) (HCC_64BYTE_CONTEXT(_hcc) ? 64 : 32)
 
 struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
@@ -214,6 +268,8 @@ struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
 void xhci_free_container_ctx(struct xhci_hcd *xhci,
                              struct xhci_container_ctx *ctx)
 {
+        if (!ctx)
+                return;
         dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma);
         kfree(ctx);
 }
@@ -248,6 +304,15 @@ struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci,
                 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params)));
 }
 
+static void xhci_init_endpoint_timer(struct xhci_hcd *xhci,
+                struct xhci_virt_ep *ep)
+{
+        init_timer(&ep->stop_cmd_timer);
+        ep->stop_cmd_timer.data = (unsigned long) ep;
+        ep->stop_cmd_timer.function = xhci_stop_endpoint_command_watchdog;
+        ep->xhci = xhci;
+}
+
 /* All the xhci_tds in the ring's TD list should be freed at this point */
 void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
 {
@@ -267,6 +332,12 @@ void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
                 if (dev->eps[i].ring)
                         xhci_ring_free(xhci, dev->eps[i].ring);
 
+        if (dev->ring_cache) {
+                for (i = 0; i < dev->num_rings_cached; i++)
+                        xhci_ring_free(xhci, dev->ring_cache[i]);
+                kfree(dev->ring_cache);
+        }
+
         if (dev->in_ctx)
                 xhci_free_container_ctx(xhci, dev->in_ctx);
         if (dev->out_ctx)
@@ -309,15 +380,25 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
         xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
                         (unsigned long long)dev->in_ctx->dma);
 
-        /* Initialize the cancellation list for each endpoint */
-        for (i = 0; i < 31; i++)
+        /* Initialize the cancellation list and watchdog timers for each ep */
+        for (i = 0; i < 31; i++) {
+                xhci_init_endpoint_timer(xhci, &dev->eps[i]);
                 INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list);
+        }
 
         /* Allocate endpoint 0 ring */
         dev->eps[0].ring = xhci_ring_alloc(xhci, 1, true, flags);
         if (!dev->eps[0].ring)
                 goto fail;
 
+        /* Allocate pointers to the ring cache */
+        dev->ring_cache = kzalloc(
+                        sizeof(struct xhci_ring *)*XHCI_MAX_RINGS_CACHED,
+                        flags);
+        if (!dev->ring_cache)
+                goto fail;
+        dev->num_rings_cached = 0;
+
         init_completion(&dev->cmd_completion);
         INIT_LIST_HEAD(&dev->cmd_list);
 
@@ -374,7 +455,7 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud
         case USB_SPEED_LOW:
                 slot_ctx->dev_info |= (u32) SLOT_SPEED_LS;
                 break;
-        case USB_SPEED_VARIABLE:
+        case USB_SPEED_WIRELESS:
                 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
                 return -EINVAL;
                 break;
@@ -418,7 +499,7 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud
         case USB_SPEED_LOW:
                 ep0_ctx->ep_info2 |= MAX_PACKET(8);
                 break;
-        case USB_SPEED_VARIABLE:
+        case USB_SPEED_WIRELESS:
                 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
                 return -EINVAL;
                 break;
@@ -486,8 +567,13 @@ static inline unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
                         if (interval < 3)
                                 interval = 3;
                         if ((1 << interval) != 8*ep->desc.bInterval)
-                                dev_warn(&udev->dev, "ep %#x - rounding interval to %d microframes\n",
-                                                ep->desc.bEndpointAddress, 1 << interval);
+                                dev_warn(&udev->dev,
+                                                "ep %#x - rounding interval"
+                                                " to %d microframes, "
+                                                "ep desc says %d microframes\n",
+                                                ep->desc.bEndpointAddress,
+                                                1 << interval,
+                                                8*ep->desc.bInterval);
                 }
                 break;
         default:
@@ -496,6 +582,19 @@ static inline unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
         return EP_INTERVAL(interval);
 }
 
+/* The "Mult" field in the endpoint context is only set for SuperSpeed devices.
+ * High speed endpoint descriptors can define "the number of additional
+ * transaction opportunities per microframe", but that goes in the Max Burst
+ * endpoint context field.
+ */
+static inline u32 xhci_get_endpoint_mult(struct usb_device *udev,
+                struct usb_host_endpoint *ep)
+{
+        if (udev->speed != USB_SPEED_SUPER || !ep->ss_ep_comp)
+                return 0;
+        return ep->ss_ep_comp->desc.bmAttributes;
+}
+
 static inline u32 xhci_get_endpoint_type(struct usb_device *udev,
                 struct usb_host_endpoint *ep)
 {
@@ -526,6 +625,36 @@ static inline u32 xhci_get_endpoint_type(struct usb_device *udev,
         return type;
 }
 
+/* Return the maximum endpoint service interval time (ESIT) payload.
+ * Basically, this is the maxpacket size, multiplied by the burst size
+ * and mult size.
+ */
+static inline u32 xhci_get_max_esit_payload(struct xhci_hcd *xhci,
+                struct usb_device *udev,
+                struct usb_host_endpoint *ep)
+{
+        int max_burst;
+        int max_packet;
+
+        /* Only applies for interrupt or isochronous endpoints */
+        if (usb_endpoint_xfer_control(&ep->desc) ||
+                        usb_endpoint_xfer_bulk(&ep->desc))
+                return 0;
+
+        if (udev->speed == USB_SPEED_SUPER) {
+                if (ep->ss_ep_comp)
+                        return ep->ss_ep_comp->desc.wBytesPerInterval;
+                xhci_warn(xhci, "WARN no SS endpoint companion descriptor.\n");
+                /* Assume no bursts, no multiple opportunities to send. */
+                return ep->desc.wMaxPacketSize;
+        }
+
+        max_packet = ep->desc.wMaxPacketSize & 0x3ff;
+        max_burst = (ep->desc.wMaxPacketSize & 0x1800) >> 11;
+        /* A 0 in max burst means 1 transfer per ESIT */
+        return max_packet * (max_burst + 1);
+}
+
 int xhci_endpoint_init(struct xhci_hcd *xhci,
                 struct xhci_virt_device *virt_dev,
                 struct usb_device *udev,
@@ -537,6 +666,7 @@ int xhci_endpoint_init(struct xhci_hcd *xhci,
         struct xhci_ring *ep_ring;
         unsigned int max_packet;
         unsigned int max_burst;
+        u32 max_esit_payload;
 
         ep_index = xhci_get_endpoint_index(&ep->desc);
         ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
@@ -544,12 +674,21 @@ int xhci_endpoint_init(struct xhci_hcd *xhci,
         /* Set up the endpoint ring */
         virt_dev->eps[ep_index].new_ring =
                 xhci_ring_alloc(xhci, 1, true, mem_flags);
-        if (!virt_dev->eps[ep_index].new_ring)
-                return -ENOMEM;
+        if (!virt_dev->eps[ep_index].new_ring) {
+                /* Attempt to use the ring cache */
+                if (virt_dev->num_rings_cached == 0)
+                        return -ENOMEM;
+                virt_dev->eps[ep_index].new_ring =
+                        virt_dev->ring_cache[virt_dev->num_rings_cached];
+                virt_dev->ring_cache[virt_dev->num_rings_cached] = NULL;
+                virt_dev->num_rings_cached--;
+                xhci_reinit_cached_ring(xhci, virt_dev->eps[ep_index].new_ring);
+        }
         ep_ring = virt_dev->eps[ep_index].new_ring;
         ep_ctx->deq = ep_ring->first_seg->dma | ep_ring->cycle_state;
 
         ep_ctx->ep_info = xhci_get_endpoint_interval(udev, ep);
+        ep_ctx->ep_info |= EP_MULT(xhci_get_endpoint_mult(udev, ep));
 
         /* FIXME dig Mult and streams info out of ep companion desc */
 
@@ -595,6 +734,26 @@ int xhci_endpoint_init(struct xhci_hcd *xhci,
         default:
                 BUG();
         }
+        max_esit_payload = xhci_get_max_esit_payload(xhci, udev, ep);
+        ep_ctx->tx_info = MAX_ESIT_PAYLOAD_FOR_EP(max_esit_payload);
+
+        /*
+         * XXX no idea how to calculate the average TRB buffer length for bulk
+         * endpoints, as the driver gives us no clue how big each scatter gather
+         * list entry (or buffer) is going to be.
+         *
+         * For isochronous and interrupt endpoints, we set it to the max
+         * available, until we have new API in the USB core to allow drivers to
+         * declare how much bandwidth they actually need.
+         *
+         * Normally, it would be calculated by taking the total of the buffer
+         * lengths in the TD and then dividing by the number of TRBs in a TD,
+         * including link TRBs, No-op TRBs, and Event data TRBs.  Since we don't
+         * use Event Data TRBs, and we don't chain in a link TRB on short
+         * transfers, we're basically dividing by 1.
+         */
+        ep_ctx->tx_info |= AVG_TRB_LENGTH_FOR_EP(max_esit_payload);
+
         /* FIXME Debug endpoint context */
         return 0;
 }
@@ -758,7 +917,8 @@ static void scratchpad_free(struct xhci_hcd *xhci)
 }
 
 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
-                bool allocate_completion, gfp_t mem_flags)
+                bool allocate_in_ctx, bool allocate_completion,
+                gfp_t mem_flags)
 {
         struct xhci_command *command;
 
@@ -766,16 +926,22 @@ struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
         if (!command)
                 return NULL;
 
-        command->in_ctx =
-                xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, mem_flags);
-        if (!command->in_ctx)
-                return NULL;
+        if (allocate_in_ctx) {
+                command->in_ctx =
+                        xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT,
+                                        mem_flags);
+                if (!command->in_ctx) {
+                        kfree(command);
+                        return NULL;
+                }
+        }
 
         if (allocate_completion) {
                 command->completion =
                         kzalloc(sizeof(struct completion), mem_flags);
                 if (!command->completion) {
                         xhci_free_container_ctx(xhci, command->in_ctx);
+                        kfree(command);
                         return NULL;
                 }
                 init_completion(command->completion);
@@ -848,6 +1014,163 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci)
         xhci->page_shift = 0;
 }
 
+static int xhci_test_trb_in_td(struct xhci_hcd *xhci,
+                struct xhci_segment *input_seg,
+                union xhci_trb *start_trb,
+                union xhci_trb *end_trb,
+                dma_addr_t input_dma,
+                struct xhci_segment *result_seg,
+                char *test_name, int test_number)
+{
+        unsigned long long start_dma;
+        unsigned long long end_dma;
+        struct xhci_segment *seg;
+
+        start_dma = xhci_trb_virt_to_dma(input_seg, start_trb);
+        end_dma = xhci_trb_virt_to_dma(input_seg, end_trb);
+
+        seg = trb_in_td(input_seg, start_trb, end_trb, input_dma);
+        if (seg != result_seg) {
+                xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n",
+                                test_name, test_number);
+                xhci_warn(xhci, "Tested TRB math w/ seg %p and "
+                                "input DMA 0x%llx\n",
+                                input_seg,
+                                (unsigned long long) input_dma);
+                xhci_warn(xhci, "starting TRB %p (0x%llx DMA), "
+                                "ending TRB %p (0x%llx DMA)\n",
+                                start_trb, start_dma,
+                                end_trb, end_dma);
+                xhci_warn(xhci, "Expected seg %p, got seg %p\n",
+                                result_seg, seg);
+                return -1;
+        }
+        return 0;
+}
+
+/* TRB math checks for xhci_trb_in_td(), using the command and event rings. */
+static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci, gfp_t mem_flags)
+{
+        struct {
+                dma_addr_t              input_dma;
+                struct xhci_segment     *result_seg;
+        } simple_test_vector [] = {
+                /* A zeroed DMA field should fail */
+                { 0, NULL },
+                /* One TRB before the ring start should fail */
+                { xhci->event_ring->first_seg->dma - 16, NULL },
+                /* One byte before the ring start should fail */
+                { xhci->event_ring->first_seg->dma - 1, NULL },
+                /* Starting TRB should succeed */
+                { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg },
+                /* Ending TRB should succeed */
+                { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16,
+                        xhci->event_ring->first_seg },
+                /* One byte after the ring end should fail */
+                { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL },
+                /* One TRB after the ring end should fail */
+                { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL },
+                /* An address of all ones should fail */
+                { (dma_addr_t) (~0), NULL },
+        };
+        struct {
+                struct xhci_segment     *input_seg;
+                union xhci_trb          *start_trb;
+                union xhci_trb          *end_trb;
+                dma_addr_t              input_dma;
+                struct xhci_segment     *result_seg;
+        } complex_test_vector [] = {
+                /* Test feeding a valid DMA address from a different ring */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = xhci->event_ring->first_seg->trbs,
+                        .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
+                        .input_dma = xhci->cmd_ring->first_seg->dma,
+                        .result_seg = NULL,
+                },
+                /* Test feeding a valid end TRB from a different ring */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = xhci->event_ring->first_seg->trbs,
+                        .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
+                        .input_dma = xhci->cmd_ring->first_seg->dma,
+                        .result_seg = NULL,
+                },
+                /* Test feeding a valid start and end TRB from a different ring */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = xhci->cmd_ring->first_seg->trbs,
+                        .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
+                        .input_dma = xhci->cmd_ring->first_seg->dma,
+                        .result_seg = NULL,
+                },
+                /* TRB in this ring, but after this TD */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = &xhci->event_ring->first_seg->trbs[0],
+                        .end_trb = &xhci->event_ring->first_seg->trbs[3],
+                        .input_dma = xhci->event_ring->first_seg->dma + 4*16,
+                        .result_seg = NULL,
+                },
+                /* TRB in this ring, but before this TD */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = &xhci->event_ring->first_seg->trbs[3],
+                        .end_trb = &xhci->event_ring->first_seg->trbs[6],
+                        .input_dma = xhci->event_ring->first_seg->dma + 2*16,
+                        .result_seg = NULL,
+                },
+                /* TRB in this ring, but after this wrapped TD */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
+                        .end_trb = &xhci->event_ring->first_seg->trbs[1],
+                        .input_dma = xhci->event_ring->first_seg->dma + 2*16,
+                        .result_seg = NULL,
+                },
+                /* TRB in this ring, but before this wrapped TD */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
+                        .end_trb = &xhci->event_ring->first_seg->trbs[1],
+                        .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16,
+                        .result_seg = NULL,
+                },
+                /* TRB not in this ring, and we have a wrapped TD */
+                {       .input_seg = xhci->event_ring->first_seg,
+                        .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
+                        .end_trb = &xhci->event_ring->first_seg->trbs[1],
+                        .input_dma = xhci->cmd_ring->first_seg->dma + 2*16,
+                        .result_seg = NULL,
+                },
+        };
+
+        unsigned int num_tests;
+        int i, ret;
+
+        num_tests = sizeof(simple_test_vector) / sizeof(simple_test_vector[0]);
+        for (i = 0; i < num_tests; i++) {
+                ret = xhci_test_trb_in_td(xhci,
+                                xhci->event_ring->first_seg,
+                                xhci->event_ring->first_seg->trbs,
+                                &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
+                                simple_test_vector[i].input_dma,
+                                simple_test_vector[i].result_seg,
+                                "Simple", i);
+                if (ret < 0)
+                        return ret;
+        }
+
+        num_tests = sizeof(complex_test_vector) / sizeof(complex_test_vector[0]);
+        for (i = 0; i < num_tests; i++) {
+                ret = xhci_test_trb_in_td(xhci,
+                                complex_test_vector[i].input_seg,
+                                complex_test_vector[i].start_trb,
+                                complex_test_vector[i].end_trb,
+                                complex_test_vector[i].input_dma,
+                                complex_test_vector[i].result_seg,
+                                "Complex", i);
+                if (ret < 0)
+                        return ret;
+        }
+        xhci_dbg(xhci, "TRB math tests passed.\n");
+        return 0;
+}
+
+
 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 {
         dma_addr_t      dma;
@@ -951,6 +1274,8 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
         xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, false, flags);
         if (!xhci->event_ring)
                 goto fail;
+        if (xhci_check_trb_in_td_math(xhci, flags) < 0)
+                goto fail;
 
         xhci->erst.entries = pci_alloc_consistent(to_pci_dev(dev),
                         sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS, &dma);