diff options
| -rw-r--r-- | drivers/net/xen-netback/common.h | 39 | ||||
| -rw-r--r-- | drivers/net/xen-netback/interface.c | 74 | ||||
| -rw-r--r-- | drivers/net/xen-netback/netback.c | 319 | ||||
| -rw-r--r-- | drivers/net/xen-netback/xenbus.c | 22 |
4 files changed, 251 insertions, 203 deletions
diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index d4eb8d2e9cb7..083ecc93fe5e 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h | |||
| @@ -176,10 +176,11 @@ struct xenvif_queue { /* Per-queue data for xenvif */ | |||
| 176 | char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */ | 176 | char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */ |
| 177 | struct xen_netif_rx_back_ring rx; | 177 | struct xen_netif_rx_back_ring rx; |
| 178 | struct sk_buff_head rx_queue; | 178 | struct sk_buff_head rx_queue; |
| 179 | RING_IDX rx_last_skb_slots; | ||
| 180 | unsigned long status; | ||
| 181 | 179 | ||
| 182 | struct timer_list rx_stalled; | 180 | unsigned int rx_queue_max; |
| 181 | unsigned int rx_queue_len; | ||
| 182 | unsigned long last_rx_time; | ||
| 183 | bool stalled; | ||
| 183 | 184 | ||
| 184 | struct gnttab_copy grant_copy_op[MAX_GRANT_COPY_OPS]; | 185 | struct gnttab_copy grant_copy_op[MAX_GRANT_COPY_OPS]; |
| 185 | 186 | ||
| @@ -199,18 +200,14 @@ struct xenvif_queue { /* Per-queue data for xenvif */ | |||
| 199 | struct xenvif_stats stats; | 200 | struct xenvif_stats stats; |
| 200 | }; | 201 | }; |
| 201 | 202 | ||
| 203 | /* Maximum number of Rx slots a to-guest packet may use, including the | ||
| 204 | * slot needed for GSO meta-data. | ||
| 205 | */ | ||
| 206 | #define XEN_NETBK_RX_SLOTS_MAX (MAX_SKB_FRAGS + 1) | ||
| 207 | |||
| 202 | enum state_bit_shift { | 208 | enum state_bit_shift { |
| 203 | /* This bit marks that the vif is connected */ | 209 | /* This bit marks that the vif is connected */ |
| 204 | VIF_STATUS_CONNECTED, | 210 | VIF_STATUS_CONNECTED, |
| 205 | /* This bit signals the RX thread that queuing was stopped (in | ||
| 206 | * start_xmit), and either the timer fired or an RX interrupt came | ||
| 207 | */ | ||
| 208 | QUEUE_STATUS_RX_PURGE_EVENT, | ||
| 209 | /* This bit tells the interrupt handler that this queue was the reason | ||
| 210 | * for the carrier off, so it should kick the thread. Only queues which | ||
| 211 | * brought it down can turn on the carrier. | ||
| 212 | */ | ||
| 213 | QUEUE_STATUS_RX_STALLED | ||
| 214 | }; | 211 | }; |
| 215 | 212 | ||
| 216 | struct xenvif { | 213 | struct xenvif { |
| @@ -228,9 +225,6 @@ struct xenvif { | |||
| 228 | u8 ip_csum:1; | 225 | u8 ip_csum:1; |
| 229 | u8 ipv6_csum:1; | 226 | u8 ipv6_csum:1; |
| 230 | 227 | ||
| 231 | /* Internal feature information. */ | ||
| 232 | u8 can_queue:1; /* can queue packets for receiver? */ | ||
| 233 | |||
| 234 | /* Is this interface disabled? True when backend discovers | 228 | /* Is this interface disabled? True when backend discovers |
| 235 | * frontend is rogue. | 229 | * frontend is rogue. |
| 236 | */ | 230 | */ |
| @@ -240,6 +234,9 @@ struct xenvif { | |||
| 240 | /* Queues */ | 234 | /* Queues */ |
| 241 | struct xenvif_queue *queues; | 235 | struct xenvif_queue *queues; |
| 242 | unsigned int num_queues; /* active queues, resource allocated */ | 236 | unsigned int num_queues; /* active queues, resource allocated */ |
| 237 | unsigned int stalled_queues; | ||
| 238 | |||
| 239 | spinlock_t lock; | ||
| 243 | 240 | ||
| 244 | #ifdef CONFIG_DEBUG_FS | 241 | #ifdef CONFIG_DEBUG_FS |
| 245 | struct dentry *xenvif_dbg_root; | 242 | struct dentry *xenvif_dbg_root; |
| @@ -249,6 +246,14 @@ struct xenvif { | |||
| 249 | struct net_device *dev; | 246 | struct net_device *dev; |
| 250 | }; | 247 | }; |
| 251 | 248 | ||
| 249 | struct xenvif_rx_cb { | ||
| 250 | unsigned long expires; | ||
| 251 | int meta_slots_used; | ||
| 252 | bool full_coalesce; | ||
| 253 | }; | ||
| 254 | |||
| 255 | #define XENVIF_RX_CB(skb) ((struct xenvif_rx_cb *)(skb)->cb) | ||
| 256 | |||
| 252 | static inline struct xenbus_device *xenvif_to_xenbus_device(struct xenvif *vif) | 257 | static inline struct xenbus_device *xenvif_to_xenbus_device(struct xenvif *vif) |
| 253 | { | 258 | { |
| 254 | return to_xenbus_device(vif->dev->dev.parent); | 259 | return to_xenbus_device(vif->dev->dev.parent); |
| @@ -272,8 +277,6 @@ void xenvif_xenbus_fini(void); | |||
| 272 | 277 | ||
| 273 | int xenvif_schedulable(struct xenvif *vif); | 278 | int xenvif_schedulable(struct xenvif *vif); |
| 274 | 279 | ||
| 275 | int xenvif_must_stop_queue(struct xenvif_queue *queue); | ||
| 276 | |||
| 277 | int xenvif_queue_stopped(struct xenvif_queue *queue); | 280 | int xenvif_queue_stopped(struct xenvif_queue *queue); |
| 278 | void xenvif_wake_queue(struct xenvif_queue *queue); | 281 | void xenvif_wake_queue(struct xenvif_queue *queue); |
| 279 | 282 | ||
| @@ -296,6 +299,8 @@ void xenvif_kick_thread(struct xenvif_queue *queue); | |||
| 296 | 299 | ||
| 297 | int xenvif_dealloc_kthread(void *data); | 300 | int xenvif_dealloc_kthread(void *data); |
| 298 | 301 | ||
| 302 | void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); | ||
| 303 | |||
| 299 | /* Determine whether the needed number of slots (req) are available, | 304 | /* Determine whether the needed number of slots (req) are available, |
| 300 | * and set req_event if not. | 305 | * and set req_event if not. |
| 301 | */ | 306 | */ |
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index f379689dde30..895fe84011e7 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c | |||
| @@ -43,6 +43,9 @@ | |||
| 43 | #define XENVIF_QUEUE_LENGTH 32 | 43 | #define XENVIF_QUEUE_LENGTH 32 |
| 44 | #define XENVIF_NAPI_WEIGHT 64 | 44 | #define XENVIF_NAPI_WEIGHT 64 |
| 45 | 45 | ||
| 46 | /* Number of bytes allowed on the internal guest Rx queue. */ | ||
| 47 | #define XENVIF_RX_QUEUE_BYTES (XEN_NETIF_RX_RING_SIZE/2 * PAGE_SIZE) | ||
| 48 | |||
| 46 | /* This function is used to set SKBTX_DEV_ZEROCOPY as well as | 49 | /* This function is used to set SKBTX_DEV_ZEROCOPY as well as |
| 47 | * increasing the inflight counter. We need to increase the inflight | 50 | * increasing the inflight counter. We need to increase the inflight |
| 48 | * counter because core driver calls into xenvif_zerocopy_callback | 51 | * counter because core driver calls into xenvif_zerocopy_callback |
| @@ -60,20 +63,11 @@ void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue) | |||
| 60 | atomic_dec(&queue->inflight_packets); | 63 | atomic_dec(&queue->inflight_packets); |
| 61 | } | 64 | } |
| 62 | 65 | ||
| 63 | static inline void xenvif_stop_queue(struct xenvif_queue *queue) | ||
| 64 | { | ||
| 65 | struct net_device *dev = queue->vif->dev; | ||
| 66 | |||
| 67 | if (!queue->vif->can_queue) | ||
| 68 | return; | ||
| 69 | |||
| 70 | netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); | ||
| 71 | } | ||
| 72 | |||
| 73 | int xenvif_schedulable(struct xenvif *vif) | 66 | int xenvif_schedulable(struct xenvif *vif) |
| 74 | { | 67 | { |
| 75 | return netif_running(vif->dev) && | 68 | return netif_running(vif->dev) && |
| 76 | test_bit(VIF_STATUS_CONNECTED, &vif->status); | 69 | test_bit(VIF_STATUS_CONNECTED, &vif->status) && |
| 70 | !vif->disabled; | ||
| 77 | } | 71 | } |
| 78 | 72 | ||
| 79 | static irqreturn_t xenvif_tx_interrupt(int irq, void *dev_id) | 73 | static irqreturn_t xenvif_tx_interrupt(int irq, void *dev_id) |
| @@ -114,16 +108,7 @@ int xenvif_poll(struct napi_struct *napi, int budget) | |||
| 114 | static irqreturn_t xenvif_rx_interrupt(int irq, void *dev_id) | 108 | static irqreturn_t xenvif_rx_interrupt(int irq, void *dev_id) |
| 115 | { | 109 | { |
| 116 | struct xenvif_queue *queue = dev_id; | 110 | struct xenvif_queue *queue = dev_id; |
| 117 | struct netdev_queue *net_queue = | ||
| 118 | netdev_get_tx_queue(queue->vif->dev, queue->id); | ||
| 119 | 111 | ||
| 120 | /* QUEUE_STATUS_RX_PURGE_EVENT is only set if either QDisc was off OR | ||
| 121 | * the carrier went down and this queue was previously blocked | ||
| 122 | */ | ||
| 123 | if (unlikely(netif_tx_queue_stopped(net_queue) || | ||
| 124 | (!netif_carrier_ok(queue->vif->dev) && | ||
| 125 | test_bit(QUEUE_STATUS_RX_STALLED, &queue->status)))) | ||
| 126 | set_bit(QUEUE_STATUS_RX_PURGE_EVENT, &queue->status); | ||
| 127 | xenvif_kick_thread(queue); | 112 | xenvif_kick_thread(queue); |
| 128 | 113 | ||
| 129 | return IRQ_HANDLED; | 114 | return IRQ_HANDLED; |
| @@ -151,24 +136,13 @@ void xenvif_wake_queue(struct xenvif_queue *queue) | |||
| 151 | netif_tx_wake_queue(netdev_get_tx_queue(dev, id)); | 136 | netif_tx_wake_queue(netdev_get_tx_queue(dev, id)); |
| 152 | } | 137 | } |
| 153 | 138 | ||
| 154 | /* Callback to wake the queue's thread and turn the carrier off on timeout */ | ||
| 155 | static void xenvif_rx_stalled(unsigned long data) | ||
| 156 | { | ||
| 157 | struct xenvif_queue *queue = (struct xenvif_queue *)data; | ||
| 158 | |||
| 159 | if (xenvif_queue_stopped(queue)) { | ||
| 160 | set_bit(QUEUE_STATUS_RX_PURGE_EVENT, &queue->status); | ||
| 161 | xenvif_kick_thread(queue); | ||
| 162 | } | ||
| 163 | } | ||
| 164 | |||
| 165 | static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev) | 139 | static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev) |
| 166 | { | 140 | { |
| 167 | struct xenvif *vif = netdev_priv(dev); | 141 | struct xenvif *vif = netdev_priv(dev); |
| 168 | struct xenvif_queue *queue = NULL; | 142 | struct xenvif_queue *queue = NULL; |
| 169 | unsigned int num_queues = vif->num_queues; | 143 | unsigned int num_queues = vif->num_queues; |
| 170 | u16 index; | 144 | u16 index; |
| 171 | int min_slots_needed; | 145 | struct xenvif_rx_cb *cb; |
| 172 | 146 | ||
| 173 | BUG_ON(skb->dev != dev); | 147 | BUG_ON(skb->dev != dev); |
| 174 | 148 | ||
| @@ -191,30 +165,10 @@ static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev) | |||
| 191 | !xenvif_schedulable(vif)) | 165 | !xenvif_schedulable(vif)) |
| 192 | goto drop; | 166 | goto drop; |
| 193 | 167 | ||
| 194 | /* At best we'll need one slot for the header and one for each | 168 | cb = XENVIF_RX_CB(skb); |
| 195 | * frag. | 169 | cb->expires = jiffies + rx_drain_timeout_jiffies; |
| 196 | */ | ||
| 197 | min_slots_needed = 1 + skb_shinfo(skb)->nr_frags; | ||
| 198 | |||
| 199 | /* If the skb is GSO then we'll also need an extra slot for the | ||
| 200 | * metadata. | ||
| 201 | */ | ||
| 202 | if (skb_is_gso(skb)) | ||
| 203 | min_slots_needed++; | ||
| 204 | 170 | ||
| 205 | /* If the skb can't possibly fit in the remaining slots | 171 | xenvif_rx_queue_tail(queue, skb); |
| 206 | * then turn off the queue to give the ring a chance to | ||
| 207 | * drain. | ||
| 208 | */ | ||
| 209 | if (!xenvif_rx_ring_slots_available(queue, min_slots_needed)) { | ||
| 210 | queue->rx_stalled.function = xenvif_rx_stalled; | ||
| 211 | queue->rx_stalled.data = (unsigned long)queue; | ||
| 212 | xenvif_stop_queue(queue); | ||
| 213 | mod_timer(&queue->rx_stalled, | ||
| 214 | jiffies + rx_drain_timeout_jiffies); | ||
| 215 | } | ||
| 216 | |||
| 217 | skb_queue_tail(&queue->rx_queue, skb); | ||
| 218 | xenvif_kick_thread(queue); | 172 | xenvif_kick_thread(queue); |
| 219 | 173 | ||
| 220 | return NETDEV_TX_OK; | 174 | return NETDEV_TX_OK; |
| @@ -465,6 +419,8 @@ struct xenvif *xenvif_alloc(struct device *parent, domid_t domid, | |||
| 465 | vif->queues = NULL; | 419 | vif->queues = NULL; |
| 466 | vif->num_queues = 0; | 420 | vif->num_queues = 0; |
| 467 | 421 | ||
| 422 | spin_lock_init(&vif->lock); | ||
| 423 | |||
| 468 | dev->netdev_ops = &xenvif_netdev_ops; | 424 | dev->netdev_ops = &xenvif_netdev_ops; |
| 469 | dev->hw_features = NETIF_F_SG | | 425 | dev->hw_features = NETIF_F_SG | |
| 470 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | | 426 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | |
| @@ -508,6 +464,8 @@ int xenvif_init_queue(struct xenvif_queue *queue) | |||
| 508 | init_timer(&queue->credit_timeout); | 464 | init_timer(&queue->credit_timeout); |
| 509 | queue->credit_window_start = get_jiffies_64(); | 465 | queue->credit_window_start = get_jiffies_64(); |
| 510 | 466 | ||
| 467 | queue->rx_queue_max = XENVIF_RX_QUEUE_BYTES; | ||
| 468 | |||
| 511 | skb_queue_head_init(&queue->rx_queue); | 469 | skb_queue_head_init(&queue->rx_queue); |
| 512 | skb_queue_head_init(&queue->tx_queue); | 470 | skb_queue_head_init(&queue->tx_queue); |
| 513 | 471 | ||
| @@ -539,8 +497,6 @@ int xenvif_init_queue(struct xenvif_queue *queue) | |||
| 539 | queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE; | 497 | queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE; |
| 540 | } | 498 | } |
| 541 | 499 | ||
| 542 | init_timer(&queue->rx_stalled); | ||
| 543 | |||
| 544 | return 0; | 500 | return 0; |
| 545 | } | 501 | } |
| 546 | 502 | ||
| @@ -551,7 +507,6 @@ void xenvif_carrier_on(struct xenvif *vif) | |||
| 551 | dev_set_mtu(vif->dev, ETH_DATA_LEN); | 507 | dev_set_mtu(vif->dev, ETH_DATA_LEN); |
| 552 | netdev_update_features(vif->dev); | 508 | netdev_update_features(vif->dev); |
| 553 | set_bit(VIF_STATUS_CONNECTED, &vif->status); | 509 | set_bit(VIF_STATUS_CONNECTED, &vif->status); |
| 554 | netif_carrier_on(vif->dev); | ||
| 555 | if (netif_running(vif->dev)) | 510 | if (netif_running(vif->dev)) |
| 556 | xenvif_up(vif); | 511 | xenvif_up(vif); |
| 557 | rtnl_unlock(); | 512 | rtnl_unlock(); |
| @@ -611,6 +566,8 @@ int xenvif_connect(struct xenvif_queue *queue, unsigned long tx_ring_ref, | |||
| 611 | disable_irq(queue->rx_irq); | 566 | disable_irq(queue->rx_irq); |
| 612 | } | 567 | } |
| 613 | 568 | ||
| 569 | queue->stalled = true; | ||
| 570 | |||
| 614 | task = kthread_create(xenvif_kthread_guest_rx, | 571 | task = kthread_create(xenvif_kthread_guest_rx, |
| 615 | (void *)queue, "%s-guest-rx", queue->name); | 572 | (void *)queue, "%s-guest-rx", queue->name); |
| 616 | if (IS_ERR(task)) { | 573 | if (IS_ERR(task)) { |
| @@ -674,7 +631,6 @@ void xenvif_disconnect(struct xenvif *vif) | |||
| 674 | netif_napi_del(&queue->napi); | 631 | netif_napi_del(&queue->napi); |
| 675 | 632 | ||
| 676 | if (queue->task) { | 633 | if (queue->task) { |
| 677 | del_timer_sync(&queue->rx_stalled); | ||
| 678 | kthread_stop(queue->task); | 634 | kthread_stop(queue->task); |
| 679 | queue->task = NULL; | 635 | queue->task = NULL; |
| 680 | } | 636 | } |
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index 08f65996534c..6563f0713fc0 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c | |||
| @@ -55,13 +55,20 @@ | |||
| 55 | bool separate_tx_rx_irq = 1; | 55 | bool separate_tx_rx_irq = 1; |
| 56 | module_param(separate_tx_rx_irq, bool, 0644); | 56 | module_param(separate_tx_rx_irq, bool, 0644); |
| 57 | 57 | ||
| 58 | /* When guest ring is filled up, qdisc queues the packets for us, but we have | 58 | /* The time that packets can stay on the guest Rx internal queue |
| 59 | * to timeout them, otherwise other guests' packets can get stuck there | 59 | * before they are dropped. |
| 60 | */ | 60 | */ |
| 61 | unsigned int rx_drain_timeout_msecs = 10000; | 61 | unsigned int rx_drain_timeout_msecs = 10000; |
| 62 | module_param(rx_drain_timeout_msecs, uint, 0444); | 62 | module_param(rx_drain_timeout_msecs, uint, 0444); |
| 63 | unsigned int rx_drain_timeout_jiffies; | 63 | unsigned int rx_drain_timeout_jiffies; |
| 64 | 64 | ||
| 65 | /* The length of time before the frontend is considered unresponsive | ||
| 66 | * because it isn't providing Rx slots. | ||
| 67 | */ | ||
| 68 | static unsigned int rx_stall_timeout_msecs = 60000; | ||
| 69 | module_param(rx_stall_timeout_msecs, uint, 0444); | ||
| 70 | static unsigned int rx_stall_timeout_jiffies; | ||
| 71 | |||
| 65 | unsigned int xenvif_max_queues; | 72 | unsigned int xenvif_max_queues; |
| 66 | module_param_named(max_queues, xenvif_max_queues, uint, 0644); | 73 | module_param_named(max_queues, xenvif_max_queues, uint, 0644); |
| 67 | MODULE_PARM_DESC(max_queues, | 74 | MODULE_PARM_DESC(max_queues, |
| @@ -83,7 +90,6 @@ static void make_tx_response(struct xenvif_queue *queue, | |||
| 83 | s8 st); | 90 | s8 st); |
| 84 | 91 | ||
| 85 | static inline int tx_work_todo(struct xenvif_queue *queue); | 92 | static inline int tx_work_todo(struct xenvif_queue *queue); |
| 86 | static inline int rx_work_todo(struct xenvif_queue *queue); | ||
| 87 | 93 | ||
| 88 | static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue, | 94 | static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue, |
| 89 | u16 id, | 95 | u16 id, |
| @@ -163,6 +169,69 @@ bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue, int needed) | |||
| 163 | return false; | 169 | return false; |
| 164 | } | 170 | } |
| 165 | 171 | ||
| 172 | void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb) | ||
| 173 | { | ||
| 174 | unsigned long flags; | ||
| 175 | |||
| 176 | spin_lock_irqsave(&queue->rx_queue.lock, flags); | ||
| 177 | |||
| 178 | __skb_queue_tail(&queue->rx_queue, skb); | ||
| 179 | |||
| 180 | queue->rx_queue_len += skb->len; | ||
| 181 | if (queue->rx_queue_len > queue->rx_queue_max) | ||
| 182 | netif_tx_stop_queue(netdev_get_tx_queue(queue->vif->dev, queue->id)); | ||
| 183 | |||
| 184 | spin_unlock_irqrestore(&queue->rx_queue.lock, flags); | ||
| 185 | } | ||
| 186 | |||
| 187 | static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue) | ||
| 188 | { | ||
| 189 | struct sk_buff *skb; | ||
| 190 | |||
| 191 | spin_lock_irq(&queue->rx_queue.lock); | ||
| 192 | |||
| 193 | skb = __skb_dequeue(&queue->rx_queue); | ||
| 194 | if (skb) | ||
| 195 | queue->rx_queue_len -= skb->len; | ||
| 196 | |||
| 197 | spin_unlock_irq(&queue->rx_queue.lock); | ||
| 198 | |||
| 199 | return skb; | ||
| 200 | } | ||
| 201 | |||
| 202 | static void xenvif_rx_queue_maybe_wake(struct xenvif_queue *queue) | ||
| 203 | { | ||
| 204 | spin_lock_irq(&queue->rx_queue.lock); | ||
| 205 | |||
| 206 | if (queue->rx_queue_len < queue->rx_queue_max) | ||
| 207 | netif_tx_wake_queue(netdev_get_tx_queue(queue->vif->dev, queue->id)); | ||
| 208 | |||
| 209 | spin_unlock_irq(&queue->rx_queue.lock); | ||
| 210 | } | ||
| 211 | |||
| 212 | |||
| 213 | static void xenvif_rx_queue_purge(struct xenvif_queue *queue) | ||
| 214 | { | ||
| 215 | struct sk_buff *skb; | ||
| 216 | while ((skb = xenvif_rx_dequeue(queue)) != NULL) | ||
| 217 | kfree_skb(skb); | ||
| 218 | } | ||
| 219 | |||
| 220 | static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue) | ||
| 221 | { | ||
| 222 | struct sk_buff *skb; | ||
| 223 | |||
| 224 | for(;;) { | ||
| 225 | skb = skb_peek(&queue->rx_queue); | ||
| 226 | if (!skb) | ||
| 227 | break; | ||
| 228 | if (time_before(jiffies, XENVIF_RX_CB(skb)->expires)) | ||
| 229 | break; | ||
| 230 | xenvif_rx_dequeue(queue); | ||
| 231 | kfree_skb(skb); | ||
| 232 | } | ||
| 233 | } | ||
| 234 | |||
| 166 | /* | 235 | /* |
| 167 | * Returns true if we should start a new receive buffer instead of | 236 | * Returns true if we should start a new receive buffer instead of |
| 168 | * adding 'size' bytes to a buffer which currently contains 'offset' | 237 | * adding 'size' bytes to a buffer which currently contains 'offset' |
| @@ -237,13 +306,6 @@ static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue, | |||
| 237 | return meta; | 306 | return meta; |
| 238 | } | 307 | } |
| 239 | 308 | ||
| 240 | struct xenvif_rx_cb { | ||
| 241 | int meta_slots_used; | ||
| 242 | bool full_coalesce; | ||
| 243 | }; | ||
| 244 | |||
| 245 | #define XENVIF_RX_CB(skb) ((struct xenvif_rx_cb *)(skb)->cb) | ||
| 246 | |||
| 247 | /* | 309 | /* |
| 248 | * Set up the grant operations for this fragment. If it's a flipping | 310 | * Set up the grant operations for this fragment. If it's a flipping |
| 249 | * interface, we also set up the unmap request from here. | 311 | * interface, we also set up the unmap request from here. |
| @@ -587,12 +649,15 @@ static void xenvif_rx_action(struct xenvif_queue *queue) | |||
| 587 | 649 | ||
| 588 | skb_queue_head_init(&rxq); | 650 | skb_queue_head_init(&rxq); |
| 589 | 651 | ||
| 590 | while ((skb = skb_dequeue(&queue->rx_queue)) != NULL) { | 652 | while (xenvif_rx_ring_slots_available(queue, XEN_NETBK_RX_SLOTS_MAX) |
| 653 | && (skb = xenvif_rx_dequeue(queue)) != NULL) { | ||
| 591 | RING_IDX max_slots_needed; | 654 | RING_IDX max_slots_needed; |
| 592 | RING_IDX old_req_cons; | 655 | RING_IDX old_req_cons; |
| 593 | RING_IDX ring_slots_used; | 656 | RING_IDX ring_slots_used; |
| 594 | int i; | 657 | int i; |
| 595 | 658 | ||
| 659 | queue->last_rx_time = jiffies; | ||
| 660 | |||
| 596 | /* We need a cheap worse case estimate for the number of | 661 | /* We need a cheap worse case estimate for the number of |
| 597 | * slots we'll use. | 662 | * slots we'll use. |
| 598 | */ | 663 | */ |
| @@ -634,15 +699,6 @@ static void xenvif_rx_action(struct xenvif_queue *queue) | |||
| 634 | skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)) | 699 | skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)) |
| 635 | max_slots_needed++; | 700 | max_slots_needed++; |
| 636 | 701 | ||
| 637 | /* If the skb may not fit then bail out now */ | ||
| 638 | if (!xenvif_rx_ring_slots_available(queue, max_slots_needed)) { | ||
| 639 | skb_queue_head(&queue->rx_queue, skb); | ||
| 640 | need_to_notify = true; | ||
| 641 | queue->rx_last_skb_slots = max_slots_needed; | ||
| 642 | break; | ||
| 643 | } else | ||
| 644 | queue->rx_last_skb_slots = 0; | ||
| 645 | |||
| 646 | old_req_cons = queue->rx.req_cons; | 702 | old_req_cons = queue->rx.req_cons; |
| 647 | XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo, queue); | 703 | XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo, queue); |
| 648 | ring_slots_used = queue->rx.req_cons - old_req_cons; | 704 | ring_slots_used = queue->rx.req_cons - old_req_cons; |
| @@ -1869,12 +1925,6 @@ void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx) | |||
| 1869 | } | 1925 | } |
| 1870 | } | 1926 | } |
| 1871 | 1927 | ||
| 1872 | static inline int rx_work_todo(struct xenvif_queue *queue) | ||
| 1873 | { | ||
| 1874 | return (!skb_queue_empty(&queue->rx_queue) && | ||
| 1875 | xenvif_rx_ring_slots_available(queue, queue->rx_last_skb_slots)); | ||
| 1876 | } | ||
| 1877 | |||
| 1878 | static inline int tx_work_todo(struct xenvif_queue *queue) | 1928 | static inline int tx_work_todo(struct xenvif_queue *queue) |
| 1879 | { | 1929 | { |
| 1880 | if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx))) | 1930 | if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx))) |
| @@ -1931,92 +1981,121 @@ err: | |||
| 1931 | return err; | 1981 | return err; |
| 1932 | } | 1982 | } |
| 1933 | 1983 | ||
| 1934 | static void xenvif_start_queue(struct xenvif_queue *queue) | 1984 | static void xenvif_queue_carrier_off(struct xenvif_queue *queue) |
| 1935 | { | 1985 | { |
| 1936 | if (xenvif_schedulable(queue->vif)) | 1986 | struct xenvif *vif = queue->vif; |
| 1937 | xenvif_wake_queue(queue); | 1987 | |
| 1988 | queue->stalled = true; | ||
| 1989 | |||
| 1990 | /* At least one queue has stalled? Disable the carrier. */ | ||
| 1991 | spin_lock(&vif->lock); | ||
| 1992 | if (vif->stalled_queues++ == 0) { | ||
| 1993 | netdev_info(vif->dev, "Guest Rx stalled"); | ||
| 1994 | netif_carrier_off(vif->dev); | ||
| 1995 | } | ||
| 1996 | spin_unlock(&vif->lock); | ||
| 1938 | } | 1997 | } |
| 1939 | 1998 | ||
| 1940 | /* Only called from the queue's thread, it handles the situation when the guest | 1999 | static void xenvif_queue_carrier_on(struct xenvif_queue *queue) |
| 1941 | * doesn't post enough requests on the receiving ring. | ||
| 1942 | * First xenvif_start_xmit disables QDisc and start a timer, and then either the | ||
| 1943 | * timer fires, or the guest send an interrupt after posting new request. If it | ||
| 1944 | * is the timer, the carrier is turned off here. | ||
| 1945 | * */ | ||
| 1946 | static void xenvif_rx_purge_event(struct xenvif_queue *queue) | ||
| 1947 | { | 2000 | { |
| 1948 | /* Either the last unsuccesful skb or at least 1 slot should fit */ | 2001 | struct xenvif *vif = queue->vif; |
| 1949 | int needed = queue->rx_last_skb_slots ? | ||
| 1950 | queue->rx_last_skb_slots : 1; | ||
| 1951 | 2002 | ||
| 1952 | /* It is assumed that if the guest post new slots after this, the RX | 2003 | queue->last_rx_time = jiffies; /* Reset Rx stall detection. */ |
| 1953 | * interrupt will set the QUEUE_STATUS_RX_PURGE_EVENT bit and wake up | 2004 | queue->stalled = false; |
| 1954 | * the thread again | ||
| 1955 | */ | ||
| 1956 | set_bit(QUEUE_STATUS_RX_STALLED, &queue->status); | ||
| 1957 | if (!xenvif_rx_ring_slots_available(queue, needed)) { | ||
| 1958 | rtnl_lock(); | ||
| 1959 | if (netif_carrier_ok(queue->vif->dev)) { | ||
| 1960 | /* Timer fired and there are still no slots. Turn off | ||
| 1961 | * everything except the interrupts | ||
| 1962 | */ | ||
| 1963 | netif_carrier_off(queue->vif->dev); | ||
| 1964 | skb_queue_purge(&queue->rx_queue); | ||
| 1965 | queue->rx_last_skb_slots = 0; | ||
| 1966 | if (net_ratelimit()) | ||
| 1967 | netdev_err(queue->vif->dev, "Carrier off due to lack of guest response on queue %d\n", queue->id); | ||
| 1968 | } else { | ||
| 1969 | /* Probably an another queue already turned the carrier | ||
| 1970 | * off, make sure nothing is stucked in the internal | ||
| 1971 | * queue of this queue | ||
| 1972 | */ | ||
| 1973 | skb_queue_purge(&queue->rx_queue); | ||
| 1974 | queue->rx_last_skb_slots = 0; | ||
| 1975 | } | ||
| 1976 | rtnl_unlock(); | ||
| 1977 | } else if (!netif_carrier_ok(queue->vif->dev)) { | ||
| 1978 | unsigned int num_queues = queue->vif->num_queues; | ||
| 1979 | unsigned int i; | ||
| 1980 | /* The carrier was down, but an interrupt kicked | ||
| 1981 | * the thread again after new requests were | ||
| 1982 | * posted | ||
| 1983 | */ | ||
| 1984 | clear_bit(QUEUE_STATUS_RX_STALLED, | ||
| 1985 | &queue->status); | ||
| 1986 | rtnl_lock(); | ||
| 1987 | netif_carrier_on(queue->vif->dev); | ||
| 1988 | netif_tx_wake_all_queues(queue->vif->dev); | ||
| 1989 | rtnl_unlock(); | ||
| 1990 | 2005 | ||
| 1991 | for (i = 0; i < num_queues; i++) { | 2006 | /* All queues are ready? Enable the carrier. */ |
| 1992 | struct xenvif_queue *temp = &queue->vif->queues[i]; | 2007 | spin_lock(&vif->lock); |
| 2008 | if (--vif->stalled_queues == 0) { | ||
| 2009 | netdev_info(vif->dev, "Guest Rx ready"); | ||
| 2010 | netif_carrier_on(vif->dev); | ||
| 2011 | } | ||
| 2012 | spin_unlock(&vif->lock); | ||
| 2013 | } | ||
| 1993 | 2014 | ||
| 1994 | xenvif_napi_schedule_or_enable_events(temp); | 2015 | static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue) |
| 1995 | } | 2016 | { |
| 1996 | if (net_ratelimit()) | 2017 | RING_IDX prod, cons; |
| 1997 | netdev_err(queue->vif->dev, "Carrier on again\n"); | 2018 | |
| 1998 | } else { | 2019 | prod = queue->rx.sring->req_prod; |
| 1999 | /* Queuing were stopped, but the guest posted | 2020 | cons = queue->rx.req_cons; |
| 2000 | * new requests and sent an interrupt | 2021 | |
| 2001 | */ | 2022 | return !queue->stalled |
| 2002 | clear_bit(QUEUE_STATUS_RX_STALLED, | 2023 | && prod - cons < XEN_NETBK_RX_SLOTS_MAX |
| 2003 | &queue->status); | 2024 | && time_after(jiffies, |
| 2004 | del_timer_sync(&queue->rx_stalled); | 2025 | queue->last_rx_time + rx_stall_timeout_jiffies); |
| 2005 | xenvif_start_queue(queue); | 2026 | } |
| 2027 | |||
| 2028 | static bool xenvif_rx_queue_ready(struct xenvif_queue *queue) | ||
| 2029 | { | ||
| 2030 | RING_IDX prod, cons; | ||
| 2031 | |||
| 2032 | prod = queue->rx.sring->req_prod; | ||
| 2033 | cons = queue->rx.req_cons; | ||
| 2034 | |||
| 2035 | return queue->stalled | ||
| 2036 | && prod - cons >= XEN_NETBK_RX_SLOTS_MAX; | ||
| 2037 | } | ||
| 2038 | |||
| 2039 | static bool xenvif_have_rx_work(struct xenvif_queue *queue) | ||
| 2040 | { | ||
| 2041 | return (!skb_queue_empty(&queue->rx_queue) | ||
| 2042 | && xenvif_rx_ring_slots_available(queue, XEN_NETBK_RX_SLOTS_MAX)) | ||
| 2043 | || xenvif_rx_queue_stalled(queue) | ||
| 2044 | || xenvif_rx_queue_ready(queue) | ||
| 2045 | || kthread_should_stop() | ||
| 2046 | || queue->vif->disabled; | ||
| 2047 | } | ||
| 2048 | |||
| 2049 | static long xenvif_rx_queue_timeout(struct xenvif_queue *queue) | ||
| 2050 | { | ||
| 2051 | struct sk_buff *skb; | ||
| 2052 | long timeout; | ||
| 2053 | |||
| 2054 | skb = skb_peek(&queue->rx_queue); | ||
| 2055 | if (!skb) | ||
| 2056 | return MAX_SCHEDULE_TIMEOUT; | ||
| 2057 | |||
| 2058 | timeout = XENVIF_RX_CB(skb)->expires - jiffies; | ||
| 2059 | return timeout < 0 ? 0 : timeout; | ||
| 2060 | } | ||
| 2061 | |||
| 2062 | /* Wait until the guest Rx thread has work. | ||
| 2063 | * | ||
| 2064 | * The timeout needs to be adjusted based on the current head of the | ||
| 2065 | * queue (and not just the head at the beginning). In particular, if | ||
| 2066 | * the queue is initially empty an infinite timeout is used and this | ||
| 2067 | * needs to be reduced when a skb is queued. | ||
| 2068 | * | ||
| 2069 | * This cannot be done with wait_event_timeout() because it only | ||
| 2070 | * calculates the timeout once. | ||
| 2071 | */ | ||
| 2072 | static void xenvif_wait_for_rx_work(struct xenvif_queue *queue) | ||
| 2073 | { | ||
| 2074 | DEFINE_WAIT(wait); | ||
| 2075 | |||
| 2076 | if (xenvif_have_rx_work(queue)) | ||
| 2077 | return; | ||
| 2078 | |||
| 2079 | for (;;) { | ||
| 2080 | long ret; | ||
| 2081 | |||
| 2082 | prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE); | ||
| 2083 | if (xenvif_have_rx_work(queue)) | ||
| 2084 | break; | ||
| 2085 | ret = schedule_timeout(xenvif_rx_queue_timeout(queue)); | ||
| 2086 | if (!ret) | ||
| 2087 | break; | ||
| 2006 | } | 2088 | } |
| 2089 | finish_wait(&queue->wq, &wait); | ||
| 2007 | } | 2090 | } |
| 2008 | 2091 | ||
| 2009 | int xenvif_kthread_guest_rx(void *data) | 2092 | int xenvif_kthread_guest_rx(void *data) |
| 2010 | { | 2093 | { |
| 2011 | struct xenvif_queue *queue = data; | 2094 | struct xenvif_queue *queue = data; |
| 2012 | struct sk_buff *skb; | 2095 | struct xenvif *vif = queue->vif; |
| 2013 | 2096 | ||
| 2014 | while (!kthread_should_stop()) { | 2097 | for (;;) { |
| 2015 | wait_event_interruptible(queue->wq, | 2098 | xenvif_wait_for_rx_work(queue); |
| 2016 | rx_work_todo(queue) || | ||
| 2017 | queue->vif->disabled || | ||
| 2018 | test_bit(QUEUE_STATUS_RX_PURGE_EVENT, &queue->status) || | ||
| 2019 | kthread_should_stop()); | ||
| 2020 | 2099 | ||
| 2021 | if (kthread_should_stop()) | 2100 | if (kthread_should_stop()) |
| 2022 | break; | 2101 | break; |
| @@ -2028,35 +2107,38 @@ int xenvif_kthread_guest_rx(void *data) | |||
| 2028 | * context so we defer it here, if this thread is | 2107 | * context so we defer it here, if this thread is |
| 2029 | * associated with queue 0. | 2108 | * associated with queue 0. |
| 2030 | */ | 2109 | */ |
| 2031 | if (unlikely(queue->vif->disabled && queue->id == 0)) { | 2110 | if (unlikely(vif->disabled && queue->id == 0)) { |
| 2032 | xenvif_carrier_off(queue->vif); | 2111 | xenvif_carrier_off(vif); |
| 2033 | } else if (unlikely(queue->vif->disabled)) { | 2112 | xenvif_rx_queue_purge(queue); |
| 2034 | /* kthread_stop() would be called upon this thread soon, | 2113 | continue; |
| 2035 | * be a bit proactive | ||
| 2036 | */ | ||
| 2037 | skb_queue_purge(&queue->rx_queue); | ||
| 2038 | queue->rx_last_skb_slots = 0; | ||
| 2039 | } else if (unlikely(test_and_clear_bit(QUEUE_STATUS_RX_PURGE_EVENT, | ||
| 2040 | &queue->status))) { | ||
| 2041 | xenvif_rx_purge_event(queue); | ||
| 2042 | } else if (!netif_carrier_ok(queue->vif->dev)) { | ||
| 2043 | /* Another queue stalled and turned the carrier off, so | ||
| 2044 | * purge the internal queue of queues which were not | ||
| 2045 | * blocked | ||
| 2046 | */ | ||
| 2047 | skb_queue_purge(&queue->rx_queue); | ||
| 2048 | queue->rx_last_skb_slots = 0; | ||
| 2049 | } | 2114 | } |
| 2050 | 2115 | ||
| 2051 | if (!skb_queue_empty(&queue->rx_queue)) | 2116 | if (!skb_queue_empty(&queue->rx_queue)) |
| 2052 | xenvif_rx_action(queue); | 2117 | xenvif_rx_action(queue); |
| 2053 | 2118 | ||
| 2119 | /* If the guest hasn't provided any Rx slots for a | ||
| 2120 | * while it's probably not responsive, drop the | ||
| 2121 | * carrier so packets are dropped earlier. | ||
| 2122 | */ | ||
| 2123 | if (xenvif_rx_queue_stalled(queue)) | ||
| 2124 | xenvif_queue_carrier_off(queue); | ||
| 2125 | else if (xenvif_rx_queue_ready(queue)) | ||
| 2126 | xenvif_queue_carrier_on(queue); | ||
| 2127 | |||
| 2128 | /* Queued packets may have foreign pages from other | ||
| 2129 | * domains. These cannot be queued indefinitely as | ||
| 2130 | * this would starve guests of grant refs and transmit | ||
| 2131 | * slots. | ||
| 2132 | */ | ||
| 2133 | xenvif_rx_queue_drop_expired(queue); | ||
| 2134 | |||
| 2135 | xenvif_rx_queue_maybe_wake(queue); | ||
| 2136 | |||
| 2054 | cond_resched(); | 2137 | cond_resched(); |
| 2055 | } | 2138 | } |
| 2056 | 2139 | ||
| 2057 | /* Bin any remaining skbs */ | 2140 | /* Bin any remaining skbs */ |
| 2058 | while ((skb = skb_dequeue(&queue->rx_queue)) != NULL) | 2141 | xenvif_rx_queue_purge(queue); |
| 2059 | dev_kfree_skb(skb); | ||
| 2060 | 2142 | ||
| 2061 | return 0; | 2143 | return 0; |
| 2062 | } | 2144 | } |
| @@ -2113,6 +2195,7 @@ static int __init netback_init(void) | |||
| 2113 | goto failed_init; | 2195 | goto failed_init; |
| 2114 | 2196 | ||
| 2115 | rx_drain_timeout_jiffies = msecs_to_jiffies(rx_drain_timeout_msecs); | 2197 | rx_drain_timeout_jiffies = msecs_to_jiffies(rx_drain_timeout_msecs); |
| 2198 | rx_stall_timeout_jiffies = msecs_to_jiffies(rx_stall_timeout_msecs); | ||
| 2116 | 2199 | ||
| 2117 | #ifdef CONFIG_DEBUG_FS | 2200 | #ifdef CONFIG_DEBUG_FS |
| 2118 | xen_netback_dbg_root = debugfs_create_dir("xen-netback", NULL); | 2201 | xen_netback_dbg_root = debugfs_create_dir("xen-netback", NULL); |
diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index 8079c31ac5e6..4e56a27f9689 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c | |||
| @@ -52,6 +52,7 @@ static int xenvif_read_io_ring(struct seq_file *m, void *v) | |||
| 52 | struct xenvif_queue *queue = m->private; | 52 | struct xenvif_queue *queue = m->private; |
| 53 | struct xen_netif_tx_back_ring *tx_ring = &queue->tx; | 53 | struct xen_netif_tx_back_ring *tx_ring = &queue->tx; |
| 54 | struct xen_netif_rx_back_ring *rx_ring = &queue->rx; | 54 | struct xen_netif_rx_back_ring *rx_ring = &queue->rx; |
| 55 | struct netdev_queue *dev_queue; | ||
| 55 | 56 | ||
| 56 | if (tx_ring->sring) { | 57 | if (tx_ring->sring) { |
| 57 | struct xen_netif_tx_sring *sring = tx_ring->sring; | 58 | struct xen_netif_tx_sring *sring = tx_ring->sring; |
| @@ -112,6 +113,13 @@ static int xenvif_read_io_ring(struct seq_file *m, void *v) | |||
| 112 | queue->credit_timeout.expires, | 113 | queue->credit_timeout.expires, |
| 113 | jiffies); | 114 | jiffies); |
| 114 | 115 | ||
| 116 | dev_queue = netdev_get_tx_queue(queue->vif->dev, queue->id); | ||
| 117 | |||
| 118 | seq_printf(m, "\nRx internal queue: len %u max %u pkts %u %s\n", | ||
| 119 | queue->rx_queue_len, queue->rx_queue_max, | ||
| 120 | skb_queue_len(&queue->rx_queue), | ||
| 121 | netif_tx_queue_stopped(dev_queue) ? "stopped" : "running"); | ||
| 122 | |||
| 115 | return 0; | 123 | return 0; |
| 116 | } | 124 | } |
| 117 | 125 | ||
| @@ -703,6 +711,7 @@ static void connect(struct backend_info *be) | |||
| 703 | be->vif->queues = vzalloc(requested_num_queues * | 711 | be->vif->queues = vzalloc(requested_num_queues * |
| 704 | sizeof(struct xenvif_queue)); | 712 | sizeof(struct xenvif_queue)); |
| 705 | be->vif->num_queues = requested_num_queues; | 713 | be->vif->num_queues = requested_num_queues; |
| 714 | be->vif->stalled_queues = requested_num_queues; | ||
| 706 | 715 | ||
| 707 | for (queue_index = 0; queue_index < requested_num_queues; ++queue_index) { | 716 | for (queue_index = 0; queue_index < requested_num_queues; ++queue_index) { |
| 708 | queue = &be->vif->queues[queue_index]; | 717 | queue = &be->vif->queues[queue_index]; |
| @@ -873,15 +882,10 @@ static int read_xenbus_vif_flags(struct backend_info *be) | |||
| 873 | if (!rx_copy) | 882 | if (!rx_copy) |
| 874 | return -EOPNOTSUPP; | 883 | return -EOPNOTSUPP; |
| 875 | 884 | ||
| 876 | if (vif->dev->tx_queue_len != 0) { | 885 | if (xenbus_scanf(XBT_NIL, dev->otherend, |
| 877 | if (xenbus_scanf(XBT_NIL, dev->otherend, | 886 | "feature-rx-notify", "%d", &val) < 0 || val == 0) { |
| 878 | "feature-rx-notify", "%d", &val) < 0) | 887 | xenbus_dev_fatal(dev, -EINVAL, "feature-rx-notify is mandatory"); |
| 879 | val = 0; | 888 | return -EINVAL; |
| 880 | if (val) | ||
| 881 | vif->can_queue = 1; | ||
| 882 | else | ||
| 883 | /* Must be non-zero for pfifo_fast to work. */ | ||
| 884 | vif->dev->tx_queue_len = 1; | ||
| 885 | } | 889 | } |
| 886 | 890 | ||
| 887 | if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg", | 891 | if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg", |
