aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Durrant <Paul.Durrant@citrix.com>2016-10-04 05:29:12 -0400
committerDavid S. Miller <davem@davemloft.net>2016-10-06 20:37:35 -0400
commit3254f83694fe519ac18b8334a2f481d80c3a8a3a (patch)
treedd0bf4b1a2f62b0ed62ccc005821728b7cd9bd5f
parent00c06ed779bee562603b28b088d7a8143dd2d94f (diff)
xen-netback: separate guest side rx code into separate module
The netback source module has become very large and somewhat confusing. This patch simply moves all code related to the backend to frontend (i.e guest side rx) data-path into a separate rx source module. This patch contains no functional change, it is code movement and minimal changes to avoid patch style-check issues. Signed-off-by: Paul Durrant <paul.durrant@citrix.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--drivers/net/xen-netback/Makefile2
-rw-r--r--drivers/net/xen-netback/netback.c754
-rw-r--r--drivers/net/xen-netback/rx.c789
3 files changed, 790 insertions, 755 deletions
diff --git a/drivers/net/xen-netback/Makefile b/drivers/net/xen-netback/Makefile
index 11e02be9db1a..d49798a46b51 100644
--- a/drivers/net/xen-netback/Makefile
+++ b/drivers/net/xen-netback/Makefile
@@ -1,3 +1,3 @@
1obj-$(CONFIG_XEN_NETDEV_BACKEND) := xen-netback.o 1obj-$(CONFIG_XEN_NETDEV_BACKEND) := xen-netback.o
2 2
3xen-netback-y := netback.o xenbus.o interface.o hash.o 3xen-netback-y := netback.o xenbus.o interface.o hash.o rx.o
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 3d0c989384b5..47b481095d77 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -106,13 +106,6 @@ static void push_tx_responses(struct xenvif_queue *queue);
106 106
107static inline int tx_work_todo(struct xenvif_queue *queue); 107static inline int tx_work_todo(struct xenvif_queue *queue);
108 108
109static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
110 u16 id,
111 s8 st,
112 u16 offset,
113 u16 size,
114 u16 flags);
115
116static inline unsigned long idx_to_pfn(struct xenvif_queue *queue, 109static inline unsigned long idx_to_pfn(struct xenvif_queue *queue,
117 u16 idx) 110 u16 idx)
118{ 111{
@@ -155,571 +148,11 @@ static inline pending_ring_idx_t pending_index(unsigned i)
155 return i & (MAX_PENDING_REQS-1); 148 return i & (MAX_PENDING_REQS-1);
156} 149}
157 150
158static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
159{
160 RING_IDX prod, cons;
161 struct sk_buff *skb;
162 int needed;
163
164 skb = skb_peek(&queue->rx_queue);
165 if (!skb)
166 return false;
167
168 needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
169 if (skb_is_gso(skb))
170 needed++;
171 if (skb->sw_hash)
172 needed++;
173
174 do {
175 prod = queue->rx.sring->req_prod;
176 cons = queue->rx.req_cons;
177
178 if (prod - cons >= needed)
179 return true;
180
181 queue->rx.sring->req_event = prod + 1;
182
183 /* Make sure event is visible before we check prod
184 * again.
185 */
186 mb();
187 } while (queue->rx.sring->req_prod != prod);
188
189 return false;
190}
191
192void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb)
193{
194 unsigned long flags;
195
196 spin_lock_irqsave(&queue->rx_queue.lock, flags);
197
198 __skb_queue_tail(&queue->rx_queue, skb);
199
200 queue->rx_queue_len += skb->len;
201 if (queue->rx_queue_len > queue->rx_queue_max)
202 netif_tx_stop_queue(netdev_get_tx_queue(queue->vif->dev, queue->id));
203
204 spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
205}
206
207static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue)
208{
209 struct sk_buff *skb;
210
211 spin_lock_irq(&queue->rx_queue.lock);
212
213 skb = __skb_dequeue(&queue->rx_queue);
214 if (skb)
215 queue->rx_queue_len -= skb->len;
216
217 spin_unlock_irq(&queue->rx_queue.lock);
218
219 return skb;
220}
221
222static void xenvif_rx_queue_maybe_wake(struct xenvif_queue *queue)
223{
224 spin_lock_irq(&queue->rx_queue.lock);
225
226 if (queue->rx_queue_len < queue->rx_queue_max)
227 netif_tx_wake_queue(netdev_get_tx_queue(queue->vif->dev, queue->id));
228
229 spin_unlock_irq(&queue->rx_queue.lock);
230}
231
232
233static void xenvif_rx_queue_purge(struct xenvif_queue *queue)
234{
235 struct sk_buff *skb;
236 while ((skb = xenvif_rx_dequeue(queue)) != NULL)
237 kfree_skb(skb);
238}
239
240static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)
241{
242 struct sk_buff *skb;
243
244 for(;;) {
245 skb = skb_peek(&queue->rx_queue);
246 if (!skb)
247 break;
248 if (time_before(jiffies, XENVIF_RX_CB(skb)->expires))
249 break;
250 xenvif_rx_dequeue(queue);
251 kfree_skb(skb);
252 }
253}
254
255struct netrx_pending_operations {
256 unsigned copy_prod, copy_cons;
257 unsigned meta_prod, meta_cons;
258 struct gnttab_copy *copy;
259 struct xenvif_rx_meta *meta;
260 int copy_off;
261 grant_ref_t copy_gref;
262};
263
264static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue,
265 struct netrx_pending_operations *npo)
266{
267 struct xenvif_rx_meta *meta;
268 struct xen_netif_rx_request req;
269
270 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
271
272 meta = npo->meta + npo->meta_prod++;
273 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
274 meta->gso_size = 0;
275 meta->size = 0;
276 meta->id = req.id;
277
278 npo->copy_off = 0;
279 npo->copy_gref = req.gref;
280
281 return meta;
282}
283
284struct gop_frag_copy {
285 struct xenvif_queue *queue;
286 struct netrx_pending_operations *npo;
287 struct xenvif_rx_meta *meta;
288 int head;
289 int gso_type;
290 int protocol;
291 int hash_present;
292
293 struct page *page;
294};
295
296static void xenvif_setup_copy_gop(unsigned long gfn,
297 unsigned int offset,
298 unsigned int *len,
299 struct gop_frag_copy *info)
300{
301 struct gnttab_copy *copy_gop;
302 struct xen_page_foreign *foreign;
303 /* Convenient aliases */
304 struct xenvif_queue *queue = info->queue;
305 struct netrx_pending_operations *npo = info->npo;
306 struct page *page = info->page;
307
308 BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
309
310 if (npo->copy_off == MAX_BUFFER_OFFSET)
311 info->meta = get_next_rx_buffer(queue, npo);
312
313 if (npo->copy_off + *len > MAX_BUFFER_OFFSET)
314 *len = MAX_BUFFER_OFFSET - npo->copy_off;
315
316 copy_gop = npo->copy + npo->copy_prod++;
317 copy_gop->flags = GNTCOPY_dest_gref;
318 copy_gop->len = *len;
319
320 foreign = xen_page_foreign(page);
321 if (foreign) {
322 copy_gop->source.domid = foreign->domid;
323 copy_gop->source.u.ref = foreign->gref;
324 copy_gop->flags |= GNTCOPY_source_gref;
325 } else {
326 copy_gop->source.domid = DOMID_SELF;
327 copy_gop->source.u.gmfn = gfn;
328 }
329 copy_gop->source.offset = offset;
330
331 copy_gop->dest.domid = queue->vif->domid;
332 copy_gop->dest.offset = npo->copy_off;
333 copy_gop->dest.u.ref = npo->copy_gref;
334
335 npo->copy_off += *len;
336 info->meta->size += *len;
337
338 if (!info->head)
339 return;
340
341 /* Leave a gap for the GSO descriptor. */
342 if ((1 << info->gso_type) & queue->vif->gso_mask)
343 queue->rx.req_cons++;
344
345 /* Leave a gap for the hash extra segment. */
346 if (info->hash_present)
347 queue->rx.req_cons++;
348
349 info->head = 0; /* There must be something in this buffer now */
350}
351
352static void xenvif_gop_frag_copy_grant(unsigned long gfn,
353 unsigned offset,
354 unsigned int len,
355 void *data)
356{
357 unsigned int bytes;
358
359 while (len) {
360 bytes = len;
361 xenvif_setup_copy_gop(gfn, offset, &bytes, data);
362 offset += bytes;
363 len -= bytes;
364 }
365}
366
367/*
368 * Set up the grant operations for this fragment. If it's a flipping
369 * interface, we also set up the unmap request from here.
370 */
371static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb,
372 struct netrx_pending_operations *npo,
373 struct page *page, unsigned long size,
374 unsigned long offset, int *head)
375{
376 struct gop_frag_copy info = {
377 .queue = queue,
378 .npo = npo,
379 .head = *head,
380 .gso_type = XEN_NETIF_GSO_TYPE_NONE,
381 /* xenvif_set_skb_hash() will have either set a s/w
382 * hash or cleared the hash depending on
383 * whether the the frontend wants a hash for this skb.
384 */
385 .hash_present = skb->sw_hash,
386 };
387 unsigned long bytes;
388
389 if (skb_is_gso(skb)) {
390 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
391 info.gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
392 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
393 info.gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
394 }
395
396 /* Data must not cross a page boundary. */
397 BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
398
399 info.meta = npo->meta + npo->meta_prod - 1;
400
401 /* Skip unused frames from start of page */
402 page += offset >> PAGE_SHIFT;
403 offset &= ~PAGE_MASK;
404
405 while (size > 0) {
406 BUG_ON(offset >= PAGE_SIZE);
407
408 bytes = PAGE_SIZE - offset;
409 if (bytes > size)
410 bytes = size;
411
412 info.page = page;
413 gnttab_foreach_grant_in_range(page, offset, bytes,
414 xenvif_gop_frag_copy_grant,
415 &info);
416 size -= bytes;
417 offset = 0;
418
419 /* Next page */
420 if (size) {
421 BUG_ON(!PageCompound(page));
422 page++;
423 }
424 }
425
426 *head = info.head;
427}
428
429/*
430 * Prepare an SKB to be transmitted to the frontend.
431 *
432 * This function is responsible for allocating grant operations, meta
433 * structures, etc.
434 *
435 * It returns the number of meta structures consumed. The number of
436 * ring slots used is always equal to the number of meta slots used
437 * plus the number of GSO descriptors used. Currently, we use either
438 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
439 * frontend-side LRO).
440 */
441static int xenvif_gop_skb(struct sk_buff *skb,
442 struct netrx_pending_operations *npo,
443 struct xenvif_queue *queue)
444{
445 struct xenvif *vif = netdev_priv(skb->dev);
446 int nr_frags = skb_shinfo(skb)->nr_frags;
447 int i;
448 struct xen_netif_rx_request req;
449 struct xenvif_rx_meta *meta;
450 unsigned char *data;
451 int head = 1;
452 int old_meta_prod;
453 int gso_type;
454
455 old_meta_prod = npo->meta_prod;
456
457 gso_type = XEN_NETIF_GSO_TYPE_NONE;
458 if (skb_is_gso(skb)) {
459 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
460 gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
461 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
462 gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
463 }
464
465 /* Set up a GSO prefix descriptor, if necessary */
466 if ((1 << gso_type) & vif->gso_prefix_mask) {
467 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
468 meta = npo->meta + npo->meta_prod++;
469 meta->gso_type = gso_type;
470 meta->gso_size = skb_shinfo(skb)->gso_size;
471 meta->size = 0;
472 meta->id = req.id;
473 }
474
475 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
476 meta = npo->meta + npo->meta_prod++;
477
478 if ((1 << gso_type) & vif->gso_mask) {
479 meta->gso_type = gso_type;
480 meta->gso_size = skb_shinfo(skb)->gso_size;
481 } else {
482 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
483 meta->gso_size = 0;
484 }
485
486 meta->size = 0;
487 meta->id = req.id;
488 npo->copy_off = 0;
489 npo->copy_gref = req.gref;
490
491 data = skb->data;
492 while (data < skb_tail_pointer(skb)) {
493 unsigned int offset = offset_in_page(data);
494 unsigned int len = PAGE_SIZE - offset;
495
496 if (data + len > skb_tail_pointer(skb))
497 len = skb_tail_pointer(skb) - data;
498
499 xenvif_gop_frag_copy(queue, skb, npo,
500 virt_to_page(data), len, offset, &head);
501 data += len;
502 }
503
504 for (i = 0; i < nr_frags; i++) {
505 xenvif_gop_frag_copy(queue, skb, npo,
506 skb_frag_page(&skb_shinfo(skb)->frags[i]),
507 skb_frag_size(&skb_shinfo(skb)->frags[i]),
508 skb_shinfo(skb)->frags[i].page_offset,
509 &head);
510 }
511
512 return npo->meta_prod - old_meta_prod;
513}
514
515/*
516 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
517 * used to set up the operations on the top of
518 * netrx_pending_operations, which have since been done. Check that
519 * they didn't give any errors and advance over them.
520 */
521static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
522 struct netrx_pending_operations *npo)
523{
524 struct gnttab_copy *copy_op;
525 int status = XEN_NETIF_RSP_OKAY;
526 int i;
527
528 for (i = 0; i < nr_meta_slots; i++) {
529 copy_op = npo->copy + npo->copy_cons++;
530 if (copy_op->status != GNTST_okay) {
531 netdev_dbg(vif->dev,
532 "Bad status %d from copy to DOM%d.\n",
533 copy_op->status, vif->domid);
534 status = XEN_NETIF_RSP_ERROR;
535 }
536 }
537
538 return status;
539}
540
541static void xenvif_add_frag_responses(struct xenvif_queue *queue, int status,
542 struct xenvif_rx_meta *meta,
543 int nr_meta_slots)
544{
545 int i;
546 unsigned long offset;
547
548 /* No fragments used */
549 if (nr_meta_slots <= 1)
550 return;
551
552 nr_meta_slots--;
553
554 for (i = 0; i < nr_meta_slots; i++) {
555 int flags;
556 if (i == nr_meta_slots - 1)
557 flags = 0;
558 else
559 flags = XEN_NETRXF_more_data;
560
561 offset = 0;
562 make_rx_response(queue, meta[i].id, status, offset,
563 meta[i].size, flags);
564 }
565}
566
567void xenvif_kick_thread(struct xenvif_queue *queue) 151void xenvif_kick_thread(struct xenvif_queue *queue)
568{ 152{
569 wake_up(&queue->wq); 153 wake_up(&queue->wq);
570} 154}
571 155
572static void xenvif_rx_action(struct xenvif_queue *queue)
573{
574 struct xenvif *vif = queue->vif;
575 s8 status;
576 u16 flags;
577 struct xen_netif_rx_response *resp;
578 struct sk_buff_head rxq;
579 struct sk_buff *skb;
580 LIST_HEAD(notify);
581 int ret;
582 unsigned long offset;
583 bool need_to_notify = false;
584
585 struct netrx_pending_operations npo = {
586 .copy = queue->grant_copy_op,
587 .meta = queue->meta,
588 };
589
590 skb_queue_head_init(&rxq);
591
592 while (xenvif_rx_ring_slots_available(queue)
593 && (skb = xenvif_rx_dequeue(queue)) != NULL) {
594 queue->last_rx_time = jiffies;
595
596 XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo, queue);
597
598 __skb_queue_tail(&rxq, skb);
599 }
600
601 BUG_ON(npo.meta_prod > ARRAY_SIZE(queue->meta));
602
603 if (!npo.copy_prod)
604 goto done;
605
606 BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS);
607 gnttab_batch_copy(queue->grant_copy_op, npo.copy_prod);
608
609 while ((skb = __skb_dequeue(&rxq)) != NULL) {
610 struct xen_netif_extra_info *extra = NULL;
611
612 if ((1 << queue->meta[npo.meta_cons].gso_type) &
613 vif->gso_prefix_mask) {
614 resp = RING_GET_RESPONSE(&queue->rx,
615 queue->rx.rsp_prod_pvt++);
616
617 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;
618
619 resp->offset = queue->meta[npo.meta_cons].gso_size;
620 resp->id = queue->meta[npo.meta_cons].id;
621 resp->status = XENVIF_RX_CB(skb)->meta_slots_used;
622
623 npo.meta_cons++;
624 XENVIF_RX_CB(skb)->meta_slots_used--;
625 }
626
627
628 queue->stats.tx_bytes += skb->len;
629 queue->stats.tx_packets++;
630
631 status = xenvif_check_gop(vif,
632 XENVIF_RX_CB(skb)->meta_slots_used,
633 &npo);
634
635 if (XENVIF_RX_CB(skb)->meta_slots_used == 1)
636 flags = 0;
637 else
638 flags = XEN_NETRXF_more_data;
639
640 if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */
641 flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated;
642 else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
643 /* remote but checksummed. */
644 flags |= XEN_NETRXF_data_validated;
645
646 offset = 0;
647 resp = make_rx_response(queue, queue->meta[npo.meta_cons].id,
648 status, offset,
649 queue->meta[npo.meta_cons].size,
650 flags);
651
652 if ((1 << queue->meta[npo.meta_cons].gso_type) &
653 vif->gso_mask) {
654 extra = (struct xen_netif_extra_info *)
655 RING_GET_RESPONSE(&queue->rx,
656 queue->rx.rsp_prod_pvt++);
657
658 resp->flags |= XEN_NETRXF_extra_info;
659
660 extra->u.gso.type = queue->meta[npo.meta_cons].gso_type;
661 extra->u.gso.size = queue->meta[npo.meta_cons].gso_size;
662 extra->u.gso.pad = 0;
663 extra->u.gso.features = 0;
664
665 extra->type = XEN_NETIF_EXTRA_TYPE_GSO;
666 extra->flags = 0;
667 }
668
669 if (skb->sw_hash) {
670 /* Since the skb got here via xenvif_select_queue()
671 * we know that the hash has been re-calculated
672 * according to a configuration set by the frontend
673 * and therefore we know that it is legitimate to
674 * pass it to the frontend.
675 */
676 if (resp->flags & XEN_NETRXF_extra_info)
677 extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE;
678 else
679 resp->flags |= XEN_NETRXF_extra_info;
680
681 extra = (struct xen_netif_extra_info *)
682 RING_GET_RESPONSE(&queue->rx,
683 queue->rx.rsp_prod_pvt++);
684
685 extra->u.hash.algorithm =
686 XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ;
687
688 if (skb->l4_hash)
689 extra->u.hash.type =
690 skb->protocol == htons(ETH_P_IP) ?
691 _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP :
692 _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
693 else
694 extra->u.hash.type =
695 skb->protocol == htons(ETH_P_IP) ?
696 _XEN_NETIF_CTRL_HASH_TYPE_IPV4 :
697 _XEN_NETIF_CTRL_HASH_TYPE_IPV6;
698
699 *(uint32_t *)extra->u.hash.value =
700 skb_get_hash_raw(skb);
701
702 extra->type = XEN_NETIF_EXTRA_TYPE_HASH;
703 extra->flags = 0;
704 }
705
706 xenvif_add_frag_responses(queue, status,
707 queue->meta + npo.meta_cons + 1,
708 XENVIF_RX_CB(skb)->meta_slots_used);
709
710 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, ret);
711
712 need_to_notify |= !!ret;
713
714 npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used;
715 dev_kfree_skb(skb);
716 }
717
718done:
719 if (need_to_notify)
720 notify_remote_via_irq(queue->rx_irq);
721}
722
723void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue) 156void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue)
724{ 157{
725 int more_to_do; 158 int more_to_do;
@@ -1951,29 +1384,6 @@ static void push_tx_responses(struct xenvif_queue *queue)
1951 notify_remote_via_irq(queue->tx_irq); 1384 notify_remote_via_irq(queue->tx_irq);
1952} 1385}
1953 1386
1954static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
1955 u16 id,
1956 s8 st,
1957 u16 offset,
1958 u16 size,
1959 u16 flags)
1960{
1961 RING_IDX i = queue->rx.rsp_prod_pvt;
1962 struct xen_netif_rx_response *resp;
1963
1964 resp = RING_GET_RESPONSE(&queue->rx, i);
1965 resp->offset = offset;
1966 resp->flags = flags;
1967 resp->id = id;
1968 resp->status = (s16)size;
1969 if (st < 0)
1970 resp->status = (s16)st;
1971
1972 queue->rx.rsp_prod_pvt = ++i;
1973
1974 return resp;
1975}
1976
1977void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx) 1387void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx)
1978{ 1388{
1979 int ret; 1389 int ret;
@@ -2055,170 +1465,6 @@ err:
2055 return err; 1465 return err;
2056} 1466}
2057 1467
2058static void xenvif_queue_carrier_off(struct xenvif_queue *queue)
2059{
2060 struct xenvif *vif = queue->vif;
2061
2062 queue->stalled = true;
2063
2064 /* At least one queue has stalled? Disable the carrier. */
2065 spin_lock(&vif->lock);
2066 if (vif->stalled_queues++ == 0) {
2067 netdev_info(vif->dev, "Guest Rx stalled");
2068 netif_carrier_off(vif->dev);
2069 }
2070 spin_unlock(&vif->lock);
2071}
2072
2073static void xenvif_queue_carrier_on(struct xenvif_queue *queue)
2074{
2075 struct xenvif *vif = queue->vif;
2076
2077 queue->last_rx_time = jiffies; /* Reset Rx stall detection. */
2078 queue->stalled = false;
2079
2080 /* All queues are ready? Enable the carrier. */
2081 spin_lock(&vif->lock);
2082 if (--vif->stalled_queues == 0) {
2083 netdev_info(vif->dev, "Guest Rx ready");
2084 netif_carrier_on(vif->dev);
2085 }
2086 spin_unlock(&vif->lock);
2087}
2088
2089static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)
2090{
2091 RING_IDX prod, cons;
2092
2093 prod = queue->rx.sring->req_prod;
2094 cons = queue->rx.req_cons;
2095
2096 return !queue->stalled && prod - cons < 1
2097 && time_after(jiffies,
2098 queue->last_rx_time + queue->vif->stall_timeout);
2099}
2100
2101static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)
2102{
2103 RING_IDX prod, cons;
2104
2105 prod = queue->rx.sring->req_prod;
2106 cons = queue->rx.req_cons;
2107
2108 return queue->stalled && prod - cons >= 1;
2109}
2110
2111static bool xenvif_have_rx_work(struct xenvif_queue *queue)
2112{
2113 return xenvif_rx_ring_slots_available(queue)
2114 || (queue->vif->stall_timeout &&
2115 (xenvif_rx_queue_stalled(queue)
2116 || xenvif_rx_queue_ready(queue)))
2117 || kthread_should_stop()
2118 || queue->vif->disabled;
2119}
2120
2121static long xenvif_rx_queue_timeout(struct xenvif_queue *queue)
2122{
2123 struct sk_buff *skb;
2124 long timeout;
2125
2126 skb = skb_peek(&queue->rx_queue);
2127 if (!skb)
2128 return MAX_SCHEDULE_TIMEOUT;
2129
2130 timeout = XENVIF_RX_CB(skb)->expires - jiffies;
2131 return timeout < 0 ? 0 : timeout;
2132}
2133
2134/* Wait until the guest Rx thread has work.
2135 *
2136 * The timeout needs to be adjusted based on the current head of the
2137 * queue (and not just the head at the beginning). In particular, if
2138 * the queue is initially empty an infinite timeout is used and this
2139 * needs to be reduced when a skb is queued.
2140 *
2141 * This cannot be done with wait_event_timeout() because it only
2142 * calculates the timeout once.
2143 */
2144static void xenvif_wait_for_rx_work(struct xenvif_queue *queue)
2145{
2146 DEFINE_WAIT(wait);
2147
2148 if (xenvif_have_rx_work(queue))
2149 return;
2150
2151 for (;;) {
2152 long ret;
2153
2154 prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE);
2155 if (xenvif_have_rx_work(queue))
2156 break;
2157 ret = schedule_timeout(xenvif_rx_queue_timeout(queue));
2158 if (!ret)
2159 break;
2160 }
2161 finish_wait(&queue->wq, &wait);
2162}
2163
2164int xenvif_kthread_guest_rx(void *data)
2165{
2166 struct xenvif_queue *queue = data;
2167 struct xenvif *vif = queue->vif;
2168
2169 if (!vif->stall_timeout)
2170 xenvif_queue_carrier_on(queue);
2171
2172 for (;;) {
2173 xenvif_wait_for_rx_work(queue);
2174
2175 if (kthread_should_stop())
2176 break;
2177
2178 /* This frontend is found to be rogue, disable it in
2179 * kthread context. Currently this is only set when
2180 * netback finds out frontend sends malformed packet,
2181 * but we cannot disable the interface in softirq
2182 * context so we defer it here, if this thread is
2183 * associated with queue 0.
2184 */
2185 if (unlikely(vif->disabled && queue->id == 0)) {
2186 xenvif_carrier_off(vif);
2187 break;
2188 }
2189
2190 if (!skb_queue_empty(&queue->rx_queue))
2191 xenvif_rx_action(queue);
2192
2193 /* If the guest hasn't provided any Rx slots for a
2194 * while it's probably not responsive, drop the
2195 * carrier so packets are dropped earlier.
2196 */
2197 if (vif->stall_timeout) {
2198 if (xenvif_rx_queue_stalled(queue))
2199 xenvif_queue_carrier_off(queue);
2200 else if (xenvif_rx_queue_ready(queue))
2201 xenvif_queue_carrier_on(queue);
2202 }
2203
2204 /* Queued packets may have foreign pages from other
2205 * domains. These cannot be queued indefinitely as
2206 * this would starve guests of grant refs and transmit
2207 * slots.
2208 */
2209 xenvif_rx_queue_drop_expired(queue);
2210
2211 xenvif_rx_queue_maybe_wake(queue);
2212
2213 cond_resched();
2214 }
2215
2216 /* Bin any remaining skbs */
2217 xenvif_rx_queue_purge(queue);
2218
2219 return 0;
2220}
2221
2222static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue) 1468static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue)
2223{ 1469{
2224 /* Dealloc thread must remain running until all inflight 1470 /* Dealloc thread must remain running until all inflight
diff --git a/drivers/net/xen-netback/rx.c b/drivers/net/xen-netback/rx.c
new file mode 100644
index 000000000000..03836aaac1c2
--- /dev/null
+++ b/drivers/net/xen-netback/rx.c
@@ -0,0 +1,789 @@
1/*
2 * Copyright (c) 2016 Citrix Systems Inc.
3 * Copyright (c) 2002-2005, K A Fraser
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation; or, when distributed
8 * separately from the Linux kernel or incorporated into other
9 * software packages, subject to the following license:
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a copy
12 * of this source file (the "Software"), to deal in the Software without
13 * restriction, including without limitation the rights to use, copy, modify,
14 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
15 * and to permit persons to whom the Software is furnished to do so, subject to
16 * the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included in
19 * all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 * IN THE SOFTWARE.
28 */
29
30#include "common.h"
31
32#include <linux/kthread.h>
33
34#include <xen/xen.h>
35#include <xen/events.h>
36
37static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
38{
39 RING_IDX prod, cons;
40 struct sk_buff *skb;
41 int needed;
42
43 skb = skb_peek(&queue->rx_queue);
44 if (!skb)
45 return false;
46
47 needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
48 if (skb_is_gso(skb))
49 needed++;
50 if (skb->sw_hash)
51 needed++;
52
53 do {
54 prod = queue->rx.sring->req_prod;
55 cons = queue->rx.req_cons;
56
57 if (prod - cons >= needed)
58 return true;
59
60 queue->rx.sring->req_event = prod + 1;
61
62 /* Make sure event is visible before we check prod
63 * again.
64 */
65 mb();
66 } while (queue->rx.sring->req_prod != prod);
67
68 return false;
69}
70
71void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb)
72{
73 unsigned long flags;
74
75 spin_lock_irqsave(&queue->rx_queue.lock, flags);
76
77 __skb_queue_tail(&queue->rx_queue, skb);
78
79 queue->rx_queue_len += skb->len;
80 if (queue->rx_queue_len > queue->rx_queue_max) {
81 struct net_device *dev = queue->vif->dev;
82
83 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
84 }
85
86 spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
87}
88
89static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue)
90{
91 struct sk_buff *skb;
92
93 spin_lock_irq(&queue->rx_queue.lock);
94
95 skb = __skb_dequeue(&queue->rx_queue);
96 if (skb)
97 queue->rx_queue_len -= skb->len;
98
99 spin_unlock_irq(&queue->rx_queue.lock);
100
101 return skb;
102}
103
104static void xenvif_rx_queue_maybe_wake(struct xenvif_queue *queue)
105{
106 spin_lock_irq(&queue->rx_queue.lock);
107
108 if (queue->rx_queue_len < queue->rx_queue_max) {
109 struct net_device *dev = queue->vif->dev;
110
111 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id));
112 }
113
114 spin_unlock_irq(&queue->rx_queue.lock);
115}
116
117static void xenvif_rx_queue_purge(struct xenvif_queue *queue)
118{
119 struct sk_buff *skb;
120
121 while ((skb = xenvif_rx_dequeue(queue)) != NULL)
122 kfree_skb(skb);
123}
124
125static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)
126{
127 struct sk_buff *skb;
128
129 for (;;) {
130 skb = skb_peek(&queue->rx_queue);
131 if (!skb)
132 break;
133 if (time_before(jiffies, XENVIF_RX_CB(skb)->expires))
134 break;
135 xenvif_rx_dequeue(queue);
136 kfree_skb(skb);
137 }
138}
139
140struct netrx_pending_operations {
141 unsigned int copy_prod, copy_cons;
142 unsigned int meta_prod, meta_cons;
143 struct gnttab_copy *copy;
144 struct xenvif_rx_meta *meta;
145 int copy_off;
146 grant_ref_t copy_gref;
147};
148
149static struct xenvif_rx_meta *get_next_rx_buffer(
150 struct xenvif_queue *queue,
151 struct netrx_pending_operations *npo)
152{
153 struct xenvif_rx_meta *meta;
154 struct xen_netif_rx_request req;
155
156 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
157
158 meta = npo->meta + npo->meta_prod++;
159 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
160 meta->gso_size = 0;
161 meta->size = 0;
162 meta->id = req.id;
163
164 npo->copy_off = 0;
165 npo->copy_gref = req.gref;
166
167 return meta;
168}
169
170struct gop_frag_copy {
171 struct xenvif_queue *queue;
172 struct netrx_pending_operations *npo;
173 struct xenvif_rx_meta *meta;
174 int head;
175 int gso_type;
176 int protocol;
177 int hash_present;
178
179 struct page *page;
180};
181
182static void xenvif_setup_copy_gop(unsigned long gfn,
183 unsigned int offset,
184 unsigned int *len,
185 struct gop_frag_copy *info)
186{
187 struct gnttab_copy *copy_gop;
188 struct xen_page_foreign *foreign;
189 /* Convenient aliases */
190 struct xenvif_queue *queue = info->queue;
191 struct netrx_pending_operations *npo = info->npo;
192 struct page *page = info->page;
193
194 WARN_ON(npo->copy_off > MAX_BUFFER_OFFSET);
195
196 if (npo->copy_off == MAX_BUFFER_OFFSET)
197 info->meta = get_next_rx_buffer(queue, npo);
198
199 if (npo->copy_off + *len > MAX_BUFFER_OFFSET)
200 *len = MAX_BUFFER_OFFSET - npo->copy_off;
201
202 copy_gop = npo->copy + npo->copy_prod++;
203 copy_gop->flags = GNTCOPY_dest_gref;
204 copy_gop->len = *len;
205
206 foreign = xen_page_foreign(page);
207 if (foreign) {
208 copy_gop->source.domid = foreign->domid;
209 copy_gop->source.u.ref = foreign->gref;
210 copy_gop->flags |= GNTCOPY_source_gref;
211 } else {
212 copy_gop->source.domid = DOMID_SELF;
213 copy_gop->source.u.gmfn = gfn;
214 }
215 copy_gop->source.offset = offset;
216
217 copy_gop->dest.domid = queue->vif->domid;
218 copy_gop->dest.offset = npo->copy_off;
219 copy_gop->dest.u.ref = npo->copy_gref;
220
221 npo->copy_off += *len;
222 info->meta->size += *len;
223
224 if (!info->head)
225 return;
226
227 /* Leave a gap for the GSO descriptor. */
228 if ((1 << info->gso_type) & queue->vif->gso_mask)
229 queue->rx.req_cons++;
230
231 /* Leave a gap for the hash extra segment. */
232 if (info->hash_present)
233 queue->rx.req_cons++;
234
235 info->head = 0; /* There must be something in this buffer now */
236}
237
238static void xenvif_gop_frag_copy_grant(unsigned long gfn,
239 unsigned int offset,
240 unsigned int len,
241 void *data)
242{
243 unsigned int bytes;
244
245 while (len) {
246 bytes = len;
247 xenvif_setup_copy_gop(gfn, offset, &bytes, data);
248 offset += bytes;
249 len -= bytes;
250 }
251}
252
253/* Set up the grant operations for this fragment. If it's a flipping
254 * interface, we also set up the unmap request from here.
255 */
256static void xenvif_gop_frag_copy(struct xenvif_queue *queue,
257 struct sk_buff *skb,
258 struct netrx_pending_operations *npo,
259 struct page *page, unsigned long size,
260 unsigned long offset, int *head)
261{
262 struct gop_frag_copy info = {
263 .queue = queue,
264 .npo = npo,
265 .head = *head,
266 .gso_type = XEN_NETIF_GSO_TYPE_NONE,
267 /* xenvif_set_skb_hash() will have either set a s/w
268 * hash or cleared the hash depending on
269 * whether the the frontend wants a hash for this skb.
270 */
271 .hash_present = skb->sw_hash,
272 };
273 unsigned long bytes;
274
275 if (skb_is_gso(skb)) {
276 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
277 info.gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
278 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
279 info.gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
280 }
281
282 /* Data must not cross a page boundary. */
283 WARN_ON(size + offset > (PAGE_SIZE << compound_order(page)));
284
285 info.meta = npo->meta + npo->meta_prod - 1;
286
287 /* Skip unused frames from start of page */
288 page += offset >> PAGE_SHIFT;
289 offset &= ~PAGE_MASK;
290
291 while (size > 0) {
292 WARN_ON(offset >= PAGE_SIZE);
293
294 bytes = PAGE_SIZE - offset;
295 if (bytes > size)
296 bytes = size;
297
298 info.page = page;
299 gnttab_foreach_grant_in_range(page, offset, bytes,
300 xenvif_gop_frag_copy_grant,
301 &info);
302 size -= bytes;
303 offset = 0;
304
305 /* Next page */
306 if (size) {
307 WARN_ON(!PageCompound(page));
308 page++;
309 }
310 }
311
312 *head = info.head;
313}
314
315/* Prepare an SKB to be transmitted to the frontend.
316 *
317 * This function is responsible for allocating grant operations, meta
318 * structures, etc.
319 *
320 * It returns the number of meta structures consumed. The number of
321 * ring slots used is always equal to the number of meta slots used
322 * plus the number of GSO descriptors used. Currently, we use either
323 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
324 * frontend-side LRO).
325 */
326static int xenvif_gop_skb(struct sk_buff *skb,
327 struct netrx_pending_operations *npo,
328 struct xenvif_queue *queue)
329{
330 struct xenvif *vif = netdev_priv(skb->dev);
331 int nr_frags = skb_shinfo(skb)->nr_frags;
332 int i;
333 struct xen_netif_rx_request req;
334 struct xenvif_rx_meta *meta;
335 unsigned char *data;
336 int head = 1;
337 int old_meta_prod;
338 int gso_type;
339
340 old_meta_prod = npo->meta_prod;
341
342 gso_type = XEN_NETIF_GSO_TYPE_NONE;
343 if (skb_is_gso(skb)) {
344 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
345 gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
346 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
347 gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
348 }
349
350 /* Set up a GSO prefix descriptor, if necessary */
351 if ((1 << gso_type) & vif->gso_prefix_mask) {
352 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
353 meta = npo->meta + npo->meta_prod++;
354 meta->gso_type = gso_type;
355 meta->gso_size = skb_shinfo(skb)->gso_size;
356 meta->size = 0;
357 meta->id = req.id;
358 }
359
360 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
361 meta = npo->meta + npo->meta_prod++;
362
363 if ((1 << gso_type) & vif->gso_mask) {
364 meta->gso_type = gso_type;
365 meta->gso_size = skb_shinfo(skb)->gso_size;
366 } else {
367 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
368 meta->gso_size = 0;
369 }
370
371 meta->size = 0;
372 meta->id = req.id;
373 npo->copy_off = 0;
374 npo->copy_gref = req.gref;
375
376 data = skb->data;
377 while (data < skb_tail_pointer(skb)) {
378 unsigned int offset = offset_in_page(data);
379 unsigned int len = PAGE_SIZE - offset;
380
381 if (data + len > skb_tail_pointer(skb))
382 len = skb_tail_pointer(skb) - data;
383
384 xenvif_gop_frag_copy(queue, skb, npo,
385 virt_to_page(data), len, offset, &head);
386 data += len;
387 }
388
389 for (i = 0; i < nr_frags; i++) {
390 xenvif_gop_frag_copy(queue, skb, npo,
391 skb_frag_page(&skb_shinfo(skb)->frags[i]),
392 skb_frag_size(&skb_shinfo(skb)->frags[i]),
393 skb_shinfo(skb)->frags[i].page_offset,
394 &head);
395 }
396
397 return npo->meta_prod - old_meta_prod;
398}
399
400/* This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
401 * used to set up the operations on the top of
402 * netrx_pending_operations, which have since been done. Check that
403 * they didn't give any errors and advance over them.
404 */
405static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
406 struct netrx_pending_operations *npo)
407{
408 struct gnttab_copy *copy_op;
409 int status = XEN_NETIF_RSP_OKAY;
410 int i;
411
412 for (i = 0; i < nr_meta_slots; i++) {
413 copy_op = npo->copy + npo->copy_cons++;
414 if (copy_op->status != GNTST_okay) {
415 netdev_dbg(vif->dev,
416 "Bad status %d from copy to DOM%d.\n",
417 copy_op->status, vif->domid);
418 status = XEN_NETIF_RSP_ERROR;
419 }
420 }
421
422 return status;
423}
424
425static struct xen_netif_rx_response *make_rx_response(
426 struct xenvif_queue *queue, u16 id, s8 st, u16 offset, u16 size,
427 u16 flags)
428{
429 RING_IDX i = queue->rx.rsp_prod_pvt;
430 struct xen_netif_rx_response *resp;
431
432 resp = RING_GET_RESPONSE(&queue->rx, i);
433 resp->offset = offset;
434 resp->flags = flags;
435 resp->id = id;
436 resp->status = (s16)size;
437 if (st < 0)
438 resp->status = (s16)st;
439
440 queue->rx.rsp_prod_pvt = ++i;
441
442 return resp;
443}
444
445static void xenvif_add_frag_responses(struct xenvif_queue *queue,
446 int status,
447 struct xenvif_rx_meta *meta,
448 int nr_meta_slots)
449{
450 int i;
451 unsigned long offset;
452
453 /* No fragments used */
454 if (nr_meta_slots <= 1)
455 return;
456
457 nr_meta_slots--;
458
459 for (i = 0; i < nr_meta_slots; i++) {
460 int flags;
461
462 if (i == nr_meta_slots - 1)
463 flags = 0;
464 else
465 flags = XEN_NETRXF_more_data;
466
467 offset = 0;
468 make_rx_response(queue, meta[i].id, status, offset,
469 meta[i].size, flags);
470 }
471}
472
473static void xenvif_rx_action(struct xenvif_queue *queue)
474{
475 struct xenvif *vif = queue->vif;
476 s8 status;
477 u16 flags;
478 struct xen_netif_rx_response *resp;
479 struct sk_buff_head rxq;
480 struct sk_buff *skb;
481 LIST_HEAD(notify);
482 int ret;
483 unsigned long offset;
484 bool need_to_notify = false;
485
486 struct netrx_pending_operations npo = {
487 .copy = queue->grant_copy_op,
488 .meta = queue->meta,
489 };
490
491 skb_queue_head_init(&rxq);
492
493 while (xenvif_rx_ring_slots_available(queue) &&
494 (skb = xenvif_rx_dequeue(queue)) != NULL) {
495 queue->last_rx_time = jiffies;
496
497 XENVIF_RX_CB(skb)->meta_slots_used =
498 xenvif_gop_skb(skb, &npo, queue);
499
500 __skb_queue_tail(&rxq, skb);
501 }
502
503 WARN_ON(npo.meta_prod > ARRAY_SIZE(queue->meta));
504
505 if (!npo.copy_prod)
506 goto done;
507
508 WARN_ON(npo.copy_prod > MAX_GRANT_COPY_OPS);
509 gnttab_batch_copy(queue->grant_copy_op, npo.copy_prod);
510
511 while ((skb = __skb_dequeue(&rxq)) != NULL) {
512 struct xen_netif_extra_info *extra = NULL;
513
514 if ((1 << queue->meta[npo.meta_cons].gso_type) &
515 vif->gso_prefix_mask) {
516 resp = RING_GET_RESPONSE(&queue->rx,
517 queue->rx.rsp_prod_pvt++);
518
519 resp->flags = XEN_NETRXF_gso_prefix |
520 XEN_NETRXF_more_data;
521
522 resp->offset = queue->meta[npo.meta_cons].gso_size;
523 resp->id = queue->meta[npo.meta_cons].id;
524 resp->status = XENVIF_RX_CB(skb)->meta_slots_used;
525
526 npo.meta_cons++;
527 XENVIF_RX_CB(skb)->meta_slots_used--;
528 }
529
530 queue->stats.tx_bytes += skb->len;
531 queue->stats.tx_packets++;
532
533 status = xenvif_check_gop(vif,
534 XENVIF_RX_CB(skb)->meta_slots_used,
535 &npo);
536
537 if (XENVIF_RX_CB(skb)->meta_slots_used == 1)
538 flags = 0;
539 else
540 flags = XEN_NETRXF_more_data;
541
542 if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */
543 flags |= XEN_NETRXF_csum_blank |
544 XEN_NETRXF_data_validated;
545 else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
546 /* remote but checksummed. */
547 flags |= XEN_NETRXF_data_validated;
548
549 offset = 0;
550 resp = make_rx_response(queue, queue->meta[npo.meta_cons].id,
551 status, offset,
552 queue->meta[npo.meta_cons].size,
553 flags);
554
555 if ((1 << queue->meta[npo.meta_cons].gso_type) &
556 vif->gso_mask) {
557 extra = (struct xen_netif_extra_info *)
558 RING_GET_RESPONSE(&queue->rx,
559 queue->rx.rsp_prod_pvt++);
560
561 resp->flags |= XEN_NETRXF_extra_info;
562
563 extra->u.gso.type = queue->meta[npo.meta_cons].gso_type;
564 extra->u.gso.size = queue->meta[npo.meta_cons].gso_size;
565 extra->u.gso.pad = 0;
566 extra->u.gso.features = 0;
567
568 extra->type = XEN_NETIF_EXTRA_TYPE_GSO;
569 extra->flags = 0;
570 }
571
572 if (skb->sw_hash) {
573 /* Since the skb got here via xenvif_select_queue()
574 * we know that the hash has been re-calculated
575 * according to a configuration set by the frontend
576 * and therefore we know that it is legitimate to
577 * pass it to the frontend.
578 */
579 if (resp->flags & XEN_NETRXF_extra_info)
580 extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE;
581 else
582 resp->flags |= XEN_NETRXF_extra_info;
583
584 extra = (struct xen_netif_extra_info *)
585 RING_GET_RESPONSE(&queue->rx,
586 queue->rx.rsp_prod_pvt++);
587
588 extra->u.hash.algorithm =
589 XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ;
590
591 if (skb->l4_hash)
592 extra->u.hash.type =
593 skb->protocol == htons(ETH_P_IP) ?
594 _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP :
595 _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
596 else
597 extra->u.hash.type =
598 skb->protocol == htons(ETH_P_IP) ?
599 _XEN_NETIF_CTRL_HASH_TYPE_IPV4 :
600 _XEN_NETIF_CTRL_HASH_TYPE_IPV6;
601
602 *(uint32_t *)extra->u.hash.value =
603 skb_get_hash_raw(skb);
604
605 extra->type = XEN_NETIF_EXTRA_TYPE_HASH;
606 extra->flags = 0;
607 }
608
609 xenvif_add_frag_responses(queue, status,
610 queue->meta + npo.meta_cons + 1,
611 XENVIF_RX_CB(skb)->meta_slots_used);
612
613 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, ret);
614
615 need_to_notify |= !!ret;
616
617 npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used;
618 dev_kfree_skb(skb);
619 }
620
621done:
622 if (need_to_notify)
623 notify_remote_via_irq(queue->rx_irq);
624}
625
626static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)
627{
628 RING_IDX prod, cons;
629
630 prod = queue->rx.sring->req_prod;
631 cons = queue->rx.req_cons;
632
633 return !queue->stalled &&
634 prod - cons < 1 &&
635 time_after(jiffies,
636 queue->last_rx_time + queue->vif->stall_timeout);
637}
638
639static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)
640{
641 RING_IDX prod, cons;
642
643 prod = queue->rx.sring->req_prod;
644 cons = queue->rx.req_cons;
645
646 return queue->stalled && prod - cons >= 1;
647}
648
649static bool xenvif_have_rx_work(struct xenvif_queue *queue)
650{
651 return xenvif_rx_ring_slots_available(queue) ||
652 (queue->vif->stall_timeout &&
653 (xenvif_rx_queue_stalled(queue) ||
654 xenvif_rx_queue_ready(queue))) ||
655 kthread_should_stop() ||
656 queue->vif->disabled;
657}
658
659static long xenvif_rx_queue_timeout(struct xenvif_queue *queue)
660{
661 struct sk_buff *skb;
662 long timeout;
663
664 skb = skb_peek(&queue->rx_queue);
665 if (!skb)
666 return MAX_SCHEDULE_TIMEOUT;
667
668 timeout = XENVIF_RX_CB(skb)->expires - jiffies;
669 return timeout < 0 ? 0 : timeout;
670}
671
672/* Wait until the guest Rx thread has work.
673 *
674 * The timeout needs to be adjusted based on the current head of the
675 * queue (and not just the head at the beginning). In particular, if
676 * the queue is initially empty an infinite timeout is used and this
677 * needs to be reduced when a skb is queued.
678 *
679 * This cannot be done with wait_event_timeout() because it only
680 * calculates the timeout once.
681 */
682static void xenvif_wait_for_rx_work(struct xenvif_queue *queue)
683{
684 DEFINE_WAIT(wait);
685
686 if (xenvif_have_rx_work(queue))
687 return;
688
689 for (;;) {
690 long ret;
691
692 prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE);
693 if (xenvif_have_rx_work(queue))
694 break;
695 ret = schedule_timeout(xenvif_rx_queue_timeout(queue));
696 if (!ret)
697 break;
698 }
699 finish_wait(&queue->wq, &wait);
700}
701
702static void xenvif_queue_carrier_off(struct xenvif_queue *queue)
703{
704 struct xenvif *vif = queue->vif;
705
706 queue->stalled = true;
707
708 /* At least one queue has stalled? Disable the carrier. */
709 spin_lock(&vif->lock);
710 if (vif->stalled_queues++ == 0) {
711 netdev_info(vif->dev, "Guest Rx stalled");
712 netif_carrier_off(vif->dev);
713 }
714 spin_unlock(&vif->lock);
715}
716
717static void xenvif_queue_carrier_on(struct xenvif_queue *queue)
718{
719 struct xenvif *vif = queue->vif;
720
721 queue->last_rx_time = jiffies; /* Reset Rx stall detection. */
722 queue->stalled = false;
723
724 /* All queues are ready? Enable the carrier. */
725 spin_lock(&vif->lock);
726 if (--vif->stalled_queues == 0) {
727 netdev_info(vif->dev, "Guest Rx ready");
728 netif_carrier_on(vif->dev);
729 }
730 spin_unlock(&vif->lock);
731}
732
733int xenvif_kthread_guest_rx(void *data)
734{
735 struct xenvif_queue *queue = data;
736 struct xenvif *vif = queue->vif;
737
738 if (!vif->stall_timeout)
739 xenvif_queue_carrier_on(queue);
740
741 for (;;) {
742 xenvif_wait_for_rx_work(queue);
743
744 if (kthread_should_stop())
745 break;
746
747 /* This frontend is found to be rogue, disable it in
748 * kthread context. Currently this is only set when
749 * netback finds out frontend sends malformed packet,
750 * but we cannot disable the interface in softirq
751 * context so we defer it here, if this thread is
752 * associated with queue 0.
753 */
754 if (unlikely(vif->disabled && queue->id == 0)) {
755 xenvif_carrier_off(vif);
756 break;
757 }
758
759 if (!skb_queue_empty(&queue->rx_queue))
760 xenvif_rx_action(queue);
761
762 /* If the guest hasn't provided any Rx slots for a
763 * while it's probably not responsive, drop the
764 * carrier so packets are dropped earlier.
765 */
766 if (vif->stall_timeout) {
767 if (xenvif_rx_queue_stalled(queue))
768 xenvif_queue_carrier_off(queue);
769 else if (xenvif_rx_queue_ready(queue))
770 xenvif_queue_carrier_on(queue);
771 }
772
773 /* Queued packets may have foreign pages from other
774 * domains. These cannot be queued indefinitely as
775 * this would starve guests of grant refs and transmit
776 * slots.
777 */
778 xenvif_rx_queue_drop_expired(queue);
779
780 xenvif_rx_queue_maybe_wake(queue);
781
782 cond_resched();
783 }
784
785 /* Bin any remaining skbs */
786 xenvif_rx_queue_purge(queue);
787
788 return 0;
789}
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-28 13:40:55 -0400