aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Vrabel <david.vrabel@citrix.com>2016-10-04 05:29:14 -0400
committerDavid S. Miller <davem@davemloft.net>2016-10-06 20:37:35 -0400
commiteb1723a29b9a75dd787510a39096a68dba6cc200 (patch)
tree7f6c52ddcb5b98accb591c01b94def17d100007e
parentfedbc8c132bcf836358103195d8b6df6c03d9daf (diff)
xen-netback: refactor guest rx
Refactor the to-guest (rx) path to: 1. Push responses for completed skbs earlier, reducing latency. 2. Reduce the per-queue memory overhead by greatly reducing the maximum number of grant copy ops in each hypercall (from 4352 to 64). Each struct xenvif_queue is now only 44 kB instead of 220 kB. 3. Make the code more maintainable. Signed-off-by: David Vrabel <david.vrabel@citrix.com> [re-based] Signed-off-by: Paul Durrant <paul.durrant@citrix.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--drivers/net/xen-netback/common.h23
-rw-r--r--drivers/net/xen-netback/rx.c576
2 files changed, 215 insertions, 384 deletions
diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 0ba59106b1a5..7d12a388afc6 100644
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -91,13 +91,6 @@ struct xenvif_rx_meta {
91 */ 91 */
92#define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1) 92#define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1)
93 93
94/* It's possible for an skb to have a maximal number of frags
95 * but still be less than MAX_BUFFER_OFFSET in size. Thus the
96 * worst-case number of copy operations is MAX_XEN_SKB_FRAGS per
97 * ring slot.
98 */
99#define MAX_GRANT_COPY_OPS (MAX_XEN_SKB_FRAGS * XEN_NETIF_RX_RING_SIZE)
100
101#define NETBACK_INVALID_HANDLE -1 94#define NETBACK_INVALID_HANDLE -1
102 95
103/* To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating 96/* To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
@@ -133,6 +126,14 @@ struct xenvif_stats {
133 unsigned long tx_frag_overflow; 126 unsigned long tx_frag_overflow;
134}; 127};
135 128
129#define COPY_BATCH_SIZE 64
130
131struct xenvif_copy_state {
132 struct gnttab_copy op[COPY_BATCH_SIZE];
133 RING_IDX idx[COPY_BATCH_SIZE];
134 unsigned int num;
135};
136
136struct xenvif_queue { /* Per-queue data for xenvif */ 137struct xenvif_queue { /* Per-queue data for xenvif */
137 unsigned int id; /* Queue ID, 0-based */ 138 unsigned int id; /* Queue ID, 0-based */
138 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */ 139 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
@@ -189,12 +190,7 @@ struct xenvif_queue { /* Per-queue data for xenvif */
189 unsigned long last_rx_time; 190 unsigned long last_rx_time;
190 bool stalled; 191 bool stalled;
191 192
192 struct gnttab_copy grant_copy_op[MAX_GRANT_COPY_OPS]; 193 struct xenvif_copy_state rx_copy;
193
194 /* We create one meta structure per ring request we consume, so
195 * the maximum number is the same as the ring size.
196 */
197 struct xenvif_rx_meta meta[XEN_NETIF_RX_RING_SIZE];
198 194
199 /* Transmit shaping: allow 'credit_bytes' every 'credit_usec'. */ 195 /* Transmit shaping: allow 'credit_bytes' every 'credit_usec'. */
200 unsigned long credit_bytes; 196 unsigned long credit_bytes;
@@ -358,6 +354,7 @@ int xenvif_dealloc_kthread(void *data);
358 354
359irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data); 355irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data);
360 356
357void xenvif_rx_action(struct xenvif_queue *queue);
361void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); 358void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);
362 359
363void xenvif_carrier_on(struct xenvif *vif); 360void xenvif_carrier_on(struct xenvif *vif);
diff --git a/drivers/net/xen-netback/rx.c b/drivers/net/xen-netback/rx.c
index 6bd7d6e84b8e..b0ce4c6e9b21 100644
--- a/drivers/net/xen-netback/rx.c
+++ b/drivers/net/xen-netback/rx.c
@@ -26,7 +26,6 @@
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 * IN THE SOFTWARE. 27 * IN THE SOFTWARE.
28 */ 28 */
29
30#include "common.h" 29#include "common.h"
31 30
32#include <linux/kthread.h> 31#include <linux/kthread.h>
@@ -137,464 +136,299 @@ static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)
137 } 136 }
138} 137}
139 138
140struct netrx_pending_operations { 139static void xenvif_rx_copy_flush(struct xenvif_queue *queue)
141 unsigned int copy_prod, copy_cons;
142 unsigned int meta_prod, meta_cons;
143 struct gnttab_copy *copy;
144 struct xenvif_rx_meta *meta;
145 int copy_off;
146 grant_ref_t copy_gref;
147};
148
149static struct xenvif_rx_meta *get_next_rx_buffer(
150 struct xenvif_queue *queue,
151 struct netrx_pending_operations *npo)
152{ 140{
153 struct xenvif_rx_meta *meta; 141 unsigned int i;
154 struct xen_netif_rx_request req;
155 142
156 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req); 143 gnttab_batch_copy(queue->rx_copy.op, queue->rx_copy.num);
157 144
158 meta = npo->meta + npo->meta_prod++; 145 for (i = 0; i < queue->rx_copy.num; i++) {
159 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE; 146 struct gnttab_copy *op;
160 meta->gso_size = 0;
161 meta->size = 0;
162 meta->id = req.id;
163 147
164 npo->copy_off = 0; 148 op = &queue->rx_copy.op[i];
165 npo->copy_gref = req.gref;
166 149
167 return meta; 150 /* If the copy failed, overwrite the status field in
168} 151 * the corresponding response.
152 */
153 if (unlikely(op->status != GNTST_okay)) {
154 struct xen_netif_rx_response *rsp;
169 155
170struct gop_frag_copy { 156 rsp = RING_GET_RESPONSE(&queue->rx,
171 struct xenvif_queue *queue; 157 queue->rx_copy.idx[i]);
172 struct netrx_pending_operations *npo; 158 rsp->status = op->status;
173 struct xenvif_rx_meta *meta; 159 }
174 int head; 160 }
175 int gso_type;
176 int protocol;
177 int hash_present;
178 161
179 struct page *page; 162 queue->rx_copy.num = 0;
180}; 163}
181 164
182static void xenvif_setup_copy_gop(unsigned long gfn, 165static void xenvif_rx_copy_add(struct xenvif_queue *queue,
183 unsigned int offset, 166 struct xen_netif_rx_request *req,
184 unsigned int *len, 167 unsigned int offset, void *data, size_t len)
185 struct gop_frag_copy *info)
186{ 168{
187 struct gnttab_copy *copy_gop; 169 struct gnttab_copy *op;
170 struct page *page;
188 struct xen_page_foreign *foreign; 171 struct xen_page_foreign *foreign;
189 /* Convenient aliases */
190 struct xenvif_queue *queue = info->queue;
191 struct netrx_pending_operations *npo = info->npo;
192 struct page *page = info->page;
193 172
194 WARN_ON(npo->copy_off > MAX_BUFFER_OFFSET); 173 if (queue->rx_copy.num == COPY_BATCH_SIZE)
174 xenvif_rx_copy_flush(queue);
195 175
196 if (npo->copy_off == MAX_BUFFER_OFFSET) 176 op = &queue->rx_copy.op[queue->rx_copy.num];
197 info->meta = get_next_rx_buffer(queue, npo);
198 177
199 if (npo->copy_off + *len > MAX_BUFFER_OFFSET) 178 page = virt_to_page(data);
200 *len = MAX_BUFFER_OFFSET - npo->copy_off;
201 179
202 copy_gop = npo->copy + npo->copy_prod++; 180 op->flags = GNTCOPY_dest_gref;
203 copy_gop->flags = GNTCOPY_dest_gref;
204 copy_gop->len = *len;
205 181
206 foreign = xen_page_foreign(page); 182 foreign = xen_page_foreign(page);
207 if (foreign) { 183 if (foreign) {
208 copy_gop->source.domid = foreign->domid; 184 op->source.domid = foreign->domid;
209 copy_gop->source.u.ref = foreign->gref; 185 op->source.u.ref = foreign->gref;
210 copy_gop->flags |= GNTCOPY_source_gref; 186 op->flags |= GNTCOPY_source_gref;
211 } else { 187 } else {
212 copy_gop->source.domid = DOMID_SELF; 188 op->source.u.gmfn = virt_to_gfn(data);
213 copy_gop->source.u.gmfn = gfn; 189 op->source.domid = DOMID_SELF;
214 } 190 }
215 copy_gop->source.offset = offset;
216 191
217 copy_gop->dest.domid = queue->vif->domid; 192 op->source.offset = xen_offset_in_page(data);
218 copy_gop->dest.offset = npo->copy_off; 193 op->dest.u.ref = req->gref;
219 copy_gop->dest.u.ref = npo->copy_gref; 194 op->dest.domid = queue->vif->domid;
195 op->dest.offset = offset;
196 op->len = len;
220 197
221 npo->copy_off += *len; 198 queue->rx_copy.idx[queue->rx_copy.num] = queue->rx.req_cons;
222 info->meta->size += *len; 199 queue->rx_copy.num++;
223
224 if (!info->head)
225 return;
226
227 /* Leave a gap for the GSO descriptor. */
228 if ((1 << info->gso_type) & queue->vif->gso_mask)
229 queue->rx.req_cons++;
230
231 /* Leave a gap for the hash extra segment. */
232 if (info->hash_present)
233 queue->rx.req_cons++;
234
235 info->head = 0; /* There must be something in this buffer now */
236} 200}
237 201
238static void xenvif_gop_frag_copy_grant(unsigned long gfn, 202static unsigned int xenvif_gso_type(struct sk_buff *skb)
239 unsigned int offset,
240 unsigned int len,
241 void *data)
242{ 203{
243 unsigned int bytes;
244
245 while (len) {
246 bytes = len;
247 xenvif_setup_copy_gop(gfn, offset, &bytes, data);
248 offset += bytes;
249 len -= bytes;
250 }
251}
252
253/* Set up the grant operations for this fragment. If it's a flipping
254 * interface, we also set up the unmap request from here.
255 */
256static void xenvif_gop_frag_copy(struct xenvif_queue *queue,
257 struct sk_buff *skb,
258 struct netrx_pending_operations *npo,
259 struct page *page, unsigned long size,
260 unsigned long offset, int *head)
261{
262 struct gop_frag_copy info = {
263 .queue = queue,
264 .npo = npo,
265 .head = *head,
266 .gso_type = XEN_NETIF_GSO_TYPE_NONE,
267 /* xenvif_set_skb_hash() will have either set a s/w
268 * hash or cleared the hash depending on
269 * whether the the frontend wants a hash for this skb.
270 */
271 .hash_present = skb->sw_hash,
272 };
273 unsigned long bytes;
274
275 if (skb_is_gso(skb)) { 204 if (skb_is_gso(skb)) {
276 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) 205 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
277 info.gso_type = XEN_NETIF_GSO_TYPE_TCPV4; 206 return XEN_NETIF_GSO_TYPE_TCPV4;
278 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) 207 else
279 info.gso_type = XEN_NETIF_GSO_TYPE_TCPV6; 208 return XEN_NETIF_GSO_TYPE_TCPV6;
280 } 209 }
210 return XEN_NETIF_GSO_TYPE_NONE;
211}
281 212
282 /* Data must not cross a page boundary. */ 213struct xenvif_pkt_state {
283 WARN_ON(size + offset > (PAGE_SIZE << compound_order(page))); 214 struct sk_buff *skb;
215 size_t remaining_len;
216 int frag; /* frag == -1 => skb->head */
217 unsigned int frag_offset;
218 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
219 unsigned int extra_count;
220 unsigned int slot;
221};
284 222
285 info.meta = npo->meta + npo->meta_prod - 1; 223static void xenvif_rx_next_skb(struct xenvif_queue *queue,
224 struct xenvif_pkt_state *pkt)
225{
226 struct sk_buff *skb;
227 unsigned int gso_type;
286 228
287 /* Skip unused frames from start of page */ 229 skb = xenvif_rx_dequeue(queue);
288 page += offset >> PAGE_SHIFT;
289 offset &= ~PAGE_MASK;
290 230
291 while (size > 0) { 231 queue->stats.tx_bytes += skb->len;
292 WARN_ON(offset >= PAGE_SIZE); 232 queue->stats.tx_packets++;
293 233
294 bytes = PAGE_SIZE - offset; 234 /* Reset packet state. */
295 if (bytes > size) 235 memset(pkt, 0, sizeof(struct xenvif_pkt_state));
296 bytes = size;
297 236
298 info.page = page; 237 pkt->skb = skb;
299 gnttab_foreach_grant_in_range(page, offset, bytes, 238 pkt->remaining_len = skb->len;
300 xenvif_gop_frag_copy_grant, 239 pkt->frag = -1;
301 &info);
302 size -= bytes;
303 offset = 0;
304 240
305 /* Next page */ 241 gso_type = xenvif_gso_type(skb);
306 if (size) { 242 if ((1 << gso_type) & queue->vif->gso_mask) {
307 WARN_ON(!PageCompound(page)); 243 struct xen_netif_extra_info *extra;
308 page++;
309 }
310 }
311
312 *head = info.head;
313}
314 244
315/* Prepare an SKB to be transmitted to the frontend. 245 extra = &pkt->extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
316 *
317 * This function is responsible for allocating grant operations, meta
318 * structures, etc.
319 *
320 * It returns the number of meta structures consumed. The number of
321 * ring slots used is always equal to the number of meta slots used
322 * plus the number of GSO descriptors used. Currently, we use either
323 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
324 * frontend-side LRO).
325 */
326static int xenvif_gop_skb(struct sk_buff *skb,
327 struct netrx_pending_operations *npo,
328 struct xenvif_queue *queue)
329{
330 struct xenvif *vif = netdev_priv(skb->dev);
331 int nr_frags = skb_shinfo(skb)->nr_frags;
332 int i;
333 struct xen_netif_rx_request req;
334 struct xenvif_rx_meta *meta;
335 unsigned char *data;
336 int head = 1;
337 int old_meta_prod;
338 int gso_type;
339
340 old_meta_prod = npo->meta_prod;
341
342 gso_type = XEN_NETIF_GSO_TYPE_NONE;
343 if (skb_is_gso(skb)) {
344 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
345 gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
346 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
347 gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
348 }
349 246
350 RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req); 247 extra->u.gso.type = gso_type;
351 meta = npo->meta + npo->meta_prod++; 248 extra->u.gso.size = skb_shinfo(skb)->gso_size;
249 extra->u.gso.pad = 0;
250 extra->u.gso.features = 0;
251 extra->type = XEN_NETIF_EXTRA_TYPE_GSO;
252 extra->flags = 0;
352 253
353 if ((1 << gso_type) & vif->gso_mask) { 254 pkt->extra_count++;
354 meta->gso_type = gso_type;
355 meta->gso_size = skb_shinfo(skb)->gso_size;
356 } else {
357 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
358 meta->gso_size = 0;
359 } 255 }
360 256
361 meta->size = 0; 257 if (skb->sw_hash) {
362 meta->id = req.id; 258 struct xen_netif_extra_info *extra;
363 npo->copy_off = 0;
364 npo->copy_gref = req.gref;
365 259
366 data = skb->data; 260 extra = &pkt->extras[XEN_NETIF_EXTRA_TYPE_HASH - 1];
367 while (data < skb_tail_pointer(skb)) {
368 unsigned int offset = offset_in_page(data);
369 unsigned int len = PAGE_SIZE - offset;
370 261
371 if (data + len > skb_tail_pointer(skb)) 262 extra->u.hash.algorithm =
372 len = skb_tail_pointer(skb) - data; 263 XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ;
373 264
374 xenvif_gop_frag_copy(queue, skb, npo, 265 if (skb->l4_hash)
375 virt_to_page(data), len, offset, &head); 266 extra->u.hash.type =
376 data += len; 267 skb->protocol == htons(ETH_P_IP) ?
377 } 268 _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP :
269 _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
270 else
271 extra->u.hash.type =
272 skb->protocol == htons(ETH_P_IP) ?
273 _XEN_NETIF_CTRL_HASH_TYPE_IPV4 :
274 _XEN_NETIF_CTRL_HASH_TYPE_IPV6;
378 275
379 for (i = 0; i < nr_frags; i++) { 276 *(uint32_t *)extra->u.hash.value = skb_get_hash_raw(skb);
380 xenvif_gop_frag_copy(queue, skb, npo,
381 skb_frag_page(&skb_shinfo(skb)->frags[i]),
382 skb_frag_size(&skb_shinfo(skb)->frags[i]),
383 skb_shinfo(skb)->frags[i].page_offset,
384 &head);
385 }
386 277
387 return npo->meta_prod - old_meta_prod; 278 extra->type = XEN_NETIF_EXTRA_TYPE_HASH;
388} 279 extra->flags = 0;
389 280
390/* This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was 281 pkt->extra_count++;
391 * used to set up the operations on the top of
392 * netrx_pending_operations, which have since been done. Check that
393 * they didn't give any errors and advance over them.
394 */
395static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
396 struct netrx_pending_operations *npo)
397{
398 struct gnttab_copy *copy_op;
399 int status = XEN_NETIF_RSP_OKAY;
400 int i;
401
402 for (i = 0; i < nr_meta_slots; i++) {
403 copy_op = npo->copy + npo->copy_cons++;
404 if (copy_op->status != GNTST_okay) {
405 netdev_dbg(vif->dev,
406 "Bad status %d from copy to DOM%d.\n",
407 copy_op->status, vif->domid);
408 status = XEN_NETIF_RSP_ERROR;
409 }
410 } 282 }
411
412 return status;
413} 283}
414 284
415static struct xen_netif_rx_response *make_rx_response( 285static void xenvif_rx_complete(struct xenvif_queue *queue,
416 struct xenvif_queue *queue, u16 id, s8 st, u16 offset, u16 size, 286 struct xenvif_pkt_state *pkt)
417 u16 flags)
418{ 287{
419 RING_IDX i = queue->rx.rsp_prod_pvt; 288 int notify;
420 struct xen_netif_rx_response *resp;
421 289
422 resp = RING_GET_RESPONSE(&queue->rx, i); 290 /* Complete any outstanding copy ops for this skb. */
423 resp->offset = offset; 291 xenvif_rx_copy_flush(queue);
424 resp->flags = flags;
425 resp->id = id;
426 resp->status = (s16)size;
427 if (st < 0)
428 resp->status = (s16)st;
429 292
430 queue->rx.rsp_prod_pvt = ++i; 293 /* Push responses and notify. */
294 queue->rx.rsp_prod_pvt = queue->rx.req_cons;
295 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, notify);
296 if (notify)
297 notify_remote_via_irq(queue->rx_irq);
431 298
432 return resp; 299 dev_kfree_skb(pkt->skb);
433} 300}
434 301
435static void xenvif_add_frag_responses(struct xenvif_queue *queue, 302static void xenvif_rx_next_chunk(struct xenvif_queue *queue,
436 int status, 303 struct xenvif_pkt_state *pkt,
437 struct xenvif_rx_meta *meta, 304 unsigned int offset, void **data,
438 int nr_meta_slots) 305 size_t *len)
439{ 306{
440 int i; 307 struct sk_buff *skb = pkt->skb;
441 unsigned long offset; 308 void *frag_data;
309 size_t frag_len, chunk_len;
442 310
443 /* No fragments used */ 311 if (pkt->frag == -1) {
444 if (nr_meta_slots <= 1) 312 frag_data = skb->data;
445 return; 313 frag_len = skb_headlen(skb);
446 314 } else {
447 nr_meta_slots--; 315 skb_frag_t *frag = &skb_shinfo(skb)->frags[pkt->frag];
448
449 for (i = 0; i < nr_meta_slots; i++) {
450 int flags;
451
452 if (i == nr_meta_slots - 1)
453 flags = 0;
454 else
455 flags = XEN_NETRXF_more_data;
456 316
457 offset = 0; 317 frag_data = skb_frag_address(frag);
458 make_rx_response(queue, meta[i].id, status, offset, 318 frag_len = skb_frag_size(frag);
459 meta[i].size, flags);
460 } 319 }
461}
462
463static void xenvif_rx_action(struct xenvif_queue *queue)
464{
465 struct xenvif *vif = queue->vif;
466 s8 status;
467 u16 flags;
468 struct xen_netif_rx_response *resp;
469 struct sk_buff_head rxq;
470 struct sk_buff *skb;
471 LIST_HEAD(notify);
472 int ret;
473 unsigned long offset;
474 bool need_to_notify = false;
475 320
476 struct netrx_pending_operations npo = { 321 frag_data += pkt->frag_offset;
477 .copy = queue->grant_copy_op, 322 frag_len -= pkt->frag_offset;
478 .meta = queue->meta,
479 };
480 323
481 skb_queue_head_init(&rxq); 324 chunk_len = min(frag_len, XEN_PAGE_SIZE - offset);
325 chunk_len = min(chunk_len,
326 XEN_PAGE_SIZE - xen_offset_in_page(frag_data));
482 327
483 while (xenvif_rx_ring_slots_available(queue) && 328 pkt->frag_offset += chunk_len;
484 (skb = xenvif_rx_dequeue(queue)) != NULL) {
485 queue->last_rx_time = jiffies;
486 329
487 XENVIF_RX_CB(skb)->meta_slots_used = 330 /* Advance to next frag? */
488 xenvif_gop_skb(skb, &npo, queue); 331 if (frag_len == chunk_len) {
489 332 pkt->frag++;
490 __skb_queue_tail(&rxq, skb); 333 pkt->frag_offset = 0;
491 } 334 }
492 335
493 WARN_ON(npo.meta_prod > ARRAY_SIZE(queue->meta)); 336 *data = frag_data;
337 *len = chunk_len;
338}
339
340static void xenvif_rx_data_slot(struct xenvif_queue *queue,
341 struct xenvif_pkt_state *pkt,
342 struct xen_netif_rx_request *req,
343 struct xen_netif_rx_response *rsp)
344{
345 unsigned int offset = 0;
346 unsigned int flags;
494 347
495 if (!npo.copy_prod) 348 do {
496 goto done; 349 size_t len;
350 void *data;
497 351
498 WARN_ON(npo.copy_prod > MAX_GRANT_COPY_OPS); 352 xenvif_rx_next_chunk(queue, pkt, offset, &data, &len);
499 gnttab_batch_copy(queue->grant_copy_op, npo.copy_prod); 353 xenvif_rx_copy_add(queue, req, offset, data, len);
500 354
501 while ((skb = __skb_dequeue(&rxq)) != NULL) { 355 offset += len;
502 struct xen_netif_extra_info *extra = NULL; 356 pkt->remaining_len -= len;
503 357
504 queue->stats.tx_bytes += skb->len; 358 } while (offset < XEN_PAGE_SIZE && pkt->remaining_len > 0);
505 queue->stats.tx_packets++;
506 359
507 status = xenvif_check_gop(vif, 360 if (pkt->remaining_len > 0)
508 XENVIF_RX_CB(skb)->meta_slots_used, 361 flags = XEN_NETRXF_more_data;
509 &npo); 362 else
363 flags = 0;
510 364
511 if (XENVIF_RX_CB(skb)->meta_slots_used == 1) 365 if (pkt->slot == 0) {
512 flags = 0; 366 struct sk_buff *skb = pkt->skb;
513 else
514 flags = XEN_NETRXF_more_data;
515 367
516 if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */ 368 if (skb->ip_summed == CHECKSUM_PARTIAL)
517 flags |= XEN_NETRXF_csum_blank | 369 flags |= XEN_NETRXF_csum_blank |
518 XEN_NETRXF_data_validated; 370 XEN_NETRXF_data_validated;
519 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) 371 else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
520 /* remote but checksummed. */
521 flags |= XEN_NETRXF_data_validated; 372 flags |= XEN_NETRXF_data_validated;
522 373
523 offset = 0; 374 if (pkt->extra_count != 0)
524 resp = make_rx_response(queue, queue->meta[npo.meta_cons].id, 375 flags |= XEN_NETRXF_extra_info;
525 status, offset, 376 }
526 queue->meta[npo.meta_cons].size,
527 flags);
528 377
529 if ((1 << queue->meta[npo.meta_cons].gso_type) & 378 rsp->offset = 0;
530 vif->gso_mask) { 379 rsp->flags = flags;
531 extra = (struct xen_netif_extra_info *) 380 rsp->id = req->id;
532 RING_GET_RESPONSE(&queue->rx, 381 rsp->status = (s16)offset;
533 queue->rx.rsp_prod_pvt++); 382}
534 383
535 resp->flags |= XEN_NETRXF_extra_info; 384static void xenvif_rx_extra_slot(struct xenvif_queue *queue,
385 struct xenvif_pkt_state *pkt,
386 struct xen_netif_rx_request *req,
387 struct xen_netif_rx_response *rsp)
388{
389 struct xen_netif_extra_info *extra = (void *)rsp;
390 unsigned int i;
536 391
537 extra->u.gso.type = queue->meta[npo.meta_cons].gso_type; 392 pkt->extra_count--;
538 extra->u.gso.size = queue->meta[npo.meta_cons].gso_size;
539 extra->u.gso.pad = 0;
540 extra->u.gso.features = 0;
541 393
542 extra->type = XEN_NETIF_EXTRA_TYPE_GSO; 394 for (i = 0; i < ARRAY_SIZE(pkt->extras); i++) {
543 extra->flags = 0; 395 if (pkt->extras[i].type) {
544 } 396 *extra = pkt->extras[i];
545 397
546 if (skb->sw_hash) { 398 if (pkt->extra_count != 0)
547 /* Since the skb got here via xenvif_select_queue()
548 * we know that the hash has been re-calculated
549 * according to a configuration set by the frontend
550 * and therefore we know that it is legitimate to
551 * pass it to the frontend.
552 */
553 if (resp->flags & XEN_NETRXF_extra_info)
554 extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE; 399 extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE;
555 else 400
556 resp->flags |= XEN_NETRXF_extra_info; 401 pkt->extras[i].type = 0;
557 402 return;
558 extra = (struct xen_netif_extra_info *)
559 RING_GET_RESPONSE(&queue->rx,
560 queue->rx.rsp_prod_pvt++);
561
562 extra->u.hash.algorithm =
563 XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ;
564
565 if (skb->l4_hash)
566 extra->u.hash.type =
567 skb->protocol == htons(ETH_P_IP) ?
568 _XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP :
569 _XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
570 else
571 extra->u.hash.type =
572 skb->protocol == htons(ETH_P_IP) ?
573 _XEN_NETIF_CTRL_HASH_TYPE_IPV4 :
574 _XEN_NETIF_CTRL_HASH_TYPE_IPV6;
575
576 *(uint32_t *)extra->u.hash.value =
577 skb_get_hash_raw(skb);
578
579 extra->type = XEN_NETIF_EXTRA_TYPE_HASH;
580 extra->flags = 0;
581 } 403 }
404 }
405 BUG();
406}
582 407
583 xenvif_add_frag_responses(queue, status, 408void xenvif_rx_action(struct xenvif_queue *queue)
584 queue->meta + npo.meta_cons + 1, 409{
585 XENVIF_RX_CB(skb)->meta_slots_used); 410 struct xenvif_pkt_state pkt;
586 411
587 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, ret); 412 xenvif_rx_next_skb(queue, &pkt);
588 413
589 need_to_notify |= !!ret; 414 do {
415 struct xen_netif_rx_request *req;
416 struct xen_netif_rx_response *rsp;
590 417
591 npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used; 418 req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons);
592 dev_kfree_skb(skb); 419 rsp = RING_GET_RESPONSE(&queue->rx, queue->rx.req_cons);
593 }
594 420
595done: 421 /* Extras must go after the first data slot */
596 if (need_to_notify) 422 if (pkt.slot != 0 && pkt.extra_count != 0)
597 notify_remote_via_irq(queue->rx_irq); 423 xenvif_rx_extra_slot(queue, &pkt, req, rsp);
424 else
425 xenvif_rx_data_slot(queue, &pkt, req, rsp);
426
427 queue->rx.req_cons++;
428 pkt.slot++;
429 } while (pkt.remaining_len > 0 || pkt.extra_count != 0);
430
431 xenvif_rx_complete(queue, &pkt);
598} 432}
599 433
600static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue) 434static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-28 13:41:01 -0400