aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/xen-netback/netback.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/xen-netback/netback.c')
-rw-r--r--drivers/net/xen-netback/netback.c927
1 files changed, 318 insertions, 609 deletions
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 64828de25d9a..f3e591c611de 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -70,131 +70,26 @@ module_param(fatal_skb_slots, uint, 0444);
70 */ 70 */
71#define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN 71#define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
72 72
73typedef unsigned int pending_ring_idx_t;
74#define INVALID_PENDING_RING_IDX (~0U)
75
76struct pending_tx_info {
77 struct xen_netif_tx_request req; /* coalesced tx request */
78 struct xenvif *vif;
79 pending_ring_idx_t head; /* head != INVALID_PENDING_RING_IDX
80 * if it is head of one or more tx
81 * reqs
82 */
83};
84
85struct netbk_rx_meta {
86 int id;
87 int size;
88 int gso_size;
89};
90
91#define MAX_PENDING_REQS 256
92
93/* Discriminate from any valid pending_idx value. */
94#define INVALID_PENDING_IDX 0xFFFF
95
96#define MAX_BUFFER_OFFSET PAGE_SIZE
97
98/* extra field used in struct page */
99union page_ext {
100 struct {
101#if BITS_PER_LONG < 64
102#define IDX_WIDTH 8
103#define GROUP_WIDTH (BITS_PER_LONG - IDX_WIDTH)
104 unsigned int group:GROUP_WIDTH;
105 unsigned int idx:IDX_WIDTH;
106#else
107 unsigned int group, idx;
108#endif
109 } e;
110 void *mapping;
111};
112
113struct xen_netbk {
114 wait_queue_head_t wq;
115 struct task_struct *task;
116
117 struct sk_buff_head rx_queue;
118 struct sk_buff_head tx_queue;
119
120 struct timer_list net_timer;
121
122 struct page *mmap_pages[MAX_PENDING_REQS];
123
124 pending_ring_idx_t pending_prod;
125 pending_ring_idx_t pending_cons;
126 struct list_head net_schedule_list;
127
128 /* Protect the net_schedule_list in netif. */
129 spinlock_t net_schedule_list_lock;
130
131 atomic_t netfront_count;
132
133 struct pending_tx_info pending_tx_info[MAX_PENDING_REQS];
134 /* Coalescing tx requests before copying makes number of grant
135 * copy ops greater or equal to number of slots required. In
136 * worst case a tx request consumes 2 gnttab_copy.
137 */
138 struct gnttab_copy tx_copy_ops[2*MAX_PENDING_REQS];
139
140 u16 pending_ring[MAX_PENDING_REQS];
141
142 /*
143 * Given MAX_BUFFER_OFFSET of 4096 the worst case is that each
144 * head/fragment page uses 2 copy operations because it
145 * straddles two buffers in the frontend.
146 */
147 struct gnttab_copy grant_copy_op[2*XEN_NETIF_RX_RING_SIZE];
148 struct netbk_rx_meta meta[2*XEN_NETIF_RX_RING_SIZE];
149};
150
151static struct xen_netbk *xen_netbk;
152static int xen_netbk_group_nr;
153
154/* 73/*
155 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of 74 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of
156 * one or more merged tx requests, otherwise it is the continuation of 75 * one or more merged tx requests, otherwise it is the continuation of
157 * previous tx request. 76 * previous tx request.
158 */ 77 */
159static inline int pending_tx_is_head(struct xen_netbk *netbk, RING_IDX idx) 78static inline int pending_tx_is_head(struct xenvif *vif, RING_IDX idx)
160{ 79{
161 return netbk->pending_tx_info[idx].head != INVALID_PENDING_RING_IDX; 80 return vif->pending_tx_info[idx].head != INVALID_PENDING_RING_IDX;
162}
163
164void xen_netbk_add_xenvif(struct xenvif *vif)
165{
166 int i;
167 int min_netfront_count;
168 int min_group = 0;
169 struct xen_netbk *netbk;
170
171 min_netfront_count = atomic_read(&xen_netbk[0].netfront_count);
172 for (i = 0; i < xen_netbk_group_nr; i++) {
173 int netfront_count = atomic_read(&xen_netbk[i].netfront_count);
174 if (netfront_count < min_netfront_count) {
175 min_group = i;
176 min_netfront_count = netfront_count;
177 }
178 }
179
180 netbk = &xen_netbk[min_group];
181
182 vif->netbk = netbk;
183 atomic_inc(&netbk->netfront_count);
184} 81}
185 82
186void xen_netbk_remove_xenvif(struct xenvif *vif) 83static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx,
187{ 84 u8 status);
188 struct xen_netbk *netbk = vif->netbk;
189 vif->netbk = NULL;
190 atomic_dec(&netbk->netfront_count);
191}
192 85
193static void xen_netbk_idx_release(struct xen_netbk *netbk, u16 pending_idx,
194 u8 status);
195static void make_tx_response(struct xenvif *vif, 86static void make_tx_response(struct xenvif *vif,
196 struct xen_netif_tx_request *txp, 87 struct xen_netif_tx_request *txp,
197 s8 st); 88 s8 st);
89
90static inline int tx_work_todo(struct xenvif *vif);
91static inline int rx_work_todo(struct xenvif *vif);
92
198static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif, 93static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
199 u16 id, 94 u16 id,
200 s8 st, 95 s8 st,
@@ -202,55 +97,16 @@ static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
202 u16 size, 97 u16 size,
203 u16 flags); 98 u16 flags);
204 99
205static inline unsigned long idx_to_pfn(struct xen_netbk *netbk, 100static inline unsigned long idx_to_pfn(struct xenvif *vif,
206 u16 idx) 101 u16 idx)
207{ 102{
208 return page_to_pfn(netbk->mmap_pages[idx]); 103 return page_to_pfn(vif->mmap_pages[idx]);
209} 104}
210 105
211static inline unsigned long idx_to_kaddr(struct xen_netbk *netbk, 106static inline unsigned long idx_to_kaddr(struct xenvif *vif,
212 u16 idx) 107 u16 idx)
213{ 108{
214 return (unsigned long)pfn_to_kaddr(idx_to_pfn(netbk, idx)); 109 return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif, idx));
215}
216
217/* extra field used in struct page */
218static inline void set_page_ext(struct page *pg, struct xen_netbk *netbk,
219 unsigned int idx)
220{
221 unsigned int group = netbk - xen_netbk;
222 union page_ext ext = { .e = { .group = group + 1, .idx = idx } };
223
224 BUILD_BUG_ON(sizeof(ext) > sizeof(ext.mapping));
225 pg->mapping = ext.mapping;
226}
227
228static int get_page_ext(struct page *pg,
229 unsigned int *pgroup, unsigned int *pidx)
230{
231 union page_ext ext = { .mapping = pg->mapping };
232 struct xen_netbk *netbk;
233 unsigned int group, idx;
234
235 group = ext.e.group - 1;
236
237 if (group < 0 || group >= xen_netbk_group_nr)
238 return 0;
239
240 netbk = &xen_netbk[group];
241
242 idx = ext.e.idx;
243
244 if ((idx < 0) || (idx >= MAX_PENDING_REQS))
245 return 0;
246
247 if (netbk->mmap_pages[idx] != pg)
248 return 0;
249
250 *pgroup = group;
251 *pidx = idx;
252
253 return 1;
254} 110}
255 111
256/* 112/*
@@ -278,15 +134,10 @@ static inline pending_ring_idx_t pending_index(unsigned i)
278 return i & (MAX_PENDING_REQS-1); 134 return i & (MAX_PENDING_REQS-1);
279} 135}
280 136
281static inline pending_ring_idx_t nr_pending_reqs(struct xen_netbk *netbk) 137static inline pending_ring_idx_t nr_pending_reqs(struct xenvif *vif)
282{ 138{
283 return MAX_PENDING_REQS - 139 return MAX_PENDING_REQS -
284 netbk->pending_prod + netbk->pending_cons; 140 vif->pending_prod + vif->pending_cons;
285}
286
287static void xen_netbk_kick_thread(struct xen_netbk *netbk)
288{
289 wake_up(&netbk->wq);
290} 141}
291 142
292static int max_required_rx_slots(struct xenvif *vif) 143static int max_required_rx_slots(struct xenvif *vif)
@@ -300,7 +151,7 @@ static int max_required_rx_slots(struct xenvif *vif)
300 return max; 151 return max;
301} 152}
302 153
303int xen_netbk_rx_ring_full(struct xenvif *vif) 154int xenvif_rx_ring_full(struct xenvif *vif)
304{ 155{
305 RING_IDX peek = vif->rx_req_cons_peek; 156 RING_IDX peek = vif->rx_req_cons_peek;
306 RING_IDX needed = max_required_rx_slots(vif); 157 RING_IDX needed = max_required_rx_slots(vif);
@@ -309,16 +160,16 @@ int xen_netbk_rx_ring_full(struct xenvif *vif)
309 ((vif->rx.rsp_prod_pvt + XEN_NETIF_RX_RING_SIZE - peek) < needed); 160 ((vif->rx.rsp_prod_pvt + XEN_NETIF_RX_RING_SIZE - peek) < needed);
310} 161}
311 162
312int xen_netbk_must_stop_queue(struct xenvif *vif) 163int xenvif_must_stop_queue(struct xenvif *vif)
313{ 164{
314 if (!xen_netbk_rx_ring_full(vif)) 165 if (!xenvif_rx_ring_full(vif))
315 return 0; 166 return 0;
316 167
317 vif->rx.sring->req_event = vif->rx_req_cons_peek + 168 vif->rx.sring->req_event = vif->rx_req_cons_peek +
318 max_required_rx_slots(vif); 169 max_required_rx_slots(vif);
319 mb(); /* request notification /then/ check the queue */ 170 mb(); /* request notification /then/ check the queue */
320 171
321 return xen_netbk_rx_ring_full(vif); 172 return xenvif_rx_ring_full(vif);
322} 173}
323 174
324/* 175/*
@@ -361,55 +212,89 @@ static bool start_new_rx_buffer(int offset, unsigned long size, int head)
361 return false; 212 return false;
362} 213}
363 214
215struct xenvif_count_slot_state {
216 unsigned long copy_off;
217 bool head;
218};
219
220unsigned int xenvif_count_frag_slots(struct xenvif *vif,
221 unsigned long offset, unsigned long size,
222 struct xenvif_count_slot_state *state)
223{
224 unsigned count = 0;
225
226 offset &= ~PAGE_MASK;
227
228 while (size > 0) {
229 unsigned long bytes;
230
231 bytes = PAGE_SIZE - offset;
232
233 if (bytes > size)
234 bytes = size;
235
236 if (start_new_rx_buffer(state->copy_off, bytes, state->head)) {
237 count++;
238 state->copy_off = 0;
239 }
240
241 if (state->copy_off + bytes > MAX_BUFFER_OFFSET)
242 bytes = MAX_BUFFER_OFFSET - state->copy_off;
243
244 state->copy_off += bytes;
245
246 offset += bytes;
247 size -= bytes;
248
249 if (offset == PAGE_SIZE)
250 offset = 0;
251
252 state->head = false;
253 }
254
255 return count;
256}
257
364/* 258/*
365 * Figure out how many ring slots we're going to need to send @skb to 259 * Figure out how many ring slots we're going to need to send @skb to
366 * the guest. This function is essentially a dry run of 260 * the guest. This function is essentially a dry run of
367 * netbk_gop_frag_copy. 261 * xenvif_gop_frag_copy.
368 */ 262 */
369unsigned int xen_netbk_count_skb_slots(struct xenvif *vif, struct sk_buff *skb) 263unsigned int xenvif_count_skb_slots(struct xenvif *vif, struct sk_buff *skb)
370{ 264{
265 struct xenvif_count_slot_state state;
371 unsigned int count; 266 unsigned int count;
372 int i, copy_off; 267 unsigned char *data;
268 unsigned i;
373 269
374 count = DIV_ROUND_UP(skb_headlen(skb), PAGE_SIZE); 270 state.head = true;
271 state.copy_off = 0;
375 272
376 copy_off = skb_headlen(skb) % PAGE_SIZE; 273 /* Slot for the first (partial) page of data. */
274 count = 1;
377 275
276 /* Need a slot for the GSO prefix for GSO extra data? */
378 if (skb_shinfo(skb)->gso_size) 277 if (skb_shinfo(skb)->gso_size)
379 count++; 278 count++;
380 279
381 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 280 data = skb->data;
382 unsigned long size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 281 while (data < skb_tail_pointer(skb)) {
383 unsigned long offset = skb_shinfo(skb)->frags[i].page_offset; 282 unsigned long offset = offset_in_page(data);
384 unsigned long bytes; 283 unsigned long size = PAGE_SIZE - offset;
385
386 offset &= ~PAGE_MASK;
387
388 while (size > 0) {
389 BUG_ON(offset >= PAGE_SIZE);
390 BUG_ON(copy_off > MAX_BUFFER_OFFSET);
391
392 bytes = PAGE_SIZE - offset;
393
394 if (bytes > size)
395 bytes = size;
396 284
397 if (start_new_rx_buffer(copy_off, bytes, 0)) { 285 if (data + size > skb_tail_pointer(skb))
398 count++; 286 size = skb_tail_pointer(skb) - data;
399 copy_off = 0;
400 }
401 287
402 if (copy_off + bytes > MAX_BUFFER_OFFSET) 288 count += xenvif_count_frag_slots(vif, offset, size, &state);
403 bytes = MAX_BUFFER_OFFSET - copy_off;
404 289
405 copy_off += bytes; 290 data += size;
291 }
406 292
407 offset += bytes; 293 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
408 size -= bytes; 294 unsigned long size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
295 unsigned long offset = skb_shinfo(skb)->frags[i].page_offset;
409 296
410 if (offset == PAGE_SIZE) 297 count += xenvif_count_frag_slots(vif, offset, size, &state);
411 offset = 0;
412 }
413 } 298 }
414 return count; 299 return count;
415} 300}
@@ -418,15 +303,15 @@ struct netrx_pending_operations {
418 unsigned copy_prod, copy_cons; 303 unsigned copy_prod, copy_cons;
419 unsigned meta_prod, meta_cons; 304 unsigned meta_prod, meta_cons;
420 struct gnttab_copy *copy; 305 struct gnttab_copy *copy;
421 struct netbk_rx_meta *meta; 306 struct xenvif_rx_meta *meta;
422 int copy_off; 307 int copy_off;
423 grant_ref_t copy_gref; 308 grant_ref_t copy_gref;
424}; 309};
425 310
426static struct netbk_rx_meta *get_next_rx_buffer(struct xenvif *vif, 311static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif *vif,
427 struct netrx_pending_operations *npo) 312 struct netrx_pending_operations *npo)
428{ 313{
429 struct netbk_rx_meta *meta; 314 struct xenvif_rx_meta *meta;
430 struct xen_netif_rx_request *req; 315 struct xen_netif_rx_request *req;
431 316
432 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 317 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
@@ -446,19 +331,13 @@ static struct netbk_rx_meta *get_next_rx_buffer(struct xenvif *vif,
446 * Set up the grant operations for this fragment. If it's a flipping 331 * Set up the grant operations for this fragment. If it's a flipping
447 * interface, we also set up the unmap request from here. 332 * interface, we also set up the unmap request from here.
448 */ 333 */
449static void netbk_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb, 334static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
450 struct netrx_pending_operations *npo, 335 struct netrx_pending_operations *npo,
451 struct page *page, unsigned long size, 336 struct page *page, unsigned long size,
452 unsigned long offset, int *head) 337 unsigned long offset, int *head)
453{ 338{
454 struct gnttab_copy *copy_gop; 339 struct gnttab_copy *copy_gop;
455 struct netbk_rx_meta *meta; 340 struct xenvif_rx_meta *meta;
456 /*
457 * These variables are used iff get_page_ext returns true,
458 * in which case they are guaranteed to be initialized.
459 */
460 unsigned int uninitialized_var(group), uninitialized_var(idx);
461 int foreign = get_page_ext(page, &group, &idx);
462 unsigned long bytes; 341 unsigned long bytes;
463 342
464 /* Data must not cross a page boundary. */ 343 /* Data must not cross a page boundary. */
@@ -494,26 +373,15 @@ static void netbk_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
494 373
495 copy_gop = npo->copy + npo->copy_prod++; 374 copy_gop = npo->copy + npo->copy_prod++;
496 copy_gop->flags = GNTCOPY_dest_gref; 375 copy_gop->flags = GNTCOPY_dest_gref;
497 if (foreign) { 376 copy_gop->len = bytes;
498 struct xen_netbk *netbk = &xen_netbk[group];
499 struct pending_tx_info *src_pend;
500
501 src_pend = &netbk->pending_tx_info[idx];
502 377
503 copy_gop->source.domid = src_pend->vif->domid; 378 copy_gop->source.domid = DOMID_SELF;
504 copy_gop->source.u.ref = src_pend->req.gref; 379 copy_gop->source.u.gmfn = virt_to_mfn(page_address(page));
505 copy_gop->flags |= GNTCOPY_source_gref;
506 } else {
507 void *vaddr = page_address(page);
508 copy_gop->source.domid = DOMID_SELF;
509 copy_gop->source.u.gmfn = virt_to_mfn(vaddr);
510 }
511 copy_gop->source.offset = offset; 380 copy_gop->source.offset = offset;
512 copy_gop->dest.domid = vif->domid;
513 381
382 copy_gop->dest.domid = vif->domid;
514 copy_gop->dest.offset = npo->copy_off; 383 copy_gop->dest.offset = npo->copy_off;
515 copy_gop->dest.u.ref = npo->copy_gref; 384 copy_gop->dest.u.ref = npo->copy_gref;
516 copy_gop->len = bytes;
517 385
518 npo->copy_off += bytes; 386 npo->copy_off += bytes;
519 meta->size += bytes; 387 meta->size += bytes;
@@ -549,14 +417,14 @@ static void netbk_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
549 * zero GSO descriptors (for non-GSO packets) or one descriptor (for 417 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
550 * frontend-side LRO). 418 * frontend-side LRO).
551 */ 419 */
552static int netbk_gop_skb(struct sk_buff *skb, 420static int xenvif_gop_skb(struct sk_buff *skb,
553 struct netrx_pending_operations *npo) 421 struct netrx_pending_operations *npo)
554{ 422{
555 struct xenvif *vif = netdev_priv(skb->dev); 423 struct xenvif *vif = netdev_priv(skb->dev);
556 int nr_frags = skb_shinfo(skb)->nr_frags; 424 int nr_frags = skb_shinfo(skb)->nr_frags;
557 int i; 425 int i;
558 struct xen_netif_rx_request *req; 426 struct xen_netif_rx_request *req;
559 struct netbk_rx_meta *meta; 427 struct xenvif_rx_meta *meta;
560 unsigned char *data; 428 unsigned char *data;
561 int head = 1; 429 int head = 1;
562 int old_meta_prod; 430 int old_meta_prod;
@@ -593,30 +461,30 @@ static int netbk_gop_skb(struct sk_buff *skb,
593 if (data + len > skb_tail_pointer(skb)) 461 if (data + len > skb_tail_pointer(skb))
594 len = skb_tail_pointer(skb) - data; 462 len = skb_tail_pointer(skb) - data;
595 463
596 netbk_gop_frag_copy(vif, skb, npo, 464 xenvif_gop_frag_copy(vif, skb, npo,
597 virt_to_page(data), len, offset, &head); 465 virt_to_page(data), len, offset, &head);
598 data += len; 466 data += len;
599 } 467 }
600 468
601 for (i = 0; i < nr_frags; i++) { 469 for (i = 0; i < nr_frags; i++) {
602 netbk_gop_frag_copy(vif, skb, npo, 470 xenvif_gop_frag_copy(vif, skb, npo,
603 skb_frag_page(&skb_shinfo(skb)->frags[i]), 471 skb_frag_page(&skb_shinfo(skb)->frags[i]),
604 skb_frag_size(&skb_shinfo(skb)->frags[i]), 472 skb_frag_size(&skb_shinfo(skb)->frags[i]),
605 skb_shinfo(skb)->frags[i].page_offset, 473 skb_shinfo(skb)->frags[i].page_offset,
606 &head); 474 &head);
607 } 475 }
608 476
609 return npo->meta_prod - old_meta_prod; 477 return npo->meta_prod - old_meta_prod;
610} 478}
611 479
612/* 480/*
613 * This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was 481 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
614 * used to set up the operations on the top of 482 * used to set up the operations on the top of
615 * netrx_pending_operations, which have since been done. Check that 483 * netrx_pending_operations, which have since been done. Check that
616 * they didn't give any errors and advance over them. 484 * they didn't give any errors and advance over them.
617 */ 485 */
618static int netbk_check_gop(struct xenvif *vif, int nr_meta_slots, 486static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
619 struct netrx_pending_operations *npo) 487 struct netrx_pending_operations *npo)
620{ 488{
621 struct gnttab_copy *copy_op; 489 struct gnttab_copy *copy_op;
622 int status = XEN_NETIF_RSP_OKAY; 490 int status = XEN_NETIF_RSP_OKAY;
@@ -635,9 +503,9 @@ static int netbk_check_gop(struct xenvif *vif, int nr_meta_slots,
635 return status; 503 return status;
636} 504}
637 505
638static void netbk_add_frag_responses(struct xenvif *vif, int status, 506static void xenvif_add_frag_responses(struct xenvif *vif, int status,
639 struct netbk_rx_meta *meta, 507 struct xenvif_rx_meta *meta,
640 int nr_meta_slots) 508 int nr_meta_slots)
641{ 509{
642 int i; 510 int i;
643 unsigned long offset; 511 unsigned long offset;
@@ -665,9 +533,13 @@ struct skb_cb_overlay {
665 int meta_slots_used; 533 int meta_slots_used;
666}; 534};
667 535
668static void xen_netbk_rx_action(struct xen_netbk *netbk) 536static void xenvif_kick_thread(struct xenvif *vif)
537{
538 wake_up(&vif->wq);
539}
540
541void xenvif_rx_action(struct xenvif *vif)
669{ 542{
670 struct xenvif *vif = NULL, *tmp;
671 s8 status; 543 s8 status;
672 u16 flags; 544 u16 flags;
673 struct xen_netif_rx_response *resp; 545 struct xen_netif_rx_response *resp;
@@ -679,22 +551,23 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
679 int count; 551 int count;
680 unsigned long offset; 552 unsigned long offset;
681 struct skb_cb_overlay *sco; 553 struct skb_cb_overlay *sco;
554 int need_to_notify = 0;
682 555
683 struct netrx_pending_operations npo = { 556 struct netrx_pending_operations npo = {
684 .copy = netbk->grant_copy_op, 557 .copy = vif->grant_copy_op,
685 .meta = netbk->meta, 558 .meta = vif->meta,
686 }; 559 };
687 560
688 skb_queue_head_init(&rxq); 561 skb_queue_head_init(&rxq);
689 562
690 count = 0; 563 count = 0;
691 564
692 while ((skb = skb_dequeue(&netbk->rx_queue)) != NULL) { 565 while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) {
693 vif = netdev_priv(skb->dev); 566 vif = netdev_priv(skb->dev);
694 nr_frags = skb_shinfo(skb)->nr_frags; 567 nr_frags = skb_shinfo(skb)->nr_frags;
695 568
696 sco = (struct skb_cb_overlay *)skb->cb; 569 sco = (struct skb_cb_overlay *)skb->cb;
697 sco->meta_slots_used = netbk_gop_skb(skb, &npo); 570 sco->meta_slots_used = xenvif_gop_skb(skb, &npo);
698 571
699 count += nr_frags + 1; 572 count += nr_frags + 1;
700 573
@@ -706,27 +579,27 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
706 break; 579 break;
707 } 580 }
708 581
709 BUG_ON(npo.meta_prod > ARRAY_SIZE(netbk->meta)); 582 BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta));
710 583
711 if (!npo.copy_prod) 584 if (!npo.copy_prod)
712 return; 585 return;
713 586
714 BUG_ON(npo.copy_prod > ARRAY_SIZE(netbk->grant_copy_op)); 587 BUG_ON(npo.copy_prod > ARRAY_SIZE(vif->grant_copy_op));
715 gnttab_batch_copy(netbk->grant_copy_op, npo.copy_prod); 588 gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod);
716 589
717 while ((skb = __skb_dequeue(&rxq)) != NULL) { 590 while ((skb = __skb_dequeue(&rxq)) != NULL) {
718 sco = (struct skb_cb_overlay *)skb->cb; 591 sco = (struct skb_cb_overlay *)skb->cb;
719 592
720 vif = netdev_priv(skb->dev); 593 vif = netdev_priv(skb->dev);
721 594
722 if (netbk->meta[npo.meta_cons].gso_size && vif->gso_prefix) { 595 if (vif->meta[npo.meta_cons].gso_size && vif->gso_prefix) {
723 resp = RING_GET_RESPONSE(&vif->rx, 596 resp = RING_GET_RESPONSE(&vif->rx,
724 vif->rx.rsp_prod_pvt++); 597 vif->rx.rsp_prod_pvt++);
725 598
726 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data; 599 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;
727 600
728 resp->offset = netbk->meta[npo.meta_cons].gso_size; 601 resp->offset = vif->meta[npo.meta_cons].gso_size;
729 resp->id = netbk->meta[npo.meta_cons].id; 602 resp->id = vif->meta[npo.meta_cons].id;
730 resp->status = sco->meta_slots_used; 603 resp->status = sco->meta_slots_used;
731 604
732 npo.meta_cons++; 605 npo.meta_cons++;
@@ -737,7 +610,7 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
737 vif->dev->stats.tx_bytes += skb->len; 610 vif->dev->stats.tx_bytes += skb->len;
738 vif->dev->stats.tx_packets++; 611 vif->dev->stats.tx_packets++;
739 612
740 status = netbk_check_gop(vif, sco->meta_slots_used, &npo); 613 status = xenvif_check_gop(vif, sco->meta_slots_used, &npo);
741 614
742 if (sco->meta_slots_used == 1) 615 if (sco->meta_slots_used == 1)
743 flags = 0; 616 flags = 0;
@@ -751,12 +624,12 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
751 flags |= XEN_NETRXF_data_validated; 624 flags |= XEN_NETRXF_data_validated;
752 625
753 offset = 0; 626 offset = 0;
754 resp = make_rx_response(vif, netbk->meta[npo.meta_cons].id, 627 resp = make_rx_response(vif, vif->meta[npo.meta_cons].id,
755 status, offset, 628 status, offset,
756 netbk->meta[npo.meta_cons].size, 629 vif->meta[npo.meta_cons].size,
757 flags); 630 flags);
758 631
759 if (netbk->meta[npo.meta_cons].gso_size && !vif->gso_prefix) { 632 if (vif->meta[npo.meta_cons].gso_size && !vif->gso_prefix) {
760 struct xen_netif_extra_info *gso = 633 struct xen_netif_extra_info *gso =
761 (struct xen_netif_extra_info *) 634 (struct xen_netif_extra_info *)
762 RING_GET_RESPONSE(&vif->rx, 635 RING_GET_RESPONSE(&vif->rx,
@@ -764,7 +637,7 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
764 637
765 resp->flags |= XEN_NETRXF_extra_info; 638 resp->flags |= XEN_NETRXF_extra_info;
766 639
767 gso->u.gso.size = netbk->meta[npo.meta_cons].gso_size; 640 gso->u.gso.size = vif->meta[npo.meta_cons].gso_size;
768 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; 641 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
769 gso->u.gso.pad = 0; 642 gso->u.gso.pad = 0;
770 gso->u.gso.features = 0; 643 gso->u.gso.features = 0;
@@ -773,123 +646,44 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
773 gso->flags = 0; 646 gso->flags = 0;
774 } 647 }
775 648
776 netbk_add_frag_responses(vif, status, 649 xenvif_add_frag_responses(vif, status,
777 netbk->meta + npo.meta_cons + 1, 650 vif->meta + npo.meta_cons + 1,
778 sco->meta_slots_used); 651 sco->meta_slots_used);
779 652
780 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret); 653 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret);
781 654
655 if (ret)
656 need_to_notify = 1;
657
782 xenvif_notify_tx_completion(vif); 658 xenvif_notify_tx_completion(vif);
783 659
784 if (ret && list_empty(&vif->notify_list))
785 list_add_tail(&vif->notify_list, &notify);
786 else
787 xenvif_put(vif);
788 npo.meta_cons += sco->meta_slots_used; 660 npo.meta_cons += sco->meta_slots_used;
789 dev_kfree_skb(skb); 661 dev_kfree_skb(skb);
790 } 662 }
791 663
792 list_for_each_entry_safe(vif, tmp, &notify, notify_list) { 664 if (need_to_notify)
793 notify_remote_via_irq(vif->rx_irq); 665 notify_remote_via_irq(vif->rx_irq);
794 list_del_init(&vif->notify_list);
795 xenvif_put(vif);
796 }
797 666
798 /* More work to do? */ 667 /* More work to do? */
799 if (!skb_queue_empty(&netbk->rx_queue) && 668 if (!skb_queue_empty(&vif->rx_queue))
800 !timer_pending(&netbk->net_timer)) 669 xenvif_kick_thread(vif);
801 xen_netbk_kick_thread(netbk);
802}
803
804void xen_netbk_queue_tx_skb(struct xenvif *vif, struct sk_buff *skb)
805{
806 struct xen_netbk *netbk = vif->netbk;
807
808 skb_queue_tail(&netbk->rx_queue, skb);
809
810 xen_netbk_kick_thread(netbk);
811}
812
813static void xen_netbk_alarm(unsigned long data)
814{
815 struct xen_netbk *netbk = (struct xen_netbk *)data;
816 xen_netbk_kick_thread(netbk);
817}
818
819static int __on_net_schedule_list(struct xenvif *vif)
820{
821 return !list_empty(&vif->schedule_list);
822} 670}
823 671
824/* Must be called with net_schedule_list_lock held */ 672void xenvif_queue_tx_skb(struct xenvif *vif, struct sk_buff *skb)
825static void remove_from_net_schedule_list(struct xenvif *vif)
826{ 673{
827 if (likely(__on_net_schedule_list(vif))) { 674 skb_queue_tail(&vif->rx_queue, skb);
828 list_del_init(&vif->schedule_list);
829 xenvif_put(vif);
830 }
831}
832
833static struct xenvif *poll_net_schedule_list(struct xen_netbk *netbk)
834{
835 struct xenvif *vif = NULL;
836
837 spin_lock_irq(&netbk->net_schedule_list_lock);
838 if (list_empty(&netbk->net_schedule_list))
839 goto out;
840
841 vif = list_first_entry(&netbk->net_schedule_list,
842 struct xenvif, schedule_list);
843 if (!vif)
844 goto out;
845
846 xenvif_get(vif);
847 675
848 remove_from_net_schedule_list(vif); 676 xenvif_kick_thread(vif);
849out:
850 spin_unlock_irq(&netbk->net_schedule_list_lock);
851 return vif;
852} 677}
853 678
854void xen_netbk_schedule_xenvif(struct xenvif *vif) 679void xenvif_check_rx_xenvif(struct xenvif *vif)
855{
856 unsigned long flags;
857 struct xen_netbk *netbk = vif->netbk;
858
859 if (__on_net_schedule_list(vif))
860 goto kick;
861
862 spin_lock_irqsave(&netbk->net_schedule_list_lock, flags);
863 if (!__on_net_schedule_list(vif) &&
864 likely(xenvif_schedulable(vif))) {
865 list_add_tail(&vif->schedule_list, &netbk->net_schedule_list);
866 xenvif_get(vif);
867 }
868 spin_unlock_irqrestore(&netbk->net_schedule_list_lock, flags);
869
870kick:
871 smp_mb();
872 if ((nr_pending_reqs(netbk) < (MAX_PENDING_REQS/2)) &&
873 !list_empty(&netbk->net_schedule_list))
874 xen_netbk_kick_thread(netbk);
875}
876
877void xen_netbk_deschedule_xenvif(struct xenvif *vif)
878{
879 struct xen_netbk *netbk = vif->netbk;
880 spin_lock_irq(&netbk->net_schedule_list_lock);
881 remove_from_net_schedule_list(vif);
882 spin_unlock_irq(&netbk->net_schedule_list_lock);
883}
884
885void xen_netbk_check_rx_xenvif(struct xenvif *vif)
886{ 680{
887 int more_to_do; 681 int more_to_do;
888 682
889 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do); 683 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do);
890 684
891 if (more_to_do) 685 if (more_to_do)
892 xen_netbk_schedule_xenvif(vif); 686 napi_schedule(&vif->napi);
893} 687}
894 688
895static void tx_add_credit(struct xenvif *vif) 689static void tx_add_credit(struct xenvif *vif)
@@ -916,11 +710,11 @@ static void tx_credit_callback(unsigned long data)
916{ 710{
917 struct xenvif *vif = (struct xenvif *)data; 711 struct xenvif *vif = (struct xenvif *)data;
918 tx_add_credit(vif); 712 tx_add_credit(vif);
919 xen_netbk_check_rx_xenvif(vif); 713 xenvif_check_rx_xenvif(vif);
920} 714}
921 715
922static void netbk_tx_err(struct xenvif *vif, 716static void xenvif_tx_err(struct xenvif *vif,
923 struct xen_netif_tx_request *txp, RING_IDX end) 717 struct xen_netif_tx_request *txp, RING_IDX end)
924{ 718{
925 RING_IDX cons = vif->tx.req_cons; 719 RING_IDX cons = vif->tx.req_cons;
926 720
@@ -931,21 +725,18 @@ static void netbk_tx_err(struct xenvif *vif,
931 txp = RING_GET_REQUEST(&vif->tx, cons++); 725 txp = RING_GET_REQUEST(&vif->tx, cons++);
932 } while (1); 726 } while (1);
933 vif->tx.req_cons = cons; 727 vif->tx.req_cons = cons;
934 xen_netbk_check_rx_xenvif(vif);
935 xenvif_put(vif);
936} 728}
937 729
938static void netbk_fatal_tx_err(struct xenvif *vif) 730static void xenvif_fatal_tx_err(struct xenvif *vif)
939{ 731{
940 netdev_err(vif->dev, "fatal error; disabling device\n"); 732 netdev_err(vif->dev, "fatal error; disabling device\n");
941 xenvif_carrier_off(vif); 733 xenvif_carrier_off(vif);
942 xenvif_put(vif);
943} 734}
944 735
945static int netbk_count_requests(struct xenvif *vif, 736static int xenvif_count_requests(struct xenvif *vif,
946 struct xen_netif_tx_request *first, 737 struct xen_netif_tx_request *first,
947 struct xen_netif_tx_request *txp, 738 struct xen_netif_tx_request *txp,
948 int work_to_do) 739 int work_to_do)
949{ 740{
950 RING_IDX cons = vif->tx.req_cons; 741 RING_IDX cons = vif->tx.req_cons;
951 int slots = 0; 742 int slots = 0;
@@ -962,7 +753,7 @@ static int netbk_count_requests(struct xenvif *vif,
962 netdev_err(vif->dev, 753 netdev_err(vif->dev,
963 "Asked for %d slots but exceeds this limit\n", 754 "Asked for %d slots but exceeds this limit\n",
964 work_to_do); 755 work_to_do);
965 netbk_fatal_tx_err(vif); 756 xenvif_fatal_tx_err(vif);
966 return -ENODATA; 757 return -ENODATA;
967 } 758 }
968 759
@@ -973,7 +764,7 @@ static int netbk_count_requests(struct xenvif *vif,
973 netdev_err(vif->dev, 764 netdev_err(vif->dev,
974 "Malicious frontend using %d slots, threshold %u\n", 765 "Malicious frontend using %d slots, threshold %u\n",
975 slots, fatal_skb_slots); 766 slots, fatal_skb_slots);
976 netbk_fatal_tx_err(vif); 767 xenvif_fatal_tx_err(vif);
977 return -E2BIG; 768 return -E2BIG;
978 } 769 }
979 770
@@ -1021,7 +812,7 @@ static int netbk_count_requests(struct xenvif *vif,
1021 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) { 812 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
1022 netdev_err(vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n", 813 netdev_err(vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n",
1023 txp->offset, txp->size); 814 txp->offset, txp->size);
1024 netbk_fatal_tx_err(vif); 815 xenvif_fatal_tx_err(vif);
1025 return -EINVAL; 816 return -EINVAL;
1026 } 817 }
1027 818
@@ -1033,30 +824,30 @@ static int netbk_count_requests(struct xenvif *vif,
1033 } while (more_data); 824 } while (more_data);
1034 825
1035 if (drop_err) { 826 if (drop_err) {
1036 netbk_tx_err(vif, first, cons + slots); 827 xenvif_tx_err(vif, first, cons + slots);
1037 return drop_err; 828 return drop_err;
1038 } 829 }
1039 830
1040 return slots; 831 return slots;
1041} 832}
1042 833
1043static struct page *xen_netbk_alloc_page(struct xen_netbk *netbk, 834static struct page *xenvif_alloc_page(struct xenvif *vif,
1044 u16 pending_idx) 835 u16 pending_idx)
1045{ 836{
1046 struct page *page; 837 struct page *page;
1047 page = alloc_page(GFP_KERNEL|__GFP_COLD); 838
839 page = alloc_page(GFP_ATOMIC|__GFP_COLD);
1048 if (!page) 840 if (!page)
1049 return NULL; 841 return NULL;
1050 set_page_ext(page, netbk, pending_idx); 842 vif->mmap_pages[pending_idx] = page;
1051 netbk->mmap_pages[pending_idx] = page; 843
1052 return page; 844 return page;
1053} 845}
1054 846
1055static struct gnttab_copy *xen_netbk_get_requests(struct xen_netbk *netbk, 847static struct gnttab_copy *xenvif_get_requests(struct xenvif *vif,
1056 struct xenvif *vif, 848 struct sk_buff *skb,
1057 struct sk_buff *skb, 849 struct xen_netif_tx_request *txp,
1058 struct xen_netif_tx_request *txp, 850 struct gnttab_copy *gop)
1059 struct gnttab_copy *gop)
1060{ 851{
1061 struct skb_shared_info *shinfo = skb_shinfo(skb); 852 struct skb_shared_info *shinfo = skb_shinfo(skb);
1062 skb_frag_t *frags = shinfo->frags; 853 skb_frag_t *frags = shinfo->frags;
@@ -1079,14 +870,14 @@ static struct gnttab_copy *xen_netbk_get_requests(struct xen_netbk *netbk,
1079 870
1080 /* Coalesce tx requests, at this point the packet passed in 871 /* Coalesce tx requests, at this point the packet passed in
1081 * should be <= 64K. Any packets larger than 64K have been 872 * should be <= 64K. Any packets larger than 64K have been
1082 * handled in netbk_count_requests(). 873 * handled in xenvif_count_requests().
1083 */ 874 */
1084 for (shinfo->nr_frags = slot = start; slot < nr_slots; 875 for (shinfo->nr_frags = slot = start; slot < nr_slots;
1085 shinfo->nr_frags++) { 876 shinfo->nr_frags++) {
1086 struct pending_tx_info *pending_tx_info = 877 struct pending_tx_info *pending_tx_info =
1087 netbk->pending_tx_info; 878 vif->pending_tx_info;
1088 879
1089 page = alloc_page(GFP_KERNEL|__GFP_COLD); 880 page = alloc_page(GFP_ATOMIC|__GFP_COLD);
1090 if (!page) 881 if (!page)
1091 goto err; 882 goto err;
1092 883
@@ -1121,21 +912,18 @@ static struct gnttab_copy *xen_netbk_get_requests(struct xen_netbk *netbk,
1121 gop->len = txp->size; 912 gop->len = txp->size;
1122 dst_offset += gop->len; 913 dst_offset += gop->len;
1123 914
1124 index = pending_index(netbk->pending_cons++); 915 index = pending_index(vif->pending_cons++);
1125 916
1126 pending_idx = netbk->pending_ring[index]; 917 pending_idx = vif->pending_ring[index];
1127 918
1128 memcpy(&pending_tx_info[pending_idx].req, txp, 919 memcpy(&pending_tx_info[pending_idx].req, txp,
1129 sizeof(*txp)); 920 sizeof(*txp));
1130 xenvif_get(vif);
1131
1132 pending_tx_info[pending_idx].vif = vif;
1133 921
1134 /* Poison these fields, corresponding 922 /* Poison these fields, corresponding
1135 * fields for head tx req will be set 923 * fields for head tx req will be set
1136 * to correct values after the loop. 924 * to correct values after the loop.
1137 */ 925 */
1138 netbk->mmap_pages[pending_idx] = (void *)(~0UL); 926 vif->mmap_pages[pending_idx] = (void *)(~0UL);
1139 pending_tx_info[pending_idx].head = 927 pending_tx_info[pending_idx].head =
1140 INVALID_PENDING_RING_IDX; 928 INVALID_PENDING_RING_IDX;
1141 929
@@ -1155,8 +943,7 @@ static struct gnttab_copy *xen_netbk_get_requests(struct xen_netbk *netbk,
1155 first->req.offset = 0; 943 first->req.offset = 0;
1156 first->req.size = dst_offset; 944 first->req.size = dst_offset;
1157 first->head = start_idx; 945 first->head = start_idx;
1158 set_page_ext(page, netbk, head_idx); 946 vif->mmap_pages[head_idx] = page;
1159 netbk->mmap_pages[head_idx] = page;
1160 frag_set_pending_idx(&frags[shinfo->nr_frags], head_idx); 947 frag_set_pending_idx(&frags[shinfo->nr_frags], head_idx);
1161 } 948 }
1162 949
@@ -1166,20 +953,20 @@ static struct gnttab_copy *xen_netbk_get_requests(struct xen_netbk *netbk,
1166err: 953err:
1167 /* Unwind, freeing all pages and sending error responses. */ 954 /* Unwind, freeing all pages and sending error responses. */
1168 while (shinfo->nr_frags-- > start) { 955 while (shinfo->nr_frags-- > start) {
1169 xen_netbk_idx_release(netbk, 956 xenvif_idx_release(vif,
1170 frag_get_pending_idx(&frags[shinfo->nr_frags]), 957 frag_get_pending_idx(&frags[shinfo->nr_frags]),
1171 XEN_NETIF_RSP_ERROR); 958 XEN_NETIF_RSP_ERROR);
1172 } 959 }
1173 /* The head too, if necessary. */ 960 /* The head too, if necessary. */
1174 if (start) 961 if (start)
1175 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_ERROR); 962 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
1176 963
1177 return NULL; 964 return NULL;
1178} 965}
1179 966
1180static int xen_netbk_tx_check_gop(struct xen_netbk *netbk, 967static int xenvif_tx_check_gop(struct xenvif *vif,
1181 struct sk_buff *skb, 968 struct sk_buff *skb,
1182 struct gnttab_copy **gopp) 969 struct gnttab_copy **gopp)
1183{ 970{
1184 struct gnttab_copy *gop = *gopp; 971 struct gnttab_copy *gop = *gopp;
1185 u16 pending_idx = *((u16 *)skb->data); 972 u16 pending_idx = *((u16 *)skb->data);
@@ -1192,7 +979,7 @@ static int xen_netbk_tx_check_gop(struct xen_netbk *netbk,
1192 /* Check status of header. */ 979 /* Check status of header. */
1193 err = gop->status; 980 err = gop->status;
1194 if (unlikely(err)) 981 if (unlikely(err))
1195 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_ERROR); 982 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
1196 983
1197 /* Skip first skb fragment if it is on same page as header fragment. */ 984 /* Skip first skb fragment if it is on same page as header fragment. */
1198 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); 985 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
@@ -1202,7 +989,7 @@ static int xen_netbk_tx_check_gop(struct xen_netbk *netbk,
1202 pending_ring_idx_t head; 989 pending_ring_idx_t head;
1203 990
1204 pending_idx = frag_get_pending_idx(&shinfo->frags[i]); 991 pending_idx = frag_get_pending_idx(&shinfo->frags[i]);
1205 tx_info = &netbk->pending_tx_info[pending_idx]; 992 tx_info = &vif->pending_tx_info[pending_idx];
1206 head = tx_info->head; 993 head = tx_info->head;
1207 994
1208 /* Check error status: if okay then remember grant handle. */ 995 /* Check error status: if okay then remember grant handle. */
@@ -1210,18 +997,19 @@ static int xen_netbk_tx_check_gop(struct xen_netbk *netbk,
1210 newerr = (++gop)->status; 997 newerr = (++gop)->status;
1211 if (newerr) 998 if (newerr)
1212 break; 999 break;
1213 peek = netbk->pending_ring[pending_index(++head)]; 1000 peek = vif->pending_ring[pending_index(++head)];
1214 } while (!pending_tx_is_head(netbk, peek)); 1001 } while (!pending_tx_is_head(vif, peek));
1215 1002
1216 if (likely(!newerr)) { 1003 if (likely(!newerr)) {
1217 /* Had a previous error? Invalidate this fragment. */ 1004 /* Had a previous error? Invalidate this fragment. */
1218 if (unlikely(err)) 1005 if (unlikely(err))
1219 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1006 xenvif_idx_release(vif, pending_idx,
1007 XEN_NETIF_RSP_OKAY);
1220 continue; 1008 continue;
1221 } 1009 }
1222 1010
1223 /* Error on this fragment: respond to client with an error. */ 1011 /* Error on this fragment: respond to client with an error. */
1224 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_ERROR); 1012 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
1225 1013
1226 /* Not the first error? Preceding frags already invalidated. */ 1014 /* Not the first error? Preceding frags already invalidated. */
1227 if (err) 1015 if (err)
@@ -1229,10 +1017,11 @@ static int xen_netbk_tx_check_gop(struct xen_netbk *netbk,
1229 1017
1230 /* First error: invalidate header and preceding fragments. */ 1018 /* First error: invalidate header and preceding fragments. */
1231 pending_idx = *((u16 *)skb->data); 1019 pending_idx = *((u16 *)skb->data);
1232 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1020 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY);
1233 for (j = start; j < i; j++) { 1021 for (j = start; j < i; j++) {
1234 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 1022 pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
1235 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1023 xenvif_idx_release(vif, pending_idx,
1024 XEN_NETIF_RSP_OKAY);
1236 } 1025 }
1237 1026
1238 /* Remember the error: invalidate all subsequent fragments. */ 1027 /* Remember the error: invalidate all subsequent fragments. */
@@ -1243,7 +1032,7 @@ static int xen_netbk_tx_check_gop(struct xen_netbk *netbk,
1243 return err; 1032 return err;
1244} 1033}
1245 1034
1246static void xen_netbk_fill_frags(struct xen_netbk *netbk, struct sk_buff *skb) 1035static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
1247{ 1036{
1248 struct skb_shared_info *shinfo = skb_shinfo(skb); 1037 struct skb_shared_info *shinfo = skb_shinfo(skb);
1249 int nr_frags = shinfo->nr_frags; 1038 int nr_frags = shinfo->nr_frags;
@@ -1257,20 +1046,20 @@ static void xen_netbk_fill_frags(struct xen_netbk *netbk, struct sk_buff *skb)
1257 1046
1258 pending_idx = frag_get_pending_idx(frag); 1047 pending_idx = frag_get_pending_idx(frag);
1259 1048
1260 txp = &netbk->pending_tx_info[pending_idx].req; 1049 txp = &vif->pending_tx_info[pending_idx].req;
1261 page = virt_to_page(idx_to_kaddr(netbk, pending_idx)); 1050 page = virt_to_page(idx_to_kaddr(vif, pending_idx));
1262 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); 1051 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size);
1263 skb->len += txp->size; 1052 skb->len += txp->size;
1264 skb->data_len += txp->size; 1053 skb->data_len += txp->size;
1265 skb->truesize += txp->size; 1054 skb->truesize += txp->size;
1266 1055
1267 /* Take an extra reference to offset xen_netbk_idx_release */ 1056 /* Take an extra reference to offset xenvif_idx_release */
1268 get_page(netbk->mmap_pages[pending_idx]); 1057 get_page(vif->mmap_pages[pending_idx]);
1269 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1058 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY);
1270 } 1059 }
1271} 1060}
1272 1061
1273static int xen_netbk_get_extras(struct xenvif *vif, 1062static int xenvif_get_extras(struct xenvif *vif,
1274 struct xen_netif_extra_info *extras, 1063 struct xen_netif_extra_info *extras,
1275 int work_to_do) 1064 int work_to_do)
1276{ 1065{
@@ -1280,7 +1069,7 @@ static int xen_netbk_get_extras(struct xenvif *vif,
1280 do { 1069 do {
1281 if (unlikely(work_to_do-- <= 0)) { 1070 if (unlikely(work_to_do-- <= 0)) {
1282 netdev_err(vif->dev, "Missing extra info\n"); 1071 netdev_err(vif->dev, "Missing extra info\n");
1283 netbk_fatal_tx_err(vif); 1072 xenvif_fatal_tx_err(vif);
1284 return -EBADR; 1073 return -EBADR;
1285 } 1074 }
1286 1075
@@ -1291,7 +1080,7 @@ static int xen_netbk_get_extras(struct xenvif *vif,
1291 vif->tx.req_cons = ++cons; 1080 vif->tx.req_cons = ++cons;
1292 netdev_err(vif->dev, 1081 netdev_err(vif->dev,
1293 "Invalid extra type: %d\n", extra.type); 1082 "Invalid extra type: %d\n", extra.type);
1294 netbk_fatal_tx_err(vif); 1083 xenvif_fatal_tx_err(vif);
1295 return -EINVAL; 1084 return -EINVAL;
1296 } 1085 }
1297 1086
@@ -1302,20 +1091,20 @@ static int xen_netbk_get_extras(struct xenvif *vif,
1302 return work_to_do; 1091 return work_to_do;
1303} 1092}
1304 1093
1305static int netbk_set_skb_gso(struct xenvif *vif, 1094static int xenvif_set_skb_gso(struct xenvif *vif,
1306 struct sk_buff *skb, 1095 struct sk_buff *skb,
1307 struct xen_netif_extra_info *gso) 1096 struct xen_netif_extra_info *gso)
1308{ 1097{
1309 if (!gso->u.gso.size) { 1098 if (!gso->u.gso.size) {
1310 netdev_err(vif->dev, "GSO size must not be zero.\n"); 1099 netdev_err(vif->dev, "GSO size must not be zero.\n");
1311 netbk_fatal_tx_err(vif); 1100 xenvif_fatal_tx_err(vif);
1312 return -EINVAL; 1101 return -EINVAL;
1313 } 1102 }
1314 1103
1315 /* Currently only TCPv4 S.O. is supported. */ 1104 /* Currently only TCPv4 S.O. is supported. */
1316 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) { 1105 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) {
1317 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type); 1106 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type);
1318 netbk_fatal_tx_err(vif); 1107 xenvif_fatal_tx_err(vif);
1319 return -EINVAL; 1108 return -EINVAL;
1320 } 1109 }
1321 1110
@@ -1426,16 +1215,14 @@ static bool tx_credit_exceeded(struct xenvif *vif, unsigned size)
1426 return false; 1215 return false;
1427} 1216}
1428 1217
1429static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk) 1218static unsigned xenvif_tx_build_gops(struct xenvif *vif)
1430{ 1219{
1431 struct gnttab_copy *gop = netbk->tx_copy_ops, *request_gop; 1220 struct gnttab_copy *gop = vif->tx_copy_ops, *request_gop;
1432 struct sk_buff *skb; 1221 struct sk_buff *skb;
1433 int ret; 1222 int ret;
1434 1223
1435 while ((nr_pending_reqs(netbk) + XEN_NETBK_LEGACY_SLOTS_MAX 1224 while ((nr_pending_reqs(vif) + XEN_NETBK_LEGACY_SLOTS_MAX
1436 < MAX_PENDING_REQS) && 1225 < MAX_PENDING_REQS)) {
1437 !list_empty(&netbk->net_schedule_list)) {
1438 struct xenvif *vif;
1439 struct xen_netif_tx_request txreq; 1226 struct xen_netif_tx_request txreq;
1440 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX]; 1227 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX];
1441 struct page *page; 1228 struct page *page;
@@ -1446,16 +1233,6 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1446 unsigned int data_len; 1233 unsigned int data_len;
1447 pending_ring_idx_t index; 1234 pending_ring_idx_t index;
1448 1235
1449 /* Get a netif from the list with work to do. */
1450 vif = poll_net_schedule_list(netbk);
1451 /* This can sometimes happen because the test of
1452 * list_empty(net_schedule_list) at the top of the
1453 * loop is unlocked. Just go back and have another
1454 * look.
1455 */
1456 if (!vif)
1457 continue;
1458
1459 if (vif->tx.sring->req_prod - vif->tx.req_cons > 1236 if (vif->tx.sring->req_prod - vif->tx.req_cons >
1460 XEN_NETIF_TX_RING_SIZE) { 1237 XEN_NETIF_TX_RING_SIZE) {
1461 netdev_err(vif->dev, 1238 netdev_err(vif->dev,
@@ -1463,15 +1240,13 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1463 "req_prod %d, req_cons %d, size %ld\n", 1240 "req_prod %d, req_cons %d, size %ld\n",
1464 vif->tx.sring->req_prod, vif->tx.req_cons, 1241 vif->tx.sring->req_prod, vif->tx.req_cons,
1465 XEN_NETIF_TX_RING_SIZE); 1242 XEN_NETIF_TX_RING_SIZE);
1466 netbk_fatal_tx_err(vif); 1243 xenvif_fatal_tx_err(vif);
1467 continue; 1244 continue;
1468 } 1245 }
1469 1246
1470 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, work_to_do); 1247 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, work_to_do);
1471 if (!work_to_do) { 1248 if (!work_to_do)
1472 xenvif_put(vif); 1249 break;
1473 continue;
1474 }
1475 1250
1476 idx = vif->tx.req_cons; 1251 idx = vif->tx.req_cons;
1477 rmb(); /* Ensure that we see the request before we copy it. */ 1252 rmb(); /* Ensure that we see the request before we copy it. */
@@ -1479,10 +1254,8 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1479 1254
1480 /* Credit-based scheduling. */ 1255 /* Credit-based scheduling. */
1481 if (txreq.size > vif->remaining_credit && 1256 if (txreq.size > vif->remaining_credit &&
1482 tx_credit_exceeded(vif, txreq.size)) { 1257 tx_credit_exceeded(vif, txreq.size))
1483 xenvif_put(vif); 1258 break;
1484 continue;
1485 }
1486 1259
1487 vif->remaining_credit -= txreq.size; 1260 vif->remaining_credit -= txreq.size;
1488 1261
@@ -1491,24 +1264,24 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1491 1264
1492 memset(extras, 0, sizeof(extras)); 1265 memset(extras, 0, sizeof(extras));
1493 if (txreq.flags & XEN_NETTXF_extra_info) { 1266 if (txreq.flags & XEN_NETTXF_extra_info) {
1494 work_to_do = xen_netbk_get_extras(vif, extras, 1267 work_to_do = xenvif_get_extras(vif, extras,
1495 work_to_do); 1268 work_to_do);
1496 idx = vif->tx.req_cons; 1269 idx = vif->tx.req_cons;
1497 if (unlikely(work_to_do < 0)) 1270 if (unlikely(work_to_do < 0))
1498 continue; 1271 break;
1499 } 1272 }
1500 1273
1501 ret = netbk_count_requests(vif, &txreq, txfrags, work_to_do); 1274 ret = xenvif_count_requests(vif, &txreq, txfrags, work_to_do);
1502 if (unlikely(ret < 0)) 1275 if (unlikely(ret < 0))
1503 continue; 1276 break;
1504 1277
1505 idx += ret; 1278 idx += ret;
1506 1279
1507 if (unlikely(txreq.size < ETH_HLEN)) { 1280 if (unlikely(txreq.size < ETH_HLEN)) {
1508 netdev_dbg(vif->dev, 1281 netdev_dbg(vif->dev,
1509 "Bad packet size: %d\n", txreq.size); 1282 "Bad packet size: %d\n", txreq.size);
1510 netbk_tx_err(vif, &txreq, idx); 1283 xenvif_tx_err(vif, &txreq, idx);
1511 continue; 1284 break;
1512 } 1285 }
1513 1286
1514 /* No crossing a page as the payload mustn't fragment. */ 1287 /* No crossing a page as the payload mustn't fragment. */
@@ -1517,12 +1290,12 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1517 "txreq.offset: %x, size: %u, end: %lu\n", 1290 "txreq.offset: %x, size: %u, end: %lu\n",
1518 txreq.offset, txreq.size, 1291 txreq.offset, txreq.size,
1519 (txreq.offset&~PAGE_MASK) + txreq.size); 1292 (txreq.offset&~PAGE_MASK) + txreq.size);
1520 netbk_fatal_tx_err(vif); 1293 xenvif_fatal_tx_err(vif);
1521 continue; 1294 break;
1522 } 1295 }
1523 1296
1524 index = pending_index(netbk->pending_cons); 1297 index = pending_index(vif->pending_cons);
1525 pending_idx = netbk->pending_ring[index]; 1298 pending_idx = vif->pending_ring[index];
1526 1299
1527 data_len = (txreq.size > PKT_PROT_LEN && 1300 data_len = (txreq.size > PKT_PROT_LEN &&
1528 ret < XEN_NETBK_LEGACY_SLOTS_MAX) ? 1301 ret < XEN_NETBK_LEGACY_SLOTS_MAX) ?
@@ -1533,7 +1306,7 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1533 if (unlikely(skb == NULL)) { 1306 if (unlikely(skb == NULL)) {
1534 netdev_dbg(vif->dev, 1307 netdev_dbg(vif->dev,
1535 "Can't allocate a skb in start_xmit.\n"); 1308 "Can't allocate a skb in start_xmit.\n");
1536 netbk_tx_err(vif, &txreq, idx); 1309 xenvif_tx_err(vif, &txreq, idx);
1537 break; 1310 break;
1538 } 1311 }
1539 1312
@@ -1544,19 +1317,19 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1544 struct xen_netif_extra_info *gso; 1317 struct xen_netif_extra_info *gso;
1545 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1318 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1546 1319
1547 if (netbk_set_skb_gso(vif, skb, gso)) { 1320 if (xenvif_set_skb_gso(vif, skb, gso)) {
1548 /* Failure in netbk_set_skb_gso is fatal. */ 1321 /* Failure in xenvif_set_skb_gso is fatal. */
1549 kfree_skb(skb); 1322 kfree_skb(skb);
1550 continue; 1323 break;
1551 } 1324 }
1552 } 1325 }
1553 1326
1554 /* XXX could copy straight to head */ 1327 /* XXX could copy straight to head */
1555 page = xen_netbk_alloc_page(netbk, pending_idx); 1328 page = xenvif_alloc_page(vif, pending_idx);
1556 if (!page) { 1329 if (!page) {
1557 kfree_skb(skb); 1330 kfree_skb(skb);
1558 netbk_tx_err(vif, &txreq, idx); 1331 xenvif_tx_err(vif, &txreq, idx);
1559 continue; 1332 break;
1560 } 1333 }
1561 1334
1562 gop->source.u.ref = txreq.gref; 1335 gop->source.u.ref = txreq.gref;
@@ -1572,10 +1345,9 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1572 1345
1573 gop++; 1346 gop++;
1574 1347
1575 memcpy(&netbk->pending_tx_info[pending_idx].req, 1348 memcpy(&vif->pending_tx_info[pending_idx].req,
1576 &txreq, sizeof(txreq)); 1349 &txreq, sizeof(txreq));
1577 netbk->pending_tx_info[pending_idx].vif = vif; 1350 vif->pending_tx_info[pending_idx].head = index;
1578 netbk->pending_tx_info[pending_idx].head = index;
1579 *((u16 *)skb->data) = pending_idx; 1351 *((u16 *)skb->data) = pending_idx;
1580 1352
1581 __skb_put(skb, data_len); 1353 __skb_put(skb, data_len);
@@ -1590,46 +1362,45 @@ static unsigned xen_netbk_tx_build_gops(struct xen_netbk *netbk)
1590 INVALID_PENDING_IDX); 1362 INVALID_PENDING_IDX);
1591 } 1363 }
1592 1364
1593 netbk->pending_cons++; 1365 vif->pending_cons++;
1594 1366
1595 request_gop = xen_netbk_get_requests(netbk, vif, 1367 request_gop = xenvif_get_requests(vif, skb, txfrags, gop);
1596 skb, txfrags, gop);
1597 if (request_gop == NULL) { 1368 if (request_gop == NULL) {
1598 kfree_skb(skb); 1369 kfree_skb(skb);
1599 netbk_tx_err(vif, &txreq, idx); 1370 xenvif_tx_err(vif, &txreq, idx);
1600 continue; 1371 break;
1601 } 1372 }
1602 gop = request_gop; 1373 gop = request_gop;
1603 1374
1604 __skb_queue_tail(&netbk->tx_queue, skb); 1375 __skb_queue_tail(&vif->tx_queue, skb);
1605 1376
1606 vif->tx.req_cons = idx; 1377 vif->tx.req_cons = idx;
1607 xen_netbk_check_rx_xenvif(vif);
1608 1378
1609 if ((gop-netbk->tx_copy_ops) >= ARRAY_SIZE(netbk->tx_copy_ops)) 1379 if ((gop-vif->tx_copy_ops) >= ARRAY_SIZE(vif->tx_copy_ops))
1610 break; 1380 break;
1611 } 1381 }
1612 1382
1613 return gop - netbk->tx_copy_ops; 1383 return gop - vif->tx_copy_ops;
1614} 1384}
1615 1385
1616static void xen_netbk_tx_submit(struct xen_netbk *netbk) 1386
1387static int xenvif_tx_submit(struct xenvif *vif, int budget)
1617{ 1388{
1618 struct gnttab_copy *gop = netbk->tx_copy_ops; 1389 struct gnttab_copy *gop = vif->tx_copy_ops;
1619 struct sk_buff *skb; 1390 struct sk_buff *skb;
1391 int work_done = 0;
1620 1392
1621 while ((skb = __skb_dequeue(&netbk->tx_queue)) != NULL) { 1393 while (work_done < budget &&
1394 (skb = __skb_dequeue(&vif->tx_queue)) != NULL) {
1622 struct xen_netif_tx_request *txp; 1395 struct xen_netif_tx_request *txp;
1623 struct xenvif *vif;
1624 u16 pending_idx; 1396 u16 pending_idx;
1625 unsigned data_len; 1397 unsigned data_len;
1626 1398
1627 pending_idx = *((u16 *)skb->data); 1399 pending_idx = *((u16 *)skb->data);
1628 vif = netbk->pending_tx_info[pending_idx].vif; 1400 txp = &vif->pending_tx_info[pending_idx].req;
1629 txp = &netbk->pending_tx_info[pending_idx].req;
1630 1401
1631 /* Check the remap error code. */ 1402 /* Check the remap error code. */
1632 if (unlikely(xen_netbk_tx_check_gop(netbk, skb, &gop))) { 1403 if (unlikely(xenvif_tx_check_gop(vif, skb, &gop))) {
1633 netdev_dbg(vif->dev, "netback grant failed.\n"); 1404 netdev_dbg(vif->dev, "netback grant failed.\n");
1634 skb_shinfo(skb)->nr_frags = 0; 1405 skb_shinfo(skb)->nr_frags = 0;
1635 kfree_skb(skb); 1406 kfree_skb(skb);
@@ -1638,7 +1409,7 @@ static void xen_netbk_tx_submit(struct xen_netbk *netbk)
1638 1409
1639 data_len = skb->len; 1410 data_len = skb->len;
1640 memcpy(skb->data, 1411 memcpy(skb->data,
1641 (void *)(idx_to_kaddr(netbk, pending_idx)|txp->offset), 1412 (void *)(idx_to_kaddr(vif, pending_idx)|txp->offset),
1642 data_len); 1413 data_len);
1643 if (data_len < txp->size) { 1414 if (data_len < txp->size) {
1644 /* Append the packet payload as a fragment. */ 1415 /* Append the packet payload as a fragment. */
@@ -1646,7 +1417,8 @@ static void xen_netbk_tx_submit(struct xen_netbk *netbk)
1646 txp->size -= data_len; 1417 txp->size -= data_len;
1647 } else { 1418 } else {
1648 /* Schedule a response immediately. */ 1419 /* Schedule a response immediately. */
1649 xen_netbk_idx_release(netbk, pending_idx, XEN_NETIF_RSP_OKAY); 1420 xenvif_idx_release(vif, pending_idx,
1421 XEN_NETIF_RSP_OKAY);
1650 } 1422 }
1651 1423
1652 if (txp->flags & XEN_NETTXF_csum_blank) 1424 if (txp->flags & XEN_NETTXF_csum_blank)
@@ -1654,7 +1426,7 @@ static void xen_netbk_tx_submit(struct xen_netbk *netbk)
1654 else if (txp->flags & XEN_NETTXF_data_validated) 1426 else if (txp->flags & XEN_NETTXF_data_validated)
1655 skb->ip_summed = CHECKSUM_UNNECESSARY; 1427 skb->ip_summed = CHECKSUM_UNNECESSARY;
1656 1428
1657 xen_netbk_fill_frags(netbk, skb); 1429 xenvif_fill_frags(vif, skb);
1658 1430
1659 /* 1431 /*
1660 * If the initial fragment was < PKT_PROT_LEN then 1432 * If the initial fragment was < PKT_PROT_LEN then
@@ -1682,53 +1454,61 @@ static void xen_netbk_tx_submit(struct xen_netbk *netbk)
1682 vif->dev->stats.rx_bytes += skb->len; 1454 vif->dev->stats.rx_bytes += skb->len;
1683 vif->dev->stats.rx_packets++; 1455 vif->dev->stats.rx_packets++;
1684 1456
1685 xenvif_receive_skb(vif, skb); 1457 work_done++;
1458
1459 netif_receive_skb(skb);
1686 } 1460 }
1461
1462 return work_done;
1687} 1463}
1688 1464
1689/* Called after netfront has transmitted */ 1465/* Called after netfront has transmitted */
1690static void xen_netbk_tx_action(struct xen_netbk *netbk) 1466int xenvif_tx_action(struct xenvif *vif, int budget)
1691{ 1467{
1692 unsigned nr_gops; 1468 unsigned nr_gops;
1469 int work_done;
1693 1470
1694 nr_gops = xen_netbk_tx_build_gops(netbk); 1471 if (unlikely(!tx_work_todo(vif)))
1472 return 0;
1473
1474 nr_gops = xenvif_tx_build_gops(vif);
1695 1475
1696 if (nr_gops == 0) 1476 if (nr_gops == 0)
1697 return; 1477 return 0;
1698 1478
1699 gnttab_batch_copy(netbk->tx_copy_ops, nr_gops); 1479 gnttab_batch_copy(vif->tx_copy_ops, nr_gops);
1700 1480
1701 xen_netbk_tx_submit(netbk); 1481 work_done = xenvif_tx_submit(vif, nr_gops);
1482
1483 return work_done;
1702} 1484}
1703 1485
1704static void xen_netbk_idx_release(struct xen_netbk *netbk, u16 pending_idx, 1486static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx,
1705 u8 status) 1487 u8 status)
1706{ 1488{
1707 struct xenvif *vif;
1708 struct pending_tx_info *pending_tx_info; 1489 struct pending_tx_info *pending_tx_info;
1709 pending_ring_idx_t head; 1490 pending_ring_idx_t head;
1710 u16 peek; /* peek into next tx request */ 1491 u16 peek; /* peek into next tx request */
1711 1492
1712 BUG_ON(netbk->mmap_pages[pending_idx] == (void *)(~0UL)); 1493 BUG_ON(vif->mmap_pages[pending_idx] == (void *)(~0UL));
1713 1494
1714 /* Already complete? */ 1495 /* Already complete? */
1715 if (netbk->mmap_pages[pending_idx] == NULL) 1496 if (vif->mmap_pages[pending_idx] == NULL)
1716 return; 1497 return;
1717 1498
1718 pending_tx_info = &netbk->pending_tx_info[pending_idx]; 1499 pending_tx_info = &vif->pending_tx_info[pending_idx];
1719 1500
1720 vif = pending_tx_info->vif;
1721 head = pending_tx_info->head; 1501 head = pending_tx_info->head;
1722 1502
1723 BUG_ON(!pending_tx_is_head(netbk, head)); 1503 BUG_ON(!pending_tx_is_head(vif, head));
1724 BUG_ON(netbk->pending_ring[pending_index(head)] != pending_idx); 1504 BUG_ON(vif->pending_ring[pending_index(head)] != pending_idx);
1725 1505
1726 do { 1506 do {
1727 pending_ring_idx_t index; 1507 pending_ring_idx_t index;
1728 pending_ring_idx_t idx = pending_index(head); 1508 pending_ring_idx_t idx = pending_index(head);
1729 u16 info_idx = netbk->pending_ring[idx]; 1509 u16 info_idx = vif->pending_ring[idx];
1730 1510
1731 pending_tx_info = &netbk->pending_tx_info[info_idx]; 1511 pending_tx_info = &vif->pending_tx_info[info_idx];
1732 make_tx_response(vif, &pending_tx_info->req, status); 1512 make_tx_response(vif, &pending_tx_info->req, status);
1733 1513
1734 /* Setting any number other than 1514 /* Setting any number other than
@@ -1737,18 +1517,15 @@ static void xen_netbk_idx_release(struct xen_netbk *netbk, u16 pending_idx,
1737 */ 1517 */
1738 pending_tx_info->head = 0; 1518 pending_tx_info->head = 0;
1739 1519
1740 index = pending_index(netbk->pending_prod++); 1520 index = pending_index(vif->pending_prod++);
1741 netbk->pending_ring[index] = netbk->pending_ring[info_idx]; 1521 vif->pending_ring[index] = vif->pending_ring[info_idx];
1742
1743 xenvif_put(vif);
1744 1522
1745 peek = netbk->pending_ring[pending_index(++head)]; 1523 peek = vif->pending_ring[pending_index(++head)];
1746 1524
1747 } while (!pending_tx_is_head(netbk, peek)); 1525 } while (!pending_tx_is_head(vif, peek));
1748 1526
1749 netbk->mmap_pages[pending_idx]->mapping = 0; 1527 put_page(vif->mmap_pages[pending_idx]);
1750 put_page(netbk->mmap_pages[pending_idx]); 1528 vif->mmap_pages[pending_idx] = NULL;
1751 netbk->mmap_pages[pending_idx] = NULL;
1752} 1529}
1753 1530
1754 1531
@@ -1796,46 +1573,23 @@ static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
1796 return resp; 1573 return resp;
1797} 1574}
1798 1575
1799static inline int rx_work_todo(struct xen_netbk *netbk) 1576static inline int rx_work_todo(struct xenvif *vif)
1800{ 1577{
1801 return !skb_queue_empty(&netbk->rx_queue); 1578 return !skb_queue_empty(&vif->rx_queue);
1802} 1579}
1803 1580
1804static inline int tx_work_todo(struct xen_netbk *netbk) 1581static inline int tx_work_todo(struct xenvif *vif)
1805{ 1582{
1806 1583
1807 if ((nr_pending_reqs(netbk) + XEN_NETBK_LEGACY_SLOTS_MAX 1584 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif->tx)) &&
1808 < MAX_PENDING_REQS) && 1585 (nr_pending_reqs(vif) + XEN_NETBK_LEGACY_SLOTS_MAX
1809 !list_empty(&netbk->net_schedule_list)) 1586 < MAX_PENDING_REQS))
1810 return 1; 1587 return 1;
1811 1588
1812 return 0; 1589 return 0;
1813} 1590}
1814 1591
1815static int xen_netbk_kthread(void *data) 1592void xenvif_unmap_frontend_rings(struct xenvif *vif)
1816{
1817 struct xen_netbk *netbk = data;
1818 while (!kthread_should_stop()) {
1819 wait_event_interruptible(netbk->wq,
1820 rx_work_todo(netbk) ||
1821 tx_work_todo(netbk) ||
1822 kthread_should_stop());
1823 cond_resched();
1824
1825 if (kthread_should_stop())
1826 break;
1827
1828 if (rx_work_todo(netbk))
1829 xen_netbk_rx_action(netbk);
1830
1831 if (tx_work_todo(netbk))
1832 xen_netbk_tx_action(netbk);
1833 }
1834
1835 return 0;
1836}
1837
1838void xen_netbk_unmap_frontend_rings(struct xenvif *vif)
1839{ 1593{
1840 if (vif->tx.sring) 1594 if (vif->tx.sring)
1841 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif), 1595 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif),
@@ -1845,9 +1599,9 @@ void xen_netbk_unmap_frontend_rings(struct xenvif *vif)
1845 vif->rx.sring); 1599 vif->rx.sring);
1846} 1600}
1847 1601
1848int xen_netbk_map_frontend_rings(struct xenvif *vif, 1602int xenvif_map_frontend_rings(struct xenvif *vif,
1849 grant_ref_t tx_ring_ref, 1603 grant_ref_t tx_ring_ref,
1850 grant_ref_t rx_ring_ref) 1604 grant_ref_t rx_ring_ref)
1851{ 1605{
1852 void *addr; 1606 void *addr;
1853 struct xen_netif_tx_sring *txs; 1607 struct xen_netif_tx_sring *txs;
@@ -1876,15 +1630,33 @@ int xen_netbk_map_frontend_rings(struct xenvif *vif,
1876 return 0; 1630 return 0;
1877 1631
1878err: 1632err:
1879 xen_netbk_unmap_frontend_rings(vif); 1633 xenvif_unmap_frontend_rings(vif);
1880 return err; 1634 return err;
1881} 1635}
1882 1636
1637int xenvif_kthread(void *data)
1638{
1639 struct xenvif *vif = data;
1640
1641 while (!kthread_should_stop()) {
1642 wait_event_interruptible(vif->wq,
1643 rx_work_todo(vif) ||
1644 kthread_should_stop());
1645 if (kthread_should_stop())
1646 break;
1647
1648 if (rx_work_todo(vif))
1649 xenvif_rx_action(vif);
1650
1651 cond_resched();
1652 }
1653
1654 return 0;
1655}
1656
1883static int __init netback_init(void) 1657static int __init netback_init(void)
1884{ 1658{
1885 int i;
1886 int rc = 0; 1659 int rc = 0;
1887 int group;
1888 1660
1889 if (!xen_domain()) 1661 if (!xen_domain())
1890 return -ENODEV; 1662 return -ENODEV;
@@ -1895,48 +1667,6 @@ static int __init netback_init(void)
1895 fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX; 1667 fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX;
1896 } 1668 }
1897 1669
1898 xen_netbk_group_nr = num_online_cpus();
1899 xen_netbk = vzalloc(sizeof(struct xen_netbk) * xen_netbk_group_nr);
1900 if (!xen_netbk)
1901 return -ENOMEM;
1902
1903 for (group = 0; group < xen_netbk_group_nr; group++) {
1904 struct xen_netbk *netbk = &xen_netbk[group];
1905 skb_queue_head_init(&netbk->rx_queue);
1906 skb_queue_head_init(&netbk->tx_queue);
1907
1908 init_timer(&netbk->net_timer);
1909 netbk->net_timer.data = (unsigned long)netbk;
1910 netbk->net_timer.function = xen_netbk_alarm;
1911
1912 netbk->pending_cons = 0;
1913 netbk->pending_prod = MAX_PENDING_REQS;
1914 for (i = 0; i < MAX_PENDING_REQS; i++)
1915 netbk->pending_ring[i] = i;
1916
1917 init_waitqueue_head(&netbk->wq);
1918 netbk->task = kthread_create(xen_netbk_kthread,
1919 (void *)netbk,
1920 "netback/%u", group);
1921
1922 if (IS_ERR(netbk->task)) {
1923 pr_alert("kthread_create() fails at netback\n");
1924 del_timer(&netbk->net_timer);
1925 rc = PTR_ERR(netbk->task);
1926 goto failed_init;
1927 }
1928
1929 kthread_bind(netbk->task, group);
1930
1931 INIT_LIST_HEAD(&netbk->net_schedule_list);
1932
1933 spin_lock_init(&netbk->net_schedule_list_lock);
1934
1935 atomic_set(&netbk->netfront_count, 0);
1936
1937 wake_up_process(netbk->task);
1938 }
1939
1940 rc = xenvif_xenbus_init(); 1670 rc = xenvif_xenbus_init();
1941 if (rc) 1671 if (rc)
1942 goto failed_init; 1672 goto failed_init;
@@ -1944,35 +1674,14 @@ static int __init netback_init(void)
1944 return 0; 1674 return 0;
1945 1675
1946failed_init: 1676failed_init:
1947 while (--group >= 0) {
1948 struct xen_netbk *netbk = &xen_netbk[group];
1949 del_timer(&netbk->net_timer);
1950 kthread_stop(netbk->task);
1951 }
1952 vfree(xen_netbk);
1953 return rc; 1677 return rc;
1954
1955} 1678}
1956 1679
1957module_init(netback_init); 1680module_init(netback_init);
1958 1681
1959static void __exit netback_fini(void) 1682static void __exit netback_fini(void)
1960{ 1683{
1961 int i, j;
1962
1963 xenvif_xenbus_fini(); 1684 xenvif_xenbus_fini();
1964
1965 for (i = 0; i < xen_netbk_group_nr; i++) {
1966 struct xen_netbk *netbk = &xen_netbk[i];
1967 del_timer_sync(&netbk->net_timer);
1968 kthread_stop(netbk->task);
1969 for (j = 0; j < MAX_PENDING_REQS; j++) {
1970 if (netbk->mmap_pages[j])
1971 __free_page(netbk->mmap_pages[j]);
1972 }
1973 }
1974
1975 vfree(xen_netbk);
1976} 1685}
1977module_exit(netback_fini); 1686module_exit(netback_fini);
1978 1687
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-03 07:37:25 -0400