aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/xen-netback
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/xen-netback')
-rw-r--r--drivers/net/xen-netback/common.h107
-rw-r--r--drivers/net/xen-netback/interface.c549
-rw-r--r--drivers/net/xen-netback/netback.c838
-rw-r--r--drivers/net/xen-netback/xenbus.c182
4 files changed, 1023 insertions, 653 deletions
diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 630a3fcf65bc..4dd7c4a1923b 100644
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -99,22 +99,43 @@ struct xenvif_rx_meta {
99 */ 99 */
100#define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN 100#define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
101 101
102struct xenvif { 102/* Queue name is interface name with "-qNNN" appended */
103 /* Unique identifier for this interface. */ 103#define QUEUE_NAME_SIZE (IFNAMSIZ + 5)
104 domid_t domid;
105 unsigned int handle;
106 104
107 /* Is this interface disabled? True when backend discovers 105/* IRQ name is queue name with "-tx" or "-rx" appended */
108 * frontend is rogue. 106#define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3)
107
108struct xenvif;
109
110struct xenvif_stats {
111 /* Stats fields to be updated per-queue.
112 * A subset of struct net_device_stats that contains only the
113 * fields that are updated in netback.c for each queue.
109 */ 114 */
110 bool disabled; 115 unsigned int rx_bytes;
116 unsigned int rx_packets;
117 unsigned int tx_bytes;
118 unsigned int tx_packets;
119
120 /* Additional stats used by xenvif */
121 unsigned long rx_gso_checksum_fixup;
122 unsigned long tx_zerocopy_sent;
123 unsigned long tx_zerocopy_success;
124 unsigned long tx_zerocopy_fail;
125 unsigned long tx_frag_overflow;
126};
127
128struct xenvif_queue { /* Per-queue data for xenvif */
129 unsigned int id; /* Queue ID, 0-based */
130 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
131 struct xenvif *vif; /* Parent VIF */
111 132
112 /* Use NAPI for guest TX */ 133 /* Use NAPI for guest TX */
113 struct napi_struct napi; 134 struct napi_struct napi;
114 /* When feature-split-event-channels = 0, tx_irq = rx_irq. */ 135 /* When feature-split-event-channels = 0, tx_irq = rx_irq. */
115 unsigned int tx_irq; 136 unsigned int tx_irq;
116 /* Only used when feature-split-event-channels = 1 */ 137 /* Only used when feature-split-event-channels = 1 */
117 char tx_irq_name[IFNAMSIZ+4]; /* DEVNAME-tx */ 138 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
118 struct xen_netif_tx_back_ring tx; 139 struct xen_netif_tx_back_ring tx;
119 struct sk_buff_head tx_queue; 140 struct sk_buff_head tx_queue;
120 struct page *mmap_pages[MAX_PENDING_REQS]; 141 struct page *mmap_pages[MAX_PENDING_REQS];
@@ -150,7 +171,7 @@ struct xenvif {
150 /* When feature-split-event-channels = 0, tx_irq = rx_irq. */ 171 /* When feature-split-event-channels = 0, tx_irq = rx_irq. */
151 unsigned int rx_irq; 172 unsigned int rx_irq;
152 /* Only used when feature-split-event-channels = 1 */ 173 /* Only used when feature-split-event-channels = 1 */
153 char rx_irq_name[IFNAMSIZ+4]; /* DEVNAME-rx */ 174 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
154 struct xen_netif_rx_back_ring rx; 175 struct xen_netif_rx_back_ring rx;
155 struct sk_buff_head rx_queue; 176 struct sk_buff_head rx_queue;
156 RING_IDX rx_last_skb_slots; 177 RING_IDX rx_last_skb_slots;
@@ -158,14 +179,29 @@ struct xenvif {
158 179
159 struct timer_list wake_queue; 180 struct timer_list wake_queue;
160 181
161 /* This array is allocated seperately as it is large */ 182 struct gnttab_copy grant_copy_op[MAX_GRANT_COPY_OPS];
162 struct gnttab_copy *grant_copy_op;
163 183
164 /* We create one meta structure per ring request we consume, so 184 /* We create one meta structure per ring request we consume, so
165 * the maximum number is the same as the ring size. 185 * the maximum number is the same as the ring size.
166 */ 186 */
167 struct xenvif_rx_meta meta[XEN_NETIF_RX_RING_SIZE]; 187 struct xenvif_rx_meta meta[XEN_NETIF_RX_RING_SIZE];
168 188
189 /* Transmit shaping: allow 'credit_bytes' every 'credit_usec'. */
190 unsigned long credit_bytes;
191 unsigned long credit_usec;
192 unsigned long remaining_credit;
193 struct timer_list credit_timeout;
194 u64 credit_window_start;
195
196 /* Statistics */
197 struct xenvif_stats stats;
198};
199
200struct xenvif {
201 /* Unique identifier for this interface. */
202 domid_t domid;
203 unsigned int handle;
204
169 u8 fe_dev_addr[6]; 205 u8 fe_dev_addr[6];
170 206
171 /* Frontend feature information. */ 207 /* Frontend feature information. */
@@ -179,19 +215,13 @@ struct xenvif {
179 /* Internal feature information. */ 215 /* Internal feature information. */
180 u8 can_queue:1; /* can queue packets for receiver? */ 216 u8 can_queue:1; /* can queue packets for receiver? */
181 217
182 /* Transmit shaping: allow 'credit_bytes' every 'credit_usec'. */ 218 /* Is this interface disabled? True when backend discovers
183 unsigned long credit_bytes; 219 * frontend is rogue.
184 unsigned long credit_usec; 220 */
185 unsigned long remaining_credit; 221 bool disabled;
186 struct timer_list credit_timeout;
187 u64 credit_window_start;
188 222
189 /* Statistics */ 223 /* Queues */
190 unsigned long rx_gso_checksum_fixup; 224 struct xenvif_queue *queues;
191 unsigned long tx_zerocopy_sent;
192 unsigned long tx_zerocopy_success;
193 unsigned long tx_zerocopy_fail;
194 unsigned long tx_frag_overflow;
195 225
196 /* Miscellaneous private stuff. */ 226 /* Miscellaneous private stuff. */
197 struct net_device *dev; 227 struct net_device *dev;
@@ -206,7 +236,10 @@ struct xenvif *xenvif_alloc(struct device *parent,
206 domid_t domid, 236 domid_t domid,
207 unsigned int handle); 237 unsigned int handle);
208 238
209int xenvif_connect(struct xenvif *vif, unsigned long tx_ring_ref, 239int xenvif_init_queue(struct xenvif_queue *queue);
240void xenvif_deinit_queue(struct xenvif_queue *queue);
241
242int xenvif_connect(struct xenvif_queue *queue, unsigned long tx_ring_ref,
210 unsigned long rx_ring_ref, unsigned int tx_evtchn, 243 unsigned long rx_ring_ref, unsigned int tx_evtchn,
211 unsigned int rx_evtchn); 244 unsigned int rx_evtchn);
212void xenvif_disconnect(struct xenvif *vif); 245void xenvif_disconnect(struct xenvif *vif);
@@ -217,44 +250,47 @@ void xenvif_xenbus_fini(void);
217 250
218int xenvif_schedulable(struct xenvif *vif); 251int xenvif_schedulable(struct xenvif *vif);
219 252
220int xenvif_must_stop_queue(struct xenvif *vif); 253int xenvif_must_stop_queue(struct xenvif_queue *queue);
254
255int xenvif_queue_stopped(struct xenvif_queue *queue);
256void xenvif_wake_queue(struct xenvif_queue *queue);
221 257
222/* (Un)Map communication rings. */ 258/* (Un)Map communication rings. */
223void xenvif_unmap_frontend_rings(struct xenvif *vif); 259void xenvif_unmap_frontend_rings(struct xenvif_queue *queue);
224int xenvif_map_frontend_rings(struct xenvif *vif, 260int xenvif_map_frontend_rings(struct xenvif_queue *queue,
225 grant_ref_t tx_ring_ref, 261 grant_ref_t tx_ring_ref,
226 grant_ref_t rx_ring_ref); 262 grant_ref_t rx_ring_ref);
227 263
228/* Check for SKBs from frontend and schedule backend processing */ 264/* Check for SKBs from frontend and schedule backend processing */
229void xenvif_check_rx_xenvif(struct xenvif *vif); 265void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue);
230 266
231/* Prevent the device from generating any further traffic. */ 267/* Prevent the device from generating any further traffic. */
232void xenvif_carrier_off(struct xenvif *vif); 268void xenvif_carrier_off(struct xenvif *vif);
233 269
234int xenvif_tx_action(struct xenvif *vif, int budget); 270int xenvif_tx_action(struct xenvif_queue *queue, int budget);
235 271
236int xenvif_kthread_guest_rx(void *data); 272int xenvif_kthread_guest_rx(void *data);
237void xenvif_kick_thread(struct xenvif *vif); 273void xenvif_kick_thread(struct xenvif_queue *queue);
238 274
239int xenvif_dealloc_kthread(void *data); 275int xenvif_dealloc_kthread(void *data);
240 276
241/* Determine whether the needed number of slots (req) are available, 277/* Determine whether the needed number of slots (req) are available,
242 * and set req_event if not. 278 * and set req_event if not.
243 */ 279 */
244bool xenvif_rx_ring_slots_available(struct xenvif *vif, int needed); 280bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue, int needed);
245 281
246void xenvif_stop_queue(struct xenvif *vif); 282void xenvif_carrier_on(struct xenvif *vif);
247 283
248/* Callback from stack when TX packet can be released */ 284/* Callback from stack when TX packet can be released */
249void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success); 285void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success);
250 286
251/* Unmap a pending page and release it back to the guest */ 287/* Unmap a pending page and release it back to the guest */
252void xenvif_idx_unmap(struct xenvif *vif, u16 pending_idx); 288void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx);
253 289
254static inline pending_ring_idx_t nr_pending_reqs(struct xenvif *vif) 290static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)
255{ 291{
256 return MAX_PENDING_REQS - 292 return MAX_PENDING_REQS -
257 vif->pending_prod + vif->pending_cons; 293 queue->pending_prod + queue->pending_cons;
258} 294}
259 295
260/* Callback from stack when TX packet can be released */ 296/* Callback from stack when TX packet can be released */
@@ -264,5 +300,6 @@ extern bool separate_tx_rx_irq;
264 300
265extern unsigned int rx_drain_timeout_msecs; 301extern unsigned int rx_drain_timeout_msecs;
266extern unsigned int rx_drain_timeout_jiffies; 302extern unsigned int rx_drain_timeout_jiffies;
303extern unsigned int xenvif_max_queues;
267 304
268#endif /* __XEN_NETBACK__COMMON_H__ */ 305#endif /* __XEN_NETBACK__COMMON_H__ */
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index ef05c5c49d41..852da34b8961 100644
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -43,6 +43,16 @@
43#define XENVIF_QUEUE_LENGTH 32 43#define XENVIF_QUEUE_LENGTH 32
44#define XENVIF_NAPI_WEIGHT 64 44#define XENVIF_NAPI_WEIGHT 64
45 45
46static inline void xenvif_stop_queue(struct xenvif_queue *queue)
47{
48 struct net_device *dev = queue->vif->dev;
49
50 if (!queue->vif->can_queue)
51 return;
52
53 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
54}
55
46int xenvif_schedulable(struct xenvif *vif) 56int xenvif_schedulable(struct xenvif *vif)
47{ 57{
48 return netif_running(vif->dev) && netif_carrier_ok(vif->dev); 58 return netif_running(vif->dev) && netif_carrier_ok(vif->dev);
@@ -50,57 +60,34 @@ int xenvif_schedulable(struct xenvif *vif)
50 60
51static irqreturn_t xenvif_tx_interrupt(int irq, void *dev_id) 61static irqreturn_t xenvif_tx_interrupt(int irq, void *dev_id)
52{ 62{
53 struct xenvif *vif = dev_id; 63 struct xenvif_queue *queue = dev_id;
54 64
55 if (RING_HAS_UNCONSUMED_REQUESTS(&vif->tx)) 65 if (RING_HAS_UNCONSUMED_REQUESTS(&queue->tx))
56 napi_schedule(&vif->napi); 66 napi_schedule(&queue->napi);
57 67
58 return IRQ_HANDLED; 68 return IRQ_HANDLED;
59} 69}
60 70
61static int xenvif_poll(struct napi_struct *napi, int budget) 71int xenvif_poll(struct napi_struct *napi, int budget)
62{ 72{
63 struct xenvif *vif = container_of(napi, struct xenvif, napi); 73 struct xenvif_queue *queue =
74 container_of(napi, struct xenvif_queue, napi);
64 int work_done; 75 int work_done;
65 76
66 /* This vif is rogue, we pretend we've there is nothing to do 77 /* This vif is rogue, we pretend we've there is nothing to do
67 * for this vif to deschedule it from NAPI. But this interface 78 * for this vif to deschedule it from NAPI. But this interface
68 * will be turned off in thread context later. 79 * will be turned off in thread context later.
69 */ 80 */
70 if (unlikely(vif->disabled)) { 81 if (unlikely(queue->vif->disabled)) {
71 napi_complete(napi); 82 napi_complete(napi);
72 return 0; 83 return 0;
73 } 84 }
74 85
75 work_done = xenvif_tx_action(vif, budget); 86 work_done = xenvif_tx_action(queue, budget);
76 87
77 if (work_done < budget) { 88 if (work_done < budget) {
78 int more_to_do = 0; 89 napi_complete(napi);
79 unsigned long flags; 90 xenvif_napi_schedule_or_enable_events(queue);
80
81 /* It is necessary to disable IRQ before calling
82 * RING_HAS_UNCONSUMED_REQUESTS. Otherwise we might
83 * lose event from the frontend.
84 *
85 * Consider:
86 * RING_HAS_UNCONSUMED_REQUESTS
87 * <frontend generates event to trigger napi_schedule>
88 * __napi_complete
89 *
90 * This handler is still in scheduled state so the
91 * event has no effect at all. After __napi_complete
92 * this handler is descheduled and cannot get
93 * scheduled again. We lose event in this case and the ring
94 * will be completely stalled.
95 */
96
97 local_irq_save(flags);
98
99 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do);
100 if (!more_to_do)
101 __napi_complete(napi);
102
103 local_irq_restore(flags);
104 } 91 }
105 92
106 return work_done; 93 return work_done;
@@ -108,9 +95,9 @@ static int xenvif_poll(struct napi_struct *napi, int budget)
108 95
109static irqreturn_t xenvif_rx_interrupt(int irq, void *dev_id) 96static irqreturn_t xenvif_rx_interrupt(int irq, void *dev_id)
110{ 97{
111 struct xenvif *vif = dev_id; 98 struct xenvif_queue *queue = dev_id;
112 99
113 xenvif_kick_thread(vif); 100 xenvif_kick_thread(queue);
114 101
115 return IRQ_HANDLED; 102 return IRQ_HANDLED;
116} 103}
@@ -123,28 +110,80 @@ static irqreturn_t xenvif_interrupt(int irq, void *dev_id)
123 return IRQ_HANDLED; 110 return IRQ_HANDLED;
124} 111}
125 112
126static void xenvif_wake_queue(unsigned long data) 113int xenvif_queue_stopped(struct xenvif_queue *queue)
127{ 114{
128 struct xenvif *vif = (struct xenvif *)data; 115 struct net_device *dev = queue->vif->dev;
116 unsigned int id = queue->id;
117 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, id));
118}
119
120void xenvif_wake_queue(struct xenvif_queue *queue)
121{
122 struct net_device *dev = queue->vif->dev;
123 unsigned int id = queue->id;
124 netif_tx_wake_queue(netdev_get_tx_queue(dev, id));
125}
129 126
130 if (netif_queue_stopped(vif->dev)) { 127/* Callback to wake the queue and drain it on timeout */
131 netdev_err(vif->dev, "draining TX queue\n"); 128static void xenvif_wake_queue_callback(unsigned long data)
132 vif->rx_queue_purge = true; 129{
133 xenvif_kick_thread(vif); 130 struct xenvif_queue *queue = (struct xenvif_queue *)data;
134 netif_wake_queue(vif->dev); 131
132 if (xenvif_queue_stopped(queue)) {
133 netdev_err(queue->vif->dev, "draining TX queue\n");
134 queue->rx_queue_purge = true;
135 xenvif_kick_thread(queue);
136 xenvif_wake_queue(queue);
135 } 137 }
136} 138}
137 139
140static u16 xenvif_select_queue(struct net_device *dev, struct sk_buff *skb,
141 void *accel_priv, select_queue_fallback_t fallback)
142{
143 unsigned int num_queues = dev->real_num_tx_queues;
144 u32 hash;
145 u16 queue_index;
146
147 /* First, check if there is only one queue to optimise the
148 * single-queue or old frontend scenario.
149 */
150 if (num_queues == 1) {
151 queue_index = 0;
152 } else {
153 /* Use skb_get_hash to obtain an L4 hash if available */
154 hash = skb_get_hash(skb);
155 queue_index = hash % num_queues;
156 }
157
158 return queue_index;
159}
160
138static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev) 161static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev)
139{ 162{
140 struct xenvif *vif = netdev_priv(dev); 163 struct xenvif *vif = netdev_priv(dev);
164 struct xenvif_queue *queue = NULL;
165 unsigned int num_queues = dev->real_num_tx_queues;
166 u16 index;
141 int min_slots_needed; 167 int min_slots_needed;
142 168
143 BUG_ON(skb->dev != dev); 169 BUG_ON(skb->dev != dev);
144 170
145 /* Drop the packet if vif is not ready */ 171 /* Drop the packet if queues are not set up */
146 if (vif->task == NULL || 172 if (num_queues < 1)
147 vif->dealloc_task == NULL || 173 goto drop;
174
175 /* Obtain the queue to be used to transmit this packet */
176 index = skb_get_queue_mapping(skb);
177 if (index >= num_queues) {
178 pr_warn_ratelimited("Invalid queue %hu for packet on interface %s\n.",
179 index, vif->dev->name);
180 index %= num_queues;
181 }
182 queue = &vif->queues[index];
183
184 /* Drop the packet if queue is not ready */
185 if (queue->task == NULL ||
186 queue->dealloc_task == NULL ||
148 !xenvif_schedulable(vif)) 187 !xenvif_schedulable(vif))
149 goto drop; 188 goto drop;
150 189
@@ -163,16 +202,16 @@ static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev)
163 * then turn off the queue to give the ring a chance to 202 * then turn off the queue to give the ring a chance to
164 * drain. 203 * drain.
165 */ 204 */
166 if (!xenvif_rx_ring_slots_available(vif, min_slots_needed)) { 205 if (!xenvif_rx_ring_slots_available(queue, min_slots_needed)) {
167 vif->wake_queue.function = xenvif_wake_queue; 206 queue->wake_queue.function = xenvif_wake_queue_callback;
168 vif->wake_queue.data = (unsigned long)vif; 207 queue->wake_queue.data = (unsigned long)queue;
169 xenvif_stop_queue(vif); 208 xenvif_stop_queue(queue);
170 mod_timer(&vif->wake_queue, 209 mod_timer(&queue->wake_queue,
171 jiffies + rx_drain_timeout_jiffies); 210 jiffies + rx_drain_timeout_jiffies);
172 } 211 }
173 212
174 skb_queue_tail(&vif->rx_queue, skb); 213 skb_queue_tail(&queue->rx_queue, skb);
175 xenvif_kick_thread(vif); 214 xenvif_kick_thread(queue);
176 215
177 return NETDEV_TX_OK; 216 return NETDEV_TX_OK;
178 217
@@ -185,25 +224,65 @@ static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev)
185static struct net_device_stats *xenvif_get_stats(struct net_device *dev) 224static struct net_device_stats *xenvif_get_stats(struct net_device *dev)
186{ 225{
187 struct xenvif *vif = netdev_priv(dev); 226 struct xenvif *vif = netdev_priv(dev);
227 struct xenvif_queue *queue = NULL;
228 unsigned int num_queues = dev->real_num_tx_queues;
229 unsigned long rx_bytes = 0;
230 unsigned long rx_packets = 0;
231 unsigned long tx_bytes = 0;
232 unsigned long tx_packets = 0;
233 unsigned int index;
234
235 if (vif->queues == NULL)
236 goto out;
237
238 /* Aggregate tx and rx stats from each queue */
239 for (index = 0; index < num_queues; ++index) {
240 queue = &vif->queues[index];
241 rx_bytes += queue->stats.rx_bytes;
242 rx_packets += queue->stats.rx_packets;
243 tx_bytes += queue->stats.tx_bytes;
244 tx_packets += queue->stats.tx_packets;
245 }
246
247out:
248 vif->dev->stats.rx_bytes = rx_bytes;
249 vif->dev->stats.rx_packets = rx_packets;
250 vif->dev->stats.tx_bytes = tx_bytes;
251 vif->dev->stats.tx_packets = tx_packets;
252
188 return &vif->dev->stats; 253 return &vif->dev->stats;
189} 254}
190 255
191static void xenvif_up(struct xenvif *vif) 256static void xenvif_up(struct xenvif *vif)
192{ 257{
193 napi_enable(&vif->napi); 258 struct xenvif_queue *queue = NULL;
194 enable_irq(vif->tx_irq); 259 unsigned int num_queues = vif->dev->real_num_tx_queues;
195 if (vif->tx_irq != vif->rx_irq) 260 unsigned int queue_index;
196 enable_irq(vif->rx_irq); 261
197 xenvif_check_rx_xenvif(vif); 262 for (queue_index = 0; queue_index < num_queues; ++queue_index) {
263 queue = &vif->queues[queue_index];
264 napi_enable(&queue->napi);
265 enable_irq(queue->tx_irq);
266 if (queue->tx_irq != queue->rx_irq)
267 enable_irq(queue->rx_irq);
268 xenvif_napi_schedule_or_enable_events(queue);
269 }
198} 270}
199 271
200static void xenvif_down(struct xenvif *vif) 272static void xenvif_down(struct xenvif *vif)
201{ 273{
202 napi_disable(&vif->napi); 274 struct xenvif_queue *queue = NULL;
203 disable_irq(vif->tx_irq); 275 unsigned int num_queues = vif->dev->real_num_tx_queues;
204 if (vif->tx_irq != vif->rx_irq) 276 unsigned int queue_index;
205 disable_irq(vif->rx_irq); 277
206 del_timer_sync(&vif->credit_timeout); 278 for (queue_index = 0; queue_index < num_queues; ++queue_index) {
279 queue = &vif->queues[queue_index];
280 napi_disable(&queue->napi);
281 disable_irq(queue->tx_irq);
282 if (queue->tx_irq != queue->rx_irq)
283 disable_irq(queue->rx_irq);
284 del_timer_sync(&queue->credit_timeout);
285 }
207} 286}
208 287
209static int xenvif_open(struct net_device *dev) 288static int xenvif_open(struct net_device *dev)
@@ -211,7 +290,7 @@ static int xenvif_open(struct net_device *dev)
211 struct xenvif *vif = netdev_priv(dev); 290 struct xenvif *vif = netdev_priv(dev);
212 if (netif_carrier_ok(dev)) 291 if (netif_carrier_ok(dev))
213 xenvif_up(vif); 292 xenvif_up(vif);
214 netif_start_queue(dev); 293 netif_tx_start_all_queues(dev);
215 return 0; 294 return 0;
216} 295}
217 296
@@ -220,7 +299,7 @@ static int xenvif_close(struct net_device *dev)
220 struct xenvif *vif = netdev_priv(dev); 299 struct xenvif *vif = netdev_priv(dev);
221 if (netif_carrier_ok(dev)) 300 if (netif_carrier_ok(dev))
222 xenvif_down(vif); 301 xenvif_down(vif);
223 netif_stop_queue(dev); 302 netif_tx_stop_all_queues(dev);
224 return 0; 303 return 0;
225} 304}
226 305
@@ -260,29 +339,29 @@ static const struct xenvif_stat {
260} xenvif_stats[] = { 339} xenvif_stats[] = {
261 { 340 {
262 "rx_gso_checksum_fixup", 341 "rx_gso_checksum_fixup",
263 offsetof(struct xenvif, rx_gso_checksum_fixup) 342 offsetof(struct xenvif_stats, rx_gso_checksum_fixup)
264 }, 343 },
265 /* If (sent != success + fail), there are probably packets never 344 /* If (sent != success + fail), there are probably packets never
266 * freed up properly! 345 * freed up properly!
267 */ 346 */
268 { 347 {
269 "tx_zerocopy_sent", 348 "tx_zerocopy_sent",
270 offsetof(struct xenvif, tx_zerocopy_sent), 349 offsetof(struct xenvif_stats, tx_zerocopy_sent),
271 }, 350 },
272 { 351 {
273 "tx_zerocopy_success", 352 "tx_zerocopy_success",
274 offsetof(struct xenvif, tx_zerocopy_success), 353 offsetof(struct xenvif_stats, tx_zerocopy_success),
275 }, 354 },
276 { 355 {
277 "tx_zerocopy_fail", 356 "tx_zerocopy_fail",
278 offsetof(struct xenvif, tx_zerocopy_fail) 357 offsetof(struct xenvif_stats, tx_zerocopy_fail)
279 }, 358 },
280 /* Number of packets exceeding MAX_SKB_FRAG slots. You should use 359 /* Number of packets exceeding MAX_SKB_FRAG slots. You should use
281 * a guest with the same MAX_SKB_FRAG 360 * a guest with the same MAX_SKB_FRAG
282 */ 361 */
283 { 362 {
284 "tx_frag_overflow", 363 "tx_frag_overflow",
285 offsetof(struct xenvif, tx_frag_overflow) 364 offsetof(struct xenvif_stats, tx_frag_overflow)
286 }, 365 },
287}; 366};
288 367
@@ -299,11 +378,20 @@ static int xenvif_get_sset_count(struct net_device *dev, int string_set)
299static void xenvif_get_ethtool_stats(struct net_device *dev, 378static void xenvif_get_ethtool_stats(struct net_device *dev,
300 struct ethtool_stats *stats, u64 * data) 379 struct ethtool_stats *stats, u64 * data)
301{ 380{
302 void *vif = netdev_priv(dev); 381 struct xenvif *vif = netdev_priv(dev);
382 unsigned int num_queues = dev->real_num_tx_queues;
303 int i; 383 int i;
304 384 unsigned int queue_index;
305 for (i = 0; i < ARRAY_SIZE(xenvif_stats); i++) 385 struct xenvif_stats *vif_stats;
306 data[i] = *(unsigned long *)(vif + xenvif_stats[i].offset); 386
387 for (i = 0; i < ARRAY_SIZE(xenvif_stats); i++) {
388 unsigned long accum = 0;
389 for (queue_index = 0; queue_index < num_queues; ++queue_index) {
390 vif_stats = &vif->queues[queue_index].stats;
391 accum += *(unsigned long *)(vif_stats + xenvif_stats[i].offset);
392 }
393 data[i] = accum;
394 }
307} 395}
308 396
309static void xenvif_get_strings(struct net_device *dev, u32 stringset, u8 * data) 397static void xenvif_get_strings(struct net_device *dev, u32 stringset, u8 * data)
@@ -336,6 +424,7 @@ static const struct net_device_ops xenvif_netdev_ops = {
336 .ndo_fix_features = xenvif_fix_features, 424 .ndo_fix_features = xenvif_fix_features,
337 .ndo_set_mac_address = eth_mac_addr, 425 .ndo_set_mac_address = eth_mac_addr,
338 .ndo_validate_addr = eth_validate_addr, 426 .ndo_validate_addr = eth_validate_addr,
427 .ndo_select_queue = xenvif_select_queue,
339}; 428};
340 429
341struct xenvif *xenvif_alloc(struct device *parent, domid_t domid, 430struct xenvif *xenvif_alloc(struct device *parent, domid_t domid,
@@ -345,10 +434,14 @@ struct xenvif *xenvif_alloc(struct device *parent, domid_t domid,
345 struct net_device *dev; 434 struct net_device *dev;
346 struct xenvif *vif; 435 struct xenvif *vif;
347 char name[IFNAMSIZ] = {}; 436 char name[IFNAMSIZ] = {};
348 int i;
349 437
350 snprintf(name, IFNAMSIZ - 1, "vif%u.%u", domid, handle); 438 snprintf(name, IFNAMSIZ - 1, "vif%u.%u", domid, handle);
351 dev = alloc_netdev(sizeof(struct xenvif), name, ether_setup); 439 /* Allocate a netdev with the max. supported number of queues.
440 * When the guest selects the desired number, it will be updated
441 * via netif_set_real_num_tx_queues().
442 */
443 dev = alloc_netdev_mq(sizeof(struct xenvif), name, ether_setup,
444 xenvif_max_queues);
352 if (dev == NULL) { 445 if (dev == NULL) {
353 pr_warn("Could not allocate netdev for %s\n", name); 446 pr_warn("Could not allocate netdev for %s\n", name);
354 return ERR_PTR(-ENOMEM); 447 return ERR_PTR(-ENOMEM);
@@ -358,66 +451,28 @@ struct xenvif *xenvif_alloc(struct device *parent, domid_t domid,
358 451
359 vif = netdev_priv(dev); 452 vif = netdev_priv(dev);
360 453
361 vif->grant_copy_op = vmalloc(sizeof(struct gnttab_copy) *
362 MAX_GRANT_COPY_OPS);
363 if (vif->grant_copy_op == NULL) {
364 pr_warn("Could not allocate grant copy space for %s\n", name);
365 free_netdev(dev);
366 return ERR_PTR(-ENOMEM);
367 }
368
369 vif->domid = domid; 454 vif->domid = domid;
370 vif->handle = handle; 455 vif->handle = handle;
371 vif->can_sg = 1; 456 vif->can_sg = 1;
372 vif->ip_csum = 1; 457 vif->ip_csum = 1;
373 vif->dev = dev; 458 vif->dev = dev;
374
375 vif->disabled = false; 459 vif->disabled = false;
376 460
377 vif->credit_bytes = vif->remaining_credit = ~0UL; 461 /* Start out with no queues. The call below does not require
378 vif->credit_usec = 0UL; 462 * rtnl_lock() as it happens before register_netdev().
379 init_timer(&vif->credit_timeout); 463 */
380 vif->credit_window_start = get_jiffies_64(); 464 vif->queues = NULL;
381 465 netif_set_real_num_tx_queues(dev, 0);
382 init_timer(&vif->wake_queue);
383 466
384 dev->netdev_ops = &xenvif_netdev_ops; 467 dev->netdev_ops = &xenvif_netdev_ops;
385 dev->hw_features = NETIF_F_SG | 468 dev->hw_features = NETIF_F_SG |
386 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 469 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
387 NETIF_F_TSO | NETIF_F_TSO6; 470 NETIF_F_TSO | NETIF_F_TSO6;
388 dev->features = dev->hw_features | NETIF_F_RXCSUM; 471 dev->features = dev->hw_features | NETIF_F_RXCSUM;
389 SET_ETHTOOL_OPS(dev, &xenvif_ethtool_ops); 472 dev->ethtool_ops = &xenvif_ethtool_ops;
390 473
391 dev->tx_queue_len = XENVIF_QUEUE_LENGTH; 474 dev->tx_queue_len = XENVIF_QUEUE_LENGTH;
392 475
393 skb_queue_head_init(&vif->rx_queue);
394 skb_queue_head_init(&vif->tx_queue);
395
396 vif->pending_cons = 0;
397 vif->pending_prod = MAX_PENDING_REQS;
398 for (i = 0; i < MAX_PENDING_REQS; i++)
399 vif->pending_ring[i] = i;
400 spin_lock_init(&vif->callback_lock);
401 spin_lock_init(&vif->response_lock);
402 /* If ballooning is disabled, this will consume real memory, so you
403 * better enable it. The long term solution would be to use just a
404 * bunch of valid page descriptors, without dependency on ballooning
405 */
406 err = alloc_xenballooned_pages(MAX_PENDING_REQS,
407 vif->mmap_pages,
408 false);
409 if (err) {
410 netdev_err(dev, "Could not reserve mmap_pages\n");
411 return ERR_PTR(-ENOMEM);
412 }
413 for (i = 0; i < MAX_PENDING_REQS; i++) {
414 vif->pending_tx_info[i].callback_struct = (struct ubuf_info)
415 { .callback = xenvif_zerocopy_callback,
416 .ctx = NULL,
417 .desc = i };
418 vif->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;
419 }
420
421 /* 476 /*
422 * Initialise a dummy MAC address. We choose the numerically 477 * Initialise a dummy MAC address. We choose the numerically
423 * largest non-broadcast address to prevent the address getting 478 * largest non-broadcast address to prevent the address getting
@@ -427,8 +482,6 @@ struct xenvif *xenvif_alloc(struct device *parent, domid_t domid,
427 memset(dev->dev_addr, 0xFF, ETH_ALEN); 482 memset(dev->dev_addr, 0xFF, ETH_ALEN);
428 dev->dev_addr[0] &= ~0x01; 483 dev->dev_addr[0] &= ~0x01;
429 484
430 netif_napi_add(dev, &vif->napi, xenvif_poll, XENVIF_NAPI_WEIGHT);
431
432 netif_carrier_off(dev); 485 netif_carrier_off(dev);
433 486
434 err = register_netdev(dev); 487 err = register_netdev(dev);
@@ -445,98 +498,147 @@ struct xenvif *xenvif_alloc(struct device *parent, domid_t domid,
445 return vif; 498 return vif;
446} 499}
447 500
448int xenvif_connect(struct xenvif *vif, unsigned long tx_ring_ref, 501int xenvif_init_queue(struct xenvif_queue *queue)
502{
503 int err, i;
504
505 queue->credit_bytes = queue->remaining_credit = ~0UL;
506 queue->credit_usec = 0UL;
507 init_timer(&queue->credit_timeout);
508 queue->credit_window_start = get_jiffies_64();
509
510 skb_queue_head_init(&queue->rx_queue);
511 skb_queue_head_init(&queue->tx_queue);
512
513 queue->pending_cons = 0;
514 queue->pending_prod = MAX_PENDING_REQS;
515 for (i = 0; i < MAX_PENDING_REQS; ++i)
516 queue->pending_ring[i] = i;
517
518 spin_lock_init(&queue->callback_lock);
519 spin_lock_init(&queue->response_lock);
520
521 /* If ballooning is disabled, this will consume real memory, so you
522 * better enable it. The long term solution would be to use just a
523 * bunch of valid page descriptors, without dependency on ballooning
524 */
525 err = alloc_xenballooned_pages(MAX_PENDING_REQS,
526 queue->mmap_pages,
527 false);
528 if (err) {
529 netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n");
530 return -ENOMEM;
531 }
532
533 for (i = 0; i < MAX_PENDING_REQS; i++) {
534 queue->pending_tx_info[i].callback_struct = (struct ubuf_info)
535 { .callback = xenvif_zerocopy_callback,
536 .ctx = NULL,
537 .desc = i };
538 queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;
539 }
540
541 init_timer(&queue->wake_queue);
542
543 netif_napi_add(queue->vif->dev, &queue->napi, xenvif_poll,
544 XENVIF_NAPI_WEIGHT);
545
546 return 0;
547}
548
549void xenvif_carrier_on(struct xenvif *vif)
550{
551 rtnl_lock();
552 if (!vif->can_sg && vif->dev->mtu > ETH_DATA_LEN)
553 dev_set_mtu(vif->dev, ETH_DATA_LEN);
554 netdev_update_features(vif->dev);
555 netif_carrier_on(vif->dev);
556 if (netif_running(vif->dev))
557 xenvif_up(vif);
558 rtnl_unlock();
559}
560
561int xenvif_connect(struct xenvif_queue *queue, unsigned long tx_ring_ref,
449 unsigned long rx_ring_ref, unsigned int tx_evtchn, 562 unsigned long rx_ring_ref, unsigned int tx_evtchn,
450 unsigned int rx_evtchn) 563 unsigned int rx_evtchn)
451{ 564{
452 struct task_struct *task; 565 struct task_struct *task;
453 int err = -ENOMEM; 566 int err = -ENOMEM;
454 567
455 BUG_ON(vif->tx_irq); 568 BUG_ON(queue->tx_irq);
456 BUG_ON(vif->task); 569 BUG_ON(queue->task);
457 BUG_ON(vif->dealloc_task); 570 BUG_ON(queue->dealloc_task);
458 571
459 err = xenvif_map_frontend_rings(vif, tx_ring_ref, rx_ring_ref); 572 err = xenvif_map_frontend_rings(queue, tx_ring_ref, rx_ring_ref);
460 if (err < 0) 573 if (err < 0)
461 goto err; 574 goto err;
462 575
463 init_waitqueue_head(&vif->wq); 576 init_waitqueue_head(&queue->wq);
464 init_waitqueue_head(&vif->dealloc_wq); 577 init_waitqueue_head(&queue->dealloc_wq);
465 578
466 if (tx_evtchn == rx_evtchn) { 579 if (tx_evtchn == rx_evtchn) {
467 /* feature-split-event-channels == 0 */ 580 /* feature-split-event-channels == 0 */
468 err = bind_interdomain_evtchn_to_irqhandler( 581 err = bind_interdomain_evtchn_to_irqhandler(
469 vif->domid, tx_evtchn, xenvif_interrupt, 0, 582 queue->vif->domid, tx_evtchn, xenvif_interrupt, 0,
470 vif->dev->name, vif); 583 queue->name, queue);
471 if (err < 0) 584 if (err < 0)
472 goto err_unmap; 585 goto err_unmap;
473 vif->tx_irq = vif->rx_irq = err; 586 queue->tx_irq = queue->rx_irq = err;
474 disable_irq(vif->tx_irq); 587 disable_irq(queue->tx_irq);
475 } else { 588 } else {
476 /* feature-split-event-channels == 1 */ 589 /* feature-split-event-channels == 1 */
477 snprintf(vif->tx_irq_name, sizeof(vif->tx_irq_name), 590 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),
478 "%s-tx", vif->dev->name); 591 "%s-tx", queue->name);
479 err = bind_interdomain_evtchn_to_irqhandler( 592 err = bind_interdomain_evtchn_to_irqhandler(
480 vif->domid, tx_evtchn, xenvif_tx_interrupt, 0, 593 queue->vif->domid, tx_evtchn, xenvif_tx_interrupt, 0,
481 vif->tx_irq_name, vif); 594 queue->tx_irq_name, queue);
482 if (err < 0) 595 if (err < 0)
483 goto err_unmap; 596 goto err_unmap;
484 vif->tx_irq = err; 597 queue->tx_irq = err;
485 disable_irq(vif->tx_irq); 598 disable_irq(queue->tx_irq);
486 599
487 snprintf(vif->rx_irq_name, sizeof(vif->rx_irq_name), 600 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),
488 "%s-rx", vif->dev->name); 601 "%s-rx", queue->name);
489 err = bind_interdomain_evtchn_to_irqhandler( 602 err = bind_interdomain_evtchn_to_irqhandler(
490 vif->domid, rx_evtchn, xenvif_rx_interrupt, 0, 603 queue->vif->domid, rx_evtchn, xenvif_rx_interrupt, 0,
491 vif->rx_irq_name, vif); 604 queue->rx_irq_name, queue);
492 if (err < 0) 605 if (err < 0)
493 goto err_tx_unbind; 606 goto err_tx_unbind;
494 vif->rx_irq = err; 607 queue->rx_irq = err;
495 disable_irq(vif->rx_irq); 608 disable_irq(queue->rx_irq);
496 } 609 }
497 610
498 task = kthread_create(xenvif_kthread_guest_rx, 611 task = kthread_create(xenvif_kthread_guest_rx,
499 (void *)vif, "%s-guest-rx", vif->dev->name); 612 (void *)queue, "%s-guest-rx", queue->name);
500 if (IS_ERR(task)) { 613 if (IS_ERR(task)) {
501 pr_warn("Could not allocate kthread for %s\n", vif->dev->name); 614 pr_warn("Could not allocate kthread for %s\n", queue->name);
502 err = PTR_ERR(task); 615 err = PTR_ERR(task);
503 goto err_rx_unbind; 616 goto err_rx_unbind;
504 } 617 }
505 618 queue->task = task;
506 vif->task = task;
507 619
508 task = kthread_create(xenvif_dealloc_kthread, 620 task = kthread_create(xenvif_dealloc_kthread,
509 (void *)vif, "%s-dealloc", vif->dev->name); 621 (void *)queue, "%s-dealloc", queue->name);
510 if (IS_ERR(task)) { 622 if (IS_ERR(task)) {
511 pr_warn("Could not allocate kthread for %s\n", vif->dev->name); 623 pr_warn("Could not allocate kthread for %s\n", queue->name);
512 err = PTR_ERR(task); 624 err = PTR_ERR(task);
513 goto err_rx_unbind; 625 goto err_rx_unbind;
514 } 626 }
627 queue->dealloc_task = task;
515 628
516 vif->dealloc_task = task; 629 wake_up_process(queue->task);
517 630 wake_up_process(queue->dealloc_task);
518 rtnl_lock();
519 if (!vif->can_sg && vif->dev->mtu > ETH_DATA_LEN)
520 dev_set_mtu(vif->dev, ETH_DATA_LEN);
521 netdev_update_features(vif->dev);
522 netif_carrier_on(vif->dev);
523 if (netif_running(vif->dev))
524 xenvif_up(vif);
525 rtnl_unlock();
526
527 wake_up_process(vif->task);
528 wake_up_process(vif->dealloc_task);
529 631
530 return 0; 632 return 0;
531 633
532err_rx_unbind: 634err_rx_unbind:
533 unbind_from_irqhandler(vif->rx_irq, vif); 635 unbind_from_irqhandler(queue->rx_irq, queue);
534 vif->rx_irq = 0; 636 queue->rx_irq = 0;
535err_tx_unbind: 637err_tx_unbind:
536 unbind_from_irqhandler(vif->tx_irq, vif); 638 unbind_from_irqhandler(queue->tx_irq, queue);
537 vif->tx_irq = 0; 639 queue->tx_irq = 0;
538err_unmap: 640err_unmap:
539 xenvif_unmap_frontend_rings(vif); 641 xenvif_unmap_frontend_rings(queue);
540err: 642err:
541 module_put(THIS_MODULE); 643 module_put(THIS_MODULE);
542 return err; 644 return err;
@@ -553,38 +655,77 @@ void xenvif_carrier_off(struct xenvif *vif)
553 rtnl_unlock(); 655 rtnl_unlock();
554} 656}
555 657
658static void xenvif_wait_unmap_timeout(struct xenvif_queue *queue,
659 unsigned int worst_case_skb_lifetime)
660{
661 int i, unmap_timeout = 0;
662
663 for (i = 0; i < MAX_PENDING_REQS; ++i) {
664 if (queue->grant_tx_handle[i] != NETBACK_INVALID_HANDLE) {
665 unmap_timeout++;
666 schedule_timeout(msecs_to_jiffies(1000));
667 if (unmap_timeout > worst_case_skb_lifetime &&
668 net_ratelimit())
669 netdev_err(queue->vif->dev,
670 "Page still granted! Index: %x\n",
671 i);
672 i = -1;
673 }
674 }
675}
676
556void xenvif_disconnect(struct xenvif *vif) 677void xenvif_disconnect(struct xenvif *vif)
557{ 678{
679 struct xenvif_queue *queue = NULL;
680 unsigned int num_queues = vif->dev->real_num_tx_queues;
681 unsigned int queue_index;
682
558 if (netif_carrier_ok(vif->dev)) 683 if (netif_carrier_ok(vif->dev))
559 xenvif_carrier_off(vif); 684 xenvif_carrier_off(vif);
560 685
561 if (vif->task) { 686 for (queue_index = 0; queue_index < num_queues; ++queue_index) {
562 del_timer_sync(&vif->wake_queue); 687 queue = &vif->queues[queue_index];
563 kthread_stop(vif->task);
564 vif->task = NULL;
565 }
566 688
567 if (vif->dealloc_task) { 689 if (queue->task) {
568 kthread_stop(vif->dealloc_task); 690 del_timer_sync(&queue->wake_queue);
569 vif->dealloc_task = NULL; 691 kthread_stop(queue->task);
570 } 692 queue->task = NULL;
693 }
571 694
572 if (vif->tx_irq) { 695 if (queue->dealloc_task) {
573 if (vif->tx_irq == vif->rx_irq) 696 kthread_stop(queue->dealloc_task);
574 unbind_from_irqhandler(vif->tx_irq, vif); 697 queue->dealloc_task = NULL;
575 else {
576 unbind_from_irqhandler(vif->tx_irq, vif);
577 unbind_from_irqhandler(vif->rx_irq, vif);
578 } 698 }
579 vif->tx_irq = 0; 699
700 if (queue->tx_irq) {
701 if (queue->tx_irq == queue->rx_irq)
702 unbind_from_irqhandler(queue->tx_irq, queue);
703 else {
704 unbind_from_irqhandler(queue->tx_irq, queue);
705 unbind_from_irqhandler(queue->rx_irq, queue);
706 }
707 queue->tx_irq = 0;
708 }
709
710 xenvif_unmap_frontend_rings(queue);
580 } 711 }
712}
581 713
582 xenvif_unmap_frontend_rings(vif); 714/* Reverse the relevant parts of xenvif_init_queue().
715 * Used for queue teardown from xenvif_free(), and on the
716 * error handling paths in xenbus.c:connect().
717 */
718void xenvif_deinit_queue(struct xenvif_queue *queue)
719{
720 free_xenballooned_pages(MAX_PENDING_REQS, queue->mmap_pages);
721 netif_napi_del(&queue->napi);
583} 722}
584 723
585void xenvif_free(struct xenvif *vif) 724void xenvif_free(struct xenvif *vif)
586{ 725{
587 int i, unmap_timeout = 0; 726 struct xenvif_queue *queue = NULL;
727 unsigned int num_queues = vif->dev->real_num_tx_queues;
728 unsigned int queue_index;
588 /* Here we want to avoid timeout messages if an skb can be legitimately 729 /* Here we want to avoid timeout messages if an skb can be legitimately
589 * stuck somewhere else. Realistically this could be an another vif's 730 * stuck somewhere else. Realistically this could be an another vif's
590 * internal or QDisc queue. That another vif also has this 731 * internal or QDisc queue. That another vif also has this
@@ -599,33 +740,21 @@ void xenvif_free(struct xenvif *vif)
599 unsigned int worst_case_skb_lifetime = (rx_drain_timeout_msecs/1000) * 740 unsigned int worst_case_skb_lifetime = (rx_drain_timeout_msecs/1000) *
600 DIV_ROUND_UP(XENVIF_QUEUE_LENGTH, (XEN_NETIF_RX_RING_SIZE / MAX_SKB_FRAGS)); 741 DIV_ROUND_UP(XENVIF_QUEUE_LENGTH, (XEN_NETIF_RX_RING_SIZE / MAX_SKB_FRAGS));
601 742
602 for (i = 0; i < MAX_PENDING_REQS; ++i) { 743 unregister_netdev(vif->dev);
603 if (vif->grant_tx_handle[i] != NETBACK_INVALID_HANDLE) {
604 unmap_timeout++;
605 schedule_timeout(msecs_to_jiffies(1000));
606 if (unmap_timeout > worst_case_skb_lifetime &&
607 net_ratelimit())
608 netdev_err(vif->dev,
609 "Page still granted! Index: %x\n",
610 i);
611 /* If there are still unmapped pages, reset the loop to
612 * start checking again. We shouldn't exit here until
613 * dealloc thread and NAPI instance release all the
614 * pages. If a kernel bug causes the skbs to stall
615 * somewhere, the interface cannot be brought down
616 * properly.
617 */
618 i = -1;
619 }
620 }
621
622 free_xenballooned_pages(MAX_PENDING_REQS, vif->mmap_pages);
623 744
624 netif_napi_del(&vif->napi); 745 for (queue_index = 0; queue_index < num_queues; ++queue_index) {
746 queue = &vif->queues[queue_index];
747 xenvif_wait_unmap_timeout(queue, worst_case_skb_lifetime);
748 xenvif_deinit_queue(queue);
749 }
625 750
626 unregister_netdev(vif->dev); 751 /* Free the array of queues. The call below does not require
752 * rtnl_lock() because it happens after unregister_netdev().
753 */
754 netif_set_real_num_tx_queues(vif->dev, 0);
755 vfree(vif->queues);
756 vif->queues = NULL;
627 757
628 vfree(vif->grant_copy_op);
629 free_netdev(vif->dev); 758 free_netdev(vif->dev);
630 759
631 module_put(THIS_MODULE); 760 module_put(THIS_MODULE);
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 76665405c5aa..1844a47636b6 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -62,6 +62,11 @@ unsigned int rx_drain_timeout_msecs = 10000;
62module_param(rx_drain_timeout_msecs, uint, 0444); 62module_param(rx_drain_timeout_msecs, uint, 0444);
63unsigned int rx_drain_timeout_jiffies; 63unsigned int rx_drain_timeout_jiffies;
64 64
65unsigned int xenvif_max_queues;
66module_param_named(max_queues, xenvif_max_queues, uint, 0644);
67MODULE_PARM_DESC(max_queues,
68 "Maximum number of queues per virtual interface");
69
65/* 70/*
66 * This is the maximum slots a skb can have. If a guest sends a skb 71 * This is the maximum slots a skb can have. If a guest sends a skb
67 * which exceeds this limit it is considered malicious. 72 * which exceeds this limit it is considered malicious.
@@ -70,33 +75,33 @@ unsigned int rx_drain_timeout_jiffies;
70static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT; 75static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT;
71module_param(fatal_skb_slots, uint, 0444); 76module_param(fatal_skb_slots, uint, 0444);
72 77
73static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx, 78static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx,
74 u8 status); 79 u8 status);
75 80
76static void make_tx_response(struct xenvif *vif, 81static void make_tx_response(struct xenvif_queue *queue,
77 struct xen_netif_tx_request *txp, 82 struct xen_netif_tx_request *txp,
78 s8 st); 83 s8 st);
79 84
80static inline int tx_work_todo(struct xenvif *vif); 85static inline int tx_work_todo(struct xenvif_queue *queue);
81static inline int rx_work_todo(struct xenvif *vif); 86static inline int rx_work_todo(struct xenvif_queue *queue);
82 87
83static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif, 88static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
84 u16 id, 89 u16 id,
85 s8 st, 90 s8 st,
86 u16 offset, 91 u16 offset,
87 u16 size, 92 u16 size,
88 u16 flags); 93 u16 flags);
89 94
90static inline unsigned long idx_to_pfn(struct xenvif *vif, 95static inline unsigned long idx_to_pfn(struct xenvif_queue *queue,
91 u16 idx) 96 u16 idx)
92{ 97{
93 return page_to_pfn(vif->mmap_pages[idx]); 98 return page_to_pfn(queue->mmap_pages[idx]);
94} 99}
95 100
96static inline unsigned long idx_to_kaddr(struct xenvif *vif, 101static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue,
97 u16 idx) 102 u16 idx)
98{ 103{
99 return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif, idx)); 104 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx));
100} 105}
101 106
102#define callback_param(vif, pending_idx) \ 107#define callback_param(vif, pending_idx) \
@@ -104,13 +109,13 @@ static inline unsigned long idx_to_kaddr(struct xenvif *vif,
104 109
105/* Find the containing VIF's structure from a pointer in pending_tx_info array 110/* Find the containing VIF's structure from a pointer in pending_tx_info array
106 */ 111 */
107static inline struct xenvif* ubuf_to_vif(struct ubuf_info *ubuf) 112static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info *ubuf)
108{ 113{
109 u16 pending_idx = ubuf->desc; 114 u16 pending_idx = ubuf->desc;
110 struct pending_tx_info *temp = 115 struct pending_tx_info *temp =
111 container_of(ubuf, struct pending_tx_info, callback_struct); 116 container_of(ubuf, struct pending_tx_info, callback_struct);
112 return container_of(temp - pending_idx, 117 return container_of(temp - pending_idx,
113 struct xenvif, 118 struct xenvif_queue,
114 pending_tx_info[0]); 119 pending_tx_info[0]);
115} 120}
116 121
@@ -136,24 +141,24 @@ static inline pending_ring_idx_t pending_index(unsigned i)
136 return i & (MAX_PENDING_REQS-1); 141 return i & (MAX_PENDING_REQS-1);
137} 142}
138 143
139bool xenvif_rx_ring_slots_available(struct xenvif *vif, int needed) 144bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue, int needed)
140{ 145{
141 RING_IDX prod, cons; 146 RING_IDX prod, cons;
142 147
143 do { 148 do {
144 prod = vif->rx.sring->req_prod; 149 prod = queue->rx.sring->req_prod;
145 cons = vif->rx.req_cons; 150 cons = queue->rx.req_cons;
146 151
147 if (prod - cons >= needed) 152 if (prod - cons >= needed)
148 return true; 153 return true;
149 154
150 vif->rx.sring->req_event = prod + 1; 155 queue->rx.sring->req_event = prod + 1;
151 156
152 /* Make sure event is visible before we check prod 157 /* Make sure event is visible before we check prod
153 * again. 158 * again.
154 */ 159 */
155 mb(); 160 mb();
156 } while (vif->rx.sring->req_prod != prod); 161 } while (queue->rx.sring->req_prod != prod);
157 162
158 return false; 163 return false;
159} 164}
@@ -163,7 +168,8 @@ bool xenvif_rx_ring_slots_available(struct xenvif *vif, int needed)
163 * adding 'size' bytes to a buffer which currently contains 'offset' 168 * adding 'size' bytes to a buffer which currently contains 'offset'
164 * bytes. 169 * bytes.
165 */ 170 */
166static bool start_new_rx_buffer(int offset, unsigned long size, int head) 171static bool start_new_rx_buffer(int offset, unsigned long size, int head,
172 bool full_coalesce)
167{ 173{
168 /* simple case: we have completely filled the current buffer. */ 174 /* simple case: we have completely filled the current buffer. */
169 if (offset == MAX_BUFFER_OFFSET) 175 if (offset == MAX_BUFFER_OFFSET)
@@ -175,6 +181,7 @@ static bool start_new_rx_buffer(int offset, unsigned long size, int head)
175 * (i) this frag would fit completely in the next buffer 181 * (i) this frag would fit completely in the next buffer
176 * and (ii) there is already some data in the current buffer 182 * and (ii) there is already some data in the current buffer
177 * and (iii) this is not the head buffer. 183 * and (iii) this is not the head buffer.
184 * and (iv) there is no need to fully utilize the buffers
178 * 185 *
179 * Where: 186 * Where:
180 * - (i) stops us splitting a frag into two copies 187 * - (i) stops us splitting a frag into two copies
@@ -185,6 +192,8 @@ static bool start_new_rx_buffer(int offset, unsigned long size, int head)
185 * by (ii) but is explicitly checked because 192 * by (ii) but is explicitly checked because
186 * netfront relies on the first buffer being 193 * netfront relies on the first buffer being
187 * non-empty and can crash otherwise. 194 * non-empty and can crash otherwise.
195 * - (iv) is needed for skbs which can use up more than MAX_SKB_FRAGS
196 * slot
188 * 197 *
189 * This means we will effectively linearise small 198 * This means we will effectively linearise small
190 * frags but do not needlessly split large buffers 199 * frags but do not needlessly split large buffers
@@ -192,7 +201,8 @@ static bool start_new_rx_buffer(int offset, unsigned long size, int head)
192 * own buffers as before. 201 * own buffers as before.
193 */ 202 */
194 BUG_ON(size > MAX_BUFFER_OFFSET); 203 BUG_ON(size > MAX_BUFFER_OFFSET);
195 if ((offset + size > MAX_BUFFER_OFFSET) && offset && !head) 204 if ((offset + size > MAX_BUFFER_OFFSET) && offset && !head &&
205 !full_coalesce)
196 return true; 206 return true;
197 207
198 return false; 208 return false;
@@ -207,13 +217,13 @@ struct netrx_pending_operations {
207 grant_ref_t copy_gref; 217 grant_ref_t copy_gref;
208}; 218};
209 219
210static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif *vif, 220static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue,
211 struct netrx_pending_operations *npo) 221 struct netrx_pending_operations *npo)
212{ 222{
213 struct xenvif_rx_meta *meta; 223 struct xenvif_rx_meta *meta;
214 struct xen_netif_rx_request *req; 224 struct xen_netif_rx_request *req;
215 225
216 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 226 req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons++);
217 227
218 meta = npo->meta + npo->meta_prod++; 228 meta = npo->meta + npo->meta_prod++;
219 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE; 229 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
@@ -227,15 +237,22 @@ static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif *vif,
227 return meta; 237 return meta;
228} 238}
229 239
240struct xenvif_rx_cb {
241 int meta_slots_used;
242 bool full_coalesce;
243};
244
245#define XENVIF_RX_CB(skb) ((struct xenvif_rx_cb *)(skb)->cb)
246
230/* 247/*
231 * Set up the grant operations for this fragment. If it's a flipping 248 * Set up the grant operations for this fragment. If it's a flipping
232 * interface, we also set up the unmap request from here. 249 * interface, we also set up the unmap request from here.
233 */ 250 */
234static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb, 251static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb,
235 struct netrx_pending_operations *npo, 252 struct netrx_pending_operations *npo,
236 struct page *page, unsigned long size, 253 struct page *page, unsigned long size,
237 unsigned long offset, int *head, 254 unsigned long offset, int *head,
238 struct xenvif *foreign_vif, 255 struct xenvif_queue *foreign_queue,
239 grant_ref_t foreign_gref) 256 grant_ref_t foreign_gref)
240{ 257{
241 struct gnttab_copy *copy_gop; 258 struct gnttab_copy *copy_gop;
@@ -261,14 +278,17 @@ static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
261 if (bytes > size) 278 if (bytes > size)
262 bytes = size; 279 bytes = size;
263 280
264 if (start_new_rx_buffer(npo->copy_off, bytes, *head)) { 281 if (start_new_rx_buffer(npo->copy_off,
282 bytes,
283 *head,
284 XENVIF_RX_CB(skb)->full_coalesce)) {
265 /* 285 /*
266 * Netfront requires there to be some data in the head 286 * Netfront requires there to be some data in the head
267 * buffer. 287 * buffer.
268 */ 288 */
269 BUG_ON(*head); 289 BUG_ON(*head);
270 290
271 meta = get_next_rx_buffer(vif, npo); 291 meta = get_next_rx_buffer(queue, npo);
272 } 292 }
273 293
274 if (npo->copy_off + bytes > MAX_BUFFER_OFFSET) 294 if (npo->copy_off + bytes > MAX_BUFFER_OFFSET)
@@ -278,8 +298,8 @@ static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
278 copy_gop->flags = GNTCOPY_dest_gref; 298 copy_gop->flags = GNTCOPY_dest_gref;
279 copy_gop->len = bytes; 299 copy_gop->len = bytes;
280 300
281 if (foreign_vif) { 301 if (foreign_queue) {
282 copy_gop->source.domid = foreign_vif->domid; 302 copy_gop->source.domid = foreign_queue->vif->domid;
283 copy_gop->source.u.ref = foreign_gref; 303 copy_gop->source.u.ref = foreign_gref;
284 copy_gop->flags |= GNTCOPY_source_gref; 304 copy_gop->flags |= GNTCOPY_source_gref;
285 } else { 305 } else {
@@ -289,7 +309,7 @@ static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
289 } 309 }
290 copy_gop->source.offset = offset; 310 copy_gop->source.offset = offset;
291 311
292 copy_gop->dest.domid = vif->domid; 312 copy_gop->dest.domid = queue->vif->domid;
293 copy_gop->dest.offset = npo->copy_off; 313 copy_gop->dest.offset = npo->copy_off;
294 copy_gop->dest.u.ref = npo->copy_gref; 314 copy_gop->dest.u.ref = npo->copy_gref;
295 315
@@ -314,8 +334,8 @@ static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
314 gso_type = XEN_NETIF_GSO_TYPE_TCPV6; 334 gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
315 } 335 }
316 336
317 if (*head && ((1 << gso_type) & vif->gso_mask)) 337 if (*head && ((1 << gso_type) & queue->vif->gso_mask))
318 vif->rx.req_cons++; 338 queue->rx.req_cons++;
319 339
320 *head = 0; /* There must be something in this buffer now. */ 340 *head = 0; /* There must be something in this buffer now. */
321 341
@@ -323,6 +343,35 @@ static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
323} 343}
324 344
325/* 345/*
346 * Find the grant ref for a given frag in a chain of struct ubuf_info's
347 * skb: the skb itself
348 * i: the frag's number
349 * ubuf: a pointer to an element in the chain. It should not be NULL
350 *
351 * Returns a pointer to the element in the chain where the page were found. If
352 * not found, returns NULL.
353 * See the definition of callback_struct in common.h for more details about
354 * the chain.
355 */
356static const struct ubuf_info *xenvif_find_gref(const struct sk_buff *const skb,
357 const int i,
358 const struct ubuf_info *ubuf)
359{
360 struct xenvif_queue *foreign_queue = ubuf_to_queue(ubuf);
361
362 do {
363 u16 pending_idx = ubuf->desc;
364
365 if (skb_shinfo(skb)->frags[i].page.p ==
366 foreign_queue->mmap_pages[pending_idx])
367 break;
368 ubuf = (struct ubuf_info *) ubuf->ctx;
369 } while (ubuf);
370
371 return ubuf;
372}
373
374/*
326 * Prepare an SKB to be transmitted to the frontend. 375 * Prepare an SKB to be transmitted to the frontend.
327 * 376 *
328 * This function is responsible for allocating grant operations, meta 377 * This function is responsible for allocating grant operations, meta
@@ -335,7 +384,8 @@ static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
335 * frontend-side LRO). 384 * frontend-side LRO).
336 */ 385 */
337static int xenvif_gop_skb(struct sk_buff *skb, 386static int xenvif_gop_skb(struct sk_buff *skb,
338 struct netrx_pending_operations *npo) 387 struct netrx_pending_operations *npo,
388 struct xenvif_queue *queue)
339{ 389{
340 struct xenvif *vif = netdev_priv(skb->dev); 390 struct xenvif *vif = netdev_priv(skb->dev);
341 int nr_frags = skb_shinfo(skb)->nr_frags; 391 int nr_frags = skb_shinfo(skb)->nr_frags;
@@ -346,9 +396,8 @@ static int xenvif_gop_skb(struct sk_buff *skb,
346 int head = 1; 396 int head = 1;
347 int old_meta_prod; 397 int old_meta_prod;
348 int gso_type; 398 int gso_type;
349 struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg; 399 const struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg;
350 grant_ref_t foreign_grefs[MAX_SKB_FRAGS]; 400 const struct ubuf_info *const head_ubuf = ubuf;
351 struct xenvif *foreign_vif = NULL;
352 401
353 old_meta_prod = npo->meta_prod; 402 old_meta_prod = npo->meta_prod;
354 403
@@ -362,7 +411,7 @@ static int xenvif_gop_skb(struct sk_buff *skb,
362 411
363 /* Set up a GSO prefix descriptor, if necessary */ 412 /* Set up a GSO prefix descriptor, if necessary */
364 if ((1 << gso_type) & vif->gso_prefix_mask) { 413 if ((1 << gso_type) & vif->gso_prefix_mask) {
365 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 414 req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons++);
366 meta = npo->meta + npo->meta_prod++; 415 meta = npo->meta + npo->meta_prod++;
367 meta->gso_type = gso_type; 416 meta->gso_type = gso_type;
368 meta->gso_size = skb_shinfo(skb)->gso_size; 417 meta->gso_size = skb_shinfo(skb)->gso_size;
@@ -370,7 +419,7 @@ static int xenvif_gop_skb(struct sk_buff *skb,
370 meta->id = req->id; 419 meta->id = req->id;
371 } 420 }
372 421
373 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++); 422 req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons++);
374 meta = npo->meta + npo->meta_prod++; 423 meta = npo->meta + npo->meta_prod++;
375 424
376 if ((1 << gso_type) & vif->gso_mask) { 425 if ((1 << gso_type) & vif->gso_mask) {
@@ -386,19 +435,6 @@ static int xenvif_gop_skb(struct sk_buff *skb,
386 npo->copy_off = 0; 435 npo->copy_off = 0;
387 npo->copy_gref = req->gref; 436 npo->copy_gref = req->gref;
388 437
389 if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
390 (ubuf->callback == &xenvif_zerocopy_callback)) {
391 int i = 0;
392 foreign_vif = ubuf_to_vif(ubuf);
393
394 do {
395 u16 pending_idx = ubuf->desc;
396 foreign_grefs[i++] =
397 foreign_vif->pending_tx_info[pending_idx].req.gref;
398 ubuf = (struct ubuf_info *) ubuf->ctx;
399 } while (ubuf);
400 }
401
402 data = skb->data; 438 data = skb->data;
403 while (data < skb_tail_pointer(skb)) { 439 while (data < skb_tail_pointer(skb)) {
404 unsigned int offset = offset_in_page(data); 440 unsigned int offset = offset_in_page(data);
@@ -407,7 +443,7 @@ static int xenvif_gop_skb(struct sk_buff *skb,
407 if (data + len > skb_tail_pointer(skb)) 443 if (data + len > skb_tail_pointer(skb))
408 len = skb_tail_pointer(skb) - data; 444 len = skb_tail_pointer(skb) - data;
409 445
410 xenvif_gop_frag_copy(vif, skb, npo, 446 xenvif_gop_frag_copy(queue, skb, npo,
411 virt_to_page(data), len, offset, &head, 447 virt_to_page(data), len, offset, &head,
412 NULL, 448 NULL,
413 0); 449 0);
@@ -415,13 +451,61 @@ static int xenvif_gop_skb(struct sk_buff *skb,
415 } 451 }
416 452
417 for (i = 0; i < nr_frags; i++) { 453 for (i = 0; i < nr_frags; i++) {
418 xenvif_gop_frag_copy(vif, skb, npo, 454 /* This variable also signals whether foreign_gref has a real
455 * value or not.
456 */
457 struct xenvif_queue *foreign_queue = NULL;
458 grant_ref_t foreign_gref;
459
460 if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
461 (ubuf->callback == &xenvif_zerocopy_callback)) {
462 const struct ubuf_info *const startpoint = ubuf;
463
464 /* Ideally ubuf points to the chain element which
465 * belongs to this frag. Or if frags were removed from
466 * the beginning, then shortly before it.
467 */
468 ubuf = xenvif_find_gref(skb, i, ubuf);
469
470 /* Try again from the beginning of the list, if we
471 * haven't tried from there. This only makes sense in
472 * the unlikely event of reordering the original frags.
473 * For injected local pages it's an unnecessary second
474 * run.
475 */
476 if (unlikely(!ubuf) && startpoint != head_ubuf)
477 ubuf = xenvif_find_gref(skb, i, head_ubuf);
478
479 if (likely(ubuf)) {
480 u16 pending_idx = ubuf->desc;
481
482 foreign_queue = ubuf_to_queue(ubuf);
483 foreign_gref =
484 foreign_queue->pending_tx_info[pending_idx].req.gref;
485 /* Just a safety measure. If this was the last
486 * element on the list, the for loop will
487 * iterate again if a local page were added to
488 * the end. Using head_ubuf here prevents the
489 * second search on the chain. Or the original
490 * frags changed order, but that's less likely.
491 * In any way, ubuf shouldn't be NULL.
492 */
493 ubuf = ubuf->ctx ?
494 (struct ubuf_info *) ubuf->ctx :
495 head_ubuf;
496 } else
497 /* This frag was a local page, added to the
498 * array after the skb left netback.
499 */
500 ubuf = head_ubuf;
501 }
502 xenvif_gop_frag_copy(queue, skb, npo,
419 skb_frag_page(&skb_shinfo(skb)->frags[i]), 503 skb_frag_page(&skb_shinfo(skb)->frags[i]),
420 skb_frag_size(&skb_shinfo(skb)->frags[i]), 504 skb_frag_size(&skb_shinfo(skb)->frags[i]),
421 skb_shinfo(skb)->frags[i].page_offset, 505 skb_shinfo(skb)->frags[i].page_offset,
422 &head, 506 &head,
423 foreign_vif, 507 foreign_queue,
424 foreign_grefs[i]); 508 foreign_queue ? foreign_gref : UINT_MAX);
425 } 509 }
426 510
427 return npo->meta_prod - old_meta_prod; 511 return npo->meta_prod - old_meta_prod;
@@ -453,7 +537,7 @@ static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
453 return status; 537 return status;
454} 538}
455 539
456static void xenvif_add_frag_responses(struct xenvif *vif, int status, 540static void xenvif_add_frag_responses(struct xenvif_queue *queue, int status,
457 struct xenvif_rx_meta *meta, 541 struct xenvif_rx_meta *meta,
458 int nr_meta_slots) 542 int nr_meta_slots)
459{ 543{
@@ -474,23 +558,17 @@ static void xenvif_add_frag_responses(struct xenvif *vif, int status,
474 flags = XEN_NETRXF_more_data; 558 flags = XEN_NETRXF_more_data;
475 559
476 offset = 0; 560 offset = 0;
477 make_rx_response(vif, meta[i].id, status, offset, 561 make_rx_response(queue, meta[i].id, status, offset,
478 meta[i].size, flags); 562 meta[i].size, flags);
479 } 563 }
480} 564}
481 565
482struct xenvif_rx_cb { 566void xenvif_kick_thread(struct xenvif_queue *queue)
483 int meta_slots_used;
484};
485
486#define XENVIF_RX_CB(skb) ((struct xenvif_rx_cb *)(skb)->cb)
487
488void xenvif_kick_thread(struct xenvif *vif)
489{ 567{
490 wake_up(&vif->wq); 568 wake_up(&queue->wq);
491} 569}
492 570
493static void xenvif_rx_action(struct xenvif *vif) 571static void xenvif_rx_action(struct xenvif_queue *queue)
494{ 572{
495 s8 status; 573 s8 status;
496 u16 flags; 574 u16 flags;
@@ -503,13 +581,13 @@ static void xenvif_rx_action(struct xenvif *vif)
503 bool need_to_notify = false; 581 bool need_to_notify = false;
504 582
505 struct netrx_pending_operations npo = { 583 struct netrx_pending_operations npo = {
506 .copy = vif->grant_copy_op, 584 .copy = queue->grant_copy_op,
507 .meta = vif->meta, 585 .meta = queue->meta,
508 }; 586 };
509 587
510 skb_queue_head_init(&rxq); 588 skb_queue_head_init(&rxq);
511 589
512 while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) { 590 while ((skb = skb_dequeue(&queue->rx_queue)) != NULL) {
513 RING_IDX max_slots_needed; 591 RING_IDX max_slots_needed;
514 RING_IDX old_req_cons; 592 RING_IDX old_req_cons;
515 RING_IDX ring_slots_used; 593 RING_IDX ring_slots_used;
@@ -540,10 +618,15 @@ static void xenvif_rx_action(struct xenvif *vif)
540 618
541 /* To avoid the estimate becoming too pessimal for some 619 /* To avoid the estimate becoming too pessimal for some
542 * frontends that limit posted rx requests, cap the estimate 620 * frontends that limit posted rx requests, cap the estimate
543 * at MAX_SKB_FRAGS. 621 * at MAX_SKB_FRAGS. In this case netback will fully coalesce
622 * the skb into the provided slots.
544 */ 623 */
545 if (max_slots_needed > MAX_SKB_FRAGS) 624 if (max_slots_needed > MAX_SKB_FRAGS) {
546 max_slots_needed = MAX_SKB_FRAGS; 625 max_slots_needed = MAX_SKB_FRAGS;
626 XENVIF_RX_CB(skb)->full_coalesce = true;
627 } else {
628 XENVIF_RX_CB(skb)->full_coalesce = false;
629 }
547 630
548 /* We may need one more slot for GSO metadata */ 631 /* We may need one more slot for GSO metadata */
549 if (skb_is_gso(skb) && 632 if (skb_is_gso(skb) &&
@@ -552,42 +635,42 @@ static void xenvif_rx_action(struct xenvif *vif)
552 max_slots_needed++; 635 max_slots_needed++;
553 636
554 /* If the skb may not fit then bail out now */ 637 /* If the skb may not fit then bail out now */
555 if (!xenvif_rx_ring_slots_available(vif, max_slots_needed)) { 638 if (!xenvif_rx_ring_slots_available(queue, max_slots_needed)) {
556 skb_queue_head(&vif->rx_queue, skb); 639 skb_queue_head(&queue->rx_queue, skb);
557 need_to_notify = true; 640 need_to_notify = true;
558 vif->rx_last_skb_slots = max_slots_needed; 641 queue->rx_last_skb_slots = max_slots_needed;
559 break; 642 break;
560 } else 643 } else
561 vif->rx_last_skb_slots = 0; 644 queue->rx_last_skb_slots = 0;
562 645
563 old_req_cons = vif->rx.req_cons; 646 old_req_cons = queue->rx.req_cons;
564 XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo); 647 XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo, queue);
565 ring_slots_used = vif->rx.req_cons - old_req_cons; 648 ring_slots_used = queue->rx.req_cons - old_req_cons;
566 649
567 BUG_ON(ring_slots_used > max_slots_needed); 650 BUG_ON(ring_slots_used > max_slots_needed);
568 651
569 __skb_queue_tail(&rxq, skb); 652 __skb_queue_tail(&rxq, skb);
570 } 653 }
571 654
572 BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta)); 655 BUG_ON(npo.meta_prod > ARRAY_SIZE(queue->meta));
573 656
574 if (!npo.copy_prod) 657 if (!npo.copy_prod)
575 goto done; 658 goto done;
576 659
577 BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS); 660 BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS);
578 gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod); 661 gnttab_batch_copy(queue->grant_copy_op, npo.copy_prod);
579 662
580 while ((skb = __skb_dequeue(&rxq)) != NULL) { 663 while ((skb = __skb_dequeue(&rxq)) != NULL) {
581 664
582 if ((1 << vif->meta[npo.meta_cons].gso_type) & 665 if ((1 << queue->meta[npo.meta_cons].gso_type) &
583 vif->gso_prefix_mask) { 666 queue->vif->gso_prefix_mask) {
584 resp = RING_GET_RESPONSE(&vif->rx, 667 resp = RING_GET_RESPONSE(&queue->rx,
585 vif->rx.rsp_prod_pvt++); 668 queue->rx.rsp_prod_pvt++);
586 669
587 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data; 670 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;
588 671
589 resp->offset = vif->meta[npo.meta_cons].gso_size; 672 resp->offset = queue->meta[npo.meta_cons].gso_size;
590 resp->id = vif->meta[npo.meta_cons].id; 673 resp->id = queue->meta[npo.meta_cons].id;
591 resp->status = XENVIF_RX_CB(skb)->meta_slots_used; 674 resp->status = XENVIF_RX_CB(skb)->meta_slots_used;
592 675
593 npo.meta_cons++; 676 npo.meta_cons++;
@@ -595,10 +678,10 @@ static void xenvif_rx_action(struct xenvif *vif)
595 } 678 }
596 679
597 680
598 vif->dev->stats.tx_bytes += skb->len; 681 queue->stats.tx_bytes += skb->len;
599 vif->dev->stats.tx_packets++; 682 queue->stats.tx_packets++;
600 683
601 status = xenvif_check_gop(vif, 684 status = xenvif_check_gop(queue->vif,
602 XENVIF_RX_CB(skb)->meta_slots_used, 685 XENVIF_RX_CB(skb)->meta_slots_used,
603 &npo); 686 &npo);
604 687
@@ -614,22 +697,22 @@ static void xenvif_rx_action(struct xenvif *vif)
614 flags |= XEN_NETRXF_data_validated; 697 flags |= XEN_NETRXF_data_validated;
615 698
616 offset = 0; 699 offset = 0;
617 resp = make_rx_response(vif, vif->meta[npo.meta_cons].id, 700 resp = make_rx_response(queue, queue->meta[npo.meta_cons].id,
618 status, offset, 701 status, offset,
619 vif->meta[npo.meta_cons].size, 702 queue->meta[npo.meta_cons].size,
620 flags); 703 flags);
621 704
622 if ((1 << vif->meta[npo.meta_cons].gso_type) & 705 if ((1 << queue->meta[npo.meta_cons].gso_type) &
623 vif->gso_mask) { 706 queue->vif->gso_mask) {
624 struct xen_netif_extra_info *gso = 707 struct xen_netif_extra_info *gso =
625 (struct xen_netif_extra_info *) 708 (struct xen_netif_extra_info *)
626 RING_GET_RESPONSE(&vif->rx, 709 RING_GET_RESPONSE(&queue->rx,
627 vif->rx.rsp_prod_pvt++); 710 queue->rx.rsp_prod_pvt++);
628 711
629 resp->flags |= XEN_NETRXF_extra_info; 712 resp->flags |= XEN_NETRXF_extra_info;
630 713
631 gso->u.gso.type = vif->meta[npo.meta_cons].gso_type; 714 gso->u.gso.type = queue->meta[npo.meta_cons].gso_type;
632 gso->u.gso.size = vif->meta[npo.meta_cons].gso_size; 715 gso->u.gso.size = queue->meta[npo.meta_cons].gso_size;
633 gso->u.gso.pad = 0; 716 gso->u.gso.pad = 0;
634 gso->u.gso.features = 0; 717 gso->u.gso.features = 0;
635 718
@@ -637,11 +720,11 @@ static void xenvif_rx_action(struct xenvif *vif)
637 gso->flags = 0; 720 gso->flags = 0;
638 } 721 }
639 722
640 xenvif_add_frag_responses(vif, status, 723 xenvif_add_frag_responses(queue, status,
641 vif->meta + npo.meta_cons + 1, 724 queue->meta + npo.meta_cons + 1,
642 XENVIF_RX_CB(skb)->meta_slots_used); 725 XENVIF_RX_CB(skb)->meta_slots_used);
643 726
644 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret); 727 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, ret);
645 728
646 need_to_notify |= !!ret; 729 need_to_notify |= !!ret;
647 730
@@ -651,20 +734,20 @@ static void xenvif_rx_action(struct xenvif *vif)
651 734
652done: 735done:
653 if (need_to_notify) 736 if (need_to_notify)
654 notify_remote_via_irq(vif->rx_irq); 737 notify_remote_via_irq(queue->rx_irq);
655} 738}
656 739
657void xenvif_check_rx_xenvif(struct xenvif *vif) 740void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue)
658{ 741{
659 int more_to_do; 742 int more_to_do;
660 743
661 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do); 744 RING_FINAL_CHECK_FOR_REQUESTS(&queue->tx, more_to_do);
662 745
663 if (more_to_do) 746 if (more_to_do)
664 napi_schedule(&vif->napi); 747 napi_schedule(&queue->napi);
665} 748}
666 749
667static void tx_add_credit(struct xenvif *vif) 750static void tx_add_credit(struct xenvif_queue *queue)
668{ 751{
669 unsigned long max_burst, max_credit; 752 unsigned long max_burst, max_credit;
670 753
@@ -672,55 +755,57 @@ static void tx_add_credit(struct xenvif *vif)
672 * Allow a burst big enough to transmit a jumbo packet of up to 128kB. 755 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
673 * Otherwise the interface can seize up due to insufficient credit. 756 * Otherwise the interface can seize up due to insufficient credit.
674 */ 757 */
675 max_burst = RING_GET_REQUEST(&vif->tx, vif->tx.req_cons)->size; 758 max_burst = RING_GET_REQUEST(&queue->tx, queue->tx.req_cons)->size;
676 max_burst = min(max_burst, 131072UL); 759 max_burst = min(max_burst, 131072UL);
677 max_burst = max(max_burst, vif->credit_bytes); 760 max_burst = max(max_burst, queue->credit_bytes);
678 761
679 /* Take care that adding a new chunk of credit doesn't wrap to zero. */ 762 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
680 max_credit = vif->remaining_credit + vif->credit_bytes; 763 max_credit = queue->remaining_credit + queue->credit_bytes;
681 if (max_credit < vif->remaining_credit) 764 if (max_credit < queue->remaining_credit)
682 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */ 765 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */
683 766
684 vif->remaining_credit = min(max_credit, max_burst); 767 queue->remaining_credit = min(max_credit, max_burst);
685} 768}
686 769
687static void tx_credit_callback(unsigned long data) 770static void tx_credit_callback(unsigned long data)
688{ 771{
689 struct xenvif *vif = (struct xenvif *)data; 772 struct xenvif_queue *queue = (struct xenvif_queue *)data;
690 tx_add_credit(vif); 773 tx_add_credit(queue);
691 xenvif_check_rx_xenvif(vif); 774 xenvif_napi_schedule_or_enable_events(queue);
692} 775}
693 776
694static void xenvif_tx_err(struct xenvif *vif, 777static void xenvif_tx_err(struct xenvif_queue *queue,
695 struct xen_netif_tx_request *txp, RING_IDX end) 778 struct xen_netif_tx_request *txp, RING_IDX end)
696{ 779{
697 RING_IDX cons = vif->tx.req_cons; 780 RING_IDX cons = queue->tx.req_cons;
698 unsigned long flags; 781 unsigned long flags;
699 782
700 do { 783 do {
701 spin_lock_irqsave(&vif->response_lock, flags); 784 spin_lock_irqsave(&queue->response_lock, flags);
702 make_tx_response(vif, txp, XEN_NETIF_RSP_ERROR); 785 make_tx_response(queue, txp, XEN_NETIF_RSP_ERROR);
703 spin_unlock_irqrestore(&vif->response_lock, flags); 786 spin_unlock_irqrestore(&queue->response_lock, flags);
704 if (cons == end) 787 if (cons == end)
705 break; 788 break;
706 txp = RING_GET_REQUEST(&vif->tx, cons++); 789 txp = RING_GET_REQUEST(&queue->tx, cons++);
707 } while (1); 790 } while (1);
708 vif->tx.req_cons = cons; 791 queue->tx.req_cons = cons;
709} 792}
710 793
711static void xenvif_fatal_tx_err(struct xenvif *vif) 794static void xenvif_fatal_tx_err(struct xenvif *vif)
712{ 795{
713 netdev_err(vif->dev, "fatal error; disabling device\n"); 796 netdev_err(vif->dev, "fatal error; disabling device\n");
714 vif->disabled = true; 797 vif->disabled = true;
715 xenvif_kick_thread(vif); 798 /* Disable the vif from queue 0's kthread */
799 if (vif->queues)
800 xenvif_kick_thread(&vif->queues[0]);
716} 801}
717 802
718static int xenvif_count_requests(struct xenvif *vif, 803static int xenvif_count_requests(struct xenvif_queue *queue,
719 struct xen_netif_tx_request *first, 804 struct xen_netif_tx_request *first,
720 struct xen_netif_tx_request *txp, 805 struct xen_netif_tx_request *txp,
721 int work_to_do) 806 int work_to_do)
722{ 807{
723 RING_IDX cons = vif->tx.req_cons; 808 RING_IDX cons = queue->tx.req_cons;
724 int slots = 0; 809 int slots = 0;
725 int drop_err = 0; 810 int drop_err = 0;
726 int more_data; 811 int more_data;
@@ -732,10 +817,10 @@ static int xenvif_count_requests(struct xenvif *vif,
732 struct xen_netif_tx_request dropped_tx = { 0 }; 817 struct xen_netif_tx_request dropped_tx = { 0 };
733 818
734 if (slots >= work_to_do) { 819 if (slots >= work_to_do) {
735 netdev_err(vif->dev, 820 netdev_err(queue->vif->dev,
736 "Asked for %d slots but exceeds this limit\n", 821 "Asked for %d slots but exceeds this limit\n",
737 work_to_do); 822 work_to_do);
738 xenvif_fatal_tx_err(vif); 823 xenvif_fatal_tx_err(queue->vif);
739 return -ENODATA; 824 return -ENODATA;
740 } 825 }
741 826
@@ -743,10 +828,10 @@ static int xenvif_count_requests(struct xenvif *vif,
743 * considered malicious. 828 * considered malicious.
744 */ 829 */
745 if (unlikely(slots >= fatal_skb_slots)) { 830 if (unlikely(slots >= fatal_skb_slots)) {
746 netdev_err(vif->dev, 831 netdev_err(queue->vif->dev,
747 "Malicious frontend using %d slots, threshold %u\n", 832 "Malicious frontend using %d slots, threshold %u\n",
748 slots, fatal_skb_slots); 833 slots, fatal_skb_slots);
749 xenvif_fatal_tx_err(vif); 834 xenvif_fatal_tx_err(queue->vif);
750 return -E2BIG; 835 return -E2BIG;
751 } 836 }
752 837
@@ -759,7 +844,7 @@ static int xenvif_count_requests(struct xenvif *vif,
759 */ 844 */
760 if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) { 845 if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) {
761 if (net_ratelimit()) 846 if (net_ratelimit())
762 netdev_dbg(vif->dev, 847 netdev_dbg(queue->vif->dev,
763 "Too many slots (%d) exceeding limit (%d), dropping packet\n", 848 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
764 slots, XEN_NETBK_LEGACY_SLOTS_MAX); 849 slots, XEN_NETBK_LEGACY_SLOTS_MAX);
765 drop_err = -E2BIG; 850 drop_err = -E2BIG;
@@ -768,7 +853,7 @@ static int xenvif_count_requests(struct xenvif *vif,
768 if (drop_err) 853 if (drop_err)
769 txp = &dropped_tx; 854 txp = &dropped_tx;
770 855
771 memcpy(txp, RING_GET_REQUEST(&vif->tx, cons + slots), 856 memcpy(txp, RING_GET_REQUEST(&queue->tx, cons + slots),
772 sizeof(*txp)); 857 sizeof(*txp));
773 858
774 /* If the guest submitted a frame >= 64 KiB then 859 /* If the guest submitted a frame >= 64 KiB then
@@ -782,7 +867,7 @@ static int xenvif_count_requests(struct xenvif *vif,
782 */ 867 */
783 if (!drop_err && txp->size > first->size) { 868 if (!drop_err && txp->size > first->size) {
784 if (net_ratelimit()) 869 if (net_ratelimit())
785 netdev_dbg(vif->dev, 870 netdev_dbg(queue->vif->dev,
786 "Invalid tx request, slot size %u > remaining size %u\n", 871 "Invalid tx request, slot size %u > remaining size %u\n",
787 txp->size, first->size); 872 txp->size, first->size);
788 drop_err = -EIO; 873 drop_err = -EIO;
@@ -792,9 +877,9 @@ static int xenvif_count_requests(struct xenvif *vif,
792 slots++; 877 slots++;
793 878
794 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) { 879 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
795 netdev_err(vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n", 880 netdev_err(queue->vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n",
796 txp->offset, txp->size); 881 txp->offset, txp->size);
797 xenvif_fatal_tx_err(vif); 882 xenvif_fatal_tx_err(queue->vif);
798 return -EINVAL; 883 return -EINVAL;
799 } 884 }
800 885
@@ -806,7 +891,7 @@ static int xenvif_count_requests(struct xenvif *vif,
806 } while (more_data); 891 } while (more_data);
807 892
808 if (drop_err) { 893 if (drop_err) {
809 xenvif_tx_err(vif, first, cons + slots); 894 xenvif_tx_err(queue, first, cons + slots);
810 return drop_err; 895 return drop_err;
811 } 896 }
812 897
@@ -820,17 +905,17 @@ struct xenvif_tx_cb {
820 905
821#define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb) 906#define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)
822 907
823static inline void xenvif_tx_create_map_op(struct xenvif *vif, 908static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue,
824 u16 pending_idx, 909 u16 pending_idx,
825 struct xen_netif_tx_request *txp, 910 struct xen_netif_tx_request *txp,
826 struct gnttab_map_grant_ref *mop) 911 struct gnttab_map_grant_ref *mop)
827{ 912{
828 vif->pages_to_map[mop-vif->tx_map_ops] = vif->mmap_pages[pending_idx]; 913 queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx];
829 gnttab_set_map_op(mop, idx_to_kaddr(vif, pending_idx), 914 gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx),
830 GNTMAP_host_map | GNTMAP_readonly, 915 GNTMAP_host_map | GNTMAP_readonly,
831 txp->gref, vif->domid); 916 txp->gref, queue->vif->domid);
832 917
833 memcpy(&vif->pending_tx_info[pending_idx].req, txp, 918 memcpy(&queue->pending_tx_info[pending_idx].req, txp,
834 sizeof(*txp)); 919 sizeof(*txp));
835} 920}
836 921
@@ -851,7 +936,7 @@ static inline struct sk_buff *xenvif_alloc_skb(unsigned int size)
851 return skb; 936 return skb;
852} 937}
853 938
854static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif, 939static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif_queue *queue,
855 struct sk_buff *skb, 940 struct sk_buff *skb,
856 struct xen_netif_tx_request *txp, 941 struct xen_netif_tx_request *txp,
857 struct gnttab_map_grant_ref *gop) 942 struct gnttab_map_grant_ref *gop)
@@ -878,9 +963,9 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
878 963
879 for (shinfo->nr_frags = start; shinfo->nr_frags < nr_slots; 964 for (shinfo->nr_frags = start; shinfo->nr_frags < nr_slots;
880 shinfo->nr_frags++, txp++, gop++) { 965 shinfo->nr_frags++, txp++, gop++) {
881 index = pending_index(vif->pending_cons++); 966 index = pending_index(queue->pending_cons++);
882 pending_idx = vif->pending_ring[index]; 967 pending_idx = queue->pending_ring[index];
883 xenvif_tx_create_map_op(vif, pending_idx, txp, gop); 968 xenvif_tx_create_map_op(queue, pending_idx, txp, gop);
884 frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); 969 frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx);
885 } 970 }
886 971
@@ -888,7 +973,7 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
888 struct sk_buff *nskb = xenvif_alloc_skb(0); 973 struct sk_buff *nskb = xenvif_alloc_skb(0);
889 if (unlikely(nskb == NULL)) { 974 if (unlikely(nskb == NULL)) {
890 if (net_ratelimit()) 975 if (net_ratelimit())
891 netdev_err(vif->dev, 976 netdev_err(queue->vif->dev,
892 "Can't allocate the frag_list skb.\n"); 977 "Can't allocate the frag_list skb.\n");
893 return NULL; 978 return NULL;
894 } 979 }
@@ -898,9 +983,9 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
898 983
899 for (shinfo->nr_frags = 0; shinfo->nr_frags < frag_overflow; 984 for (shinfo->nr_frags = 0; shinfo->nr_frags < frag_overflow;
900 shinfo->nr_frags++, txp++, gop++) { 985 shinfo->nr_frags++, txp++, gop++) {
901 index = pending_index(vif->pending_cons++); 986 index = pending_index(queue->pending_cons++);
902 pending_idx = vif->pending_ring[index]; 987 pending_idx = queue->pending_ring[index];
903 xenvif_tx_create_map_op(vif, pending_idx, txp, gop); 988 xenvif_tx_create_map_op(queue, pending_idx, txp, gop);
904 frag_set_pending_idx(&frags[shinfo->nr_frags], 989 frag_set_pending_idx(&frags[shinfo->nr_frags],
905 pending_idx); 990 pending_idx);
906 } 991 }
@@ -911,34 +996,34 @@ static struct gnttab_map_grant_ref *xenvif_get_requests(struct xenvif *vif,
911 return gop; 996 return gop;
912} 997}
913 998
914static inline void xenvif_grant_handle_set(struct xenvif *vif, 999static inline void xenvif_grant_handle_set(struct xenvif_queue *queue,
915 u16 pending_idx, 1000 u16 pending_idx,
916 grant_handle_t handle) 1001 grant_handle_t handle)
917{ 1002{
918 if (unlikely(vif->grant_tx_handle[pending_idx] != 1003 if (unlikely(queue->grant_tx_handle[pending_idx] !=
919 NETBACK_INVALID_HANDLE)) { 1004 NETBACK_INVALID_HANDLE)) {
920 netdev_err(vif->dev, 1005 netdev_err(queue->vif->dev,
921 "Trying to overwrite active handle! pending_idx: %x\n", 1006 "Trying to overwrite active handle! pending_idx: %x\n",
922 pending_idx); 1007 pending_idx);
923 BUG(); 1008 BUG();
924 } 1009 }
925 vif->grant_tx_handle[pending_idx] = handle; 1010 queue->grant_tx_handle[pending_idx] = handle;
926} 1011}
927 1012
928static inline void xenvif_grant_handle_reset(struct xenvif *vif, 1013static inline void xenvif_grant_handle_reset(struct xenvif_queue *queue,
929 u16 pending_idx) 1014 u16 pending_idx)
930{ 1015{
931 if (unlikely(vif->grant_tx_handle[pending_idx] == 1016 if (unlikely(queue->grant_tx_handle[pending_idx] ==
932 NETBACK_INVALID_HANDLE)) { 1017 NETBACK_INVALID_HANDLE)) {
933 netdev_err(vif->dev, 1018 netdev_err(queue->vif->dev,
934 "Trying to unmap invalid handle! pending_idx: %x\n", 1019 "Trying to unmap invalid handle! pending_idx: %x\n",
935 pending_idx); 1020 pending_idx);
936 BUG(); 1021 BUG();
937 } 1022 }
938 vif->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE; 1023 queue->grant_tx_handle[pending_idx] = NETBACK_INVALID_HANDLE;
939} 1024}
940 1025
941static int xenvif_tx_check_gop(struct xenvif *vif, 1026static int xenvif_tx_check_gop(struct xenvif_queue *queue,
942 struct sk_buff *skb, 1027 struct sk_buff *skb,
943 struct gnttab_map_grant_ref **gopp_map, 1028 struct gnttab_map_grant_ref **gopp_map,
944 struct gnttab_copy **gopp_copy) 1029 struct gnttab_copy **gopp_copy)
@@ -955,12 +1040,12 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
955 (*gopp_copy)++; 1040 (*gopp_copy)++;
956 if (unlikely(err)) { 1041 if (unlikely(err)) {
957 if (net_ratelimit()) 1042 if (net_ratelimit())
958 netdev_dbg(vif->dev, 1043 netdev_dbg(queue->vif->dev,
959 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n", 1044 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n",
960 (*gopp_copy)->status, 1045 (*gopp_copy)->status,
961 pending_idx, 1046 pending_idx,
962 (*gopp_copy)->source.u.ref); 1047 (*gopp_copy)->source.u.ref);
963 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR); 1048 xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR);
964 } 1049 }
965 1050
966check_frags: 1051check_frags:
@@ -973,24 +1058,24 @@ check_frags:
973 newerr = gop_map->status; 1058 newerr = gop_map->status;
974 1059
975 if (likely(!newerr)) { 1060 if (likely(!newerr)) {
976 xenvif_grant_handle_set(vif, 1061 xenvif_grant_handle_set(queue,
977 pending_idx, 1062 pending_idx,
978 gop_map->handle); 1063 gop_map->handle);
979 /* Had a previous error? Invalidate this fragment. */ 1064 /* Had a previous error? Invalidate this fragment. */
980 if (unlikely(err)) 1065 if (unlikely(err))
981 xenvif_idx_unmap(vif, pending_idx); 1066 xenvif_idx_unmap(queue, pending_idx);
982 continue; 1067 continue;
983 } 1068 }
984 1069
985 /* Error on this fragment: respond to client with an error. */ 1070 /* Error on this fragment: respond to client with an error. */
986 if (net_ratelimit()) 1071 if (net_ratelimit())
987 netdev_dbg(vif->dev, 1072 netdev_dbg(queue->vif->dev,
988 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n", 1073 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n",
989 i, 1074 i,
990 gop_map->status, 1075 gop_map->status,
991 pending_idx, 1076 pending_idx,
992 gop_map->ref); 1077 gop_map->ref);
993 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR); 1078 xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_ERROR);
994 1079
995 /* Not the first error? Preceding frags already invalidated. */ 1080 /* Not the first error? Preceding frags already invalidated. */
996 if (err) 1081 if (err)
@@ -998,7 +1083,7 @@ check_frags:
998 /* First error: invalidate preceding fragments. */ 1083 /* First error: invalidate preceding fragments. */
999 for (j = 0; j < i; j++) { 1084 for (j = 0; j < i; j++) {
1000 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 1085 pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
1001 xenvif_idx_unmap(vif, pending_idx); 1086 xenvif_idx_unmap(queue, pending_idx);
1002 } 1087 }
1003 1088
1004 /* Remember the error: invalidate all subsequent fragments. */ 1089 /* Remember the error: invalidate all subsequent fragments. */
@@ -1022,7 +1107,7 @@ check_frags:
1022 shinfo = skb_shinfo(first_skb); 1107 shinfo = skb_shinfo(first_skb);
1023 for (j = 0; j < shinfo->nr_frags; j++) { 1108 for (j = 0; j < shinfo->nr_frags; j++) {
1024 pending_idx = frag_get_pending_idx(&shinfo->frags[j]); 1109 pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
1025 xenvif_idx_unmap(vif, pending_idx); 1110 xenvif_idx_unmap(queue, pending_idx);
1026 } 1111 }
1027 } 1112 }
1028 1113
@@ -1030,7 +1115,7 @@ check_frags:
1030 return err; 1115 return err;
1031} 1116}
1032 1117
1033static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb) 1118static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb)
1034{ 1119{
1035 struct skb_shared_info *shinfo = skb_shinfo(skb); 1120 struct skb_shared_info *shinfo = skb_shinfo(skb);
1036 int nr_frags = shinfo->nr_frags; 1121 int nr_frags = shinfo->nr_frags;
@@ -1048,23 +1133,23 @@ static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
1048 /* If this is not the first frag, chain it to the previous*/ 1133 /* If this is not the first frag, chain it to the previous*/
1049 if (prev_pending_idx == INVALID_PENDING_IDX) 1134 if (prev_pending_idx == INVALID_PENDING_IDX)
1050 skb_shinfo(skb)->destructor_arg = 1135 skb_shinfo(skb)->destructor_arg =
1051 &callback_param(vif, pending_idx); 1136 &callback_param(queue, pending_idx);
1052 else 1137 else
1053 callback_param(vif, prev_pending_idx).ctx = 1138 callback_param(queue, prev_pending_idx).ctx =
1054 &callback_param(vif, pending_idx); 1139 &callback_param(queue, pending_idx);
1055 1140
1056 callback_param(vif, pending_idx).ctx = NULL; 1141 callback_param(queue, pending_idx).ctx = NULL;
1057 prev_pending_idx = pending_idx; 1142 prev_pending_idx = pending_idx;
1058 1143
1059 txp = &vif->pending_tx_info[pending_idx].req; 1144 txp = &queue->pending_tx_info[pending_idx].req;
1060 page = virt_to_page(idx_to_kaddr(vif, pending_idx)); 1145 page = virt_to_page(idx_to_kaddr(queue, pending_idx));
1061 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); 1146 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size);
1062 skb->len += txp->size; 1147 skb->len += txp->size;
1063 skb->data_len += txp->size; 1148 skb->data_len += txp->size;
1064 skb->truesize += txp->size; 1149 skb->truesize += txp->size;
1065 1150
1066 /* Take an extra reference to offset network stack's put_page */ 1151 /* Take an extra reference to offset network stack's put_page */
1067 get_page(vif->mmap_pages[pending_idx]); 1152 get_page(queue->mmap_pages[pending_idx]);
1068 } 1153 }
1069 /* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc 1154 /* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc
1070 * overlaps with "index", and "mapping" is not set. I think mapping 1155 * overlaps with "index", and "mapping" is not set. I think mapping
@@ -1074,33 +1159,33 @@ static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
1074 skb->pfmemalloc = false; 1159 skb->pfmemalloc = false;
1075} 1160}
1076 1161
1077static int xenvif_get_extras(struct xenvif *vif, 1162static int xenvif_get_extras(struct xenvif_queue *queue,
1078 struct xen_netif_extra_info *extras, 1163 struct xen_netif_extra_info *extras,
1079 int work_to_do) 1164 int work_to_do)
1080{ 1165{
1081 struct xen_netif_extra_info extra; 1166 struct xen_netif_extra_info extra;
1082 RING_IDX cons = vif->tx.req_cons; 1167 RING_IDX cons = queue->tx.req_cons;
1083 1168
1084 do { 1169 do {
1085 if (unlikely(work_to_do-- <= 0)) { 1170 if (unlikely(work_to_do-- <= 0)) {
1086 netdev_err(vif->dev, "Missing extra info\n"); 1171 netdev_err(queue->vif->dev, "Missing extra info\n");
1087 xenvif_fatal_tx_err(vif); 1172 xenvif_fatal_tx_err(queue->vif);
1088 return -EBADR; 1173 return -EBADR;
1089 } 1174 }
1090 1175
1091 memcpy(&extra, RING_GET_REQUEST(&vif->tx, cons), 1176 memcpy(&extra, RING_GET_REQUEST(&queue->tx, cons),
1092 sizeof(extra)); 1177 sizeof(extra));
1093 if (unlikely(!extra.type || 1178 if (unlikely(!extra.type ||
1094 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 1179 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1095 vif->tx.req_cons = ++cons; 1180 queue->tx.req_cons = ++cons;
1096 netdev_err(vif->dev, 1181 netdev_err(queue->vif->dev,
1097 "Invalid extra type: %d\n", extra.type); 1182 "Invalid extra type: %d\n", extra.type);
1098 xenvif_fatal_tx_err(vif); 1183 xenvif_fatal_tx_err(queue->vif);
1099 return -EINVAL; 1184 return -EINVAL;
1100 } 1185 }
1101 1186
1102 memcpy(&extras[extra.type - 1], &extra, sizeof(extra)); 1187 memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
1103 vif->tx.req_cons = ++cons; 1188 queue->tx.req_cons = ++cons;
1104 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); 1189 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
1105 1190
1106 return work_to_do; 1191 return work_to_do;
@@ -1135,7 +1220,7 @@ static int xenvif_set_skb_gso(struct xenvif *vif,
1135 return 0; 1220 return 0;
1136} 1221}
1137 1222
1138static int checksum_setup(struct xenvif *vif, struct sk_buff *skb) 1223static int checksum_setup(struct xenvif_queue *queue, struct sk_buff *skb)
1139{ 1224{
1140 bool recalculate_partial_csum = false; 1225 bool recalculate_partial_csum = false;
1141 1226
@@ -1145,7 +1230,7 @@ static int checksum_setup(struct xenvif *vif, struct sk_buff *skb)
1145 * recalculate the partial checksum. 1230 * recalculate the partial checksum.
1146 */ 1231 */
1147 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { 1232 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
1148 vif->rx_gso_checksum_fixup++; 1233 queue->stats.rx_gso_checksum_fixup++;
1149 skb->ip_summed = CHECKSUM_PARTIAL; 1234 skb->ip_summed = CHECKSUM_PARTIAL;
1150 recalculate_partial_csum = true; 1235 recalculate_partial_csum = true;
1151 } 1236 }
@@ -1157,31 +1242,31 @@ static int checksum_setup(struct xenvif *vif, struct sk_buff *skb)
1157 return skb_checksum_setup(skb, recalculate_partial_csum); 1242 return skb_checksum_setup(skb, recalculate_partial_csum);
1158} 1243}
1159 1244
1160static bool tx_credit_exceeded(struct xenvif *vif, unsigned size) 1245static bool tx_credit_exceeded(struct xenvif_queue *queue, unsigned size)
1161{ 1246{
1162 u64 now = get_jiffies_64(); 1247 u64 now = get_jiffies_64();
1163 u64 next_credit = vif->credit_window_start + 1248 u64 next_credit = queue->credit_window_start +
1164 msecs_to_jiffies(vif->credit_usec / 1000); 1249 msecs_to_jiffies(queue->credit_usec / 1000);
1165 1250
1166 /* Timer could already be pending in rare cases. */ 1251 /* Timer could already be pending in rare cases. */
1167 if (timer_pending(&vif->credit_timeout)) 1252 if (timer_pending(&queue->credit_timeout))
1168 return true; 1253 return true;
1169 1254
1170 /* Passed the point where we can replenish credit? */ 1255 /* Passed the point where we can replenish credit? */
1171 if (time_after_eq64(now, next_credit)) { 1256 if (time_after_eq64(now, next_credit)) {
1172 vif->credit_window_start = now; 1257 queue->credit_window_start = now;
1173 tx_add_credit(vif); 1258 tx_add_credit(queue);
1174 } 1259 }
1175 1260
1176 /* Still too big to send right now? Set a callback. */ 1261 /* Still too big to send right now? Set a callback. */
1177 if (size > vif->remaining_credit) { 1262 if (size > queue->remaining_credit) {
1178 vif->credit_timeout.data = 1263 queue->credit_timeout.data =
1179 (unsigned long)vif; 1264 (unsigned long)queue;
1180 vif->credit_timeout.function = 1265 queue->credit_timeout.function =
1181 tx_credit_callback; 1266 tx_credit_callback;
1182 mod_timer(&vif->credit_timeout, 1267 mod_timer(&queue->credit_timeout,
1183 next_credit); 1268 next_credit);
1184 vif->credit_window_start = next_credit; 1269 queue->credit_window_start = next_credit;
1185 1270
1186 return true; 1271 return true;
1187 } 1272 }
@@ -1189,16 +1274,16 @@ static bool tx_credit_exceeded(struct xenvif *vif, unsigned size)
1189 return false; 1274 return false;
1190} 1275}
1191 1276
1192static void xenvif_tx_build_gops(struct xenvif *vif, 1277static void xenvif_tx_build_gops(struct xenvif_queue *queue,
1193 int budget, 1278 int budget,
1194 unsigned *copy_ops, 1279 unsigned *copy_ops,
1195 unsigned *map_ops) 1280 unsigned *map_ops)
1196{ 1281{
1197 struct gnttab_map_grant_ref *gop = vif->tx_map_ops, *request_gop; 1282 struct gnttab_map_grant_ref *gop = queue->tx_map_ops, *request_gop;
1198 struct sk_buff *skb; 1283 struct sk_buff *skb;
1199 int ret; 1284 int ret;
1200 1285
1201 while (skb_queue_len(&vif->tx_queue) < budget) { 1286 while (skb_queue_len(&queue->tx_queue) < budget) {
1202 struct xen_netif_tx_request txreq; 1287 struct xen_netif_tx_request txreq;
1203 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX]; 1288 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX];
1204 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1]; 1289 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1];
@@ -1208,69 +1293,69 @@ static void xenvif_tx_build_gops(struct xenvif *vif,
1208 unsigned int data_len; 1293 unsigned int data_len;
1209 pending_ring_idx_t index; 1294 pending_ring_idx_t index;
1210 1295
1211 if (vif->tx.sring->req_prod - vif->tx.req_cons > 1296 if (queue->tx.sring->req_prod - queue->tx.req_cons >
1212 XEN_NETIF_TX_RING_SIZE) { 1297 XEN_NETIF_TX_RING_SIZE) {
1213 netdev_err(vif->dev, 1298 netdev_err(queue->vif->dev,
1214 "Impossible number of requests. " 1299 "Impossible number of requests. "
1215 "req_prod %d, req_cons %d, size %ld\n", 1300 "req_prod %d, req_cons %d, size %ld\n",
1216 vif->tx.sring->req_prod, vif->tx.req_cons, 1301 queue->tx.sring->req_prod, queue->tx.req_cons,
1217 XEN_NETIF_TX_RING_SIZE); 1302 XEN_NETIF_TX_RING_SIZE);
1218 xenvif_fatal_tx_err(vif); 1303 xenvif_fatal_tx_err(queue->vif);
1219 break; 1304 break;
1220 } 1305 }
1221 1306
1222 work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&vif->tx); 1307 work_to_do = RING_HAS_UNCONSUMED_REQUESTS(&queue->tx);
1223 if (!work_to_do) 1308 if (!work_to_do)
1224 break; 1309 break;
1225 1310
1226 idx = vif->tx.req_cons; 1311 idx = queue->tx.req_cons;
1227 rmb(); /* Ensure that we see the request before we copy it. */ 1312 rmb(); /* Ensure that we see the request before we copy it. */
1228 memcpy(&txreq, RING_GET_REQUEST(&vif->tx, idx), sizeof(txreq)); 1313 memcpy(&txreq, RING_GET_REQUEST(&queue->tx, idx), sizeof(txreq));
1229 1314
1230 /* Credit-based scheduling. */ 1315 /* Credit-based scheduling. */
1231 if (txreq.size > vif->remaining_credit && 1316 if (txreq.size > queue->remaining_credit &&
1232 tx_credit_exceeded(vif, txreq.size)) 1317 tx_credit_exceeded(queue, txreq.size))
1233 break; 1318 break;
1234 1319
1235 vif->remaining_credit -= txreq.size; 1320 queue->remaining_credit -= txreq.size;
1236 1321
1237 work_to_do--; 1322 work_to_do--;
1238 vif->tx.req_cons = ++idx; 1323 queue->tx.req_cons = ++idx;
1239 1324
1240 memset(extras, 0, sizeof(extras)); 1325 memset(extras, 0, sizeof(extras));
1241 if (txreq.flags & XEN_NETTXF_extra_info) { 1326 if (txreq.flags & XEN_NETTXF_extra_info) {
1242 work_to_do = xenvif_get_extras(vif, extras, 1327 work_to_do = xenvif_get_extras(queue, extras,
1243 work_to_do); 1328 work_to_do);
1244 idx = vif->tx.req_cons; 1329 idx = queue->tx.req_cons;
1245 if (unlikely(work_to_do < 0)) 1330 if (unlikely(work_to_do < 0))
1246 break; 1331 break;
1247 } 1332 }
1248 1333
1249 ret = xenvif_count_requests(vif, &txreq, txfrags, work_to_do); 1334 ret = xenvif_count_requests(queue, &txreq, txfrags, work_to_do);
1250 if (unlikely(ret < 0)) 1335 if (unlikely(ret < 0))
1251 break; 1336 break;
1252 1337
1253 idx += ret; 1338 idx += ret;
1254 1339
1255 if (unlikely(txreq.size < ETH_HLEN)) { 1340 if (unlikely(txreq.size < ETH_HLEN)) {
1256 netdev_dbg(vif->dev, 1341 netdev_dbg(queue->vif->dev,
1257 "Bad packet size: %d\n", txreq.size); 1342 "Bad packet size: %d\n", txreq.size);
1258 xenvif_tx_err(vif, &txreq, idx); 1343 xenvif_tx_err(queue, &txreq, idx);
1259 break; 1344 break;
1260 } 1345 }
1261 1346
1262 /* No crossing a page as the payload mustn't fragment. */ 1347 /* No crossing a page as the payload mustn't fragment. */
1263 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) { 1348 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) {
1264 netdev_err(vif->dev, 1349 netdev_err(queue->vif->dev,
1265 "txreq.offset: %x, size: %u, end: %lu\n", 1350 "txreq.offset: %x, size: %u, end: %lu\n",
1266 txreq.offset, txreq.size, 1351 txreq.offset, txreq.size,
1267 (txreq.offset&~PAGE_MASK) + txreq.size); 1352 (txreq.offset&~PAGE_MASK) + txreq.size);
1268 xenvif_fatal_tx_err(vif); 1353 xenvif_fatal_tx_err(queue->vif);
1269 break; 1354 break;
1270 } 1355 }
1271 1356
1272 index = pending_index(vif->pending_cons); 1357 index = pending_index(queue->pending_cons);
1273 pending_idx = vif->pending_ring[index]; 1358 pending_idx = queue->pending_ring[index];
1274 1359
1275 data_len = (txreq.size > PKT_PROT_LEN && 1360 data_len = (txreq.size > PKT_PROT_LEN &&
1276 ret < XEN_NETBK_LEGACY_SLOTS_MAX) ? 1361 ret < XEN_NETBK_LEGACY_SLOTS_MAX) ?
@@ -1278,9 +1363,9 @@ static void xenvif_tx_build_gops(struct xenvif *vif,
1278 1363
1279 skb = xenvif_alloc_skb(data_len); 1364 skb = xenvif_alloc_skb(data_len);
1280 if (unlikely(skb == NULL)) { 1365 if (unlikely(skb == NULL)) {
1281 netdev_dbg(vif->dev, 1366 netdev_dbg(queue->vif->dev,
1282 "Can't allocate a skb in start_xmit.\n"); 1367 "Can't allocate a skb in start_xmit.\n");
1283 xenvif_tx_err(vif, &txreq, idx); 1368 xenvif_tx_err(queue, &txreq, idx);
1284 break; 1369 break;
1285 } 1370 }
1286 1371
@@ -1288,7 +1373,7 @@ static void xenvif_tx_build_gops(struct xenvif *vif,
1288 struct xen_netif_extra_info *gso; 1373 struct xen_netif_extra_info *gso;
1289 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; 1374 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1290 1375
1291 if (xenvif_set_skb_gso(vif, skb, gso)) { 1376 if (xenvif_set_skb_gso(queue->vif, skb, gso)) {
1292 /* Failure in xenvif_set_skb_gso is fatal. */ 1377 /* Failure in xenvif_set_skb_gso is fatal. */
1293 kfree_skb(skb); 1378 kfree_skb(skb);
1294 break; 1379 break;
@@ -1298,18 +1383,18 @@ static void xenvif_tx_build_gops(struct xenvif *vif,
1298 XENVIF_TX_CB(skb)->pending_idx = pending_idx; 1383 XENVIF_TX_CB(skb)->pending_idx = pending_idx;
1299 1384
1300 __skb_put(skb, data_len); 1385 __skb_put(skb, data_len);
1301 vif->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref; 1386 queue->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref;
1302 vif->tx_copy_ops[*copy_ops].source.domid = vif->domid; 1387 queue->tx_copy_ops[*copy_ops].source.domid = queue->vif->domid;
1303 vif->tx_copy_ops[*copy_ops].source.offset = txreq.offset; 1388 queue->tx_copy_ops[*copy_ops].source.offset = txreq.offset;
1304 1389
1305 vif->tx_copy_ops[*copy_ops].dest.u.gmfn = 1390 queue->tx_copy_ops[*copy_ops].dest.u.gmfn =
1306 virt_to_mfn(skb->data); 1391 virt_to_mfn(skb->data);
1307 vif->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF; 1392 queue->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF;
1308 vif->tx_copy_ops[*copy_ops].dest.offset = 1393 queue->tx_copy_ops[*copy_ops].dest.offset =
1309 offset_in_page(skb->data); 1394 offset_in_page(skb->data);
1310 1395
1311 vif->tx_copy_ops[*copy_ops].len = data_len; 1396 queue->tx_copy_ops[*copy_ops].len = data_len;
1312 vif->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref; 1397 queue->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref;
1313 1398
1314 (*copy_ops)++; 1399 (*copy_ops)++;
1315 1400
@@ -1318,42 +1403,42 @@ static void xenvif_tx_build_gops(struct xenvif *vif,
1318 skb_shinfo(skb)->nr_frags++; 1403 skb_shinfo(skb)->nr_frags++;
1319 frag_set_pending_idx(&skb_shinfo(skb)->frags[0], 1404 frag_set_pending_idx(&skb_shinfo(skb)->frags[0],
1320 pending_idx); 1405 pending_idx);
1321 xenvif_tx_create_map_op(vif, pending_idx, &txreq, gop); 1406 xenvif_tx_create_map_op(queue, pending_idx, &txreq, gop);
1322 gop++; 1407 gop++;
1323 } else { 1408 } else {
1324 frag_set_pending_idx(&skb_shinfo(skb)->frags[0], 1409 frag_set_pending_idx(&skb_shinfo(skb)->frags[0],
1325 INVALID_PENDING_IDX); 1410 INVALID_PENDING_IDX);
1326 memcpy(&vif->pending_tx_info[pending_idx].req, &txreq, 1411 memcpy(&queue->pending_tx_info[pending_idx].req, &txreq,
1327 sizeof(txreq)); 1412 sizeof(txreq));
1328 } 1413 }
1329 1414
1330 vif->pending_cons++; 1415 queue->pending_cons++;
1331 1416
1332 request_gop = xenvif_get_requests(vif, skb, txfrags, gop); 1417 request_gop = xenvif_get_requests(queue, skb, txfrags, gop);
1333 if (request_gop == NULL) { 1418 if (request_gop == NULL) {
1334 kfree_skb(skb); 1419 kfree_skb(skb);
1335 xenvif_tx_err(vif, &txreq, idx); 1420 xenvif_tx_err(queue, &txreq, idx);
1336 break; 1421 break;
1337 } 1422 }
1338 gop = request_gop; 1423 gop = request_gop;
1339 1424
1340 __skb_queue_tail(&vif->tx_queue, skb); 1425 __skb_queue_tail(&queue->tx_queue, skb);
1341 1426
1342 vif->tx.req_cons = idx; 1427 queue->tx.req_cons = idx;
1343 1428
1344 if (((gop-vif->tx_map_ops) >= ARRAY_SIZE(vif->tx_map_ops)) || 1429 if (((gop-queue->tx_map_ops) >= ARRAY_SIZE(queue->tx_map_ops)) ||
1345 (*copy_ops >= ARRAY_SIZE(vif->tx_copy_ops))) 1430 (*copy_ops >= ARRAY_SIZE(queue->tx_copy_ops)))
1346 break; 1431 break;
1347 } 1432 }
1348 1433
1349 (*map_ops) = gop - vif->tx_map_ops; 1434 (*map_ops) = gop - queue->tx_map_ops;
1350 return; 1435 return;
1351} 1436}
1352 1437
1353/* Consolidate skb with a frag_list into a brand new one with local pages on 1438/* Consolidate skb with a frag_list into a brand new one with local pages on
1354 * frags. Returns 0 or -ENOMEM if can't allocate new pages. 1439 * frags. Returns 0 or -ENOMEM if can't allocate new pages.
1355 */ 1440 */
1356static int xenvif_handle_frag_list(struct xenvif *vif, struct sk_buff *skb) 1441static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *skb)
1357{ 1442{
1358 unsigned int offset = skb_headlen(skb); 1443 unsigned int offset = skb_headlen(skb);
1359 skb_frag_t frags[MAX_SKB_FRAGS]; 1444 skb_frag_t frags[MAX_SKB_FRAGS];
@@ -1361,10 +1446,10 @@ static int xenvif_handle_frag_list(struct xenvif *vif, struct sk_buff *skb)
1361 struct ubuf_info *uarg; 1446 struct ubuf_info *uarg;
1362 struct sk_buff *nskb = skb_shinfo(skb)->frag_list; 1447 struct sk_buff *nskb = skb_shinfo(skb)->frag_list;
1363 1448
1364 vif->tx_zerocopy_sent += 2; 1449 queue->stats.tx_zerocopy_sent += 2;
1365 vif->tx_frag_overflow++; 1450 queue->stats.tx_frag_overflow++;
1366 1451
1367 xenvif_fill_frags(vif, nskb); 1452 xenvif_fill_frags(queue, nskb);
1368 /* Subtract frags size, we will correct it later */ 1453 /* Subtract frags size, we will correct it later */
1369 skb->truesize -= skb->data_len; 1454 skb->truesize -= skb->data_len;
1370 skb->len += nskb->len; 1455 skb->len += nskb->len;
@@ -1416,37 +1501,37 @@ static int xenvif_handle_frag_list(struct xenvif *vif, struct sk_buff *skb)
1416 return 0; 1501 return 0;
1417} 1502}
1418 1503
1419static int xenvif_tx_submit(struct xenvif *vif) 1504static int xenvif_tx_submit(struct xenvif_queue *queue)
1420{ 1505{
1421 struct gnttab_map_grant_ref *gop_map = vif->tx_map_ops; 1506 struct gnttab_map_grant_ref *gop_map = queue->tx_map_ops;
1422 struct gnttab_copy *gop_copy = vif->tx_copy_ops; 1507 struct gnttab_copy *gop_copy = queue->tx_copy_ops;
1423 struct sk_buff *skb; 1508 struct sk_buff *skb;
1424 int work_done = 0; 1509 int work_done = 0;
1425 1510
1426 while ((skb = __skb_dequeue(&vif->tx_queue)) != NULL) { 1511 while ((skb = __skb_dequeue(&queue->tx_queue)) != NULL) {
1427 struct xen_netif_tx_request *txp; 1512 struct xen_netif_tx_request *txp;
1428 u16 pending_idx; 1513 u16 pending_idx;
1429 unsigned data_len; 1514 unsigned data_len;
1430 1515
1431 pending_idx = XENVIF_TX_CB(skb)->pending_idx; 1516 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
1432 txp = &vif->pending_tx_info[pending_idx].req; 1517 txp = &queue->pending_tx_info[pending_idx].req;
1433 1518
1434 /* Check the remap error code. */ 1519 /* Check the remap error code. */
1435 if (unlikely(xenvif_tx_check_gop(vif, skb, &gop_map, &gop_copy))) { 1520 if (unlikely(xenvif_tx_check_gop(queue, skb, &gop_map, &gop_copy))) {
1436 skb_shinfo(skb)->nr_frags = 0; 1521 skb_shinfo(skb)->nr_frags = 0;
1437 kfree_skb(skb); 1522 kfree_skb(skb);
1438 continue; 1523 continue;
1439 } 1524 }
1440 1525
1441 data_len = skb->len; 1526 data_len = skb->len;
1442 callback_param(vif, pending_idx).ctx = NULL; 1527 callback_param(queue, pending_idx).ctx = NULL;
1443 if (data_len < txp->size) { 1528 if (data_len < txp->size) {
1444 /* Append the packet payload as a fragment. */ 1529 /* Append the packet payload as a fragment. */
1445 txp->offset += data_len; 1530 txp->offset += data_len;
1446 txp->size -= data_len; 1531 txp->size -= data_len;
1447 } else { 1532 } else {
1448 /* Schedule a response immediately. */ 1533 /* Schedule a response immediately. */
1449 xenvif_idx_release(vif, pending_idx, 1534 xenvif_idx_release(queue, pending_idx,
1450 XEN_NETIF_RSP_OKAY); 1535 XEN_NETIF_RSP_OKAY);
1451 } 1536 }
1452 1537
@@ -1455,12 +1540,12 @@ static int xenvif_tx_submit(struct xenvif *vif)
1455 else if (txp->flags & XEN_NETTXF_data_validated) 1540 else if (txp->flags & XEN_NETTXF_data_validated)
1456 skb->ip_summed = CHECKSUM_UNNECESSARY; 1541 skb->ip_summed = CHECKSUM_UNNECESSARY;
1457 1542
1458 xenvif_fill_frags(vif, skb); 1543 xenvif_fill_frags(queue, skb);
1459 1544
1460 if (unlikely(skb_has_frag_list(skb))) { 1545 if (unlikely(skb_has_frag_list(skb))) {
1461 if (xenvif_handle_frag_list(vif, skb)) { 1546 if (xenvif_handle_frag_list(queue, skb)) {
1462 if (net_ratelimit()) 1547 if (net_ratelimit())
1463 netdev_err(vif->dev, 1548 netdev_err(queue->vif->dev,
1464 "Not enough memory to consolidate frag_list!\n"); 1549 "Not enough memory to consolidate frag_list!\n");
1465 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; 1550 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
1466 kfree_skb(skb); 1551 kfree_skb(skb);
@@ -1473,12 +1558,12 @@ static int xenvif_tx_submit(struct xenvif *vif)
1473 __pskb_pull_tail(skb, target - skb_headlen(skb)); 1558 __pskb_pull_tail(skb, target - skb_headlen(skb));
1474 } 1559 }
1475 1560
1476 skb->dev = vif->dev; 1561 skb->dev = queue->vif->dev;
1477 skb->protocol = eth_type_trans(skb, skb->dev); 1562 skb->protocol = eth_type_trans(skb, skb->dev);
1478 skb_reset_network_header(skb); 1563 skb_reset_network_header(skb);
1479 1564
1480 if (checksum_setup(vif, skb)) { 1565 if (checksum_setup(queue, skb)) {
1481 netdev_dbg(vif->dev, 1566 netdev_dbg(queue->vif->dev,
1482 "Can't setup checksum in net_tx_action\n"); 1567 "Can't setup checksum in net_tx_action\n");
1483 /* We have to set this flag to trigger the callback */ 1568 /* We have to set this flag to trigger the callback */
1484 if (skb_shinfo(skb)->destructor_arg) 1569 if (skb_shinfo(skb)->destructor_arg)
@@ -1503,8 +1588,8 @@ static int xenvif_tx_submit(struct xenvif *vif)
1503 DIV_ROUND_UP(skb->len - hdrlen, mss); 1588 DIV_ROUND_UP(skb->len - hdrlen, mss);
1504 } 1589 }
1505 1590
1506 vif->dev->stats.rx_bytes += skb->len; 1591 queue->stats.rx_bytes += skb->len;
1507 vif->dev->stats.rx_packets++; 1592 queue->stats.rx_packets++;
1508 1593
1509 work_done++; 1594 work_done++;
1510 1595
@@ -1515,7 +1600,7 @@ static int xenvif_tx_submit(struct xenvif *vif)
1515 */ 1600 */
1516 if (skb_shinfo(skb)->destructor_arg) { 1601 if (skb_shinfo(skb)->destructor_arg) {
1517 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; 1602 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
1518 vif->tx_zerocopy_sent++; 1603 queue->stats.tx_zerocopy_sent++;
1519 } 1604 }
1520 1605
1521 netif_receive_skb(skb); 1606 netif_receive_skb(skb);
@@ -1528,47 +1613,47 @@ void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success)
1528{ 1613{
1529 unsigned long flags; 1614 unsigned long flags;
1530 pending_ring_idx_t index; 1615 pending_ring_idx_t index;
1531 struct xenvif *vif = ubuf_to_vif(ubuf); 1616 struct xenvif_queue *queue = ubuf_to_queue(ubuf);
1532 1617
1533 /* This is the only place where we grab this lock, to protect callbacks 1618 /* This is the only place where we grab this lock, to protect callbacks
1534 * from each other. 1619 * from each other.
1535 */ 1620 */
1536 spin_lock_irqsave(&vif->callback_lock, flags); 1621 spin_lock_irqsave(&queue->callback_lock, flags);
1537 do { 1622 do {
1538 u16 pending_idx = ubuf->desc; 1623 u16 pending_idx = ubuf->desc;
1539 ubuf = (struct ubuf_info *) ubuf->ctx; 1624 ubuf = (struct ubuf_info *) ubuf->ctx;
1540 BUG_ON(vif->dealloc_prod - vif->dealloc_cons >= 1625 BUG_ON(queue->dealloc_prod - queue->dealloc_cons >=
1541 MAX_PENDING_REQS); 1626 MAX_PENDING_REQS);
1542 index = pending_index(vif->dealloc_prod); 1627 index = pending_index(queue->dealloc_prod);
1543 vif->dealloc_ring[index] = pending_idx; 1628 queue->dealloc_ring[index] = pending_idx;
1544 /* Sync with xenvif_tx_dealloc_action: 1629 /* Sync with xenvif_tx_dealloc_action:
1545 * insert idx then incr producer. 1630 * insert idx then incr producer.
1546 */ 1631 */
1547 smp_wmb(); 1632 smp_wmb();
1548 vif->dealloc_prod++; 1633 queue->dealloc_prod++;
1549 } while (ubuf); 1634 } while (ubuf);
1550 wake_up(&vif->dealloc_wq); 1635 wake_up(&queue->dealloc_wq);
1551 spin_unlock_irqrestore(&vif->callback_lock, flags); 1636 spin_unlock_irqrestore(&queue->callback_lock, flags);
1552 1637
1553 if (likely(zerocopy_success)) 1638 if (likely(zerocopy_success))
1554 vif->tx_zerocopy_success++; 1639 queue->stats.tx_zerocopy_success++;
1555 else 1640 else
1556 vif->tx_zerocopy_fail++; 1641 queue->stats.tx_zerocopy_fail++;
1557} 1642}
1558 1643
1559static inline void xenvif_tx_dealloc_action(struct xenvif *vif) 1644static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)
1560{ 1645{
1561 struct gnttab_unmap_grant_ref *gop; 1646 struct gnttab_unmap_grant_ref *gop;
1562 pending_ring_idx_t dc, dp; 1647 pending_ring_idx_t dc, dp;
1563 u16 pending_idx, pending_idx_release[MAX_PENDING_REQS]; 1648 u16 pending_idx, pending_idx_release[MAX_PENDING_REQS];
1564 unsigned int i = 0; 1649 unsigned int i = 0;
1565 1650
1566 dc = vif->dealloc_cons; 1651 dc = queue->dealloc_cons;
1567 gop = vif->tx_unmap_ops; 1652 gop = queue->tx_unmap_ops;
1568 1653
1569 /* Free up any grants we have finished using */ 1654 /* Free up any grants we have finished using */
1570 do { 1655 do {
1571 dp = vif->dealloc_prod; 1656 dp = queue->dealloc_prod;
1572 1657
1573 /* Ensure we see all indices enqueued by all 1658 /* Ensure we see all indices enqueued by all
1574 * xenvif_zerocopy_callback(). 1659 * xenvif_zerocopy_callback().
@@ -1576,38 +1661,38 @@ static inline void xenvif_tx_dealloc_action(struct xenvif *vif)
1576 smp_rmb(); 1661 smp_rmb();
1577 1662
1578 while (dc != dp) { 1663 while (dc != dp) {
1579 BUG_ON(gop - vif->tx_unmap_ops > MAX_PENDING_REQS); 1664 BUG_ON(gop - queue->tx_unmap_ops > MAX_PENDING_REQS);
1580 pending_idx = 1665 pending_idx =
1581 vif->dealloc_ring[pending_index(dc++)]; 1666 queue->dealloc_ring[pending_index(dc++)];
1582 1667
1583 pending_idx_release[gop-vif->tx_unmap_ops] = 1668 pending_idx_release[gop-queue->tx_unmap_ops] =
1584 pending_idx; 1669 pending_idx;
1585 vif->pages_to_unmap[gop-vif->tx_unmap_ops] = 1670 queue->pages_to_unmap[gop-queue->tx_unmap_ops] =
1586 vif->mmap_pages[pending_idx]; 1671 queue->mmap_pages[pending_idx];
1587 gnttab_set_unmap_op(gop, 1672 gnttab_set_unmap_op(gop,
1588 idx_to_kaddr(vif, pending_idx), 1673 idx_to_kaddr(queue, pending_idx),
1589 GNTMAP_host_map, 1674 GNTMAP_host_map,
1590 vif->grant_tx_handle[pending_idx]); 1675 queue->grant_tx_handle[pending_idx]);
1591 xenvif_grant_handle_reset(vif, pending_idx); 1676 xenvif_grant_handle_reset(queue, pending_idx);
1592 ++gop; 1677 ++gop;
1593 } 1678 }
1594 1679
1595 } while (dp != vif->dealloc_prod); 1680 } while (dp != queue->dealloc_prod);
1596 1681
1597 vif->dealloc_cons = dc; 1682 queue->dealloc_cons = dc;
1598 1683
1599 if (gop - vif->tx_unmap_ops > 0) { 1684 if (gop - queue->tx_unmap_ops > 0) {
1600 int ret; 1685 int ret;
1601 ret = gnttab_unmap_refs(vif->tx_unmap_ops, 1686 ret = gnttab_unmap_refs(queue->tx_unmap_ops,
1602 NULL, 1687 NULL,
1603 vif->pages_to_unmap, 1688 queue->pages_to_unmap,
1604 gop - vif->tx_unmap_ops); 1689 gop - queue->tx_unmap_ops);
1605 if (ret) { 1690 if (ret) {
1606 netdev_err(vif->dev, "Unmap fail: nr_ops %tx ret %d\n", 1691 netdev_err(queue->vif->dev, "Unmap fail: nr_ops %tx ret %d\n",
1607 gop - vif->tx_unmap_ops, ret); 1692 gop - queue->tx_unmap_ops, ret);
1608 for (i = 0; i < gop - vif->tx_unmap_ops; ++i) { 1693 for (i = 0; i < gop - queue->tx_unmap_ops; ++i) {
1609 if (gop[i].status != GNTST_okay) 1694 if (gop[i].status != GNTST_okay)
1610 netdev_err(vif->dev, 1695 netdev_err(queue->vif->dev,
1611 " host_addr: %llx handle: %x status: %d\n", 1696 " host_addr: %llx handle: %x status: %d\n",
1612 gop[i].host_addr, 1697 gop[i].host_addr,
1613 gop[i].handle, 1698 gop[i].handle,
@@ -1617,91 +1702,91 @@ static inline void xenvif_tx_dealloc_action(struct xenvif *vif)
1617 } 1702 }
1618 } 1703 }
1619 1704
1620 for (i = 0; i < gop - vif->tx_unmap_ops; ++i) 1705 for (i = 0; i < gop - queue->tx_unmap_ops; ++i)
1621 xenvif_idx_release(vif, pending_idx_release[i], 1706 xenvif_idx_release(queue, pending_idx_release[i],
1622 XEN_NETIF_RSP_OKAY); 1707 XEN_NETIF_RSP_OKAY);
1623} 1708}
1624 1709
1625 1710
1626/* Called after netfront has transmitted */ 1711/* Called after netfront has transmitted */
1627int xenvif_tx_action(struct xenvif *vif, int budget) 1712int xenvif_tx_action(struct xenvif_queue *queue, int budget)
1628{ 1713{
1629 unsigned nr_mops, nr_cops = 0; 1714 unsigned nr_mops, nr_cops = 0;
1630 int work_done, ret; 1715 int work_done, ret;
1631 1716
1632 if (unlikely(!tx_work_todo(vif))) 1717 if (unlikely(!tx_work_todo(queue)))
1633 return 0; 1718 return 0;
1634 1719
1635 xenvif_tx_build_gops(vif, budget, &nr_cops, &nr_mops); 1720 xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops);
1636 1721
1637 if (nr_cops == 0) 1722 if (nr_cops == 0)
1638 return 0; 1723 return 0;
1639 1724
1640 gnttab_batch_copy(vif->tx_copy_ops, nr_cops); 1725 gnttab_batch_copy(queue->tx_copy_ops, nr_cops);
1641 if (nr_mops != 0) { 1726 if (nr_mops != 0) {
1642 ret = gnttab_map_refs(vif->tx_map_ops, 1727 ret = gnttab_map_refs(queue->tx_map_ops,
1643 NULL, 1728 NULL,
1644 vif->pages_to_map, 1729 queue->pages_to_map,
1645 nr_mops); 1730 nr_mops);
1646 BUG_ON(ret); 1731 BUG_ON(ret);
1647 } 1732 }
1648 1733
1649 work_done = xenvif_tx_submit(vif); 1734 work_done = xenvif_tx_submit(queue);
1650 1735
1651 return work_done; 1736 return work_done;
1652} 1737}
1653 1738
1654static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx, 1739static void xenvif_idx_release(struct xenvif_queue *queue, u16 pending_idx,
1655 u8 status) 1740 u8 status)
1656{ 1741{
1657 struct pending_tx_info *pending_tx_info; 1742 struct pending_tx_info *pending_tx_info;
1658 pending_ring_idx_t index; 1743 pending_ring_idx_t index;
1659 unsigned long flags; 1744 unsigned long flags;
1660 1745
1661 pending_tx_info = &vif->pending_tx_info[pending_idx]; 1746 pending_tx_info = &queue->pending_tx_info[pending_idx];
1662 spin_lock_irqsave(&vif->response_lock, flags); 1747 spin_lock_irqsave(&queue->response_lock, flags);
1663 make_tx_response(vif, &pending_tx_info->req, status); 1748 make_tx_response(queue, &pending_tx_info->req, status);
1664 index = pending_index(vif->pending_prod); 1749 index = pending_index(queue->pending_prod);
1665 vif->pending_ring[index] = pending_idx; 1750 queue->pending_ring[index] = pending_idx;
1666 /* TX shouldn't use the index before we give it back here */ 1751 /* TX shouldn't use the index before we give it back here */
1667 mb(); 1752 mb();
1668 vif->pending_prod++; 1753 queue->pending_prod++;
1669 spin_unlock_irqrestore(&vif->response_lock, flags); 1754 spin_unlock_irqrestore(&queue->response_lock, flags);
1670} 1755}
1671 1756
1672 1757
1673static void make_tx_response(struct xenvif *vif, 1758static void make_tx_response(struct xenvif_queue *queue,
1674 struct xen_netif_tx_request *txp, 1759 struct xen_netif_tx_request *txp,
1675 s8 st) 1760 s8 st)
1676{ 1761{
1677 RING_IDX i = vif->tx.rsp_prod_pvt; 1762 RING_IDX i = queue->tx.rsp_prod_pvt;
1678 struct xen_netif_tx_response *resp; 1763 struct xen_netif_tx_response *resp;
1679 int notify; 1764 int notify;
1680 1765
1681 resp = RING_GET_RESPONSE(&vif->tx, i); 1766 resp = RING_GET_RESPONSE(&queue->tx, i);
1682 resp->id = txp->id; 1767 resp->id = txp->id;
1683 resp->status = st; 1768 resp->status = st;
1684 1769
1685 if (txp->flags & XEN_NETTXF_extra_info) 1770 if (txp->flags & XEN_NETTXF_extra_info)
1686 RING_GET_RESPONSE(&vif->tx, ++i)->status = XEN_NETIF_RSP_NULL; 1771 RING_GET_RESPONSE(&queue->tx, ++i)->status = XEN_NETIF_RSP_NULL;
1687 1772
1688 vif->tx.rsp_prod_pvt = ++i; 1773 queue->tx.rsp_prod_pvt = ++i;
1689 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->tx, notify); 1774 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->tx, notify);
1690 if (notify) 1775 if (notify)
1691 notify_remote_via_irq(vif->tx_irq); 1776 notify_remote_via_irq(queue->tx_irq);
1692} 1777}
1693 1778
1694static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif, 1779static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
1695 u16 id, 1780 u16 id,
1696 s8 st, 1781 s8 st,
1697 u16 offset, 1782 u16 offset,
1698 u16 size, 1783 u16 size,
1699 u16 flags) 1784 u16 flags)
1700{ 1785{
1701 RING_IDX i = vif->rx.rsp_prod_pvt; 1786 RING_IDX i = queue->rx.rsp_prod_pvt;
1702 struct xen_netif_rx_response *resp; 1787 struct xen_netif_rx_response *resp;
1703 1788
1704 resp = RING_GET_RESPONSE(&vif->rx, i); 1789 resp = RING_GET_RESPONSE(&queue->rx, i);
1705 resp->offset = offset; 1790 resp->offset = offset;
1706 resp->flags = flags; 1791 resp->flags = flags;
1707 resp->id = id; 1792 resp->id = id;
@@ -1709,26 +1794,26 @@ static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
1709 if (st < 0) 1794 if (st < 0)
1710 resp->status = (s16)st; 1795 resp->status = (s16)st;
1711 1796
1712 vif->rx.rsp_prod_pvt = ++i; 1797 queue->rx.rsp_prod_pvt = ++i;
1713 1798
1714 return resp; 1799 return resp;
1715} 1800}
1716 1801
1717void xenvif_idx_unmap(struct xenvif *vif, u16 pending_idx) 1802void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx)
1718{ 1803{
1719 int ret; 1804 int ret;
1720 struct gnttab_unmap_grant_ref tx_unmap_op; 1805 struct gnttab_unmap_grant_ref tx_unmap_op;
1721 1806
1722 gnttab_set_unmap_op(&tx_unmap_op, 1807 gnttab_set_unmap_op(&tx_unmap_op,
1723 idx_to_kaddr(vif, pending_idx), 1808 idx_to_kaddr(queue, pending_idx),
1724 GNTMAP_host_map, 1809 GNTMAP_host_map,
1725 vif->grant_tx_handle[pending_idx]); 1810 queue->grant_tx_handle[pending_idx]);
1726 xenvif_grant_handle_reset(vif, pending_idx); 1811 xenvif_grant_handle_reset(queue, pending_idx);
1727 1812
1728 ret = gnttab_unmap_refs(&tx_unmap_op, NULL, 1813 ret = gnttab_unmap_refs(&tx_unmap_op, NULL,
1729 &vif->mmap_pages[pending_idx], 1); 1814 &queue->mmap_pages[pending_idx], 1);
1730 if (ret) { 1815 if (ret) {
1731 netdev_err(vif->dev, 1816 netdev_err(queue->vif->dev,
1732 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: %x status: %d\n", 1817 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: %x status: %d\n",
1733 ret, 1818 ret,
1734 pending_idx, 1819 pending_idx,
@@ -1738,41 +1823,40 @@ void xenvif_idx_unmap(struct xenvif *vif, u16 pending_idx)
1738 BUG(); 1823 BUG();
1739 } 1824 }
1740 1825
1741 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY); 1826 xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_OKAY);
1742} 1827}
1743 1828
1744static inline int rx_work_todo(struct xenvif *vif) 1829static inline int rx_work_todo(struct xenvif_queue *queue)
1745{ 1830{
1746 return (!skb_queue_empty(&vif->rx_queue) && 1831 return (!skb_queue_empty(&queue->rx_queue) &&
1747 xenvif_rx_ring_slots_available(vif, vif->rx_last_skb_slots)) || 1832 xenvif_rx_ring_slots_available(queue, queue->rx_last_skb_slots)) ||
1748 vif->rx_queue_purge; 1833 queue->rx_queue_purge;
1749} 1834}
1750 1835
1751static inline int tx_work_todo(struct xenvif *vif) 1836static inline int tx_work_todo(struct xenvif_queue *queue)
1752{ 1837{
1753 1838 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue->tx)))
1754 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif->tx)))
1755 return 1; 1839 return 1;
1756 1840
1757 return 0; 1841 return 0;
1758} 1842}
1759 1843
1760static inline bool tx_dealloc_work_todo(struct xenvif *vif) 1844static inline bool tx_dealloc_work_todo(struct xenvif_queue *queue)
1761{ 1845{
1762 return vif->dealloc_cons != vif->dealloc_prod; 1846 return queue->dealloc_cons != queue->dealloc_prod;
1763} 1847}
1764 1848
1765void xenvif_unmap_frontend_rings(struct xenvif *vif) 1849void xenvif_unmap_frontend_rings(struct xenvif_queue *queue)
1766{ 1850{
1767 if (vif->tx.sring) 1851 if (queue->tx.sring)
1768 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif), 1852 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif),
1769 vif->tx.sring); 1853 queue->tx.sring);
1770 if (vif->rx.sring) 1854 if (queue->rx.sring)
1771 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif), 1855 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue->vif),
1772 vif->rx.sring); 1856 queue->rx.sring);
1773} 1857}
1774 1858
1775int xenvif_map_frontend_rings(struct xenvif *vif, 1859int xenvif_map_frontend_rings(struct xenvif_queue *queue,
1776 grant_ref_t tx_ring_ref, 1860 grant_ref_t tx_ring_ref,
1777 grant_ref_t rx_ring_ref) 1861 grant_ref_t rx_ring_ref)
1778{ 1862{
@@ -1782,85 +1866,78 @@ int xenvif_map_frontend_rings(struct xenvif *vif,
1782 1866
1783 int err = -ENOMEM; 1867 int err = -ENOMEM;
1784 1868
1785 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif), 1869 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif),
1786 tx_ring_ref, &addr); 1870 tx_ring_ref, &addr);
1787 if (err) 1871 if (err)
1788 goto err; 1872 goto err;
1789 1873
1790 txs = (struct xen_netif_tx_sring *)addr; 1874 txs = (struct xen_netif_tx_sring *)addr;
1791 BACK_RING_INIT(&vif->tx, txs, PAGE_SIZE); 1875 BACK_RING_INIT(&queue->tx, txs, PAGE_SIZE);
1792 1876
1793 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif), 1877 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue->vif),
1794 rx_ring_ref, &addr); 1878 rx_ring_ref, &addr);
1795 if (err) 1879 if (err)
1796 goto err; 1880 goto err;
1797 1881
1798 rxs = (struct xen_netif_rx_sring *)addr; 1882 rxs = (struct xen_netif_rx_sring *)addr;
1799 BACK_RING_INIT(&vif->rx, rxs, PAGE_SIZE); 1883 BACK_RING_INIT(&queue->rx, rxs, PAGE_SIZE);
1800 1884
1801 return 0; 1885 return 0;
1802 1886
1803err: 1887err:
1804 xenvif_unmap_frontend_rings(vif); 1888 xenvif_unmap_frontend_rings(queue);
1805 return err; 1889 return err;
1806} 1890}
1807 1891
1808void xenvif_stop_queue(struct xenvif *vif) 1892static void xenvif_start_queue(struct xenvif_queue *queue)
1809{ 1893{
1810 if (!vif->can_queue) 1894 if (xenvif_schedulable(queue->vif))
1811 return; 1895 xenvif_wake_queue(queue);
1812
1813 netif_stop_queue(vif->dev);
1814}
1815
1816static void xenvif_start_queue(struct xenvif *vif)
1817{
1818 if (xenvif_schedulable(vif))
1819 netif_wake_queue(vif->dev);
1820} 1896}
1821 1897
1822int xenvif_kthread_guest_rx(void *data) 1898int xenvif_kthread_guest_rx(void *data)
1823{ 1899{
1824 struct xenvif *vif = data; 1900 struct xenvif_queue *queue = data;
1825 struct sk_buff *skb; 1901 struct sk_buff *skb;
1826 1902
1827 while (!kthread_should_stop()) { 1903 while (!kthread_should_stop()) {
1828 wait_event_interruptible(vif->wq, 1904 wait_event_interruptible(queue->wq,
1829 rx_work_todo(vif) || 1905 rx_work_todo(queue) ||
1830 vif->disabled || 1906 queue->vif->disabled ||
1831 kthread_should_stop()); 1907 kthread_should_stop());
1832 1908
1833 /* This frontend is found to be rogue, disable it in 1909 /* This frontend is found to be rogue, disable it in
1834 * kthread context. Currently this is only set when 1910 * kthread context. Currently this is only set when
1835 * netback finds out frontend sends malformed packet, 1911 * netback finds out frontend sends malformed packet,
1836 * but we cannot disable the interface in softirq 1912 * but we cannot disable the interface in softirq
1837 * context so we defer it here. 1913 * context so we defer it here, if this thread is
1914 * associated with queue 0.
1838 */ 1915 */
1839 if (unlikely(vif->disabled && netif_carrier_ok(vif->dev))) 1916 if (unlikely(queue->vif->disabled && netif_carrier_ok(queue->vif->dev) && queue->id == 0))
1840 xenvif_carrier_off(vif); 1917 xenvif_carrier_off(queue->vif);
1841 1918
1842 if (kthread_should_stop()) 1919 if (kthread_should_stop())
1843 break; 1920 break;
1844 1921
1845 if (vif->rx_queue_purge) { 1922 if (queue->rx_queue_purge) {
1846 skb_queue_purge(&vif->rx_queue); 1923 skb_queue_purge(&queue->rx_queue);
1847 vif->rx_queue_purge = false; 1924 queue->rx_queue_purge = false;
1848 } 1925 }
1849 1926
1850 if (!skb_queue_empty(&vif->rx_queue)) 1927 if (!skb_queue_empty(&queue->rx_queue))
1851 xenvif_rx_action(vif); 1928 xenvif_rx_action(queue);
1852 1929
1853 if (skb_queue_empty(&vif->rx_queue) && 1930 if (skb_queue_empty(&queue->rx_queue) &&
1854 netif_queue_stopped(vif->dev)) { 1931 xenvif_queue_stopped(queue)) {
1855 del_timer_sync(&vif->wake_queue); 1932 del_timer_sync(&queue->wake_queue);
1856 xenvif_start_queue(vif); 1933 xenvif_start_queue(queue);
1857 } 1934 }
1858 1935
1859 cond_resched(); 1936 cond_resched();
1860 } 1937 }
1861 1938
1862 /* Bin any remaining skbs */ 1939 /* Bin any remaining skbs */
1863 while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) 1940 while ((skb = skb_dequeue(&queue->rx_queue)) != NULL)
1864 dev_kfree_skb(skb); 1941 dev_kfree_skb(skb);
1865 1942
1866 return 0; 1943 return 0;
@@ -1868,22 +1945,22 @@ int xenvif_kthread_guest_rx(void *data)
1868 1945
1869int xenvif_dealloc_kthread(void *data) 1946int xenvif_dealloc_kthread(void *data)
1870{ 1947{
1871 struct xenvif *vif = data; 1948 struct xenvif_queue *queue = data;
1872 1949
1873 while (!kthread_should_stop()) { 1950 while (!kthread_should_stop()) {
1874 wait_event_interruptible(vif->dealloc_wq, 1951 wait_event_interruptible(queue->dealloc_wq,
1875 tx_dealloc_work_todo(vif) || 1952 tx_dealloc_work_todo(queue) ||
1876 kthread_should_stop()); 1953 kthread_should_stop());
1877 if (kthread_should_stop()) 1954 if (kthread_should_stop())
1878 break; 1955 break;
1879 1956
1880 xenvif_tx_dealloc_action(vif); 1957 xenvif_tx_dealloc_action(queue);
1881 cond_resched(); 1958 cond_resched();
1882 } 1959 }
1883 1960
1884 /* Unmap anything remaining*/ 1961 /* Unmap anything remaining*/
1885 if (tx_dealloc_work_todo(vif)) 1962 if (tx_dealloc_work_todo(queue))
1886 xenvif_tx_dealloc_action(vif); 1963 xenvif_tx_dealloc_action(queue);
1887 1964
1888 return 0; 1965 return 0;
1889} 1966}
@@ -1895,6 +1972,9 @@ static int __init netback_init(void)
1895 if (!xen_domain()) 1972 if (!xen_domain())
1896 return -ENODEV; 1973 return -ENODEV;
1897 1974
1975 /* Allow as many queues as there are CPUs, by default */
1976 xenvif_max_queues = num_online_cpus();
1977
1898 if (fatal_skb_slots < XEN_NETBK_LEGACY_SLOTS_MAX) { 1978 if (fatal_skb_slots < XEN_NETBK_LEGACY_SLOTS_MAX) {
1899 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n", 1979 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
1900 fatal_skb_slots, XEN_NETBK_LEGACY_SLOTS_MAX); 1980 fatal_skb_slots, XEN_NETBK_LEGACY_SLOTS_MAX);
diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c
index 7a206cffb062..96c63dc2509e 100644
--- a/drivers/net/xen-netback/xenbus.c
+++ b/drivers/net/xen-netback/xenbus.c
@@ -19,6 +19,8 @@
19*/ 19*/
20 20
21#include "common.h" 21#include "common.h"
22#include <linux/vmalloc.h>
23#include <linux/rtnetlink.h>
22 24
23struct backend_info { 25struct backend_info {
24 struct xenbus_device *dev; 26 struct xenbus_device *dev;
@@ -34,8 +36,9 @@ struct backend_info {
34 u8 have_hotplug_status_watch:1; 36 u8 have_hotplug_status_watch:1;
35}; 37};
36 38
37static int connect_rings(struct backend_info *); 39static int connect_rings(struct backend_info *be, struct xenvif_queue *queue);
38static void connect(struct backend_info *); 40static void connect(struct backend_info *be);
41static int read_xenbus_vif_flags(struct backend_info *be);
39static void backend_create_xenvif(struct backend_info *be); 42static void backend_create_xenvif(struct backend_info *be);
40static void unregister_hotplug_status_watch(struct backend_info *be); 43static void unregister_hotplug_status_watch(struct backend_info *be);
41static void set_backend_state(struct backend_info *be, 44static void set_backend_state(struct backend_info *be,
@@ -157,6 +160,12 @@ static int netback_probe(struct xenbus_device *dev,
157 if (err) 160 if (err)
158 pr_debug("Error writing feature-split-event-channels\n"); 161 pr_debug("Error writing feature-split-event-channels\n");
159 162
163 /* Multi-queue support: This is an optional feature. */
164 err = xenbus_printf(XBT_NIL, dev->nodename,
165 "multi-queue-max-queues", "%u", xenvif_max_queues);
166 if (err)
167 pr_debug("Error writing multi-queue-max-queues\n");
168
160 err = xenbus_switch_state(dev, XenbusStateInitWait); 169 err = xenbus_switch_state(dev, XenbusStateInitWait);
161 if (err) 170 if (err)
162 goto fail; 171 goto fail;
@@ -485,10 +494,26 @@ static void connect(struct backend_info *be)
485{ 494{
486 int err; 495 int err;
487 struct xenbus_device *dev = be->dev; 496 struct xenbus_device *dev = be->dev;
497 unsigned long credit_bytes, credit_usec;
498 unsigned int queue_index;
499 unsigned int requested_num_queues;
500 struct xenvif_queue *queue;
488 501
489 err = connect_rings(be); 502 /* Check whether the frontend requested multiple queues
490 if (err) 503 * and read the number requested.
504 */
505 err = xenbus_scanf(XBT_NIL, dev->otherend,
506 "multi-queue-num-queues",
507 "%u", &requested_num_queues);
508 if (err < 0) {
509 requested_num_queues = 1; /* Fall back to single queue */
510 } else if (requested_num_queues > xenvif_max_queues) {
511 /* buggy or malicious guest */
512 xenbus_dev_fatal(dev, err,
513 "guest requested %u queues, exceeding the maximum of %u.",
514 requested_num_queues, xenvif_max_queues);
491 return; 515 return;
516 }
492 517
493 err = xen_net_read_mac(dev, be->vif->fe_dev_addr); 518 err = xen_net_read_mac(dev, be->vif->fe_dev_addr);
494 if (err) { 519 if (err) {
@@ -496,9 +521,54 @@ static void connect(struct backend_info *be)
496 return; 521 return;
497 } 522 }
498 523
499 xen_net_read_rate(dev, &be->vif->credit_bytes, 524 xen_net_read_rate(dev, &credit_bytes, &credit_usec);
500 &be->vif->credit_usec); 525 read_xenbus_vif_flags(be);
501 be->vif->remaining_credit = be->vif->credit_bytes; 526
527 /* Use the number of queues requested by the frontend */
528 be->vif->queues = vzalloc(requested_num_queues *
529 sizeof(struct xenvif_queue));
530 rtnl_lock();
531 netif_set_real_num_tx_queues(be->vif->dev, requested_num_queues);
532 rtnl_unlock();
533
534 for (queue_index = 0; queue_index < requested_num_queues; ++queue_index) {
535 queue = &be->vif->queues[queue_index];
536 queue->vif = be->vif;
537 queue->id = queue_index;
538 snprintf(queue->name, sizeof(queue->name), "%s-q%u",
539 be->vif->dev->name, queue->id);
540
541 err = xenvif_init_queue(queue);
542 if (err) {
543 /* xenvif_init_queue() cleans up after itself on
544 * failure, but we need to clean up any previously
545 * initialised queues. Set num_queues to i so that
546 * earlier queues can be destroyed using the regular
547 * disconnect logic.
548 */
549 rtnl_lock();
550 netif_set_real_num_tx_queues(be->vif->dev, queue_index);
551 rtnl_unlock();
552 goto err;
553 }
554
555 queue->remaining_credit = credit_bytes;
556
557 err = connect_rings(be, queue);
558 if (err) {
559 /* connect_rings() cleans up after itself on failure,
560 * but we need to clean up after xenvif_init_queue() here,
561 * and also clean up any previously initialised queues.
562 */
563 xenvif_deinit_queue(queue);
564 rtnl_lock();
565 netif_set_real_num_tx_queues(be->vif->dev, queue_index);
566 rtnl_unlock();
567 goto err;
568 }
569 }
570
571 xenvif_carrier_on(be->vif);
502 572
503 unregister_hotplug_status_watch(be); 573 unregister_hotplug_status_watch(be);
504 err = xenbus_watch_pathfmt(dev, &be->hotplug_status_watch, 574 err = xenbus_watch_pathfmt(dev, &be->hotplug_status_watch,
@@ -507,45 +577,109 @@ static void connect(struct backend_info *be)
507 if (!err) 577 if (!err)
508 be->have_hotplug_status_watch = 1; 578 be->have_hotplug_status_watch = 1;
509 579
510 netif_wake_queue(be->vif->dev); 580 netif_tx_wake_all_queues(be->vif->dev);
581
582 return;
583
584err:
585 if (be->vif->dev->real_num_tx_queues > 0)
586 xenvif_disconnect(be->vif); /* Clean up existing queues */
587 vfree(be->vif->queues);
588 be->vif->queues = NULL;
589 rtnl_lock();
590 netif_set_real_num_tx_queues(be->vif->dev, 0);
591 rtnl_unlock();
592 return;
511} 593}
512 594
513 595
514static int connect_rings(struct backend_info *be) 596static int connect_rings(struct backend_info *be, struct xenvif_queue *queue)
515{ 597{
516 struct xenvif *vif = be->vif;
517 struct xenbus_device *dev = be->dev; 598 struct xenbus_device *dev = be->dev;
599 unsigned int num_queues = queue->vif->dev->real_num_tx_queues;
518 unsigned long tx_ring_ref, rx_ring_ref; 600 unsigned long tx_ring_ref, rx_ring_ref;
519 unsigned int tx_evtchn, rx_evtchn, rx_copy; 601 unsigned int tx_evtchn, rx_evtchn;
520 int err; 602 int err;
521 int val; 603 char *xspath;
604 size_t xspathsize;
605 const size_t xenstore_path_ext_size = 11; /* sufficient for "/queue-NNN" */
606
607 /* If the frontend requested 1 queue, or we have fallen back
608 * to single queue due to lack of frontend support for multi-
609 * queue, expect the remaining XenStore keys in the toplevel
610 * directory. Otherwise, expect them in a subdirectory called
611 * queue-N.
612 */
613 if (num_queues == 1) {
614 xspath = kzalloc(strlen(dev->otherend) + 1, GFP_KERNEL);
615 if (!xspath) {
616 xenbus_dev_fatal(dev, -ENOMEM,
617 "reading ring references");
618 return -ENOMEM;
619 }
620 strcpy(xspath, dev->otherend);
621 } else {
622 xspathsize = strlen(dev->otherend) + xenstore_path_ext_size;
623 xspath = kzalloc(xspathsize, GFP_KERNEL);
624 if (!xspath) {
625 xenbus_dev_fatal(dev, -ENOMEM,
626 "reading ring references");
627 return -ENOMEM;
628 }
629 snprintf(xspath, xspathsize, "%s/queue-%u", dev->otherend,
630 queue->id);
631 }
522 632
523 err = xenbus_gather(XBT_NIL, dev->otherend, 633 err = xenbus_gather(XBT_NIL, xspath,
524 "tx-ring-ref", "%lu", &tx_ring_ref, 634 "tx-ring-ref", "%lu", &tx_ring_ref,
525 "rx-ring-ref", "%lu", &rx_ring_ref, NULL); 635 "rx-ring-ref", "%lu", &rx_ring_ref, NULL);
526 if (err) { 636 if (err) {
527 xenbus_dev_fatal(dev, err, 637 xenbus_dev_fatal(dev, err,
528 "reading %s/ring-ref", 638 "reading %s/ring-ref",
529 dev->otherend); 639 xspath);
530 return err; 640 goto err;
531 } 641 }
532 642
533 /* Try split event channels first, then single event channel. */ 643 /* Try split event channels first, then single event channel. */
534 err = xenbus_gather(XBT_NIL, dev->otherend, 644 err = xenbus_gather(XBT_NIL, xspath,
535 "event-channel-tx", "%u", &tx_evtchn, 645 "event-channel-tx", "%u", &tx_evtchn,
536 "event-channel-rx", "%u", &rx_evtchn, NULL); 646 "event-channel-rx", "%u", &rx_evtchn, NULL);
537 if (err < 0) { 647 if (err < 0) {
538 err = xenbus_scanf(XBT_NIL, dev->otherend, 648 err = xenbus_scanf(XBT_NIL, xspath,
539 "event-channel", "%u", &tx_evtchn); 649 "event-channel", "%u", &tx_evtchn);
540 if (err < 0) { 650 if (err < 0) {
541 xenbus_dev_fatal(dev, err, 651 xenbus_dev_fatal(dev, err,
542 "reading %s/event-channel(-tx/rx)", 652 "reading %s/event-channel(-tx/rx)",
543 dev->otherend); 653 xspath);
544 return err; 654 goto err;
545 } 655 }
546 rx_evtchn = tx_evtchn; 656 rx_evtchn = tx_evtchn;
547 } 657 }
548 658
659 /* Map the shared frame, irq etc. */
660 err = xenvif_connect(queue, tx_ring_ref, rx_ring_ref,
661 tx_evtchn, rx_evtchn);
662 if (err) {
663 xenbus_dev_fatal(dev, err,
664 "mapping shared-frames %lu/%lu port tx %u rx %u",
665 tx_ring_ref, rx_ring_ref,
666 tx_evtchn, rx_evtchn);
667 goto err;
668 }
669
670 err = 0;
671err: /* Regular return falls through with err == 0 */
672 kfree(xspath);
673 return err;
674}
675
676static int read_xenbus_vif_flags(struct backend_info *be)
677{
678 struct xenvif *vif = be->vif;
679 struct xenbus_device *dev = be->dev;
680 unsigned int rx_copy;
681 int err, val;
682
549 err = xenbus_scanf(XBT_NIL, dev->otherend, "request-rx-copy", "%u", 683 err = xenbus_scanf(XBT_NIL, dev->otherend, "request-rx-copy", "%u",
550 &rx_copy); 684 &rx_copy);
551 if (err == -ENOENT) { 685 if (err == -ENOENT) {
@@ -621,16 +755,6 @@ static int connect_rings(struct backend_info *be)
621 val = 0; 755 val = 0;
622 vif->ipv6_csum = !!val; 756 vif->ipv6_csum = !!val;
623 757
624 /* Map the shared frame, irq etc. */
625 err = xenvif_connect(vif, tx_ring_ref, rx_ring_ref,
626 tx_evtchn, rx_evtchn);
627 if (err) {
628 xenbus_dev_fatal(dev, err,
629 "mapping shared-frames %lu/%lu port tx %u rx %u",
630 tx_ring_ref, rx_ring_ref,
631 tx_evtchn, rx_evtchn);
632 return err;
633 }
634 return 0; 758 return 0;
635} 759}
636 760
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-24 12:52:21 -0500