aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband/hw/cxgb4/cm.c
diff options
context:
space:
mode:
authorSteve Wise <swise@opengridcomputing.com>2010-04-21 18:30:06 -0400
committerRoland Dreier <rolandd@cisco.com>2010-04-21 18:30:06 -0400
commitcfdda9d764362ab77b11a410bb928400e6520d57 (patch)
tree3634e5aca12414d40f4e50a3d73543cc479b525f /drivers/infiniband/hw/cxgb4/cm.c
parent0eddb519b9127c73d53db4bf3ec1d45b13f844d1 (diff)
RDMA/cxgb4: Add driver for Chelsio T4 RNIC
Add an RDMA/iWARP driver for Chelsio T4 Ethernet adapters. Signed-off-by: Steve Wise <swise@opengridcomputing.com> Signed-off-by: Roland Dreier <rolandd@cisco.com>
Diffstat (limited to 'drivers/infiniband/hw/cxgb4/cm.c')
-rw-r--r--drivers/infiniband/hw/cxgb4/cm.c2330
1 files changed, 2330 insertions, 0 deletions
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
new file mode 100644
index 000000000000..07b068be0cfa
--- /dev/null
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -0,0 +1,2330 @@
1/*
2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/module.h>
33#include <linux/list.h>
34#include <linux/workqueue.h>
35#include <linux/skbuff.h>
36#include <linux/timer.h>
37#include <linux/notifier.h>
38#include <linux/inetdevice.h>
39#include <linux/ip.h>
40#include <linux/tcp.h>
41
42#include <net/neighbour.h>
43#include <net/netevent.h>
44#include <net/route.h>
45
46#include "iw_cxgb4.h"
47
48static char *states[] = {
49 "idle",
50 "listen",
51 "connecting",
52 "mpa_wait_req",
53 "mpa_req_sent",
54 "mpa_req_rcvd",
55 "mpa_rep_sent",
56 "fpdu_mode",
57 "aborting",
58 "closing",
59 "moribund",
60 "dead",
61 NULL,
62};
63
64static int enable_tcp_timestamps;
65module_param(enable_tcp_timestamps, int, 0644);
66MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)");
67
68static int enable_tcp_sack;
69module_param(enable_tcp_sack, int, 0644);
70MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)");
71
72static int enable_tcp_window_scaling = 1;
73module_param(enable_tcp_window_scaling, int, 0644);
74MODULE_PARM_DESC(enable_tcp_window_scaling,
75 "Enable tcp window scaling (default=1)");
76
77int c4iw_debug;
78module_param(c4iw_debug, int, 0644);
79MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)");
80
81static int peer2peer;
82module_param(peer2peer, int, 0644);
83MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)");
84
85static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ;
86module_param(p2p_type, int, 0644);
87MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: "
88 "1=RDMA_READ 0=RDMA_WRITE (default 1)");
89
90static int ep_timeout_secs = 60;
91module_param(ep_timeout_secs, int, 0644);
92MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "
93 "in seconds (default=60)");
94
95static int mpa_rev = 1;
96module_param(mpa_rev, int, 0644);
97MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, "
98 "1 is spec compliant. (default=1)");
99
100static int markers_enabled;
101module_param(markers_enabled, int, 0644);
102MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)");
103
104static int crc_enabled = 1;
105module_param(crc_enabled, int, 0644);
106MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)");
107
108static int rcv_win = 256 * 1024;
109module_param(rcv_win, int, 0644);
110MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)");
111
112static int snd_win = 32 * 1024;
113module_param(snd_win, int, 0644);
114MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=32KB)");
115
116static void process_work(struct work_struct *work);
117static struct workqueue_struct *workq;
118static DECLARE_WORK(skb_work, process_work);
119
120static struct sk_buff_head rxq;
121static c4iw_handler_func work_handlers[NUM_CPL_CMDS];
122c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
123
124static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp);
125static void ep_timeout(unsigned long arg);
126static void connect_reply_upcall(struct c4iw_ep *ep, int status);
127
128static void start_ep_timer(struct c4iw_ep *ep)
129{
130 PDBG("%s ep %p\n", __func__, ep);
131 if (timer_pending(&ep->timer)) {
132 PDBG("%s stopped / restarted timer ep %p\n", __func__, ep);
133 del_timer_sync(&ep->timer);
134 } else
135 c4iw_get_ep(&ep->com);
136 ep->timer.expires = jiffies + ep_timeout_secs * HZ;
137 ep->timer.data = (unsigned long)ep;
138 ep->timer.function = ep_timeout;
139 add_timer(&ep->timer);
140}
141
142static void stop_ep_timer(struct c4iw_ep *ep)
143{
144 PDBG("%s ep %p\n", __func__, ep);
145 if (!timer_pending(&ep->timer)) {
146 printk(KERN_ERR "%s timer stopped when its not running! "
147 "ep %p state %u\n", __func__, ep, ep->com.state);
148 WARN_ON(1);
149 return;
150 }
151 del_timer_sync(&ep->timer);
152 c4iw_put_ep(&ep->com);
153}
154
155static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb,
156 struct l2t_entry *l2e)
157{
158 int error = 0;
159
160 if (c4iw_fatal_error(rdev)) {
161 kfree_skb(skb);
162 PDBG("%s - device in error state - dropping\n", __func__);
163 return -EIO;
164 }
165 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e);
166 if (error < 0)
167 kfree_skb(skb);
168 return error;
169}
170
171int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb)
172{
173 int error = 0;
174
175 if (c4iw_fatal_error(rdev)) {
176 kfree_skb(skb);
177 PDBG("%s - device in error state - dropping\n", __func__);
178 return -EIO;
179 }
180 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb);
181 if (error < 0)
182 kfree_skb(skb);
183 return error;
184}
185
186static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb)
187{
188 struct cpl_tid_release *req;
189
190 skb = get_skb(skb, sizeof *req, GFP_KERNEL);
191 if (!skb)
192 return;
193 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req));
194 INIT_TP_WR(req, hwtid);
195 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));
196 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
197 c4iw_ofld_send(rdev, skb);
198 return;
199}
200
201static void set_emss(struct c4iw_ep *ep, u16 opt)
202{
203 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40;
204 ep->mss = ep->emss;
205 if (GET_TCPOPT_TSTAMP(opt))
206 ep->emss -= 12;
207 if (ep->emss < 128)
208 ep->emss = 128;
209 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt),
210 ep->mss, ep->emss);
211}
212
213static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc)
214{
215 unsigned long flags;
216 enum c4iw_ep_state state;
217
218 spin_lock_irqsave(&epc->lock, flags);
219 state = epc->state;
220 spin_unlock_irqrestore(&epc->lock, flags);
221 return state;
222}
223
224static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new)
225{
226 epc->state = new;
227}
228
229static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new)
230{
231 unsigned long flags;
232
233 spin_lock_irqsave(&epc->lock, flags);
234 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]);
235 __state_set(epc, new);
236 spin_unlock_irqrestore(&epc->lock, flags);
237 return;
238}
239
240static void *alloc_ep(int size, gfp_t gfp)
241{
242 struct c4iw_ep_common *epc;
243
244 epc = kzalloc(size, gfp);
245 if (epc) {
246 kref_init(&epc->kref);
247 spin_lock_init(&epc->lock);
248 init_waitqueue_head(&epc->waitq);
249 }
250 PDBG("%s alloc ep %p\n", __func__, epc);
251 return epc;
252}
253
254void _c4iw_free_ep(struct kref *kref)
255{
256 struct c4iw_ep *ep;
257
258 ep = container_of(kref, struct c4iw_ep, com.kref);
259 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]);
260 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) {
261 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid);
262 dst_release(ep->dst);
263 cxgb4_l2t_release(ep->l2t);
264 }
265 kfree(ep);
266}
267
268static void release_ep_resources(struct c4iw_ep *ep)
269{
270 set_bit(RELEASE_RESOURCES, &ep->com.flags);
271 c4iw_put_ep(&ep->com);
272}
273
274static void process_work(struct work_struct *work)
275{
276 struct sk_buff *skb = NULL;
277 struct c4iw_dev *dev;
278 struct cpl_act_establish *rpl = cplhdr(skb);
279 unsigned int opcode;
280 int ret;
281
282 while ((skb = skb_dequeue(&rxq))) {
283 rpl = cplhdr(skb);
284 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *)));
285 opcode = rpl->ot.opcode;
286
287 BUG_ON(!work_handlers[opcode]);
288 ret = work_handlers[opcode](dev, skb);
289 if (!ret)
290 kfree_skb(skb);
291 }
292}
293
294static int status2errno(int status)
295{
296 switch (status) {
297 case CPL_ERR_NONE:
298 return 0;
299 case CPL_ERR_CONN_RESET:
300 return -ECONNRESET;
301 case CPL_ERR_ARP_MISS:
302 return -EHOSTUNREACH;
303 case CPL_ERR_CONN_TIMEDOUT:
304 return -ETIMEDOUT;
305 case CPL_ERR_TCAM_FULL:
306 return -ENOMEM;
307 case CPL_ERR_CONN_EXIST:
308 return -EADDRINUSE;
309 default:
310 return -EIO;
311 }
312}
313
314/*
315 * Try and reuse skbs already allocated...
316 */
317static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)
318{
319 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) {
320 skb_trim(skb, 0);
321 skb_get(skb);
322 skb_reset_transport_header(skb);
323 } else {
324 skb = alloc_skb(len, gfp);
325 }
326 return skb;
327}
328
329static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip,
330 __be32 peer_ip, __be16 local_port,
331 __be16 peer_port, u8 tos)
332{
333 struct rtable *rt;
334 struct flowi fl = {
335 .oif = 0,
336 .nl_u = {
337 .ip4_u = {
338 .daddr = peer_ip,
339 .saddr = local_ip,
340 .tos = tos}
341 },
342 .proto = IPPROTO_TCP,
343 .uli_u = {
344 .ports = {
345 .sport = local_port,
346 .dport = peer_port}
347 }
348 };
349
350 if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
351 return NULL;
352 return rt;
353}
354
355static void arp_failure_discard(void *handle, struct sk_buff *skb)
356{
357 PDBG("%s c4iw_dev %p\n", __func__, handle);
358 kfree_skb(skb);
359}
360
361/*
362 * Handle an ARP failure for an active open.
363 */
364static void act_open_req_arp_failure(void *handle, struct sk_buff *skb)
365{
366 printk(KERN_ERR MOD "ARP failure duing connect\n");
367 kfree_skb(skb);
368}
369
370/*
371 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
372 * and send it along.
373 */
374static void abort_arp_failure(void *handle, struct sk_buff *skb)
375{
376 struct c4iw_rdev *rdev = handle;
377 struct cpl_abort_req *req = cplhdr(skb);
378
379 PDBG("%s rdev %p\n", __func__, rdev);
380 req->cmd = CPL_ABORT_NO_RST;
381 c4iw_ofld_send(rdev, skb);
382}
383
384static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb)
385{
386 unsigned int flowclen = 80;
387 struct fw_flowc_wr *flowc;
388 int i;
389
390 skb = get_skb(skb, flowclen, GFP_KERNEL);
391 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen);
392
393 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) |
394 FW_FLOWC_WR_NPARAMS(8));
395 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen,
396 16)) | FW_WR_FLOWID(ep->hwtid));
397
398 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
399 flowc->mnemval[0].val = cpu_to_be32(0);
400 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
401 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan);
402 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
403 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan);
404 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
405 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid);
406 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
407 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq);
408 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
409 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq);
410 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
411 flowc->mnemval[6].val = cpu_to_be32(snd_win);
412 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
413 flowc->mnemval[7].val = cpu_to_be32(ep->emss);
414 /* Pad WR to 16 byte boundary */
415 flowc->mnemval[8].mnemonic = 0;
416 flowc->mnemval[8].val = 0;
417 for (i = 0; i < 9; i++) {
418 flowc->mnemval[i].r4[0] = 0;
419 flowc->mnemval[i].r4[1] = 0;
420 flowc->mnemval[i].r4[2] = 0;
421 }
422
423 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
424 c4iw_ofld_send(&ep->com.dev->rdev, skb);
425}
426
427static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp)
428{
429 struct cpl_close_con_req *req;
430 struct sk_buff *skb;
431 int wrlen = roundup(sizeof *req, 16);
432
433 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
434 skb = get_skb(NULL, wrlen, gfp);
435 if (!skb) {
436 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
437 return -ENOMEM;
438 }
439 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
440 t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
441 req = (struct cpl_close_con_req *) skb_put(skb, wrlen);
442 memset(req, 0, wrlen);
443 INIT_TP_WR(req, ep->hwtid);
444 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ,
445 ep->hwtid));
446 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
447}
448
449static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp)
450{
451 struct cpl_abort_req *req;
452 int wrlen = roundup(sizeof *req, 16);
453
454 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
455 skb = get_skb(skb, wrlen, gfp);
456 if (!skb) {
457 printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
458 __func__);
459 return -ENOMEM;
460 }
461 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
462 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure);
463 req = (struct cpl_abort_req *) skb_put(skb, wrlen);
464 memset(req, 0, wrlen);
465 INIT_TP_WR(req, ep->hwtid);
466 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid));
467 req->cmd = CPL_ABORT_SEND_RST;
468 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
469}
470
471static int send_connect(struct c4iw_ep *ep)
472{
473 struct cpl_act_open_req *req;
474 struct sk_buff *skb;
475 u64 opt0;
476 u32 opt2;
477 unsigned int mtu_idx;
478 int wscale;
479 int wrlen = roundup(sizeof *req, 16);
480
481 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid);
482
483 skb = get_skb(NULL, wrlen, GFP_KERNEL);
484 if (!skb) {
485 printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
486 __func__);
487 return -ENOMEM;
488 }
489 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx);
490
491 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx);
492 wscale = compute_wscale(rcv_win);
493 opt0 = KEEP_ALIVE(1) |
494 WND_SCALE(wscale) |
495 MSS_IDX(mtu_idx) |
496 L2T_IDX(ep->l2t->idx) |
497 TX_CHAN(ep->tx_chan) |
498 SMAC_SEL(ep->smac_idx) |
499 DSCP(ep->tos) |
500 RCV_BUFSIZ(rcv_win>>10);
501 opt2 = RX_CHANNEL(0) |
502 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid);
503 if (enable_tcp_timestamps)
504 opt2 |= TSTAMPS_EN(1);
505 if (enable_tcp_sack)
506 opt2 |= SACK_EN(1);
507 if (wscale && enable_tcp_window_scaling)
508 opt2 |= WND_SCALE_EN(1);
509 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure);
510
511 req = (struct cpl_act_open_req *) skb_put(skb, wrlen);
512 INIT_TP_WR(req, 0);
513 OPCODE_TID(req) = cpu_to_be32(
514 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ((ep->rss_qid<<14)|ep->atid)));
515 req->local_port = ep->com.local_addr.sin_port;
516 req->peer_port = ep->com.remote_addr.sin_port;
517 req->local_ip = ep->com.local_addr.sin_addr.s_addr;
518 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr;
519 req->opt0 = cpu_to_be64(opt0);
520 req->params = 0;
521 req->opt2 = cpu_to_be32(opt2);
522 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
523}
524
525static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb)
526{
527 int mpalen, wrlen;
528 struct fw_ofld_tx_data_wr *req;
529 struct mpa_message *mpa;
530
531 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);
532
533 BUG_ON(skb_cloned(skb));
534
535 mpalen = sizeof(*mpa) + ep->plen;
536 wrlen = roundup(mpalen + sizeof *req, 16);
537 skb = get_skb(skb, wrlen, GFP_KERNEL);
538 if (!skb) {
539 connect_reply_upcall(ep, -ENOMEM);
540 return;
541 }
542 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
543
544 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen);
545 memset(req, 0, wrlen);
546 req->op_to_immdlen = cpu_to_be32(
547 FW_WR_OP(FW_OFLD_TX_DATA_WR) |
548 FW_WR_COMPL(1) |
549 FW_WR_IMMDLEN(mpalen));
550 req->flowid_len16 = cpu_to_be32(
551 FW_WR_FLOWID(ep->hwtid) |
552 FW_WR_LEN16(wrlen >> 4));
553 req->plen = cpu_to_be32(mpalen);
554 req->tunnel_to_proxy = cpu_to_be32(
555 FW_OFLD_TX_DATA_WR_FLUSH(1) |
556 FW_OFLD_TX_DATA_WR_SHOVE(1));
557
558 mpa = (struct mpa_message *)(req + 1);
559 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
560 mpa->flags = (crc_enabled ? MPA_CRC : 0) |
561 (markers_enabled ? MPA_MARKERS : 0);
562 mpa->private_data_size = htons(ep->plen);
563 mpa->revision = mpa_rev;
564
565 if (ep->plen)
566 memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen);
567
568 /*
569 * Reference the mpa skb. This ensures the data area
570 * will remain in memory until the hw acks the tx.
571 * Function fw4_ack() will deref it.
572 */
573 skb_get(skb);
574 t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
575 BUG_ON(ep->mpa_skb);
576 ep->mpa_skb = skb;
577 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
578 start_ep_timer(ep);
579 state_set(&ep->com, MPA_REQ_SENT);
580 ep->mpa_attr.initiator = 1;
581 return;
582}
583
584static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)
585{
586 int mpalen, wrlen;
587 struct fw_ofld_tx_data_wr *req;
588 struct mpa_message *mpa;
589 struct sk_buff *skb;
590
591 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);
592
593 mpalen = sizeof(*mpa) + plen;
594 wrlen = roundup(mpalen + sizeof *req, 16);
595
596 skb = get_skb(NULL, wrlen, GFP_KERNEL);
597 if (!skb) {
598 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
599 return -ENOMEM;
600 }
601 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
602
603 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen);
604 memset(req, 0, wrlen);
605 req->op_to_immdlen = cpu_to_be32(
606 FW_WR_OP(FW_OFLD_TX_DATA_WR) |
607 FW_WR_COMPL(1) |
608 FW_WR_IMMDLEN(mpalen));
609 req->flowid_len16 = cpu_to_be32(
610 FW_WR_FLOWID(ep->hwtid) |
611 FW_WR_LEN16(wrlen >> 4));
612 req->plen = cpu_to_be32(mpalen);
613 req->tunnel_to_proxy = cpu_to_be32(
614 FW_OFLD_TX_DATA_WR_FLUSH(1) |
615 FW_OFLD_TX_DATA_WR_SHOVE(1));
616
617 mpa = (struct mpa_message *)(req + 1);
618 memset(mpa, 0, sizeof(*mpa));
619 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
620 mpa->flags = MPA_REJECT;
621 mpa->revision = mpa_rev;
622 mpa->private_data_size = htons(plen);
623 if (plen)
624 memcpy(mpa->private_data, pdata, plen);
625
626 /*
627 * Reference the mpa skb again. This ensures the data area
628 * will remain in memory until the hw acks the tx.
629 * Function fw4_ack() will deref it.
630 */
631 skb_get(skb);
632 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
633 t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
634 BUG_ON(ep->mpa_skb);
635 ep->mpa_skb = skb;
636 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
637}
638
639static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)
640{
641 int mpalen, wrlen;
642 struct fw_ofld_tx_data_wr *req;
643 struct mpa_message *mpa;
644 struct sk_buff *skb;
645
646 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen);
647
648 mpalen = sizeof(*mpa) + plen;
649 wrlen = roundup(mpalen + sizeof *req, 16);
650
651 skb = get_skb(NULL, wrlen, GFP_KERNEL);
652 if (!skb) {
653 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
654 return -ENOMEM;
655 }
656 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
657
658 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen);
659 memset(req, 0, wrlen);
660 req->op_to_immdlen = cpu_to_be32(
661 FW_WR_OP(FW_OFLD_TX_DATA_WR) |
662 FW_WR_COMPL(1) |
663 FW_WR_IMMDLEN(mpalen));
664 req->flowid_len16 = cpu_to_be32(
665 FW_WR_FLOWID(ep->hwtid) |
666 FW_WR_LEN16(wrlen >> 4));
667 req->plen = cpu_to_be32(mpalen);
668 req->tunnel_to_proxy = cpu_to_be32(
669 FW_OFLD_TX_DATA_WR_FLUSH(1) |
670 FW_OFLD_TX_DATA_WR_SHOVE(1));
671
672 mpa = (struct mpa_message *)(req + 1);
673 memset(mpa, 0, sizeof(*mpa));
674 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
675 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
676 (markers_enabled ? MPA_MARKERS : 0);
677 mpa->revision = mpa_rev;
678 mpa->private_data_size = htons(plen);
679 if (plen)
680 memcpy(mpa->private_data, pdata, plen);
681
682 /*
683 * Reference the mpa skb. This ensures the data area
684 * will remain in memory until the hw acks the tx.
685 * Function fw4_ack() will deref it.
686 */
687 skb_get(skb);
688 t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
689 ep->mpa_skb = skb;
690 state_set(&ep->com, MPA_REP_SENT);
691 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
692}
693
694static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb)
695{
696 struct c4iw_ep *ep;
697 struct cpl_act_establish *req = cplhdr(skb);
698 unsigned int tid = GET_TID(req);
699 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid));
700 struct tid_info *t = dev->rdev.lldi.tids;
701
702 ep = lookup_atid(t, atid);
703
704 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid,
705 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn));
706
707 dst_confirm(ep->dst);
708
709 /* setup the hwtid for this connection */
710 ep->hwtid = tid;
711 cxgb4_insert_tid(t, ep, tid);
712
713 ep->snd_seq = be32_to_cpu(req->snd_isn);
714 ep->rcv_seq = be32_to_cpu(req->rcv_isn);
715
716 set_emss(ep, ntohs(req->tcp_opt));
717
718 /* dealloc the atid */
719 cxgb4_free_atid(t, atid);
720
721 /* start MPA negotiation */
722 send_flowc(ep, NULL);
723 send_mpa_req(ep, skb);
724
725 return 0;
726}
727
728static void close_complete_upcall(struct c4iw_ep *ep)
729{
730 struct iw_cm_event event;
731
732 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
733 memset(&event, 0, sizeof(event));
734 event.event = IW_CM_EVENT_CLOSE;
735 if (ep->com.cm_id) {
736 PDBG("close complete delivered ep %p cm_id %p tid %u\n",
737 ep, ep->com.cm_id, ep->hwtid);
738 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
739 ep->com.cm_id->rem_ref(ep->com.cm_id);
740 ep->com.cm_id = NULL;
741 ep->com.qp = NULL;
742 }
743}
744
745static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp)
746{
747 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
748 close_complete_upcall(ep);
749 state_set(&ep->com, ABORTING);
750 return send_abort(ep, skb, gfp);
751}
752
753static void peer_close_upcall(struct c4iw_ep *ep)
754{
755 struct iw_cm_event event;
756
757 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
758 memset(&event, 0, sizeof(event));
759 event.event = IW_CM_EVENT_DISCONNECT;
760 if (ep->com.cm_id) {
761 PDBG("peer close delivered ep %p cm_id %p tid %u\n",
762 ep, ep->com.cm_id, ep->hwtid);
763 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
764 }
765}
766
767static void peer_abort_upcall(struct c4iw_ep *ep)
768{
769 struct iw_cm_event event;
770
771 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
772 memset(&event, 0, sizeof(event));
773 event.event = IW_CM_EVENT_CLOSE;
774 event.status = -ECONNRESET;
775 if (ep->com.cm_id) {
776 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep,
777 ep->com.cm_id, ep->hwtid);
778 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
779 ep->com.cm_id->rem_ref(ep->com.cm_id);
780 ep->com.cm_id = NULL;
781 ep->com.qp = NULL;
782 }
783}
784
785static void connect_reply_upcall(struct c4iw_ep *ep, int status)
786{
787 struct iw_cm_event event;
788
789 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status);
790 memset(&event, 0, sizeof(event));
791 event.event = IW_CM_EVENT_CONNECT_REPLY;
792 event.status = status;
793 event.local_addr = ep->com.local_addr;
794 event.remote_addr = ep->com.remote_addr;
795
796 if ((status == 0) || (status == -ECONNREFUSED)) {
797 event.private_data_len = ep->plen;
798 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
799 }
800 if (ep->com.cm_id) {
801 PDBG("%s ep %p tid %u status %d\n", __func__, ep,
802 ep->hwtid, status);
803 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
804 }
805 if (status < 0) {
806 ep->com.cm_id->rem_ref(ep->com.cm_id);
807 ep->com.cm_id = NULL;
808 ep->com.qp = NULL;
809 }
810}
811
812static void connect_request_upcall(struct c4iw_ep *ep)
813{
814 struct iw_cm_event event;
815
816 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
817 memset(&event, 0, sizeof(event));
818 event.event = IW_CM_EVENT_CONNECT_REQUEST;
819 event.local_addr = ep->com.local_addr;
820 event.remote_addr = ep->com.remote_addr;
821 event.private_data_len = ep->plen;
822 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
823 event.provider_data = ep;
824 if (state_read(&ep->parent_ep->com) != DEAD) {
825 c4iw_get_ep(&ep->com);
826 ep->parent_ep->com.cm_id->event_handler(
827 ep->parent_ep->com.cm_id,
828 &event);
829 }
830 c4iw_put_ep(&ep->parent_ep->com);
831 ep->parent_ep = NULL;
832}
833
834static void established_upcall(struct c4iw_ep *ep)
835{
836 struct iw_cm_event event;
837
838 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
839 memset(&event, 0, sizeof(event));
840 event.event = IW_CM_EVENT_ESTABLISHED;
841 if (ep->com.cm_id) {
842 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
843 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
844 }
845}
846
847static int update_rx_credits(struct c4iw_ep *ep, u32 credits)
848{
849 struct cpl_rx_data_ack *req;
850 struct sk_buff *skb;
851 int wrlen = roundup(sizeof *req, 16);
852
853 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits);
854 skb = get_skb(NULL, wrlen, GFP_KERNEL);
855 if (!skb) {
856 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n");
857 return 0;
858 }
859
860 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen);
861 memset(req, 0, wrlen);
862 INIT_TP_WR(req, ep->hwtid);
863 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
864 ep->hwtid));
865 req->credit_dack = cpu_to_be32(credits);
866 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->txq_idx);
867 c4iw_ofld_send(&ep->com.dev->rdev, skb);
868 return credits;
869}
870
871static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb)
872{
873 struct mpa_message *mpa;
874 u16 plen;
875 struct c4iw_qp_attributes attrs;
876 enum c4iw_qp_attr_mask mask;
877 int err;
878
879 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
880
881 /*
882 * Stop mpa timer. If it expired, then the state has
883 * changed and we bail since ep_timeout already aborted
884 * the connection.
885 */
886 stop_ep_timer(ep);
887 if (state_read(&ep->com) != MPA_REQ_SENT)
888 return;
889
890 /*
891 * If we get more than the supported amount of private data
892 * then we must fail this connection.
893 */
894 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
895 err = -EINVAL;
896 goto err;
897 }
898
899 /*
900 * copy the new data into our accumulation buffer.
901 */
902 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
903 skb->len);
904 ep->mpa_pkt_len += skb->len;
905
906 /*
907 * if we don't even have the mpa message, then bail.
908 */
909 if (ep->mpa_pkt_len < sizeof(*mpa))
910 return;
911 mpa = (struct mpa_message *) ep->mpa_pkt;
912
913 /* Validate MPA header. */
914 if (mpa->revision != mpa_rev) {
915 err = -EPROTO;
916 goto err;
917 }
918 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
919 err = -EPROTO;
920 goto err;
921 }
922
923 plen = ntohs(mpa->private_data_size);
924
925 /*
926 * Fail if there's too much private data.
927 */
928 if (plen > MPA_MAX_PRIVATE_DATA) {
929 err = -EPROTO;
930 goto err;
931 }
932
933 /*
934 * If plen does not account for pkt size
935 */
936 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
937 err = -EPROTO;
938 goto err;
939 }
940
941 ep->plen = (u8) plen;
942
943 /*
944 * If we don't have all the pdata yet, then bail.
945 * We'll continue process when more data arrives.
946 */
947 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
948 return;
949
950 if (mpa->flags & MPA_REJECT) {
951 err = -ECONNREFUSED;
952 goto err;
953 }
954
955 /*
956 * If we get here we have accumulated the entire mpa
957 * start reply message including private data. And
958 * the MPA header is valid.
959 */
960 state_set(&ep->com, FPDU_MODE);
961 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
962 ep->mpa_attr.recv_marker_enabled = markers_enabled;
963 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
964 ep->mpa_attr.version = mpa_rev;
965 ep->mpa_attr.p2p_type = peer2peer ? p2p_type :
966 FW_RI_INIT_P2PTYPE_DISABLED;
967 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
968 "xmit_marker_enabled=%d, version=%d\n", __func__,
969 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
970 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
971
972 attrs.mpa_attr = ep->mpa_attr;
973 attrs.max_ird = ep->ird;
974 attrs.max_ord = ep->ord;
975 attrs.llp_stream_handle = ep;
976 attrs.next_state = C4IW_QP_STATE_RTS;
977
978 mask = C4IW_QP_ATTR_NEXT_STATE |
979 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR |
980 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD;
981
982 /* bind QP and TID with INIT_WR */
983 err = c4iw_modify_qp(ep->com.qp->rhp,
984 ep->com.qp, mask, &attrs, 1);
985 if (err)
986 goto err;
987 goto out;
988err:
989 abort_connection(ep, skb, GFP_KERNEL);
990out:
991 connect_reply_upcall(ep, err);
992 return;
993}
994
995static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
996{
997 struct mpa_message *mpa;
998 u16 plen;
999
1000 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1001
1002 if (state_read(&ep->com) != MPA_REQ_WAIT)
1003 return;
1004
1005 /*
1006 * If we get more than the supported amount of private data
1007 * then we must fail this connection.
1008 */
1009 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
1010 stop_ep_timer(ep);
1011 abort_connection(ep, skb, GFP_KERNEL);
1012 return;
1013 }
1014
1015 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
1016
1017 /*
1018 * Copy the new data into our accumulation buffer.
1019 */
1020 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
1021 skb->len);
1022 ep->mpa_pkt_len += skb->len;
1023
1024 /*
1025 * If we don't even have the mpa message, then bail.
1026 * We'll continue process when more data arrives.
1027 */
1028 if (ep->mpa_pkt_len < sizeof(*mpa))
1029 return;
1030
1031 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
1032 stop_ep_timer(ep);
1033 mpa = (struct mpa_message *) ep->mpa_pkt;
1034
1035 /*
1036 * Validate MPA Header.
1037 */
1038 if (mpa->revision != mpa_rev) {
1039 abort_connection(ep, skb, GFP_KERNEL);
1040 return;
1041 }
1042
1043 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
1044 abort_connection(ep, skb, GFP_KERNEL);
1045 return;
1046 }
1047
1048 plen = ntohs(mpa->private_data_size);
1049
1050 /*
1051 * Fail if there's too much private data.
1052 */
1053 if (plen > MPA_MAX_PRIVATE_DATA) {
1054 abort_connection(ep, skb, GFP_KERNEL);
1055 return;
1056 }
1057
1058 /*
1059 * If plen does not account for pkt size
1060 */
1061 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
1062 abort_connection(ep, skb, GFP_KERNEL);
1063 return;
1064 }
1065 ep->plen = (u8) plen;
1066
1067 /*
1068 * If we don't have all the pdata yet, then bail.
1069 */
1070 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
1071 return;
1072
1073 /*
1074 * If we get here we have accumulated the entire mpa
1075 * start reply message including private data.
1076 */
1077 ep->mpa_attr.initiator = 0;
1078 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
1079 ep->mpa_attr.recv_marker_enabled = markers_enabled;
1080 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
1081 ep->mpa_attr.version = mpa_rev;
1082 ep->mpa_attr.p2p_type = peer2peer ? p2p_type :
1083 FW_RI_INIT_P2PTYPE_DISABLED;
1084 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
1085 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__,
1086 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
1087 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version,
1088 ep->mpa_attr.p2p_type);
1089
1090 state_set(&ep->com, MPA_REQ_RCVD);
1091
1092 /* drive upcall */
1093 connect_request_upcall(ep);
1094 return;
1095}
1096
1097static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb)
1098{
1099 struct c4iw_ep *ep;
1100 struct cpl_rx_data *hdr = cplhdr(skb);
1101 unsigned int dlen = ntohs(hdr->len);
1102 unsigned int tid = GET_TID(hdr);
1103 struct tid_info *t = dev->rdev.lldi.tids;
1104
1105 ep = lookup_tid(t, tid);
1106 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen);
1107 skb_pull(skb, sizeof(*hdr));
1108 skb_trim(skb, dlen);
1109
1110 ep->rcv_seq += dlen;
1111 BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen));
1112
1113 /* update RX credits */
1114 update_rx_credits(ep, dlen);
1115
1116 switch (state_read(&ep->com)) {
1117 case MPA_REQ_SENT:
1118 process_mpa_reply(ep, skb);
1119 break;
1120 case MPA_REQ_WAIT:
1121 process_mpa_request(ep, skb);
1122 break;
1123 case MPA_REP_SENT:
1124 break;
1125 default:
1126 printk(KERN_ERR MOD "%s Unexpected streaming data."
1127 " ep %p state %d tid %u\n",
1128 __func__, ep, state_read(&ep->com), ep->hwtid);
1129
1130 /*
1131 * The ep will timeout and inform the ULP of the failure.
1132 * See ep_timeout().
1133 */
1134 break;
1135 }
1136 return 0;
1137}
1138
1139static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1140{
1141 struct c4iw_ep *ep;
1142 struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1143 unsigned long flags;
1144 int release = 0;
1145 unsigned int tid = GET_TID(rpl);
1146 struct tid_info *t = dev->rdev.lldi.tids;
1147
1148 ep = lookup_tid(t, tid);
1149 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1150 BUG_ON(!ep);
1151 spin_lock_irqsave(&ep->com.lock, flags);
1152 switch (ep->com.state) {
1153 case ABORTING:
1154 __state_set(&ep->com, DEAD);
1155 release = 1;
1156 break;
1157 default:
1158 printk(KERN_ERR "%s ep %p state %d\n",
1159 __func__, ep, ep->com.state);
1160 break;
1161 }
1162 spin_unlock_irqrestore(&ep->com.lock, flags);
1163
1164 if (release)
1165 release_ep_resources(ep);
1166 return 0;
1167}
1168
1169/*
1170 * Return whether a failed active open has allocated a TID
1171 */
1172static inline int act_open_has_tid(int status)
1173{
1174 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
1175 status != CPL_ERR_ARP_MISS;
1176}
1177
1178static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1179{
1180 struct c4iw_ep *ep;
1181 struct cpl_act_open_rpl *rpl = cplhdr(skb);
1182 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID(
1183 ntohl(rpl->atid_status)));
1184 struct tid_info *t = dev->rdev.lldi.tids;
1185 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status));
1186
1187 ep = lookup_atid(t, atid);
1188
1189 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid,
1190 status, status2errno(status));
1191
1192 if (status == CPL_ERR_RTX_NEG_ADVICE) {
1193 printk(KERN_WARNING MOD "Connection problems for atid %u\n",
1194 atid);
1195 return 0;
1196 }
1197
1198 connect_reply_upcall(ep, status2errno(status));
1199 state_set(&ep->com, DEAD);
1200
1201 if (status && act_open_has_tid(status))
1202 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl));
1203
1204 cxgb4_free_atid(t, atid);
1205 dst_release(ep->dst);
1206 cxgb4_l2t_release(ep->l2t);
1207 c4iw_put_ep(&ep->com);
1208
1209 return 0;
1210}
1211
1212static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1213{
1214 struct cpl_pass_open_rpl *rpl = cplhdr(skb);
1215 struct tid_info *t = dev->rdev.lldi.tids;
1216 unsigned int stid = GET_TID(rpl);
1217 struct c4iw_listen_ep *ep = lookup_stid(t, stid);
1218
1219 if (!ep) {
1220 printk(KERN_ERR MOD "stid %d lookup failure!\n", stid);
1221 return 0;
1222 }
1223 PDBG("%s ep %p status %d error %d\n", __func__, ep,
1224 rpl->status, status2errno(rpl->status));
1225 ep->com.rpl_err = status2errno(rpl->status);
1226 ep->com.rpl_done = 1;
1227 wake_up(&ep->com.waitq);
1228
1229 return 0;
1230}
1231
1232static int listen_stop(struct c4iw_listen_ep *ep)
1233{
1234 struct sk_buff *skb;
1235 struct cpl_close_listsvr_req *req;
1236
1237 PDBG("%s ep %p\n", __func__, ep);
1238 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
1239 if (!skb) {
1240 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
1241 return -ENOMEM;
1242 }
1243 req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req));
1244 INIT_TP_WR(req, 0);
1245 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ,
1246 ep->stid));
1247 req->reply_ctrl = cpu_to_be16(
1248 QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0]));
1249 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
1250 return c4iw_ofld_send(&ep->com.dev->rdev, skb);
1251}
1252
1253static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1254{
1255 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb);
1256 struct tid_info *t = dev->rdev.lldi.tids;
1257 unsigned int stid = GET_TID(rpl);
1258 struct c4iw_listen_ep *ep = lookup_stid(t, stid);
1259
1260 PDBG("%s ep %p\n", __func__, ep);
1261 ep->com.rpl_err = status2errno(rpl->status);
1262 ep->com.rpl_done = 1;
1263 wake_up(&ep->com.waitq);
1264 return 0;
1265}
1266
1267static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb,
1268 struct cpl_pass_accept_req *req)
1269{
1270 struct cpl_pass_accept_rpl *rpl;
1271 unsigned int mtu_idx;
1272 u64 opt0;
1273 u32 opt2;
1274 int wscale;
1275
1276 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1277 BUG_ON(skb_cloned(skb));
1278 skb_trim(skb, sizeof(*rpl));
1279 skb_get(skb);
1280 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx);
1281 wscale = compute_wscale(rcv_win);
1282 opt0 = KEEP_ALIVE(1) |
1283 WND_SCALE(wscale) |
1284 MSS_IDX(mtu_idx) |
1285 L2T_IDX(ep->l2t->idx) |
1286 TX_CHAN(ep->tx_chan) |
1287 SMAC_SEL(ep->smac_idx) |
1288 DSCP(ep->tos) |
1289 RCV_BUFSIZ(rcv_win>>10);
1290 opt2 = RX_CHANNEL(0) |
1291 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid);
1292
1293 if (enable_tcp_timestamps && req->tcpopt.tstamp)
1294 opt2 |= TSTAMPS_EN(1);
1295 if (enable_tcp_sack && req->tcpopt.sack)
1296 opt2 |= SACK_EN(1);
1297 if (wscale && enable_tcp_window_scaling)
1298 opt2 |= WND_SCALE_EN(1);
1299
1300 rpl = cplhdr(skb);
1301 INIT_TP_WR(rpl, ep->hwtid);
1302 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1303 ep->hwtid));
1304 rpl->opt0 = cpu_to_be64(opt0);
1305 rpl->opt2 = cpu_to_be32(opt2);
1306 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx);
1307 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
1308
1309 return;
1310}
1311
1312static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip,
1313 struct sk_buff *skb)
1314{
1315 PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid,
1316 peer_ip);
1317 BUG_ON(skb_cloned(skb));
1318 skb_trim(skb, sizeof(struct cpl_tid_release));
1319 skb_get(skb);
1320 release_tid(&dev->rdev, hwtid, skb);
1321 return;
1322}
1323
1324static void get_4tuple(struct cpl_pass_accept_req *req,
1325 __be32 *local_ip, __be32 *peer_ip,
1326 __be16 *local_port, __be16 *peer_port)
1327{
1328 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len));
1329 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len));
1330 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len);
1331 struct tcphdr *tcp = (struct tcphdr *)
1332 ((u8 *)(req + 1) + eth_len + ip_len);
1333
1334 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__,
1335 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source),
1336 ntohs(tcp->dest));
1337
1338 *peer_ip = ip->saddr;
1339 *local_ip = ip->daddr;
1340 *peer_port = tcp->source;
1341 *local_port = tcp->dest;
1342
1343 return;
1344}
1345
1346static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
1347{
1348 struct c4iw_ep *child_ep, *parent_ep;
1349 struct cpl_pass_accept_req *req = cplhdr(skb);
1350 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid));
1351 struct tid_info *t = dev->rdev.lldi.tids;
1352 unsigned int hwtid = GET_TID(req);
1353 struct dst_entry *dst;
1354 struct l2t_entry *l2t;
1355 struct rtable *rt;
1356 __be32 local_ip, peer_ip;
1357 __be16 local_port, peer_port;
1358 struct net_device *pdev;
1359 u32 tx_chan, smac_idx;
1360 u16 rss_qid;
1361 u32 mtu;
1362 int step;
1363 int txq_idx;
1364
1365 parent_ep = lookup_stid(t, stid);
1366 PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid);
1367
1368 get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port);
1369
1370 if (state_read(&parent_ep->com) != LISTEN) {
1371 printk(KERN_ERR "%s - listening ep not in LISTEN\n",
1372 __func__);
1373 goto reject;
1374 }
1375
1376 /* Find output route */
1377 rt = find_route(dev, local_ip, peer_ip, local_port, peer_port,
1378 GET_POPEN_TOS(ntohl(req->tos_stid)));
1379 if (!rt) {
1380 printk(KERN_ERR MOD "%s - failed to find dst entry!\n",
1381 __func__);
1382 goto reject;
1383 }
1384 dst = &rt->u.dst;
1385 if (dst->neighbour->dev->flags & IFF_LOOPBACK) {
1386 pdev = ip_dev_find(&init_net, peer_ip);
1387 BUG_ON(!pdev);
1388 l2t = cxgb4_l2t_get(dev->rdev.lldi.l2t, dst->neighbour,
1389 pdev, 0);
1390 mtu = pdev->mtu;
1391 tx_chan = cxgb4_port_chan(pdev);
1392 smac_idx = tx_chan << 1;
1393 step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan;
1394 txq_idx = cxgb4_port_idx(pdev) * step;
1395 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan;
1396 rss_qid = dev->rdev.lldi.rxq_ids[cxgb4_port_idx(pdev) * step];
1397 dev_put(pdev);
1398 } else {
1399 l2t = cxgb4_l2t_get(dev->rdev.lldi.l2t, dst->neighbour,
1400 dst->neighbour->dev, 0);
1401 mtu = dst_mtu(dst);
1402 tx_chan = cxgb4_port_chan(dst->neighbour->dev);
1403 smac_idx = tx_chan << 1;
1404 step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan;
1405 txq_idx = cxgb4_port_idx(dst->neighbour->dev) * step;
1406 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan;
1407 rss_qid = dev->rdev.lldi.rxq_ids[
1408 cxgb4_port_idx(dst->neighbour->dev) * step];
1409 }
1410 if (!l2t) {
1411 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n",
1412 __func__);
1413 dst_release(dst);
1414 goto reject;
1415 }
1416
1417 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL);
1418 if (!child_ep) {
1419 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n",
1420 __func__);
1421 cxgb4_l2t_release(l2t);
1422 dst_release(dst);
1423 goto reject;
1424 }
1425 state_set(&child_ep->com, CONNECTING);
1426 child_ep->com.dev = dev;
1427 child_ep->com.cm_id = NULL;
1428 child_ep->com.local_addr.sin_family = PF_INET;
1429 child_ep->com.local_addr.sin_port = local_port;
1430 child_ep->com.local_addr.sin_addr.s_addr = local_ip;
1431 child_ep->com.remote_addr.sin_family = PF_INET;
1432 child_ep->com.remote_addr.sin_port = peer_port;
1433 child_ep->com.remote_addr.sin_addr.s_addr = peer_ip;
1434 c4iw_get_ep(&parent_ep->com);
1435 child_ep->parent_ep = parent_ep;
1436 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid));
1437 child_ep->l2t = l2t;
1438 child_ep->dst = dst;
1439 child_ep->hwtid = hwtid;
1440 child_ep->tx_chan = tx_chan;
1441 child_ep->smac_idx = smac_idx;
1442 child_ep->rss_qid = rss_qid;
1443 child_ep->mtu = mtu;
1444 child_ep->txq_idx = txq_idx;
1445
1446 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__,
1447 tx_chan, smac_idx, rss_qid);
1448
1449 init_timer(&child_ep->timer);
1450 cxgb4_insert_tid(t, child_ep, hwtid);
1451 accept_cr(child_ep, peer_ip, skb, req);
1452 goto out;
1453reject:
1454 reject_cr(dev, hwtid, peer_ip, skb);
1455out:
1456 return 0;
1457}
1458
1459static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb)
1460{
1461 struct c4iw_ep *ep;
1462 struct cpl_pass_establish *req = cplhdr(skb);
1463 struct tid_info *t = dev->rdev.lldi.tids;
1464 unsigned int tid = GET_TID(req);
1465
1466 ep = lookup_tid(t, tid);
1467 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1468 ep->snd_seq = be32_to_cpu(req->snd_isn);
1469 ep->rcv_seq = be32_to_cpu(req->rcv_isn);
1470
1471 set_emss(ep, ntohs(req->tcp_opt));
1472
1473 dst_confirm(ep->dst);
1474 state_set(&ep->com, MPA_REQ_WAIT);
1475 start_ep_timer(ep);
1476 send_flowc(ep, skb);
1477
1478 return 0;
1479}
1480
1481static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
1482{
1483 struct cpl_peer_close *hdr = cplhdr(skb);
1484 struct c4iw_ep *ep;
1485 struct c4iw_qp_attributes attrs;
1486 unsigned long flags;
1487 int disconnect = 1;
1488 int release = 0;
1489 int closing = 0;
1490 struct tid_info *t = dev->rdev.lldi.tids;
1491 unsigned int tid = GET_TID(hdr);
1492 int start_timer = 0;
1493 int stop_timer = 0;
1494
1495 ep = lookup_tid(t, tid);
1496 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1497 dst_confirm(ep->dst);
1498
1499 spin_lock_irqsave(&ep->com.lock, flags);
1500 switch (ep->com.state) {
1501 case MPA_REQ_WAIT:
1502 __state_set(&ep->com, CLOSING);
1503 break;
1504 case MPA_REQ_SENT:
1505 __state_set(&ep->com, CLOSING);
1506 connect_reply_upcall(ep, -ECONNRESET);
1507 break;
1508 case MPA_REQ_RCVD:
1509
1510 /*
1511 * We're gonna mark this puppy DEAD, but keep
1512 * the reference on it until the ULP accepts or
1513 * rejects the CR. Also wake up anyone waiting
1514 * in rdma connection migration (see c4iw_accept_cr()).
1515 */
1516 __state_set(&ep->com, CLOSING);
1517 ep->com.rpl_done = 1;
1518 ep->com.rpl_err = -ECONNRESET;
1519 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid);
1520 wake_up(&ep->com.waitq);
1521 break;
1522 case MPA_REP_SENT:
1523 __state_set(&ep->com, CLOSING);
1524 ep->com.rpl_done = 1;
1525 ep->com.rpl_err = -ECONNRESET;
1526 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid);
1527 wake_up(&ep->com.waitq);
1528 break;
1529 case FPDU_MODE:
1530 start_timer = 1;
1531 __state_set(&ep->com, CLOSING);
1532 closing = 1;
1533 peer_close_upcall(ep);
1534 break;
1535 case ABORTING:
1536 disconnect = 0;
1537 break;
1538 case CLOSING:
1539 __state_set(&ep->com, MORIBUND);
1540 disconnect = 0;
1541 break;
1542 case MORIBUND:
1543 stop_timer = 1;
1544 if (ep->com.cm_id && ep->com.qp) {
1545 attrs.next_state = C4IW_QP_STATE_IDLE;
1546 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
1547 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
1548 }
1549 close_complete_upcall(ep);
1550 __state_set(&ep->com, DEAD);
1551 release = 1;
1552 disconnect = 0;
1553 break;
1554 case DEAD:
1555 disconnect = 0;
1556 break;
1557 default:
1558 BUG_ON(1);
1559 }
1560 spin_unlock_irqrestore(&ep->com.lock, flags);
1561 if (closing) {
1562 attrs.next_state = C4IW_QP_STATE_CLOSING;
1563 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
1564 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
1565 }
1566 if (start_timer)
1567 start_ep_timer(ep);
1568 if (stop_timer)
1569 stop_ep_timer(ep);
1570 if (disconnect)
1571 c4iw_ep_disconnect(ep, 0, GFP_KERNEL);
1572 if (release)
1573 release_ep_resources(ep);
1574 return 0;
1575}
1576
1577/*
1578 * Returns whether an ABORT_REQ_RSS message is a negative advice.
1579 */
1580static int is_neg_adv_abort(unsigned int status)
1581{
1582 return status == CPL_ERR_RTX_NEG_ADVICE ||
1583 status == CPL_ERR_PERSIST_NEG_ADVICE;
1584}
1585
1586static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
1587{
1588 struct cpl_abort_req_rss *req = cplhdr(skb);
1589 struct c4iw_ep *ep;
1590 struct cpl_abort_rpl *rpl;
1591 struct sk_buff *rpl_skb;
1592 struct c4iw_qp_attributes attrs;
1593 int ret;
1594 int release = 0;
1595 unsigned long flags;
1596 struct tid_info *t = dev->rdev.lldi.tids;
1597 unsigned int tid = GET_TID(req);
1598 int stop_timer = 0;
1599
1600 ep = lookup_tid(t, tid);
1601 if (is_neg_adv_abort(req->status)) {
1602 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep,
1603 ep->hwtid);
1604 return 0;
1605 }
1606 spin_lock_irqsave(&ep->com.lock, flags);
1607 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid,
1608 ep->com.state);
1609 switch (ep->com.state) {
1610 case CONNECTING:
1611 break;
1612 case MPA_REQ_WAIT:
1613 stop_timer = 1;
1614 break;
1615 case MPA_REQ_SENT:
1616 stop_timer = 1;
1617 connect_reply_upcall(ep, -ECONNRESET);
1618 break;
1619 case MPA_REP_SENT:
1620 ep->com.rpl_done = 1;
1621 ep->com.rpl_err = -ECONNRESET;
1622 PDBG("waking up ep %p\n", ep);
1623 wake_up(&ep->com.waitq);
1624 break;
1625 case MPA_REQ_RCVD:
1626
1627 /*
1628 * We're gonna mark this puppy DEAD, but keep
1629 * the reference on it until the ULP accepts or
1630 * rejects the CR. Also wake up anyone waiting
1631 * in rdma connection migration (see c4iw_accept_cr()).
1632 */
1633 ep->com.rpl_done = 1;
1634 ep->com.rpl_err = -ECONNRESET;
1635 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid);
1636 wake_up(&ep->com.waitq);
1637 break;
1638 case MORIBUND:
1639 case CLOSING:
1640 stop_timer = 1;
1641 /*FALLTHROUGH*/
1642 case FPDU_MODE:
1643 if (ep->com.cm_id && ep->com.qp) {
1644 attrs.next_state = C4IW_QP_STATE_ERROR;
1645 ret = c4iw_modify_qp(ep->com.qp->rhp,
1646 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE,
1647 &attrs, 1);
1648 if (ret)
1649 printk(KERN_ERR MOD
1650 "%s - qp <- error failed!\n",
1651 __func__);
1652 }
1653 peer_abort_upcall(ep);
1654 break;
1655 case ABORTING:
1656 break;
1657 case DEAD:
1658 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__);
1659 spin_unlock_irqrestore(&ep->com.lock, flags);
1660 return 0;
1661 default:
1662 BUG_ON(1);
1663 break;
1664 }
1665 dst_confirm(ep->dst);
1666 if (ep->com.state != ABORTING) {
1667 __state_set(&ep->com, DEAD);
1668 release = 1;
1669 }
1670 spin_unlock_irqrestore(&ep->com.lock, flags);
1671
1672 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);
1673 if (!rpl_skb) {
1674 printk(KERN_ERR MOD "%s - cannot allocate skb!\n",
1675 __func__);
1676 release = 1;
1677 goto out;
1678 }
1679 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
1680 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl));
1681 INIT_TP_WR(rpl, ep->hwtid);
1682 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid));
1683 rpl->cmd = CPL_ABORT_NO_RST;
1684 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb);
1685out:
1686 if (stop_timer)
1687 stop_ep_timer(ep);
1688 if (release)
1689 release_ep_resources(ep);
1690 return 0;
1691}
1692
1693static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1694{
1695 struct c4iw_ep *ep;
1696 struct c4iw_qp_attributes attrs;
1697 struct cpl_close_con_rpl *rpl = cplhdr(skb);
1698 unsigned long flags;
1699 int release = 0;
1700 struct tid_info *t = dev->rdev.lldi.tids;
1701 unsigned int tid = GET_TID(rpl);
1702 int stop_timer = 0;
1703
1704 ep = lookup_tid(t, tid);
1705
1706 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1707 BUG_ON(!ep);
1708
1709 /* The cm_id may be null if we failed to connect */
1710 spin_lock_irqsave(&ep->com.lock, flags);
1711 switch (ep->com.state) {
1712 case CLOSING:
1713 __state_set(&ep->com, MORIBUND);
1714 break;
1715 case MORIBUND:
1716 stop_timer = 1;
1717 if ((ep->com.cm_id) && (ep->com.qp)) {
1718 attrs.next_state = C4IW_QP_STATE_IDLE;
1719 c4iw_modify_qp(ep->com.qp->rhp,
1720 ep->com.qp,
1721 C4IW_QP_ATTR_NEXT_STATE,
1722 &attrs, 1);
1723 }
1724 close_complete_upcall(ep);
1725 __state_set(&ep->com, DEAD);
1726 release = 1;
1727 break;
1728 case ABORTING:
1729 case DEAD:
1730 break;
1731 default:
1732 BUG_ON(1);
1733 break;
1734 }
1735 spin_unlock_irqrestore(&ep->com.lock, flags);
1736 if (stop_timer)
1737 stop_ep_timer(ep);
1738 if (release)
1739 release_ep_resources(ep);
1740 return 0;
1741}
1742
1743static int terminate(struct c4iw_dev *dev, struct sk_buff *skb)
1744{
1745 struct c4iw_ep *ep;
1746 struct cpl_rdma_terminate *term = cplhdr(skb);
1747 struct tid_info *t = dev->rdev.lldi.tids;
1748 unsigned int tid = GET_TID(term);
1749
1750 ep = lookup_tid(t, tid);
1751
1752 if (state_read(&ep->com) != FPDU_MODE)
1753 return 0;
1754
1755 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1756 skb_pull(skb, sizeof *term);
1757 PDBG("%s saving %d bytes of term msg\n", __func__, skb->len);
1758 skb_copy_from_linear_data(skb, ep->com.qp->attr.terminate_buffer,
1759 skb->len);
1760 ep->com.qp->attr.terminate_msg_len = skb->len;
1761 ep->com.qp->attr.is_terminate_local = 0;
1762 return 0;
1763}
1764
1765/*
1766 * Upcall from the adapter indicating data has been transmitted.
1767 * For us its just the single MPA request or reply. We can now free
1768 * the skb holding the mpa message.
1769 */
1770static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb)
1771{
1772 struct c4iw_ep *ep;
1773 struct cpl_fw4_ack *hdr = cplhdr(skb);
1774 u8 credits = hdr->credits;
1775 unsigned int tid = GET_TID(hdr);
1776 struct tid_info *t = dev->rdev.lldi.tids;
1777
1778
1779 ep = lookup_tid(t, tid);
1780 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits);
1781 if (credits == 0) {
1782 PDBG(KERN_ERR "%s 0 credit ack ep %p tid %u state %u\n",
1783 __func__, ep, ep->hwtid, state_read(&ep->com));
1784 return 0;
1785 }
1786
1787 dst_confirm(ep->dst);
1788 if (ep->mpa_skb) {
1789 PDBG("%s last streaming msg ack ep %p tid %u state %u "
1790 "initiator %u freeing skb\n", __func__, ep, ep->hwtid,
1791 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0);
1792 kfree_skb(ep->mpa_skb);
1793 ep->mpa_skb = NULL;
1794 }
1795 return 0;
1796}
1797
1798static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb)
1799{
1800 struct cpl_fw6_msg *rpl = cplhdr(skb);
1801 struct c4iw_wr_wait *wr_waitp;
1802 int ret;
1803
1804 PDBG("%s type %u\n", __func__, rpl->type);
1805
1806 switch (rpl->type) {
1807 case 1:
1808 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff);
1809 wr_waitp = (__force struct c4iw_wr_wait *)rpl->data[1];
1810 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret);
1811 if (wr_waitp) {
1812 wr_waitp->ret = ret;
1813 wr_waitp->done = 1;
1814 wake_up(&wr_waitp->wait);
1815 }
1816 break;
1817 case 2:
1818 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]);
1819 break;
1820 default:
1821 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__,
1822 rpl->type);
1823 break;
1824 }
1825 return 0;
1826}
1827
1828static void ep_timeout(unsigned long arg)
1829{
1830 struct c4iw_ep *ep = (struct c4iw_ep *)arg;
1831 struct c4iw_qp_attributes attrs;
1832 unsigned long flags;
1833 int abort = 1;
1834
1835 spin_lock_irqsave(&ep->com.lock, flags);
1836 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid,
1837 ep->com.state);
1838 switch (ep->com.state) {
1839 case MPA_REQ_SENT:
1840 __state_set(&ep->com, ABORTING);
1841 connect_reply_upcall(ep, -ETIMEDOUT);
1842 break;
1843 case MPA_REQ_WAIT:
1844 __state_set(&ep->com, ABORTING);
1845 break;
1846 case CLOSING:
1847 case MORIBUND:
1848 if (ep->com.cm_id && ep->com.qp) {
1849 attrs.next_state = C4IW_QP_STATE_ERROR;
1850 c4iw_modify_qp(ep->com.qp->rhp,
1851 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE,
1852 &attrs, 1);
1853 }
1854 __state_set(&ep->com, ABORTING);
1855 break;
1856 default:
1857 printk(KERN_ERR "%s unexpected state ep %p tid %u state %u\n",
1858 __func__, ep, ep->hwtid, ep->com.state);
1859 WARN_ON(1);
1860 abort = 0;
1861 }
1862 spin_unlock_irqrestore(&ep->com.lock, flags);
1863 if (abort)
1864 abort_connection(ep, NULL, GFP_ATOMIC);
1865 c4iw_put_ep(&ep->com);
1866}
1867
1868int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
1869{
1870 int err;
1871 struct c4iw_ep *ep = to_ep(cm_id);
1872 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1873
1874 if (state_read(&ep->com) == DEAD) {
1875 c4iw_put_ep(&ep->com);
1876 return -ECONNRESET;
1877 }
1878 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
1879 if (mpa_rev == 0)
1880 abort_connection(ep, NULL, GFP_KERNEL);
1881 else {
1882 err = send_mpa_reject(ep, pdata, pdata_len);
1883 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL);
1884 }
1885 c4iw_put_ep(&ep->com);
1886 return 0;
1887}
1888
1889int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1890{
1891 int err;
1892 struct c4iw_qp_attributes attrs;
1893 enum c4iw_qp_attr_mask mask;
1894 struct c4iw_ep *ep = to_ep(cm_id);
1895 struct c4iw_dev *h = to_c4iw_dev(cm_id->device);
1896 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn);
1897
1898 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1899 if (state_read(&ep->com) == DEAD) {
1900 err = -ECONNRESET;
1901 goto err;
1902 }
1903
1904 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
1905 BUG_ON(!qp);
1906
1907 if ((conn_param->ord > T4_MAX_READ_DEPTH) ||
1908 (conn_param->ird > T4_MAX_READ_DEPTH)) {
1909 abort_connection(ep, NULL, GFP_KERNEL);
1910 err = -EINVAL;
1911 goto err;
1912 }
1913
1914 cm_id->add_ref(cm_id);
1915 ep->com.cm_id = cm_id;
1916 ep->com.qp = qp;
1917
1918 ep->ird = conn_param->ird;
1919 ep->ord = conn_param->ord;
1920
1921 if (peer2peer && ep->ird == 0)
1922 ep->ird = 1;
1923
1924 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord);
1925
1926 /* bind QP to EP and move to RTS */
1927 attrs.mpa_attr = ep->mpa_attr;
1928 attrs.max_ird = ep->ird;
1929 attrs.max_ord = ep->ord;
1930 attrs.llp_stream_handle = ep;
1931 attrs.next_state = C4IW_QP_STATE_RTS;
1932
1933 /* bind QP and TID with INIT_WR */
1934 mask = C4IW_QP_ATTR_NEXT_STATE |
1935 C4IW_QP_ATTR_LLP_STREAM_HANDLE |
1936 C4IW_QP_ATTR_MPA_ATTR |
1937 C4IW_QP_ATTR_MAX_IRD |
1938 C4IW_QP_ATTR_MAX_ORD;
1939
1940 err = c4iw_modify_qp(ep->com.qp->rhp,
1941 ep->com.qp, mask, &attrs, 1);
1942 if (err)
1943 goto err1;
1944 err = send_mpa_reply(ep, conn_param->private_data,
1945 conn_param->private_data_len);
1946 if (err)
1947 goto err1;
1948
1949 state_set(&ep->com, FPDU_MODE);
1950 established_upcall(ep);
1951 c4iw_put_ep(&ep->com);
1952 return 0;
1953err1:
1954 ep->com.cm_id = NULL;
1955 ep->com.qp = NULL;
1956 cm_id->rem_ref(cm_id);
1957err:
1958 c4iw_put_ep(&ep->com);
1959 return err;
1960}
1961
1962int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1963{
1964 int err = 0;
1965 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device);
1966 struct c4iw_ep *ep;
1967 struct rtable *rt;
1968 struct net_device *pdev;
1969 int step;
1970
1971 ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
1972 if (!ep) {
1973 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
1974 err = -ENOMEM;
1975 goto out;
1976 }
1977 init_timer(&ep->timer);
1978 ep->plen = conn_param->private_data_len;
1979 if (ep->plen)
1980 memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
1981 conn_param->private_data, ep->plen);
1982 ep->ird = conn_param->ird;
1983 ep->ord = conn_param->ord;
1984
1985 if (peer2peer && ep->ord == 0)
1986 ep->ord = 1;
1987
1988 cm_id->add_ref(cm_id);
1989 ep->com.dev = dev;
1990 ep->com.cm_id = cm_id;
1991 ep->com.qp = get_qhp(dev, conn_param->qpn);
1992 BUG_ON(!ep->com.qp);
1993 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn,
1994 ep->com.qp, cm_id);
1995
1996 /*
1997 * Allocate an active TID to initiate a TCP connection.
1998 */
1999 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep);
2000 if (ep->atid == -1) {
2001 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
2002 err = -ENOMEM;
2003 goto fail2;
2004 }
2005
2006 PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__,
2007 ntohl(cm_id->local_addr.sin_addr.s_addr),
2008 ntohs(cm_id->local_addr.sin_port),
2009 ntohl(cm_id->remote_addr.sin_addr.s_addr),
2010 ntohs(cm_id->remote_addr.sin_port));
2011
2012 /* find a route */
2013 rt = find_route(dev,
2014 cm_id->local_addr.sin_addr.s_addr,
2015 cm_id->remote_addr.sin_addr.s_addr,
2016 cm_id->local_addr.sin_port,
2017 cm_id->remote_addr.sin_port, 0);
2018 if (!rt) {
2019 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
2020 err = -EHOSTUNREACH;
2021 goto fail3;
2022 }
2023 ep->dst = &rt->u.dst;
2024
2025 /* get a l2t entry */
2026 if (ep->dst->neighbour->dev->flags & IFF_LOOPBACK) {
2027 PDBG("%s LOOPBACK\n", __func__);
2028 pdev = ip_dev_find(&init_net,
2029 cm_id->remote_addr.sin_addr.s_addr);
2030 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t,
2031 ep->dst->neighbour,
2032 pdev, 0);
2033 ep->mtu = pdev->mtu;
2034 ep->tx_chan = cxgb4_port_chan(pdev);
2035 ep->smac_idx = ep->tx_chan << 1;
2036 step = ep->com.dev->rdev.lldi.ntxq /
2037 ep->com.dev->rdev.lldi.nchan;
2038 ep->txq_idx = cxgb4_port_idx(pdev) * step;
2039 step = ep->com.dev->rdev.lldi.nrxq /
2040 ep->com.dev->rdev.lldi.nchan;
2041 ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[
2042 cxgb4_port_idx(pdev) * step];
2043 dev_put(pdev);
2044 } else {
2045 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t,
2046 ep->dst->neighbour,
2047 ep->dst->neighbour->dev, 0);
2048 ep->mtu = dst_mtu(ep->dst);
2049 ep->tx_chan = cxgb4_port_chan(ep->dst->neighbour->dev);
2050 ep->smac_idx = ep->tx_chan << 1;
2051 step = ep->com.dev->rdev.lldi.ntxq /
2052 ep->com.dev->rdev.lldi.nchan;
2053 ep->txq_idx = cxgb4_port_idx(ep->dst->neighbour->dev) * step;
2054 step = ep->com.dev->rdev.lldi.nrxq /
2055 ep->com.dev->rdev.lldi.nchan;
2056 ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[
2057 cxgb4_port_idx(ep->dst->neighbour->dev) * step];
2058 }
2059 if (!ep->l2t) {
2060 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__);
2061 err = -ENOMEM;
2062 goto fail4;
2063 }
2064
2065 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n",
2066 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid,
2067 ep->l2t->idx);
2068
2069 state_set(&ep->com, CONNECTING);
2070 ep->tos = 0;
2071 ep->com.local_addr = cm_id->local_addr;
2072 ep->com.remote_addr = cm_id->remote_addr;
2073
2074 /* send connect request to rnic */
2075 err = send_connect(ep);
2076 if (!err)
2077 goto out;
2078
2079 cxgb4_l2t_release(ep->l2t);
2080fail4:
2081 dst_release(ep->dst);
2082fail3:
2083 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid);
2084fail2:
2085 cm_id->rem_ref(cm_id);
2086 c4iw_put_ep(&ep->com);
2087out:
2088 return err;
2089}
2090
2091int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
2092{
2093 int err = 0;
2094 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device);
2095 struct c4iw_listen_ep *ep;
2096
2097
2098 might_sleep();
2099
2100 ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
2101 if (!ep) {
2102 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
2103 err = -ENOMEM;
2104 goto fail1;
2105 }
2106 PDBG("%s ep %p\n", __func__, ep);
2107 cm_id->add_ref(cm_id);
2108 ep->com.cm_id = cm_id;
2109 ep->com.dev = dev;
2110 ep->backlog = backlog;
2111 ep->com.local_addr = cm_id->local_addr;
2112
2113 /*
2114 * Allocate a server TID.
2115 */
2116 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep);
2117 if (ep->stid == -1) {
2118 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
2119 err = -ENOMEM;
2120 goto fail2;
2121 }
2122
2123 state_set(&ep->com, LISTEN);
2124 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], ep->stid,
2125 ep->com.local_addr.sin_addr.s_addr,
2126 ep->com.local_addr.sin_port,
2127 ep->com.dev->rdev.lldi.rxq_ids[0]);
2128 if (err)
2129 goto fail3;
2130
2131 /* wait for pass_open_rpl */
2132 wait_event(ep->com.waitq, ep->com.rpl_done);
2133 err = ep->com.rpl_err;
2134 if (!err) {
2135 cm_id->provider_data = ep;
2136 goto out;
2137 }
2138fail3:
2139 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET);
2140fail2:
2141 cm_id->rem_ref(cm_id);
2142 c4iw_put_ep(&ep->com);
2143fail1:
2144out:
2145 return err;
2146}
2147
2148int c4iw_destroy_listen(struct iw_cm_id *cm_id)
2149{
2150 int err;
2151 struct c4iw_listen_ep *ep = to_listen_ep(cm_id);
2152
2153 PDBG("%s ep %p\n", __func__, ep);
2154
2155 might_sleep();
2156 state_set(&ep->com, DEAD);
2157 ep->com.rpl_done = 0;
2158 ep->com.rpl_err = 0;
2159 err = listen_stop(ep);
2160 if (err)
2161 goto done;
2162 wait_event(ep->com.waitq, ep->com.rpl_done);
2163 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET);
2164done:
2165 err = ep->com.rpl_err;
2166 cm_id->rem_ref(cm_id);
2167 c4iw_put_ep(&ep->com);
2168 return err;
2169}
2170
2171int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
2172{
2173 int ret = 0;
2174 unsigned long flags;
2175 int close = 0;
2176 int fatal = 0;
2177 struct c4iw_rdev *rdev;
2178 int start_timer = 0;
2179 int stop_timer = 0;
2180
2181 spin_lock_irqsave(&ep->com.lock, flags);
2182
2183 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep,
2184 states[ep->com.state], abrupt);
2185
2186 rdev = &ep->com.dev->rdev;
2187 if (c4iw_fatal_error(rdev)) {
2188 fatal = 1;
2189 close_complete_upcall(ep);
2190 ep->com.state = DEAD;
2191 }
2192 switch (ep->com.state) {
2193 case MPA_REQ_WAIT:
2194 case MPA_REQ_SENT:
2195 case MPA_REQ_RCVD:
2196 case MPA_REP_SENT:
2197 case FPDU_MODE:
2198 close = 1;
2199 if (abrupt)
2200 ep->com.state = ABORTING;
2201 else {
2202 ep->com.state = CLOSING;
2203 start_timer = 1;
2204 }
2205 set_bit(CLOSE_SENT, &ep->com.flags);
2206 break;
2207 case CLOSING:
2208 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
2209 close = 1;
2210 if (abrupt) {
2211 stop_timer = 1;
2212 ep->com.state = ABORTING;
2213 } else
2214 ep->com.state = MORIBUND;
2215 }
2216 break;
2217 case MORIBUND:
2218 case ABORTING:
2219 case DEAD:
2220 PDBG("%s ignoring disconnect ep %p state %u\n",
2221 __func__, ep, ep->com.state);
2222 break;
2223 default:
2224 BUG();
2225 break;
2226 }
2227
2228 spin_unlock_irqrestore(&ep->com.lock, flags);
2229 if (start_timer)
2230 start_ep_timer(ep);
2231 if (stop_timer)
2232 stop_ep_timer(ep);
2233 if (close) {
2234 if (abrupt)
2235 ret = abort_connection(ep, NULL, gfp);
2236 else
2237 ret = send_halfclose(ep, gfp);
2238 if (ret)
2239 fatal = 1;
2240 }
2241 if (fatal)
2242 release_ep_resources(ep);
2243 return ret;
2244}
2245
2246/*
2247 * All the CM events are handled on a work queue to have a safe context.
2248 */
2249static int sched(struct c4iw_dev *dev, struct sk_buff *skb)
2250{
2251
2252 /*
2253 * Save dev in the skb->cb area.
2254 */
2255 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev;
2256
2257 /*
2258 * Queue the skb and schedule the worker thread.
2259 */
2260 skb_queue_tail(&rxq, skb);
2261 queue_work(workq, &skb_work);
2262 return 0;
2263}
2264
2265static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
2266{
2267 struct cpl_set_tcb_rpl *rpl = cplhdr(skb);
2268
2269 if (rpl->status != CPL_ERR_NONE) {
2270 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u "
2271 "for tid %u\n", rpl->status, GET_TID(rpl));
2272 }
2273 return 0;
2274}
2275
2276int __init c4iw_cm_init(void)
2277{
2278 skb_queue_head_init(&rxq);
2279
2280 workq = create_singlethread_workqueue("iw_cxgb4");
2281 if (!workq)
2282 return -ENOMEM;
2283
2284 /*
2285 * Most upcalls from the T4 Core go to sched() to
2286 * schedule the processing on a work queue.
2287 */
2288 c4iw_handlers[CPL_ACT_ESTABLISH] = sched;
2289 c4iw_handlers[CPL_ACT_OPEN_RPL] = sched;
2290 c4iw_handlers[CPL_RX_DATA] = sched;
2291 c4iw_handlers[CPL_ABORT_RPL_RSS] = sched;
2292 c4iw_handlers[CPL_ABORT_RPL] = sched;
2293 c4iw_handlers[CPL_PASS_OPEN_RPL] = sched;
2294 c4iw_handlers[CPL_CLOSE_LISTSRV_RPL] = sched;
2295 c4iw_handlers[CPL_PASS_ACCEPT_REQ] = sched;
2296 c4iw_handlers[CPL_PASS_ESTABLISH] = sched;
2297 c4iw_handlers[CPL_PEER_CLOSE] = sched;
2298 c4iw_handlers[CPL_CLOSE_CON_RPL] = sched;
2299 c4iw_handlers[CPL_ABORT_REQ_RSS] = sched;
2300 c4iw_handlers[CPL_RDMA_TERMINATE] = sched;
2301 c4iw_handlers[CPL_FW4_ACK] = sched;
2302 c4iw_handlers[CPL_SET_TCB_RPL] = set_tcb_rpl;
2303 c4iw_handlers[CPL_FW6_MSG] = fw6_msg;
2304
2305 /*
2306 * These are the real handlers that are called from a
2307 * work queue.
2308 */
2309 work_handlers[CPL_ACT_ESTABLISH] = act_establish;
2310 work_handlers[CPL_ACT_OPEN_RPL] = act_open_rpl;
2311 work_handlers[CPL_RX_DATA] = rx_data;
2312 work_handlers[CPL_ABORT_RPL_RSS] = abort_rpl;
2313 work_handlers[CPL_ABORT_RPL] = abort_rpl;
2314 work_handlers[CPL_PASS_OPEN_RPL] = pass_open_rpl;
2315 work_handlers[CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl;
2316 work_handlers[CPL_PASS_ACCEPT_REQ] = pass_accept_req;
2317 work_handlers[CPL_PASS_ESTABLISH] = pass_establish;
2318 work_handlers[CPL_PEER_CLOSE] = peer_close;
2319 work_handlers[CPL_ABORT_REQ_RSS] = peer_abort;
2320 work_handlers[CPL_CLOSE_CON_RPL] = close_con_rpl;
2321 work_handlers[CPL_RDMA_TERMINATE] = terminate;
2322 work_handlers[CPL_FW4_ACK] = fw4_ack;
2323 return 0;
2324}
2325
2326void __exit c4iw_cm_term(void)
2327{
2328 flush_workqueue(workq);
2329 destroy_workqueue(workq);
2330}
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-06 00:48:17 -0400