aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAtul Gupta <atul.gupta@chelsio.com>2018-03-31 12:12:00 -0400
committerDavid S. Miller <davem@davemloft.net>2018-03-31 23:37:32 -0400
commit36bedb3f2e5b81832b5895363ed3fedb9ff1e8d0 (patch)
tree74b25eaa0590ea6af3f17d1979214f35063b3e84
parentcc35c88ae4db219611e204375d6a4248bc0e84d6 (diff)
crypto: chtls - Inline TLS record Tx
TLS handler for record transmit. Create Inline TLS work request and post to FW. Create Inline TLS record CPLs for hardware Signed-off-by: Atul Gupta <atul.gupta@chelsio.com> Signed-off-by: Michael Werner <werner@chelsio.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_io.c1222
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_main.c2
2 files changed, 1224 insertions, 0 deletions
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c
new file mode 100644
index 000000000000..6974d3ed0ff5
--- /dev/null
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -0,0 +1,1222 @@
1/*
2 * Copyright (c) 2018 Chelsio Communications, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * Written by: Atul Gupta (atul.gupta@chelsio.com)
9 */
10
11#include <linux/module.h>
12#include <linux/list.h>
13#include <linux/workqueue.h>
14#include <linux/skbuff.h>
15#include <linux/timer.h>
16#include <linux/notifier.h>
17#include <linux/inetdevice.h>
18#include <linux/ip.h>
19#include <linux/tcp.h>
20#include <linux/sched/signal.h>
21#include <net/tcp.h>
22#include <net/busy_poll.h>
23#include <crypto/aes.h>
24
25#include "chtls.h"
26#include "chtls_cm.h"
27
28static bool is_tls_tx(struct chtls_sock *csk)
29{
30 return csk->tlshws.txkey >= 0;
31}
32
33static int data_sgl_len(const struct sk_buff *skb)
34{
35 unsigned int cnt;
36
37 cnt = skb_shinfo(skb)->nr_frags;
38 return sgl_len(cnt) * 8;
39}
40
41static int nos_ivs(struct sock *sk, unsigned int size)
42{
43 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
44
45 return DIV_ROUND_UP(size, csk->tlshws.mfs);
46}
47
48static int set_ivs_imm(struct sock *sk, const struct sk_buff *skb)
49{
50 int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
51 int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
52
53 if ((hlen + KEY_ON_MEM_SZ + ivs_size) <
54 MAX_IMM_OFLD_TX_DATA_WR_LEN) {
55 ULP_SKB_CB(skb)->ulp.tls.iv = 1;
56 return 1;
57 }
58 ULP_SKB_CB(skb)->ulp.tls.iv = 0;
59 return 0;
60}
61
62static int max_ivs_size(struct sock *sk, int size)
63{
64 return nos_ivs(sk, size) * CIPHER_BLOCK_SIZE;
65}
66
67static int ivs_size(struct sock *sk, const struct sk_buff *skb)
68{
69 return set_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
70 CIPHER_BLOCK_SIZE) : 0;
71}
72
73static int flowc_wr_credits(int nparams, int *flowclenp)
74{
75 int flowclen16, flowclen;
76
77 flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
78 flowclen16 = DIV_ROUND_UP(flowclen, 16);
79 flowclen = flowclen16 * 16;
80
81 if (flowclenp)
82 *flowclenp = flowclen;
83
84 return flowclen16;
85}
86
87static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
88 struct fw_flowc_wr *flowc,
89 int flowclen)
90{
91 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
92 struct sk_buff *skb;
93
94 skb = alloc_skb(flowclen, GFP_ATOMIC);
95 if (!skb)
96 return NULL;
97
98 memcpy(__skb_put(skb, flowclen), flowc, flowclen);
99 skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
100
101 return skb;
102}
103
104static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
105 int flowclen)
106{
107 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
108 struct tcp_sock *tp = tcp_sk(sk);
109 int flowclen16 = flowclen / 16;
110 struct sk_buff *skb;
111 int ret;
112
113 if (csk_flag(sk, CSK_TX_DATA_SENT)) {
114 skb = create_flowc_wr_skb(sk, flowc, flowclen);
115 if (!skb)
116 return -ENOMEM;
117
118 skb_entail(sk, skb,
119 ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
120 return 0;
121 }
122
123 ret = cxgb4_immdata_send(csk->egress_dev,
124 csk->txq_idx,
125 flowc, flowclen);
126 if (!ret)
127 return flowclen16;
128 skb = create_flowc_wr_skb(sk, flowc, flowclen);
129 if (!skb)
130 return -ENOMEM;
131 send_or_defer(sk, tp, skb, 0);
132 return flowclen16;
133}
134
135static u8 tcp_state_to_flowc_state(u8 state)
136{
137 switch (state) {
138 case TCP_ESTABLISHED:
139 return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED;
140 case TCP_CLOSE_WAIT:
141 return FW_FLOWC_MNEM_TCPSTATE_CLOSEWAIT;
142 case TCP_FIN_WAIT1:
143 return FW_FLOWC_MNEM_TCPSTATE_FINWAIT1;
144 case TCP_CLOSING:
145 return FW_FLOWC_MNEM_TCPSTATE_CLOSING;
146 case TCP_LAST_ACK:
147 return FW_FLOWC_MNEM_TCPSTATE_LASTACK;
148 case TCP_FIN_WAIT2:
149 return FW_FLOWC_MNEM_TCPSTATE_FINWAIT2;
150 }
151
152 return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED;
153}
154
155int send_tx_flowc_wr(struct sock *sk, int compl,
156 u32 snd_nxt, u32 rcv_nxt)
157{
158 struct flowc_packed {
159 struct fw_flowc_wr fc;
160 struct fw_flowc_mnemval mnemval[FW_FLOWC_MNEM_MAX];
161 } __packed sflowc;
162 int nparams, paramidx, flowclen16, flowclen;
163 struct fw_flowc_wr *flowc;
164 struct chtls_sock *csk;
165 struct tcp_sock *tp;
166
167 csk = rcu_dereference_sk_user_data(sk);
168 tp = tcp_sk(sk);
169 memset(&sflowc, 0, sizeof(sflowc));
170 flowc = &sflowc.fc;
171
172#define FLOWC_PARAM(__m, __v) \
173 do { \
174 flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
175 flowc->mnemval[paramidx].val = cpu_to_be32(__v); \
176 paramidx++; \
177 } while (0)
178
179 paramidx = 0;
180
181 FLOWC_PARAM(PFNVFN, FW_PFVF_CMD_PFN_V(csk->cdev->lldi->pf));
182 FLOWC_PARAM(CH, csk->tx_chan);
183 FLOWC_PARAM(PORT, csk->tx_chan);
184 FLOWC_PARAM(IQID, csk->rss_qid);
185 FLOWC_PARAM(SNDNXT, tp->snd_nxt);
186 FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
187 FLOWC_PARAM(SNDBUF, csk->sndbuf);
188 FLOWC_PARAM(MSS, tp->mss_cache);
189 FLOWC_PARAM(TCPSTATE, tcp_state_to_flowc_state(sk->sk_state));
190
191 if (SND_WSCALE(tp))
192 FLOWC_PARAM(RCV_SCALE, SND_WSCALE(tp));
193
194 if (csk->ulp_mode == ULP_MODE_TLS)
195 FLOWC_PARAM(ULD_MODE, ULP_MODE_TLS);
196
197 if (csk->tlshws.fcplenmax)
198 FLOWC_PARAM(TXDATAPLEN_MAX, csk->tlshws.fcplenmax);
199
200 nparams = paramidx;
201#undef FLOWC_PARAM
202
203 flowclen16 = flowc_wr_credits(nparams, &flowclen);
204 flowc->op_to_nparams =
205 cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |
206 FW_WR_COMPL_V(compl) |
207 FW_FLOWC_WR_NPARAMS_V(nparams));
208 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) |
209 FW_WR_FLOWID_V(csk->tid));
210
211 return send_flowc_wr(sk, flowc, flowclen);
212}
213
214/* Copy IVs to WR */
215static int tls_copy_ivs(struct sock *sk, struct sk_buff *skb)
216
217{
218 struct chtls_sock *csk;
219 unsigned char *iv_loc;
220 struct chtls_hws *hws;
221 unsigned char *ivs;
222 u16 number_of_ivs;
223 struct page *page;
224 int err = 0;
225
226 csk = rcu_dereference_sk_user_data(sk);
227 hws = &csk->tlshws;
228 number_of_ivs = nos_ivs(sk, skb->len);
229
230 if (number_of_ivs > MAX_IVS_PAGE) {
231 pr_warn("MAX IVs in PAGE exceeded %d\n", number_of_ivs);
232 return -ENOMEM;
233 }
234
235 /* generate the IVs */
236 ivs = kmalloc(number_of_ivs * CIPHER_BLOCK_SIZE, GFP_ATOMIC);
237 if (!ivs)
238 return -ENOMEM;
239 get_random_bytes(ivs, number_of_ivs * CIPHER_BLOCK_SIZE);
240
241 if (skb_ulp_tls_iv_imm(skb)) {
242 /* send the IVs as immediate data in the WR */
243 iv_loc = (unsigned char *)__skb_push(skb, number_of_ivs *
244 CIPHER_BLOCK_SIZE);
245 if (iv_loc)
246 memcpy(iv_loc, ivs, number_of_ivs * CIPHER_BLOCK_SIZE);
247
248 hws->ivsize = number_of_ivs * CIPHER_BLOCK_SIZE;
249 } else {
250 /* Send the IVs as sgls */
251 /* Already accounted IV DSGL for credits */
252 skb_shinfo(skb)->nr_frags--;
253 page = alloc_pages(sk->sk_allocation | __GFP_COMP, 0);
254 if (!page) {
255 pr_info("%s : Page allocation for IVs failed\n",
256 __func__);
257 err = -ENOMEM;
258 goto out;
259 }
260 memcpy(page_address(page), ivs, number_of_ivs *
261 CIPHER_BLOCK_SIZE);
262 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page, 0,
263 number_of_ivs * CIPHER_BLOCK_SIZE);
264 hws->ivsize = 0;
265 }
266out:
267 kfree(ivs);
268 return err;
269}
270
271/* Copy Key to WR */
272static void tls_copy_tx_key(struct sock *sk, struct sk_buff *skb)
273{
274 struct ulptx_sc_memrd *sc_memrd;
275 struct chtls_sock *csk;
276 struct chtls_dev *cdev;
277 struct ulptx_idata *sc;
278 struct chtls_hws *hws;
279 u32 immdlen;
280 int kaddr;
281
282 csk = rcu_dereference_sk_user_data(sk);
283 hws = &csk->tlshws;
284 cdev = csk->cdev;
285
286 immdlen = sizeof(*sc) + sizeof(*sc_memrd);
287 kaddr = keyid_to_addr(cdev->kmap.start, hws->txkey);
288 sc = (struct ulptx_idata *)__skb_push(skb, immdlen);
289 if (sc) {
290 sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
291 sc->len = htonl(0);
292 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
293 sc_memrd->cmd_to_len =
294 htonl(ULPTX_CMD_V(ULP_TX_SC_MEMRD) |
295 ULP_TX_SC_MORE_V(1) |
296 ULPTX_LEN16_V(hws->keylen >> 4));
297 sc_memrd->addr = htonl(kaddr);
298 }
299}
300
301static u64 tlstx_incr_seqnum(struct chtls_hws *hws)
302{
303 return hws->tx_seq_no++;
304}
305
306static bool is_sg_request(const struct sk_buff *skb)
307{
308 return skb->peeked ||
309 (skb->len > MAX_IMM_ULPTX_WR_LEN);
310}
311
312/*
313 * Returns true if an sk_buff carries urgent data.
314 */
315static bool skb_urgent(struct sk_buff *skb)
316{
317 return ULP_SKB_CB(skb)->flags & ULPCB_FLAG_URG;
318}
319
320/* TLS content type for CPL SFO */
321static unsigned char tls_content_type(unsigned char content_type)
322{
323 switch (content_type) {
324 case TLS_HDR_TYPE_CCS:
325 return CPL_TX_TLS_SFO_TYPE_CCS;
326 case TLS_HDR_TYPE_ALERT:
327 return CPL_TX_TLS_SFO_TYPE_ALERT;
328 case TLS_HDR_TYPE_HANDSHAKE:
329 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
330 case TLS_HDR_TYPE_HEARTBEAT:
331 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
332 }
333 return CPL_TX_TLS_SFO_TYPE_DATA;
334}
335
336static void tls_tx_data_wr(struct sock *sk, struct sk_buff *skb,
337 int dlen, int tls_immd, u32 credits,
338 int expn, int pdus)
339{
340 struct fw_tlstx_data_wr *req_wr;
341 struct cpl_tx_tls_sfo *req_cpl;
342 unsigned int wr_ulp_mode_force;
343 struct tls_scmd *updated_scmd;
344 unsigned char data_type;
345 struct chtls_sock *csk;
346 struct net_device *dev;
347 struct chtls_hws *hws;
348 struct tls_scmd *scmd;
349 struct adapter *adap;
350 unsigned char *req;
351 int immd_len;
352 int iv_imm;
353 int len;
354
355 csk = rcu_dereference_sk_user_data(sk);
356 iv_imm = skb_ulp_tls_iv_imm(skb);
357 dev = csk->egress_dev;
358 adap = netdev2adap(dev);
359 hws = &csk->tlshws;
360 scmd = &hws->scmd;
361 len = dlen + expn;
362
363 dlen = (dlen < hws->mfs) ? dlen : hws->mfs;
364 atomic_inc(&adap->chcr_stats.tls_pdu_tx);
365
366 updated_scmd = scmd;
367 updated_scmd->seqno_numivs &= 0xffffff80;
368 updated_scmd->seqno_numivs |= SCMD_NUM_IVS_V(pdus);
369 hws->scmd = *updated_scmd;
370
371 req = (unsigned char *)__skb_push(skb, sizeof(struct cpl_tx_tls_sfo));
372 req_cpl = (struct cpl_tx_tls_sfo *)req;
373 req = (unsigned char *)__skb_push(skb, (sizeof(struct
374 fw_tlstx_data_wr)));
375
376 req_wr = (struct fw_tlstx_data_wr *)req;
377 immd_len = (tls_immd ? dlen : 0);
378 req_wr->op_to_immdlen =
379 htonl(FW_WR_OP_V(FW_TLSTX_DATA_WR) |
380 FW_TLSTX_DATA_WR_COMPL_V(1) |
381 FW_TLSTX_DATA_WR_IMMDLEN_V(immd_len));
382 req_wr->flowid_len16 = htonl(FW_TLSTX_DATA_WR_FLOWID_V(csk->tid) |
383 FW_TLSTX_DATA_WR_LEN16_V(credits));
384 wr_ulp_mode_force = TX_ULP_MODE_V(ULP_MODE_TLS);
385
386 if (is_sg_request(skb))
387 wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F |
388 ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 :
389 FW_OFLD_TX_DATA_WR_SHOVE_F);
390
391 req_wr->lsodisable_to_flags =
392 htonl(TX_ULP_MODE_V(ULP_MODE_TLS) |
393 FW_OFLD_TX_DATA_WR_URGENT_V(skb_urgent(skb)) |
394 T6_TX_FORCE_F | wr_ulp_mode_force |
395 TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) &&
396 skb_queue_empty(&csk->txq)));
397
398 req_wr->ctxloc_to_exp =
399 htonl(FW_TLSTX_DATA_WR_NUMIVS_V(pdus) |
400 FW_TLSTX_DATA_WR_EXP_V(expn) |
401 FW_TLSTX_DATA_WR_CTXLOC_V(CHTLS_KEY_CONTEXT_DDR) |
402 FW_TLSTX_DATA_WR_IVDSGL_V(!iv_imm) |
403 FW_TLSTX_DATA_WR_KEYSIZE_V(hws->keylen >> 4));
404
405 /* Fill in the length */
406 req_wr->plen = htonl(len);
407 req_wr->mfs = htons(hws->mfs);
408 req_wr->adjustedplen_pkd =
409 htons(FW_TLSTX_DATA_WR_ADJUSTEDPLEN_V(hws->adjustlen));
410 req_wr->expinplenmax_pkd =
411 htons(FW_TLSTX_DATA_WR_EXPINPLENMAX_V(hws->expansion));
412 req_wr->pdusinplenmax_pkd =
413 FW_TLSTX_DATA_WR_PDUSINPLENMAX_V(hws->pdus);
414 req_wr->r10 = 0;
415
416 data_type = tls_content_type(ULP_SKB_CB(skb)->ulp.tls.type);
417 req_cpl->op_to_seg_len = htonl(CPL_TX_TLS_SFO_OPCODE_V(CPL_TX_TLS_SFO) |
418 CPL_TX_TLS_SFO_DATA_TYPE_V(data_type) |
419 CPL_TX_TLS_SFO_CPL_LEN_V(2) |
420 CPL_TX_TLS_SFO_SEG_LEN_V(dlen));
421 req_cpl->pld_len = htonl(len - expn);
422
423 req_cpl->type_protover = htonl(CPL_TX_TLS_SFO_TYPE_V
424 ((data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) ?
425 TLS_HDR_TYPE_HEARTBEAT : 0) |
426 CPL_TX_TLS_SFO_PROTOVER_V(0));
427
428 /* create the s-command */
429 req_cpl->r1_lo = 0;
430 req_cpl->seqno_numivs = cpu_to_be32(hws->scmd.seqno_numivs);
431 req_cpl->ivgen_hdrlen = cpu_to_be32(hws->scmd.ivgen_hdrlen);
432 req_cpl->scmd1 = cpu_to_be64(tlstx_incr_seqnum(hws));
433}
434
435/*
436 * Calculate the TLS data expansion size
437 */
438static int chtls_expansion_size(struct sock *sk, int data_len,
439 int fullpdu,
440 unsigned short *pducnt)
441{
442 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
443 struct chtls_hws *hws = &csk->tlshws;
444 struct tls_scmd *scmd = &hws->scmd;
445 int fragsize = hws->mfs;
446 int expnsize = 0;
447 int fragleft;
448 int fragcnt;
449 int expppdu;
450
451 if (SCMD_CIPH_MODE_G(scmd->seqno_numivs) ==
452 SCMD_CIPH_MODE_AES_GCM) {
453 expppdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
454 TLS_HEADER_LENGTH;
455
456 if (fullpdu) {
457 *pducnt = data_len / (expppdu + fragsize);
458 if (*pducnt > 32)
459 *pducnt = 32;
460 else if (!*pducnt)
461 *pducnt = 1;
462 expnsize = (*pducnt) * expppdu;
463 return expnsize;
464 }
465 fragcnt = (data_len / fragsize);
466 expnsize = fragcnt * expppdu;
467 fragleft = data_len % fragsize;
468 if (fragleft > 0)
469 expnsize += expppdu;
470 }
471 return expnsize;
472}
473
474/* WR with IV, KEY and CPL SFO added */
475static void make_tlstx_data_wr(struct sock *sk, struct sk_buff *skb,
476 int tls_tx_imm, int tls_len, u32 credits)
477{
478 unsigned short pdus_per_ulp = 0;
479 struct chtls_sock *csk;
480 struct chtls_hws *hws;
481 int expn_sz;
482 int pdus;
483
484 csk = rcu_dereference_sk_user_data(sk);
485 hws = &csk->tlshws;
486 pdus = DIV_ROUND_UP(tls_len, hws->mfs);
487 expn_sz = chtls_expansion_size(sk, tls_len, 0, NULL);
488 if (!hws->compute) {
489 hws->expansion = chtls_expansion_size(sk,
490 hws->fcplenmax,
491 1, &pdus_per_ulp);
492 hws->pdus = pdus_per_ulp;
493 hws->adjustlen = hws->pdus *
494 ((hws->expansion / hws->pdus) + hws->mfs);
495 hws->compute = 1;
496 }
497 if (tls_copy_ivs(sk, skb))
498 return;
499 tls_copy_tx_key(sk, skb);
500 tls_tx_data_wr(sk, skb, tls_len, tls_tx_imm, credits, expn_sz, pdus);
501 hws->tx_seq_no += (pdus - 1);
502}
503
504static void make_tx_data_wr(struct sock *sk, struct sk_buff *skb,
505 unsigned int immdlen, int len,
506 u32 credits, u32 compl)
507{
508 struct fw_ofld_tx_data_wr *req;
509 unsigned int wr_ulp_mode_force;
510 struct chtls_sock *csk;
511 unsigned int opcode;
512
513 csk = rcu_dereference_sk_user_data(sk);
514 opcode = FW_OFLD_TX_DATA_WR;
515
516 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req));
517 req->op_to_immdlen = htonl(WR_OP_V(opcode) |
518 FW_WR_COMPL_V(compl) |
519 FW_WR_IMMDLEN_V(immdlen));
520 req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |
521 FW_WR_LEN16_V(credits));
522
523 wr_ulp_mode_force = TX_ULP_MODE_V(csk->ulp_mode);
524 if (is_sg_request(skb))
525 wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F |
526 ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 :
527 FW_OFLD_TX_DATA_WR_SHOVE_F);
528
529 req->tunnel_to_proxy = htonl(wr_ulp_mode_force |
530 FW_OFLD_TX_DATA_WR_URGENT_V(skb_urgent(skb)) |
531 FW_OFLD_TX_DATA_WR_SHOVE_V((!csk_flag
532 (sk, CSK_TX_MORE_DATA)) &&
533 skb_queue_empty(&csk->txq)));
534 req->plen = htonl(len);
535}
536
537static int chtls_wr_size(struct chtls_sock *csk, const struct sk_buff *skb,
538 bool size)
539{
540 int wr_size;
541
542 wr_size = TLS_WR_CPL_LEN;
543 wr_size += KEY_ON_MEM_SZ;
544 wr_size += ivs_size(csk->sk, skb);
545
546 if (size)
547 return wr_size;
548
549 /* frags counted for IV dsgl */
550 if (!skb_ulp_tls_iv_imm(skb))
551 skb_shinfo(skb)->nr_frags++;
552
553 return wr_size;
554}
555
556static bool is_ofld_imm(struct chtls_sock *csk, const struct sk_buff *skb)
557{
558 int length = skb->len;
559
560 if (skb->peeked || skb->len > MAX_IMM_ULPTX_WR_LEN)
561 return false;
562
563 if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) {
564 /* Check TLS header len for Immediate */
565 if (csk->ulp_mode == ULP_MODE_TLS &&
566 skb_ulp_tls_inline(skb))
567 length += chtls_wr_size(csk, skb, true);
568 else
569 length += sizeof(struct fw_ofld_tx_data_wr);
570
571 return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
572 }
573 return true;
574}
575
576static unsigned int calc_tx_flits(const struct sk_buff *skb,
577 unsigned int immdlen)
578{
579 unsigned int flits, cnt;
580
581 flits = immdlen / 8; /* headers */
582 cnt = skb_shinfo(skb)->nr_frags;
583 if (skb_tail_pointer(skb) != skb_transport_header(skb))
584 cnt++;
585 return flits + sgl_len(cnt);
586}
587
588static void arp_failure_discard(void *handle, struct sk_buff *skb)
589{
590 kfree_skb(skb);
591}
592
593int chtls_push_frames(struct chtls_sock *csk, int comp)
594{
595 struct chtls_hws *hws = &csk->tlshws;
596 struct tcp_sock *tp;
597 struct sk_buff *skb;
598 int total_size = 0;
599 struct sock *sk;
600 int wr_size;
601
602 wr_size = sizeof(struct fw_ofld_tx_data_wr);
603 sk = csk->sk;
604 tp = tcp_sk(sk);
605
606 if (unlikely(sk_in_state(sk, TCPF_SYN_SENT | TCPF_CLOSE)))
607 return 0;
608
609 if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN)))
610 return 0;
611
612 while (csk->wr_credits && (skb = skb_peek(&csk->txq)) &&
613 (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_HOLD) ||
614 skb_queue_len(&csk->txq) > 1)) {
615 unsigned int credit_len = skb->len;
616 unsigned int credits_needed;
617 unsigned int completion = 0;
618 int tls_len = skb->len;/* TLS data len before IV/key */
619 unsigned int immdlen;
620 int len = skb->len; /* length [ulp bytes] inserted by hw */
621 int flowclen16 = 0;
622 int tls_tx_imm = 0;
623
624 immdlen = skb->len;
625 if (!is_ofld_imm(csk, skb)) {
626 immdlen = skb_transport_offset(skb);
627 if (skb_ulp_tls_inline(skb))
628 wr_size = chtls_wr_size(csk, skb, false);
629 credit_len = 8 * calc_tx_flits(skb, immdlen);
630 } else {
631 if (skb_ulp_tls_inline(skb)) {
632 wr_size = chtls_wr_size(csk, skb, false);
633 tls_tx_imm = 1;
634 }
635 }
636 if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR))
637 credit_len += wr_size;
638 credits_needed = DIV_ROUND_UP(credit_len, 16);
639 if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
640 flowclen16 = send_tx_flowc_wr(sk, 1, tp->snd_nxt,
641 tp->rcv_nxt);
642 if (flowclen16 <= 0)
643 break;
644 csk->wr_credits -= flowclen16;
645 csk->wr_unacked += flowclen16;
646 csk->wr_nondata += flowclen16;
647 csk_set_flag(csk, CSK_TX_DATA_SENT);
648 }
649
650 if (csk->wr_credits < credits_needed) {
651 if (skb_ulp_tls_inline(skb) &&
652 !skb_ulp_tls_iv_imm(skb))
653 skb_shinfo(skb)->nr_frags--;
654 break;
655 }
656
657 __skb_unlink(skb, &csk->txq);
658 skb_set_queue_mapping(skb, (csk->txq_idx << 1) |
659 CPL_PRIORITY_DATA);
660 if (hws->ofld)
661 hws->txqid = (skb->queue_mapping >> 1);
662 skb->csum = (__force __wsum)(credits_needed + csk->wr_nondata);
663 csk->wr_credits -= credits_needed;
664 csk->wr_unacked += credits_needed;
665 csk->wr_nondata = 0;
666 enqueue_wr(csk, skb);
667
668 if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) {
669 if ((comp && csk->wr_unacked == credits_needed) ||
670 (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) ||
671 csk->wr_unacked >= csk->wr_max_credits / 2) {
672 completion = 1;
673 csk->wr_unacked = 0;
674 }
675 if (skb_ulp_tls_inline(skb))
676 make_tlstx_data_wr(sk, skb, tls_tx_imm,
677 tls_len, credits_needed);
678 else
679 make_tx_data_wr(sk, skb, immdlen, len,
680 credits_needed, completion);
681 tp->snd_nxt += len;
682 tp->lsndtime = tcp_time_stamp(tp);
683 if (completion)
684 ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_NEED_HDR;
685 } else {
686 struct cpl_close_con_req *req = cplhdr(skb);
687 unsigned int cmd = CPL_OPCODE_G(ntohl
688 (OPCODE_TID(req)));
689
690 if (cmd == CPL_CLOSE_CON_REQ)
691 csk_set_flag(csk,
692 CSK_CLOSE_CON_REQUESTED);
693
694 if ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) &&
695 (csk->wr_unacked >= csk->wr_max_credits / 2)) {
696 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
697 csk->wr_unacked = 0;
698 }
699 }
700 total_size += skb->truesize;
701 if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_BARRIER)
702 csk_set_flag(csk, CSK_TX_WAIT_IDLE);
703 t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
704 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
705 }
706 sk->sk_wmem_queued -= total_size;
707 return total_size;
708}
709
710static void mark_urg(struct tcp_sock *tp, int flags,
711 struct sk_buff *skb)
712{
713 if (unlikely(flags & MSG_OOB)) {
714 tp->snd_up = tp->write_seq;
715 ULP_SKB_CB(skb)->flags = ULPCB_FLAG_URG |
716 ULPCB_FLAG_BARRIER |
717 ULPCB_FLAG_NO_APPEND |
718 ULPCB_FLAG_NEED_HDR;
719 }
720}
721
722/*
723 * Returns true if a connection should send more data to TCP engine
724 */
725static bool should_push(struct sock *sk)
726{
727 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
728 struct chtls_dev *cdev = csk->cdev;
729 struct tcp_sock *tp = tcp_sk(sk);
730
731 /*
732 * If we've released our offload resources there's nothing to do ...
733 */
734 if (!cdev)
735 return false;
736
737 /*
738 * If there aren't any work requests in flight, or there isn't enough
739 * data in flight, or Nagle is off then send the current TX_DATA
740 * otherwise hold it and wait to accumulate more data.
741 */
742 return csk->wr_credits == csk->wr_max_credits ||
743 (tp->nonagle & TCP_NAGLE_OFF);
744}
745
746/*
747 * Returns true if a TCP socket is corked.
748 */
749static bool corked(const struct tcp_sock *tp, int flags)
750{
751 return (flags & MSG_MORE) || (tp->nonagle & TCP_NAGLE_CORK);
752}
753
754/*
755 * Returns true if a send should try to push new data.
756 */
757static bool send_should_push(struct sock *sk, int flags)
758{
759 return should_push(sk) && !corked(tcp_sk(sk), flags);
760}
761
762void chtls_tcp_push(struct sock *sk, int flags)
763{
764 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
765 int qlen = skb_queue_len(&csk->txq);
766
767 if (likely(qlen)) {
768 struct sk_buff *skb = skb_peek_tail(&csk->txq);
769 struct tcp_sock *tp = tcp_sk(sk);
770
771 mark_urg(tp, flags, skb);
772
773 if (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) &&
774 corked(tp, flags)) {
775 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_HOLD;
776 return;
777 }
778
779 ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_HOLD;
780 if (qlen == 1 &&
781 ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
782 should_push(sk)))
783 chtls_push_frames(csk, 1);
784 }
785}
786
787/*
788 * Calculate the size for a new send sk_buff. It's maximum size so we can
789 * pack lots of data into it, unless we plan to send it immediately, in which
790 * case we size it more tightly.
791 *
792 * Note: we don't bother compensating for MSS < PAGE_SIZE because it doesn't
793 * arise in normal cases and when it does we are just wasting memory.
794 */
795static int select_size(struct sock *sk, int io_len, int flags, int len)
796{
797 const int pgbreak = SKB_MAX_HEAD(len);
798
799 /*
800 * If the data wouldn't fit in the main body anyway, put only the
801 * header in the main body so it can use immediate data and place all
802 * the payload in page fragments.
803 */
804 if (io_len > pgbreak)
805 return 0;
806
807 /*
808 * If we will be accumulating payload get a large main body.
809 */
810 if (!send_should_push(sk, flags))
811 return pgbreak;
812
813 return io_len;
814}
815
816void skb_entail(struct sock *sk, struct sk_buff *skb, int flags)
817{
818 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
819 struct tcp_sock *tp = tcp_sk(sk);
820
821 ULP_SKB_CB(skb)->seq = tp->write_seq;
822 ULP_SKB_CB(skb)->flags = flags;
823 __skb_queue_tail(&csk->txq, skb);
824 sk->sk_wmem_queued += skb->truesize;
825
826 if (TCP_PAGE(sk) && TCP_OFF(sk)) {
827 put_page(TCP_PAGE(sk));
828 TCP_PAGE(sk) = NULL;
829 TCP_OFF(sk) = 0;
830 }
831}
832
833static struct sk_buff *get_tx_skb(struct sock *sk, int size)
834{
835 struct sk_buff *skb;
836
837 skb = alloc_skb(size + TX_HEADER_LEN, sk->sk_allocation);
838 if (likely(skb)) {
839 skb_reserve(skb, TX_HEADER_LEN);
840 skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR);
841 skb_reset_transport_header(skb);
842 }
843 return skb;
844}
845
846static struct sk_buff *get_record_skb(struct sock *sk, int size, bool zcopy)
847{
848 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
849 struct sk_buff *skb;
850
851 skb = alloc_skb(((zcopy ? 0 : size) + TX_TLSHDR_LEN +
852 KEY_ON_MEM_SZ + max_ivs_size(sk, size)),
853 sk->sk_allocation);
854 if (likely(skb)) {
855 skb_reserve(skb, (TX_TLSHDR_LEN +
856 KEY_ON_MEM_SZ + max_ivs_size(sk, size)));
857 skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR);
858 skb_reset_transport_header(skb);
859 ULP_SKB_CB(skb)->ulp.tls.ofld = 1;
860 ULP_SKB_CB(skb)->ulp.tls.type = csk->tlshws.type;
861 }
862 return skb;
863}
864
865static void tx_skb_finalize(struct sk_buff *skb)
866{
867 struct ulp_skb_cb *cb = ULP_SKB_CB(skb);
868
869 if (!(cb->flags & ULPCB_FLAG_NO_HDR))
870 cb->flags = ULPCB_FLAG_NEED_HDR;
871 cb->flags |= ULPCB_FLAG_NO_APPEND;
872}
873
874static void push_frames_if_head(struct sock *sk)
875{
876 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
877
878 if (skb_queue_len(&csk->txq) == 1)
879 chtls_push_frames(csk, 1);
880}
881
882static int chtls_skb_copy_to_page_nocache(struct sock *sk,
883 struct iov_iter *from,
884 struct sk_buff *skb,
885 struct page *page,
886 int off, int copy)
887{
888 int err;
889
890 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) +
891 off, copy, skb->len);
892 if (err)
893 return err;
894
895 skb->len += copy;
896 skb->data_len += copy;
897 skb->truesize += copy;
898 sk->sk_wmem_queued += copy;
899 return 0;
900}
901
902/* Read TLS header to find content type and data length */
903static u16 tls_header_read(struct tls_hdr *thdr, struct iov_iter *from)
904{
905 if (copy_from_iter(thdr, sizeof(*thdr), from) != sizeof(*thdr))
906 return -EFAULT;
907 return (__force u16)cpu_to_be16(thdr->length);
908}
909
910int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
911{
912 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
913 struct chtls_dev *cdev = csk->cdev;
914 struct tcp_sock *tp = tcp_sk(sk);
915 struct sk_buff *skb;
916 int mss, flags, err;
917 int recordsz = 0;
918 int copied = 0;
919 int hdrlen = 0;
920 long timeo;
921
922 lock_sock(sk);
923 flags = msg->msg_flags;
924 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
925
926 if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
927 err = sk_stream_wait_connect(sk, &timeo);
928 if (err)
929 goto out_err;
930 }
931
932 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
933 err = -EPIPE;
934 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
935 goto out_err;
936
937 mss = csk->mss;
938 csk_set_flag(csk, CSK_TX_MORE_DATA);
939
940 while (msg_data_left(msg)) {
941 int copy = 0;
942
943 skb = skb_peek_tail(&csk->txq);
944 if (skb) {
945 copy = mss - skb->len;
946 skb->ip_summed = CHECKSUM_UNNECESSARY;
947 }
948
949 if (is_tls_tx(csk) && !csk->tlshws.txleft) {
950 struct tls_hdr hdr;
951
952 recordsz = tls_header_read(&hdr, &msg->msg_iter);
953 size -= TLS_HEADER_LENGTH;
954 hdrlen += TLS_HEADER_LENGTH;
955 csk->tlshws.txleft = recordsz;
956 csk->tlshws.type = hdr.type;
957 if (skb)
958 ULP_SKB_CB(skb)->ulp.tls.type = hdr.type;
959 }
960
961 if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
962 copy <= 0) {
963new_buf:
964 if (skb) {
965 tx_skb_finalize(skb);
966 push_frames_if_head(sk);
967 }
968
969 if (is_tls_tx(csk)) {
970 skb = get_record_skb(sk,
971 select_size(sk,
972 recordsz,
973 flags,
974 TX_TLSHDR_LEN),
975 false);
976 } else {
977 skb = get_tx_skb(sk,
978 select_size(sk, size, flags,
979 TX_HEADER_LEN));
980 }
981 if (unlikely(!skb))
982 goto wait_for_memory;
983
984 skb->ip_summed = CHECKSUM_UNNECESSARY;
985 copy = mss;
986 }
987 if (copy > size)
988 copy = size;
989
990 if (skb_tailroom(skb) > 0) {
991 copy = min(copy, skb_tailroom(skb));
992 if (is_tls_tx(csk))
993 copy = min_t(int, copy, csk->tlshws.txleft);
994 err = skb_add_data_nocache(sk, skb,
995 &msg->msg_iter, copy);
996 if (err)
997 goto do_fault;
998 } else {
999 int i = skb_shinfo(skb)->nr_frags;
1000 struct page *page = TCP_PAGE(sk);
1001 int pg_size = PAGE_SIZE;
1002 int off = TCP_OFF(sk);
1003 bool merge;
1004
1005 if (page)
1006 pg_size <<= compound_order(page);
1007
1008 if (off < pg_size &&
1009 skb_can_coalesce(skb, i, page, off)) {
1010 merge = 1;
1011 goto copy;
1012 }
1013 merge = 0;
1014 if (i == (is_tls_tx(csk) ? (MAX_SKB_FRAGS - 1) :
1015 MAX_SKB_FRAGS))
1016 goto new_buf;
1017
1018 if (page && off == pg_size) {
1019 put_page(page);
1020 TCP_PAGE(sk) = page = NULL;
1021 pg_size = PAGE_SIZE;
1022 }
1023
1024 if (!page) {
1025 gfp_t gfp = sk->sk_allocation;
1026 int order = cdev->send_page_order;
1027
1028 if (order) {
1029 page = alloc_pages(gfp | __GFP_COMP |
1030 __GFP_NOWARN |
1031 __GFP_NORETRY,
1032 order);
1033 if (page)
1034 pg_size <<=
1035 compound_order(page);
1036 }
1037 if (!page) {
1038 page = alloc_page(gfp);
1039 pg_size = PAGE_SIZE;
1040 }
1041 if (!page)
1042 goto wait_for_memory;
1043 off = 0;
1044 }
1045copy:
1046 if (copy > pg_size - off)
1047 copy = pg_size - off;
1048 if (is_tls_tx(csk))
1049 copy = min_t(int, copy, csk->tlshws.txleft);
1050
1051 err = chtls_skb_copy_to_page_nocache(sk, &msg->msg_iter,
1052 skb, page,
1053 off, copy);
1054 if (unlikely(err)) {
1055 if (!TCP_PAGE(sk)) {
1056 TCP_PAGE(sk) = page;
1057 TCP_OFF(sk) = 0;
1058 }
1059 goto do_fault;
1060 }
1061 /* Update the skb. */
1062 if (merge) {
1063 skb_shinfo(skb)->frags[i - 1].size += copy;
1064 } else {
1065 skb_fill_page_desc(skb, i, page, off, copy);
1066 if (off + copy < pg_size) {
1067 /* space left keep page */
1068 get_page(page);
1069 TCP_PAGE(sk) = page;
1070 } else {
1071 TCP_PAGE(sk) = NULL;
1072 }
1073 }
1074 TCP_OFF(sk) = off + copy;
1075 }
1076 if (unlikely(skb->len == mss))
1077 tx_skb_finalize(skb);
1078 tp->write_seq += copy;
1079 copied += copy;
1080 size -= copy;
1081
1082 if (is_tls_tx(csk))
1083 csk->tlshws.txleft -= copy;
1084
1085 if (corked(tp, flags) &&
1086 (sk_stream_wspace(sk) < sk_stream_min_wspace(sk)))
1087 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND;
1088
1089 if (size == 0)
1090 goto out;
1091
1092 if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)
1093 push_frames_if_head(sk);
1094 continue;
1095wait_for_memory:
1096 err = sk_stream_wait_memory(sk, &timeo);
1097 if (err)
1098 goto do_error;
1099 }
1100out:
1101 csk_reset_flag(csk, CSK_TX_MORE_DATA);
1102 if (copied)
1103 chtls_tcp_push(sk, flags);
1104done:
1105 release_sock(sk);
1106 return copied + hdrlen;
1107do_fault:
1108 if (!skb->len) {
1109 __skb_unlink(skb, &csk->txq);
1110 sk->sk_wmem_queued -= skb->truesize;
1111 __kfree_skb(skb);
1112 }
1113do_error:
1114 if (copied)
1115 goto out;
1116out_err:
1117 if (csk_conn_inline(csk))
1118 csk_reset_flag(csk, CSK_TX_MORE_DATA);
1119 copied = sk_stream_error(sk, flags, err);
1120 goto done;
1121}
1122
1123int chtls_sendpage(struct sock *sk, struct page *page,
1124 int offset, size_t size, int flags)
1125{
1126 struct chtls_sock *csk;
1127 int mss, err, copied;
1128 struct tcp_sock *tp;
1129 long timeo;
1130
1131 tp = tcp_sk(sk);
1132 copied = 0;
1133 csk = rcu_dereference_sk_user_data(sk);
1134 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1135
1136 err = sk_stream_wait_connect(sk, &timeo);
1137 if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
1138 err != 0)
1139 goto out_err;
1140
1141 mss = csk->mss;
1142 csk_set_flag(csk, CSK_TX_MORE_DATA);
1143
1144 while (size > 0) {
1145 struct sk_buff *skb = skb_peek_tail(&csk->txq);
1146 int copy, i;
1147
1148 copy = mss - skb->len;
1149 if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
1150 copy <= 0) {
1151new_buf:
1152
1153 if (is_tls_tx(csk)) {
1154 skb = get_record_skb(sk,
1155 select_size(sk, size,
1156 flags,
1157 TX_TLSHDR_LEN),
1158 true);
1159 } else {
1160 skb = get_tx_skb(sk, 0);
1161 }
1162 if (!skb)
1163 goto do_error;
1164 copy = mss;
1165 }
1166 if (copy > size)
1167 copy = size;
1168
1169 i = skb_shinfo(skb)->nr_frags;
1170 if (skb_can_coalesce(skb, i, page, offset)) {
1171 skb_shinfo(skb)->frags[i - 1].size += copy;
1172 } else if (i < MAX_SKB_FRAGS) {
1173 get_page(page);
1174 skb_fill_page_desc(skb, i, page, offset, copy);
1175 } else {
1176 tx_skb_finalize(skb);
1177 push_frames_if_head(sk);
1178 goto new_buf;
1179 }
1180
1181 skb->len += copy;
1182 if (skb->len == mss)
1183 tx_skb_finalize(skb);
1184 skb->data_len += copy;
1185 skb->truesize += copy;
1186 sk->sk_wmem_queued += copy;
1187 tp->write_seq += copy;
1188 copied += copy;
1189 offset += copy;
1190 size -= copy;
1191
1192 if (corked(tp, flags) &&
1193 (sk_stream_wspace(sk) < sk_stream_min_wspace(sk)))
1194 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND;
1195
1196 if (!size)
1197 break;
1198
1199 if (unlikely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND))
1200 push_frames_if_head(sk);
1201 continue;
1202
1203 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1204 }
1205out:
1206 csk_reset_flag(csk, CSK_TX_MORE_DATA);
1207 if (copied)
1208 chtls_tcp_push(sk, flags);
1209done:
1210 release_sock(sk);
1211 return copied;
1212
1213do_error:
1214 if (copied)
1215 goto out;
1216
1217out_err:
1218 if (csk_conn_inline(csk))
1219 csk_reset_flag(csk, CSK_TX_MORE_DATA);
1220 copied = sk_stream_error(sk, flags, err);
1221 goto done;
1222}
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
index 04b316f86ebd..e5e543a45542 100644
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -547,6 +547,8 @@ static void __init chtls_init_ulp_ops(void)
547 chtls_cpl_prot.disconnect = chtls_disconnect; 547 chtls_cpl_prot.disconnect = chtls_disconnect;
548 chtls_cpl_prot.destroy = chtls_destroy_sock; 548 chtls_cpl_prot.destroy = chtls_destroy_sock;
549 chtls_cpl_prot.shutdown = chtls_shutdown; 549 chtls_cpl_prot.shutdown = chtls_shutdown;
550 chtls_cpl_prot.sendmsg = chtls_sendmsg;
551 chtls_cpl_prot.sendpage = chtls_sendpage;
550 chtls_cpl_prot.setsockopt = chtls_setsockopt; 552 chtls_cpl_prot.setsockopt = chtls_setsockopt;
551 chtls_cpl_prot.getsockopt = chtls_getsockopt; 553 chtls_cpl_prot.getsockopt = chtls_getsockopt;
552} 554}
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-23 14:36:52 -0400