summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTrond Myklebust <trond.myklebust@hammerspace.com>2019-07-11 16:52:47 -0400
committerTrond Myklebust <trond.myklebust@hammerspace.com>2019-07-12 12:11:01 -0400
commit347543e64082782379627cb21162cb859590f3c7 (patch)
tree1bd1dd4ceeaf4ad9a56834ce31b5eb31b976e95e
parent80d3c45fd765fbf4f10981b60ff6b1384bdbc706 (diff)
parent62a92ba97a31c544802bbf13d3a998e86796d548 (diff)
Merge tag 'nfs-rdma-for-5.3-1' of git://git.linux-nfs.org/projects/anna/linux-nfs
NFSoRDMA client updates for 5.3 New features: - Add a way to place MRs back on the free list - Reduce context switching - Add new trace events Bugfixes and cleanups: - Fix a BUG when tracing is enabled with NFSv4.1 - Fix a use-after-free in rpcrdma_post_recvs - Replace use of xdr_stream_pos in rpcrdma_marshal_req - Fix occasional transport deadlock - Fix show_nfs_errors macros, other tracing improvements - Remove RPCRDMA_REQ_F_PENDING and fr_state - Various simplifications and refactors
Diffstat
-rw-r--r--fs/nfs/callback_proc.c28
-rw-r--r--fs/nfs/nfs2xdr.c2
-rw-r--r--fs/nfs/nfs3xdr.c2
-rw-r--r--fs/nfs/nfs4trace.h207
-rw-r--r--fs/nfs/nfs4xdr.c2
-rw-r--r--fs/nfs/nfstrace.h233
-rw-r--r--include/linux/sunrpc/xprt.h3
-rw-r--r--include/trace/events/rpcrdma.h90
-rw-r--r--net/sunrpc/sched.c1
-rw-r--r--net/sunrpc/xprt.c32
-rw-r--r--net/sunrpc/xprtrdma/frwr_ops.c327
-rw-r--r--net/sunrpc/xprtrdma/rpc_rdma.c148
-rw-r--r--net/sunrpc/xprtrdma/transport.c83
-rw-r--r--net/sunrpc/xprtrdma/verbs.c115
-rw-r--r--net/sunrpc/xprtrdma/xprt_rdma.h44
-rw-r--r--net/sunrpc/xprtsock.c23
16 files changed, 837 insertions, 503 deletions
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 315967354954..f39924ba050b 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -414,27 +414,39 @@ static __be32
414validate_seqid(const struct nfs4_slot_table *tbl, const struct nfs4_slot *slot, 414validate_seqid(const struct nfs4_slot_table *tbl, const struct nfs4_slot *slot,
415 const struct cb_sequenceargs * args) 415 const struct cb_sequenceargs * args)
416{ 416{
417 __be32 ret;
418
419 ret = cpu_to_be32(NFS4ERR_BADSLOT);
417 if (args->csa_slotid > tbl->server_highest_slotid) 420 if (args->csa_slotid > tbl->server_highest_slotid)
418 return htonl(NFS4ERR_BADSLOT); 421 goto out_err;
419 422
420 /* Replay */ 423 /* Replay */
421 if (args->csa_sequenceid == slot->seq_nr) { 424 if (args->csa_sequenceid == slot->seq_nr) {
425 ret = cpu_to_be32(NFS4ERR_DELAY);
422 if (nfs4_test_locked_slot(tbl, slot->slot_nr)) 426 if (nfs4_test_locked_slot(tbl, slot->slot_nr))
423 return htonl(NFS4ERR_DELAY); 427 goto out_err;
428
424 /* Signal process_op to set this error on next op */ 429 /* Signal process_op to set this error on next op */
430 ret = cpu_to_be32(NFS4ERR_RETRY_UNCACHED_REP);
425 if (args->csa_cachethis == 0) 431 if (args->csa_cachethis == 0)
426 return htonl(NFS4ERR_RETRY_UNCACHED_REP); 432 goto out_err;
427 433
428 /* Liar! We never allowed you to set csa_cachethis != 0 */ 434 /* Liar! We never allowed you to set csa_cachethis != 0 */
429 return htonl(NFS4ERR_SEQ_FALSE_RETRY); 435 ret = cpu_to_be32(NFS4ERR_SEQ_FALSE_RETRY);
436 goto out_err;
430 } 437 }
431 438
432 /* Note: wraparound relies on seq_nr being of type u32 */ 439 /* Note: wraparound relies on seq_nr being of type u32 */
433 if (likely(args->csa_sequenceid == slot->seq_nr + 1))
434 return htonl(NFS4_OK);
435
436 /* Misordered request */ 440 /* Misordered request */
437 return htonl(NFS4ERR_SEQ_MISORDERED); 441 ret = cpu_to_be32(NFS4ERR_SEQ_MISORDERED);
442 if (args->csa_sequenceid != slot->seq_nr + 1)
443 goto out_err;
444
445 return cpu_to_be32(NFS4_OK);
446
447out_err:
448 trace_nfs4_cb_seqid_err(args, ret);
449 return ret;
438} 450}
439 451
440/* 452/*
diff --git a/fs/nfs/nfs2xdr.c b/fs/nfs/nfs2xdr.c
index 572794dab4b1..cbc17a203248 100644
--- a/fs/nfs/nfs2xdr.c
+++ b/fs/nfs/nfs2xdr.c
@@ -151,7 +151,7 @@ static int decode_stat(struct xdr_stream *xdr, enum nfs_stat *status)
151 return 0; 151 return 0;
152out_status: 152out_status:
153 *status = be32_to_cpup(p); 153 *status = be32_to_cpup(p);
154 trace_nfs_xdr_status((int)*status); 154 trace_nfs_xdr_status(xdr, (int)*status);
155 return 0; 155 return 0;
156} 156}
157 157
diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c
index abbbdde97e31..602767850b36 100644
--- a/fs/nfs/nfs3xdr.c
+++ b/fs/nfs/nfs3xdr.c
@@ -343,7 +343,7 @@ static int decode_nfsstat3(struct xdr_stream *xdr, enum nfs_stat *status)
343 return 0; 343 return 0;
344out_status: 344out_status:
345 *status = be32_to_cpup(p); 345 *status = be32_to_cpup(p);
346 trace_nfs_xdr_status((int)*status); 346 trace_nfs_xdr_status(xdr, (int)*status);
347 return 0; 347 return 0;
348} 348}
349 349
diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h
index cd1a5c08da9a..d85f20945a2b 100644
--- a/fs/nfs/nfs4trace.h
+++ b/fs/nfs/nfs4trace.h
@@ -156,7 +156,7 @@ TRACE_DEFINE_ENUM(NFS4ERR_WRONG_TYPE);
156TRACE_DEFINE_ENUM(NFS4ERR_XDEV); 156TRACE_DEFINE_ENUM(NFS4ERR_XDEV);
157 157
158#define show_nfsv4_errors(error) \ 158#define show_nfsv4_errors(error) \
159 __print_symbolic(-(error), \ 159 __print_symbolic(error, \
160 { NFS4_OK, "OK" }, \ 160 { NFS4_OK, "OK" }, \
161 /* Mapped by nfs4_stat_to_errno() */ \ 161 /* Mapped by nfs4_stat_to_errno() */ \
162 { EPERM, "EPERM" }, \ 162 { EPERM, "EPERM" }, \
@@ -348,7 +348,7 @@ DECLARE_EVENT_CLASS(nfs4_clientid_event,
348 348
349 TP_STRUCT__entry( 349 TP_STRUCT__entry(
350 __string(dstaddr, clp->cl_hostname) 350 __string(dstaddr, clp->cl_hostname)
351 __field(int, error) 351 __field(unsigned long, error)
352 ), 352 ),
353 353
354 TP_fast_assign( 354 TP_fast_assign(
@@ -357,8 +357,8 @@ DECLARE_EVENT_CLASS(nfs4_clientid_event,
357 ), 357 ),
358 358
359 TP_printk( 359 TP_printk(
360 "error=%d (%s) dstaddr=%s", 360 "error=%ld (%s) dstaddr=%s",
361 __entry->error, 361 -__entry->error,
362 show_nfsv4_errors(__entry->error), 362 show_nfsv4_errors(__entry->error),
363 __get_str(dstaddr) 363 __get_str(dstaddr)
364 ) 364 )
@@ -420,7 +420,7 @@ TRACE_EVENT(nfs4_sequence_done,
420 __field(unsigned int, highest_slotid) 420 __field(unsigned int, highest_slotid)
421 __field(unsigned int, target_highest_slotid) 421 __field(unsigned int, target_highest_slotid)
422 __field(unsigned int, status_flags) 422 __field(unsigned int, status_flags)
423 __field(int, error) 423 __field(unsigned long, error)
424 ), 424 ),
425 425
426 TP_fast_assign( 426 TP_fast_assign(
@@ -435,10 +435,10 @@ TRACE_EVENT(nfs4_sequence_done,
435 __entry->error = res->sr_status; 435 __entry->error = res->sr_status;
436 ), 436 ),
437 TP_printk( 437 TP_printk(
438 "error=%d (%s) session=0x%08x slot_nr=%u seq_nr=%u " 438 "error=%ld (%s) session=0x%08x slot_nr=%u seq_nr=%u "
439 "highest_slotid=%u target_highest_slotid=%u " 439 "highest_slotid=%u target_highest_slotid=%u "
440 "status_flags=%u (%s)", 440 "status_flags=%u (%s)",
441 __entry->error, 441 -__entry->error,
442 show_nfsv4_errors(__entry->error), 442 show_nfsv4_errors(__entry->error),
443 __entry->session, 443 __entry->session,
444 __entry->slot_nr, 444 __entry->slot_nr,
@@ -467,7 +467,7 @@ TRACE_EVENT(nfs4_cb_sequence,
467 __field(unsigned int, seq_nr) 467 __field(unsigned int, seq_nr)
468 __field(unsigned int, highest_slotid) 468 __field(unsigned int, highest_slotid)
469 __field(unsigned int, cachethis) 469 __field(unsigned int, cachethis)
470 __field(int, error) 470 __field(unsigned long, error)
471 ), 471 ),
472 472
473 TP_fast_assign( 473 TP_fast_assign(
@@ -476,13 +476,13 @@ TRACE_EVENT(nfs4_cb_sequence,
476 __entry->seq_nr = args->csa_sequenceid; 476 __entry->seq_nr = args->csa_sequenceid;
477 __entry->highest_slotid = args->csa_highestslotid; 477 __entry->highest_slotid = args->csa_highestslotid;
478 __entry->cachethis = args->csa_cachethis; 478 __entry->cachethis = args->csa_cachethis;
479 __entry->error = -be32_to_cpu(status); 479 __entry->error = be32_to_cpu(status);
480 ), 480 ),
481 481
482 TP_printk( 482 TP_printk(
483 "error=%d (%s) session=0x%08x slot_nr=%u seq_nr=%u " 483 "error=%ld (%s) session=0x%08x slot_nr=%u seq_nr=%u "
484 "highest_slotid=%u", 484 "highest_slotid=%u",
485 __entry->error, 485 -__entry->error,
486 show_nfsv4_errors(__entry->error), 486 show_nfsv4_errors(__entry->error),
487 __entry->session, 487 __entry->session,
488 __entry->slot_nr, 488 __entry->slot_nr,
@@ -490,6 +490,44 @@ TRACE_EVENT(nfs4_cb_sequence,
490 __entry->highest_slotid 490 __entry->highest_slotid
491 ) 491 )
492); 492);
493
494TRACE_EVENT(nfs4_cb_seqid_err,
495 TP_PROTO(
496 const struct cb_sequenceargs *args,
497 __be32 status
498 ),
499 TP_ARGS(args, status),
500
501 TP_STRUCT__entry(
502 __field(unsigned int, session)
503 __field(unsigned int, slot_nr)
504 __field(unsigned int, seq_nr)
505 __field(unsigned int, highest_slotid)
506 __field(unsigned int, cachethis)
507 __field(unsigned long, error)
508 ),
509
510 TP_fast_assign(
511 __entry->session = nfs_session_id_hash(&args->csa_sessionid);
512 __entry->slot_nr = args->csa_slotid;
513 __entry->seq_nr = args->csa_sequenceid;
514 __entry->highest_slotid = args->csa_highestslotid;
515 __entry->cachethis = args->csa_cachethis;
516 __entry->error = be32_to_cpu(status);
517 ),
518
519 TP_printk(
520 "error=%ld (%s) session=0x%08x slot_nr=%u seq_nr=%u "
521 "highest_slotid=%u",
522 -__entry->error,
523 show_nfsv4_errors(__entry->error),
524 __entry->session,
525 __entry->slot_nr,
526 __entry->seq_nr,
527 __entry->highest_slotid
528 )
529);
530
493#endif /* CONFIG_NFS_V4_1 */ 531#endif /* CONFIG_NFS_V4_1 */
494 532
495TRACE_EVENT(nfs4_setup_sequence, 533TRACE_EVENT(nfs4_setup_sequence,
@@ -526,26 +564,37 @@ TRACE_EVENT(nfs4_setup_sequence,
526 564
527TRACE_EVENT(nfs4_xdr_status, 565TRACE_EVENT(nfs4_xdr_status,
528 TP_PROTO( 566 TP_PROTO(
567 const struct xdr_stream *xdr,
529 u32 op, 568 u32 op,
530 int error 569 int error
531 ), 570 ),
532 571
533 TP_ARGS(op, error), 572 TP_ARGS(xdr, op, error),
534 573
535 TP_STRUCT__entry( 574 TP_STRUCT__entry(
575 __field(unsigned int, task_id)
576 __field(unsigned int, client_id)
577 __field(u32, xid)
536 __field(u32, op) 578 __field(u32, op)
537 __field(int, error) 579 __field(unsigned long, error)
538 ), 580 ),
539 581
540 TP_fast_assign( 582 TP_fast_assign(
583 const struct rpc_rqst *rqstp = xdr->rqst;
584 const struct rpc_task *task = rqstp->rq_task;
585
586 __entry->task_id = task->tk_pid;
587 __entry->client_id = task->tk_client->cl_clid;
588 __entry->xid = be32_to_cpu(rqstp->rq_xid);
541 __entry->op = op; 589 __entry->op = op;
542 __entry->error = -error; 590 __entry->error = error;
543 ), 591 ),
544 592
545 TP_printk( 593 TP_printk(
546 "operation %d: nfs status %d (%s)", 594 "task:%u@%d xid=0x%08x error=%ld (%s) operation=%u",
547 __entry->op, 595 __entry->task_id, __entry->client_id, __entry->xid,
548 __entry->error, show_nfsv4_errors(__entry->error) 596 -__entry->error, show_nfsv4_errors(__entry->error),
597 __entry->op
549 ) 598 )
550); 599);
551 600
@@ -559,7 +608,7 @@ DECLARE_EVENT_CLASS(nfs4_open_event,
559 TP_ARGS(ctx, flags, error), 608 TP_ARGS(ctx, flags, error),
560 609
561 TP_STRUCT__entry( 610 TP_STRUCT__entry(
562 __field(int, error) 611 __field(unsigned long, error)
563 __field(unsigned int, flags) 612 __field(unsigned int, flags)
564 __field(unsigned int, fmode) 613 __field(unsigned int, fmode)
565 __field(dev_t, dev) 614 __field(dev_t, dev)
@@ -577,7 +626,7 @@ DECLARE_EVENT_CLASS(nfs4_open_event,
577 const struct nfs4_state *state = ctx->state; 626 const struct nfs4_state *state = ctx->state;
578 const struct inode *inode = NULL; 627 const struct inode *inode = NULL;
579 628
580 __entry->error = error; 629 __entry->error = -error;
581 __entry->flags = flags; 630 __entry->flags = flags;
582 __entry->fmode = (__force unsigned int)ctx->mode; 631 __entry->fmode = (__force unsigned int)ctx->mode;
583 __entry->dev = ctx->dentry->d_sb->s_dev; 632 __entry->dev = ctx->dentry->d_sb->s_dev;
@@ -609,11 +658,11 @@ DECLARE_EVENT_CLASS(nfs4_open_event,
609 ), 658 ),
610 659
611 TP_printk( 660 TP_printk(
612 "error=%d (%s) flags=%d (%s) fmode=%s " 661 "error=%ld (%s) flags=%d (%s) fmode=%s "
613 "fileid=%02x:%02x:%llu fhandle=0x%08x " 662 "fileid=%02x:%02x:%llu fhandle=0x%08x "
614 "name=%02x:%02x:%llu/%s stateid=%d:0x%08x " 663 "name=%02x:%02x:%llu/%s stateid=%d:0x%08x "
615 "openstateid=%d:0x%08x", 664 "openstateid=%d:0x%08x",
616 __entry->error, 665 -__entry->error,
617 show_nfsv4_errors(__entry->error), 666 show_nfsv4_errors(__entry->error),
618 __entry->flags, 667 __entry->flags,
619 show_open_flags(__entry->flags), 668 show_open_flags(__entry->flags),
@@ -695,7 +744,7 @@ TRACE_EVENT(nfs4_close,
695 __field(u32, fhandle) 744 __field(u32, fhandle)
696 __field(u64, fileid) 745 __field(u64, fileid)
697 __field(unsigned int, fmode) 746 __field(unsigned int, fmode)
698 __field(int, error) 747 __field(unsigned long, error)
699 __field(int, stateid_seq) 748 __field(int, stateid_seq)
700 __field(u32, stateid_hash) 749 __field(u32, stateid_hash)
701 ), 750 ),
@@ -715,9 +764,9 @@ TRACE_EVENT(nfs4_close,
715 ), 764 ),
716 765
717 TP_printk( 766 TP_printk(
718 "error=%d (%s) fmode=%s fileid=%02x:%02x:%llu " 767 "error=%ld (%s) fmode=%s fileid=%02x:%02x:%llu "
719 "fhandle=0x%08x openstateid=%d:0x%08x", 768 "fhandle=0x%08x openstateid=%d:0x%08x",
720 __entry->error, 769 -__entry->error,
721 show_nfsv4_errors(__entry->error), 770 show_nfsv4_errors(__entry->error),
722 __entry->fmode ? show_fmode_flags(__entry->fmode) : 771 __entry->fmode ? show_fmode_flags(__entry->fmode) :
723 "closed", 772 "closed",
@@ -757,7 +806,7 @@ DECLARE_EVENT_CLASS(nfs4_lock_event,
757 TP_ARGS(request, state, cmd, error), 806 TP_ARGS(request, state, cmd, error),
758 807
759 TP_STRUCT__entry( 808 TP_STRUCT__entry(
760 __field(int, error) 809 __field(unsigned long, error)
761 __field(int, cmd) 810 __field(int, cmd)
762 __field(char, type) 811 __field(char, type)
763 __field(loff_t, start) 812 __field(loff_t, start)
@@ -787,10 +836,10 @@ DECLARE_EVENT_CLASS(nfs4_lock_event,
787 ), 836 ),
788 837
789 TP_printk( 838 TP_printk(
790 "error=%d (%s) cmd=%s:%s range=%lld:%lld " 839 "error=%ld (%s) cmd=%s:%s range=%lld:%lld "
791 "fileid=%02x:%02x:%llu fhandle=0x%08x " 840 "fileid=%02x:%02x:%llu fhandle=0x%08x "
792 "stateid=%d:0x%08x", 841 "stateid=%d:0x%08x",
793 __entry->error, 842 -__entry->error,
794 show_nfsv4_errors(__entry->error), 843 show_nfsv4_errors(__entry->error),
795 show_lock_cmd(__entry->cmd), 844 show_lock_cmd(__entry->cmd),
796 show_lock_type(__entry->type), 845 show_lock_type(__entry->type),
@@ -827,7 +876,7 @@ TRACE_EVENT(nfs4_set_lock,
827 TP_ARGS(request, state, lockstateid, cmd, error), 876 TP_ARGS(request, state, lockstateid, cmd, error),
828 877
829 TP_STRUCT__entry( 878 TP_STRUCT__entry(
830 __field(int, error) 879 __field(unsigned long, error)
831 __field(int, cmd) 880 __field(int, cmd)
832 __field(char, type) 881 __field(char, type)
833 __field(loff_t, start) 882 __field(loff_t, start)
@@ -863,10 +912,10 @@ TRACE_EVENT(nfs4_set_lock,
863 ), 912 ),
864 913
865 TP_printk( 914 TP_printk(
866 "error=%d (%s) cmd=%s:%s range=%lld:%lld " 915 "error=%ld (%s) cmd=%s:%s range=%lld:%lld "
867 "fileid=%02x:%02x:%llu fhandle=0x%08x " 916 "fileid=%02x:%02x:%llu fhandle=0x%08x "
868 "stateid=%d:0x%08x lockstateid=%d:0x%08x", 917 "stateid=%d:0x%08x lockstateid=%d:0x%08x",
869 __entry->error, 918 -__entry->error,
870 show_nfsv4_errors(__entry->error), 919 show_nfsv4_errors(__entry->error),
871 show_lock_cmd(__entry->cmd), 920 show_lock_cmd(__entry->cmd),
872 show_lock_type(__entry->type), 921 show_lock_type(__entry->type),
@@ -932,7 +981,7 @@ TRACE_EVENT(nfs4_delegreturn_exit,
932 TP_STRUCT__entry( 981 TP_STRUCT__entry(
933 __field(dev_t, dev) 982 __field(dev_t, dev)
934 __field(u32, fhandle) 983 __field(u32, fhandle)
935 __field(int, error) 984 __field(unsigned long, error)
936 __field(int, stateid_seq) 985 __field(int, stateid_seq)
937 __field(u32, stateid_hash) 986 __field(u32, stateid_hash)
938 ), 987 ),
@@ -948,9 +997,9 @@ TRACE_EVENT(nfs4_delegreturn_exit,
948 ), 997 ),
949 998
950 TP_printk( 999 TP_printk(
951 "error=%d (%s) dev=%02x:%02x fhandle=0x%08x " 1000 "error=%ld (%s) dev=%02x:%02x fhandle=0x%08x "
952 "stateid=%d:0x%08x", 1001 "stateid=%d:0x%08x",
953 __entry->error, 1002 -__entry->error,
954 show_nfsv4_errors(__entry->error), 1003 show_nfsv4_errors(__entry->error),
955 MAJOR(__entry->dev), MINOR(__entry->dev), 1004 MAJOR(__entry->dev), MINOR(__entry->dev),
956 __entry->fhandle, 1005 __entry->fhandle,
@@ -969,7 +1018,7 @@ DECLARE_EVENT_CLASS(nfs4_test_stateid_event,
969 TP_ARGS(state, lsp, error), 1018 TP_ARGS(state, lsp, error),
970 1019
971 TP_STRUCT__entry( 1020 TP_STRUCT__entry(
972 __field(int, error) 1021 __field(unsigned long, error)
973 __field(dev_t, dev) 1022 __field(dev_t, dev)
974 __field(u32, fhandle) 1023 __field(u32, fhandle)
975 __field(u64, fileid) 1024 __field(u64, fileid)
@@ -991,9 +1040,9 @@ DECLARE_EVENT_CLASS(nfs4_test_stateid_event,
991 ), 1040 ),
992 1041
993 TP_printk( 1042 TP_printk(
994 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1043 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
995 "stateid=%d:0x%08x", 1044 "stateid=%d:0x%08x",
996 __entry->error, 1045 -__entry->error,
997 show_nfsv4_errors(__entry->error), 1046 show_nfsv4_errors(__entry->error),
998 MAJOR(__entry->dev), MINOR(__entry->dev), 1047 MAJOR(__entry->dev), MINOR(__entry->dev),
999 (unsigned long long)__entry->fileid, 1048 (unsigned long long)__entry->fileid,
@@ -1026,7 +1075,7 @@ DECLARE_EVENT_CLASS(nfs4_lookup_event,
1026 1075
1027 TP_STRUCT__entry( 1076 TP_STRUCT__entry(
1028 __field(dev_t, dev) 1077 __field(dev_t, dev)
1029 __field(int, error) 1078 __field(unsigned long, error)
1030 __field(u64, dir) 1079 __field(u64, dir)
1031 __string(name, name->name) 1080 __string(name, name->name)
1032 ), 1081 ),
@@ -1034,13 +1083,13 @@ DECLARE_EVENT_CLASS(nfs4_lookup_event,
1034 TP_fast_assign( 1083 TP_fast_assign(
1035 __entry->dev = dir->i_sb->s_dev; 1084 __entry->dev = dir->i_sb->s_dev;
1036 __entry->dir = NFS_FILEID(dir); 1085 __entry->dir = NFS_FILEID(dir);
1037 __entry->error = error; 1086 __entry->error = -error;
1038 __assign_str(name, name->name); 1087 __assign_str(name, name->name);
1039 ), 1088 ),
1040 1089
1041 TP_printk( 1090 TP_printk(
1042 "error=%d (%s) name=%02x:%02x:%llu/%s", 1091 "error=%ld (%s) name=%02x:%02x:%llu/%s",
1043 __entry->error, 1092 -__entry->error,
1044 show_nfsv4_errors(__entry->error), 1093 show_nfsv4_errors(__entry->error),
1045 MAJOR(__entry->dev), MINOR(__entry->dev), 1094 MAJOR(__entry->dev), MINOR(__entry->dev),
1046 (unsigned long long)__entry->dir, 1095 (unsigned long long)__entry->dir,
@@ -1076,7 +1125,7 @@ TRACE_EVENT(nfs4_lookupp,
1076 TP_STRUCT__entry( 1125 TP_STRUCT__entry(
1077 __field(dev_t, dev) 1126 __field(dev_t, dev)
1078 __field(u64, ino) 1127 __field(u64, ino)
1079 __field(int, error) 1128 __field(unsigned long, error)
1080 ), 1129 ),
1081 1130
1082 TP_fast_assign( 1131 TP_fast_assign(
@@ -1086,8 +1135,8 @@ TRACE_EVENT(nfs4_lookupp,
1086 ), 1135 ),
1087 1136
1088 TP_printk( 1137 TP_printk(
1089 "error=%d (%s) inode=%02x:%02x:%llu", 1138 "error=%ld (%s) inode=%02x:%02x:%llu",
1090 __entry->error, 1139 -__entry->error,
1091 show_nfsv4_errors(__entry->error), 1140 show_nfsv4_errors(__entry->error),
1092 MAJOR(__entry->dev), MINOR(__entry->dev), 1141 MAJOR(__entry->dev), MINOR(__entry->dev),
1093 (unsigned long long)__entry->ino 1142 (unsigned long long)__entry->ino
@@ -1107,7 +1156,7 @@ TRACE_EVENT(nfs4_rename,
1107 1156
1108 TP_STRUCT__entry( 1157 TP_STRUCT__entry(
1109 __field(dev_t, dev) 1158 __field(dev_t, dev)
1110 __field(int, error) 1159 __field(unsigned long, error)
1111 __field(u64, olddir) 1160 __field(u64, olddir)
1112 __string(oldname, oldname->name) 1161 __string(oldname, oldname->name)
1113 __field(u64, newdir) 1162 __field(u64, newdir)
@@ -1124,9 +1173,9 @@ TRACE_EVENT(nfs4_rename,
1124 ), 1173 ),
1125 1174
1126 TP_printk( 1175 TP_printk(
1127 "error=%d (%s) oldname=%02x:%02x:%llu/%s " 1176 "error=%ld (%s) oldname=%02x:%02x:%llu/%s "
1128 "newname=%02x:%02x:%llu/%s", 1177 "newname=%02x:%02x:%llu/%s",
1129 __entry->error, 1178 -__entry->error,
1130 show_nfsv4_errors(__entry->error), 1179 show_nfsv4_errors(__entry->error),
1131 MAJOR(__entry->dev), MINOR(__entry->dev), 1180 MAJOR(__entry->dev), MINOR(__entry->dev),
1132 (unsigned long long)__entry->olddir, 1181 (unsigned long long)__entry->olddir,
@@ -1149,19 +1198,19 @@ DECLARE_EVENT_CLASS(nfs4_inode_event,
1149 __field(dev_t, dev) 1198 __field(dev_t, dev)
1150 __field(u32, fhandle) 1199 __field(u32, fhandle)
1151 __field(u64, fileid) 1200 __field(u64, fileid)
1152 __field(int, error) 1201 __field(unsigned long, error)
1153 ), 1202 ),
1154 1203
1155 TP_fast_assign( 1204 TP_fast_assign(
1156 __entry->dev = inode->i_sb->s_dev; 1205 __entry->dev = inode->i_sb->s_dev;
1157 __entry->fileid = NFS_FILEID(inode); 1206 __entry->fileid = NFS_FILEID(inode);
1158 __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); 1207 __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
1159 __entry->error = error; 1208 __entry->error = error < 0 ? -error : 0;
1160 ), 1209 ),
1161 1210
1162 TP_printk( 1211 TP_printk(
1163 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x", 1212 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x",
1164 __entry->error, 1213 -__entry->error,
1165 show_nfsv4_errors(__entry->error), 1214 show_nfsv4_errors(__entry->error),
1166 MAJOR(__entry->dev), MINOR(__entry->dev), 1215 MAJOR(__entry->dev), MINOR(__entry->dev),
1167 (unsigned long long)__entry->fileid, 1216 (unsigned long long)__entry->fileid,
@@ -1200,7 +1249,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_event,
1200 __field(dev_t, dev) 1249 __field(dev_t, dev)
1201 __field(u32, fhandle) 1250 __field(u32, fhandle)
1202 __field(u64, fileid) 1251 __field(u64, fileid)
1203 __field(int, error) 1252 __field(unsigned long, error)
1204 __field(int, stateid_seq) 1253 __field(int, stateid_seq)
1205 __field(u32, stateid_hash) 1254 __field(u32, stateid_hash)
1206 ), 1255 ),
@@ -1217,9 +1266,9 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_event,
1217 ), 1266 ),
1218 1267
1219 TP_printk( 1268 TP_printk(
1220 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1269 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1221 "stateid=%d:0x%08x", 1270 "stateid=%d:0x%08x",
1222 __entry->error, 1271 -__entry->error,
1223 show_nfsv4_errors(__entry->error), 1272 show_nfsv4_errors(__entry->error),
1224 MAJOR(__entry->dev), MINOR(__entry->dev), 1273 MAJOR(__entry->dev), MINOR(__entry->dev),
1225 (unsigned long long)__entry->fileid, 1274 (unsigned long long)__entry->fileid,
@@ -1257,7 +1306,7 @@ DECLARE_EVENT_CLASS(nfs4_getattr_event,
1257 __field(u32, fhandle) 1306 __field(u32, fhandle)
1258 __field(u64, fileid) 1307 __field(u64, fileid)
1259 __field(unsigned int, valid) 1308 __field(unsigned int, valid)
1260 __field(int, error) 1309 __field(unsigned long, error)
1261 ), 1310 ),
1262 1311
1263 TP_fast_assign( 1312 TP_fast_assign(
@@ -1269,9 +1318,9 @@ DECLARE_EVENT_CLASS(nfs4_getattr_event,
1269 ), 1318 ),
1270 1319
1271 TP_printk( 1320 TP_printk(
1272 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1321 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1273 "valid=%s", 1322 "valid=%s",
1274 __entry->error, 1323 -__entry->error,
1275 show_nfsv4_errors(__entry->error), 1324 show_nfsv4_errors(__entry->error),
1276 MAJOR(__entry->dev), MINOR(__entry->dev), 1325 MAJOR(__entry->dev), MINOR(__entry->dev),
1277 (unsigned long long)__entry->fileid, 1326 (unsigned long long)__entry->fileid,
@@ -1304,7 +1353,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_callback_event,
1304 TP_ARGS(clp, fhandle, inode, error), 1353 TP_ARGS(clp, fhandle, inode, error),
1305 1354
1306 TP_STRUCT__entry( 1355 TP_STRUCT__entry(
1307 __field(int, error) 1356 __field(unsigned long, error)
1308 __field(dev_t, dev) 1357 __field(dev_t, dev)
1309 __field(u32, fhandle) 1358 __field(u32, fhandle)
1310 __field(u64, fileid) 1359 __field(u64, fileid)
@@ -1325,9 +1374,9 @@ DECLARE_EVENT_CLASS(nfs4_inode_callback_event,
1325 ), 1374 ),
1326 1375
1327 TP_printk( 1376 TP_printk(
1328 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1377 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1329 "dstaddr=%s", 1378 "dstaddr=%s",
1330 __entry->error, 1379 -__entry->error,
1331 show_nfsv4_errors(__entry->error), 1380 show_nfsv4_errors(__entry->error),
1332 MAJOR(__entry->dev), MINOR(__entry->dev), 1381 MAJOR(__entry->dev), MINOR(__entry->dev),
1333 (unsigned long long)__entry->fileid, 1382 (unsigned long long)__entry->fileid,
@@ -1359,7 +1408,7 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_callback_event,
1359 TP_ARGS(clp, fhandle, inode, stateid, error), 1408 TP_ARGS(clp, fhandle, inode, stateid, error),
1360 1409
1361 TP_STRUCT__entry( 1410 TP_STRUCT__entry(
1362 __field(int, error) 1411 __field(unsigned long, error)
1363 __field(dev_t, dev) 1412 __field(dev_t, dev)
1364 __field(u32, fhandle) 1413 __field(u32, fhandle)
1365 __field(u64, fileid) 1414 __field(u64, fileid)
@@ -1386,9 +1435,9 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_callback_event,
1386 ), 1435 ),
1387 1436
1388 TP_printk( 1437 TP_printk(
1389 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1438 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1390 "stateid=%d:0x%08x dstaddr=%s", 1439 "stateid=%d:0x%08x dstaddr=%s",
1391 __entry->error, 1440 -__entry->error,
1392 show_nfsv4_errors(__entry->error), 1441 show_nfsv4_errors(__entry->error),
1393 MAJOR(__entry->dev), MINOR(__entry->dev), 1442 MAJOR(__entry->dev), MINOR(__entry->dev),
1394 (unsigned long long)__entry->fileid, 1443 (unsigned long long)__entry->fileid,
@@ -1422,7 +1471,7 @@ DECLARE_EVENT_CLASS(nfs4_idmap_event,
1422 TP_ARGS(name, len, id, error), 1471 TP_ARGS(name, len, id, error),
1423 1472
1424 TP_STRUCT__entry( 1473 TP_STRUCT__entry(
1425 __field(int, error) 1474 __field(unsigned long, error)
1426 __field(u32, id) 1475 __field(u32, id)
1427 __dynamic_array(char, name, len > 0 ? len + 1 : 1) 1476 __dynamic_array(char, name, len > 0 ? len + 1 : 1)
1428 ), 1477 ),
@@ -1437,8 +1486,8 @@ DECLARE_EVENT_CLASS(nfs4_idmap_event,
1437 ), 1486 ),
1438 1487
1439 TP_printk( 1488 TP_printk(
1440 "error=%d id=%u name=%s", 1489 "error=%ld (%s) id=%u name=%s",
1441 __entry->error, 1490 -__entry->error, show_nfsv4_errors(__entry->error),
1442 __entry->id, 1491 __entry->id,
1443 __get_str(name) 1492 __get_str(name)
1444 ) 1493 )
@@ -1471,7 +1520,7 @@ DECLARE_EVENT_CLASS(nfs4_read_event,
1471 __field(u64, fileid) 1520 __field(u64, fileid)
1472 __field(loff_t, offset) 1521 __field(loff_t, offset)
1473 __field(size_t, count) 1522 __field(size_t, count)
1474 __field(int, error) 1523 __field(unsigned long, error)
1475 __field(int, stateid_seq) 1524 __field(int, stateid_seq)
1476 __field(u32, stateid_hash) 1525 __field(u32, stateid_hash)
1477 ), 1526 ),
@@ -1485,7 +1534,7 @@ DECLARE_EVENT_CLASS(nfs4_read_event,
1485 __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); 1534 __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
1486 __entry->offset = hdr->args.offset; 1535 __entry->offset = hdr->args.offset;
1487 __entry->count = hdr->args.count; 1536 __entry->count = hdr->args.count;
1488 __entry->error = error; 1537 __entry->error = error < 0 ? -error : 0;
1489 __entry->stateid_seq = 1538 __entry->stateid_seq =
1490 be32_to_cpu(state->stateid.seqid); 1539 be32_to_cpu(state->stateid.seqid);
1491 __entry->stateid_hash = 1540 __entry->stateid_hash =
@@ -1493,9 +1542,9 @@ DECLARE_EVENT_CLASS(nfs4_read_event,
1493 ), 1542 ),
1494 1543
1495 TP_printk( 1544 TP_printk(
1496 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1545 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1497 "offset=%lld count=%zu stateid=%d:0x%08x", 1546 "offset=%lld count=%zu stateid=%d:0x%08x",
1498 __entry->error, 1547 -__entry->error,
1499 show_nfsv4_errors(__entry->error), 1548 show_nfsv4_errors(__entry->error),
1500 MAJOR(__entry->dev), MINOR(__entry->dev), 1549 MAJOR(__entry->dev), MINOR(__entry->dev),
1501 (unsigned long long)__entry->fileid, 1550 (unsigned long long)__entry->fileid,
@@ -1531,7 +1580,7 @@ DECLARE_EVENT_CLASS(nfs4_write_event,
1531 __field(u64, fileid) 1580 __field(u64, fileid)
1532 __field(loff_t, offset) 1581 __field(loff_t, offset)
1533 __field(size_t, count) 1582 __field(size_t, count)
1534 __field(int, error) 1583 __field(unsigned long, error)
1535 __field(int, stateid_seq) 1584 __field(int, stateid_seq)
1536 __field(u32, stateid_hash) 1585 __field(u32, stateid_hash)
1537 ), 1586 ),
@@ -1545,7 +1594,7 @@ DECLARE_EVENT_CLASS(nfs4_write_event,
1545 __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); 1594 __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
1546 __entry->offset = hdr->args.offset; 1595 __entry->offset = hdr->args.offset;
1547 __entry->count = hdr->args.count; 1596 __entry->count = hdr->args.count;
1548 __entry->error = error; 1597 __entry->error = error < 0 ? -error : 0;
1549 __entry->stateid_seq = 1598 __entry->stateid_seq =
1550 be32_to_cpu(state->stateid.seqid); 1599 be32_to_cpu(state->stateid.seqid);
1551 __entry->stateid_hash = 1600 __entry->stateid_hash =
@@ -1553,9 +1602,9 @@ DECLARE_EVENT_CLASS(nfs4_write_event,
1553 ), 1602 ),
1554 1603
1555 TP_printk( 1604 TP_printk(
1556 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1605 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1557 "offset=%lld count=%zu stateid=%d:0x%08x", 1606 "offset=%lld count=%zu stateid=%d:0x%08x",
1558 __entry->error, 1607 -__entry->error,
1559 show_nfsv4_errors(__entry->error), 1608 show_nfsv4_errors(__entry->error),
1560 MAJOR(__entry->dev), MINOR(__entry->dev), 1609 MAJOR(__entry->dev), MINOR(__entry->dev),
1561 (unsigned long long)__entry->fileid, 1610 (unsigned long long)__entry->fileid,
@@ -1592,7 +1641,7 @@ DECLARE_EVENT_CLASS(nfs4_commit_event,
1592 __field(u64, fileid) 1641 __field(u64, fileid)
1593 __field(loff_t, offset) 1642 __field(loff_t, offset)
1594 __field(size_t, count) 1643 __field(size_t, count)
1595 __field(int, error) 1644 __field(unsigned long, error)
1596 ), 1645 ),
1597 1646
1598 TP_fast_assign( 1647 TP_fast_assign(
@@ -1606,9 +1655,9 @@ DECLARE_EVENT_CLASS(nfs4_commit_event,
1606 ), 1655 ),
1607 1656
1608 TP_printk( 1657 TP_printk(
1609 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1658 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1610 "offset=%lld count=%zu", 1659 "offset=%lld count=%zu",
1611 __entry->error, 1660 -__entry->error,
1612 show_nfsv4_errors(__entry->error), 1661 show_nfsv4_errors(__entry->error),
1613 MAJOR(__entry->dev), MINOR(__entry->dev), 1662 MAJOR(__entry->dev), MINOR(__entry->dev),
1614 (unsigned long long)__entry->fileid, 1663 (unsigned long long)__entry->fileid,
@@ -1656,7 +1705,7 @@ TRACE_EVENT(nfs4_layoutget,
1656 __field(u32, iomode) 1705 __field(u32, iomode)
1657 __field(u64, offset) 1706 __field(u64, offset)
1658 __field(u64, count) 1707 __field(u64, count)
1659 __field(int, error) 1708 __field(unsigned long, error)
1660 __field(int, stateid_seq) 1709 __field(int, stateid_seq)
1661 __field(u32, stateid_hash) 1710 __field(u32, stateid_hash)
1662 __field(int, layoutstateid_seq) 1711 __field(int, layoutstateid_seq)
@@ -1689,10 +1738,10 @@ TRACE_EVENT(nfs4_layoutget,
1689 ), 1738 ),
1690 1739
1691 TP_printk( 1740 TP_printk(
1692 "error=%d (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " 1741 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
1693 "iomode=%s offset=%llu count=%llu stateid=%d:0x%08x " 1742 "iomode=%s offset=%llu count=%llu stateid=%d:0x%08x "
1694 "layoutstateid=%d:0x%08x", 1743 "layoutstateid=%d:0x%08x",
1695 __entry->error, 1744 -__entry->error,
1696 show_nfsv4_errors(__entry->error), 1745 show_nfsv4_errors(__entry->error),
1697 MAJOR(__entry->dev), MINOR(__entry->dev), 1746 MAJOR(__entry->dev), MINOR(__entry->dev),
1698 (unsigned long long)__entry->fileid, 1747 (unsigned long long)__entry->fileid,
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index 602446158bfb..d974ff3164ba 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -3187,7 +3187,7 @@ static bool __decode_op_hdr(struct xdr_stream *xdr, enum nfs_opnum4 expected,
3187 return true; 3187 return true;
3188out_status: 3188out_status:
3189 nfserr = be32_to_cpup(p); 3189 nfserr = be32_to_cpup(p);
3190 trace_nfs4_xdr_status(opnum, nfserr); 3190 trace_nfs4_xdr_status(xdr, opnum, nfserr);
3191 *nfs_retval = nfs4_stat_to_errno(nfserr); 3191 *nfs_retval = nfs4_stat_to_errno(nfserr);
3192 return true; 3192 return true;
3193out_bad_operation: 3193out_bad_operation:
diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h
index a0d6910aa03a..976d4089e267 100644
--- a/fs/nfs/nfstrace.h
+++ b/fs/nfs/nfstrace.h
@@ -11,6 +11,16 @@
11#include <linux/tracepoint.h> 11#include <linux/tracepoint.h>
12#include <linux/iversion.h> 12#include <linux/iversion.h>
13 13
14TRACE_DEFINE_ENUM(DT_UNKNOWN);
15TRACE_DEFINE_ENUM(DT_FIFO);
16TRACE_DEFINE_ENUM(DT_CHR);
17TRACE_DEFINE_ENUM(DT_DIR);
18TRACE_DEFINE_ENUM(DT_BLK);
19TRACE_DEFINE_ENUM(DT_REG);
20TRACE_DEFINE_ENUM(DT_LNK);
21TRACE_DEFINE_ENUM(DT_SOCK);
22TRACE_DEFINE_ENUM(DT_WHT);
23
14#define nfs_show_file_type(ftype) \ 24#define nfs_show_file_type(ftype) \
15 __print_symbolic(ftype, \ 25 __print_symbolic(ftype, \
16 { DT_UNKNOWN, "UNKNOWN" }, \ 26 { DT_UNKNOWN, "UNKNOWN" }, \
@@ -23,25 +33,57 @@
23 { DT_SOCK, "SOCK" }, \ 33 { DT_SOCK, "SOCK" }, \
24 { DT_WHT, "WHT" }) 34 { DT_WHT, "WHT" })
25 35
36TRACE_DEFINE_ENUM(NFS_INO_INVALID_DATA);
37TRACE_DEFINE_ENUM(NFS_INO_INVALID_ATIME);
38TRACE_DEFINE_ENUM(NFS_INO_INVALID_ACCESS);
39TRACE_DEFINE_ENUM(NFS_INO_INVALID_ACL);
40TRACE_DEFINE_ENUM(NFS_INO_REVAL_PAGECACHE);
41TRACE_DEFINE_ENUM(NFS_INO_REVAL_FORCED);
42TRACE_DEFINE_ENUM(NFS_INO_INVALID_LABEL);
43TRACE_DEFINE_ENUM(NFS_INO_INVALID_CHANGE);
44TRACE_DEFINE_ENUM(NFS_INO_INVALID_CTIME);
45TRACE_DEFINE_ENUM(NFS_INO_INVALID_MTIME);
46TRACE_DEFINE_ENUM(NFS_INO_INVALID_SIZE);
47TRACE_DEFINE_ENUM(NFS_INO_INVALID_OTHER);
48
26#define nfs_show_cache_validity(v) \ 49#define nfs_show_cache_validity(v) \
27 __print_flags(v, "|", \ 50 __print_flags(v, "|", \
28 { NFS_INO_INVALID_ATTR, "INVALID_ATTR" }, \
29 { NFS_INO_INVALID_DATA, "INVALID_DATA" }, \ 51 { NFS_INO_INVALID_DATA, "INVALID_DATA" }, \
30 { NFS_INO_INVALID_ATIME, "INVALID_ATIME" }, \ 52 { NFS_INO_INVALID_ATIME, "INVALID_ATIME" }, \
31 { NFS_INO_INVALID_ACCESS, "INVALID_ACCESS" }, \ 53 { NFS_INO_INVALID_ACCESS, "INVALID_ACCESS" }, \
32 { NFS_INO_INVALID_ACL, "INVALID_ACL" }, \ 54 { NFS_INO_INVALID_ACL, "INVALID_ACL" }, \
33 { NFS_INO_REVAL_PAGECACHE, "REVAL_PAGECACHE" }, \ 55 { NFS_INO_REVAL_PAGECACHE, "REVAL_PAGECACHE" }, \
34 { NFS_INO_REVAL_FORCED, "REVAL_FORCED" }, \ 56 { NFS_INO_REVAL_FORCED, "REVAL_FORCED" }, \
35 { NFS_INO_INVALID_LABEL, "INVALID_LABEL" }) 57 { NFS_INO_INVALID_LABEL, "INVALID_LABEL" }, \
58 { NFS_INO_INVALID_CHANGE, "INVALID_CHANGE" }, \
59 { NFS_INO_INVALID_CTIME, "INVALID_CTIME" }, \
60 { NFS_INO_INVALID_MTIME, "INVALID_MTIME" }, \
61 { NFS_INO_INVALID_SIZE, "INVALID_SIZE" }, \
62 { NFS_INO_INVALID_OTHER, "INVALID_OTHER" })
63
64TRACE_DEFINE_ENUM(NFS_INO_ADVISE_RDPLUS);
65TRACE_DEFINE_ENUM(NFS_INO_STALE);
66TRACE_DEFINE_ENUM(NFS_INO_ACL_LRU_SET);
67TRACE_DEFINE_ENUM(NFS_INO_INVALIDATING);
68TRACE_DEFINE_ENUM(NFS_INO_FSCACHE);
69TRACE_DEFINE_ENUM(NFS_INO_FSCACHE_LOCK);
70TRACE_DEFINE_ENUM(NFS_INO_LAYOUTCOMMIT);
71TRACE_DEFINE_ENUM(NFS_INO_LAYOUTCOMMITTING);
72TRACE_DEFINE_ENUM(NFS_INO_LAYOUTSTATS);
73TRACE_DEFINE_ENUM(NFS_INO_ODIRECT);
36 74
37#define nfs_show_nfsi_flags(v) \ 75#define nfs_show_nfsi_flags(v) \
38 __print_flags(v, "|", \ 76 __print_flags(v, "|", \
39 { 1 << NFS_INO_ADVISE_RDPLUS, "ADVISE_RDPLUS" }, \ 77 { BIT(NFS_INO_ADVISE_RDPLUS), "ADVISE_RDPLUS" }, \
40 { 1 << NFS_INO_STALE, "STALE" }, \ 78 { BIT(NFS_INO_STALE), "STALE" }, \
41 { 1 << NFS_INO_INVALIDATING, "INVALIDATING" }, \ 79 { BIT(NFS_INO_ACL_LRU_SET), "ACL_LRU_SET" }, \
42 { 1 << NFS_INO_FSCACHE, "FSCACHE" }, \ 80 { BIT(NFS_INO_INVALIDATING), "INVALIDATING" }, \
43 { 1 << NFS_INO_LAYOUTCOMMIT, "NEED_LAYOUTCOMMIT" }, \ 81 { BIT(NFS_INO_FSCACHE), "FSCACHE" }, \
44 { 1 << NFS_INO_LAYOUTCOMMITTING, "LAYOUTCOMMIT" }) 82 { BIT(NFS_INO_FSCACHE_LOCK), "FSCACHE_LOCK" }, \
83 { BIT(NFS_INO_LAYOUTCOMMIT), "NEED_LAYOUTCOMMIT" }, \
84 { BIT(NFS_INO_LAYOUTCOMMITTING), "LAYOUTCOMMIT" }, \
85 { BIT(NFS_INO_LAYOUTSTATS), "LAYOUTSTATS" }, \
86 { BIT(NFS_INO_ODIRECT), "ODIRECT" })
45 87
46DECLARE_EVENT_CLASS(nfs_inode_event, 88DECLARE_EVENT_CLASS(nfs_inode_event,
47 TP_PROTO( 89 TP_PROTO(
@@ -83,7 +125,7 @@ DECLARE_EVENT_CLASS(nfs_inode_event_done,
83 TP_ARGS(inode, error), 125 TP_ARGS(inode, error),
84 126
85 TP_STRUCT__entry( 127 TP_STRUCT__entry(
86 __field(int, error) 128 __field(unsigned long, error)
87 __field(dev_t, dev) 129 __field(dev_t, dev)
88 __field(u32, fhandle) 130 __field(u32, fhandle)
89 __field(unsigned char, type) 131 __field(unsigned char, type)
@@ -96,7 +138,7 @@ DECLARE_EVENT_CLASS(nfs_inode_event_done,
96 138
97 TP_fast_assign( 139 TP_fast_assign(
98 const struct nfs_inode *nfsi = NFS_I(inode); 140 const struct nfs_inode *nfsi = NFS_I(inode);
99 __entry->error = error; 141 __entry->error = error < 0 ? -error : 0;
100 __entry->dev = inode->i_sb->s_dev; 142 __entry->dev = inode->i_sb->s_dev;
101 __entry->fileid = nfsi->fileid; 143 __entry->fileid = nfsi->fileid;
102 __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); 144 __entry->fhandle = nfs_fhandle_hash(&nfsi->fh);
@@ -108,10 +150,10 @@ DECLARE_EVENT_CLASS(nfs_inode_event_done,
108 ), 150 ),
109 151
110 TP_printk( 152 TP_printk(
111 "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " 153 "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
112 "type=%u (%s) version=%llu size=%lld " 154 "type=%u (%s) version=%llu size=%lld "
113 "cache_validity=%lu (%s) nfs_flags=%ld (%s)", 155 "cache_validity=0x%lx (%s) nfs_flags=0x%lx (%s)",
114 __entry->error, 156 -__entry->error, nfs_show_status(__entry->error),
115 MAJOR(__entry->dev), MINOR(__entry->dev), 157 MAJOR(__entry->dev), MINOR(__entry->dev),
116 (unsigned long long)__entry->fileid, 158 (unsigned long long)__entry->fileid,
117 __entry->fhandle, 159 __entry->fhandle,
@@ -158,13 +200,41 @@ DEFINE_NFS_INODE_EVENT_DONE(nfs_fsync_exit);
158DEFINE_NFS_INODE_EVENT(nfs_access_enter); 200DEFINE_NFS_INODE_EVENT(nfs_access_enter);
159DEFINE_NFS_INODE_EVENT_DONE(nfs_access_exit); 201DEFINE_NFS_INODE_EVENT_DONE(nfs_access_exit);
160 202
203TRACE_DEFINE_ENUM(LOOKUP_FOLLOW);
204TRACE_DEFINE_ENUM(LOOKUP_DIRECTORY);
205TRACE_DEFINE_ENUM(LOOKUP_AUTOMOUNT);
206TRACE_DEFINE_ENUM(LOOKUP_PARENT);
207TRACE_DEFINE_ENUM(LOOKUP_REVAL);
208TRACE_DEFINE_ENUM(LOOKUP_RCU);
209TRACE_DEFINE_ENUM(LOOKUP_NO_REVAL);
210TRACE_DEFINE_ENUM(LOOKUP_NO_EVAL);
211TRACE_DEFINE_ENUM(LOOKUP_OPEN);
212TRACE_DEFINE_ENUM(LOOKUP_CREATE);
213TRACE_DEFINE_ENUM(LOOKUP_EXCL);
214TRACE_DEFINE_ENUM(LOOKUP_RENAME_TARGET);
215TRACE_DEFINE_ENUM(LOOKUP_JUMPED);
216TRACE_DEFINE_ENUM(LOOKUP_ROOT);
217TRACE_DEFINE_ENUM(LOOKUP_EMPTY);
218TRACE_DEFINE_ENUM(LOOKUP_DOWN);
219
161#define show_lookup_flags(flags) \ 220#define show_lookup_flags(flags) \
162 __print_flags((unsigned long)flags, "|", \ 221 __print_flags(flags, "|", \
163 { LOOKUP_AUTOMOUNT, "AUTOMOUNT" }, \ 222 { LOOKUP_FOLLOW, "FOLLOW" }, \
164 { LOOKUP_DIRECTORY, "DIRECTORY" }, \ 223 { LOOKUP_DIRECTORY, "DIRECTORY" }, \
224 { LOOKUP_AUTOMOUNT, "AUTOMOUNT" }, \
225 { LOOKUP_PARENT, "PARENT" }, \
226 { LOOKUP_REVAL, "REVAL" }, \
227 { LOOKUP_RCU, "RCU" }, \
228 { LOOKUP_NO_REVAL, "NO_REVAL" }, \
229 { LOOKUP_NO_EVAL, "NO_EVAL" }, \
165 { LOOKUP_OPEN, "OPEN" }, \ 230 { LOOKUP_OPEN, "OPEN" }, \
166 { LOOKUP_CREATE, "CREATE" }, \ 231 { LOOKUP_CREATE, "CREATE" }, \
167 { LOOKUP_EXCL, "EXCL" }) 232 { LOOKUP_EXCL, "EXCL" }, \
233 { LOOKUP_RENAME_TARGET, "RENAME_TARGET" }, \
234 { LOOKUP_JUMPED, "JUMPED" }, \
235 { LOOKUP_ROOT, "ROOT" }, \
236 { LOOKUP_EMPTY, "EMPTY" }, \
237 { LOOKUP_DOWN, "DOWN" })
168 238
169DECLARE_EVENT_CLASS(nfs_lookup_event, 239DECLARE_EVENT_CLASS(nfs_lookup_event,
170 TP_PROTO( 240 TP_PROTO(
@@ -176,7 +246,7 @@ DECLARE_EVENT_CLASS(nfs_lookup_event,
176 TP_ARGS(dir, dentry, flags), 246 TP_ARGS(dir, dentry, flags),
177 247
178 TP_STRUCT__entry( 248 TP_STRUCT__entry(
179 __field(unsigned int, flags) 249 __field(unsigned long, flags)
180 __field(dev_t, dev) 250 __field(dev_t, dev)
181 __field(u64, dir) 251 __field(u64, dir)
182 __string(name, dentry->d_name.name) 252 __string(name, dentry->d_name.name)
@@ -190,7 +260,7 @@ DECLARE_EVENT_CLASS(nfs_lookup_event,
190 ), 260 ),
191 261
192 TP_printk( 262 TP_printk(
193 "flags=%u (%s) name=%02x:%02x:%llu/%s", 263 "flags=0x%lx (%s) name=%02x:%02x:%llu/%s",
194 __entry->flags, 264 __entry->flags,
195 show_lookup_flags(__entry->flags), 265 show_lookup_flags(__entry->flags),
196 MAJOR(__entry->dev), MINOR(__entry->dev), 266 MAJOR(__entry->dev), MINOR(__entry->dev),
@@ -219,8 +289,8 @@ DECLARE_EVENT_CLASS(nfs_lookup_event_done,
219 TP_ARGS(dir, dentry, flags, error), 289 TP_ARGS(dir, dentry, flags, error),
220 290
221 TP_STRUCT__entry( 291 TP_STRUCT__entry(
222 __field(int, error) 292 __field(unsigned long, error)
223 __field(unsigned int, flags) 293 __field(unsigned long, flags)
224 __field(dev_t, dev) 294 __field(dev_t, dev)
225 __field(u64, dir) 295 __field(u64, dir)
226 __string(name, dentry->d_name.name) 296 __string(name, dentry->d_name.name)
@@ -229,14 +299,14 @@ DECLARE_EVENT_CLASS(nfs_lookup_event_done,
229 TP_fast_assign( 299 TP_fast_assign(
230 __entry->dev = dir->i_sb->s_dev; 300 __entry->dev = dir->i_sb->s_dev;
231 __entry->dir = NFS_FILEID(dir); 301 __entry->dir = NFS_FILEID(dir);
232 __entry->error = error; 302 __entry->error = error < 0 ? -error : 0;
233 __entry->flags = flags; 303 __entry->flags = flags;
234 __assign_str(name, dentry->d_name.name); 304 __assign_str(name, dentry->d_name.name);
235 ), 305 ),
236 306
237 TP_printk( 307 TP_printk(
238 "error=%d flags=%u (%s) name=%02x:%02x:%llu/%s", 308 "error=%ld (%s) flags=0x%lx (%s) name=%02x:%02x:%llu/%s",
239 __entry->error, 309 -__entry->error, nfs_show_status(__entry->error),
240 __entry->flags, 310 __entry->flags,
241 show_lookup_flags(__entry->flags), 311 show_lookup_flags(__entry->flags),
242 MAJOR(__entry->dev), MINOR(__entry->dev), 312 MAJOR(__entry->dev), MINOR(__entry->dev),
@@ -260,15 +330,43 @@ DEFINE_NFS_LOOKUP_EVENT_DONE(nfs_lookup_exit);
260DEFINE_NFS_LOOKUP_EVENT(nfs_lookup_revalidate_enter); 330DEFINE_NFS_LOOKUP_EVENT(nfs_lookup_revalidate_enter);
261DEFINE_NFS_LOOKUP_EVENT_DONE(nfs_lookup_revalidate_exit); 331DEFINE_NFS_LOOKUP_EVENT_DONE(nfs_lookup_revalidate_exit);
262 332
333TRACE_DEFINE_ENUM(O_WRONLY);
334TRACE_DEFINE_ENUM(O_RDWR);
335TRACE_DEFINE_ENUM(O_CREAT);
336TRACE_DEFINE_ENUM(O_EXCL);
337TRACE_DEFINE_ENUM(O_NOCTTY);
338TRACE_DEFINE_ENUM(O_TRUNC);
339TRACE_DEFINE_ENUM(O_APPEND);
340TRACE_DEFINE_ENUM(O_NONBLOCK);
341TRACE_DEFINE_ENUM(O_DSYNC);
342TRACE_DEFINE_ENUM(O_DIRECT);
343TRACE_DEFINE_ENUM(O_LARGEFILE);
344TRACE_DEFINE_ENUM(O_DIRECTORY);
345TRACE_DEFINE_ENUM(O_NOFOLLOW);
346TRACE_DEFINE_ENUM(O_NOATIME);
347TRACE_DEFINE_ENUM(O_CLOEXEC);
348
263#define show_open_flags(flags) \ 349#define show_open_flags(flags) \
264 __print_flags((unsigned long)flags, "|", \ 350 __print_flags(flags, "|", \
351 { O_WRONLY, "O_WRONLY" }, \
352 { O_RDWR, "O_RDWR" }, \
265 { O_CREAT, "O_CREAT" }, \ 353 { O_CREAT, "O_CREAT" }, \
266 { O_EXCL, "O_EXCL" }, \ 354 { O_EXCL, "O_EXCL" }, \
355 { O_NOCTTY, "O_NOCTTY" }, \
267 { O_TRUNC, "O_TRUNC" }, \ 356 { O_TRUNC, "O_TRUNC" }, \
268 { O_APPEND, "O_APPEND" }, \ 357 { O_APPEND, "O_APPEND" }, \
358 { O_NONBLOCK, "O_NONBLOCK" }, \
269 { O_DSYNC, "O_DSYNC" }, \ 359 { O_DSYNC, "O_DSYNC" }, \
270 { O_DIRECT, "O_DIRECT" }, \ 360 { O_DIRECT, "O_DIRECT" }, \
271 { O_DIRECTORY, "O_DIRECTORY" }) 361 { O_LARGEFILE, "O_LARGEFILE" }, \
362 { O_DIRECTORY, "O_DIRECTORY" }, \
363 { O_NOFOLLOW, "O_NOFOLLOW" }, \
364 { O_NOATIME, "O_NOATIME" }, \
365 { O_CLOEXEC, "O_CLOEXEC" })
366
367TRACE_DEFINE_ENUM(FMODE_READ);
368TRACE_DEFINE_ENUM(FMODE_WRITE);
369TRACE_DEFINE_ENUM(FMODE_EXEC);
272 370
273#define show_fmode_flags(mode) \ 371#define show_fmode_flags(mode) \
274 __print_flags(mode, "|", \ 372 __print_flags(mode, "|", \
@@ -286,7 +384,7 @@ TRACE_EVENT(nfs_atomic_open_enter,
286 TP_ARGS(dir, ctx, flags), 384 TP_ARGS(dir, ctx, flags),
287 385
288 TP_STRUCT__entry( 386 TP_STRUCT__entry(
289 __field(unsigned int, flags) 387 __field(unsigned long, flags)
290 __field(unsigned int, fmode) 388 __field(unsigned int, fmode)
291 __field(dev_t, dev) 389 __field(dev_t, dev)
292 __field(u64, dir) 390 __field(u64, dir)
@@ -302,7 +400,7 @@ TRACE_EVENT(nfs_atomic_open_enter,
302 ), 400 ),
303 401
304 TP_printk( 402 TP_printk(
305 "flags=%u (%s) fmode=%s name=%02x:%02x:%llu/%s", 403 "flags=0x%lx (%s) fmode=%s name=%02x:%02x:%llu/%s",
306 __entry->flags, 404 __entry->flags,
307 show_open_flags(__entry->flags), 405 show_open_flags(__entry->flags),
308 show_fmode_flags(__entry->fmode), 406 show_fmode_flags(__entry->fmode),
@@ -323,8 +421,8 @@ TRACE_EVENT(nfs_atomic_open_exit,
323 TP_ARGS(dir, ctx, flags, error), 421 TP_ARGS(dir, ctx, flags, error),
324 422
325 TP_STRUCT__entry( 423 TP_STRUCT__entry(
326 __field(int, error) 424 __field(unsigned long, error)
327 __field(unsigned int, flags) 425 __field(unsigned long, flags)
328 __field(unsigned int, fmode) 426 __field(unsigned int, fmode)
329 __field(dev_t, dev) 427 __field(dev_t, dev)
330 __field(u64, dir) 428 __field(u64, dir)
@@ -332,7 +430,7 @@ TRACE_EVENT(nfs_atomic_open_exit,
332 ), 430 ),
333 431
334 TP_fast_assign( 432 TP_fast_assign(
335 __entry->error = error; 433 __entry->error = -error;
336 __entry->dev = dir->i_sb->s_dev; 434 __entry->dev = dir->i_sb->s_dev;
337 __entry->dir = NFS_FILEID(dir); 435 __entry->dir = NFS_FILEID(dir);
338 __entry->flags = flags; 436 __entry->flags = flags;
@@ -341,9 +439,9 @@ TRACE_EVENT(nfs_atomic_open_exit,
341 ), 439 ),
342 440
343 TP_printk( 441 TP_printk(
344 "error=%d flags=%u (%s) fmode=%s " 442 "error=%ld (%s) flags=0x%lx (%s) fmode=%s "
345 "name=%02x:%02x:%llu/%s", 443 "name=%02x:%02x:%llu/%s",
346 __entry->error, 444 -__entry->error, nfs_show_status(__entry->error),
347 __entry->flags, 445 __entry->flags,
348 show_open_flags(__entry->flags), 446 show_open_flags(__entry->flags),
349 show_fmode_flags(__entry->fmode), 447 show_fmode_flags(__entry->fmode),
@@ -363,7 +461,7 @@ TRACE_EVENT(nfs_create_enter,
363 TP_ARGS(dir, dentry, flags), 461 TP_ARGS(dir, dentry, flags),
364 462
365 TP_STRUCT__entry( 463 TP_STRUCT__entry(
366 __field(unsigned int, flags) 464 __field(unsigned long, flags)
367 __field(dev_t, dev) 465 __field(dev_t, dev)
368 __field(u64, dir) 466 __field(u64, dir)
369 __string(name, dentry->d_name.name) 467 __string(name, dentry->d_name.name)
@@ -377,7 +475,7 @@ TRACE_EVENT(nfs_create_enter,
377 ), 475 ),
378 476
379 TP_printk( 477 TP_printk(
380 "flags=%u (%s) name=%02x:%02x:%llu/%s", 478 "flags=0x%lx (%s) name=%02x:%02x:%llu/%s",
381 __entry->flags, 479 __entry->flags,
382 show_open_flags(__entry->flags), 480 show_open_flags(__entry->flags),
383 MAJOR(__entry->dev), MINOR(__entry->dev), 481 MAJOR(__entry->dev), MINOR(__entry->dev),
@@ -397,15 +495,15 @@ TRACE_EVENT(nfs_create_exit,
397 TP_ARGS(dir, dentry, flags, error), 495 TP_ARGS(dir, dentry, flags, error),
398 496
399 TP_STRUCT__entry( 497 TP_STRUCT__entry(
400 __field(int, error) 498 __field(unsigned long, error)
401 __field(unsigned int, flags) 499 __field(unsigned long, flags)
402 __field(dev_t, dev) 500 __field(dev_t, dev)
403 __field(u64, dir) 501 __field(u64, dir)
404 __string(name, dentry->d_name.name) 502 __string(name, dentry->d_name.name)
405 ), 503 ),
406 504
407 TP_fast_assign( 505 TP_fast_assign(
408 __entry->error = error; 506 __entry->error = -error;
409 __entry->dev = dir->i_sb->s_dev; 507 __entry->dev = dir->i_sb->s_dev;
410 __entry->dir = NFS_FILEID(dir); 508 __entry->dir = NFS_FILEID(dir);
411 __entry->flags = flags; 509 __entry->flags = flags;
@@ -413,8 +511,8 @@ TRACE_EVENT(nfs_create_exit,
413 ), 511 ),
414 512
415 TP_printk( 513 TP_printk(
416 "error=%d flags=%u (%s) name=%02x:%02x:%llu/%s", 514 "error=%ld (%s) flags=0x%lx (%s) name=%02x:%02x:%llu/%s",
417 __entry->error, 515 -__entry->error, nfs_show_status(__entry->error),
418 __entry->flags, 516 __entry->flags,
419 show_open_flags(__entry->flags), 517 show_open_flags(__entry->flags),
420 MAJOR(__entry->dev), MINOR(__entry->dev), 518 MAJOR(__entry->dev), MINOR(__entry->dev),
@@ -469,7 +567,7 @@ DECLARE_EVENT_CLASS(nfs_directory_event_done,
469 TP_ARGS(dir, dentry, error), 567 TP_ARGS(dir, dentry, error),
470 568
471 TP_STRUCT__entry( 569 TP_STRUCT__entry(
472 __field(int, error) 570 __field(unsigned long, error)
473 __field(dev_t, dev) 571 __field(dev_t, dev)
474 __field(u64, dir) 572 __field(u64, dir)
475 __string(name, dentry->d_name.name) 573 __string(name, dentry->d_name.name)
@@ -478,13 +576,13 @@ DECLARE_EVENT_CLASS(nfs_directory_event_done,
478 TP_fast_assign( 576 TP_fast_assign(
479 __entry->dev = dir->i_sb->s_dev; 577 __entry->dev = dir->i_sb->s_dev;
480 __entry->dir = NFS_FILEID(dir); 578 __entry->dir = NFS_FILEID(dir);
481 __entry->error = error; 579 __entry->error = error < 0 ? -error : 0;
482 __assign_str(name, dentry->d_name.name); 580 __assign_str(name, dentry->d_name.name);
483 ), 581 ),
484 582
485 TP_printk( 583 TP_printk(
486 "error=%d name=%02x:%02x:%llu/%s", 584 "error=%ld (%s) name=%02x:%02x:%llu/%s",
487 __entry->error, 585 -__entry->error, nfs_show_status(__entry->error),
488 MAJOR(__entry->dev), MINOR(__entry->dev), 586 MAJOR(__entry->dev), MINOR(__entry->dev),
489 (unsigned long long)__entry->dir, 587 (unsigned long long)__entry->dir,
490 __get_str(name) 588 __get_str(name)
@@ -557,7 +655,7 @@ TRACE_EVENT(nfs_link_exit,
557 TP_ARGS(inode, dir, dentry, error), 655 TP_ARGS(inode, dir, dentry, error),
558 656
559 TP_STRUCT__entry( 657 TP_STRUCT__entry(
560 __field(int, error) 658 __field(unsigned long, error)
561 __field(dev_t, dev) 659 __field(dev_t, dev)
562 __field(u64, fileid) 660 __field(u64, fileid)
563 __field(u64, dir) 661 __field(u64, dir)
@@ -568,13 +666,13 @@ TRACE_EVENT(nfs_link_exit,
568 __entry->dev = inode->i_sb->s_dev; 666 __entry->dev = inode->i_sb->s_dev;
569 __entry->fileid = NFS_FILEID(inode); 667 __entry->fileid = NFS_FILEID(inode);
570 __entry->dir = NFS_FILEID(dir); 668 __entry->dir = NFS_FILEID(dir);
571 __entry->error = error; 669 __entry->error = error < 0 ? -error : 0;
572 __assign_str(name, dentry->d_name.name); 670 __assign_str(name, dentry->d_name.name);
573 ), 671 ),
574 672
575 TP_printk( 673 TP_printk(
576 "error=%d fileid=%02x:%02x:%llu name=%02x:%02x:%llu/%s", 674 "error=%ld (%s) fileid=%02x:%02x:%llu name=%02x:%02x:%llu/%s",
577 __entry->error, 675 -__entry->error, nfs_show_status(__entry->error),
578 MAJOR(__entry->dev), MINOR(__entry->dev), 676 MAJOR(__entry->dev), MINOR(__entry->dev),
579 __entry->fileid, 677 __entry->fileid,
580 MAJOR(__entry->dev), MINOR(__entry->dev), 678 MAJOR(__entry->dev), MINOR(__entry->dev),
@@ -642,7 +740,7 @@ DECLARE_EVENT_CLASS(nfs_rename_event_done,
642 740
643 TP_STRUCT__entry( 741 TP_STRUCT__entry(
644 __field(dev_t, dev) 742 __field(dev_t, dev)
645 __field(int, error) 743 __field(unsigned long, error)
646 __field(u64, old_dir) 744 __field(u64, old_dir)
647 __string(old_name, old_dentry->d_name.name) 745 __string(old_name, old_dentry->d_name.name)
648 __field(u64, new_dir) 746 __field(u64, new_dir)
@@ -651,17 +749,17 @@ DECLARE_EVENT_CLASS(nfs_rename_event_done,
651 749
652 TP_fast_assign( 750 TP_fast_assign(
653 __entry->dev = old_dir->i_sb->s_dev; 751 __entry->dev = old_dir->i_sb->s_dev;
752 __entry->error = -error;
654 __entry->old_dir = NFS_FILEID(old_dir); 753 __entry->old_dir = NFS_FILEID(old_dir);
655 __entry->new_dir = NFS_FILEID(new_dir); 754 __entry->new_dir = NFS_FILEID(new_dir);
656 __entry->error = error;
657 __assign_str(old_name, old_dentry->d_name.name); 755 __assign_str(old_name, old_dentry->d_name.name);
658 __assign_str(new_name, new_dentry->d_name.name); 756 __assign_str(new_name, new_dentry->d_name.name);
659 ), 757 ),
660 758
661 TP_printk( 759 TP_printk(
662 "error=%d old_name=%02x:%02x:%llu/%s " 760 "error=%ld (%s) old_name=%02x:%02x:%llu/%s "
663 "new_name=%02x:%02x:%llu/%s", 761 "new_name=%02x:%02x:%llu/%s",
664 __entry->error, 762 -__entry->error, nfs_show_status(__entry->error),
665 MAJOR(__entry->dev), MINOR(__entry->dev), 763 MAJOR(__entry->dev), MINOR(__entry->dev),
666 (unsigned long long)__entry->old_dir, 764 (unsigned long long)__entry->old_dir,
667 __get_str(old_name), 765 __get_str(old_name),
@@ -697,7 +795,7 @@ TRACE_EVENT(nfs_sillyrename_unlink,
697 795
698 TP_STRUCT__entry( 796 TP_STRUCT__entry(
699 __field(dev_t, dev) 797 __field(dev_t, dev)
700 __field(int, error) 798 __field(unsigned long, error)
701 __field(u64, dir) 799 __field(u64, dir)
702 __dynamic_array(char, name, data->args.name.len + 1) 800 __dynamic_array(char, name, data->args.name.len + 1)
703 ), 801 ),
@@ -707,15 +805,15 @@ TRACE_EVENT(nfs_sillyrename_unlink,
707 size_t len = data->args.name.len; 805 size_t len = data->args.name.len;
708 __entry->dev = dir->i_sb->s_dev; 806 __entry->dev = dir->i_sb->s_dev;
709 __entry->dir = NFS_FILEID(dir); 807 __entry->dir = NFS_FILEID(dir);
710 __entry->error = error; 808 __entry->error = -error;
711 memcpy(__get_str(name), 809 memcpy(__get_str(name),
712 data->args.name.name, len); 810 data->args.name.name, len);
713 __get_str(name)[len] = 0; 811 __get_str(name)[len] = 0;
714 ), 812 ),
715 813
716 TP_printk( 814 TP_printk(
717 "error=%d name=%02x:%02x:%llu/%s", 815 "error=%ld (%s) name=%02x:%02x:%llu/%s",
718 __entry->error, 816 -__entry->error, nfs_show_status(__entry->error),
719 MAJOR(__entry->dev), MINOR(__entry->dev), 817 MAJOR(__entry->dev), MINOR(__entry->dev),
720 (unsigned long long)__entry->dir, 818 (unsigned long long)__entry->dir,
721 __get_str(name) 819 __get_str(name)
@@ -974,6 +1072,8 @@ TRACE_DEFINE_ENUM(NFSERR_PERM);
974TRACE_DEFINE_ENUM(NFSERR_NOENT); 1072TRACE_DEFINE_ENUM(NFSERR_NOENT);
975TRACE_DEFINE_ENUM(NFSERR_IO); 1073TRACE_DEFINE_ENUM(NFSERR_IO);
976TRACE_DEFINE_ENUM(NFSERR_NXIO); 1074TRACE_DEFINE_ENUM(NFSERR_NXIO);
1075TRACE_DEFINE_ENUM(ECHILD);
1076TRACE_DEFINE_ENUM(NFSERR_EAGAIN);
977TRACE_DEFINE_ENUM(NFSERR_ACCES); 1077TRACE_DEFINE_ENUM(NFSERR_ACCES);
978TRACE_DEFINE_ENUM(NFSERR_EXIST); 1078TRACE_DEFINE_ENUM(NFSERR_EXIST);
979TRACE_DEFINE_ENUM(NFSERR_XDEV); 1079TRACE_DEFINE_ENUM(NFSERR_XDEV);
@@ -985,6 +1085,7 @@ TRACE_DEFINE_ENUM(NFSERR_FBIG);
985TRACE_DEFINE_ENUM(NFSERR_NOSPC); 1085TRACE_DEFINE_ENUM(NFSERR_NOSPC);
986TRACE_DEFINE_ENUM(NFSERR_ROFS); 1086TRACE_DEFINE_ENUM(NFSERR_ROFS);
987TRACE_DEFINE_ENUM(NFSERR_MLINK); 1087TRACE_DEFINE_ENUM(NFSERR_MLINK);
1088TRACE_DEFINE_ENUM(NFSERR_OPNOTSUPP);
988TRACE_DEFINE_ENUM(NFSERR_NAMETOOLONG); 1089TRACE_DEFINE_ENUM(NFSERR_NAMETOOLONG);
989TRACE_DEFINE_ENUM(NFSERR_NOTEMPTY); 1090TRACE_DEFINE_ENUM(NFSERR_NOTEMPTY);
990TRACE_DEFINE_ENUM(NFSERR_DQUOT); 1091TRACE_DEFINE_ENUM(NFSERR_DQUOT);
@@ -1007,6 +1108,8 @@ TRACE_DEFINE_ENUM(NFSERR_JUKEBOX);
1007 { NFSERR_NOENT, "NOENT" }, \ 1108 { NFSERR_NOENT, "NOENT" }, \
1008 { NFSERR_IO, "IO" }, \ 1109 { NFSERR_IO, "IO" }, \
1009 { NFSERR_NXIO, "NXIO" }, \ 1110 { NFSERR_NXIO, "NXIO" }, \
1111 { ECHILD, "CHILD" }, \
1112 { NFSERR_EAGAIN, "AGAIN" }, \
1010 { NFSERR_ACCES, "ACCES" }, \ 1113 { NFSERR_ACCES, "ACCES" }, \
1011 { NFSERR_EXIST, "EXIST" }, \ 1114 { NFSERR_EXIST, "EXIST" }, \
1012 { NFSERR_XDEV, "XDEV" }, \ 1115 { NFSERR_XDEV, "XDEV" }, \
@@ -1018,6 +1121,7 @@ TRACE_DEFINE_ENUM(NFSERR_JUKEBOX);
1018 { NFSERR_NOSPC, "NOSPC" }, \ 1121 { NFSERR_NOSPC, "NOSPC" }, \
1019 { NFSERR_ROFS, "ROFS" }, \ 1122 { NFSERR_ROFS, "ROFS" }, \
1020 { NFSERR_MLINK, "MLINK" }, \ 1123 { NFSERR_MLINK, "MLINK" }, \
1124 { NFSERR_OPNOTSUPP, "OPNOTSUPP" }, \
1021 { NFSERR_NAMETOOLONG, "NAMETOOLONG" }, \ 1125 { NFSERR_NAMETOOLONG, "NAMETOOLONG" }, \
1022 { NFSERR_NOTEMPTY, "NOTEMPTY" }, \ 1126 { NFSERR_NOTEMPTY, "NOTEMPTY" }, \
1023 { NFSERR_DQUOT, "DQUOT" }, \ 1127 { NFSERR_DQUOT, "DQUOT" }, \
@@ -1035,22 +1139,33 @@ TRACE_DEFINE_ENUM(NFSERR_JUKEBOX);
1035 1139
1036TRACE_EVENT(nfs_xdr_status, 1140TRACE_EVENT(nfs_xdr_status,
1037 TP_PROTO( 1141 TP_PROTO(
1142 const struct xdr_stream *xdr,
1038 int error 1143 int error
1039 ), 1144 ),
1040 1145
1041 TP_ARGS(error), 1146 TP_ARGS(xdr, error),
1042 1147
1043 TP_STRUCT__entry( 1148 TP_STRUCT__entry(
1044 __field(int, error) 1149 __field(unsigned int, task_id)
1150 __field(unsigned int, client_id)
1151 __field(u32, xid)
1152 __field(unsigned long, error)
1045 ), 1153 ),
1046 1154
1047 TP_fast_assign( 1155 TP_fast_assign(
1156 const struct rpc_rqst *rqstp = xdr->rqst;
1157 const struct rpc_task *task = rqstp->rq_task;
1158
1159 __entry->task_id = task->tk_pid;
1160 __entry->client_id = task->tk_client->cl_clid;
1161 __entry->xid = be32_to_cpu(rqstp->rq_xid);
1048 __entry->error = error; 1162 __entry->error = error;
1049 ), 1163 ),
1050 1164
1051 TP_printk( 1165 TP_printk(
1052 "error=%d (%s)", 1166 "task:%u@%d xid=0x%08x error=%ld (%s)",
1053 __entry->error, nfs_show_status(__entry->error) 1167 __entry->task_id, __entry->client_id, __entry->xid,
1168 -__entry->error, nfs_show_status(__entry->error)
1054 ) 1169 )
1055); 1170);
1056 1171
diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 15322c1d9c8c..ed76e5fb36c1 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -335,6 +335,9 @@ struct xprt_class {
335 */ 335 */
336struct rpc_xprt *xprt_create_transport(struct xprt_create *args); 336struct rpc_xprt *xprt_create_transport(struct xprt_create *args);
337void xprt_connect(struct rpc_task *task); 337void xprt_connect(struct rpc_task *task);
338unsigned long xprt_reconnect_delay(const struct rpc_xprt *xprt);
339void xprt_reconnect_backoff(struct rpc_xprt *xprt,
340 unsigned long init_to);
338void xprt_reserve(struct rpc_task *task); 341void xprt_reserve(struct rpc_task *task);
339void xprt_retry_reserve(struct rpc_task *task); 342void xprt_retry_reserve(struct rpc_task *task);
340int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task); 343int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task);
diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h
index df9851cb82b2..f6a4eaa85a3e 100644
--- a/include/trace/events/rpcrdma.h
+++ b/include/trace/events/rpcrdma.h
@@ -181,18 +181,6 @@ DECLARE_EVENT_CLASS(xprtrdma_wrch_event,
181 ), \ 181 ), \
182 TP_ARGS(task, mr, nsegs)) 182 TP_ARGS(task, mr, nsegs))
183 183
184TRACE_DEFINE_ENUM(FRWR_IS_INVALID);
185TRACE_DEFINE_ENUM(FRWR_IS_VALID);
186TRACE_DEFINE_ENUM(FRWR_FLUSHED_FR);
187TRACE_DEFINE_ENUM(FRWR_FLUSHED_LI);
188
189#define xprtrdma_show_frwr_state(x) \
190 __print_symbolic(x, \
191 { FRWR_IS_INVALID, "INVALID" }, \
192 { FRWR_IS_VALID, "VALID" }, \
193 { FRWR_FLUSHED_FR, "FLUSHED_FR" }, \
194 { FRWR_FLUSHED_LI, "FLUSHED_LI" })
195
196DECLARE_EVENT_CLASS(xprtrdma_frwr_done, 184DECLARE_EVENT_CLASS(xprtrdma_frwr_done,
197 TP_PROTO( 185 TP_PROTO(
198 const struct ib_wc *wc, 186 const struct ib_wc *wc,
@@ -203,22 +191,19 @@ DECLARE_EVENT_CLASS(xprtrdma_frwr_done,
203 191
204 TP_STRUCT__entry( 192 TP_STRUCT__entry(
205 __field(const void *, mr) 193 __field(const void *, mr)
206 __field(unsigned int, state)
207 __field(unsigned int, status) 194 __field(unsigned int, status)
208 __field(unsigned int, vendor_err) 195 __field(unsigned int, vendor_err)
209 ), 196 ),
210 197
211 TP_fast_assign( 198 TP_fast_assign(
212 __entry->mr = container_of(frwr, struct rpcrdma_mr, frwr); 199 __entry->mr = container_of(frwr, struct rpcrdma_mr, frwr);
213 __entry->state = frwr->fr_state;
214 __entry->status = wc->status; 200 __entry->status = wc->status;
215 __entry->vendor_err = __entry->status ? wc->vendor_err : 0; 201 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
216 ), 202 ),
217 203
218 TP_printk( 204 TP_printk(
219 "mr=%p state=%s: %s (%u/0x%x)", 205 "mr=%p: %s (%u/0x%x)",
220 __entry->mr, xprtrdma_show_frwr_state(__entry->state), 206 __entry->mr, rdma_show_wc_status(__entry->status),
221 rdma_show_wc_status(__entry->status),
222 __entry->status, __entry->vendor_err 207 __entry->status, __entry->vendor_err
223 ) 208 )
224); 209);
@@ -390,6 +375,37 @@ DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
390DEFINE_RXPRT_EVENT(xprtrdma_op_close); 375DEFINE_RXPRT_EVENT(xprtrdma_op_close);
391DEFINE_RXPRT_EVENT(xprtrdma_op_connect); 376DEFINE_RXPRT_EVENT(xprtrdma_op_connect);
392 377
378TRACE_EVENT(xprtrdma_op_set_cto,
379 TP_PROTO(
380 const struct rpcrdma_xprt *r_xprt,
381 unsigned long connect,
382 unsigned long reconnect
383 ),
384
385 TP_ARGS(r_xprt, connect, reconnect),
386
387 TP_STRUCT__entry(
388 __field(const void *, r_xprt)
389 __field(unsigned long, connect)
390 __field(unsigned long, reconnect)
391 __string(addr, rpcrdma_addrstr(r_xprt))
392 __string(port, rpcrdma_portstr(r_xprt))
393 ),
394
395 TP_fast_assign(
396 __entry->r_xprt = r_xprt;
397 __entry->connect = connect;
398 __entry->reconnect = reconnect;
399 __assign_str(addr, rpcrdma_addrstr(r_xprt));
400 __assign_str(port, rpcrdma_portstr(r_xprt));
401 ),
402
403 TP_printk("peer=[%s]:%s r_xprt=%p: connect=%lu reconnect=%lu",
404 __get_str(addr), __get_str(port), __entry->r_xprt,
405 __entry->connect / HZ, __entry->reconnect / HZ
406 )
407);
408
393TRACE_EVENT(xprtrdma_qp_event, 409TRACE_EVENT(xprtrdma_qp_event,
394 TP_PROTO( 410 TP_PROTO(
395 const struct rpcrdma_xprt *r_xprt, 411 const struct rpcrdma_xprt *r_xprt,
@@ -470,13 +486,12 @@ TRACE_DEFINE_ENUM(rpcrdma_replych);
470 486
471TRACE_EVENT(xprtrdma_marshal, 487TRACE_EVENT(xprtrdma_marshal,
472 TP_PROTO( 488 TP_PROTO(
473 const struct rpc_rqst *rqst, 489 const struct rpcrdma_req *req,
474 unsigned int hdrlen,
475 unsigned int rtype, 490 unsigned int rtype,
476 unsigned int wtype 491 unsigned int wtype
477 ), 492 ),
478 493
479 TP_ARGS(rqst, hdrlen, rtype, wtype), 494 TP_ARGS(req, rtype, wtype),
480 495
481 TP_STRUCT__entry( 496 TP_STRUCT__entry(
482 __field(unsigned int, task_id) 497 __field(unsigned int, task_id)
@@ -491,10 +506,12 @@ TRACE_EVENT(xprtrdma_marshal,
491 ), 506 ),
492 507
493 TP_fast_assign( 508 TP_fast_assign(
509 const struct rpc_rqst *rqst = &req->rl_slot;
510
494 __entry->task_id = rqst->rq_task->tk_pid; 511 __entry->task_id = rqst->rq_task->tk_pid;
495 __entry->client_id = rqst->rq_task->tk_client->cl_clid; 512 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
496 __entry->xid = be32_to_cpu(rqst->rq_xid); 513 __entry->xid = be32_to_cpu(rqst->rq_xid);
497 __entry->hdrlen = hdrlen; 514 __entry->hdrlen = req->rl_hdrbuf.len;
498 __entry->headlen = rqst->rq_snd_buf.head[0].iov_len; 515 __entry->headlen = rqst->rq_snd_buf.head[0].iov_len;
499 __entry->pagelen = rqst->rq_snd_buf.page_len; 516 __entry->pagelen = rqst->rq_snd_buf.page_len;
500 __entry->taillen = rqst->rq_snd_buf.tail[0].iov_len; 517 __entry->taillen = rqst->rq_snd_buf.tail[0].iov_len;
@@ -538,6 +555,33 @@ TRACE_EVENT(xprtrdma_marshal_failed,
538 ) 555 )
539); 556);
540 557
558TRACE_EVENT(xprtrdma_prepsend_failed,
559 TP_PROTO(const struct rpc_rqst *rqst,
560 int ret
561 ),
562
563 TP_ARGS(rqst, ret),
564
565 TP_STRUCT__entry(
566 __field(unsigned int, task_id)
567 __field(unsigned int, client_id)
568 __field(u32, xid)
569 __field(int, ret)
570 ),
571
572 TP_fast_assign(
573 __entry->task_id = rqst->rq_task->tk_pid;
574 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
575 __entry->xid = be32_to_cpu(rqst->rq_xid);
576 __entry->ret = ret;
577 ),
578
579 TP_printk("task:%u@%u xid=0x%08x: ret=%d",
580 __entry->task_id, __entry->client_id, __entry->xid,
581 __entry->ret
582 )
583);
584
541TRACE_EVENT(xprtrdma_post_send, 585TRACE_EVENT(xprtrdma_post_send,
542 TP_PROTO( 586 TP_PROTO(
543 const struct rpcrdma_req *req, 587 const struct rpcrdma_req *req,
@@ -559,7 +603,8 @@ TRACE_EVENT(xprtrdma_post_send,
559 const struct rpc_rqst *rqst = &req->rl_slot; 603 const struct rpc_rqst *rqst = &req->rl_slot;
560 604
561 __entry->task_id = rqst->rq_task->tk_pid; 605 __entry->task_id = rqst->rq_task->tk_pid;
562 __entry->client_id = rqst->rq_task->tk_client->cl_clid; 606 __entry->client_id = rqst->rq_task->tk_client ?
607 rqst->rq_task->tk_client->cl_clid : -1;
563 __entry->req = req; 608 __entry->req = req;
564 __entry->num_sge = req->rl_sendctx->sc_wr.num_sge; 609 __entry->num_sge = req->rl_sendctx->sc_wr.num_sge;
565 __entry->signaled = req->rl_sendctx->sc_wr.send_flags & 610 __entry->signaled = req->rl_sendctx->sc_wr.send_flags &
@@ -698,6 +743,7 @@ TRACE_EVENT(xprtrdma_wc_receive,
698DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg); 743DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);
699DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li); 744DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);
700DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake); 745DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);
746DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_done);
701 747
702TRACE_EVENT(xprtrdma_frwr_alloc, 748TRACE_EVENT(xprtrdma_frwr_alloc,
703 TP_PROTO( 749 TP_PROTO(
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index f820780280b5..8a0779e963f9 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -59,6 +59,7 @@ static struct rpc_wait_queue delay_queue;
59 */ 59 */
60struct workqueue_struct *rpciod_workqueue __read_mostly; 60struct workqueue_struct *rpciod_workqueue __read_mostly;
61struct workqueue_struct *xprtiod_workqueue __read_mostly; 61struct workqueue_struct *xprtiod_workqueue __read_mostly;
62EXPORT_SYMBOL_GPL(xprtiod_workqueue);
62 63
63unsigned long 64unsigned long
64rpc_task_timeout(const struct rpc_task *task) 65rpc_task_timeout(const struct rpc_task *task)
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index 70d6a1f10db9..70a704c44c6d 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -846,6 +846,38 @@ void xprt_connect(struct rpc_task *task)
846 xprt_release_write(xprt, task); 846 xprt_release_write(xprt, task);
847} 847}
848 848
849/**
850 * xprt_reconnect_delay - compute the wait before scheduling a connect
851 * @xprt: transport instance
852 *
853 */
854unsigned long xprt_reconnect_delay(const struct rpc_xprt *xprt)
855{
856 unsigned long start, now = jiffies;
857
858 start = xprt->stat.connect_start + xprt->reestablish_timeout;
859 if (time_after(start, now))
860 return start - now;
861 return 0;
862}
863EXPORT_SYMBOL_GPL(xprt_reconnect_delay);
864
865/**
866 * xprt_reconnect_backoff - compute the new re-establish timeout
867 * @xprt: transport instance
868 * @init_to: initial reestablish timeout
869 *
870 */
871void xprt_reconnect_backoff(struct rpc_xprt *xprt, unsigned long init_to)
872{
873 xprt->reestablish_timeout <<= 1;
874 if (xprt->reestablish_timeout > xprt->max_reconnect_timeout)
875 xprt->reestablish_timeout = xprt->max_reconnect_timeout;
876 if (xprt->reestablish_timeout < init_to)
877 xprt->reestablish_timeout = init_to;
878}
879EXPORT_SYMBOL_GPL(xprt_reconnect_backoff);
880
849enum xprt_xid_rb_cmp { 881enum xprt_xid_rb_cmp {
850 XID_RB_EQUAL, 882 XID_RB_EQUAL,
851 XID_RB_LEFT, 883 XID_RB_LEFT,
diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index 794ba4ca0994..0b6dad7580a1 100644
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -144,6 +144,26 @@ frwr_mr_recycle_worker(struct work_struct *work)
144 frwr_release_mr(mr); 144 frwr_release_mr(mr);
145} 145}
146 146
147/* frwr_reset - Place MRs back on the free list
148 * @req: request to reset
149 *
150 * Used after a failed marshal. For FRWR, this means the MRs
151 * don't have to be fully released and recreated.
152 *
153 * NB: This is safe only as long as none of @req's MRs are
154 * involved with an ongoing asynchronous FAST_REG or LOCAL_INV
155 * Work Request.
156 */
157void frwr_reset(struct rpcrdma_req *req)
158{
159 while (!list_empty(&req->rl_registered)) {
160 struct rpcrdma_mr *mr;
161
162 mr = rpcrdma_mr_pop(&req->rl_registered);
163 rpcrdma_mr_unmap_and_put(mr);
164 }
165}
166
147/** 167/**
148 * frwr_init_mr - Initialize one MR 168 * frwr_init_mr - Initialize one MR
149 * @ia: interface adapter 169 * @ia: interface adapter
@@ -168,7 +188,6 @@ int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)
168 goto out_list_err; 188 goto out_list_err;
169 189
170 mr->frwr.fr_mr = frmr; 190 mr->frwr.fr_mr = frmr;
171 mr->frwr.fr_state = FRWR_IS_INVALID;
172 mr->mr_dir = DMA_NONE; 191 mr->mr_dir = DMA_NONE;
173 INIT_LIST_HEAD(&mr->mr_list); 192 INIT_LIST_HEAD(&mr->mr_list);
174 INIT_WORK(&mr->mr_recycle, frwr_mr_recycle_worker); 193 INIT_WORK(&mr->mr_recycle, frwr_mr_recycle_worker);
@@ -298,65 +317,6 @@ size_t frwr_maxpages(struct rpcrdma_xprt *r_xprt)
298} 317}
299 318
300/** 319/**
301 * frwr_wc_fastreg - Invoked by RDMA provider for a flushed FastReg WC
302 * @cq: completion queue (ignored)
303 * @wc: completed WR
304 *
305 */
306static void
307frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc)
308{
309 struct ib_cqe *cqe = wc->wr_cqe;
310 struct rpcrdma_frwr *frwr =
311 container_of(cqe, struct rpcrdma_frwr, fr_cqe);
312
313 /* WARNING: Only wr_cqe and status are reliable at this point */
314 if (wc->status != IB_WC_SUCCESS)
315 frwr->fr_state = FRWR_FLUSHED_FR;
316 trace_xprtrdma_wc_fastreg(wc, frwr);
317}
318
319/**
320 * frwr_wc_localinv - Invoked by RDMA provider for a flushed LocalInv WC
321 * @cq: completion queue (ignored)
322 * @wc: completed WR
323 *
324 */
325static void
326frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc)
327{
328 struct ib_cqe *cqe = wc->wr_cqe;
329 struct rpcrdma_frwr *frwr = container_of(cqe, struct rpcrdma_frwr,
330 fr_cqe);
331
332 /* WARNING: Only wr_cqe and status are reliable at this point */
333 if (wc->status != IB_WC_SUCCESS)
334 frwr->fr_state = FRWR_FLUSHED_LI;
335 trace_xprtrdma_wc_li(wc, frwr);
336}
337
338/**
339 * frwr_wc_localinv_wake - Invoked by RDMA provider for a signaled LocalInv WC
340 * @cq: completion queue (ignored)
341 * @wc: completed WR
342 *
343 * Awaken anyone waiting for an MR to finish being fenced.
344 */
345static void
346frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc)
347{
348 struct ib_cqe *cqe = wc->wr_cqe;
349 struct rpcrdma_frwr *frwr = container_of(cqe, struct rpcrdma_frwr,
350 fr_cqe);
351
352 /* WARNING: Only wr_cqe and status are reliable at this point */
353 if (wc->status != IB_WC_SUCCESS)
354 frwr->fr_state = FRWR_FLUSHED_LI;
355 trace_xprtrdma_wc_li_wake(wc, frwr);
356 complete(&frwr->fr_linv_done);
357}
358
359/**
360 * frwr_map - Register a memory region 320 * frwr_map - Register a memory region
361 * @r_xprt: controlling transport 321 * @r_xprt: controlling transport
362 * @seg: memory region co-ordinates 322 * @seg: memory region co-ordinates
@@ -378,23 +338,15 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
378{ 338{
379 struct rpcrdma_ia *ia = &r_xprt->rx_ia; 339 struct rpcrdma_ia *ia = &r_xprt->rx_ia;
380 bool holes_ok = ia->ri_mrtype == IB_MR_TYPE_SG_GAPS; 340 bool holes_ok = ia->ri_mrtype == IB_MR_TYPE_SG_GAPS;
381 struct rpcrdma_frwr *frwr;
382 struct rpcrdma_mr *mr; 341 struct rpcrdma_mr *mr;
383 struct ib_mr *ibmr; 342 struct ib_mr *ibmr;
384 struct ib_reg_wr *reg_wr; 343 struct ib_reg_wr *reg_wr;
385 int i, n; 344 int i, n;
386 u8 key; 345 u8 key;
387 346
388 mr = NULL; 347 mr = rpcrdma_mr_get(r_xprt);
389 do { 348 if (!mr)
390 if (mr) 349 goto out_getmr_err;
391 rpcrdma_mr_recycle(mr);
392 mr = rpcrdma_mr_get(r_xprt);
393 if (!mr)
394 return ERR_PTR(-EAGAIN);
395 } while (mr->frwr.fr_state != FRWR_IS_INVALID);
396 frwr = &mr->frwr;
397 frwr->fr_state = FRWR_IS_VALID;
398 350
399 if (nsegs > ia->ri_max_frwr_depth) 351 if (nsegs > ia->ri_max_frwr_depth)
400 nsegs = ia->ri_max_frwr_depth; 352 nsegs = ia->ri_max_frwr_depth;
@@ -423,7 +375,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
423 if (!mr->mr_nents) 375 if (!mr->mr_nents)
424 goto out_dmamap_err; 376 goto out_dmamap_err;
425 377
426 ibmr = frwr->fr_mr; 378 ibmr = mr->frwr.fr_mr;
427 n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE); 379 n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE);
428 if (unlikely(n != mr->mr_nents)) 380 if (unlikely(n != mr->mr_nents))
429 goto out_mapmr_err; 381 goto out_mapmr_err;
@@ -433,7 +385,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
433 key = (u8)(ibmr->rkey & 0x000000FF); 385 key = (u8)(ibmr->rkey & 0x000000FF);
434 ib_update_fast_reg_key(ibmr, ++key); 386 ib_update_fast_reg_key(ibmr, ++key);
435 387
436 reg_wr = &frwr->fr_regwr; 388 reg_wr = &mr->frwr.fr_regwr;
437 reg_wr->mr = ibmr; 389 reg_wr->mr = ibmr;
438 reg_wr->key = ibmr->rkey; 390 reg_wr->key = ibmr->rkey;
439 reg_wr->access = writing ? 391 reg_wr->access = writing ?
@@ -448,6 +400,10 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
448 *out = mr; 400 *out = mr;
449 return seg; 401 return seg;
450 402
403out_getmr_err:
404 xprt_wait_for_buffer_space(&r_xprt->rx_xprt);
405 return ERR_PTR(-EAGAIN);
406
451out_dmamap_err: 407out_dmamap_err:
452 mr->mr_dir = DMA_NONE; 408 mr->mr_dir = DMA_NONE;
453 trace_xprtrdma_frwr_sgerr(mr, i); 409 trace_xprtrdma_frwr_sgerr(mr, i);
@@ -461,6 +417,23 @@ out_mapmr_err:
461} 417}
462 418
463/** 419/**
420 * frwr_wc_fastreg - Invoked by RDMA provider for a flushed FastReg WC
421 * @cq: completion queue (ignored)
422 * @wc: completed WR
423 *
424 */
425static void frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc)
426{
427 struct ib_cqe *cqe = wc->wr_cqe;
428 struct rpcrdma_frwr *frwr =
429 container_of(cqe, struct rpcrdma_frwr, fr_cqe);
430
431 /* WARNING: Only wr_cqe and status are reliable at this point */
432 trace_xprtrdma_wc_fastreg(wc, frwr);
433 /* The MR will get recycled when the associated req is retransmitted */
434}
435
436/**
464 * frwr_send - post Send WR containing the RPC Call message 437 * frwr_send - post Send WR containing the RPC Call message
465 * @ia: interface adapter 438 * @ia: interface adapter
466 * @req: Prepared RPC Call 439 * @req: Prepared RPC Call
@@ -512,31 +485,75 @@ void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs)
512 if (mr->mr_handle == rep->rr_inv_rkey) { 485 if (mr->mr_handle == rep->rr_inv_rkey) {
513 list_del_init(&mr->mr_list); 486 list_del_init(&mr->mr_list);
514 trace_xprtrdma_mr_remoteinv(mr); 487 trace_xprtrdma_mr_remoteinv(mr);
515 mr->frwr.fr_state = FRWR_IS_INVALID;
516 rpcrdma_mr_unmap_and_put(mr); 488 rpcrdma_mr_unmap_and_put(mr);
517 break; /* only one invalidated MR per RPC */ 489 break; /* only one invalidated MR per RPC */
518 } 490 }
519} 491}
520 492
493static void __frwr_release_mr(struct ib_wc *wc, struct rpcrdma_mr *mr)
494{
495 if (wc->status != IB_WC_SUCCESS)
496 rpcrdma_mr_recycle(mr);
497 else
498 rpcrdma_mr_unmap_and_put(mr);
499}
500
521/** 501/**
522 * frwr_unmap_sync - invalidate memory regions that were registered for @req 502 * frwr_wc_localinv - Invoked by RDMA provider for a LOCAL_INV WC
523 * @r_xprt: controlling transport 503 * @cq: completion queue (ignored)
524 * @mrs: list of MRs to process 504 * @wc: completed WR
505 *
506 */
507static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc)
508{
509 struct ib_cqe *cqe = wc->wr_cqe;
510 struct rpcrdma_frwr *frwr =
511 container_of(cqe, struct rpcrdma_frwr, fr_cqe);
512 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
513
514 /* WARNING: Only wr_cqe and status are reliable at this point */
515 trace_xprtrdma_wc_li(wc, frwr);
516 __frwr_release_mr(wc, mr);
517}
518
519/**
520 * frwr_wc_localinv_wake - Invoked by RDMA provider for a LOCAL_INV WC
521 * @cq: completion queue (ignored)
522 * @wc: completed WR
525 * 523 *
526 * Sleeps until it is safe for the host CPU to access the 524 * Awaken anyone waiting for an MR to finish being fenced.
527 * previously mapped memory regions. 525 */
526static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc)
527{
528 struct ib_cqe *cqe = wc->wr_cqe;
529 struct rpcrdma_frwr *frwr =
530 container_of(cqe, struct rpcrdma_frwr, fr_cqe);
531 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
532
533 /* WARNING: Only wr_cqe and status are reliable at this point */
534 trace_xprtrdma_wc_li_wake(wc, frwr);
535 complete(&frwr->fr_linv_done);
536 __frwr_release_mr(wc, mr);
537}
538
539/**
540 * frwr_unmap_sync - invalidate memory regions that were registered for @req
541 * @r_xprt: controlling transport instance
542 * @req: rpcrdma_req with a non-empty list of MRs to process
528 * 543 *
529 * Caller ensures that @mrs is not empty before the call. This 544 * Sleeps until it is safe for the host CPU to access the previously mapped
530 * function empties the list. 545 * memory regions. This guarantees that registered MRs are properly fenced
546 * from the server before the RPC consumer accesses the data in them. It
547 * also ensures proper Send flow control: waking the next RPC waits until
548 * this RPC has relinquished all its Send Queue entries.
531 */ 549 */
532void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 550void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
533{ 551{
534 struct ib_send_wr *first, **prev, *last; 552 struct ib_send_wr *first, **prev, *last;
535 const struct ib_send_wr *bad_wr; 553 const struct ib_send_wr *bad_wr;
536 struct rpcrdma_ia *ia = &r_xprt->rx_ia;
537 struct rpcrdma_frwr *frwr; 554 struct rpcrdma_frwr *frwr;
538 struct rpcrdma_mr *mr; 555 struct rpcrdma_mr *mr;
539 int count, rc; 556 int rc;
540 557
541 /* ORDER: Invalidate all of the MRs first 558 /* ORDER: Invalidate all of the MRs first
542 * 559 *
@@ -544,33 +561,32 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)
544 * a single ib_post_send() call. 561 * a single ib_post_send() call.
545 */ 562 */
546 frwr = NULL; 563 frwr = NULL;
547 count = 0;
548 prev = &first; 564 prev = &first;
549 list_for_each_entry(mr, mrs, mr_list) { 565 while (!list_empty(&req->rl_registered)) {
550 mr->frwr.fr_state = FRWR_IS_INVALID; 566 mr = rpcrdma_mr_pop(&req->rl_registered);
551 567
552 frwr = &mr->frwr;
553 trace_xprtrdma_mr_localinv(mr); 568 trace_xprtrdma_mr_localinv(mr);
569 r_xprt->rx_stats.local_inv_needed++;
554 570
571 frwr = &mr->frwr;
555 frwr->fr_cqe.done = frwr_wc_localinv; 572 frwr->fr_cqe.done = frwr_wc_localinv;
556 last = &frwr->fr_invwr; 573 last = &frwr->fr_invwr;
557 memset(last, 0, sizeof(*last)); 574 last->next = NULL;
558 last->wr_cqe = &frwr->fr_cqe; 575 last->wr_cqe = &frwr->fr_cqe;
576 last->sg_list = NULL;
577 last->num_sge = 0;
559 last->opcode = IB_WR_LOCAL_INV; 578 last->opcode = IB_WR_LOCAL_INV;
579 last->send_flags = IB_SEND_SIGNALED;
560 last->ex.invalidate_rkey = mr->mr_handle; 580 last->ex.invalidate_rkey = mr->mr_handle;
561 count++;
562 581
563 *prev = last; 582 *prev = last;
564 prev = &last->next; 583 prev = &last->next;
565 } 584 }
566 if (!frwr)
567 goto unmap;
568 585
569 /* Strong send queue ordering guarantees that when the 586 /* Strong send queue ordering guarantees that when the
570 * last WR in the chain completes, all WRs in the chain 587 * last WR in the chain completes, all WRs in the chain
571 * are complete. 588 * are complete.
572 */ 589 */
573 last->send_flags = IB_SEND_SIGNALED;
574 frwr->fr_cqe.done = frwr_wc_localinv_wake; 590 frwr->fr_cqe.done = frwr_wc_localinv_wake;
575 reinit_completion(&frwr->fr_linv_done); 591 reinit_completion(&frwr->fr_linv_done);
576 592
@@ -578,37 +594,126 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)
578 * replaces the QP. The RPC reply handler won't call us 594 * replaces the QP. The RPC reply handler won't call us
579 * unless ri_id->qp is a valid pointer. 595 * unless ri_id->qp is a valid pointer.
580 */ 596 */
581 r_xprt->rx_stats.local_inv_needed++;
582 bad_wr = NULL; 597 bad_wr = NULL;
583 rc = ib_post_send(ia->ri_id->qp, first, &bad_wr); 598 rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr);
599 trace_xprtrdma_post_send(req, rc);
600
601 /* The final LOCAL_INV WR in the chain is supposed to
602 * do the wake. If it was never posted, the wake will
603 * not happen, so don't wait in that case.
604 */
584 if (bad_wr != first) 605 if (bad_wr != first)
585 wait_for_completion(&frwr->fr_linv_done); 606 wait_for_completion(&frwr->fr_linv_done);
586 if (rc) 607 if (!rc)
587 goto out_release; 608 return;
588 609
589 /* ORDER: Now DMA unmap all of the MRs, and return 610 /* Recycle MRs in the LOCAL_INV chain that did not get posted.
590 * them to the free MR list.
591 */ 611 */
592unmap: 612 while (bad_wr) {
593 while (!list_empty(mrs)) { 613 frwr = container_of(bad_wr, struct rpcrdma_frwr,
594 mr = rpcrdma_mr_pop(mrs); 614 fr_invwr);
595 rpcrdma_mr_unmap_and_put(mr); 615 mr = container_of(frwr, struct rpcrdma_mr, frwr);
616 bad_wr = bad_wr->next;
617
618 list_del_init(&mr->mr_list);
619 rpcrdma_mr_recycle(mr);
596 } 620 }
597 return; 621}
598 622
599out_release: 623/**
600 pr_err("rpcrdma: FRWR invalidate ib_post_send returned %i\n", rc); 624 * frwr_wc_localinv_done - Invoked by RDMA provider for a signaled LOCAL_INV WC
625 * @cq: completion queue (ignored)
626 * @wc: completed WR
627 *
628 */
629static void frwr_wc_localinv_done(struct ib_cq *cq, struct ib_wc *wc)
630{
631 struct ib_cqe *cqe = wc->wr_cqe;
632 struct rpcrdma_frwr *frwr =
633 container_of(cqe, struct rpcrdma_frwr, fr_cqe);
634 struct rpcrdma_mr *mr = container_of(frwr, struct rpcrdma_mr, frwr);
601 635
602 /* Unmap and release the MRs in the LOCAL_INV WRs that did not 636 /* WARNING: Only wr_cqe and status are reliable at this point */
603 * get posted. 637 trace_xprtrdma_wc_li_done(wc, frwr);
638 rpcrdma_complete_rqst(frwr->fr_req->rl_reply);
639 __frwr_release_mr(wc, mr);
640}
641
642/**
643 * frwr_unmap_async - invalidate memory regions that were registered for @req
644 * @r_xprt: controlling transport instance
645 * @req: rpcrdma_req with a non-empty list of MRs to process
646 *
647 * This guarantees that registered MRs are properly fenced from the
648 * server before the RPC consumer accesses the data in them. It also
649 * ensures proper Send flow control: waking the next RPC waits until
650 * this RPC has relinquished all its Send Queue entries.
651 */
652void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
653{
654 struct ib_send_wr *first, *last, **prev;
655 const struct ib_send_wr *bad_wr;
656 struct rpcrdma_frwr *frwr;
657 struct rpcrdma_mr *mr;
658 int rc;
659
660 /* Chain the LOCAL_INV Work Requests and post them with
661 * a single ib_post_send() call.
662 */
663 frwr = NULL;
664 prev = &first;
665 while (!list_empty(&req->rl_registered)) {
666 mr = rpcrdma_mr_pop(&req->rl_registered);
667
668 trace_xprtrdma_mr_localinv(mr);
669 r_xprt->rx_stats.local_inv_needed++;
670
671 frwr = &mr->frwr;
672 frwr->fr_cqe.done = frwr_wc_localinv;
673 frwr->fr_req = req;
674 last = &frwr->fr_invwr;
675 last->next = NULL;
676 last->wr_cqe = &frwr->fr_cqe;
677 last->sg_list = NULL;
678 last->num_sge = 0;
679 last->opcode = IB_WR_LOCAL_INV;
680 last->send_flags = IB_SEND_SIGNALED;
681 last->ex.invalidate_rkey = mr->mr_handle;
682
683 *prev = last;
684 prev = &last->next;
685 }
686
687 /* Strong send queue ordering guarantees that when the
688 * last WR in the chain completes, all WRs in the chain
689 * are complete. The last completion will wake up the
690 * RPC waiter.
691 */
692 frwr->fr_cqe.done = frwr_wc_localinv_done;
693
694 /* Transport disconnect drains the receive CQ before it
695 * replaces the QP. The RPC reply handler won't call us
696 * unless ri_id->qp is a valid pointer.
697 */
698 bad_wr = NULL;
699 rc = ib_post_send(r_xprt->rx_ia.ri_id->qp, first, &bad_wr);
700 trace_xprtrdma_post_send(req, rc);
701 if (!rc)
702 return;
703
704 /* Recycle MRs in the LOCAL_INV chain that did not get posted.
604 */ 705 */
605 while (bad_wr) { 706 while (bad_wr) {
606 frwr = container_of(bad_wr, struct rpcrdma_frwr, 707 frwr = container_of(bad_wr, struct rpcrdma_frwr, fr_invwr);
607 fr_invwr);
608 mr = container_of(frwr, struct rpcrdma_mr, frwr); 708 mr = container_of(frwr, struct rpcrdma_mr, frwr);
609 bad_wr = bad_wr->next; 709 bad_wr = bad_wr->next;
610 710
611 list_del_init(&mr->mr_list);
612 rpcrdma_mr_recycle(mr); 711 rpcrdma_mr_recycle(mr);
613 } 712 }
713
714 /* The final LOCAL_INV WR in the chain is supposed to
715 * do the wake. If it was never posted, the wake will
716 * not happen, so wake here in that case.
717 */
718 rpcrdma_complete_rqst(req->rl_reply);
614} 719}
diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 7dc62e55f526..4345e6912392 100644
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -366,6 +366,9 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
366 unsigned int pos; 366 unsigned int pos;
367 int nsegs; 367 int nsegs;
368 368
369 if (rtype == rpcrdma_noch)
370 goto done;
371
369 pos = rqst->rq_snd_buf.head[0].iov_len; 372 pos = rqst->rq_snd_buf.head[0].iov_len;
370 if (rtype == rpcrdma_areadch) 373 if (rtype == rpcrdma_areadch)
371 pos = 0; 374 pos = 0;
@@ -389,7 +392,8 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
389 nsegs -= mr->mr_nents; 392 nsegs -= mr->mr_nents;
390 } while (nsegs); 393 } while (nsegs);
391 394
392 return 0; 395done:
396 return encode_item_not_present(xdr);
393} 397}
394 398
395/* Register and XDR encode the Write list. Supports encoding a list 399/* Register and XDR encode the Write list. Supports encoding a list
@@ -417,6 +421,9 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
417 int nsegs, nchunks; 421 int nsegs, nchunks;
418 __be32 *segcount; 422 __be32 *segcount;
419 423
424 if (wtype != rpcrdma_writech)
425 goto done;
426
420 seg = req->rl_segments; 427 seg = req->rl_segments;
421 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 428 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf,
422 rqst->rq_rcv_buf.head[0].iov_len, 429 rqst->rq_rcv_buf.head[0].iov_len,
@@ -451,7 +458,8 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
451 /* Update count of segments in this Write chunk */ 458 /* Update count of segments in this Write chunk */
452 *segcount = cpu_to_be32(nchunks); 459 *segcount = cpu_to_be32(nchunks);
453 460
454 return 0; 461done:
462 return encode_item_not_present(xdr);
455} 463}
456 464
457/* Register and XDR encode the Reply chunk. Supports encoding an array 465/* Register and XDR encode the Reply chunk. Supports encoding an array
@@ -476,6 +484,9 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
476 int nsegs, nchunks; 484 int nsegs, nchunks;
477 __be32 *segcount; 485 __be32 *segcount;
478 486
487 if (wtype != rpcrdma_replych)
488 return encode_item_not_present(xdr);
489
479 seg = req->rl_segments; 490 seg = req->rl_segments;
480 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg); 491 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg);
481 if (nsegs < 0) 492 if (nsegs < 0)
@@ -511,6 +522,16 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
511 return 0; 522 return 0;
512} 523}
513 524
525static void rpcrdma_sendctx_done(struct kref *kref)
526{
527 struct rpcrdma_req *req =
528 container_of(kref, struct rpcrdma_req, rl_kref);
529 struct rpcrdma_rep *rep = req->rl_reply;
530
531 rpcrdma_complete_rqst(rep);
532 rep->rr_rxprt->rx_stats.reply_waits_for_send++;
533}
534
514/** 535/**
515 * rpcrdma_sendctx_unmap - DMA-unmap Send buffer 536 * rpcrdma_sendctx_unmap - DMA-unmap Send buffer
516 * @sc: sendctx containing SGEs to unmap 537 * @sc: sendctx containing SGEs to unmap
@@ -520,6 +541,9 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
520{ 541{
521 struct ib_sge *sge; 542 struct ib_sge *sge;
522 543
544 if (!sc->sc_unmap_count)
545 return;
546
523 /* The first two SGEs contain the transport header and 547 /* The first two SGEs contain the transport header and
524 * the inline buffer. These are always left mapped so 548 * the inline buffer. These are always left mapped so
525 * they can be cheaply re-used. 549 * they can be cheaply re-used.
@@ -529,9 +553,7 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
529 ib_dma_unmap_page(sc->sc_device, sge->addr, sge->length, 553 ib_dma_unmap_page(sc->sc_device, sge->addr, sge->length,
530 DMA_TO_DEVICE); 554 DMA_TO_DEVICE);
531 555
532 if (test_and_clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, 556 kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done);
533 &sc->sc_req->rl_flags))
534 wake_up_bit(&sc->sc_req->rl_flags, RPCRDMA_REQ_F_TX_RESOURCES);
535} 557}
536 558
537/* Prepare an SGE for the RPC-over-RDMA transport header. 559/* Prepare an SGE for the RPC-over-RDMA transport header.
@@ -666,7 +688,7 @@ map_tail:
666out: 688out:
667 sc->sc_wr.num_sge += sge_no; 689 sc->sc_wr.num_sge += sge_no;
668 if (sc->sc_unmap_count) 690 if (sc->sc_unmap_count)
669 __set_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags); 691 kref_get(&req->rl_kref);
670 return true; 692 return true;
671 693
672out_regbuf: 694out_regbuf:
@@ -699,22 +721,28 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
699 struct rpcrdma_req *req, u32 hdrlen, 721 struct rpcrdma_req *req, u32 hdrlen,
700 struct xdr_buf *xdr, enum rpcrdma_chunktype rtype) 722 struct xdr_buf *xdr, enum rpcrdma_chunktype rtype)
701{ 723{
724 int ret;
725
726 ret = -EAGAIN;
702 req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt); 727 req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);
703 if (!req->rl_sendctx) 728 if (!req->rl_sendctx)
704 return -EAGAIN; 729 goto err;
705 req->rl_sendctx->sc_wr.num_sge = 0; 730 req->rl_sendctx->sc_wr.num_sge = 0;
706 req->rl_sendctx->sc_unmap_count = 0; 731 req->rl_sendctx->sc_unmap_count = 0;
707 req->rl_sendctx->sc_req = req; 732 req->rl_sendctx->sc_req = req;
708 __clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags); 733 kref_init(&req->rl_kref);
709 734
735 ret = -EIO;
710 if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen)) 736 if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))
711 return -EIO; 737 goto err;
712
713 if (rtype != rpcrdma_areadch) 738 if (rtype != rpcrdma_areadch)
714 if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype)) 739 if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype))
715 return -EIO; 740 goto err;
716
717 return 0; 741 return 0;
742
743err:
744 trace_xprtrdma_prepsend_failed(&req->rl_slot, ret);
745 return ret;
718} 746}
719 747
720/** 748/**
@@ -842,50 +870,28 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst)
842 * send a Call message with a Position Zero Read chunk and a 870 * send a Call message with a Position Zero Read chunk and a
843 * regular Read chunk at the same time. 871 * regular Read chunk at the same time.
844 */ 872 */
845 if (rtype != rpcrdma_noch) { 873 ret = rpcrdma_encode_read_list(r_xprt, req, rqst, rtype);
846 ret = rpcrdma_encode_read_list(r_xprt, req, rqst, rtype);
847 if (ret)
848 goto out_err;
849 }
850 ret = encode_item_not_present(xdr);
851 if (ret) 874 if (ret)
852 goto out_err; 875 goto out_err;
853 876 ret = rpcrdma_encode_write_list(r_xprt, req, rqst, wtype);
854 if (wtype == rpcrdma_writech) {
855 ret = rpcrdma_encode_write_list(r_xprt, req, rqst, wtype);
856 if (ret)
857 goto out_err;
858 }
859 ret = encode_item_not_present(xdr);
860 if (ret) 877 if (ret)
861 goto out_err; 878 goto out_err;
862 879 ret = rpcrdma_encode_reply_chunk(r_xprt, req, rqst, wtype);
863 if (wtype != rpcrdma_replych)
864 ret = encode_item_not_present(xdr);
865 else
866 ret = rpcrdma_encode_reply_chunk(r_xprt, req, rqst, wtype);
867 if (ret) 880 if (ret)
868 goto out_err; 881 goto out_err;
869 882
870 trace_xprtrdma_marshal(rqst, xdr_stream_pos(xdr), rtype, wtype); 883 ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len,
871
872 ret = rpcrdma_prepare_send_sges(r_xprt, req, xdr_stream_pos(xdr),
873 &rqst->rq_snd_buf, rtype); 884 &rqst->rq_snd_buf, rtype);
874 if (ret) 885 if (ret)
875 goto out_err; 886 goto out_err;
887
888 trace_xprtrdma_marshal(req, rtype, wtype);
876 return 0; 889 return 0;
877 890
878out_err: 891out_err:
879 trace_xprtrdma_marshal_failed(rqst, ret); 892 trace_xprtrdma_marshal_failed(rqst, ret);
880 switch (ret) { 893 r_xprt->rx_stats.failed_marshal_count++;
881 case -EAGAIN: 894 frwr_reset(req);
882 xprt_wait_for_buffer_space(rqst->rq_xprt);
883 break;
884 case -ENOBUFS:
885 break;
886 default:
887 r_xprt->rx_stats.failed_marshal_count++;
888 }
889 return ret; 895 return ret;
890} 896}
891 897
@@ -1269,51 +1275,17 @@ out_badheader:
1269 goto out; 1275 goto out;
1270} 1276}
1271 1277
1272void rpcrdma_release_rqst(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) 1278static void rpcrdma_reply_done(struct kref *kref)
1273{
1274 /* Invalidate and unmap the data payloads before waking
1275 * the waiting application. This guarantees the memory
1276 * regions are properly fenced from the server before the
1277 * application accesses the data. It also ensures proper
1278 * send flow control: waking the next RPC waits until this
1279 * RPC has relinquished all its Send Queue entries.
1280 */
1281 if (!list_empty(&req->rl_registered))
1282 frwr_unmap_sync(r_xprt, &req->rl_registered);
1283
1284 /* Ensure that any DMA mapped pages associated with
1285 * the Send of the RPC Call have been unmapped before
1286 * allowing the RPC to complete. This protects argument
1287 * memory not controlled by the RPC client from being
1288 * re-used before we're done with it.
1289 */
1290 if (test_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags)) {
1291 r_xprt->rx_stats.reply_waits_for_send++;
1292 out_of_line_wait_on_bit(&req->rl_flags,
1293 RPCRDMA_REQ_F_TX_RESOURCES,
1294 bit_wait,
1295 TASK_UNINTERRUPTIBLE);
1296 }
1297}
1298
1299/* Reply handling runs in the poll worker thread. Anything that
1300 * might wait is deferred to a separate workqueue.
1301 */
1302void rpcrdma_deferred_completion(struct work_struct *work)
1303{ 1279{
1304 struct rpcrdma_rep *rep = 1280 struct rpcrdma_req *req =
1305 container_of(work, struct rpcrdma_rep, rr_work); 1281 container_of(kref, struct rpcrdma_req, rl_kref);
1306 struct rpcrdma_req *req = rpcr_to_rdmar(rep->rr_rqst);
1307 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt;
1308 1282
1309 trace_xprtrdma_defer_cmp(rep); 1283 rpcrdma_complete_rqst(req->rl_reply);
1310 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE)
1311 frwr_reminv(rep, &req->rl_registered);
1312 rpcrdma_release_rqst(r_xprt, req);
1313 rpcrdma_complete_rqst(rep);
1314} 1284}
1315 1285
1316/* Process received RPC/RDMA messages. 1286/**
1287 * rpcrdma_reply_handler - Process received RPC/RDMA messages
1288 * @rep: Incoming rpcrdma_rep object to process
1317 * 1289 *
1318 * Errors must result in the RPC task either being awakened, or 1290 * Errors must result in the RPC task either being awakened, or
1319 * allowed to timeout, to discover the errors at that time. 1291 * allowed to timeout, to discover the errors at that time.
@@ -1373,10 +1345,16 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep)
1373 } 1345 }
1374 req->rl_reply = rep; 1346 req->rl_reply = rep;
1375 rep->rr_rqst = rqst; 1347 rep->rr_rqst = rqst;
1376 clear_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags);
1377 1348
1378 trace_xprtrdma_reply(rqst->rq_task, rep, req, credits); 1349 trace_xprtrdma_reply(rqst->rq_task, rep, req, credits);
1379 queue_work(buf->rb_completion_wq, &rep->rr_work); 1350
1351 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE)
1352 frwr_reminv(rep, &req->rl_registered);
1353 if (!list_empty(&req->rl_registered))
1354 frwr_unmap_async(r_xprt, req);
1355 /* LocalInv completion will complete the RPC */
1356 else
1357 kref_put(&req->rl_kref, rpcrdma_reply_done);
1380 return; 1358 return;
1381 1359
1382out_badversion: 1360out_badversion:
diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index 1f73a6a7e43c..4993aa49ecbe 100644
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -298,6 +298,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
298 module_put(THIS_MODULE); 298 module_put(THIS_MODULE);
299} 299}
300 300
301/* 60 second timeout, no retries */
301static const struct rpc_timeout xprt_rdma_default_timeout = { 302static const struct rpc_timeout xprt_rdma_default_timeout = {
302 .to_initval = 60 * HZ, 303 .to_initval = 60 * HZ,
303 .to_maxval = 60 * HZ, 304 .to_maxval = 60 * HZ,
@@ -323,8 +324,9 @@ xprt_setup_rdma(struct xprt_create *args)
323 if (!xprt) 324 if (!xprt)
324 return ERR_PTR(-ENOMEM); 325 return ERR_PTR(-ENOMEM);
325 326
326 /* 60 second timeout, no retries */
327 xprt->timeout = &xprt_rdma_default_timeout; 327 xprt->timeout = &xprt_rdma_default_timeout;
328 xprt->connect_timeout = xprt->timeout->to_initval;
329 xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
328 xprt->bind_timeout = RPCRDMA_BIND_TO; 330 xprt->bind_timeout = RPCRDMA_BIND_TO;
329 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; 331 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
330 xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO; 332 xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;
@@ -487,31 +489,64 @@ xprt_rdma_timer(struct rpc_xprt *xprt, struct rpc_task *task)
487} 489}
488 490
489/** 491/**
490 * xprt_rdma_connect - try to establish a transport connection 492 * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
493 * @xprt: controlling transport instance
494 * @connect_timeout: reconnect timeout after client disconnects
495 * @reconnect_timeout: reconnect timeout after server disconnects
496 *
497 */
498static void xprt_rdma_tcp_set_connect_timeout(struct rpc_xprt *xprt,
499 unsigned long connect_timeout,
500 unsigned long reconnect_timeout)
501{
502 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
503
504 trace_xprtrdma_op_set_cto(r_xprt, connect_timeout, reconnect_timeout);
505
506 spin_lock(&xprt->transport_lock);
507
508 if (connect_timeout < xprt->connect_timeout) {
509 struct rpc_timeout to;
510 unsigned long initval;
511
512 to = *xprt->timeout;
513 initval = connect_timeout;
514 if (initval < RPCRDMA_INIT_REEST_TO << 1)
515 initval = RPCRDMA_INIT_REEST_TO << 1;
516 to.to_initval = initval;
517 to.to_maxval = initval;
518 r_xprt->rx_timeout = to;
519 xprt->timeout = &r_xprt->rx_timeout;
520 xprt->connect_timeout = connect_timeout;
521 }
522
523 if (reconnect_timeout < xprt->max_reconnect_timeout)
524 xprt->max_reconnect_timeout = reconnect_timeout;
525
526 spin_unlock(&xprt->transport_lock);
527}
528
529/**
530 * xprt_rdma_connect - schedule an attempt to reconnect
491 * @xprt: transport state 531 * @xprt: transport state
492 * @task: RPC scheduler context 532 * @task: RPC scheduler context (unused)
493 * 533 *
494 */ 534 */
495static void 535static void
496xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) 536xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)
497{ 537{
498 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); 538 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
539 unsigned long delay;
499 540
500 trace_xprtrdma_op_connect(r_xprt); 541 trace_xprtrdma_op_connect(r_xprt);
542
543 delay = 0;
501 if (r_xprt->rx_ep.rep_connected != 0) { 544 if (r_xprt->rx_ep.rep_connected != 0) {
502 /* Reconnect */ 545 delay = xprt_reconnect_delay(xprt);
503 schedule_delayed_work(&r_xprt->rx_connect_worker, 546 xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO);
504 xprt->reestablish_timeout);
505 xprt->reestablish_timeout <<= 1;
506 if (xprt->reestablish_timeout > RPCRDMA_MAX_REEST_TO)
507 xprt->reestablish_timeout = RPCRDMA_MAX_REEST_TO;
508 else if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO)
509 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
510 } else {
511 schedule_delayed_work(&r_xprt->rx_connect_worker, 0);
512 if (!RPC_IS_ASYNC(task))
513 flush_delayed_work(&r_xprt->rx_connect_worker);
514 } 547 }
548 queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker,
549 delay);
515} 550}
516 551
517/** 552/**
@@ -550,8 +585,11 @@ out_sleep:
550static void 585static void
551xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst) 586xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst)
552{ 587{
588 struct rpcrdma_xprt *r_xprt =
589 container_of(xprt, struct rpcrdma_xprt, rx_xprt);
590
553 memset(rqst, 0, sizeof(*rqst)); 591 memset(rqst, 0, sizeof(*rqst));
554 rpcrdma_buffer_put(rpcr_to_rdmar(rqst)); 592 rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));
555 rpc_wake_up_next(&xprt->backlog); 593 rpc_wake_up_next(&xprt->backlog);
556} 594}
557 595
@@ -618,9 +656,16 @@ xprt_rdma_free(struct rpc_task *task)
618 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); 656 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
619 struct rpcrdma_req *req = rpcr_to_rdmar(rqst); 657 struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
620 658
621 if (test_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags))
622 rpcrdma_release_rqst(r_xprt, req);
623 trace_xprtrdma_op_free(task, req); 659 trace_xprtrdma_op_free(task, req);
660
661 if (!list_empty(&req->rl_registered))
662 frwr_unmap_sync(r_xprt, req);
663
664 /* XXX: If the RPC is completing because of a signal and
665 * not because a reply was received, we ought to ensure
666 * that the Send completion has fired, so that memory
667 * involved with the Send is not still visible to the NIC.
668 */
624} 669}
625 670
626/** 671/**
@@ -667,7 +712,6 @@ xprt_rdma_send_request(struct rpc_rqst *rqst)
667 goto drop_connection; 712 goto drop_connection;
668 rqst->rq_xtime = ktime_get(); 713 rqst->rq_xtime = ktime_get();
669 714
670 __set_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags);
671 if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req)) 715 if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req))
672 goto drop_connection; 716 goto drop_connection;
673 717
@@ -760,6 +804,7 @@ static const struct rpc_xprt_ops xprt_rdma_procs = {
760 .send_request = xprt_rdma_send_request, 804 .send_request = xprt_rdma_send_request,
761 .close = xprt_rdma_close, 805 .close = xprt_rdma_close,
762 .destroy = xprt_rdma_destroy, 806 .destroy = xprt_rdma_destroy,
807 .set_connect_timeout = xprt_rdma_tcp_set_connect_timeout,
763 .print_stats = xprt_rdma_print_stats, 808 .print_stats = xprt_rdma_print_stats,
764 .enable_swap = xprt_rdma_enable_swap, 809 .enable_swap = xprt_rdma_enable_swap,
765 .disable_swap = xprt_rdma_disable_swap, 810 .disable_swap = xprt_rdma_disable_swap,
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 84bb37924540..805b1f35e1ca 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -89,14 +89,12 @@ static void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
89 */ 89 */
90static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) 90static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)
91{ 91{
92 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
93 struct rpcrdma_ia *ia = &r_xprt->rx_ia; 92 struct rpcrdma_ia *ia = &r_xprt->rx_ia;
94 93
95 /* Flush Receives, then wait for deferred Reply work 94 /* Flush Receives, then wait for deferred Reply work
96 * to complete. 95 * to complete.
97 */ 96 */
98 ib_drain_rq(ia->ri_id->qp); 97 ib_drain_rq(ia->ri_id->qp);
99 drain_workqueue(buf->rb_completion_wq);
100 98
101 /* Deferred Reply processing might have scheduled 99 /* Deferred Reply processing might have scheduled
102 * local invalidations. 100 * local invalidations.
@@ -901,7 +899,7 @@ out_emptyq:
901 * completions recently. This is a sign the Send Queue is 899 * completions recently. This is a sign the Send Queue is
902 * backing up. Cause the caller to pause and try again. 900 * backing up. Cause the caller to pause and try again.
903 */ 901 */
904 set_bit(RPCRDMA_BUF_F_EMPTY_SCQ, &buf->rb_flags); 902 xprt_wait_for_buffer_space(&r_xprt->rx_xprt);
905 r_xprt->rx_stats.empty_sendctx_q++; 903 r_xprt->rx_stats.empty_sendctx_q++;
906 return NULL; 904 return NULL;
907} 905}
@@ -936,10 +934,7 @@ rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
936 /* Paired with READ_ONCE */ 934 /* Paired with READ_ONCE */
937 smp_store_release(&buf->rb_sc_tail, next_tail); 935 smp_store_release(&buf->rb_sc_tail, next_tail);
938 936
939 if (test_and_clear_bit(RPCRDMA_BUF_F_EMPTY_SCQ, &buf->rb_flags)) { 937 xprt_write_space(&sc->sc_xprt->rx_xprt);
940 smp_mb__after_atomic();
941 xprt_write_space(&sc->sc_xprt->rx_xprt);
942 }
943} 938}
944 939
945static void 940static void
@@ -977,8 +972,6 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)
977 r_xprt->rx_stats.mrs_allocated += count; 972 r_xprt->rx_stats.mrs_allocated += count;
978 spin_unlock(&buf->rb_mrlock); 973 spin_unlock(&buf->rb_mrlock);
979 trace_xprtrdma_createmrs(r_xprt, count); 974 trace_xprtrdma_createmrs(r_xprt, count);
980
981 xprt_write_space(&r_xprt->rx_xprt);
982} 975}
983 976
984static void 977static void
@@ -990,6 +983,7 @@ rpcrdma_mr_refresh_worker(struct work_struct *work)
990 rx_buf); 983 rx_buf);
991 984
992 rpcrdma_mrs_create(r_xprt); 985 rpcrdma_mrs_create(r_xprt);
986 xprt_write_space(&r_xprt->rx_xprt);
993} 987}
994 988
995/** 989/**
@@ -1025,7 +1019,6 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,
1025 if (!req->rl_recvbuf) 1019 if (!req->rl_recvbuf)
1026 goto out4; 1020 goto out4;
1027 1021
1028 req->rl_buffer = buffer;
1029 INIT_LIST_HEAD(&req->rl_registered); 1022 INIT_LIST_HEAD(&req->rl_registered);
1030 spin_lock(&buffer->rb_lock); 1023 spin_lock(&buffer->rb_lock);
1031 list_add(&req->rl_all, &buffer->rb_allreqs); 1024 list_add(&req->rl_all, &buffer->rb_allreqs);
@@ -1042,9 +1035,9 @@ out1:
1042 return NULL; 1035 return NULL;
1043} 1036}
1044 1037
1045static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp) 1038static struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,
1039 bool temp)
1046{ 1040{
1047 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1048 struct rpcrdma_rep *rep; 1041 struct rpcrdma_rep *rep;
1049 1042
1050 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 1043 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
@@ -1055,27 +1048,22 @@ static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp)
1055 DMA_FROM_DEVICE, GFP_KERNEL); 1048 DMA_FROM_DEVICE, GFP_KERNEL);
1056 if (!rep->rr_rdmabuf) 1049 if (!rep->rr_rdmabuf)
1057 goto out_free; 1050 goto out_free;
1051
1058 xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf), 1052 xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),
1059 rdmab_length(rep->rr_rdmabuf)); 1053 rdmab_length(rep->rr_rdmabuf));
1060
1061 rep->rr_cqe.done = rpcrdma_wc_receive; 1054 rep->rr_cqe.done = rpcrdma_wc_receive;
1062 rep->rr_rxprt = r_xprt; 1055 rep->rr_rxprt = r_xprt;
1063 INIT_WORK(&rep->rr_work, rpcrdma_deferred_completion);
1064 rep->rr_recv_wr.next = NULL; 1056 rep->rr_recv_wr.next = NULL;
1065 rep->rr_recv_wr.wr_cqe = &rep->rr_cqe; 1057 rep->rr_recv_wr.wr_cqe = &rep->rr_cqe;
1066 rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov; 1058 rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
1067 rep->rr_recv_wr.num_sge = 1; 1059 rep->rr_recv_wr.num_sge = 1;
1068 rep->rr_temp = temp; 1060 rep->rr_temp = temp;
1069 1061 return rep;
1070 spin_lock(&buf->rb_lock);
1071 list_add(&rep->rr_list, &buf->rb_recv_bufs);
1072 spin_unlock(&buf->rb_lock);
1073 return true;
1074 1062
1075out_free: 1063out_free:
1076 kfree(rep); 1064 kfree(rep);
1077out: 1065out:
1078 return false; 1066 return NULL;
1079} 1067}
1080 1068
1081/** 1069/**
@@ -1089,7 +1077,6 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
1089 struct rpcrdma_buffer *buf = &r_xprt->rx_buf; 1077 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1090 int i, rc; 1078 int i, rc;
1091 1079
1092 buf->rb_flags = 0;
1093 buf->rb_max_requests = r_xprt->rx_ep.rep_max_requests; 1080 buf->rb_max_requests = r_xprt->rx_ep.rep_max_requests;
1094 buf->rb_bc_srv_max_requests = 0; 1081 buf->rb_bc_srv_max_requests = 0;
1095 spin_lock_init(&buf->rb_mrlock); 1082 spin_lock_init(&buf->rb_mrlock);
@@ -1122,15 +1109,6 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
1122 if (rc) 1109 if (rc)
1123 goto out; 1110 goto out;
1124 1111
1125 buf->rb_completion_wq = alloc_workqueue("rpcrdma-%s",
1126 WQ_MEM_RECLAIM | WQ_HIGHPRI,
1127 0,
1128 r_xprt->rx_xprt.address_strings[RPC_DISPLAY_ADDR]);
1129 if (!buf->rb_completion_wq) {
1130 rc = -ENOMEM;
1131 goto out;
1132 }
1133
1134 return 0; 1112 return 0;
1135out: 1113out:
1136 rpcrdma_buffer_destroy(buf); 1114 rpcrdma_buffer_destroy(buf);
@@ -1204,11 +1182,6 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1204{ 1182{
1205 cancel_delayed_work_sync(&buf->rb_refresh_worker); 1183 cancel_delayed_work_sync(&buf->rb_refresh_worker);
1206 1184
1207 if (buf->rb_completion_wq) {
1208 destroy_workqueue(buf->rb_completion_wq);
1209 buf->rb_completion_wq = NULL;
1210 }
1211
1212 rpcrdma_sendctxs_destroy(buf); 1185 rpcrdma_sendctxs_destroy(buf);
1213 1186
1214 while (!list_empty(&buf->rb_recv_bufs)) { 1187 while (!list_empty(&buf->rb_recv_bufs)) {
@@ -1325,13 +1298,12 @@ rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
1325 1298
1326/** 1299/**
1327 * rpcrdma_buffer_put - Put request/reply buffers back into pool 1300 * rpcrdma_buffer_put - Put request/reply buffers back into pool
1301 * @buffers: buffer pool
1328 * @req: object to return 1302 * @req: object to return
1329 * 1303 *
1330 */ 1304 */
1331void 1305void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req)
1332rpcrdma_buffer_put(struct rpcrdma_req *req)
1333{ 1306{
1334 struct rpcrdma_buffer *buffers = req->rl_buffer;
1335 struct rpcrdma_rep *rep = req->rl_reply; 1307 struct rpcrdma_rep *rep = req->rl_reply;
1336 1308
1337 req->rl_reply = NULL; 1309 req->rl_reply = NULL;
@@ -1484,8 +1456,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
1484 struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr; 1456 struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;
1485 int rc; 1457 int rc;
1486 1458
1487 if (!ep->rep_send_count || 1459 if (!ep->rep_send_count || kref_read(&req->rl_kref) > 1) {
1488 test_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags)) {
1489 send_wr->send_flags |= IB_SEND_SIGNALED; 1460 send_wr->send_flags |= IB_SEND_SIGNALED;
1490 ep->rep_send_count = ep->rep_send_batch; 1461 ep->rep_send_count = ep->rep_send_batch;
1491 } else { 1462 } else {
@@ -1505,11 +1476,13 @@ rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
1505{ 1476{
1506 struct rpcrdma_buffer *buf = &r_xprt->rx_buf; 1477 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1507 struct rpcrdma_ep *ep = &r_xprt->rx_ep; 1478 struct rpcrdma_ep *ep = &r_xprt->rx_ep;
1508 struct ib_recv_wr *wr, *bad_wr; 1479 struct ib_recv_wr *i, *wr, *bad_wr;
1480 struct rpcrdma_rep *rep;
1509 int needed, count, rc; 1481 int needed, count, rc;
1510 1482
1511 rc = 0; 1483 rc = 0;
1512 count = 0; 1484 count = 0;
1485
1513 needed = buf->rb_credits + (buf->rb_bc_srv_max_requests << 1); 1486 needed = buf->rb_credits + (buf->rb_bc_srv_max_requests << 1);
1514 if (ep->rep_receive_count > needed) 1487 if (ep->rep_receive_count > needed)
1515 goto out; 1488 goto out;
@@ -1517,51 +1490,65 @@ rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
1517 if (!temp) 1490 if (!temp)
1518 needed += RPCRDMA_MAX_RECV_BATCH; 1491 needed += RPCRDMA_MAX_RECV_BATCH;
1519 1492
1520 count = 0; 1493 /* fast path: all needed reps can be found on the free list */
1521 wr = NULL; 1494 wr = NULL;
1495 spin_lock(&buf->rb_lock);
1522 while (needed) { 1496 while (needed) {
1523 struct rpcrdma_regbuf *rb;
1524 struct rpcrdma_rep *rep;
1525
1526 spin_lock(&buf->rb_lock);
1527 rep = list_first_entry_or_null(&buf->rb_recv_bufs, 1497 rep = list_first_entry_or_null(&buf->rb_recv_bufs,
1528 struct rpcrdma_rep, rr_list); 1498 struct rpcrdma_rep, rr_list);
1529 if (likely(rep)) 1499 if (!rep)
1530 list_del(&rep->rr_list); 1500 break;
1531 spin_unlock(&buf->rb_lock);
1532 if (!rep) {
1533 if (!rpcrdma_rep_create(r_xprt, temp))
1534 break;
1535 continue;
1536 }
1537 1501
1538 rb = rep->rr_rdmabuf; 1502 list_del(&rep->rr_list);
1539 if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) { 1503 rep->rr_recv_wr.next = wr;
1540 rpcrdma_recv_buffer_put(rep); 1504 wr = &rep->rr_recv_wr;
1505 --needed;
1506 }
1507 spin_unlock(&buf->rb_lock);
1508
1509 while (needed) {
1510 rep = rpcrdma_rep_create(r_xprt, temp);
1511 if (!rep)
1541 break; 1512 break;
1542 }
1543 1513
1544 trace_xprtrdma_post_recv(rep->rr_recv_wr.wr_cqe);
1545 rep->rr_recv_wr.next = wr; 1514 rep->rr_recv_wr.next = wr;
1546 wr = &rep->rr_recv_wr; 1515 wr = &rep->rr_recv_wr;
1547 ++count;
1548 --needed; 1516 --needed;
1549 } 1517 }
1550 if (!count) 1518 if (!wr)
1551 goto out; 1519 goto out;
1552 1520
1521 for (i = wr; i; i = i->next) {
1522 rep = container_of(i, struct rpcrdma_rep, rr_recv_wr);
1523
1524 if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))
1525 goto release_wrs;
1526
1527 trace_xprtrdma_post_recv(rep->rr_recv_wr.wr_cqe);
1528 ++count;
1529 }
1530
1553 rc = ib_post_recv(r_xprt->rx_ia.ri_id->qp, wr, 1531 rc = ib_post_recv(r_xprt->rx_ia.ri_id->qp, wr,
1554 (const struct ib_recv_wr **)&bad_wr); 1532 (const struct ib_recv_wr **)&bad_wr);
1533out:
1534 trace_xprtrdma_post_recvs(r_xprt, count, rc);
1555 if (rc) { 1535 if (rc) {
1556 for (wr = bad_wr; wr; wr = wr->next) { 1536 for (wr = bad_wr; wr;) {
1557 struct rpcrdma_rep *rep; 1537 struct rpcrdma_rep *rep;
1558 1538
1559 rep = container_of(wr, struct rpcrdma_rep, rr_recv_wr); 1539 rep = container_of(wr, struct rpcrdma_rep, rr_recv_wr);
1540 wr = wr->next;
1560 rpcrdma_recv_buffer_put(rep); 1541 rpcrdma_recv_buffer_put(rep);
1561 --count; 1542 --count;
1562 } 1543 }
1563 } 1544 }
1564 ep->rep_receive_count += count; 1545 ep->rep_receive_count += count;
1565out: 1546 return;
1566 trace_xprtrdma_post_recvs(r_xprt, count, rc); 1547
1548release_wrs:
1549 for (i = wr; i;) {
1550 rep = container_of(i, struct rpcrdma_rep, rr_recv_wr);
1551 i = i->next;
1552 rpcrdma_recv_buffer_put(rep);
1553 }
1567} 1554}
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index d1e0749bcbc4..8378f45d2da7 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -44,7 +44,8 @@
44 44
45#include <linux/wait.h> /* wait_queue_head_t, etc */ 45#include <linux/wait.h> /* wait_queue_head_t, etc */
46#include <linux/spinlock.h> /* spinlock_t, etc */ 46#include <linux/spinlock.h> /* spinlock_t, etc */
47#include <linux/atomic.h> /* atomic_t, etc */ 47#include <linux/atomic.h> /* atomic_t, etc */
48#include <linux/kref.h> /* struct kref */
48#include <linux/workqueue.h> /* struct work_struct */ 49#include <linux/workqueue.h> /* struct work_struct */
49 50
50#include <rdma/rdma_cm.h> /* RDMA connection api */ 51#include <rdma/rdma_cm.h> /* RDMA connection api */
@@ -202,10 +203,9 @@ struct rpcrdma_rep {
202 bool rr_temp; 203 bool rr_temp;
203 struct rpcrdma_regbuf *rr_rdmabuf; 204 struct rpcrdma_regbuf *rr_rdmabuf;
204 struct rpcrdma_xprt *rr_rxprt; 205 struct rpcrdma_xprt *rr_rxprt;
205 struct work_struct rr_work; 206 struct rpc_rqst *rr_rqst;
206 struct xdr_buf rr_hdrbuf; 207 struct xdr_buf rr_hdrbuf;
207 struct xdr_stream rr_stream; 208 struct xdr_stream rr_stream;
208 struct rpc_rqst *rr_rqst;
209 struct list_head rr_list; 209 struct list_head rr_list;
210 struct ib_recv_wr rr_recv_wr; 210 struct ib_recv_wr rr_recv_wr;
211}; 211};
@@ -240,18 +240,12 @@ struct rpcrdma_sendctx {
240 * An external memory region is any buffer or page that is registered 240 * An external memory region is any buffer or page that is registered
241 * on the fly (ie, not pre-registered). 241 * on the fly (ie, not pre-registered).
242 */ 242 */
243enum rpcrdma_frwr_state { 243struct rpcrdma_req;
244 FRWR_IS_INVALID, /* ready to be used */
245 FRWR_IS_VALID, /* in use */
246 FRWR_FLUSHED_FR, /* flushed FASTREG WR */
247 FRWR_FLUSHED_LI, /* flushed LOCALINV WR */
248};
249
250struct rpcrdma_frwr { 244struct rpcrdma_frwr {
251 struct ib_mr *fr_mr; 245 struct ib_mr *fr_mr;
252 struct ib_cqe fr_cqe; 246 struct ib_cqe fr_cqe;
253 enum rpcrdma_frwr_state fr_state;
254 struct completion fr_linv_done; 247 struct completion fr_linv_done;
248 struct rpcrdma_req *fr_req;
255 union { 249 union {
256 struct ib_reg_wr fr_regwr; 250 struct ib_reg_wr fr_regwr;
257 struct ib_send_wr fr_invwr; 251 struct ib_send_wr fr_invwr;
@@ -326,7 +320,6 @@ struct rpcrdma_buffer;
326struct rpcrdma_req { 320struct rpcrdma_req {
327 struct list_head rl_list; 321 struct list_head rl_list;
328 struct rpc_rqst rl_slot; 322 struct rpc_rqst rl_slot;
329 struct rpcrdma_buffer *rl_buffer;
330 struct rpcrdma_rep *rl_reply; 323 struct rpcrdma_rep *rl_reply;
331 struct xdr_stream rl_stream; 324 struct xdr_stream rl_stream;
332 struct xdr_buf rl_hdrbuf; 325 struct xdr_buf rl_hdrbuf;
@@ -336,18 +329,12 @@ struct rpcrdma_req {
336 struct rpcrdma_regbuf *rl_recvbuf; /* rq_rcv_buf */ 329 struct rpcrdma_regbuf *rl_recvbuf; /* rq_rcv_buf */
337 330
338 struct list_head rl_all; 331 struct list_head rl_all;
339 unsigned long rl_flags; 332 struct kref rl_kref;
340 333
341 struct list_head rl_registered; /* registered segments */ 334 struct list_head rl_registered; /* registered segments */
342 struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS]; 335 struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS];
343}; 336};
344 337
345/* rl_flags */
346enum {
347 RPCRDMA_REQ_F_PENDING = 0,
348 RPCRDMA_REQ_F_TX_RESOURCES,
349};
350
351static inline struct rpcrdma_req * 338static inline struct rpcrdma_req *
352rpcr_to_rdmar(const struct rpc_rqst *rqst) 339rpcr_to_rdmar(const struct rpc_rqst *rqst)
353{ 340{
@@ -391,22 +378,15 @@ struct rpcrdma_buffer {
391 struct list_head rb_recv_bufs; 378 struct list_head rb_recv_bufs;
392 struct list_head rb_allreqs; 379 struct list_head rb_allreqs;
393 380
394 unsigned long rb_flags;
395 u32 rb_max_requests; 381 u32 rb_max_requests;
396 u32 rb_credits; /* most recent credit grant */ 382 u32 rb_credits; /* most recent credit grant */
397 383
398 u32 rb_bc_srv_max_requests; 384 u32 rb_bc_srv_max_requests;
399 u32 rb_bc_max_requests; 385 u32 rb_bc_max_requests;
400 386
401 struct workqueue_struct *rb_completion_wq;
402 struct delayed_work rb_refresh_worker; 387 struct delayed_work rb_refresh_worker;
403}; 388};
404 389
405/* rb_flags */
406enum {
407 RPCRDMA_BUF_F_EMPTY_SCQ = 0,
408};
409
410/* 390/*
411 * Statistics for RPCRDMA 391 * Statistics for RPCRDMA
412 */ 392 */
@@ -452,6 +432,7 @@ struct rpcrdma_xprt {
452 struct rpcrdma_ep rx_ep; 432 struct rpcrdma_ep rx_ep;
453 struct rpcrdma_buffer rx_buf; 433 struct rpcrdma_buffer rx_buf;
454 struct delayed_work rx_connect_worker; 434 struct delayed_work rx_connect_worker;
435 struct rpc_timeout rx_timeout;
455 struct rpcrdma_stats rx_stats; 436 struct rpcrdma_stats rx_stats;
456}; 437};
457 438
@@ -518,7 +499,8 @@ rpcrdma_mr_recycle(struct rpcrdma_mr *mr)
518} 499}
519 500
520struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *); 501struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
521void rpcrdma_buffer_put(struct rpcrdma_req *); 502void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers,
503 struct rpcrdma_req *req);
522void rpcrdma_recv_buffer_put(struct rpcrdma_rep *); 504void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
523 505
524bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, 506bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size,
@@ -564,6 +546,7 @@ rpcrdma_data_dir(bool writing)
564/* Memory registration calls xprtrdma/frwr_ops.c 546/* Memory registration calls xprtrdma/frwr_ops.c
565 */ 547 */
566bool frwr_is_supported(struct ib_device *device); 548bool frwr_is_supported(struct ib_device *device);
549void frwr_reset(struct rpcrdma_req *req);
567int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep); 550int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep);
568int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr); 551int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr);
569void frwr_release_mr(struct rpcrdma_mr *mr); 552void frwr_release_mr(struct rpcrdma_mr *mr);
@@ -574,8 +557,8 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
574 struct rpcrdma_mr **mr); 557 struct rpcrdma_mr **mr);
575int frwr_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req); 558int frwr_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req);
576void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs); 559void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs);
577void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, 560void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
578 struct list_head *mrs); 561void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
579 562
580/* 563/*
581 * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c 564 * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
@@ -598,9 +581,6 @@ int rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst);
598void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *); 581void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
599void rpcrdma_complete_rqst(struct rpcrdma_rep *rep); 582void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);
600void rpcrdma_reply_handler(struct rpcrdma_rep *rep); 583void rpcrdma_reply_handler(struct rpcrdma_rep *rep);
601void rpcrdma_release_rqst(struct rpcrdma_xprt *r_xprt,
602 struct rpcrdma_req *req);
603void rpcrdma_deferred_completion(struct work_struct *work);
604 584
605static inline void rpcrdma_set_xdrlen(struct xdr_buf *xdr, size_t len) 585static inline void rpcrdma_set_xdrlen(struct xdr_buf *xdr, size_t len)
606{ 586{
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 97c15d47f343..3c2cc96afcaa 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -2414,25 +2414,6 @@ out:
2414 xprt_wake_pending_tasks(xprt, status); 2414 xprt_wake_pending_tasks(xprt, status);
2415} 2415}
2416 2416
2417static unsigned long xs_reconnect_delay(const struct rpc_xprt *xprt)
2418{
2419 unsigned long start, now = jiffies;
2420
2421 start = xprt->stat.connect_start + xprt->reestablish_timeout;
2422 if (time_after(start, now))
2423 return start - now;
2424 return 0;
2425}
2426
2427static void xs_reconnect_backoff(struct rpc_xprt *xprt)
2428{
2429 xprt->reestablish_timeout <<= 1;
2430 if (xprt->reestablish_timeout > xprt->max_reconnect_timeout)
2431 xprt->reestablish_timeout = xprt->max_reconnect_timeout;
2432 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
2433 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2434}
2435
2436/** 2417/**
2437 * xs_connect - connect a socket to a remote endpoint 2418 * xs_connect - connect a socket to a remote endpoint
2438 * @xprt: pointer to transport structure 2419 * @xprt: pointer to transport structure
@@ -2462,8 +2443,8 @@ static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
2462 /* Start by resetting any existing state */ 2443 /* Start by resetting any existing state */
2463 xs_reset_transport(transport); 2444 xs_reset_transport(transport);
2464 2445
2465 delay = xs_reconnect_delay(xprt); 2446 delay = xprt_reconnect_delay(xprt);
2466 xs_reconnect_backoff(xprt); 2447 xprt_reconnect_backoff(xprt, XS_TCP_INIT_REEST_TO);
2467 2448
2468 } else 2449 } else
2469 dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); 2450 dprintk("RPC: xs_connect scheduled xprt %p\n", xprt);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-16 21:58:16 -0400