aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/filesystems/nfs/rpc-cache.txt6
-rw-r--r--fs/lockd/host.c2
-rw-r--r--fs/nfs/dns_resolve.c15
-rw-r--r--fs/nfsd/cache.h20
-rw-r--r--fs/nfsd/export.c14
-rw-r--r--fs/nfsd/export.h2
-rw-r--r--fs/nfsd/netns.h8
-rw-r--r--fs/nfsd/nfs4callback.c98
-rw-r--r--fs/nfsd/nfs4idmap.c11
-rw-r--r--fs/nfsd/nfs4proc.c289
-rw-r--r--fs/nfsd/nfs4state.c41
-rw-r--r--fs/nfsd/nfs4xdr.c50
-rw-r--r--fs/nfsd/nfscache.c142
-rw-r--r--fs/nfsd/nfsctl.c1
-rw-r--r--fs/nfsd/state.h10
-rw-r--r--fs/nfsd/vfs.c5
-rw-r--r--fs/nfsd/xdr4.h28
-rw-r--r--fs/nfsd/xdr4cb.h10
-rw-r--r--include/linux/sunrpc/cache.h18
-rw-r--r--include/linux/sunrpc/svc_rdma.h13
-rw-r--r--include/linux/sunrpc/svcauth.h1
-rw-r--r--net/sunrpc/auth_gss/svcauth_gss.c41
-rw-r--r--net/sunrpc/cache.c153
-rw-r--r--net/sunrpc/svc_xprt.c2
-rw-r--r--net/sunrpc/svcauth.c74
-rw-r--r--net/sunrpc/svcauth_unix.c24
-rw-r--r--net/sunrpc/svcsock.c53
-rw-r--r--net/sunrpc/xprtrdma/svc_rdma_backchannel.c23
-rw-r--r--net/sunrpc/xprtrdma/svc_rdma_transport.c10
29 files changed, 858 insertions, 306 deletions
diff --git a/Documentation/filesystems/nfs/rpc-cache.txt b/Documentation/filesystems/nfs/rpc-cache.txt
index ebcaaee21616..c4dac829db0f 100644
--- a/Documentation/filesystems/nfs/rpc-cache.txt
+++ b/Documentation/filesystems/nfs/rpc-cache.txt
@@ -84,7 +84,7 @@ Creating a Cache
84 A message from user space has arrived to fill out a 84 A message from user space has arrived to fill out a
85 cache entry. It is in 'buf' of length 'len'. 85 cache entry. It is in 'buf' of length 'len'.
86 cache_parse should parse this, find the item in the 86 cache_parse should parse this, find the item in the
87 cache with sunrpc_cache_lookup, and update the item 87 cache with sunrpc_cache_lookup_rcu, and update the item
88 with sunrpc_cache_update. 88 with sunrpc_cache_update.
89 89
90 90
@@ -95,7 +95,7 @@ Creating a Cache
95Using a cache 95Using a cache
96------------- 96-------------
97 97
98To find a value in a cache, call sunrpc_cache_lookup passing a pointer 98To find a value in a cache, call sunrpc_cache_lookup_rcu passing a pointer
99to the cache_head in a sample item with the 'key' fields filled in. 99to the cache_head in a sample item with the 'key' fields filled in.
100This will be passed to ->match to identify the target entry. If no 100This will be passed to ->match to identify the target entry. If no
101entry is found, a new entry will be create, added to the cache, and 101entry is found, a new entry will be create, added to the cache, and
@@ -116,7 +116,7 @@ item does become valid, the deferred copy of the request will be
116revisited (->revisit). It is expected that this method will 116revisited (->revisit). It is expected that this method will
117reschedule the request for processing. 117reschedule the request for processing.
118 118
119The value returned by sunrpc_cache_lookup can also be passed to 119The value returned by sunrpc_cache_lookup_rcu can also be passed to
120sunrpc_cache_update to set the content for the item. A second item is 120sunrpc_cache_update to set the content for the item. A second item is
121passed which should hold the content. If the item found by _lookup 121passed which should hold the content. If the item found by _lookup
122has valid data, then it is discarded and a new item is created. This 122has valid data, then it is discarded and a new item is created. This
diff --git a/fs/lockd/host.c b/fs/lockd/host.c
index d35cd6be0675..93fb7cf0b92b 100644
--- a/fs/lockd/host.c
+++ b/fs/lockd/host.c
@@ -341,7 +341,7 @@ struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp,
341 }; 341 };
342 struct lockd_net *ln = net_generic(net, lockd_net_id); 342 struct lockd_net *ln = net_generic(net, lockd_net_id);
343 343
344 dprintk("lockd: %s(host='%*s', vers=%u, proto=%s)\n", __func__, 344 dprintk("lockd: %s(host='%.*s', vers=%u, proto=%s)\n", __func__,
345 (int)hostname_len, hostname, rqstp->rq_vers, 345 (int)hostname_len, hostname, rqstp->rq_vers,
346 (rqstp->rq_prot == IPPROTO_UDP ? "udp" : "tcp")); 346 (rqstp->rq_prot == IPPROTO_UDP ? "udp" : "tcp"));
347 347
diff --git a/fs/nfs/dns_resolve.c b/fs/nfs/dns_resolve.c
index 060c658eab66..a7d3df85736d 100644
--- a/fs/nfs/dns_resolve.c
+++ b/fs/nfs/dns_resolve.c
@@ -65,6 +65,7 @@ struct nfs_dns_ent {
65 65
66 struct sockaddr_storage addr; 66 struct sockaddr_storage addr;
67 size_t addrlen; 67 size_t addrlen;
68 struct rcu_head rcu_head;
68}; 69};
69 70
70 71
@@ -101,15 +102,23 @@ static void nfs_dns_ent_init(struct cache_head *cnew,
101 } 102 }
102} 103}
103 104
104static void nfs_dns_ent_put(struct kref *ref) 105static void nfs_dns_ent_free_rcu(struct rcu_head *head)
105{ 106{
106 struct nfs_dns_ent *item; 107 struct nfs_dns_ent *item;
107 108
108 item = container_of(ref, struct nfs_dns_ent, h.ref); 109 item = container_of(head, struct nfs_dns_ent, rcu_head);
109 kfree(item->hostname); 110 kfree(item->hostname);
110 kfree(item); 111 kfree(item);
111} 112}
112 113
114static void nfs_dns_ent_put(struct kref *ref)
115{
116 struct nfs_dns_ent *item;
117
118 item = container_of(ref, struct nfs_dns_ent, h.ref);
119 call_rcu(&item->rcu_head, nfs_dns_ent_free_rcu);
120}
121
113static struct cache_head *nfs_dns_ent_alloc(void) 122static struct cache_head *nfs_dns_ent_alloc(void)
114{ 123{
115 struct nfs_dns_ent *item = kmalloc(sizeof(*item), GFP_KERNEL); 124 struct nfs_dns_ent *item = kmalloc(sizeof(*item), GFP_KERNEL);
@@ -195,7 +204,7 @@ static struct nfs_dns_ent *nfs_dns_lookup(struct cache_detail *cd,
195{ 204{
196 struct cache_head *ch; 205 struct cache_head *ch;
197 206
198 ch = sunrpc_cache_lookup(cd, 207 ch = sunrpc_cache_lookup_rcu(cd,
199 &key->h, 208 &key->h,
200 nfs_dns_hash(key)); 209 nfs_dns_hash(key));
201 if (!ch) 210 if (!ch)
diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h
index b7559c6f2b97..4a98537efb0f 100644
--- a/fs/nfsd/cache.h
+++ b/fs/nfsd/cache.h
@@ -19,18 +19,22 @@
19 * is much larger than a sockaddr_in6. 19 * is much larger than a sockaddr_in6.
20 */ 20 */
21struct svc_cacherep { 21struct svc_cacherep {
22 struct list_head c_lru; 22 struct {
23 /* Keep often-read xid, csum in the same cache line: */
24 __be32 k_xid;
25 __wsum k_csum;
26 u32 k_proc;
27 u32 k_prot;
28 u32 k_vers;
29 unsigned int k_len;
30 struct sockaddr_in6 k_addr;
31 } c_key;
23 32
33 struct rb_node c_node;
34 struct list_head c_lru;
24 unsigned char c_state, /* unused, inprog, done */ 35 unsigned char c_state, /* unused, inprog, done */
25 c_type, /* status, buffer */ 36 c_type, /* status, buffer */
26 c_secure : 1; /* req came from port < 1024 */ 37 c_secure : 1; /* req came from port < 1024 */
27 struct sockaddr_in6 c_addr;
28 __be32 c_xid;
29 u32 c_prot;
30 u32 c_proc;
31 u32 c_vers;
32 unsigned int c_len;
33 __wsum c_csum;
34 unsigned long c_timestamp; 38 unsigned long c_timestamp;
35 union { 39 union {
36 struct kvec u_vec; 40 struct kvec u_vec;
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index a1143f7c2201..802993d8912f 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -46,7 +46,7 @@ static void expkey_put(struct kref *ref)
46 !test_bit(CACHE_NEGATIVE, &key->h.flags)) 46 !test_bit(CACHE_NEGATIVE, &key->h.flags))
47 path_put(&key->ek_path); 47 path_put(&key->ek_path);
48 auth_domain_put(key->ek_client); 48 auth_domain_put(key->ek_client);
49 kfree(key); 49 kfree_rcu(key, ek_rcu);
50} 50}
51 51
52static void expkey_request(struct cache_detail *cd, 52static void expkey_request(struct cache_detail *cd,
@@ -265,7 +265,7 @@ svc_expkey_lookup(struct cache_detail *cd, struct svc_expkey *item)
265 struct cache_head *ch; 265 struct cache_head *ch;
266 int hash = svc_expkey_hash(item); 266 int hash = svc_expkey_hash(item);
267 267
268 ch = sunrpc_cache_lookup(cd, &item->h, hash); 268 ch = sunrpc_cache_lookup_rcu(cd, &item->h, hash);
269 if (ch) 269 if (ch)
270 return container_of(ch, struct svc_expkey, h); 270 return container_of(ch, struct svc_expkey, h);
271 else 271 else
@@ -314,7 +314,7 @@ static void svc_export_put(struct kref *ref)
314 auth_domain_put(exp->ex_client); 314 auth_domain_put(exp->ex_client);
315 nfsd4_fslocs_free(&exp->ex_fslocs); 315 nfsd4_fslocs_free(&exp->ex_fslocs);
316 kfree(exp->ex_uuid); 316 kfree(exp->ex_uuid);
317 kfree(exp); 317 kfree_rcu(exp, ex_rcu);
318} 318}
319 319
320static void svc_export_request(struct cache_detail *cd, 320static void svc_export_request(struct cache_detail *cd,
@@ -780,7 +780,7 @@ svc_export_lookup(struct svc_export *exp)
780 struct cache_head *ch; 780 struct cache_head *ch;
781 int hash = svc_export_hash(exp); 781 int hash = svc_export_hash(exp);
782 782
783 ch = sunrpc_cache_lookup(exp->cd, &exp->h, hash); 783 ch = sunrpc_cache_lookup_rcu(exp->cd, &exp->h, hash);
784 if (ch) 784 if (ch)
785 return container_of(ch, struct svc_export, h); 785 return container_of(ch, struct svc_export, h);
786 else 786 else
@@ -1216,9 +1216,9 @@ static int e_show(struct seq_file *m, void *p)
1216} 1216}
1217 1217
1218const struct seq_operations nfs_exports_op = { 1218const struct seq_operations nfs_exports_op = {
1219 .start = cache_seq_start, 1219 .start = cache_seq_start_rcu,
1220 .next = cache_seq_next, 1220 .next = cache_seq_next_rcu,
1221 .stop = cache_seq_stop, 1221 .stop = cache_seq_stop_rcu,
1222 .show = e_show, 1222 .show = e_show,
1223}; 1223};
1224 1224
diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h
index c8b74126ddaa..e7daa1f246f0 100644
--- a/fs/nfsd/export.h
+++ b/fs/nfsd/export.h
@@ -61,6 +61,7 @@ struct svc_export {
61 u32 ex_layout_types; 61 u32 ex_layout_types;
62 struct nfsd4_deviceid_map *ex_devid_map; 62 struct nfsd4_deviceid_map *ex_devid_map;
63 struct cache_detail *cd; 63 struct cache_detail *cd;
64 struct rcu_head ex_rcu;
64}; 65};
65 66
66/* an "export key" (expkey) maps a filehandlefragement to an 67/* an "export key" (expkey) maps a filehandlefragement to an
@@ -75,6 +76,7 @@ struct svc_expkey {
75 u32 ek_fsid[6]; 76 u32 ek_fsid[6];
76 77
77 struct path ek_path; 78 struct path ek_path;
79 struct rcu_head ek_rcu;
78}; 80};
79 81
80#define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) 82#define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC))
diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 426f55005697..32cb8c027483 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -123,6 +123,14 @@ struct nfsd_net {
123 123
124 wait_queue_head_t ntf_wq; 124 wait_queue_head_t ntf_wq;
125 atomic_t ntf_refcnt; 125 atomic_t ntf_refcnt;
126
127 /*
128 * clientid and stateid data for construction of net unique COPY
129 * stateids.
130 */
131 u32 s2s_cp_cl_id;
132 struct idr s2s_cp_stateids;
133 spinlock_t s2s_cp_lock;
126}; 134};
127 135
128/* Simple check to find out if a given net was properly initialized */ 136/* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 601bf33c26a0..25987bcdf96f 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -39,6 +39,7 @@
39#include "state.h" 39#include "state.h"
40#include "netns.h" 40#include "netns.h"
41#include "xdr4cb.h" 41#include "xdr4cb.h"
42#include "xdr4.h"
42 43
43#define NFSDDBG_FACILITY NFSDDBG_PROC 44#define NFSDDBG_FACILITY NFSDDBG_PROC
44 45
@@ -105,6 +106,7 @@ enum nfs_cb_opnum4 {
105 OP_CB_WANTS_CANCELLED = 12, 106 OP_CB_WANTS_CANCELLED = 12,
106 OP_CB_NOTIFY_LOCK = 13, 107 OP_CB_NOTIFY_LOCK = 13,
107 OP_CB_NOTIFY_DEVICEID = 14, 108 OP_CB_NOTIFY_DEVICEID = 14,
109 OP_CB_OFFLOAD = 15,
108 OP_CB_ILLEGAL = 10044 110 OP_CB_ILLEGAL = 10044
109}; 111};
110 112
@@ -683,6 +685,101 @@ static int nfs4_xdr_dec_cb_notify_lock(struct rpc_rqst *rqstp,
683} 685}
684 686
685/* 687/*
688 * struct write_response4 {
689 * stateid4 wr_callback_id<1>;
690 * length4 wr_count;
691 * stable_how4 wr_committed;
692 * verifier4 wr_writeverf;
693 * };
694 * union offload_info4 switch (nfsstat4 coa_status) {
695 * case NFS4_OK:
696 * write_response4 coa_resok4;
697 * default:
698 * length4 coa_bytes_copied;
699 * };
700 * struct CB_OFFLOAD4args {
701 * nfs_fh4 coa_fh;
702 * stateid4 coa_stateid;
703 * offload_info4 coa_offload_info;
704 * };
705 */
706static void encode_offload_info4(struct xdr_stream *xdr,
707 __be32 nfserr,
708 const struct nfsd4_copy *cp)
709{
710 __be32 *p;
711
712 p = xdr_reserve_space(xdr, 4);
713 *p++ = nfserr;
714 if (!nfserr) {
715 p = xdr_reserve_space(xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE);
716 p = xdr_encode_empty_array(p);
717 p = xdr_encode_hyper(p, cp->cp_res.wr_bytes_written);
718 *p++ = cpu_to_be32(cp->cp_res.wr_stable_how);
719 p = xdr_encode_opaque_fixed(p, cp->cp_res.wr_verifier.data,
720 NFS4_VERIFIER_SIZE);
721 } else {
722 p = xdr_reserve_space(xdr, 8);
723 /* We always return success if bytes were written */
724 p = xdr_encode_hyper(p, 0);
725 }
726}
727
728static void encode_cb_offload4args(struct xdr_stream *xdr,
729 __be32 nfserr,
730 const struct knfsd_fh *fh,
731 const struct nfsd4_copy *cp,
732 struct nfs4_cb_compound_hdr *hdr)
733{
734 __be32 *p;
735
736 p = xdr_reserve_space(xdr, 4);
737 *p++ = cpu_to_be32(OP_CB_OFFLOAD);
738 encode_nfs_fh4(xdr, fh);
739 encode_stateid4(xdr, &cp->cp_res.cb_stateid);
740 encode_offload_info4(xdr, nfserr, cp);
741
742 hdr->nops++;
743}
744
745static void nfs4_xdr_enc_cb_offload(struct rpc_rqst *req,
746 struct xdr_stream *xdr,
747 const void *data)
748{
749 const struct nfsd4_callback *cb = data;
750 const struct nfsd4_copy *cp =
751 container_of(cb, struct nfsd4_copy, cp_cb);
752 struct nfs4_cb_compound_hdr hdr = {
753 .ident = 0,
754 .minorversion = cb->cb_clp->cl_minorversion,
755 };
756
757 encode_cb_compound4args(xdr, &hdr);
758 encode_cb_sequence4args(xdr, cb, &hdr);
759 encode_cb_offload4args(xdr, cp->nfserr, &cp->fh, cp, &hdr);
760 encode_cb_nops(&hdr);
761}
762
763static int nfs4_xdr_dec_cb_offload(struct rpc_rqst *rqstp,
764 struct xdr_stream *xdr,
765 void *data)
766{
767 struct nfsd4_callback *cb = data;
768 struct nfs4_cb_compound_hdr hdr;
769 int status;
770
771 status = decode_cb_compound4res(xdr, &hdr);
772 if (unlikely(status))
773 return status;
774
775 if (cb) {
776 status = decode_cb_sequence4res(xdr, cb);
777 if (unlikely(status || cb->cb_seq_status))
778 return status;
779 }
780 return decode_cb_op_status(xdr, OP_CB_OFFLOAD, &cb->cb_status);
781}
782/*
686 * RPC procedure tables 783 * RPC procedure tables
687 */ 784 */
688#define PROC(proc, call, argtype, restype) \ 785#define PROC(proc, call, argtype, restype) \
@@ -703,6 +800,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = {
703 PROC(CB_LAYOUT, COMPOUND, cb_layout, cb_layout), 800 PROC(CB_LAYOUT, COMPOUND, cb_layout, cb_layout),
704#endif 801#endif
705 PROC(CB_NOTIFY_LOCK, COMPOUND, cb_notify_lock, cb_notify_lock), 802 PROC(CB_NOTIFY_LOCK, COMPOUND, cb_notify_lock, cb_notify_lock),
803 PROC(CB_OFFLOAD, COMPOUND, cb_offload, cb_offload),
706}; 804};
707 805
708static unsigned int nfs4_cb_counts[ARRAY_SIZE(nfs4_cb_procedures)]; 806static unsigned int nfs4_cb_counts[ARRAY_SIZE(nfs4_cb_procedures)];
diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c
index a5bb76593ce7..bf137fec33ff 100644
--- a/fs/nfsd/nfs4idmap.c
+++ b/fs/nfsd/nfs4idmap.c
@@ -65,6 +65,7 @@ struct ent {
65 u32 id; 65 u32 id;
66 char name[IDMAP_NAMESZ]; 66 char name[IDMAP_NAMESZ];
67 char authname[IDMAP_NAMESZ]; 67 char authname[IDMAP_NAMESZ];
68 struct rcu_head rcu_head;
68}; 69};
69 70
70/* Common entry handling */ 71/* Common entry handling */
@@ -89,7 +90,7 @@ static void
89ent_put(struct kref *ref) 90ent_put(struct kref *ref)
90{ 91{
91 struct ent *map = container_of(ref, struct ent, h.ref); 92 struct ent *map = container_of(ref, struct ent, h.ref);
92 kfree(map); 93 kfree_rcu(map, rcu_head);
93} 94}
94 95
95static struct cache_head * 96static struct cache_head *
@@ -264,8 +265,8 @@ out:
264static struct ent * 265static struct ent *
265idtoname_lookup(struct cache_detail *cd, struct ent *item) 266idtoname_lookup(struct cache_detail *cd, struct ent *item)
266{ 267{
267 struct cache_head *ch = sunrpc_cache_lookup(cd, &item->h, 268 struct cache_head *ch = sunrpc_cache_lookup_rcu(cd, &item->h,
268 idtoname_hash(item)); 269 idtoname_hash(item));
269 if (ch) 270 if (ch)
270 return container_of(ch, struct ent, h); 271 return container_of(ch, struct ent, h);
271 else 272 else
@@ -422,8 +423,8 @@ out:
422static struct ent * 423static struct ent *
423nametoid_lookup(struct cache_detail *cd, struct ent *item) 424nametoid_lookup(struct cache_detail *cd, struct ent *item)
424{ 425{
425 struct cache_head *ch = sunrpc_cache_lookup(cd, &item->h, 426 struct cache_head *ch = sunrpc_cache_lookup_rcu(cd, &item->h,
426 nametoid_hash(item)); 427 nametoid_hash(item));
427 if (ch) 428 if (ch)
428 return container_of(ch, struct ent, h); 429 return container_of(ch, struct ent, h);
429 else 430 else
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index b7bc6e1a85ac..edff074d38c7 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -36,6 +36,7 @@
36#include <linux/file.h> 36#include <linux/file.h>
37#include <linux/falloc.h> 37#include <linux/falloc.h>
38#include <linux/slab.h> 38#include <linux/slab.h>
39#include <linux/kthread.h>
39 40
40#include "idmap.h" 41#include "idmap.h"
41#include "cache.h" 42#include "cache.h"
@@ -1089,36 +1090,254 @@ out:
1089 return status; 1090 return status;
1090} 1091}
1091 1092
1093void nfs4_put_copy(struct nfsd4_copy *copy)
1094{
1095 if (!refcount_dec_and_test(&copy->refcount))
1096 return;
1097 kfree(copy);
1098}
1099
1100static bool
1101check_and_set_stop_copy(struct nfsd4_copy *copy)
1102{
1103 bool value;
1104
1105 spin_lock(&copy->cp_clp->async_lock);
1106 value = copy->stopped;
1107 if (!copy->stopped)
1108 copy->stopped = true;
1109 spin_unlock(&copy->cp_clp->async_lock);
1110 return value;
1111}
1112
1113static void nfsd4_stop_copy(struct nfsd4_copy *copy)
1114{
1115 /* only 1 thread should stop the copy */
1116 if (!check_and_set_stop_copy(copy))
1117 kthread_stop(copy->copy_task);
1118 nfs4_put_copy(copy);
1119}
1120
1121static struct nfsd4_copy *nfsd4_get_copy(struct nfs4_client *clp)
1122{
1123 struct nfsd4_copy *copy = NULL;
1124
1125 spin_lock(&clp->async_lock);
1126 if (!list_empty(&clp->async_copies)) {
1127 copy = list_first_entry(&clp->async_copies, struct nfsd4_copy,
1128 copies);
1129 refcount_inc(&copy->refcount);
1130 }
1131 spin_unlock(&clp->async_lock);
1132 return copy;
1133}
1134
1135void nfsd4_shutdown_copy(struct nfs4_client *clp)
1136{
1137 struct nfsd4_copy *copy;
1138
1139 while ((copy = nfsd4_get_copy(clp)) != NULL)
1140 nfsd4_stop_copy(copy);
1141}
1142
1143static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
1144{
1145 struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
1146
1147 nfs4_put_copy(copy);
1148}
1149
1150static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
1151 struct rpc_task *task)
1152{
1153 return 1;
1154}
1155
1156static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
1157 .release = nfsd4_cb_offload_release,
1158 .done = nfsd4_cb_offload_done
1159};
1160
1161static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
1162{
1163 copy->cp_res.wr_stable_how = NFS_UNSTABLE;
1164 copy->cp_synchronous = sync;
1165 gen_boot_verifier(&copy->cp_res.wr_verifier, copy->cp_clp->net);
1166}
1167
1168static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy)
1169{
1170 ssize_t bytes_copied = 0;
1171 size_t bytes_total = copy->cp_count;
1172 u64 src_pos = copy->cp_src_pos;
1173 u64 dst_pos = copy->cp_dst_pos;
1174
1175 do {
1176 if (kthread_should_stop())
1177 break;
1178 bytes_copied = nfsd_copy_file_range(copy->file_src, src_pos,
1179 copy->file_dst, dst_pos, bytes_total);
1180 if (bytes_copied <= 0)
1181 break;
1182 bytes_total -= bytes_copied;
1183 copy->cp_res.wr_bytes_written += bytes_copied;
1184 src_pos += bytes_copied;
1185 dst_pos += bytes_copied;
1186 } while (bytes_total > 0 && !copy->cp_synchronous);
1187 return bytes_copied;
1188}
1189
1190static __be32 nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
1191{
1192 __be32 status;
1193 ssize_t bytes;
1194
1195 bytes = _nfsd_copy_file_range(copy);
1196 /* for async copy, we ignore the error, client can always retry
1197 * to get the error
1198 */
1199 if (bytes < 0 && !copy->cp_res.wr_bytes_written)
1200 status = nfserrno(bytes);
1201 else {
1202 nfsd4_init_copy_res(copy, sync);
1203 status = nfs_ok;
1204 }
1205
1206 fput(copy->file_src);
1207 fput(copy->file_dst);
1208 return status;
1209}
1210
1211static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
1212{
1213 dst->cp_src_pos = src->cp_src_pos;
1214 dst->cp_dst_pos = src->cp_dst_pos;
1215 dst->cp_count = src->cp_count;
1216 dst->cp_synchronous = src->cp_synchronous;
1217 memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
1218 memcpy(&dst->fh, &src->fh, sizeof(src->fh));
1219 dst->cp_clp = src->cp_clp;
1220 dst->file_dst = get_file(src->file_dst);
1221 dst->file_src = get_file(src->file_src);
1222 memcpy(&dst->cp_stateid, &src->cp_stateid, sizeof(src->cp_stateid));
1223}
1224
1225static void cleanup_async_copy(struct nfsd4_copy *copy)
1226{
1227 nfs4_free_cp_state(copy);
1228 fput(copy->file_dst);
1229 fput(copy->file_src);
1230 spin_lock(&copy->cp_clp->async_lock);
1231 list_del(&copy->copies);
1232 spin_unlock(&copy->cp_clp->async_lock);
1233 nfs4_put_copy(copy);
1234}
1235
1236static int nfsd4_do_async_copy(void *data)
1237{
1238 struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
1239 struct nfsd4_copy *cb_copy;
1240
1241 copy->nfserr = nfsd4_do_copy(copy, 0);
1242 cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
1243 if (!cb_copy)
1244 goto out;
1245 memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
1246 cb_copy->cp_clp = copy->cp_clp;
1247 cb_copy->nfserr = copy->nfserr;
1248 memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
1249 nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
1250 &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
1251 nfsd4_run_cb(&cb_copy->cp_cb);
1252out:
1253 cleanup_async_copy(copy);
1254 return 0;
1255}
1256
1092static __be32 1257static __be32
1093nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, 1258nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
1094 union nfsd4_op_u *u) 1259 union nfsd4_op_u *u)
1095{ 1260{
1096 struct nfsd4_copy *copy = &u->copy; 1261 struct nfsd4_copy *copy = &u->copy;
1097 struct file *src, *dst;
1098 __be32 status; 1262 __be32 status;
1099 ssize_t bytes; 1263 struct nfsd4_copy *async_copy = NULL;
1100 1264
1101 status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src, 1265 status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
1102 &copy->cp_dst_stateid, &dst); 1266 &copy->file_src, &copy->cp_dst_stateid,
1267 &copy->file_dst);
1103 if (status) 1268 if (status)
1104 goto out; 1269 goto out;
1105 1270
1106 bytes = nfsd_copy_file_range(src, copy->cp_src_pos, 1271 copy->cp_clp = cstate->clp;
1107 dst, copy->cp_dst_pos, copy->cp_count); 1272 memcpy(&copy->fh, &cstate->current_fh.fh_handle,
1273 sizeof(struct knfsd_fh));
1274 if (!copy->cp_synchronous) {
1275 struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
1108 1276
1109 if (bytes < 0) 1277 status = nfserrno(-ENOMEM);
1110 status = nfserrno(bytes); 1278 async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
1111 else { 1279 if (!async_copy)
1112 copy->cp_res.wr_bytes_written = bytes; 1280 goto out;
1113 copy->cp_res.wr_stable_how = NFS_UNSTABLE; 1281 if (!nfs4_init_cp_state(nn, copy)) {
1114 copy->cp_synchronous = 1; 1282 kfree(async_copy);
1115 gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp)); 1283 goto out;
1284 }
1285 refcount_set(&async_copy->refcount, 1);
1286 memcpy(&copy->cp_res.cb_stateid, &copy->cp_stateid,
1287 sizeof(copy->cp_stateid));
1288 dup_copy_fields(copy, async_copy);
1289 async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
1290 async_copy, "%s", "copy thread");
1291 if (IS_ERR(async_copy->copy_task))
1292 goto out_err;
1293 spin_lock(&async_copy->cp_clp->async_lock);
1294 list_add(&async_copy->copies,
1295 &async_copy->cp_clp->async_copies);
1296 spin_unlock(&async_copy->cp_clp->async_lock);
1297 wake_up_process(async_copy->copy_task);
1116 status = nfs_ok; 1298 status = nfs_ok;
1299 } else
1300 status = nfsd4_do_copy(copy, 1);
1301out:
1302 return status;
1303out_err:
1304 cleanup_async_copy(async_copy);
1305 goto out;
1306}
1307
1308struct nfsd4_copy *
1309find_async_copy(struct nfs4_client *clp, stateid_t *stateid)
1310{
1311 struct nfsd4_copy *copy;
1312
1313 spin_lock(&clp->async_lock);
1314 list_for_each_entry(copy, &clp->async_copies, copies) {
1315 if (memcmp(&copy->cp_stateid, stateid, NFS4_STATEID_SIZE))
1316 continue;
1317 refcount_inc(&copy->refcount);
1318 spin_unlock(&clp->async_lock);
1319 return copy;
1117 } 1320 }
1321 spin_unlock(&clp->async_lock);
1322 return NULL;
1323}
1324
1325static __be32
1326nfsd4_offload_cancel(struct svc_rqst *rqstp,
1327 struct nfsd4_compound_state *cstate,
1328 union nfsd4_op_u *u)
1329{
1330 struct nfsd4_offload_status *os = &u->offload_status;
1331 __be32 status = 0;
1332 struct nfsd4_copy *copy;
1333 struct nfs4_client *clp = cstate->clp;
1334
1335 copy = find_async_copy(clp, &os->stateid);
1336 if (copy)
1337 nfsd4_stop_copy(copy);
1338 else
1339 status = nfserr_bad_stateid;
1118 1340
1119 fput(src);
1120 fput(dst);
1121out:
1122 return status; 1341 return status;
1123} 1342}
1124 1343
@@ -1144,6 +1363,25 @@ nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
1144 fput(file); 1363 fput(file);
1145 return status; 1364 return status;
1146} 1365}
1366static __be32
1367nfsd4_offload_status(struct svc_rqst *rqstp,
1368 struct nfsd4_compound_state *cstate,
1369 union nfsd4_op_u *u)
1370{
1371 struct nfsd4_offload_status *os = &u->offload_status;
1372 __be32 status = 0;
1373 struct nfsd4_copy *copy;
1374 struct nfs4_client *clp = cstate->clp;
1375
1376 copy = find_async_copy(clp, &os->stateid);
1377 if (copy) {
1378 os->count = copy->cp_res.wr_bytes_written;
1379 nfs4_put_copy(copy);
1380 } else
1381 status = nfserr_bad_stateid;
1382
1383 return status;
1384}
1147 1385
1148static __be32 1386static __be32
1149nfsd4_allocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, 1387nfsd4_allocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
@@ -2047,6 +2285,14 @@ static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
2047 1 /* cr_synchronous */) * sizeof(__be32); 2285 1 /* cr_synchronous */) * sizeof(__be32);
2048} 2286}
2049 2287
2288static inline u32 nfsd4_offload_status_rsize(struct svc_rqst *rqstp,
2289 struct nfsd4_op *op)
2290{
2291 return (op_encode_hdr_size +
2292 2 /* osr_count */ +
2293 1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32);
2294}
2295
2050#ifdef CONFIG_NFSD_PNFS 2296#ifdef CONFIG_NFSD_PNFS
2051static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) 2297static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
2052{ 2298{
@@ -2460,6 +2706,17 @@ static const struct nfsd4_operation nfsd4_ops[] = {
2460 .op_name = "OP_SEEK", 2706 .op_name = "OP_SEEK",
2461 .op_rsize_bop = nfsd4_seek_rsize, 2707 .op_rsize_bop = nfsd4_seek_rsize,
2462 }, 2708 },
2709 [OP_OFFLOAD_STATUS] = {
2710 .op_func = nfsd4_offload_status,
2711 .op_name = "OP_OFFLOAD_STATUS",
2712 .op_rsize_bop = nfsd4_offload_status_rsize,
2713 },
2714 [OP_OFFLOAD_CANCEL] = {
2715 .op_func = nfsd4_offload_cancel,
2716 .op_flags = OP_MODIFIES_SOMETHING,
2717 .op_name = "OP_OFFLOAD_CANCEL",
2718 .op_rsize_bop = nfsd4_only_status_rsize,
2719 },
2463}; 2720};
2464 2721
2465/** 2722/**
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index b0ca0efd2875..f093fbe47133 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -713,6 +713,36 @@ out_free:
713 return NULL; 713 return NULL;
714} 714}
715 715
716/*
717 * Create a unique stateid_t to represent each COPY.
718 */
719int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy)
720{
721 int new_id;
722
723 idr_preload(GFP_KERNEL);
724 spin_lock(&nn->s2s_cp_lock);
725 new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, copy, 0, 0, GFP_NOWAIT);
726 spin_unlock(&nn->s2s_cp_lock);
727 idr_preload_end();
728 if (new_id < 0)
729 return 0;
730 copy->cp_stateid.si_opaque.so_id = new_id;
731 copy->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
732 copy->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
733 return 1;
734}
735
736void nfs4_free_cp_state(struct nfsd4_copy *copy)
737{
738 struct nfsd_net *nn;
739
740 nn = net_generic(copy->cp_clp->net, nfsd_net_id);
741 spin_lock(&nn->s2s_cp_lock);
742 idr_remove(&nn->s2s_cp_stateids, copy->cp_stateid.si_opaque.so_id);
743 spin_unlock(&nn->s2s_cp_lock);
744}
745
716static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp) 746static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
717{ 747{
718 struct nfs4_stid *stid; 748 struct nfs4_stid *stid;
@@ -1827,6 +1857,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
1827#ifdef CONFIG_NFSD_PNFS 1857#ifdef CONFIG_NFSD_PNFS
1828 INIT_LIST_HEAD(&clp->cl_lo_states); 1858 INIT_LIST_HEAD(&clp->cl_lo_states);
1829#endif 1859#endif
1860 INIT_LIST_HEAD(&clp->async_copies);
1861 spin_lock_init(&clp->async_lock);
1830 spin_lock_init(&clp->cl_lock); 1862 spin_lock_init(&clp->cl_lock);
1831 rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table"); 1863 rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
1832 return clp; 1864 return clp;
@@ -1942,6 +1974,7 @@ __destroy_client(struct nfs4_client *clp)
1942 } 1974 }
1943 } 1975 }
1944 nfsd4_return_all_client_layouts(clp); 1976 nfsd4_return_all_client_layouts(clp);
1977 nfsd4_shutdown_copy(clp);
1945 nfsd4_shutdown_callback(clp); 1978 nfsd4_shutdown_callback(clp);
1946 if (clp->cl_cb_conn.cb_xprt) 1979 if (clp->cl_cb_conn.cb_xprt)
1947 svc_xprt_put(clp->cl_cb_conn.cb_xprt); 1980 svc_xprt_put(clp->cl_cb_conn.cb_xprt);
@@ -2475,7 +2508,8 @@ static bool client_has_state(struct nfs4_client *clp)
2475 || !list_empty(&clp->cl_lo_states) 2508 || !list_empty(&clp->cl_lo_states)
2476#endif 2509#endif
2477 || !list_empty(&clp->cl_delegations) 2510 || !list_empty(&clp->cl_delegations)
2478 || !list_empty(&clp->cl_sessions); 2511 || !list_empty(&clp->cl_sessions)
2512 || !list_empty(&clp->async_copies);
2479} 2513}
2480 2514
2481__be32 2515__be32
@@ -4364,7 +4398,7 @@ nfs4_set_delegation(struct nfs4_client *clp, struct svc_fh *fh,
4364 4398
4365 fl = nfs4_alloc_init_lease(dp, NFS4_OPEN_DELEGATE_READ); 4399 fl = nfs4_alloc_init_lease(dp, NFS4_OPEN_DELEGATE_READ);
4366 if (!fl) 4400 if (!fl)
4367 goto out_stid; 4401 goto out_clnt_odstate;
4368 4402
4369 status = vfs_setlease(fp->fi_deleg_file, fl->fl_type, &fl, NULL); 4403 status = vfs_setlease(fp->fi_deleg_file, fl->fl_type, &fl, NULL);
4370 if (fl) 4404 if (fl)
@@ -4389,7 +4423,6 @@ out_unlock:
4389 vfs_setlease(fp->fi_deleg_file, F_UNLCK, NULL, (void **)&dp); 4423 vfs_setlease(fp->fi_deleg_file, F_UNLCK, NULL, (void **)&dp);
4390out_clnt_odstate: 4424out_clnt_odstate:
4391 put_clnt_odstate(dp->dl_clnt_odstate); 4425 put_clnt_odstate(dp->dl_clnt_odstate);
4392out_stid:
4393 nfs4_put_stid(&dp->dl_stid); 4426 nfs4_put_stid(&dp->dl_stid);
4394out_delegees: 4427out_delegees:
4395 put_deleg_file(fp); 4428 put_deleg_file(fp);
@@ -7161,6 +7194,8 @@ static int nfs4_state_create_net(struct net *net)
7161 INIT_LIST_HEAD(&nn->close_lru); 7194 INIT_LIST_HEAD(&nn->close_lru);
7162 INIT_LIST_HEAD(&nn->del_recall_lru); 7195 INIT_LIST_HEAD(&nn->del_recall_lru);
7163 spin_lock_init(&nn->client_lock); 7196 spin_lock_init(&nn->client_lock);
7197 spin_lock_init(&nn->s2s_cp_lock);
7198 idr_init(&nn->s2s_cp_stateids);
7164 7199
7165 spin_lock_init(&nn->blocked_locks_lock); 7200 spin_lock_init(&nn->blocked_locks_lock);
7166 INIT_LIST_HEAD(&nn->blocked_locks_lru); 7201 INIT_LIST_HEAD(&nn->blocked_locks_lru);
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 418fa9c78186..3de42a729093 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -1768,6 +1768,13 @@ nfsd4_decode_copy(struct nfsd4_compoundargs *argp, struct nfsd4_copy *copy)
1768} 1768}
1769 1769
1770static __be32 1770static __be32
1771nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp,
1772 struct nfsd4_offload_status *os)
1773{
1774 return nfsd4_decode_stateid(argp, &os->stateid);
1775}
1776
1777static __be32
1771nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek) 1778nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek)
1772{ 1779{
1773 DECODE_HEAD; 1780 DECODE_HEAD;
@@ -1873,8 +1880,8 @@ static const nfsd4_dec nfsd4_dec_ops[] = {
1873 [OP_IO_ADVISE] = (nfsd4_dec)nfsd4_decode_notsupp, 1880 [OP_IO_ADVISE] = (nfsd4_dec)nfsd4_decode_notsupp,
1874 [OP_LAYOUTERROR] = (nfsd4_dec)nfsd4_decode_notsupp, 1881 [OP_LAYOUTERROR] = (nfsd4_dec)nfsd4_decode_notsupp,
1875 [OP_LAYOUTSTATS] = (nfsd4_dec)nfsd4_decode_notsupp, 1882 [OP_LAYOUTSTATS] = (nfsd4_dec)nfsd4_decode_notsupp,
1876 [OP_OFFLOAD_CANCEL] = (nfsd4_dec)nfsd4_decode_notsupp, 1883 [OP_OFFLOAD_CANCEL] = (nfsd4_dec)nfsd4_decode_offload_status,
1877 [OP_OFFLOAD_STATUS] = (nfsd4_dec)nfsd4_decode_notsupp, 1884 [OP_OFFLOAD_STATUS] = (nfsd4_dec)nfsd4_decode_offload_status,
1878 [OP_READ_PLUS] = (nfsd4_dec)nfsd4_decode_notsupp, 1885 [OP_READ_PLUS] = (nfsd4_dec)nfsd4_decode_notsupp,
1879 [OP_SEEK] = (nfsd4_dec)nfsd4_decode_seek, 1886 [OP_SEEK] = (nfsd4_dec)nfsd4_decode_seek,
1880 [OP_WRITE_SAME] = (nfsd4_dec)nfsd4_decode_notsupp, 1887 [OP_WRITE_SAME] = (nfsd4_dec)nfsd4_decode_notsupp,
@@ -4224,15 +4231,27 @@ nfsd4_encode_layoutreturn(struct nfsd4_compoundres *resp, __be32 nfserr,
4224#endif /* CONFIG_NFSD_PNFS */ 4231#endif /* CONFIG_NFSD_PNFS */
4225 4232
4226static __be32 4233static __be32
4227nfsd42_encode_write_res(struct nfsd4_compoundres *resp, struct nfsd42_write_res *write) 4234nfsd42_encode_write_res(struct nfsd4_compoundres *resp,
4235 struct nfsd42_write_res *write, bool sync)
4228{ 4236{
4229 __be32 *p; 4237 __be32 *p;
4238 p = xdr_reserve_space(&resp->xdr, 4);
4239 if (!p)
4240 return nfserr_resource;
4230 4241
4231 p = xdr_reserve_space(&resp->xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE); 4242 if (sync)
4243 *p++ = cpu_to_be32(0);
4244 else {
4245 __be32 nfserr;
4246 *p++ = cpu_to_be32(1);
4247 nfserr = nfsd4_encode_stateid(&resp->xdr, &write->cb_stateid);
4248 if (nfserr)
4249 return nfserr;
4250 }
4251 p = xdr_reserve_space(&resp->xdr, 8 + 4 + NFS4_VERIFIER_SIZE);
4232 if (!p) 4252 if (!p)
4233 return nfserr_resource; 4253 return nfserr_resource;
4234 4254
4235 *p++ = cpu_to_be32(0);
4236 p = xdr_encode_hyper(p, write->wr_bytes_written); 4255 p = xdr_encode_hyper(p, write->wr_bytes_written);
4237 *p++ = cpu_to_be32(write->wr_stable_how); 4256 *p++ = cpu_to_be32(write->wr_stable_how);
4238 p = xdr_encode_opaque_fixed(p, write->wr_verifier.data, 4257 p = xdr_encode_opaque_fixed(p, write->wr_verifier.data,
@@ -4246,7 +4265,8 @@ nfsd4_encode_copy(struct nfsd4_compoundres *resp, __be32 nfserr,
4246{ 4265{
4247 __be32 *p; 4266 __be32 *p;
4248 4267
4249 nfserr = nfsd42_encode_write_res(resp, &copy->cp_res); 4268 nfserr = nfsd42_encode_write_res(resp, &copy->cp_res,
4269 copy->cp_synchronous);
4250 if (nfserr) 4270 if (nfserr)
4251 return nfserr; 4271 return nfserr;
4252 4272
@@ -4257,6 +4277,22 @@ nfsd4_encode_copy(struct nfsd4_compoundres *resp, __be32 nfserr,
4257} 4277}
4258 4278
4259static __be32 4279static __be32
4280nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
4281 struct nfsd4_offload_status *os)
4282{
4283 struct xdr_stream *xdr = &resp->xdr;
4284 __be32 *p;
4285
4286 p = xdr_reserve_space(xdr, 8 + 4);
4287 if (!p)
4288 return nfserr_resource;
4289 p = xdr_encode_hyper(p, os->count);
4290 *p++ = cpu_to_be32(0);
4291
4292 return nfserr;
4293}
4294
4295static __be32
4260nfsd4_encode_seek(struct nfsd4_compoundres *resp, __be32 nfserr, 4296nfsd4_encode_seek(struct nfsd4_compoundres *resp, __be32 nfserr,
4261 struct nfsd4_seek *seek) 4297 struct nfsd4_seek *seek)
4262{ 4298{
@@ -4359,7 +4395,7 @@ static const nfsd4_enc nfsd4_enc_ops[] = {
4359 [OP_LAYOUTERROR] = (nfsd4_enc)nfsd4_encode_noop, 4395 [OP_LAYOUTERROR] = (nfsd4_enc)nfsd4_encode_noop,
4360 [OP_LAYOUTSTATS] = (nfsd4_enc)nfsd4_encode_noop, 4396 [OP_LAYOUTSTATS] = (nfsd4_enc)nfsd4_encode_noop,
4361 [OP_OFFLOAD_CANCEL] = (nfsd4_enc)nfsd4_encode_noop, 4397 [OP_OFFLOAD_CANCEL] = (nfsd4_enc)nfsd4_encode_noop,
4362 [OP_OFFLOAD_STATUS] = (nfsd4_enc)nfsd4_encode_noop, 4398 [OP_OFFLOAD_STATUS] = (nfsd4_enc)nfsd4_encode_offload_status,
4363 [OP_READ_PLUS] = (nfsd4_enc)nfsd4_encode_noop, 4399 [OP_READ_PLUS] = (nfsd4_enc)nfsd4_encode_noop,
4364 [OP_SEEK] = (nfsd4_enc)nfsd4_encode_seek, 4400 [OP_SEEK] = (nfsd4_enc)nfsd4_encode_seek,
4365 [OP_WRITE_SAME] = (nfsd4_enc)nfsd4_encode_noop, 4401 [OP_WRITE_SAME] = (nfsd4_enc)nfsd4_encode_noop,
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index dbdeb9d6af03..e2fe0e9ce0df 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -30,6 +30,7 @@
30#define TARGET_BUCKET_SIZE 64 30#define TARGET_BUCKET_SIZE 64
31 31
32struct nfsd_drc_bucket { 32struct nfsd_drc_bucket {
33 struct rb_root rb_head;
33 struct list_head lru_head; 34 struct list_head lru_head;
34 spinlock_t cache_lock; 35 spinlock_t cache_lock;
35}; 36};
@@ -121,7 +122,7 @@ nfsd_cache_hash(__be32 xid)
121} 122}
122 123
123static struct svc_cacherep * 124static struct svc_cacherep *
124nfsd_reply_cache_alloc(void) 125nfsd_reply_cache_alloc(struct svc_rqst *rqstp, __wsum csum)
125{ 126{
126 struct svc_cacherep *rp; 127 struct svc_cacherep *rp;
127 128
@@ -129,21 +130,35 @@ nfsd_reply_cache_alloc(void)
129 if (rp) { 130 if (rp) {
130 rp->c_state = RC_UNUSED; 131 rp->c_state = RC_UNUSED;
131 rp->c_type = RC_NOCACHE; 132 rp->c_type = RC_NOCACHE;
133 RB_CLEAR_NODE(&rp->c_node);
132 INIT_LIST_HEAD(&rp->c_lru); 134 INIT_LIST_HEAD(&rp->c_lru);
135
136 memset(&rp->c_key, 0, sizeof(rp->c_key));
137 rp->c_key.k_xid = rqstp->rq_xid;
138 rp->c_key.k_proc = rqstp->rq_proc;
139 rpc_copy_addr((struct sockaddr *)&rp->c_key.k_addr, svc_addr(rqstp));
140 rpc_set_port((struct sockaddr *)&rp->c_key.k_addr, rpc_get_port(svc_addr(rqstp)));
141 rp->c_key.k_prot = rqstp->rq_prot;
142 rp->c_key.k_vers = rqstp->rq_vers;
143 rp->c_key.k_len = rqstp->rq_arg.len;
144 rp->c_key.k_csum = csum;
133 } 145 }
134 return rp; 146 return rp;
135} 147}
136 148
137static void 149static void
138nfsd_reply_cache_free_locked(struct svc_cacherep *rp) 150nfsd_reply_cache_free_locked(struct nfsd_drc_bucket *b, struct svc_cacherep *rp)
139{ 151{
140 if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base) { 152 if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base) {
141 drc_mem_usage -= rp->c_replvec.iov_len; 153 drc_mem_usage -= rp->c_replvec.iov_len;
142 kfree(rp->c_replvec.iov_base); 154 kfree(rp->c_replvec.iov_base);
143 } 155 }
144 list_del(&rp->c_lru); 156 if (rp->c_state != RC_UNUSED) {
145 atomic_dec(&num_drc_entries); 157 rb_erase(&rp->c_node, &b->rb_head);
146 drc_mem_usage -= sizeof(*rp); 158 list_del(&rp->c_lru);
159 atomic_dec(&num_drc_entries);
160 drc_mem_usage -= sizeof(*rp);
161 }
147 kmem_cache_free(drc_slab, rp); 162 kmem_cache_free(drc_slab, rp);
148} 163}
149 164
@@ -151,7 +166,7 @@ static void
151nfsd_reply_cache_free(struct nfsd_drc_bucket *b, struct svc_cacherep *rp) 166nfsd_reply_cache_free(struct nfsd_drc_bucket *b, struct svc_cacherep *rp)
152{ 167{
153 spin_lock(&b->cache_lock); 168 spin_lock(&b->cache_lock);
154 nfsd_reply_cache_free_locked(rp); 169 nfsd_reply_cache_free_locked(b, rp);
155 spin_unlock(&b->cache_lock); 170 spin_unlock(&b->cache_lock);
156} 171}
157 172
@@ -207,7 +222,7 @@ void nfsd_reply_cache_shutdown(void)
207 struct list_head *head = &drc_hashtbl[i].lru_head; 222 struct list_head *head = &drc_hashtbl[i].lru_head;
208 while (!list_empty(head)) { 223 while (!list_empty(head)) {
209 rp = list_first_entry(head, struct svc_cacherep, c_lru); 224 rp = list_first_entry(head, struct svc_cacherep, c_lru);
210 nfsd_reply_cache_free_locked(rp); 225 nfsd_reply_cache_free_locked(&drc_hashtbl[i], rp);
211 } 226 }
212 } 227 }
213 228
@@ -246,7 +261,7 @@ prune_bucket(struct nfsd_drc_bucket *b)
246 if (atomic_read(&num_drc_entries) <= max_drc_entries && 261 if (atomic_read(&num_drc_entries) <= max_drc_entries &&
247 time_before(jiffies, rp->c_timestamp + RC_EXPIRE)) 262 time_before(jiffies, rp->c_timestamp + RC_EXPIRE))
248 break; 263 break;
249 nfsd_reply_cache_free_locked(rp); 264 nfsd_reply_cache_free_locked(b, rp);
250 freed++; 265 freed++;
251 } 266 }
252 return freed; 267 return freed;
@@ -318,51 +333,48 @@ nfsd_cache_csum(struct svc_rqst *rqstp)
318 return csum; 333 return csum;
319} 334}
320 335
321static bool 336static int
322nfsd_cache_match(struct svc_rqst *rqstp, __wsum csum, struct svc_cacherep *rp) 337nfsd_cache_key_cmp(const struct svc_cacherep *key, const struct svc_cacherep *rp)
323{ 338{
324 /* Check RPC XID first */ 339 if (key->c_key.k_xid == rp->c_key.k_xid &&
325 if (rqstp->rq_xid != rp->c_xid) 340 key->c_key.k_csum != rp->c_key.k_csum)
326 return false;
327 /* compare checksum of NFS data */
328 if (csum != rp->c_csum) {
329 ++payload_misses; 341 ++payload_misses;
330 return false;
331 }
332 342
333 /* Other discriminators */ 343 return memcmp(&key->c_key, &rp->c_key, sizeof(key->c_key));
334 if (rqstp->rq_proc != rp->c_proc ||
335 rqstp->rq_prot != rp->c_prot ||
336 rqstp->rq_vers != rp->c_vers ||
337 rqstp->rq_arg.len != rp->c_len ||
338 !rpc_cmp_addr(svc_addr(rqstp), (struct sockaddr *)&rp->c_addr) ||
339 rpc_get_port(svc_addr(rqstp)) != rpc_get_port((struct sockaddr *)&rp->c_addr))
340 return false;
341
342 return true;
343} 344}
344 345
345/* 346/*
346 * Search the request hash for an entry that matches the given rqstp. 347 * Search the request hash for an entry that matches the given rqstp.
347 * Must be called with cache_lock held. Returns the found entry or 348 * Must be called with cache_lock held. Returns the found entry or
348 * NULL on failure. 349 * inserts an empty key on failure.
349 */ 350 */
350static struct svc_cacherep * 351static struct svc_cacherep *
351nfsd_cache_search(struct nfsd_drc_bucket *b, struct svc_rqst *rqstp, 352nfsd_cache_insert(struct nfsd_drc_bucket *b, struct svc_cacherep *key)
352 __wsum csum)
353{ 353{
354 struct svc_cacherep *rp, *ret = NULL; 354 struct svc_cacherep *rp, *ret = key;
355 struct list_head *rh = &b->lru_head; 355 struct rb_node **p = &b->rb_head.rb_node,
356 *parent = NULL;
356 unsigned int entries = 0; 357 unsigned int entries = 0;
358 int cmp;
357 359
358 list_for_each_entry(rp, rh, c_lru) { 360 while (*p != NULL) {
359 ++entries; 361 ++entries;
360 if (nfsd_cache_match(rqstp, csum, rp)) { 362 parent = *p;
363 rp = rb_entry(parent, struct svc_cacherep, c_node);
364
365 cmp = nfsd_cache_key_cmp(key, rp);
366 if (cmp < 0)
367 p = &parent->rb_left;
368 else if (cmp > 0)
369 p = &parent->rb_right;
370 else {
361 ret = rp; 371 ret = rp;
362 break; 372 goto out;
363 } 373 }
364 } 374 }
365 375 rb_link_node(&key->c_node, parent, p);
376 rb_insert_color(&key->c_node, &b->rb_head);
377out:
366 /* tally hash chain length stats */ 378 /* tally hash chain length stats */
367 if (entries > longest_chain) { 379 if (entries > longest_chain) {
368 longest_chain = entries; 380 longest_chain = entries;
@@ -374,6 +386,7 @@ nfsd_cache_search(struct nfsd_drc_bucket *b, struct svc_rqst *rqstp,
374 atomic_read(&num_drc_entries)); 386 atomic_read(&num_drc_entries));
375 } 387 }
376 388
389 lru_put_end(b, ret);
377 return ret; 390 return ret;
378} 391}
379 392
@@ -389,9 +402,6 @@ nfsd_cache_lookup(struct svc_rqst *rqstp)
389{ 402{
390 struct svc_cacherep *rp, *found; 403 struct svc_cacherep *rp, *found;
391 __be32 xid = rqstp->rq_xid; 404 __be32 xid = rqstp->rq_xid;
392 u32 proto = rqstp->rq_prot,
393 vers = rqstp->rq_vers,
394 proc = rqstp->rq_proc;
395 __wsum csum; 405 __wsum csum;
396 u32 hash = nfsd_cache_hash(xid); 406 u32 hash = nfsd_cache_hash(xid);
397 struct nfsd_drc_bucket *b = &drc_hashtbl[hash]; 407 struct nfsd_drc_bucket *b = &drc_hashtbl[hash];
@@ -410,60 +420,38 @@ nfsd_cache_lookup(struct svc_rqst *rqstp)
410 * Since the common case is a cache miss followed by an insert, 420 * Since the common case is a cache miss followed by an insert,
411 * preallocate an entry. 421 * preallocate an entry.
412 */ 422 */
413 rp = nfsd_reply_cache_alloc(); 423 rp = nfsd_reply_cache_alloc(rqstp, csum);
414 spin_lock(&b->cache_lock); 424 if (!rp) {
415 if (likely(rp)) { 425 dprintk("nfsd: unable to allocate DRC entry!\n");
416 atomic_inc(&num_drc_entries); 426 return rtn;
417 drc_mem_usage += sizeof(*rp);
418 } 427 }
419 428
420 /* go ahead and prune the cache */ 429 spin_lock(&b->cache_lock);
421 prune_bucket(b); 430 found = nfsd_cache_insert(b, rp);
422 431 if (found != rp) {
423 found = nfsd_cache_search(b, rqstp, csum); 432 nfsd_reply_cache_free_locked(NULL, rp);
424 if (found) {
425 if (likely(rp))
426 nfsd_reply_cache_free_locked(rp);
427 rp = found; 433 rp = found;
428 goto found_entry; 434 goto found_entry;
429 } 435 }
430 436
431 if (!rp) {
432 dprintk("nfsd: unable to allocate DRC entry!\n");
433 goto out;
434 }
435
436 nfsdstats.rcmisses++; 437 nfsdstats.rcmisses++;
437 rqstp->rq_cacherep = rp; 438 rqstp->rq_cacherep = rp;
438 rp->c_state = RC_INPROG; 439 rp->c_state = RC_INPROG;
439 rp->c_xid = xid;
440 rp->c_proc = proc;
441 rpc_copy_addr((struct sockaddr *)&rp->c_addr, svc_addr(rqstp));
442 rpc_set_port((struct sockaddr *)&rp->c_addr, rpc_get_port(svc_addr(rqstp)));
443 rp->c_prot = proto;
444 rp->c_vers = vers;
445 rp->c_len = rqstp->rq_arg.len;
446 rp->c_csum = csum;
447 440
448 lru_put_end(b, rp); 441 atomic_inc(&num_drc_entries);
442 drc_mem_usage += sizeof(*rp);
449 443
450 /* release any buffer */ 444 /* go ahead and prune the cache */
451 if (rp->c_type == RC_REPLBUFF) { 445 prune_bucket(b);
452 drc_mem_usage -= rp->c_replvec.iov_len;
453 kfree(rp->c_replvec.iov_base);
454 rp->c_replvec.iov_base = NULL;
455 }
456 rp->c_type = RC_NOCACHE;
457 out: 446 out:
458 spin_unlock(&b->cache_lock); 447 spin_unlock(&b->cache_lock);
459 return rtn; 448 return rtn;
460 449
461found_entry: 450found_entry:
462 nfsdstats.rchits++;
463 /* We found a matching entry which is either in progress or done. */ 451 /* We found a matching entry which is either in progress or done. */
464 lru_put_end(b, rp); 452 nfsdstats.rchits++;
465
466 rtn = RC_DROPIT; 453 rtn = RC_DROPIT;
454
467 /* Request being processed */ 455 /* Request being processed */
468 if (rp->c_state == RC_INPROG) 456 if (rp->c_state == RC_INPROG)
469 goto out; 457 goto out;
@@ -489,7 +477,7 @@ found_entry:
489 break; 477 break;
490 default: 478 default:
491 printk(KERN_WARNING "nfsd: bad repcache type %d\n", rp->c_type); 479 printk(KERN_WARNING "nfsd: bad repcache type %d\n", rp->c_type);
492 nfsd_reply_cache_free_locked(rp); 480 nfsd_reply_cache_free_locked(b, rp);
493 } 481 }
494 482
495 goto out; 483 goto out;
@@ -524,7 +512,7 @@ nfsd_cache_update(struct svc_rqst *rqstp, int cachetype, __be32 *statp)
524 if (!rp) 512 if (!rp)
525 return; 513 return;
526 514
527 hash = nfsd_cache_hash(rp->c_xid); 515 hash = nfsd_cache_hash(rp->c_key.k_xid);
528 b = &drc_hashtbl[hash]; 516 b = &drc_hashtbl[hash];
529 517
530 len = resv->iov_len - ((char*)statp - (char*)resv->iov_base); 518 len = resv->iov_len - ((char*)statp - (char*)resv->iov_base);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 7fb9f7c667b1..6384c9b94898 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -1242,6 +1242,7 @@ static __net_init int nfsd_init_net(struct net *net)
1242 nn->somebody_reclaimed = false; 1242 nn->somebody_reclaimed = false;
1243 nn->clverifier_counter = prandom_u32(); 1243 nn->clverifier_counter = prandom_u32();
1244 nn->clientid_counter = prandom_u32(); 1244 nn->clientid_counter = prandom_u32();
1245 nn->s2s_cp_cl_id = nn->clientid_counter++;
1245 1246
1246 atomic_set(&nn->ntf_refcnt, 0); 1247 atomic_set(&nn->ntf_refcnt, 0);
1247 init_waitqueue_head(&nn->ntf_wq); 1248 init_waitqueue_head(&nn->ntf_wq);
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 0b15dac7e609..6aacb325b6a0 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -355,6 +355,8 @@ struct nfs4_client {
355 struct rpc_wait_queue cl_cb_waitq; /* backchannel callers may */ 355 struct rpc_wait_queue cl_cb_waitq; /* backchannel callers may */
356 /* wait here for slots */ 356 /* wait here for slots */
357 struct net *net; 357 struct net *net;
358 struct list_head async_copies; /* list of async copies */
359 spinlock_t async_lock; /* lock for async copies */
358}; 360};
359 361
360/* struct nfs4_client_reset 362/* struct nfs4_client_reset
@@ -573,6 +575,7 @@ enum nfsd4_cb_op {
573 NFSPROC4_CLNT_CB_NULL = 0, 575 NFSPROC4_CLNT_CB_NULL = 0,
574 NFSPROC4_CLNT_CB_RECALL, 576 NFSPROC4_CLNT_CB_RECALL,
575 NFSPROC4_CLNT_CB_LAYOUT, 577 NFSPROC4_CLNT_CB_LAYOUT,
578 NFSPROC4_CLNT_CB_OFFLOAD,
576 NFSPROC4_CLNT_CB_SEQUENCE, 579 NFSPROC4_CLNT_CB_SEQUENCE,
577 NFSPROC4_CLNT_CB_NOTIFY_LOCK, 580 NFSPROC4_CLNT_CB_NOTIFY_LOCK,
578}; 581};
@@ -599,6 +602,7 @@ struct nfsd4_blocked_lock {
599 602
600struct nfsd4_compound_state; 603struct nfsd4_compound_state;
601struct nfsd_net; 604struct nfsd_net;
605struct nfsd4_copy;
602 606
603extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp, 607extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
604 struct nfsd4_compound_state *cstate, struct svc_fh *fhp, 608 struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
@@ -608,6 +612,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
608 struct nfs4_stid **s, struct nfsd_net *nn); 612 struct nfs4_stid **s, struct nfsd_net *nn);
609struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab, 613struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
610 void (*sc_free)(struct nfs4_stid *)); 614 void (*sc_free)(struct nfs4_stid *));
615int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
616void nfs4_free_cp_state(struct nfsd4_copy *copy);
611void nfs4_unhash_stid(struct nfs4_stid *s); 617void nfs4_unhash_stid(struct nfs4_stid *s);
612void nfs4_put_stid(struct nfs4_stid *s); 618void nfs4_put_stid(struct nfs4_stid *s);
613void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid); 619void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
@@ -626,6 +632,7 @@ extern void nfsd4_run_cb(struct nfsd4_callback *cb);
626extern int nfsd4_create_callback_queue(void); 632extern int nfsd4_create_callback_queue(void);
627extern void nfsd4_destroy_callback_queue(void); 633extern void nfsd4_destroy_callback_queue(void);
628extern void nfsd4_shutdown_callback(struct nfs4_client *); 634extern void nfsd4_shutdown_callback(struct nfs4_client *);
635extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
629extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp); 636extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
630extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name, 637extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
631 struct nfsd_net *nn); 638 struct nfsd_net *nn);
@@ -633,6 +640,9 @@ extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
633 640
634struct nfs4_file *find_file(struct knfsd_fh *fh); 641struct nfs4_file *find_file(struct knfsd_fh *fh);
635void put_nfs4_file(struct nfs4_file *fi); 642void put_nfs4_file(struct nfs4_file *fi);
643extern void nfs4_put_copy(struct nfsd4_copy *copy);
644extern struct nfsd4_copy *
645find_async_copy(struct nfs4_client *clp, stateid_t *staetid);
636static inline void get_nfs4_file(struct nfs4_file *fi) 646static inline void get_nfs4_file(struct nfs4_file *fi)
637{ 647{
638 refcount_inc(&fi->fi_ref); 648 refcount_inc(&fi->fi_ref);
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index b53e76391e52..2751976704e9 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -1276,7 +1276,6 @@ nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
1276 int type, dev_t rdev, struct svc_fh *resfhp) 1276 int type, dev_t rdev, struct svc_fh *resfhp)
1277{ 1277{
1278 struct dentry *dentry, *dchild = NULL; 1278 struct dentry *dentry, *dchild = NULL;
1279 struct inode *dirp;
1280 __be32 err; 1279 __be32 err;
1281 int host_err; 1280 int host_err;
1282 1281
@@ -1288,7 +1287,6 @@ nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
1288 return err; 1287 return err;
1289 1288
1290 dentry = fhp->fh_dentry; 1289 dentry = fhp->fh_dentry;
1291 dirp = d_inode(dentry);
1292 1290
1293 host_err = fh_want_write(fhp); 1291 host_err = fh_want_write(fhp);
1294 if (host_err) 1292 if (host_err)
@@ -1409,6 +1407,7 @@ do_nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
1409 *created = 1; 1407 *created = 1;
1410 break; 1408 break;
1411 } 1409 }
1410 /* fall through */
1412 case NFS4_CREATE_EXCLUSIVE4_1: 1411 case NFS4_CREATE_EXCLUSIVE4_1:
1413 if ( d_inode(dchild)->i_mtime.tv_sec == v_mtime 1412 if ( d_inode(dchild)->i_mtime.tv_sec == v_mtime
1414 && d_inode(dchild)->i_atime.tv_sec == v_atime 1413 && d_inode(dchild)->i_atime.tv_sec == v_atime
@@ -1417,7 +1416,7 @@ do_nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
1417 *created = 1; 1416 *created = 1;
1418 goto set_attr; 1417 goto set_attr;
1419 } 1418 }
1420 /* fallthru */ 1419 /* fall through */
1421 case NFS3_CREATE_GUARDED: 1420 case NFS3_CREATE_GUARDED:
1422 err = nfserr_exist; 1421 err = nfserr_exist;
1423 } 1422 }
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 17c453a7999c..feeb6d4bdffd 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -511,6 +511,7 @@ struct nfsd42_write_res {
511 u64 wr_bytes_written; 511 u64 wr_bytes_written;
512 u32 wr_stable_how; 512 u32 wr_stable_how;
513 nfs4_verifier wr_verifier; 513 nfs4_verifier wr_verifier;
514 stateid_t cb_stateid;
514}; 515};
515 516
516struct nfsd4_copy { 517struct nfsd4_copy {
@@ -526,6 +527,23 @@ struct nfsd4_copy {
526 527
527 /* response */ 528 /* response */
528 struct nfsd42_write_res cp_res; 529 struct nfsd42_write_res cp_res;
530
531 /* for cb_offload */
532 struct nfsd4_callback cp_cb;
533 __be32 nfserr;
534 struct knfsd_fh fh;
535
536 struct nfs4_client *cp_clp;
537
538 struct file *file_src;
539 struct file *file_dst;
540
541 stateid_t cp_stateid;
542
543 struct list_head copies;
544 struct task_struct *copy_task;
545 refcount_t refcount;
546 bool stopped;
529}; 547};
530 548
531struct nfsd4_seek { 549struct nfsd4_seek {
@@ -539,6 +557,15 @@ struct nfsd4_seek {
539 loff_t seek_pos; 557 loff_t seek_pos;
540}; 558};
541 559
560struct nfsd4_offload_status {
561 /* request */
562 stateid_t stateid;
563
564 /* response */
565 u64 count;
566 u32 status;
567};
568
542struct nfsd4_op { 569struct nfsd4_op {
543 int opnum; 570 int opnum;
544 const struct nfsd4_operation * opdesc; 571 const struct nfsd4_operation * opdesc;
@@ -597,6 +624,7 @@ struct nfsd4_op {
597 struct nfsd4_fallocate deallocate; 624 struct nfsd4_fallocate deallocate;
598 struct nfsd4_clone clone; 625 struct nfsd4_clone clone;
599 struct nfsd4_copy copy; 626 struct nfsd4_copy copy;
627 struct nfsd4_offload_status offload_status;
600 struct nfsd4_seek seek; 628 struct nfsd4_seek seek;
601 } u; 629 } u;
602 struct nfs4_replay * replay; 630 struct nfs4_replay * replay;
diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h
index 517239af0302..547cf07cf4e0 100644
--- a/fs/nfsd/xdr4cb.h
+++ b/fs/nfsd/xdr4cb.h
@@ -38,3 +38,13 @@
38#define NFS4_dec_cb_notify_lock_sz (cb_compound_dec_hdr_sz + \ 38#define NFS4_dec_cb_notify_lock_sz (cb_compound_dec_hdr_sz + \
39 cb_sequence_dec_sz + \ 39 cb_sequence_dec_sz + \
40 op_dec_sz) 40 op_dec_sz)
41#define enc_cb_offload_info_sz (1 + 1 + 2 + 1 + \
42 XDR_QUADLEN(NFS4_VERIFIER_SIZE))
43#define NFS4_enc_cb_offload_sz (cb_compound_enc_hdr_sz + \
44 cb_sequence_enc_sz + \
45 enc_nfs4_fh_sz + \
46 enc_stateid_sz + \
47 enc_cb_offload_info_sz)
48#define NFS4_dec_cb_offload_sz (cb_compound_dec_hdr_sz + \
49 cb_sequence_dec_sz + \
50 op_dec_sz)
diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h
index 40d2822f0e2f..5a3e95017fc6 100644
--- a/include/linux/sunrpc/cache.h
+++ b/include/linux/sunrpc/cache.h
@@ -67,7 +67,7 @@ struct cache_detail {
67 struct module * owner; 67 struct module * owner;
68 int hash_size; 68 int hash_size;
69 struct hlist_head * hash_table; 69 struct hlist_head * hash_table;
70 rwlock_t hash_lock; 70 spinlock_t hash_lock;
71 71
72 char *name; 72 char *name;
73 void (*cache_put)(struct kref *); 73 void (*cache_put)(struct kref *);
@@ -168,8 +168,8 @@ extern const struct file_operations content_file_operations_pipefs;
168extern const struct file_operations cache_flush_operations_pipefs; 168extern const struct file_operations cache_flush_operations_pipefs;
169 169
170extern struct cache_head * 170extern struct cache_head *
171sunrpc_cache_lookup(struct cache_detail *detail, 171sunrpc_cache_lookup_rcu(struct cache_detail *detail,
172 struct cache_head *key, int hash); 172 struct cache_head *key, int hash);
173extern struct cache_head * 173extern struct cache_head *
174sunrpc_cache_update(struct cache_detail *detail, 174sunrpc_cache_update(struct cache_detail *detail,
175 struct cache_head *new, struct cache_head *old, int hash); 175 struct cache_head *new, struct cache_head *old, int hash);
@@ -186,6 +186,12 @@ static inline struct cache_head *cache_get(struct cache_head *h)
186 return h; 186 return h;
187} 187}
188 188
189static inline struct cache_head *cache_get_rcu(struct cache_head *h)
190{
191 if (kref_get_unless_zero(&h->ref))
192 return h;
193 return NULL;
194}
189 195
190static inline void cache_put(struct cache_head *h, struct cache_detail *cd) 196static inline void cache_put(struct cache_head *h, struct cache_detail *cd)
191{ 197{
@@ -224,9 +230,9 @@ extern void sunrpc_cache_unregister_pipefs(struct cache_detail *);
224extern void sunrpc_cache_unhash(struct cache_detail *, struct cache_head *); 230extern void sunrpc_cache_unhash(struct cache_detail *, struct cache_head *);
225 231
226/* Must store cache_detail in seq_file->private if using next three functions */ 232/* Must store cache_detail in seq_file->private if using next three functions */
227extern void *cache_seq_start(struct seq_file *file, loff_t *pos); 233extern void *cache_seq_start_rcu(struct seq_file *file, loff_t *pos);
228extern void *cache_seq_next(struct seq_file *file, void *p, loff_t *pos); 234extern void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos);
229extern void cache_seq_stop(struct seq_file *file, void *p); 235extern void cache_seq_stop_rcu(struct seq_file *file, void *p);
230 236
231extern void qword_add(char **bpp, int *lp, char *str); 237extern void qword_add(char **bpp, int *lp, char *str);
232extern void qword_addhex(char **bpp, int *lp, char *buf, int blen); 238extern void qword_addhex(char **bpp, int *lp, char *buf, int blen);
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index fd78f78df5c6..e6e26918504c 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -113,13 +113,14 @@ struct svcxprt_rdma {
113/* sc_flags */ 113/* sc_flags */
114#define RDMAXPRT_CONN_PENDING 3 114#define RDMAXPRT_CONN_PENDING 3
115 115
116#define RPCRDMA_LISTEN_BACKLOG 10 116/*
117#define RPCRDMA_MAX_REQUESTS 32 117 * Default connection parameters
118
119/* Typical ULP usage of BC requests is NFSv4.1 backchannel. Our
120 * current NFSv4.1 implementation supports one backchannel slot.
121 */ 118 */
122#define RPCRDMA_MAX_BC_REQUESTS 2 119enum {
120 RPCRDMA_LISTEN_BACKLOG = 10,
121 RPCRDMA_MAX_REQUESTS = 64,
122 RPCRDMA_MAX_BC_REQUESTS = 2,
123};
123 124
124#define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD 125#define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
125 126
diff --git a/include/linux/sunrpc/svcauth.h b/include/linux/sunrpc/svcauth.h
index 04e404a07882..3e53a6e2ada7 100644
--- a/include/linux/sunrpc/svcauth.h
+++ b/include/linux/sunrpc/svcauth.h
@@ -82,6 +82,7 @@ struct auth_domain {
82 struct hlist_node hash; 82 struct hlist_node hash;
83 char *name; 83 char *name;
84 struct auth_ops *flavour; 84 struct auth_ops *flavour;
85 struct rcu_head rcu_head;
85}; 86};
86 87
87/* 88/*
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index 860f2a1bbb67..1ece4bc3eb8d 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -76,6 +76,7 @@ struct rsi {
76 struct xdr_netobj in_handle, in_token; 76 struct xdr_netobj in_handle, in_token;
77 struct xdr_netobj out_handle, out_token; 77 struct xdr_netobj out_handle, out_token;
78 int major_status, minor_status; 78 int major_status, minor_status;
79 struct rcu_head rcu_head;
79}; 80};
80 81
81static struct rsi *rsi_update(struct cache_detail *cd, struct rsi *new, struct rsi *old); 82static struct rsi *rsi_update(struct cache_detail *cd, struct rsi *new, struct rsi *old);
@@ -89,13 +90,21 @@ static void rsi_free(struct rsi *rsii)
89 kfree(rsii->out_token.data); 90 kfree(rsii->out_token.data);
90} 91}
91 92
92static void rsi_put(struct kref *ref) 93static void rsi_free_rcu(struct rcu_head *head)
93{ 94{
94 struct rsi *rsii = container_of(ref, struct rsi, h.ref); 95 struct rsi *rsii = container_of(head, struct rsi, rcu_head);
96
95 rsi_free(rsii); 97 rsi_free(rsii);
96 kfree(rsii); 98 kfree(rsii);
97} 99}
98 100
101static void rsi_put(struct kref *ref)
102{
103 struct rsi *rsii = container_of(ref, struct rsi, h.ref);
104
105 call_rcu(&rsii->rcu_head, rsi_free_rcu);
106}
107
99static inline int rsi_hash(struct rsi *item) 108static inline int rsi_hash(struct rsi *item)
100{ 109{
101 return hash_mem(item->in_handle.data, item->in_handle.len, RSI_HASHBITS) 110 return hash_mem(item->in_handle.data, item->in_handle.len, RSI_HASHBITS)
@@ -282,7 +291,7 @@ static struct rsi *rsi_lookup(struct cache_detail *cd, struct rsi *item)
282 struct cache_head *ch; 291 struct cache_head *ch;
283 int hash = rsi_hash(item); 292 int hash = rsi_hash(item);
284 293
285 ch = sunrpc_cache_lookup(cd, &item->h, hash); 294 ch = sunrpc_cache_lookup_rcu(cd, &item->h, hash);
286 if (ch) 295 if (ch)
287 return container_of(ch, struct rsi, h); 296 return container_of(ch, struct rsi, h);
288 else 297 else
@@ -330,6 +339,7 @@ struct rsc {
330 struct svc_cred cred; 339 struct svc_cred cred;
331 struct gss_svc_seq_data seqdata; 340 struct gss_svc_seq_data seqdata;
332 struct gss_ctx *mechctx; 341 struct gss_ctx *mechctx;
342 struct rcu_head rcu_head;
333}; 343};
334 344
335static struct rsc *rsc_update(struct cache_detail *cd, struct rsc *new, struct rsc *old); 345static struct rsc *rsc_update(struct cache_detail *cd, struct rsc *new, struct rsc *old);
@@ -343,12 +353,22 @@ static void rsc_free(struct rsc *rsci)
343 free_svc_cred(&rsci->cred); 353 free_svc_cred(&rsci->cred);
344} 354}
345 355
356static void rsc_free_rcu(struct rcu_head *head)
357{
358 struct rsc *rsci = container_of(head, struct rsc, rcu_head);
359
360 kfree(rsci->handle.data);
361 kfree(rsci);
362}
363
346static void rsc_put(struct kref *ref) 364static void rsc_put(struct kref *ref)
347{ 365{
348 struct rsc *rsci = container_of(ref, struct rsc, h.ref); 366 struct rsc *rsci = container_of(ref, struct rsc, h.ref);
349 367
350 rsc_free(rsci); 368 if (rsci->mechctx)
351 kfree(rsci); 369 gss_delete_sec_context(&rsci->mechctx);
370 free_svc_cred(&rsci->cred);
371 call_rcu(&rsci->rcu_head, rsc_free_rcu);
352} 372}
353 373
354static inline int 374static inline int
@@ -542,7 +562,7 @@ static struct rsc *rsc_lookup(struct cache_detail *cd, struct rsc *item)
542 struct cache_head *ch; 562 struct cache_head *ch;
543 int hash = rsc_hash(item); 563 int hash = rsc_hash(item);
544 564
545 ch = sunrpc_cache_lookup(cd, &item->h, hash); 565 ch = sunrpc_cache_lookup_rcu(cd, &item->h, hash);
546 if (ch) 566 if (ch)
547 return container_of(ch, struct rsc, h); 567 return container_of(ch, struct rsc, h);
548 else 568 else
@@ -1764,14 +1784,21 @@ out_err:
1764} 1784}
1765 1785
1766static void 1786static void
1767svcauth_gss_domain_release(struct auth_domain *dom) 1787svcauth_gss_domain_release_rcu(struct rcu_head *head)
1768{ 1788{
1789 struct auth_domain *dom = container_of(head, struct auth_domain, rcu_head);
1769 struct gss_domain *gd = container_of(dom, struct gss_domain, h); 1790 struct gss_domain *gd = container_of(dom, struct gss_domain, h);
1770 1791
1771 kfree(dom->name); 1792 kfree(dom->name);
1772 kfree(gd); 1793 kfree(gd);
1773} 1794}
1774 1795
1796static void
1797svcauth_gss_domain_release(struct auth_domain *dom)
1798{
1799 call_rcu(&dom->rcu_head, svcauth_gss_domain_release_rcu);
1800}
1801
1775static struct auth_ops svcauthops_gss = { 1802static struct auth_ops svcauthops_gss = {
1776 .name = "rpcsec_gss", 1803 .name = "rpcsec_gss",
1777 .owner = THIS_MODULE, 1804 .owner = THIS_MODULE,
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 109fbe591e7b..f96345b1180e 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -54,28 +54,33 @@ static void cache_init(struct cache_head *h, struct cache_detail *detail)
54 h->last_refresh = now; 54 h->last_refresh = now;
55} 55}
56 56
57struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail, 57static struct cache_head *sunrpc_cache_find_rcu(struct cache_detail *detail,
58 struct cache_head *key, int hash) 58 struct cache_head *key,
59 int hash)
59{ 60{
60 struct cache_head *new = NULL, *freeme = NULL, *tmp = NULL; 61 struct hlist_head *head = &detail->hash_table[hash];
61 struct hlist_head *head; 62 struct cache_head *tmp;
62
63 head = &detail->hash_table[hash];
64
65 read_lock(&detail->hash_lock);
66 63
67 hlist_for_each_entry(tmp, head, cache_list) { 64 rcu_read_lock();
65 hlist_for_each_entry_rcu(tmp, head, cache_list) {
68 if (detail->match(tmp, key)) { 66 if (detail->match(tmp, key)) {
69 if (cache_is_expired(detail, tmp)) 67 if (cache_is_expired(detail, tmp))
70 /* This entry is expired, we will discard it. */ 68 continue;
71 break; 69 tmp = cache_get_rcu(tmp);
72 cache_get(tmp); 70 rcu_read_unlock();
73 read_unlock(&detail->hash_lock);
74 return tmp; 71 return tmp;
75 } 72 }
76 } 73 }
77 read_unlock(&detail->hash_lock); 74 rcu_read_unlock();
78 /* Didn't find anything, insert an empty entry */ 75 return NULL;
76}
77
78static struct cache_head *sunrpc_cache_add_entry(struct cache_detail *detail,
79 struct cache_head *key,
80 int hash)
81{
82 struct cache_head *new, *tmp, *freeme = NULL;
83 struct hlist_head *head = &detail->hash_table[hash];
79 84
80 new = detail->alloc(); 85 new = detail->alloc();
81 if (!new) 86 if (!new)
@@ -87,35 +92,46 @@ struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail,
87 cache_init(new, detail); 92 cache_init(new, detail);
88 detail->init(new, key); 93 detail->init(new, key);
89 94
90 write_lock(&detail->hash_lock); 95 spin_lock(&detail->hash_lock);
91 96
92 /* check if entry appeared while we slept */ 97 /* check if entry appeared while we slept */
93 hlist_for_each_entry(tmp, head, cache_list) { 98 hlist_for_each_entry_rcu(tmp, head, cache_list) {
94 if (detail->match(tmp, key)) { 99 if (detail->match(tmp, key)) {
95 if (cache_is_expired(detail, tmp)) { 100 if (cache_is_expired(detail, tmp)) {
96 hlist_del_init(&tmp->cache_list); 101 hlist_del_init_rcu(&tmp->cache_list);
97 detail->entries --; 102 detail->entries --;
98 freeme = tmp; 103 freeme = tmp;
99 break; 104 break;
100 } 105 }
101 cache_get(tmp); 106 cache_get(tmp);
102 write_unlock(&detail->hash_lock); 107 spin_unlock(&detail->hash_lock);
103 cache_put(new, detail); 108 cache_put(new, detail);
104 return tmp; 109 return tmp;
105 } 110 }
106 } 111 }
107 112
108 hlist_add_head(&new->cache_list, head); 113 hlist_add_head_rcu(&new->cache_list, head);
109 detail->entries++; 114 detail->entries++;
110 cache_get(new); 115 cache_get(new);
111 write_unlock(&detail->hash_lock); 116 spin_unlock(&detail->hash_lock);
112 117
113 if (freeme) 118 if (freeme)
114 cache_put(freeme, detail); 119 cache_put(freeme, detail);
115 return new; 120 return new;
116} 121}
117EXPORT_SYMBOL_GPL(sunrpc_cache_lookup);
118 122
123struct cache_head *sunrpc_cache_lookup_rcu(struct cache_detail *detail,
124 struct cache_head *key, int hash)
125{
126 struct cache_head *ret;
127
128 ret = sunrpc_cache_find_rcu(detail, key, hash);
129 if (ret)
130 return ret;
131 /* Didn't find anything, insert an empty entry */
132 return sunrpc_cache_add_entry(detail, key, hash);
133}
134EXPORT_SYMBOL_GPL(sunrpc_cache_lookup_rcu);
119 135
120static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch); 136static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch);
121 137
@@ -151,18 +167,18 @@ struct cache_head *sunrpc_cache_update(struct cache_detail *detail,
151 struct cache_head *tmp; 167 struct cache_head *tmp;
152 168
153 if (!test_bit(CACHE_VALID, &old->flags)) { 169 if (!test_bit(CACHE_VALID, &old->flags)) {
154 write_lock(&detail->hash_lock); 170 spin_lock(&detail->hash_lock);
155 if (!test_bit(CACHE_VALID, &old->flags)) { 171 if (!test_bit(CACHE_VALID, &old->flags)) {
156 if (test_bit(CACHE_NEGATIVE, &new->flags)) 172 if (test_bit(CACHE_NEGATIVE, &new->flags))
157 set_bit(CACHE_NEGATIVE, &old->flags); 173 set_bit(CACHE_NEGATIVE, &old->flags);
158 else 174 else
159 detail->update(old, new); 175 detail->update(old, new);
160 cache_fresh_locked(old, new->expiry_time, detail); 176 cache_fresh_locked(old, new->expiry_time, detail);
161 write_unlock(&detail->hash_lock); 177 spin_unlock(&detail->hash_lock);
162 cache_fresh_unlocked(old, detail); 178 cache_fresh_unlocked(old, detail);
163 return old; 179 return old;
164 } 180 }
165 write_unlock(&detail->hash_lock); 181 spin_unlock(&detail->hash_lock);
166 } 182 }
167 /* We need to insert a new entry */ 183 /* We need to insert a new entry */
168 tmp = detail->alloc(); 184 tmp = detail->alloc();
@@ -173,7 +189,7 @@ struct cache_head *sunrpc_cache_update(struct cache_detail *detail,
173 cache_init(tmp, detail); 189 cache_init(tmp, detail);
174 detail->init(tmp, old); 190 detail->init(tmp, old);
175 191
176 write_lock(&detail->hash_lock); 192 spin_lock(&detail->hash_lock);
177 if (test_bit(CACHE_NEGATIVE, &new->flags)) 193 if (test_bit(CACHE_NEGATIVE, &new->flags))
178 set_bit(CACHE_NEGATIVE, &tmp->flags); 194 set_bit(CACHE_NEGATIVE, &tmp->flags);
179 else 195 else
@@ -183,7 +199,7 @@ struct cache_head *sunrpc_cache_update(struct cache_detail *detail,
183 cache_get(tmp); 199 cache_get(tmp);
184 cache_fresh_locked(tmp, new->expiry_time, detail); 200 cache_fresh_locked(tmp, new->expiry_time, detail);
185 cache_fresh_locked(old, 0, detail); 201 cache_fresh_locked(old, 0, detail);
186 write_unlock(&detail->hash_lock); 202 spin_unlock(&detail->hash_lock);
187 cache_fresh_unlocked(tmp, detail); 203 cache_fresh_unlocked(tmp, detail);
188 cache_fresh_unlocked(old, detail); 204 cache_fresh_unlocked(old, detail);
189 cache_put(old, detail); 205 cache_put(old, detail);
@@ -223,7 +239,7 @@ static int try_to_negate_entry(struct cache_detail *detail, struct cache_head *h
223{ 239{
224 int rv; 240 int rv;
225 241
226 write_lock(&detail->hash_lock); 242 spin_lock(&detail->hash_lock);
227 rv = cache_is_valid(h); 243 rv = cache_is_valid(h);
228 if (rv == -EAGAIN) { 244 if (rv == -EAGAIN) {
229 set_bit(CACHE_NEGATIVE, &h->flags); 245 set_bit(CACHE_NEGATIVE, &h->flags);
@@ -231,7 +247,7 @@ static int try_to_negate_entry(struct cache_detail *detail, struct cache_head *h
231 detail); 247 detail);
232 rv = -ENOENT; 248 rv = -ENOENT;
233 } 249 }
234 write_unlock(&detail->hash_lock); 250 spin_unlock(&detail->hash_lock);
235 cache_fresh_unlocked(h, detail); 251 cache_fresh_unlocked(h, detail);
236 return rv; 252 return rv;
237} 253}
@@ -341,7 +357,7 @@ static struct delayed_work cache_cleaner;
341 357
342void sunrpc_init_cache_detail(struct cache_detail *cd) 358void sunrpc_init_cache_detail(struct cache_detail *cd)
343{ 359{
344 rwlock_init(&cd->hash_lock); 360 spin_lock_init(&cd->hash_lock);
345 INIT_LIST_HEAD(&cd->queue); 361 INIT_LIST_HEAD(&cd->queue);
346 spin_lock(&cache_list_lock); 362 spin_lock(&cache_list_lock);
347 cd->nextcheck = 0; 363 cd->nextcheck = 0;
@@ -361,11 +377,11 @@ void sunrpc_destroy_cache_detail(struct cache_detail *cd)
361{ 377{
362 cache_purge(cd); 378 cache_purge(cd);
363 spin_lock(&cache_list_lock); 379 spin_lock(&cache_list_lock);
364 write_lock(&cd->hash_lock); 380 spin_lock(&cd->hash_lock);
365 if (current_detail == cd) 381 if (current_detail == cd)
366 current_detail = NULL; 382 current_detail = NULL;
367 list_del_init(&cd->others); 383 list_del_init(&cd->others);
368 write_unlock(&cd->hash_lock); 384 spin_unlock(&cd->hash_lock);
369 spin_unlock(&cache_list_lock); 385 spin_unlock(&cache_list_lock);
370 if (list_empty(&cache_list)) { 386 if (list_empty(&cache_list)) {
371 /* module must be being unloaded so its safe to kill the worker */ 387 /* module must be being unloaded so its safe to kill the worker */
@@ -422,7 +438,7 @@ static int cache_clean(void)
422 struct hlist_head *head; 438 struct hlist_head *head;
423 struct hlist_node *tmp; 439 struct hlist_node *tmp;
424 440
425 write_lock(&current_detail->hash_lock); 441 spin_lock(&current_detail->hash_lock);
426 442
427 /* Ok, now to clean this strand */ 443 /* Ok, now to clean this strand */
428 444
@@ -433,13 +449,13 @@ static int cache_clean(void)
433 if (!cache_is_expired(current_detail, ch)) 449 if (!cache_is_expired(current_detail, ch))
434 continue; 450 continue;
435 451
436 hlist_del_init(&ch->cache_list); 452 hlist_del_init_rcu(&ch->cache_list);
437 current_detail->entries--; 453 current_detail->entries--;
438 rv = 1; 454 rv = 1;
439 break; 455 break;
440 } 456 }
441 457
442 write_unlock(&current_detail->hash_lock); 458 spin_unlock(&current_detail->hash_lock);
443 d = current_detail; 459 d = current_detail;
444 if (!ch) 460 if (!ch)
445 current_index ++; 461 current_index ++;
@@ -494,9 +510,9 @@ void cache_purge(struct cache_detail *detail)
494 struct hlist_node *tmp = NULL; 510 struct hlist_node *tmp = NULL;
495 int i = 0; 511 int i = 0;
496 512
497 write_lock(&detail->hash_lock); 513 spin_lock(&detail->hash_lock);
498 if (!detail->entries) { 514 if (!detail->entries) {
499 write_unlock(&detail->hash_lock); 515 spin_unlock(&detail->hash_lock);
500 return; 516 return;
501 } 517 }
502 518
@@ -504,17 +520,17 @@ void cache_purge(struct cache_detail *detail)
504 for (i = 0; i < detail->hash_size; i++) { 520 for (i = 0; i < detail->hash_size; i++) {
505 head = &detail->hash_table[i]; 521 head = &detail->hash_table[i];
506 hlist_for_each_entry_safe(ch, tmp, head, cache_list) { 522 hlist_for_each_entry_safe(ch, tmp, head, cache_list) {
507 hlist_del_init(&ch->cache_list); 523 hlist_del_init_rcu(&ch->cache_list);
508 detail->entries--; 524 detail->entries--;
509 525
510 set_bit(CACHE_CLEANED, &ch->flags); 526 set_bit(CACHE_CLEANED, &ch->flags);
511 write_unlock(&detail->hash_lock); 527 spin_unlock(&detail->hash_lock);
512 cache_fresh_unlocked(ch, detail); 528 cache_fresh_unlocked(ch, detail);
513 cache_put(ch, detail); 529 cache_put(ch, detail);
514 write_lock(&detail->hash_lock); 530 spin_lock(&detail->hash_lock);
515 } 531 }
516 } 532 }
517 write_unlock(&detail->hash_lock); 533 spin_unlock(&detail->hash_lock);
518} 534}
519EXPORT_SYMBOL_GPL(cache_purge); 535EXPORT_SYMBOL_GPL(cache_purge);
520 536
@@ -1289,21 +1305,19 @@ EXPORT_SYMBOL_GPL(qword_get);
1289 * get a header, then pass each real item in the cache 1305 * get a header, then pass each real item in the cache
1290 */ 1306 */
1291 1307
1292void *cache_seq_start(struct seq_file *m, loff_t *pos) 1308static void *__cache_seq_start(struct seq_file *m, loff_t *pos)
1293 __acquires(cd->hash_lock)
1294{ 1309{
1295 loff_t n = *pos; 1310 loff_t n = *pos;
1296 unsigned int hash, entry; 1311 unsigned int hash, entry;
1297 struct cache_head *ch; 1312 struct cache_head *ch;
1298 struct cache_detail *cd = m->private; 1313 struct cache_detail *cd = m->private;
1299 1314
1300 read_lock(&cd->hash_lock);
1301 if (!n--) 1315 if (!n--)
1302 return SEQ_START_TOKEN; 1316 return SEQ_START_TOKEN;
1303 hash = n >> 32; 1317 hash = n >> 32;
1304 entry = n & ((1LL<<32) - 1); 1318 entry = n & ((1LL<<32) - 1);
1305 1319
1306 hlist_for_each_entry(ch, &cd->hash_table[hash], cache_list) 1320 hlist_for_each_entry_rcu(ch, &cd->hash_table[hash], cache_list)
1307 if (!entry--) 1321 if (!entry--)
1308 return ch; 1322 return ch;
1309 n &= ~((1LL<<32) - 1); 1323 n &= ~((1LL<<32) - 1);
@@ -1315,12 +1329,12 @@ void *cache_seq_start(struct seq_file *m, loff_t *pos)
1315 if (hash >= cd->hash_size) 1329 if (hash >= cd->hash_size)
1316 return NULL; 1330 return NULL;
1317 *pos = n+1; 1331 *pos = n+1;
1318 return hlist_entry_safe(cd->hash_table[hash].first, 1332 return hlist_entry_safe(rcu_dereference_raw(
1333 hlist_first_rcu(&cd->hash_table[hash])),
1319 struct cache_head, cache_list); 1334 struct cache_head, cache_list);
1320} 1335}
1321EXPORT_SYMBOL_GPL(cache_seq_start);
1322 1336
1323void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos) 1337static void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
1324{ 1338{
1325 struct cache_head *ch = p; 1339 struct cache_head *ch = p;
1326 int hash = (*pos >> 32); 1340 int hash = (*pos >> 32);
@@ -1333,7 +1347,8 @@ void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
1333 *pos += 1LL<<32; 1347 *pos += 1LL<<32;
1334 } else { 1348 } else {
1335 ++*pos; 1349 ++*pos;
1336 return hlist_entry_safe(ch->cache_list.next, 1350 return hlist_entry_safe(rcu_dereference_raw(
1351 hlist_next_rcu(&ch->cache_list)),
1337 struct cache_head, cache_list); 1352 struct cache_head, cache_list);
1338 } 1353 }
1339 *pos &= ~((1LL<<32) - 1); 1354 *pos &= ~((1LL<<32) - 1);
@@ -1345,18 +1360,32 @@ void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
1345 if (hash >= cd->hash_size) 1360 if (hash >= cd->hash_size)
1346 return NULL; 1361 return NULL;
1347 ++*pos; 1362 ++*pos;
1348 return hlist_entry_safe(cd->hash_table[hash].first, 1363 return hlist_entry_safe(rcu_dereference_raw(
1364 hlist_first_rcu(&cd->hash_table[hash])),
1349 struct cache_head, cache_list); 1365 struct cache_head, cache_list);
1350} 1366}
1351EXPORT_SYMBOL_GPL(cache_seq_next); 1367EXPORT_SYMBOL_GPL(cache_seq_next);
1352 1368
1353void cache_seq_stop(struct seq_file *m, void *p) 1369void *cache_seq_start_rcu(struct seq_file *m, loff_t *pos)
1354 __releases(cd->hash_lock) 1370 __acquires(RCU)
1355{ 1371{
1356 struct cache_detail *cd = m->private; 1372 rcu_read_lock();
1357 read_unlock(&cd->hash_lock); 1373 return __cache_seq_start(m, pos);
1374}
1375EXPORT_SYMBOL_GPL(cache_seq_start_rcu);
1376
1377void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos)
1378{
1379 return cache_seq_next(file, p, pos);
1380}
1381EXPORT_SYMBOL_GPL(cache_seq_next_rcu);
1382
1383void cache_seq_stop_rcu(struct seq_file *m, void *p)
1384 __releases(RCU)
1385{
1386 rcu_read_unlock();
1358} 1387}
1359EXPORT_SYMBOL_GPL(cache_seq_stop); 1388EXPORT_SYMBOL_GPL(cache_seq_stop_rcu);
1360 1389
1361static int c_show(struct seq_file *m, void *p) 1390static int c_show(struct seq_file *m, void *p)
1362{ 1391{
@@ -1384,9 +1413,9 @@ static int c_show(struct seq_file *m, void *p)
1384} 1413}
1385 1414
1386static const struct seq_operations cache_content_op = { 1415static const struct seq_operations cache_content_op = {
1387 .start = cache_seq_start, 1416 .start = cache_seq_start_rcu,
1388 .next = cache_seq_next, 1417 .next = cache_seq_next_rcu,
1389 .stop = cache_seq_stop, 1418 .stop = cache_seq_stop_rcu,
1390 .show = c_show, 1419 .show = c_show,
1391}; 1420};
1392 1421
@@ -1844,13 +1873,13 @@ EXPORT_SYMBOL_GPL(sunrpc_cache_unregister_pipefs);
1844 1873
1845void sunrpc_cache_unhash(struct cache_detail *cd, struct cache_head *h) 1874void sunrpc_cache_unhash(struct cache_detail *cd, struct cache_head *h)
1846{ 1875{
1847 write_lock(&cd->hash_lock); 1876 spin_lock(&cd->hash_lock);
1848 if (!hlist_unhashed(&h->cache_list)){ 1877 if (!hlist_unhashed(&h->cache_list)){
1849 hlist_del_init(&h->cache_list); 1878 hlist_del_init_rcu(&h->cache_list);
1850 cd->entries--; 1879 cd->entries--;
1851 write_unlock(&cd->hash_lock); 1880 spin_unlock(&cd->hash_lock);
1852 cache_put(h, cd); 1881 cache_put(h, cd);
1853 } else 1882 } else
1854 write_unlock(&cd->hash_lock); 1883 spin_unlock(&cd->hash_lock);
1855} 1884}
1856EXPORT_SYMBOL_GPL(sunrpc_cache_unhash); 1885EXPORT_SYMBOL_GPL(sunrpc_cache_unhash);
diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index 87533fbb96cf..51d36230b6e3 100644
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -987,7 +987,7 @@ static void call_xpt_users(struct svc_xprt *xprt)
987 spin_lock(&xprt->xpt_lock); 987 spin_lock(&xprt->xpt_lock);
988 while (!list_empty(&xprt->xpt_users)) { 988 while (!list_empty(&xprt->xpt_users)) {
989 u = list_first_entry(&xprt->xpt_users, struct svc_xpt_user, list); 989 u = list_first_entry(&xprt->xpt_users, struct svc_xpt_user, list);
990 list_del(&u->list); 990 list_del_init(&u->list);
991 u->callback(u); 991 u->callback(u);
992 } 992 }
993 spin_unlock(&xprt->xpt_lock); 993 spin_unlock(&xprt->xpt_lock);
diff --git a/net/sunrpc/svcauth.c b/net/sunrpc/svcauth.c
index bb8db3cb8032..775b8c94265b 100644
--- a/net/sunrpc/svcauth.c
+++ b/net/sunrpc/svcauth.c
@@ -27,12 +27,32 @@
27extern struct auth_ops svcauth_null; 27extern struct auth_ops svcauth_null;
28extern struct auth_ops svcauth_unix; 28extern struct auth_ops svcauth_unix;
29 29
30static DEFINE_SPINLOCK(authtab_lock); 30static struct auth_ops __rcu *authtab[RPC_AUTH_MAXFLAVOR] = {
31static struct auth_ops *authtab[RPC_AUTH_MAXFLAVOR] = { 31 [RPC_AUTH_NULL] = (struct auth_ops __force __rcu *)&svcauth_null,
32 [0] = &svcauth_null, 32 [RPC_AUTH_UNIX] = (struct auth_ops __force __rcu *)&svcauth_unix,
33 [1] = &svcauth_unix,
34}; 33};
35 34
35static struct auth_ops *
36svc_get_auth_ops(rpc_authflavor_t flavor)
37{
38 struct auth_ops *aops;
39
40 if (flavor >= RPC_AUTH_MAXFLAVOR)
41 return NULL;
42 rcu_read_lock();
43 aops = rcu_dereference(authtab[flavor]);
44 if (aops != NULL && !try_module_get(aops->owner))
45 aops = NULL;
46 rcu_read_unlock();
47 return aops;
48}
49
50static void
51svc_put_auth_ops(struct auth_ops *aops)
52{
53 module_put(aops->owner);
54}
55
36int 56int
37svc_authenticate(struct svc_rqst *rqstp, __be32 *authp) 57svc_authenticate(struct svc_rqst *rqstp, __be32 *authp)
38{ 58{
@@ -45,14 +65,11 @@ svc_authenticate(struct svc_rqst *rqstp, __be32 *authp)
45 65
46 dprintk("svc: svc_authenticate (%d)\n", flavor); 66 dprintk("svc: svc_authenticate (%d)\n", flavor);
47 67
48 spin_lock(&authtab_lock); 68 aops = svc_get_auth_ops(flavor);
49 if (flavor >= RPC_AUTH_MAXFLAVOR || !(aops = authtab[flavor]) || 69 if (aops == NULL) {
50 !try_module_get(aops->owner)) {
51 spin_unlock(&authtab_lock);
52 *authp = rpc_autherr_badcred; 70 *authp = rpc_autherr_badcred;
53 return SVC_DENIED; 71 return SVC_DENIED;
54 } 72 }
55 spin_unlock(&authtab_lock);
56 73
57 rqstp->rq_auth_slack = 0; 74 rqstp->rq_auth_slack = 0;
58 init_svc_cred(&rqstp->rq_cred); 75 init_svc_cred(&rqstp->rq_cred);
@@ -82,7 +99,7 @@ int svc_authorise(struct svc_rqst *rqstp)
82 99
83 if (aops) { 100 if (aops) {
84 rv = aops->release(rqstp); 101 rv = aops->release(rqstp);
85 module_put(aops->owner); 102 svc_put_auth_ops(aops);
86 } 103 }
87 return rv; 104 return rv;
88} 105}
@@ -90,13 +107,14 @@ int svc_authorise(struct svc_rqst *rqstp)
90int 107int
91svc_auth_register(rpc_authflavor_t flavor, struct auth_ops *aops) 108svc_auth_register(rpc_authflavor_t flavor, struct auth_ops *aops)
92{ 109{
110 struct auth_ops *old;
93 int rv = -EINVAL; 111 int rv = -EINVAL;
94 spin_lock(&authtab_lock); 112
95 if (flavor < RPC_AUTH_MAXFLAVOR && authtab[flavor] == NULL) { 113 if (flavor < RPC_AUTH_MAXFLAVOR) {
96 authtab[flavor] = aops; 114 old = cmpxchg((struct auth_ops ** __force)&authtab[flavor], NULL, aops);
97 rv = 0; 115 if (old == NULL || old == aops)
116 rv = 0;
98 } 117 }
99 spin_unlock(&authtab_lock);
100 return rv; 118 return rv;
101} 119}
102EXPORT_SYMBOL_GPL(svc_auth_register); 120EXPORT_SYMBOL_GPL(svc_auth_register);
@@ -104,10 +122,8 @@ EXPORT_SYMBOL_GPL(svc_auth_register);
104void 122void
105svc_auth_unregister(rpc_authflavor_t flavor) 123svc_auth_unregister(rpc_authflavor_t flavor)
106{ 124{
107 spin_lock(&authtab_lock);
108 if (flavor < RPC_AUTH_MAXFLAVOR) 125 if (flavor < RPC_AUTH_MAXFLAVOR)
109 authtab[flavor] = NULL; 126 rcu_assign_pointer(authtab[flavor], NULL);
110 spin_unlock(&authtab_lock);
111} 127}
112EXPORT_SYMBOL_GPL(svc_auth_unregister); 128EXPORT_SYMBOL_GPL(svc_auth_unregister);
113 129
@@ -127,10 +143,11 @@ static struct hlist_head auth_domain_table[DN_HASHMAX];
127static DEFINE_SPINLOCK(auth_domain_lock); 143static DEFINE_SPINLOCK(auth_domain_lock);
128 144
129static void auth_domain_release(struct kref *kref) 145static void auth_domain_release(struct kref *kref)
146 __releases(&auth_domain_lock)
130{ 147{
131 struct auth_domain *dom = container_of(kref, struct auth_domain, ref); 148 struct auth_domain *dom = container_of(kref, struct auth_domain, ref);
132 149
133 hlist_del(&dom->hash); 150 hlist_del_rcu(&dom->hash);
134 dom->flavour->domain_release(dom); 151 dom->flavour->domain_release(dom);
135 spin_unlock(&auth_domain_lock); 152 spin_unlock(&auth_domain_lock);
136} 153}
@@ -159,7 +176,7 @@ auth_domain_lookup(char *name, struct auth_domain *new)
159 } 176 }
160 } 177 }
161 if (new) 178 if (new)
162 hlist_add_head(&new->hash, head); 179 hlist_add_head_rcu(&new->hash, head);
163 spin_unlock(&auth_domain_lock); 180 spin_unlock(&auth_domain_lock);
164 return new; 181 return new;
165} 182}
@@ -167,6 +184,21 @@ EXPORT_SYMBOL_GPL(auth_domain_lookup);
167 184
168struct auth_domain *auth_domain_find(char *name) 185struct auth_domain *auth_domain_find(char *name)
169{ 186{
170 return auth_domain_lookup(name, NULL); 187 struct auth_domain *hp;
188 struct hlist_head *head;
189
190 head = &auth_domain_table[hash_str(name, DN_HASHBITS)];
191
192 rcu_read_lock();
193 hlist_for_each_entry_rcu(hp, head, hash) {
194 if (strcmp(hp->name, name)==0) {
195 if (!kref_get_unless_zero(&hp->ref))
196 hp = NULL;
197 rcu_read_unlock();
198 return hp;
199 }
200 }
201 rcu_read_unlock();
202 return NULL;
171} 203}
172EXPORT_SYMBOL_GPL(auth_domain_find); 204EXPORT_SYMBOL_GPL(auth_domain_find);
diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c
index af7f28fb8102..fb9041b92f72 100644
--- a/net/sunrpc/svcauth_unix.c
+++ b/net/sunrpc/svcauth_unix.c
@@ -37,20 +37,26 @@ struct unix_domain {
37extern struct auth_ops svcauth_null; 37extern struct auth_ops svcauth_null;
38extern struct auth_ops svcauth_unix; 38extern struct auth_ops svcauth_unix;
39 39
40static void svcauth_unix_domain_release(struct auth_domain *dom) 40static void svcauth_unix_domain_release_rcu(struct rcu_head *head)
41{ 41{
42 struct auth_domain *dom = container_of(head, struct auth_domain, rcu_head);
42 struct unix_domain *ud = container_of(dom, struct unix_domain, h); 43 struct unix_domain *ud = container_of(dom, struct unix_domain, h);
43 44
44 kfree(dom->name); 45 kfree(dom->name);
45 kfree(ud); 46 kfree(ud);
46} 47}
47 48
49static void svcauth_unix_domain_release(struct auth_domain *dom)
50{
51 call_rcu(&dom->rcu_head, svcauth_unix_domain_release_rcu);
52}
53
48struct auth_domain *unix_domain_find(char *name) 54struct auth_domain *unix_domain_find(char *name)
49{ 55{
50 struct auth_domain *rv; 56 struct auth_domain *rv;
51 struct unix_domain *new = NULL; 57 struct unix_domain *new = NULL;
52 58
53 rv = auth_domain_lookup(name, NULL); 59 rv = auth_domain_find(name);
54 while(1) { 60 while(1) {
55 if (rv) { 61 if (rv) {
56 if (new && rv != &new->h) 62 if (new && rv != &new->h)
@@ -91,6 +97,7 @@ struct ip_map {
91 char m_class[8]; /* e.g. "nfsd" */ 97 char m_class[8]; /* e.g. "nfsd" */
92 struct in6_addr m_addr; 98 struct in6_addr m_addr;
93 struct unix_domain *m_client; 99 struct unix_domain *m_client;
100 struct rcu_head m_rcu;
94}; 101};
95 102
96static void ip_map_put(struct kref *kref) 103static void ip_map_put(struct kref *kref)
@@ -101,7 +108,7 @@ static void ip_map_put(struct kref *kref)
101 if (test_bit(CACHE_VALID, &item->flags) && 108 if (test_bit(CACHE_VALID, &item->flags) &&
102 !test_bit(CACHE_NEGATIVE, &item->flags)) 109 !test_bit(CACHE_NEGATIVE, &item->flags))
103 auth_domain_put(&im->m_client->h); 110 auth_domain_put(&im->m_client->h);
104 kfree(im); 111 kfree_rcu(im, m_rcu);
105} 112}
106 113
107static inline int hash_ip6(const struct in6_addr *ip) 114static inline int hash_ip6(const struct in6_addr *ip)
@@ -280,9 +287,9 @@ static struct ip_map *__ip_map_lookup(struct cache_detail *cd, char *class,
280 287
281 strcpy(ip.m_class, class); 288 strcpy(ip.m_class, class);
282 ip.m_addr = *addr; 289 ip.m_addr = *addr;
283 ch = sunrpc_cache_lookup(cd, &ip.h, 290 ch = sunrpc_cache_lookup_rcu(cd, &ip.h,
284 hash_str(class, IP_HASHBITS) ^ 291 hash_str(class, IP_HASHBITS) ^
285 hash_ip6(addr)); 292 hash_ip6(addr));
286 293
287 if (ch) 294 if (ch)
288 return container_of(ch, struct ip_map, h); 295 return container_of(ch, struct ip_map, h);
@@ -412,6 +419,7 @@ struct unix_gid {
412 struct cache_head h; 419 struct cache_head h;
413 kuid_t uid; 420 kuid_t uid;
414 struct group_info *gi; 421 struct group_info *gi;
422 struct rcu_head rcu;
415}; 423};
416 424
417static int unix_gid_hash(kuid_t uid) 425static int unix_gid_hash(kuid_t uid)
@@ -426,7 +434,7 @@ static void unix_gid_put(struct kref *kref)
426 if (test_bit(CACHE_VALID, &item->flags) && 434 if (test_bit(CACHE_VALID, &item->flags) &&
427 !test_bit(CACHE_NEGATIVE, &item->flags)) 435 !test_bit(CACHE_NEGATIVE, &item->flags))
428 put_group_info(ug->gi); 436 put_group_info(ug->gi);
429 kfree(ug); 437 kfree_rcu(ug, rcu);
430} 438}
431 439
432static int unix_gid_match(struct cache_head *corig, struct cache_head *cnew) 440static int unix_gid_match(struct cache_head *corig, struct cache_head *cnew)
@@ -619,7 +627,7 @@ static struct unix_gid *unix_gid_lookup(struct cache_detail *cd, kuid_t uid)
619 struct cache_head *ch; 627 struct cache_head *ch;
620 628
621 ug.uid = uid; 629 ug.uid = uid;
622 ch = sunrpc_cache_lookup(cd, &ug.h, unix_gid_hash(uid)); 630 ch = sunrpc_cache_lookup_rcu(cd, &ug.h, unix_gid_hash(uid));
623 if (ch) 631 if (ch)
624 return container_of(ch, struct unix_gid, h); 632 return container_of(ch, struct unix_gid, h);
625 else 633 else
diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index db8bb6b3a2b0..3b525accaa68 100644
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -325,59 +325,34 @@ static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining)
325/* 325/*
326 * Generic recvfrom routine. 326 * Generic recvfrom routine.
327 */ 327 */
328static int svc_recvfrom(struct svc_rqst *rqstp, struct kvec *iov, int nr, 328static ssize_t svc_recvfrom(struct svc_rqst *rqstp, struct kvec *iov,
329 int buflen) 329 unsigned int nr, size_t buflen, unsigned int base)
330{ 330{
331 struct svc_sock *svsk = 331 struct svc_sock *svsk =
332 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); 332 container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
333 struct msghdr msg = { 333 struct msghdr msg = { NULL };
334 .msg_flags = MSG_DONTWAIT, 334 ssize_t len;
335 };
336 int len;
337 335
338 rqstp->rq_xprt_hlen = 0; 336 rqstp->rq_xprt_hlen = 0;
339 337
340 clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 338 clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
341 iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC, iov, nr, buflen); 339 iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC, iov, nr, buflen);
342 len = sock_recvmsg(svsk->sk_sock, &msg, msg.msg_flags); 340 if (base != 0) {
341 iov_iter_advance(&msg.msg_iter, base);
342 buflen -= base;
343 }
344 len = sock_recvmsg(svsk->sk_sock, &msg, MSG_DONTWAIT);
343 /* If we read a full record, then assume there may be more 345 /* If we read a full record, then assume there may be more
344 * data to read (stream based sockets only!) 346 * data to read (stream based sockets only!)
345 */ 347 */
346 if (len == buflen) 348 if (len == buflen)
347 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); 349 set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
348 350
349 dprintk("svc: socket %p recvfrom(%p, %zu) = %d\n", 351 dprintk("svc: socket %p recvfrom(%p, %zu) = %zd\n",
350 svsk, iov[0].iov_base, iov[0].iov_len, len); 352 svsk, iov[0].iov_base, iov[0].iov_len, len);
351 return len; 353 return len;
352} 354}
353 355
354static int svc_partial_recvfrom(struct svc_rqst *rqstp,
355 struct kvec *iov, int nr,
356 int buflen, unsigned int base)
357{
358 size_t save_iovlen;
359 void *save_iovbase;
360 unsigned int i;
361 int ret;
362
363 if (base == 0)
364 return svc_recvfrom(rqstp, iov, nr, buflen);
365
366 for (i = 0; i < nr; i++) {
367 if (iov[i].iov_len > base)
368 break;
369 base -= iov[i].iov_len;
370 }
371 save_iovlen = iov[i].iov_len;
372 save_iovbase = iov[i].iov_base;
373 iov[i].iov_len -= base;
374 iov[i].iov_base += base;
375 ret = svc_recvfrom(rqstp, &iov[i], nr - i, buflen);
376 iov[i].iov_len = save_iovlen;
377 iov[i].iov_base = save_iovbase;
378 return ret;
379}
380
381/* 356/*
382 * Set socket snd and rcv buffer lengths 357 * Set socket snd and rcv buffer lengths
383 */ 358 */
@@ -962,7 +937,8 @@ static int svc_tcp_recv_record(struct svc_sock *svsk, struct svc_rqst *rqstp)
962 want = sizeof(rpc_fraghdr) - svsk->sk_tcplen; 937 want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;
963 iov.iov_base = ((char *) &svsk->sk_reclen) + svsk->sk_tcplen; 938 iov.iov_base = ((char *) &svsk->sk_reclen) + svsk->sk_tcplen;
964 iov.iov_len = want; 939 iov.iov_len = want;
965 if ((len = svc_recvfrom(rqstp, &iov, 1, want)) < 0) 940 len = svc_recvfrom(rqstp, &iov, 1, want, 0);
941 if (len < 0)
966 goto error; 942 goto error;
967 svsk->sk_tcplen += len; 943 svsk->sk_tcplen += len;
968 944
@@ -1088,14 +1064,13 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
1088 1064
1089 vec = rqstp->rq_vec; 1065 vec = rqstp->rq_vec;
1090 1066
1091 pnum = copy_pages_to_kvecs(&vec[0], &rqstp->rq_pages[0], 1067 pnum = copy_pages_to_kvecs(&vec[0], &rqstp->rq_pages[0], base + want);
1092 svsk->sk_datalen + want);
1093 1068
1094 rqstp->rq_respages = &rqstp->rq_pages[pnum]; 1069 rqstp->rq_respages = &rqstp->rq_pages[pnum];
1095 rqstp->rq_next_page = rqstp->rq_respages + 1; 1070 rqstp->rq_next_page = rqstp->rq_respages + 1;
1096 1071
1097 /* Now receive data */ 1072 /* Now receive data */
1098 len = svc_partial_recvfrom(rqstp, vec, pnum, want, base); 1073 len = svc_recvfrom(rqstp, vec, pnum, base + want, base);
1099 if (len >= 0) { 1074 if (len >= 0) {
1100 svsk->sk_tcplen += len; 1075 svsk->sk_tcplen += len;
1101 svsk->sk_datalen += len; 1076 svsk->sk_datalen += len;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
index d3a1a237cee6..f3c147d70286 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
@@ -5,8 +5,6 @@
5 * Support for backward direction RPCs on RPC/RDMA (server-side). 5 * Support for backward direction RPCs on RPC/RDMA (server-side).
6 */ 6 */
7 7
8#include <linux/module.h>
9
10#include <linux/sunrpc/svc_rdma.h> 8#include <linux/sunrpc/svc_rdma.h>
11 9
12#include "xprt_rdma.h" 10#include "xprt_rdma.h"
@@ -32,7 +30,6 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp,
32 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); 30 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
33 struct kvec *dst, *src = &rcvbuf->head[0]; 31 struct kvec *dst, *src = &rcvbuf->head[0];
34 struct rpc_rqst *req; 32 struct rpc_rqst *req;
35 unsigned long cwnd;
36 u32 credits; 33 u32 credits;
37 size_t len; 34 size_t len;
38 __be32 xid; 35 __be32 xid;
@@ -66,6 +63,8 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp,
66 if (dst->iov_len < len) 63 if (dst->iov_len < len)
67 goto out_unlock; 64 goto out_unlock;
68 memcpy(dst->iov_base, p, len); 65 memcpy(dst->iov_base, p, len);
66 xprt_pin_rqst(req);
67 spin_unlock(&xprt->queue_lock);
69 68
70 credits = be32_to_cpup(rdma_resp + 2); 69 credits = be32_to_cpup(rdma_resp + 2);
71 if (credits == 0) 70 if (credits == 0)
@@ -74,15 +73,13 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp,
74 credits = r_xprt->rx_buf.rb_bc_max_requests; 73 credits = r_xprt->rx_buf.rb_bc_max_requests;
75 74
76 spin_lock_bh(&xprt->transport_lock); 75 spin_lock_bh(&xprt->transport_lock);
77 cwnd = xprt->cwnd;
78 xprt->cwnd = credits << RPC_CWNDSHIFT; 76 xprt->cwnd = credits << RPC_CWNDSHIFT;
79 if (xprt->cwnd > cwnd)
80 xprt_release_rqst_cong(req->rq_task);
81 spin_unlock_bh(&xprt->transport_lock); 77 spin_unlock_bh(&xprt->transport_lock);
82 78
83 79 spin_lock(&xprt->queue_lock);
84 ret = 0; 80 ret = 0;
85 xprt_complete_rqst(req->rq_task, rcvbuf->len); 81 xprt_complete_rqst(req->rq_task, rcvbuf->len);
82 xprt_unpin_rqst(req);
86 rcvbuf->len = 0; 83 rcvbuf->len = 0;
87 84
88out_unlock: 85out_unlock:
@@ -251,7 +248,6 @@ xprt_rdma_bc_put(struct rpc_xprt *xprt)
251 dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); 248 dprintk("svcrdma: %s: xprt %p\n", __func__, xprt);
252 249
253 xprt_free(xprt); 250 xprt_free(xprt);
254 module_put(THIS_MODULE);
255} 251}
256 252
257static const struct rpc_xprt_ops xprt_rdma_bc_procs = { 253static const struct rpc_xprt_ops xprt_rdma_bc_procs = {
@@ -323,20 +319,9 @@ xprt_setup_rdma_bc(struct xprt_create *args)
323 args->bc_xprt->xpt_bc_xprt = xprt; 319 args->bc_xprt->xpt_bc_xprt = xprt;
324 xprt->bc_xprt = args->bc_xprt; 320 xprt->bc_xprt = args->bc_xprt;
325 321
326 if (!try_module_get(THIS_MODULE))
327 goto out_fail;
328
329 /* Final put for backchannel xprt is in __svc_rdma_free */ 322 /* Final put for backchannel xprt is in __svc_rdma_free */
330 xprt_get(xprt); 323 xprt_get(xprt);
331 return xprt; 324 return xprt;
332
333out_fail:
334 xprt_rdma_free_addresses(xprt);
335 args->bc_xprt->xpt_bc_xprt = NULL;
336 args->bc_xprt->xpt_bc_xps = NULL;
337 xprt_put(xprt);
338 xprt_free(xprt);
339 return ERR_PTR(-EINVAL);
340} 325}
341 326
342struct xprt_class xprt_rdma_bc = { 327struct xprt_class xprt_rdma_bc = {
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 2848cafd4a17..2f7ec8912f49 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -475,10 +475,12 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
475 475
476 /* Qualify the transport resource defaults with the 476 /* Qualify the transport resource defaults with the
477 * capabilities of this particular device */ 477 * capabilities of this particular device */
478 newxprt->sc_max_send_sges = dev->attrs.max_send_sge; 478 /* Transport header, head iovec, tail iovec */
479 /* transport hdr, head iovec, one page list entry, tail iovec */ 479 newxprt->sc_max_send_sges = 3;
480 if (newxprt->sc_max_send_sges < 4) { 480 /* Add one SGE per page list entry */
481 pr_err("svcrdma: too few Send SGEs available (%d)\n", 481 newxprt->sc_max_send_sges += svcrdma_max_req_size / PAGE_SIZE;
482 if (newxprt->sc_max_send_sges > dev->attrs.max_send_sge) {
483 pr_err("svcrdma: too few Send SGEs available (%d needed)\n",
482 newxprt->sc_max_send_sges); 484 newxprt->sc_max_send_sges);
483 goto errout; 485 goto errout;
484 } 486 }
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-26 16:45:05 -0400