aboutsummaryrefslogtreecommitdiffstats
path: root/net
diff options
context:
space:
mode:
authorTrond Myklebust <Trond.Myklebust@netapp.com>2008-04-24 14:01:02 -0400
committerTrond Myklebust <Trond.Myklebust@netapp.com>2008-04-24 14:01:02 -0400
commit233607dbbc823caf685e778cabc49fb7f679900b (patch)
tree13840137ee10788061cbec60fcfe8ea4c274558e /net
parent3dc5063786b273f1aee545844f6bd4e9651ebffe (diff)
parentb48633bd086d21f4a2a5bea96c7e6c7ba58eb60c (diff)
Merge branch 'devel'
Diffstat (limited to 'net')
-rw-r--r--net/sunrpc/Makefile2
-rw-r--r--net/sunrpc/auth.c71
-rw-r--r--net/sunrpc/auth_generic.c177
-rw-r--r--net/sunrpc/auth_gss/auth_gss.c105
-rw-r--r--net/sunrpc/auth_null.c3
-rw-r--r--net/sunrpc/auth_unix.c59
-rw-r--r--net/sunrpc/clnt.c50
-rw-r--r--net/sunrpc/rpcb_clnt.c2
-rw-r--r--net/sunrpc/sched.c264
-rw-r--r--net/sunrpc/xprt.c80
-rw-r--r--net/sunrpc/xprtsock.c82
11 files changed, 580 insertions, 315 deletions
diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
index 92e1dbe50947..5369aa369b35 100644
--- a/net/sunrpc/Makefile
+++ b/net/sunrpc/Makefile
@@ -8,7 +8,7 @@ obj-$(CONFIG_SUNRPC_GSS) += auth_gss/
8obj-$(CONFIG_SUNRPC_XPRT_RDMA) += xprtrdma/ 8obj-$(CONFIG_SUNRPC_XPRT_RDMA) += xprtrdma/
9 9
10sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ 10sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \
11 auth.o auth_null.o auth_unix.o \ 11 auth.o auth_null.o auth_unix.o auth_generic.o \
12 svc.o svcsock.o svcauth.o svcauth_unix.o \ 12 svc.o svcsock.o svcauth.o svcauth_unix.o \
13 rpcb_clnt.o timer.o xdr.o \ 13 rpcb_clnt.o timer.o xdr.o \
14 sunrpc_syms.o cache.o rpc_pipe.o \ 14 sunrpc_syms.o cache.o rpc_pipe.o \
diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c
index eca941ce298b..6bfea9ed6869 100644
--- a/net/sunrpc/auth.c
+++ b/net/sunrpc/auth.c
@@ -11,6 +11,7 @@
11#include <linux/module.h> 11#include <linux/module.h>
12#include <linux/slab.h> 12#include <linux/slab.h>
13#include <linux/errno.h> 13#include <linux/errno.h>
14#include <linux/hash.h>
14#include <linux/sunrpc/clnt.h> 15#include <linux/sunrpc/clnt.h>
15#include <linux/spinlock.h> 16#include <linux/spinlock.h>
16 17
@@ -219,6 +220,9 @@ rpcauth_destroy_credcache(struct rpc_auth *auth)
219} 220}
220EXPORT_SYMBOL_GPL(rpcauth_destroy_credcache); 221EXPORT_SYMBOL_GPL(rpcauth_destroy_credcache);
221 222
223
224#define RPC_AUTH_EXPIRY_MORATORIUM (60 * HZ)
225
222/* 226/*
223 * Remove stale credentials. Avoid sleeping inside the loop. 227 * Remove stale credentials. Avoid sleeping inside the loop.
224 */ 228 */
@@ -227,6 +231,7 @@ rpcauth_prune_expired(struct list_head *free, int nr_to_scan)
227{ 231{
228 spinlock_t *cache_lock; 232 spinlock_t *cache_lock;
229 struct rpc_cred *cred; 233 struct rpc_cred *cred;
234 unsigned long expired = jiffies - RPC_AUTH_EXPIRY_MORATORIUM;
230 235
231 while (!list_empty(&cred_unused)) { 236 while (!list_empty(&cred_unused)) {
232 cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru); 237 cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru);
@@ -234,6 +239,10 @@ rpcauth_prune_expired(struct list_head *free, int nr_to_scan)
234 number_cred_unused--; 239 number_cred_unused--;
235 if (atomic_read(&cred->cr_count) != 0) 240 if (atomic_read(&cred->cr_count) != 0)
236 continue; 241 continue;
242 /* Enforce a 5 second garbage collection moratorium */
243 if (time_in_range(cred->cr_expire, expired, jiffies) &&
244 test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0)
245 continue;
237 cache_lock = &cred->cr_auth->au_credcache->lock; 246 cache_lock = &cred->cr_auth->au_credcache->lock;
238 spin_lock(cache_lock); 247 spin_lock(cache_lock);
239 if (atomic_read(&cred->cr_count) == 0) { 248 if (atomic_read(&cred->cr_count) == 0) {
@@ -280,10 +289,9 @@ rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred,
280 struct hlist_node *pos; 289 struct hlist_node *pos;
281 struct rpc_cred *cred = NULL, 290 struct rpc_cred *cred = NULL,
282 *entry, *new; 291 *entry, *new;
283 int nr = 0; 292 unsigned int nr;
284 293
285 if (!(flags & RPCAUTH_LOOKUP_ROOTCREDS)) 294 nr = hash_long(acred->uid, RPC_CREDCACHE_HASHBITS);
286 nr = acred->uid & RPC_CREDCACHE_MASK;
287 295
288 rcu_read_lock(); 296 rcu_read_lock();
289 hlist_for_each_entry_rcu(entry, pos, &cache->hashtable[nr], cr_hash) { 297 hlist_for_each_entry_rcu(entry, pos, &cache->hashtable[nr], cr_hash) {
@@ -356,7 +364,6 @@ rpcauth_lookupcred(struct rpc_auth *auth, int flags)
356 put_group_info(acred.group_info); 364 put_group_info(acred.group_info);
357 return ret; 365 return ret;
358} 366}
359EXPORT_SYMBOL_GPL(rpcauth_lookupcred);
360 367
361void 368void
362rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred, 369rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred,
@@ -375,41 +382,58 @@ rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred,
375} 382}
376EXPORT_SYMBOL_GPL(rpcauth_init_cred); 383EXPORT_SYMBOL_GPL(rpcauth_init_cred);
377 384
378struct rpc_cred * 385void
379rpcauth_bindcred(struct rpc_task *task) 386rpcauth_generic_bind_cred(struct rpc_task *task, struct rpc_cred *cred)
387{
388 task->tk_msg.rpc_cred = get_rpccred(cred);
389 dprintk("RPC: %5u holding %s cred %p\n", task->tk_pid,
390 cred->cr_auth->au_ops->au_name, cred);
391}
392EXPORT_SYMBOL_GPL(rpcauth_generic_bind_cred);
393
394static void
395rpcauth_bind_root_cred(struct rpc_task *task)
380{ 396{
381 struct rpc_auth *auth = task->tk_client->cl_auth; 397 struct rpc_auth *auth = task->tk_client->cl_auth;
382 struct auth_cred acred = { 398 struct auth_cred acred = {
383 .uid = current->fsuid, 399 .uid = 0,
384 .gid = current->fsgid, 400 .gid = 0,
385 .group_info = current->group_info,
386 }; 401 };
387 struct rpc_cred *ret; 402 struct rpc_cred *ret;
388 int flags = 0;
389 403
390 dprintk("RPC: %5u looking up %s cred\n", 404 dprintk("RPC: %5u looking up %s cred\n",
391 task->tk_pid, task->tk_client->cl_auth->au_ops->au_name); 405 task->tk_pid, task->tk_client->cl_auth->au_ops->au_name);
392 get_group_info(acred.group_info); 406 ret = auth->au_ops->lookup_cred(auth, &acred, 0);
393 if (task->tk_flags & RPC_TASK_ROOTCREDS) 407 if (!IS_ERR(ret))
394 flags |= RPCAUTH_LOOKUP_ROOTCREDS; 408 task->tk_msg.rpc_cred = ret;
395 ret = auth->au_ops->lookup_cred(auth, &acred, flags); 409 else
410 task->tk_status = PTR_ERR(ret);
411}
412
413static void
414rpcauth_bind_new_cred(struct rpc_task *task)
415{
416 struct rpc_auth *auth = task->tk_client->cl_auth;
417 struct rpc_cred *ret;
418
419 dprintk("RPC: %5u looking up %s cred\n",
420 task->tk_pid, auth->au_ops->au_name);
421 ret = rpcauth_lookupcred(auth, 0);
396 if (!IS_ERR(ret)) 422 if (!IS_ERR(ret))
397 task->tk_msg.rpc_cred = ret; 423 task->tk_msg.rpc_cred = ret;
398 else 424 else
399 task->tk_status = PTR_ERR(ret); 425 task->tk_status = PTR_ERR(ret);
400 put_group_info(acred.group_info);
401 return ret;
402} 426}
403 427
404void 428void
405rpcauth_holdcred(struct rpc_task *task) 429rpcauth_bindcred(struct rpc_task *task, struct rpc_cred *cred, int flags)
406{ 430{
407 struct rpc_cred *cred = task->tk_msg.rpc_cred; 431 if (cred != NULL)
408 if (cred != NULL) { 432 cred->cr_ops->crbind(task, cred);
409 get_rpccred(cred); 433 else if (flags & RPC_TASK_ROOTCREDS)
410 dprintk("RPC: %5u holding %s cred %p\n", task->tk_pid, 434 rpcauth_bind_root_cred(task);
411 cred->cr_auth->au_ops->au_name, cred); 435 else
412 } 436 rpcauth_bind_new_cred(task);
413} 437}
414 438
415void 439void
@@ -550,6 +574,7 @@ static struct shrinker rpc_cred_shrinker = {
550void __init rpcauth_init_module(void) 574void __init rpcauth_init_module(void)
551{ 575{
552 rpc_init_authunix(); 576 rpc_init_authunix();
577 rpc_init_generic_auth();
553 register_shrinker(&rpc_cred_shrinker); 578 register_shrinker(&rpc_cred_shrinker);
554} 579}
555 580
diff --git a/net/sunrpc/auth_generic.c b/net/sunrpc/auth_generic.c
new file mode 100644
index 000000000000..d927d9f57412
--- /dev/null
+++ b/net/sunrpc/auth_generic.c
@@ -0,0 +1,177 @@
1/*
2 * Generic RPC credential
3 *
4 * Copyright (C) 2008, Trond Myklebust <Trond.Myklebust@netapp.com>
5 */
6
7#include <linux/err.h>
8#include <linux/types.h>
9#include <linux/module.h>
10#include <linux/sched.h>
11#include <linux/sunrpc/auth.h>
12#include <linux/sunrpc/clnt.h>
13#include <linux/sunrpc/debug.h>
14#include <linux/sunrpc/sched.h>
15
16#ifdef RPC_DEBUG
17# define RPCDBG_FACILITY RPCDBG_AUTH
18#endif
19
20#define RPC_ANONYMOUS_USERID ((uid_t)-2)
21#define RPC_ANONYMOUS_GROUPID ((gid_t)-2)
22
23struct generic_cred {
24 struct rpc_cred gc_base;
25 struct auth_cred acred;
26};
27
28static struct rpc_auth generic_auth;
29static struct rpc_cred_cache generic_cred_cache;
30static const struct rpc_credops generic_credops;
31
32/*
33 * Public call interface
34 */
35struct rpc_cred *rpc_lookup_cred(void)
36{
37 return rpcauth_lookupcred(&generic_auth, 0);
38}
39EXPORT_SYMBOL_GPL(rpc_lookup_cred);
40
41/*
42 * Public call interface for looking up machine creds.
43 */
44struct rpc_cred *rpc_lookup_machine_cred(void)
45{
46 struct auth_cred acred = {
47 .uid = RPC_ANONYMOUS_USERID,
48 .gid = RPC_ANONYMOUS_GROUPID,
49 .machine_cred = 1,
50 };
51
52 dprintk("RPC: looking up machine cred\n");
53 return generic_auth.au_ops->lookup_cred(&generic_auth, &acred, 0);
54}
55EXPORT_SYMBOL_GPL(rpc_lookup_machine_cred);
56
57static void
58generic_bind_cred(struct rpc_task *task, struct rpc_cred *cred)
59{
60 struct rpc_auth *auth = task->tk_client->cl_auth;
61 struct auth_cred *acred = &container_of(cred, struct generic_cred, gc_base)->acred;
62 struct rpc_cred *ret;
63
64 ret = auth->au_ops->lookup_cred(auth, acred, 0);
65 if (!IS_ERR(ret))
66 task->tk_msg.rpc_cred = ret;
67 else
68 task->tk_status = PTR_ERR(ret);
69}
70
71/*
72 * Lookup generic creds for current process
73 */
74static struct rpc_cred *
75generic_lookup_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags)
76{
77 return rpcauth_lookup_credcache(&generic_auth, acred, flags);
78}
79
80static struct rpc_cred *
81generic_create_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags)
82{
83 struct generic_cred *gcred;
84
85 gcred = kmalloc(sizeof(*gcred), GFP_KERNEL);
86 if (gcred == NULL)
87 return ERR_PTR(-ENOMEM);
88
89 rpcauth_init_cred(&gcred->gc_base, acred, &generic_auth, &generic_credops);
90 gcred->gc_base.cr_flags = 1UL << RPCAUTH_CRED_UPTODATE;
91
92 gcred->acred.uid = acred->uid;
93 gcred->acred.gid = acred->gid;
94 gcred->acred.group_info = acred->group_info;
95 if (gcred->acred.group_info != NULL)
96 get_group_info(gcred->acred.group_info);
97 gcred->acred.machine_cred = acred->machine_cred;
98
99 dprintk("RPC: allocated %s cred %p for uid %d gid %d\n",
100 gcred->acred.machine_cred ? "machine" : "generic",
101 gcred, acred->uid, acred->gid);
102 return &gcred->gc_base;
103}
104
105static void
106generic_free_cred(struct rpc_cred *cred)
107{
108 struct generic_cred *gcred = container_of(cred, struct generic_cred, gc_base);
109
110 dprintk("RPC: generic_free_cred %p\n", gcred);
111 if (gcred->acred.group_info != NULL)
112 put_group_info(gcred->acred.group_info);
113 kfree(gcred);
114}
115
116static void
117generic_free_cred_callback(struct rcu_head *head)
118{
119 struct rpc_cred *cred = container_of(head, struct rpc_cred, cr_rcu);
120 generic_free_cred(cred);
121}
122
123static void
124generic_destroy_cred(struct rpc_cred *cred)
125{
126 call_rcu(&cred->cr_rcu, generic_free_cred_callback);
127}
128
129/*
130 * Match credentials against current process creds.
131 */
132static int
133generic_match(struct auth_cred *acred, struct rpc_cred *cred, int flags)
134{
135 struct generic_cred *gcred = container_of(cred, struct generic_cred, gc_base);
136
137 if (gcred->acred.uid != acred->uid ||
138 gcred->acred.gid != acred->gid ||
139 gcred->acred.group_info != acred->group_info ||
140 gcred->acred.machine_cred != acred->machine_cred)
141 return 0;
142 return 1;
143}
144
145void __init rpc_init_generic_auth(void)
146{
147 spin_lock_init(&generic_cred_cache.lock);
148}
149
150void __exit rpc_destroy_generic_auth(void)
151{
152 rpcauth_clear_credcache(&generic_cred_cache);
153}
154
155static struct rpc_cred_cache generic_cred_cache = {
156 {{ NULL, },},
157};
158
159static const struct rpc_authops generic_auth_ops = {
160 .owner = THIS_MODULE,
161 .au_name = "Generic",
162 .lookup_cred = generic_lookup_cred,
163 .crcreate = generic_create_cred,
164};
165
166static struct rpc_auth generic_auth = {
167 .au_ops = &generic_auth_ops,
168 .au_count = ATOMIC_INIT(0),
169 .au_credcache = &generic_cred_cache,
170};
171
172static const struct rpc_credops generic_credops = {
173 .cr_name = "Generic cred",
174 .crdestroy = generic_destroy_cred,
175 .crbind = generic_bind_cred,
176 .crmatch = generic_match,
177};
diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 5828e5c060ca..cc12d5f5d5da 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -114,27 +114,14 @@ static void
114gss_cred_set_ctx(struct rpc_cred *cred, struct gss_cl_ctx *ctx) 114gss_cred_set_ctx(struct rpc_cred *cred, struct gss_cl_ctx *ctx)
115{ 115{
116 struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base); 116 struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);
117 struct gss_cl_ctx *old;
118 117
119 old = gss_cred->gc_ctx; 118 if (!test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags))
119 return;
120 gss_get_ctx(ctx);
120 rcu_assign_pointer(gss_cred->gc_ctx, ctx); 121 rcu_assign_pointer(gss_cred->gc_ctx, ctx);
121 set_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); 122 set_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
123 smp_mb__before_clear_bit();
122 clear_bit(RPCAUTH_CRED_NEW, &cred->cr_flags); 124 clear_bit(RPCAUTH_CRED_NEW, &cred->cr_flags);
123 if (old)
124 gss_put_ctx(old);
125}
126
127static int
128gss_cred_is_uptodate_ctx(struct rpc_cred *cred)
129{
130 struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base);
131 int res = 0;
132
133 rcu_read_lock();
134 if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) && gss_cred->gc_ctx)
135 res = 1;
136 rcu_read_unlock();
137 return res;
138} 125}
139 126
140static const void * 127static const void *
@@ -266,6 +253,7 @@ gss_release_msg(struct gss_upcall_msg *gss_msg)
266 BUG_ON(!list_empty(&gss_msg->list)); 253 BUG_ON(!list_empty(&gss_msg->list));
267 if (gss_msg->ctx != NULL) 254 if (gss_msg->ctx != NULL)
268 gss_put_ctx(gss_msg->ctx); 255 gss_put_ctx(gss_msg->ctx);
256 rpc_destroy_wait_queue(&gss_msg->rpc_waitqueue);
269 kfree(gss_msg); 257 kfree(gss_msg);
270} 258}
271 259
@@ -339,7 +327,7 @@ gss_upcall_callback(struct rpc_task *task)
339 327
340 spin_lock(&inode->i_lock); 328 spin_lock(&inode->i_lock);
341 if (gss_msg->ctx) 329 if (gss_msg->ctx)
342 gss_cred_set_ctx(task->tk_msg.rpc_cred, gss_get_ctx(gss_msg->ctx)); 330 gss_cred_set_ctx(task->tk_msg.rpc_cred, gss_msg->ctx);
343 else 331 else
344 task->tk_status = gss_msg->msg.errno; 332 task->tk_status = gss_msg->msg.errno;
345 gss_cred->gc_upcall = NULL; 333 gss_cred->gc_upcall = NULL;
@@ -370,9 +358,16 @@ gss_alloc_msg(struct gss_auth *gss_auth, uid_t uid)
370static struct gss_upcall_msg * 358static struct gss_upcall_msg *
371gss_setup_upcall(struct rpc_clnt *clnt, struct gss_auth *gss_auth, struct rpc_cred *cred) 359gss_setup_upcall(struct rpc_clnt *clnt, struct gss_auth *gss_auth, struct rpc_cred *cred)
372{ 360{
361 struct gss_cred *gss_cred = container_of(cred,
362 struct gss_cred, gc_base);
373 struct gss_upcall_msg *gss_new, *gss_msg; 363 struct gss_upcall_msg *gss_new, *gss_msg;
364 uid_t uid = cred->cr_uid;
374 365
375 gss_new = gss_alloc_msg(gss_auth, cred->cr_uid); 366 /* Special case: rpc.gssd assumes that uid == 0 implies machine creds */
367 if (gss_cred->gc_machine_cred != 0)
368 uid = 0;
369
370 gss_new = gss_alloc_msg(gss_auth, uid);
376 if (gss_new == NULL) 371 if (gss_new == NULL)
377 return ERR_PTR(-ENOMEM); 372 return ERR_PTR(-ENOMEM);
378 gss_msg = gss_add_msg(gss_auth, gss_new); 373 gss_msg = gss_add_msg(gss_auth, gss_new);
@@ -408,13 +403,17 @@ gss_refresh_upcall(struct rpc_task *task)
408 } 403 }
409 spin_lock(&inode->i_lock); 404 spin_lock(&inode->i_lock);
410 if (gss_cred->gc_upcall != NULL) 405 if (gss_cred->gc_upcall != NULL)
411 rpc_sleep_on(&gss_cred->gc_upcall->rpc_waitqueue, task, NULL, NULL); 406 rpc_sleep_on(&gss_cred->gc_upcall->rpc_waitqueue, task, NULL);
412 else if (gss_msg->ctx == NULL && gss_msg->msg.errno >= 0) { 407 else if (gss_msg->ctx != NULL) {
408 gss_cred_set_ctx(task->tk_msg.rpc_cred, gss_msg->ctx);
409 gss_cred->gc_upcall = NULL;
410 rpc_wake_up_status(&gss_msg->rpc_waitqueue, gss_msg->msg.errno);
411 } else if (gss_msg->msg.errno >= 0) {
413 task->tk_timeout = 0; 412 task->tk_timeout = 0;
414 gss_cred->gc_upcall = gss_msg; 413 gss_cred->gc_upcall = gss_msg;
415 /* gss_upcall_callback will release the reference to gss_upcall_msg */ 414 /* gss_upcall_callback will release the reference to gss_upcall_msg */
416 atomic_inc(&gss_msg->count); 415 atomic_inc(&gss_msg->count);
417 rpc_sleep_on(&gss_msg->rpc_waitqueue, task, gss_upcall_callback, NULL); 416 rpc_sleep_on(&gss_msg->rpc_waitqueue, task, gss_upcall_callback);
418 } else 417 } else
419 err = gss_msg->msg.errno; 418 err = gss_msg->msg.errno;
420 spin_unlock(&inode->i_lock); 419 spin_unlock(&inode->i_lock);
@@ -454,7 +453,7 @@ gss_create_upcall(struct gss_auth *gss_auth, struct gss_cred *gss_cred)
454 schedule(); 453 schedule();
455 } 454 }
456 if (gss_msg->ctx) 455 if (gss_msg->ctx)
457 gss_cred_set_ctx(cred, gss_get_ctx(gss_msg->ctx)); 456 gss_cred_set_ctx(cred, gss_msg->ctx);
458 else 457 else
459 err = gss_msg->msg.errno; 458 err = gss_msg->msg.errno;
460 spin_unlock(&inode->i_lock); 459 spin_unlock(&inode->i_lock);
@@ -709,7 +708,7 @@ gss_destroying_context(struct rpc_cred *cred)
709 struct rpc_task *task; 708 struct rpc_task *task;
710 709
711 if (gss_cred->gc_ctx == NULL || 710 if (gss_cred->gc_ctx == NULL ||
712 gss_cred->gc_ctx->gc_proc == RPC_GSS_PROC_DESTROY) 711 test_and_clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0)
713 return 0; 712 return 0;
714 713
715 gss_cred->gc_ctx->gc_proc = RPC_GSS_PROC_DESTROY; 714 gss_cred->gc_ctx->gc_proc = RPC_GSS_PROC_DESTROY;
@@ -719,7 +718,7 @@ gss_destroying_context(struct rpc_cred *cred)
719 * by the RPC call or by the put_rpccred() below */ 718 * by the RPC call or by the put_rpccred() below */
720 get_rpccred(cred); 719 get_rpccred(cred);
721 720
722 task = rpc_call_null(gss_auth->client, cred, RPC_TASK_ASYNC); 721 task = rpc_call_null(gss_auth->client, cred, RPC_TASK_ASYNC|RPC_TASK_SOFT);
723 if (!IS_ERR(task)) 722 if (!IS_ERR(task))
724 rpc_put_task(task); 723 rpc_put_task(task);
725 724
@@ -817,6 +816,7 @@ gss_create_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags)
817 */ 816 */
818 cred->gc_base.cr_flags = 1UL << RPCAUTH_CRED_NEW; 817 cred->gc_base.cr_flags = 1UL << RPCAUTH_CRED_NEW;
819 cred->gc_service = gss_auth->service; 818 cred->gc_service = gss_auth->service;
819 cred->gc_machine_cred = acred->machine_cred;
820 kref_get(&gss_auth->kref); 820 kref_get(&gss_auth->kref);
821 return &cred->gc_base; 821 return &cred->gc_base;
822 822
@@ -843,17 +843,16 @@ gss_match(struct auth_cred *acred, struct rpc_cred *rc, int flags)
843{ 843{
844 struct gss_cred *gss_cred = container_of(rc, struct gss_cred, gc_base); 844 struct gss_cred *gss_cred = container_of(rc, struct gss_cred, gc_base);
845 845
846 /* 846 if (test_bit(RPCAUTH_CRED_NEW, &rc->cr_flags))
847 * If the searchflags have set RPCAUTH_LOOKUP_NEW, then
848 * we don't really care if the credential has expired or not,
849 * since the caller should be prepared to reinitialise it.
850 */
851 if ((flags & RPCAUTH_LOOKUP_NEW) && test_bit(RPCAUTH_CRED_NEW, &rc->cr_flags))
852 goto out; 847 goto out;
853 /* Don't match with creds that have expired. */ 848 /* Don't match with creds that have expired. */
854 if (gss_cred->gc_ctx && time_after(jiffies, gss_cred->gc_ctx->gc_expiry)) 849 if (time_after(jiffies, gss_cred->gc_ctx->gc_expiry))
850 return 0;
851 if (!test_bit(RPCAUTH_CRED_UPTODATE, &rc->cr_flags))
855 return 0; 852 return 0;
856out: 853out:
854 if (acred->machine_cred != gss_cred->gc_machine_cred)
855 return 0;
857 return (rc->cr_uid == acred->uid); 856 return (rc->cr_uid == acred->uid);
858} 857}
859 858
@@ -917,16 +916,48 @@ out_put_ctx:
917 return NULL; 916 return NULL;
918} 917}
919 918
919static int gss_renew_cred(struct rpc_task *task)
920{
921 struct rpc_cred *oldcred = task->tk_msg.rpc_cred;
922 struct gss_cred *gss_cred = container_of(oldcred,
923 struct gss_cred,
924 gc_base);
925 struct rpc_auth *auth = oldcred->cr_auth;
926 struct auth_cred acred = {
927 .uid = oldcred->cr_uid,
928 .machine_cred = gss_cred->gc_machine_cred,
929 };
930 struct rpc_cred *new;
931
932 new = gss_lookup_cred(auth, &acred, RPCAUTH_LOOKUP_NEW);
933 if (IS_ERR(new))
934 return PTR_ERR(new);
935 task->tk_msg.rpc_cred = new;
936 put_rpccred(oldcred);
937 return 0;
938}
939
920/* 940/*
921* Refresh credentials. XXX - finish 941* Refresh credentials. XXX - finish
922*/ 942*/
923static int 943static int
924gss_refresh(struct rpc_task *task) 944gss_refresh(struct rpc_task *task)
925{ 945{
946 struct rpc_cred *cred = task->tk_msg.rpc_cred;
947 int ret = 0;
948
949 if (!test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) &&
950 !test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags)) {
951 ret = gss_renew_cred(task);
952 if (ret < 0)
953 goto out;
954 cred = task->tk_msg.rpc_cred;
955 }
926 956
927 if (!gss_cred_is_uptodate_ctx(task->tk_msg.rpc_cred)) 957 if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags))
928 return gss_refresh_upcall(task); 958 ret = gss_refresh_upcall(task);
929 return 0; 959out:
960 return ret;
930} 961}
931 962
932/* Dummy refresh routine: used only when destroying the context */ 963/* Dummy refresh routine: used only when destroying the context */
@@ -1286,9 +1317,7 @@ out:
1286static const struct rpc_authops authgss_ops = { 1317static const struct rpc_authops authgss_ops = {
1287 .owner = THIS_MODULE, 1318 .owner = THIS_MODULE,
1288 .au_flavor = RPC_AUTH_GSS, 1319 .au_flavor = RPC_AUTH_GSS,
1289#ifdef RPC_DEBUG
1290 .au_name = "RPCSEC_GSS", 1320 .au_name = "RPCSEC_GSS",
1291#endif
1292 .create = gss_create, 1321 .create = gss_create,
1293 .destroy = gss_destroy, 1322 .destroy = gss_destroy,
1294 .lookup_cred = gss_lookup_cred, 1323 .lookup_cred = gss_lookup_cred,
@@ -1299,6 +1328,7 @@ static const struct rpc_credops gss_credops = {
1299 .cr_name = "AUTH_GSS", 1328 .cr_name = "AUTH_GSS",
1300 .crdestroy = gss_destroy_cred, 1329 .crdestroy = gss_destroy_cred,
1301 .cr_init = gss_cred_init, 1330 .cr_init = gss_cred_init,
1331 .crbind = rpcauth_generic_bind_cred,
1302 .crmatch = gss_match, 1332 .crmatch = gss_match,
1303 .crmarshal = gss_marshal, 1333 .crmarshal = gss_marshal,
1304 .crrefresh = gss_refresh, 1334 .crrefresh = gss_refresh,
@@ -1310,6 +1340,7 @@ static const struct rpc_credops gss_credops = {
1310static const struct rpc_credops gss_nullops = { 1340static const struct rpc_credops gss_nullops = {
1311 .cr_name = "AUTH_GSS", 1341 .cr_name = "AUTH_GSS",
1312 .crdestroy = gss_destroy_cred, 1342 .crdestroy = gss_destroy_cred,
1343 .crbind = rpcauth_generic_bind_cred,
1313 .crmatch = gss_match, 1344 .crmatch = gss_match,
1314 .crmarshal = gss_marshal, 1345 .crmarshal = gss_marshal,
1315 .crrefresh = gss_refresh_null, 1346 .crrefresh = gss_refresh_null,
diff --git a/net/sunrpc/auth_null.c b/net/sunrpc/auth_null.c
index 537d0e8589dd..c70dd7f5258e 100644
--- a/net/sunrpc/auth_null.c
+++ b/net/sunrpc/auth_null.c
@@ -104,9 +104,7 @@ nul_validate(struct rpc_task *task, __be32 *p)
104const struct rpc_authops authnull_ops = { 104const struct rpc_authops authnull_ops = {
105 .owner = THIS_MODULE, 105 .owner = THIS_MODULE,
106 .au_flavor = RPC_AUTH_NULL, 106 .au_flavor = RPC_AUTH_NULL,
107#ifdef RPC_DEBUG
108 .au_name = "NULL", 107 .au_name = "NULL",
109#endif
110 .create = nul_create, 108 .create = nul_create,
111 .destroy = nul_destroy, 109 .destroy = nul_destroy,
112 .lookup_cred = nul_lookup_cred, 110 .lookup_cred = nul_lookup_cred,
@@ -125,6 +123,7 @@ static
125const struct rpc_credops null_credops = { 123const struct rpc_credops null_credops = {
126 .cr_name = "AUTH_NULL", 124 .cr_name = "AUTH_NULL",
127 .crdestroy = nul_destroy_cred, 125 .crdestroy = nul_destroy_cred,
126 .crbind = rpcauth_generic_bind_cred,
128 .crmatch = nul_match, 127 .crmatch = nul_match,
129 .crmarshal = nul_marshal, 128 .crmarshal = nul_marshal,
130 .crrefresh = nul_refresh, 129 .crrefresh = nul_refresh,
diff --git a/net/sunrpc/auth_unix.c b/net/sunrpc/auth_unix.c
index 5ed91e5bcee4..44920b90bdc4 100644
--- a/net/sunrpc/auth_unix.c
+++ b/net/sunrpc/auth_unix.c
@@ -60,7 +60,8 @@ static struct rpc_cred *
60unx_create_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags) 60unx_create_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags)
61{ 61{
62 struct unx_cred *cred; 62 struct unx_cred *cred;
63 int i; 63 unsigned int groups = 0;
64 unsigned int i;
64 65
65 dprintk("RPC: allocating UNIX cred for uid %d gid %d\n", 66 dprintk("RPC: allocating UNIX cred for uid %d gid %d\n",
66 acred->uid, acred->gid); 67 acred->uid, acred->gid);
@@ -70,21 +71,17 @@ unx_create_cred(struct rpc_auth *auth, struct auth_cred *acred, int flags)
70 71
71 rpcauth_init_cred(&cred->uc_base, acred, auth, &unix_credops); 72 rpcauth_init_cred(&cred->uc_base, acred, auth, &unix_credops);
72 cred->uc_base.cr_flags = 1UL << RPCAUTH_CRED_UPTODATE; 73 cred->uc_base.cr_flags = 1UL << RPCAUTH_CRED_UPTODATE;
73 if (flags & RPCAUTH_LOOKUP_ROOTCREDS) { 74
74 cred->uc_uid = 0; 75 if (acred->group_info != NULL)
75 cred->uc_gid = 0; 76 groups = acred->group_info->ngroups;
76 cred->uc_gids[0] = NOGROUP; 77 if (groups > NFS_NGROUPS)
77 } else { 78 groups = NFS_NGROUPS;
78 int groups = acred->group_info->ngroups; 79
79 if (groups > NFS_NGROUPS) 80 cred->uc_gid = acred->gid;
80 groups = NFS_NGROUPS; 81 for (i = 0; i < groups; i++)
81 82 cred->uc_gids[i] = GROUP_AT(acred->group_info, i);
82 cred->uc_gid = acred->gid; 83 if (i < NFS_NGROUPS)
83 for (i = 0; i < groups; i++) 84 cred->uc_gids[i] = NOGROUP;
84 cred->uc_gids[i] = GROUP_AT(acred->group_info, i);
85 if (i < NFS_NGROUPS)
86 cred->uc_gids[i] = NOGROUP;
87 }
88 85
89 return &cred->uc_base; 86 return &cred->uc_base;
90} 87}
@@ -118,26 +115,21 @@ static int
118unx_match(struct auth_cred *acred, struct rpc_cred *rcred, int flags) 115unx_match(struct auth_cred *acred, struct rpc_cred *rcred, int flags)
119{ 116{
120 struct unx_cred *cred = container_of(rcred, struct unx_cred, uc_base); 117 struct unx_cred *cred = container_of(rcred, struct unx_cred, uc_base);
121 int i; 118 unsigned int groups = 0;
119 unsigned int i;
122 120
123 if (!(flags & RPCAUTH_LOOKUP_ROOTCREDS)) {
124 int groups;
125 121
126 if (cred->uc_uid != acred->uid 122 if (cred->uc_uid != acred->uid || cred->uc_gid != acred->gid)
127 || cred->uc_gid != acred->gid) 123 return 0;
128 return 0;
129 124
125 if (acred->group_info != NULL)
130 groups = acred->group_info->ngroups; 126 groups = acred->group_info->ngroups;
131 if (groups > NFS_NGROUPS) 127 if (groups > NFS_NGROUPS)
132 groups = NFS_NGROUPS; 128 groups = NFS_NGROUPS;
133 for (i = 0; i < groups ; i++) 129 for (i = 0; i < groups ; i++)
134 if (cred->uc_gids[i] != GROUP_AT(acred->group_info, i)) 130 if (cred->uc_gids[i] != GROUP_AT(acred->group_info, i))
135 return 0; 131 return 0;
136 return 1; 132 return 1;
137 }
138 return (cred->uc_uid == 0
139 && cred->uc_gid == 0
140 && cred->uc_gids[0] == (gid_t) NOGROUP);
141} 133}
142 134
143/* 135/*
@@ -218,9 +210,7 @@ void __init rpc_init_authunix(void)
218const struct rpc_authops authunix_ops = { 210const struct rpc_authops authunix_ops = {
219 .owner = THIS_MODULE, 211 .owner = THIS_MODULE,
220 .au_flavor = RPC_AUTH_UNIX, 212 .au_flavor = RPC_AUTH_UNIX,
221#ifdef RPC_DEBUG
222 .au_name = "UNIX", 213 .au_name = "UNIX",
223#endif
224 .create = unx_create, 214 .create = unx_create,
225 .destroy = unx_destroy, 215 .destroy = unx_destroy,
226 .lookup_cred = unx_lookup_cred, 216 .lookup_cred = unx_lookup_cred,
@@ -245,6 +235,7 @@ static
245const struct rpc_credops unix_credops = { 235const struct rpc_credops unix_credops = {
246 .cr_name = "AUTH_UNIX", 236 .cr_name = "AUTH_UNIX",
247 .crdestroy = unx_destroy_cred, 237 .crdestroy = unx_destroy_cred,
238 .crbind = rpcauth_generic_bind_cred,
248 .crmatch = unx_match, 239 .crmatch = unx_match,
249 .crmarshal = unx_marshal, 240 .crmarshal = unx_marshal,
250 .crrefresh = unx_refresh, 241 .crrefresh = unx_refresh,
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 7b96ff38002f..8945307556ec 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -544,7 +544,7 @@ EXPORT_SYMBOL_GPL(rpc_run_task);
544 * @msg: RPC call parameters 544 * @msg: RPC call parameters
545 * @flags: RPC call flags 545 * @flags: RPC call flags
546 */ 546 */
547int rpc_call_sync(struct rpc_clnt *clnt, struct rpc_message *msg, int flags) 547int rpc_call_sync(struct rpc_clnt *clnt, const struct rpc_message *msg, int flags)
548{ 548{
549 struct rpc_task *task; 549 struct rpc_task *task;
550 struct rpc_task_setup task_setup_data = { 550 struct rpc_task_setup task_setup_data = {
@@ -575,7 +575,7 @@ EXPORT_SYMBOL_GPL(rpc_call_sync);
575 * @data: user call data 575 * @data: user call data
576 */ 576 */
577int 577int
578rpc_call_async(struct rpc_clnt *clnt, struct rpc_message *msg, int flags, 578rpc_call_async(struct rpc_clnt *clnt, const struct rpc_message *msg, int flags,
579 const struct rpc_call_ops *tk_ops, void *data) 579 const struct rpc_call_ops *tk_ops, void *data)
580{ 580{
581 struct rpc_task *task; 581 struct rpc_task *task;
@@ -1062,7 +1062,7 @@ call_transmit(struct rpc_task *task)
1062 if (task->tk_msg.rpc_proc->p_decode != NULL) 1062 if (task->tk_msg.rpc_proc->p_decode != NULL)
1063 return; 1063 return;
1064 task->tk_action = rpc_exit_task; 1064 task->tk_action = rpc_exit_task;
1065 rpc_wake_up_task(task); 1065 rpc_wake_up_queued_task(&task->tk_xprt->pending, task);
1066} 1066}
1067 1067
1068/* 1068/*
@@ -1116,7 +1116,8 @@ call_status(struct rpc_task *task)
1116 case -ETIMEDOUT: 1116 case -ETIMEDOUT:
1117 task->tk_action = call_timeout; 1117 task->tk_action = call_timeout;
1118 if (task->tk_client->cl_discrtry) 1118 if (task->tk_client->cl_discrtry)
1119 xprt_force_disconnect(task->tk_xprt); 1119 xprt_conditional_disconnect(task->tk_xprt,
1120 req->rq_connect_cookie);
1120 break; 1121 break;
1121 case -ECONNREFUSED: 1122 case -ECONNREFUSED:
1122 case -ENOTCONN: 1123 case -ENOTCONN:
@@ -1168,6 +1169,11 @@ call_timeout(struct rpc_task *task)
1168 clnt->cl_protname, clnt->cl_server); 1169 clnt->cl_protname, clnt->cl_server);
1169 } 1170 }
1170 rpc_force_rebind(clnt); 1171 rpc_force_rebind(clnt);
1172 /*
1173 * Did our request time out due to an RPCSEC_GSS out-of-sequence
1174 * event? RFC2203 requires the server to drop all such requests.
1175 */
1176 rpcauth_invalcred(task);
1171 1177
1172retry: 1178retry:
1173 clnt->cl_stats->rpcretrans++; 1179 clnt->cl_stats->rpcretrans++;
@@ -1195,18 +1201,6 @@ call_decode(struct rpc_task *task)
1195 task->tk_flags &= ~RPC_CALL_MAJORSEEN; 1201 task->tk_flags &= ~RPC_CALL_MAJORSEEN;
1196 } 1202 }
1197 1203
1198 if (task->tk_status < 12) {
1199 if (!RPC_IS_SOFT(task)) {
1200 task->tk_action = call_bind;
1201 clnt->cl_stats->rpcretrans++;
1202 goto out_retry;
1203 }
1204 dprintk("RPC: %s: too small RPC reply size (%d bytes)\n",
1205 clnt->cl_protname, task->tk_status);
1206 task->tk_action = call_timeout;
1207 goto out_retry;
1208 }
1209
1210 /* 1204 /*
1211 * Ensure that we see all writes made by xprt_complete_rqst() 1205 * Ensure that we see all writes made by xprt_complete_rqst()
1212 * before it changed req->rq_received. 1206 * before it changed req->rq_received.
@@ -1218,6 +1212,18 @@ call_decode(struct rpc_task *task)
1218 WARN_ON(memcmp(&req->rq_rcv_buf, &req->rq_private_buf, 1212 WARN_ON(memcmp(&req->rq_rcv_buf, &req->rq_private_buf,
1219 sizeof(req->rq_rcv_buf)) != 0); 1213 sizeof(req->rq_rcv_buf)) != 0);
1220 1214
1215 if (req->rq_rcv_buf.len < 12) {
1216 if (!RPC_IS_SOFT(task)) {
1217 task->tk_action = call_bind;
1218 clnt->cl_stats->rpcretrans++;
1219 goto out_retry;
1220 }
1221 dprintk("RPC: %s: too small RPC reply size (%d bytes)\n",
1222 clnt->cl_protname, task->tk_status);
1223 task->tk_action = call_timeout;
1224 goto out_retry;
1225 }
1226
1221 /* Verify the RPC header */ 1227 /* Verify the RPC header */
1222 p = call_verify(task); 1228 p = call_verify(task);
1223 if (IS_ERR(p)) { 1229 if (IS_ERR(p)) {
@@ -1236,10 +1242,14 @@ call_decode(struct rpc_task *task)
1236 task->tk_status); 1242 task->tk_status);
1237 return; 1243 return;
1238out_retry: 1244out_retry:
1239 req->rq_received = req->rq_private_buf.len = 0;
1240 task->tk_status = 0; 1245 task->tk_status = 0;
1241 if (task->tk_client->cl_discrtry) 1246 /* Note: call_verify() may have freed the RPC slot */
1242 xprt_force_disconnect(task->tk_xprt); 1247 if (task->tk_rqstp == req) {
1248 req->rq_received = req->rq_rcv_buf.len = 0;
1249 if (task->tk_client->cl_discrtry)
1250 xprt_conditional_disconnect(task->tk_xprt,
1251 req->rq_connect_cookie);
1252 }
1243} 1253}
1244 1254
1245/* 1255/*
@@ -1531,7 +1541,7 @@ void rpc_show_tasks(void)
1531 proc = -1; 1541 proc = -1;
1532 1542
1533 if (RPC_IS_QUEUED(t)) 1543 if (RPC_IS_QUEUED(t))
1534 rpc_waitq = rpc_qname(t->u.tk_wait.rpc_waitq); 1544 rpc_waitq = rpc_qname(t->tk_waitqueue);
1535 1545
1536 printk("%5u %04d %04x %6d %8p %6d %8p %8ld %8s %8p %8p\n", 1546 printk("%5u %04d %04x %6d %8p %6d %8p %8ld %8s %8p %8p\n",
1537 t->tk_pid, proc, 1547 t->tk_pid, proc,
diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c
index 56aa018dce3a..0517967a68bf 100644
--- a/net/sunrpc/rpcb_clnt.c
+++ b/net/sunrpc/rpcb_clnt.c
@@ -298,7 +298,7 @@ void rpcb_getport_async(struct rpc_task *task)
298 298
299 /* Put self on queue before sending rpcbind request, in case 299 /* Put self on queue before sending rpcbind request, in case
300 * rpcb_getport_done completes before we return from rpc_run_task */ 300 * rpcb_getport_done completes before we return from rpc_run_task */
301 rpc_sleep_on(&xprt->binding, task, NULL, NULL); 301 rpc_sleep_on(&xprt->binding, task, NULL);
302 302
303 /* Someone else may have bound if we slept */ 303 /* Someone else may have bound if we slept */
304 if (xprt_bound(xprt)) { 304 if (xprt_bound(xprt)) {
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index 4c669121e607..6eab9bf94baf 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -38,9 +38,9 @@ static struct kmem_cache *rpc_buffer_slabp __read_mostly;
38static mempool_t *rpc_task_mempool __read_mostly; 38static mempool_t *rpc_task_mempool __read_mostly;
39static mempool_t *rpc_buffer_mempool __read_mostly; 39static mempool_t *rpc_buffer_mempool __read_mostly;
40 40
41static void __rpc_default_timer(struct rpc_task *task);
42static void rpc_async_schedule(struct work_struct *); 41static void rpc_async_schedule(struct work_struct *);
43static void rpc_release_task(struct rpc_task *task); 42static void rpc_release_task(struct rpc_task *task);
43static void __rpc_queue_timer_fn(unsigned long ptr);
44 44
45/* 45/*
46 * RPC tasks sit here while waiting for conditions to improve. 46 * RPC tasks sit here while waiting for conditions to improve.
@@ -57,41 +57,30 @@ struct workqueue_struct *rpciod_workqueue;
57 * queue->lock and bh_disabled in order to avoid races within 57 * queue->lock and bh_disabled in order to avoid races within
58 * rpc_run_timer(). 58 * rpc_run_timer().
59 */ 59 */
60static inline void 60static void
61__rpc_disable_timer(struct rpc_task *task) 61__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
62{ 62{
63 if (task->tk_timeout == 0)
64 return;
63 dprintk("RPC: %5u disabling timer\n", task->tk_pid); 65 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
64 task->tk_timeout_fn = NULL;
65 task->tk_timeout = 0; 66 task->tk_timeout = 0;
67 list_del(&task->u.tk_wait.timer_list);
68 if (list_empty(&queue->timer_list.list))
69 del_timer(&queue->timer_list.timer);
66} 70}
67 71
68/* 72static void
69 * Run a timeout function. 73rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
70 * We use the callback in order to allow __rpc_wake_up_task()
71 * and friends to disable the timer synchronously on SMP systems
72 * without calling del_timer_sync(). The latter could cause a
73 * deadlock if called while we're holding spinlocks...
74 */
75static void rpc_run_timer(struct rpc_task *task)
76{ 74{
77 void (*callback)(struct rpc_task *); 75 queue->timer_list.expires = expires;
78 76 mod_timer(&queue->timer_list.timer, expires);
79 callback = task->tk_timeout_fn;
80 task->tk_timeout_fn = NULL;
81 if (callback && RPC_IS_QUEUED(task)) {
82 dprintk("RPC: %5u running timer\n", task->tk_pid);
83 callback(task);
84 }
85 smp_mb__before_clear_bit();
86 clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate);
87 smp_mb__after_clear_bit();
88} 77}
89 78
90/* 79/*
91 * Set up a timer for the current task. 80 * Set up a timer for the current task.
92 */ 81 */
93static inline void 82static void
94__rpc_add_timer(struct rpc_task *task, rpc_action timer) 83__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
95{ 84{
96 if (!task->tk_timeout) 85 if (!task->tk_timeout)
97 return; 86 return;
@@ -99,27 +88,10 @@ __rpc_add_timer(struct rpc_task *task, rpc_action timer)
99 dprintk("RPC: %5u setting alarm for %lu ms\n", 88 dprintk("RPC: %5u setting alarm for %lu ms\n",
100 task->tk_pid, task->tk_timeout * 1000 / HZ); 89 task->tk_pid, task->tk_timeout * 1000 / HZ);
101 90
102 if (timer) 91 task->u.tk_wait.expires = jiffies + task->tk_timeout;
103 task->tk_timeout_fn = timer; 92 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
104 else 93 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
105 task->tk_timeout_fn = __rpc_default_timer; 94 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
106 set_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate);
107 mod_timer(&task->tk_timer, jiffies + task->tk_timeout);
108}
109
110/*
111 * Delete any timer for the current task. Because we use del_timer_sync(),
112 * this function should never be called while holding queue->lock.
113 */
114static void
115rpc_delete_timer(struct rpc_task *task)
116{
117 if (RPC_IS_QUEUED(task))
118 return;
119 if (test_and_clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate)) {
120 del_singleshot_timer_sync(&task->tk_timer);
121 dprintk("RPC: %5u deleting timer\n", task->tk_pid);
122 }
123} 95}
124 96
125/* 97/*
@@ -161,7 +133,7 @@ static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *
161 list_add(&task->u.tk_wait.list, &queue->tasks[0]); 133 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
162 else 134 else
163 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); 135 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
164 task->u.tk_wait.rpc_waitq = queue; 136 task->tk_waitqueue = queue;
165 queue->qlen++; 137 queue->qlen++;
166 rpc_set_queued(task); 138 rpc_set_queued(task);
167 139
@@ -181,22 +153,18 @@ static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
181 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list); 153 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
182 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links); 154 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
183 } 155 }
184 list_del(&task->u.tk_wait.list);
185} 156}
186 157
187/* 158/*
188 * Remove request from queue. 159 * Remove request from queue.
189 * Note: must be called with spin lock held. 160 * Note: must be called with spin lock held.
190 */ 161 */
191static void __rpc_remove_wait_queue(struct rpc_task *task) 162static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
192{ 163{
193 struct rpc_wait_queue *queue; 164 __rpc_disable_timer(queue, task);
194 queue = task->u.tk_wait.rpc_waitq;
195
196 if (RPC_IS_PRIORITY(queue)) 165 if (RPC_IS_PRIORITY(queue))
197 __rpc_remove_wait_queue_priority(task); 166 __rpc_remove_wait_queue_priority(task);
198 else 167 list_del(&task->u.tk_wait.list);
199 list_del(&task->u.tk_wait.list);
200 queue->qlen--; 168 queue->qlen--;
201 dprintk("RPC: %5u removed from queue %p \"%s\"\n", 169 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
202 task->tk_pid, queue, rpc_qname(queue)); 170 task->tk_pid, queue, rpc_qname(queue));
@@ -229,6 +197,9 @@ static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const c
229 INIT_LIST_HEAD(&queue->tasks[i]); 197 INIT_LIST_HEAD(&queue->tasks[i]);
230 queue->maxpriority = nr_queues - 1; 198 queue->maxpriority = nr_queues - 1;
231 rpc_reset_waitqueue_priority(queue); 199 rpc_reset_waitqueue_priority(queue);
200 queue->qlen = 0;
201 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
202 INIT_LIST_HEAD(&queue->timer_list.list);
232#ifdef RPC_DEBUG 203#ifdef RPC_DEBUG
233 queue->name = qname; 204 queue->name = qname;
234#endif 205#endif
@@ -245,6 +216,12 @@ void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
245} 216}
246EXPORT_SYMBOL_GPL(rpc_init_wait_queue); 217EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
247 218
219void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
220{
221 del_timer_sync(&queue->timer_list.timer);
222}
223EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
224
248static int rpc_wait_bit_killable(void *word) 225static int rpc_wait_bit_killable(void *word)
249{ 226{
250 if (fatal_signal_pending(current)) 227 if (fatal_signal_pending(current))
@@ -313,7 +290,6 @@ EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
313 */ 290 */
314static void rpc_make_runnable(struct rpc_task *task) 291static void rpc_make_runnable(struct rpc_task *task)
315{ 292{
316 BUG_ON(task->tk_timeout_fn);
317 rpc_clear_queued(task); 293 rpc_clear_queued(task);
318 if (rpc_test_and_set_running(task)) 294 if (rpc_test_and_set_running(task))
319 return; 295 return;
@@ -326,7 +302,7 @@ static void rpc_make_runnable(struct rpc_task *task)
326 int status; 302 int status;
327 303
328 INIT_WORK(&task->u.tk_work, rpc_async_schedule); 304 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
329 status = queue_work(task->tk_workqueue, &task->u.tk_work); 305 status = queue_work(rpciod_workqueue, &task->u.tk_work);
330 if (status < 0) { 306 if (status < 0) {
331 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status); 307 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
332 task->tk_status = status; 308 task->tk_status = status;
@@ -343,7 +319,7 @@ static void rpc_make_runnable(struct rpc_task *task)
343 * as it's on a wait queue. 319 * as it's on a wait queue.
344 */ 320 */
345static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 321static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
346 rpc_action action, rpc_action timer) 322 rpc_action action)
347{ 323{
348 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n", 324 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
349 task->tk_pid, rpc_qname(q), jiffies); 325 task->tk_pid, rpc_qname(q), jiffies);
@@ -357,11 +333,11 @@ static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
357 333
358 BUG_ON(task->tk_callback != NULL); 334 BUG_ON(task->tk_callback != NULL);
359 task->tk_callback = action; 335 task->tk_callback = action;
360 __rpc_add_timer(task, timer); 336 __rpc_add_timer(q, task);
361} 337}
362 338
363void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 339void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
364 rpc_action action, rpc_action timer) 340 rpc_action action)
365{ 341{
366 /* Mark the task as being activated if so needed */ 342 /* Mark the task as being activated if so needed */
367 rpc_set_active(task); 343 rpc_set_active(task);
@@ -370,18 +346,19 @@ void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
370 * Protect the queue operations. 346 * Protect the queue operations.
371 */ 347 */
372 spin_lock_bh(&q->lock); 348 spin_lock_bh(&q->lock);
373 __rpc_sleep_on(q, task, action, timer); 349 __rpc_sleep_on(q, task, action);
374 spin_unlock_bh(&q->lock); 350 spin_unlock_bh(&q->lock);
375} 351}
376EXPORT_SYMBOL_GPL(rpc_sleep_on); 352EXPORT_SYMBOL_GPL(rpc_sleep_on);
377 353
378/** 354/**
379 * __rpc_do_wake_up_task - wake up a single rpc_task 355 * __rpc_do_wake_up_task - wake up a single rpc_task
356 * @queue: wait queue
380 * @task: task to be woken up 357 * @task: task to be woken up
381 * 358 *
382 * Caller must hold queue->lock, and have cleared the task queued flag. 359 * Caller must hold queue->lock, and have cleared the task queued flag.
383 */ 360 */
384static void __rpc_do_wake_up_task(struct rpc_task *task) 361static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
385{ 362{
386 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n", 363 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
387 task->tk_pid, jiffies); 364 task->tk_pid, jiffies);
@@ -395,8 +372,7 @@ static void __rpc_do_wake_up_task(struct rpc_task *task)
395 return; 372 return;
396 } 373 }
397 374
398 __rpc_disable_timer(task); 375 __rpc_remove_wait_queue(queue, task);
399 __rpc_remove_wait_queue(task);
400 376
401 rpc_make_runnable(task); 377 rpc_make_runnable(task);
402 378
@@ -404,48 +380,32 @@ static void __rpc_do_wake_up_task(struct rpc_task *task)
404} 380}
405 381
406/* 382/*
407 * Wake up the specified task 383 * Wake up a queued task while the queue lock is being held
408 */ 384 */
409static void __rpc_wake_up_task(struct rpc_task *task) 385static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
410{ 386{
411 if (rpc_start_wakeup(task)) { 387 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
412 if (RPC_IS_QUEUED(task)) 388 __rpc_do_wake_up_task(queue, task);
413 __rpc_do_wake_up_task(task);
414 rpc_finish_wakeup(task);
415 }
416} 389}
417 390
418/* 391/*
419 * Default timeout handler if none specified by user 392 * Wake up a task on a specific queue
420 */ 393 */
421static void 394void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
422__rpc_default_timer(struct rpc_task *task)
423{ 395{
424 dprintk("RPC: %5u timeout (default timer)\n", task->tk_pid); 396 spin_lock_bh(&queue->lock);
425 task->tk_status = -ETIMEDOUT; 397 rpc_wake_up_task_queue_locked(queue, task);
426 rpc_wake_up_task(task); 398 spin_unlock_bh(&queue->lock);
427} 399}
400EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
428 401
429/* 402/*
430 * Wake up the specified task 403 * Wake up the specified task
431 */ 404 */
432void rpc_wake_up_task(struct rpc_task *task) 405static void rpc_wake_up_task(struct rpc_task *task)
433{ 406{
434 rcu_read_lock_bh(); 407 rpc_wake_up_queued_task(task->tk_waitqueue, task);
435 if (rpc_start_wakeup(task)) {
436 if (RPC_IS_QUEUED(task)) {
437 struct rpc_wait_queue *queue = task->u.tk_wait.rpc_waitq;
438
439 /* Note: we're already in a bh-safe context */
440 spin_lock(&queue->lock);
441 __rpc_do_wake_up_task(task);
442 spin_unlock(&queue->lock);
443 }
444 rpc_finish_wakeup(task);
445 }
446 rcu_read_unlock_bh();
447} 408}
448EXPORT_SYMBOL_GPL(rpc_wake_up_task);
449 409
450/* 410/*
451 * Wake up the next task on a priority queue. 411 * Wake up the next task on a priority queue.
@@ -495,7 +455,7 @@ new_queue:
495new_owner: 455new_owner:
496 rpc_set_waitqueue_owner(queue, task->tk_owner); 456 rpc_set_waitqueue_owner(queue, task->tk_owner);
497out: 457out:
498 __rpc_wake_up_task(task); 458 rpc_wake_up_task_queue_locked(queue, task);
499 return task; 459 return task;
500} 460}
501 461
@@ -508,16 +468,14 @@ struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
508 468
509 dprintk("RPC: wake_up_next(%p \"%s\")\n", 469 dprintk("RPC: wake_up_next(%p \"%s\")\n",
510 queue, rpc_qname(queue)); 470 queue, rpc_qname(queue));
511 rcu_read_lock_bh(); 471 spin_lock_bh(&queue->lock);
512 spin_lock(&queue->lock);
513 if (RPC_IS_PRIORITY(queue)) 472 if (RPC_IS_PRIORITY(queue))
514 task = __rpc_wake_up_next_priority(queue); 473 task = __rpc_wake_up_next_priority(queue);
515 else { 474 else {
516 task_for_first(task, &queue->tasks[0]) 475 task_for_first(task, &queue->tasks[0])
517 __rpc_wake_up_task(task); 476 rpc_wake_up_task_queue_locked(queue, task);
518 } 477 }
519 spin_unlock(&queue->lock); 478 spin_unlock_bh(&queue->lock);
520 rcu_read_unlock_bh();
521 479
522 return task; 480 return task;
523} 481}
@@ -534,18 +492,16 @@ void rpc_wake_up(struct rpc_wait_queue *queue)
534 struct rpc_task *task, *next; 492 struct rpc_task *task, *next;
535 struct list_head *head; 493 struct list_head *head;
536 494
537 rcu_read_lock_bh(); 495 spin_lock_bh(&queue->lock);
538 spin_lock(&queue->lock);
539 head = &queue->tasks[queue->maxpriority]; 496 head = &queue->tasks[queue->maxpriority];
540 for (;;) { 497 for (;;) {
541 list_for_each_entry_safe(task, next, head, u.tk_wait.list) 498 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
542 __rpc_wake_up_task(task); 499 rpc_wake_up_task_queue_locked(queue, task);
543 if (head == &queue->tasks[0]) 500 if (head == &queue->tasks[0])
544 break; 501 break;
545 head--; 502 head--;
546 } 503 }
547 spin_unlock(&queue->lock); 504 spin_unlock_bh(&queue->lock);
548 rcu_read_unlock_bh();
549} 505}
550EXPORT_SYMBOL_GPL(rpc_wake_up); 506EXPORT_SYMBOL_GPL(rpc_wake_up);
551 507
@@ -561,26 +517,48 @@ void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
561 struct rpc_task *task, *next; 517 struct rpc_task *task, *next;
562 struct list_head *head; 518 struct list_head *head;
563 519
564 rcu_read_lock_bh(); 520 spin_lock_bh(&queue->lock);
565 spin_lock(&queue->lock);
566 head = &queue->tasks[queue->maxpriority]; 521 head = &queue->tasks[queue->maxpriority];
567 for (;;) { 522 for (;;) {
568 list_for_each_entry_safe(task, next, head, u.tk_wait.list) { 523 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
569 task->tk_status = status; 524 task->tk_status = status;
570 __rpc_wake_up_task(task); 525 rpc_wake_up_task_queue_locked(queue, task);
571 } 526 }
572 if (head == &queue->tasks[0]) 527 if (head == &queue->tasks[0])
573 break; 528 break;
574 head--; 529 head--;
575 } 530 }
576 spin_unlock(&queue->lock); 531 spin_unlock_bh(&queue->lock);
577 rcu_read_unlock_bh();
578} 532}
579EXPORT_SYMBOL_GPL(rpc_wake_up_status); 533EXPORT_SYMBOL_GPL(rpc_wake_up_status);
580 534
535static void __rpc_queue_timer_fn(unsigned long ptr)
536{
537 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
538 struct rpc_task *task, *n;
539 unsigned long expires, now, timeo;
540
541 spin_lock(&queue->lock);
542 expires = now = jiffies;
543 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
544 timeo = task->u.tk_wait.expires;
545 if (time_after_eq(now, timeo)) {
546 dprintk("RPC: %5u timeout\n", task->tk_pid);
547 task->tk_status = -ETIMEDOUT;
548 rpc_wake_up_task_queue_locked(queue, task);
549 continue;
550 }
551 if (expires == now || time_after(expires, timeo))
552 expires = timeo;
553 }
554 if (!list_empty(&queue->timer_list.list))
555 rpc_set_queue_timer(queue, expires);
556 spin_unlock(&queue->lock);
557}
558
581static void __rpc_atrun(struct rpc_task *task) 559static void __rpc_atrun(struct rpc_task *task)
582{ 560{
583 rpc_wake_up_task(task); 561 task->tk_status = 0;
584} 562}
585 563
586/* 564/*
@@ -589,7 +567,7 @@ static void __rpc_atrun(struct rpc_task *task)
589void rpc_delay(struct rpc_task *task, unsigned long delay) 567void rpc_delay(struct rpc_task *task, unsigned long delay)
590{ 568{
591 task->tk_timeout = delay; 569 task->tk_timeout = delay;
592 rpc_sleep_on(&delay_queue, task, NULL, __rpc_atrun); 570 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
593} 571}
594EXPORT_SYMBOL_GPL(rpc_delay); 572EXPORT_SYMBOL_GPL(rpc_delay);
595 573
@@ -644,10 +622,6 @@ static void __rpc_execute(struct rpc_task *task)
644 BUG_ON(RPC_IS_QUEUED(task)); 622 BUG_ON(RPC_IS_QUEUED(task));
645 623
646 for (;;) { 624 for (;;) {
647 /*
648 * Garbage collection of pending timers...
649 */
650 rpc_delete_timer(task);
651 625
652 /* 626 /*
653 * Execute any pending callback. 627 * Execute any pending callback.
@@ -816,8 +790,6 @@ EXPORT_SYMBOL_GPL(rpc_free);
816static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) 790static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
817{ 791{
818 memset(task, 0, sizeof(*task)); 792 memset(task, 0, sizeof(*task));
819 setup_timer(&task->tk_timer, (void (*)(unsigned long))rpc_run_timer,
820 (unsigned long)task);
821 atomic_set(&task->tk_count, 1); 793 atomic_set(&task->tk_count, 1);
822 task->tk_flags = task_setup_data->flags; 794 task->tk_flags = task_setup_data->flags;
823 task->tk_ops = task_setup_data->callback_ops; 795 task->tk_ops = task_setup_data->callback_ops;
@@ -832,7 +804,7 @@ static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *ta
832 task->tk_owner = current->tgid; 804 task->tk_owner = current->tgid;
833 805
834 /* Initialize workqueue for async tasks */ 806 /* Initialize workqueue for async tasks */
835 task->tk_workqueue = rpciod_workqueue; 807 task->tk_workqueue = task_setup_data->workqueue;
836 808
837 task->tk_client = task_setup_data->rpc_client; 809 task->tk_client = task_setup_data->rpc_client;
838 if (task->tk_client != NULL) { 810 if (task->tk_client != NULL) {
@@ -845,12 +817,11 @@ static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *ta
845 task->tk_action = rpc_prepare_task; 817 task->tk_action = rpc_prepare_task;
846 818
847 if (task_setup_data->rpc_message != NULL) { 819 if (task_setup_data->rpc_message != NULL) {
848 memcpy(&task->tk_msg, task_setup_data->rpc_message, sizeof(task->tk_msg)); 820 task->tk_msg.rpc_proc = task_setup_data->rpc_message->rpc_proc;
821 task->tk_msg.rpc_argp = task_setup_data->rpc_message->rpc_argp;
822 task->tk_msg.rpc_resp = task_setup_data->rpc_message->rpc_resp;
849 /* Bind the user cred */ 823 /* Bind the user cred */
850 if (task->tk_msg.rpc_cred != NULL) 824 rpcauth_bindcred(task, task_setup_data->rpc_message->rpc_cred, task_setup_data->flags);
851 rpcauth_holdcred(task);
852 else
853 rpcauth_bindcred(task);
854 if (task->tk_action == NULL) 825 if (task->tk_action == NULL)
855 rpc_call_start(task); 826 rpc_call_start(task);
856 } 827 }
@@ -868,13 +839,6 @@ rpc_alloc_task(void)
868 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS); 839 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
869} 840}
870 841
871static void rpc_free_task(struct rcu_head *rcu)
872{
873 struct rpc_task *task = container_of(rcu, struct rpc_task, u.tk_rcu);
874 dprintk("RPC: %5u freeing task\n", task->tk_pid);
875 mempool_free(task, rpc_task_mempool);
876}
877
878/* 842/*
879 * Create a new task for the specified client. 843 * Create a new task for the specified client.
880 */ 844 */
@@ -898,12 +862,25 @@ out:
898 return task; 862 return task;
899} 863}
900 864
901 865static void rpc_free_task(struct rpc_task *task)
902void rpc_put_task(struct rpc_task *task)
903{ 866{
904 const struct rpc_call_ops *tk_ops = task->tk_ops; 867 const struct rpc_call_ops *tk_ops = task->tk_ops;
905 void *calldata = task->tk_calldata; 868 void *calldata = task->tk_calldata;
906 869
870 if (task->tk_flags & RPC_TASK_DYNAMIC) {
871 dprintk("RPC: %5u freeing task\n", task->tk_pid);
872 mempool_free(task, rpc_task_mempool);
873 }
874 rpc_release_calldata(tk_ops, calldata);
875}
876
877static void rpc_async_release(struct work_struct *work)
878{
879 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
880}
881
882void rpc_put_task(struct rpc_task *task)
883{
907 if (!atomic_dec_and_test(&task->tk_count)) 884 if (!atomic_dec_and_test(&task->tk_count))
908 return; 885 return;
909 /* Release resources */ 886 /* Release resources */
@@ -915,9 +892,11 @@ void rpc_put_task(struct rpc_task *task)
915 rpc_release_client(task->tk_client); 892 rpc_release_client(task->tk_client);
916 task->tk_client = NULL; 893 task->tk_client = NULL;
917 } 894 }
918 if (task->tk_flags & RPC_TASK_DYNAMIC) 895 if (task->tk_workqueue != NULL) {
919 call_rcu_bh(&task->u.tk_rcu, rpc_free_task); 896 INIT_WORK(&task->u.tk_work, rpc_async_release);
920 rpc_release_calldata(tk_ops, calldata); 897 queue_work(task->tk_workqueue, &task->u.tk_work);
898 } else
899 rpc_free_task(task);
921} 900}
922EXPORT_SYMBOL_GPL(rpc_put_task); 901EXPORT_SYMBOL_GPL(rpc_put_task);
923 902
@@ -937,9 +916,6 @@ static void rpc_release_task(struct rpc_task *task)
937 } 916 }
938 BUG_ON (RPC_IS_QUEUED(task)); 917 BUG_ON (RPC_IS_QUEUED(task));
939 918
940 /* Synchronously delete any running timer */
941 rpc_delete_timer(task);
942
943#ifdef RPC_DEBUG 919#ifdef RPC_DEBUG
944 task->tk_magic = 0; 920 task->tk_magic = 0;
945#endif 921#endif
@@ -1029,11 +1005,20 @@ rpc_destroy_mempool(void)
1029 kmem_cache_destroy(rpc_task_slabp); 1005 kmem_cache_destroy(rpc_task_slabp);
1030 if (rpc_buffer_slabp) 1006 if (rpc_buffer_slabp)
1031 kmem_cache_destroy(rpc_buffer_slabp); 1007 kmem_cache_destroy(rpc_buffer_slabp);
1008 rpc_destroy_wait_queue(&delay_queue);
1032} 1009}
1033 1010
1034int 1011int
1035rpc_init_mempool(void) 1012rpc_init_mempool(void)
1036{ 1013{
1014 /*
1015 * The following is not strictly a mempool initialisation,
1016 * but there is no harm in doing it here
1017 */
1018 rpc_init_wait_queue(&delay_queue, "delayq");
1019 if (!rpciod_start())
1020 goto err_nomem;
1021
1037 rpc_task_slabp = kmem_cache_create("rpc_tasks", 1022 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1038 sizeof(struct rpc_task), 1023 sizeof(struct rpc_task),
1039 0, SLAB_HWCACHE_ALIGN, 1024 0, SLAB_HWCACHE_ALIGN,
@@ -1054,13 +1039,6 @@ rpc_init_mempool(void)
1054 rpc_buffer_slabp); 1039 rpc_buffer_slabp);
1055 if (!rpc_buffer_mempool) 1040 if (!rpc_buffer_mempool)
1056 goto err_nomem; 1041 goto err_nomem;
1057 if (!rpciod_start())
1058 goto err_nomem;
1059 /*
1060 * The following is not strictly a mempool initialisation,
1061 * but there is no harm in doing it here
1062 */
1063 rpc_init_wait_queue(&delay_queue, "delayq");
1064 return 0; 1042 return 0;
1065err_nomem: 1043err_nomem:
1066 rpc_destroy_mempool(); 1044 rpc_destroy_mempool();
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index d5553b8179f9..75d748eee0eb 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -188,9 +188,9 @@ out_sleep:
188 task->tk_timeout = 0; 188 task->tk_timeout = 0;
189 task->tk_status = -EAGAIN; 189 task->tk_status = -EAGAIN;
190 if (req && req->rq_ntrans) 190 if (req && req->rq_ntrans)
191 rpc_sleep_on(&xprt->resend, task, NULL, NULL); 191 rpc_sleep_on(&xprt->resend, task, NULL);
192 else 192 else
193 rpc_sleep_on(&xprt->sending, task, NULL, NULL); 193 rpc_sleep_on(&xprt->sending, task, NULL);
194 return 0; 194 return 0;
195} 195}
196EXPORT_SYMBOL_GPL(xprt_reserve_xprt); 196EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
@@ -238,9 +238,9 @@ out_sleep:
238 task->tk_timeout = 0; 238 task->tk_timeout = 0;
239 task->tk_status = -EAGAIN; 239 task->tk_status = -EAGAIN;
240 if (req && req->rq_ntrans) 240 if (req && req->rq_ntrans)
241 rpc_sleep_on(&xprt->resend, task, NULL, NULL); 241 rpc_sleep_on(&xprt->resend, task, NULL);
242 else 242 else
243 rpc_sleep_on(&xprt->sending, task, NULL, NULL); 243 rpc_sleep_on(&xprt->sending, task, NULL);
244 return 0; 244 return 0;
245} 245}
246EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong); 246EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
@@ -447,13 +447,13 @@ EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
447 * @task: task to be put to sleep 447 * @task: task to be put to sleep
448 * 448 *
449 */ 449 */
450void xprt_wait_for_buffer_space(struct rpc_task *task) 450void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
451{ 451{
452 struct rpc_rqst *req = task->tk_rqstp; 452 struct rpc_rqst *req = task->tk_rqstp;
453 struct rpc_xprt *xprt = req->rq_xprt; 453 struct rpc_xprt *xprt = req->rq_xprt;
454 454
455 task->tk_timeout = req->rq_timeout; 455 task->tk_timeout = req->rq_timeout;
456 rpc_sleep_on(&xprt->pending, task, NULL, NULL); 456 rpc_sleep_on(&xprt->pending, task, action);
457} 457}
458EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space); 458EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
459 459
@@ -472,7 +472,7 @@ void xprt_write_space(struct rpc_xprt *xprt)
472 if (xprt->snd_task) { 472 if (xprt->snd_task) {
473 dprintk("RPC: write space: waking waiting task on " 473 dprintk("RPC: write space: waking waiting task on "
474 "xprt %p\n", xprt); 474 "xprt %p\n", xprt);
475 rpc_wake_up_task(xprt->snd_task); 475 rpc_wake_up_queued_task(&xprt->pending, xprt->snd_task);
476 } 476 }
477 spin_unlock_bh(&xprt->transport_lock); 477 spin_unlock_bh(&xprt->transport_lock);
478} 478}
@@ -602,11 +602,37 @@ void xprt_force_disconnect(struct rpc_xprt *xprt)
602 /* Try to schedule an autoclose RPC call */ 602 /* Try to schedule an autoclose RPC call */
603 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) 603 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
604 queue_work(rpciod_workqueue, &xprt->task_cleanup); 604 queue_work(rpciod_workqueue, &xprt->task_cleanup);
605 else if (xprt->snd_task != NULL) 605 xprt_wake_pending_tasks(xprt, -ENOTCONN);
606 rpc_wake_up_task(xprt->snd_task); 606 spin_unlock_bh(&xprt->transport_lock);
607}
608
609/**
610 * xprt_conditional_disconnect - force a transport to disconnect
611 * @xprt: transport to disconnect
612 * @cookie: 'connection cookie'
613 *
614 * This attempts to break the connection if and only if 'cookie' matches
615 * the current transport 'connection cookie'. It ensures that we don't
616 * try to break the connection more than once when we need to retransmit
617 * a batch of RPC requests.
618 *
619 */
620void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
621{
622 /* Don't race with the test_bit() in xprt_clear_locked() */
623 spin_lock_bh(&xprt->transport_lock);
624 if (cookie != xprt->connect_cookie)
625 goto out;
626 if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
627 goto out;
628 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
629 /* Try to schedule an autoclose RPC call */
630 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
631 queue_work(rpciod_workqueue, &xprt->task_cleanup);
632 xprt_wake_pending_tasks(xprt, -ENOTCONN);
633out:
607 spin_unlock_bh(&xprt->transport_lock); 634 spin_unlock_bh(&xprt->transport_lock);
608} 635}
609EXPORT_SYMBOL_GPL(xprt_force_disconnect);
610 636
611static void 637static void
612xprt_init_autodisconnect(unsigned long data) 638xprt_init_autodisconnect(unsigned long data)
@@ -653,7 +679,7 @@ void xprt_connect(struct rpc_task *task)
653 task->tk_rqstp->rq_bytes_sent = 0; 679 task->tk_rqstp->rq_bytes_sent = 0;
654 680
655 task->tk_timeout = xprt->connect_timeout; 681 task->tk_timeout = xprt->connect_timeout;
656 rpc_sleep_on(&xprt->pending, task, xprt_connect_status, NULL); 682 rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
657 xprt->stat.connect_start = jiffies; 683 xprt->stat.connect_start = jiffies;
658 xprt->ops->connect(task); 684 xprt->ops->connect(task);
659 } 685 }
@@ -749,18 +775,20 @@ EXPORT_SYMBOL_GPL(xprt_update_rtt);
749void xprt_complete_rqst(struct rpc_task *task, int copied) 775void xprt_complete_rqst(struct rpc_task *task, int copied)
750{ 776{
751 struct rpc_rqst *req = task->tk_rqstp; 777 struct rpc_rqst *req = task->tk_rqstp;
778 struct rpc_xprt *xprt = req->rq_xprt;
752 779
753 dprintk("RPC: %5u xid %08x complete (%d bytes received)\n", 780 dprintk("RPC: %5u xid %08x complete (%d bytes received)\n",
754 task->tk_pid, ntohl(req->rq_xid), copied); 781 task->tk_pid, ntohl(req->rq_xid), copied);
755 782
756 task->tk_xprt->stat.recvs++; 783 xprt->stat.recvs++;
757 task->tk_rtt = (long)jiffies - req->rq_xtime; 784 task->tk_rtt = (long)jiffies - req->rq_xtime;
758 785
759 list_del_init(&req->rq_list); 786 list_del_init(&req->rq_list);
787 req->rq_private_buf.len = copied;
760 /* Ensure all writes are done before we update req->rq_received */ 788 /* Ensure all writes are done before we update req->rq_received */
761 smp_wmb(); 789 smp_wmb();
762 req->rq_received = req->rq_private_buf.len = copied; 790 req->rq_received = copied;
763 rpc_wake_up_task(task); 791 rpc_wake_up_queued_task(&xprt->pending, task);
764} 792}
765EXPORT_SYMBOL_GPL(xprt_complete_rqst); 793EXPORT_SYMBOL_GPL(xprt_complete_rqst);
766 794
@@ -769,17 +797,17 @@ static void xprt_timer(struct rpc_task *task)
769 struct rpc_rqst *req = task->tk_rqstp; 797 struct rpc_rqst *req = task->tk_rqstp;
770 struct rpc_xprt *xprt = req->rq_xprt; 798 struct rpc_xprt *xprt = req->rq_xprt;
771 799
800 if (task->tk_status != -ETIMEDOUT)
801 return;
772 dprintk("RPC: %5u xprt_timer\n", task->tk_pid); 802 dprintk("RPC: %5u xprt_timer\n", task->tk_pid);
773 803
774 spin_lock(&xprt->transport_lock); 804 spin_lock_bh(&xprt->transport_lock);
775 if (!req->rq_received) { 805 if (!req->rq_received) {
776 if (xprt->ops->timer) 806 if (xprt->ops->timer)
777 xprt->ops->timer(task); 807 xprt->ops->timer(task);
778 task->tk_status = -ETIMEDOUT; 808 } else
779 } 809 task->tk_status = 0;
780 task->tk_timeout = 0; 810 spin_unlock_bh(&xprt->transport_lock);
781 rpc_wake_up_task(task);
782 spin_unlock(&xprt->transport_lock);
783} 811}
784 812
785/** 813/**
@@ -849,6 +877,7 @@ void xprt_transmit(struct rpc_task *task)
849 } else if (!req->rq_bytes_sent) 877 } else if (!req->rq_bytes_sent)
850 return; 878 return;
851 879
880 req->rq_connect_cookie = xprt->connect_cookie;
852 status = xprt->ops->send_request(task); 881 status = xprt->ops->send_request(task);
853 if (status == 0) { 882 if (status == 0) {
854 dprintk("RPC: %5u xmit complete\n", task->tk_pid); 883 dprintk("RPC: %5u xmit complete\n", task->tk_pid);
@@ -864,7 +893,7 @@ void xprt_transmit(struct rpc_task *task)
864 if (!xprt_connected(xprt)) 893 if (!xprt_connected(xprt))
865 task->tk_status = -ENOTCONN; 894 task->tk_status = -ENOTCONN;
866 else if (!req->rq_received) 895 else if (!req->rq_received)
867 rpc_sleep_on(&xprt->pending, task, NULL, xprt_timer); 896 rpc_sleep_on(&xprt->pending, task, xprt_timer);
868 spin_unlock_bh(&xprt->transport_lock); 897 spin_unlock_bh(&xprt->transport_lock);
869 return; 898 return;
870 } 899 }
@@ -875,7 +904,7 @@ void xprt_transmit(struct rpc_task *task)
875 */ 904 */
876 task->tk_status = status; 905 task->tk_status = status;
877 if (status == -ECONNREFUSED) 906 if (status == -ECONNREFUSED)
878 rpc_sleep_on(&xprt->sending, task, NULL, NULL); 907 rpc_sleep_on(&xprt->sending, task, NULL);
879} 908}
880 909
881static inline void do_xprt_reserve(struct rpc_task *task) 910static inline void do_xprt_reserve(struct rpc_task *task)
@@ -895,7 +924,7 @@ static inline void do_xprt_reserve(struct rpc_task *task)
895 dprintk("RPC: waiting for request slot\n"); 924 dprintk("RPC: waiting for request slot\n");
896 task->tk_status = -EAGAIN; 925 task->tk_status = -EAGAIN;
897 task->tk_timeout = 0; 926 task->tk_timeout = 0;
898 rpc_sleep_on(&xprt->backlog, task, NULL, NULL); 927 rpc_sleep_on(&xprt->backlog, task, NULL);
899} 928}
900 929
901/** 930/**
@@ -1052,6 +1081,11 @@ static void xprt_destroy(struct kref *kref)
1052 xprt->shutdown = 1; 1081 xprt->shutdown = 1;
1053 del_timer_sync(&xprt->timer); 1082 del_timer_sync(&xprt->timer);
1054 1083
1084 rpc_destroy_wait_queue(&xprt->binding);
1085 rpc_destroy_wait_queue(&xprt->pending);
1086 rpc_destroy_wait_queue(&xprt->sending);
1087 rpc_destroy_wait_queue(&xprt->resend);
1088 rpc_destroy_wait_queue(&xprt->backlog);
1055 /* 1089 /*
1056 * Tear down transport state and free the rpc_xprt 1090 * Tear down transport state and free the rpc_xprt
1057 */ 1091 */
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 613daf8c1ff7..ddbe981ab516 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -136,12 +136,6 @@ static ctl_table sunrpc_table[] = {
136#endif 136#endif
137 137
138/* 138/*
139 * How many times to try sending a request on a socket before waiting
140 * for the socket buffer to clear.
141 */
142#define XS_SENDMSG_RETRY (10U)
143
144/*
145 * Time out for an RPC UDP socket connect. UDP socket connects are 139 * Time out for an RPC UDP socket connect. UDP socket connects are
146 * synchronous, but we set a timeout anyway in case of resource 140 * synchronous, but we set a timeout anyway in case of resource
147 * exhaustion on the local host. 141 * exhaustion on the local host.
@@ -516,6 +510,14 @@ out:
516 return sent; 510 return sent;
517} 511}
518 512
513static void xs_nospace_callback(struct rpc_task *task)
514{
515 struct sock_xprt *transport = container_of(task->tk_rqstp->rq_xprt, struct sock_xprt, xprt);
516
517 transport->inet->sk_write_pending--;
518 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
519}
520
519/** 521/**
520 * xs_nospace - place task on wait queue if transmit was incomplete 522 * xs_nospace - place task on wait queue if transmit was incomplete
521 * @task: task to put to sleep 523 * @task: task to put to sleep
@@ -531,20 +533,27 @@ static void xs_nospace(struct rpc_task *task)
531 task->tk_pid, req->rq_slen - req->rq_bytes_sent, 533 task->tk_pid, req->rq_slen - req->rq_bytes_sent,
532 req->rq_slen); 534 req->rq_slen);
533 535
534 if (test_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags)) { 536 /* Protect against races with write_space */
535 /* Protect against races with write_space */ 537 spin_lock_bh(&xprt->transport_lock);
536 spin_lock_bh(&xprt->transport_lock); 538
537 539 /* Don't race with disconnect */
538 /* Don't race with disconnect */ 540 if (xprt_connected(xprt)) {
539 if (!xprt_connected(xprt)) 541 if (test_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags)) {
540 task->tk_status = -ENOTCONN; 542 /*
541 else if (test_bit(SOCK_NOSPACE, &transport->sock->flags)) 543 * Notify TCP that we're limited by the application
542 xprt_wait_for_buffer_space(task); 544 * window size
545 */
546 set_bit(SOCK_NOSPACE, &transport->sock->flags);
547 transport->inet->sk_write_pending++;
548 /* ...and wait for more buffer space */
549 xprt_wait_for_buffer_space(task, xs_nospace_callback);
550 }
551 } else {
552 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
553 task->tk_status = -ENOTCONN;
554 }
543 555
544 spin_unlock_bh(&xprt->transport_lock); 556 spin_unlock_bh(&xprt->transport_lock);
545 } else
546 /* Keep holding the socket if it is blocked */
547 rpc_delay(task, HZ>>4);
548} 557}
549 558
550/** 559/**
@@ -588,19 +597,20 @@ static int xs_udp_send_request(struct rpc_task *task)
588 } 597 }
589 598
590 switch (status) { 599 switch (status) {
600 case -EAGAIN:
601 xs_nospace(task);
602 break;
591 case -ENETUNREACH: 603 case -ENETUNREACH:
592 case -EPIPE: 604 case -EPIPE:
593 case -ECONNREFUSED: 605 case -ECONNREFUSED:
594 /* When the server has died, an ICMP port unreachable message 606 /* When the server has died, an ICMP port unreachable message
595 * prompts ECONNREFUSED. */ 607 * prompts ECONNREFUSED. */
596 break; 608 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
597 case -EAGAIN:
598 xs_nospace(task);
599 break; 609 break;
600 default: 610 default:
611 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
601 dprintk("RPC: sendmsg returned unrecognized error %d\n", 612 dprintk("RPC: sendmsg returned unrecognized error %d\n",
602 -status); 613 -status);
603 break;
604 } 614 }
605 615
606 return status; 616 return status;
@@ -650,7 +660,6 @@ static int xs_tcp_send_request(struct rpc_task *task)
650 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 660 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
651 struct xdr_buf *xdr = &req->rq_snd_buf; 661 struct xdr_buf *xdr = &req->rq_snd_buf;
652 int status; 662 int status;
653 unsigned int retry = 0;
654 663
655 xs_encode_tcp_record_marker(&req->rq_snd_buf); 664 xs_encode_tcp_record_marker(&req->rq_snd_buf);
656 665
@@ -681,9 +690,10 @@ static int xs_tcp_send_request(struct rpc_task *task)
681 return 0; 690 return 0;
682 } 691 }
683 692
693 if (status != 0)
694 continue;
684 status = -EAGAIN; 695 status = -EAGAIN;
685 if (retry++ > XS_SENDMSG_RETRY) 696 break;
686 break;
687 } 697 }
688 698
689 switch (status) { 699 switch (status) {
@@ -695,12 +705,13 @@ static int xs_tcp_send_request(struct rpc_task *task)
695 case -ENOTCONN: 705 case -ENOTCONN:
696 case -EPIPE: 706 case -EPIPE:
697 status = -ENOTCONN; 707 status = -ENOTCONN;
708 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
698 break; 709 break;
699 default: 710 default:
700 dprintk("RPC: sendmsg returned unrecognized error %d\n", 711 dprintk("RPC: sendmsg returned unrecognized error %d\n",
701 -status); 712 -status);
713 clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
702 xs_tcp_shutdown(xprt); 714 xs_tcp_shutdown(xprt);
703 break;
704 } 715 }
705 716
706 return status; 717 return status;
@@ -1073,6 +1084,7 @@ static void xs_tcp_data_ready(struct sock *sk, int bytes)
1073{ 1084{
1074 struct rpc_xprt *xprt; 1085 struct rpc_xprt *xprt;
1075 read_descriptor_t rd_desc; 1086 read_descriptor_t rd_desc;
1087 int read;
1076 1088
1077 dprintk("RPC: xs_tcp_data_ready...\n"); 1089 dprintk("RPC: xs_tcp_data_ready...\n");
1078 1090
@@ -1084,8 +1096,10 @@ static void xs_tcp_data_ready(struct sock *sk, int bytes)
1084 1096
1085 /* We use rd_desc to pass struct xprt to xs_tcp_data_recv */ 1097 /* We use rd_desc to pass struct xprt to xs_tcp_data_recv */
1086 rd_desc.arg.data = xprt; 1098 rd_desc.arg.data = xprt;
1087 rd_desc.count = 65536; 1099 do {
1088 tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv); 1100 rd_desc.count = 65536;
1101 read = tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv);
1102 } while (read > 0);
1089out: 1103out:
1090 read_unlock(&sk->sk_callback_lock); 1104 read_unlock(&sk->sk_callback_lock);
1091} 1105}
@@ -1128,6 +1142,7 @@ static void xs_tcp_state_change(struct sock *sk)
1128 break; 1142 break;
1129 case TCP_FIN_WAIT1: 1143 case TCP_FIN_WAIT1:
1130 /* The client initiated a shutdown of the socket */ 1144 /* The client initiated a shutdown of the socket */
1145 xprt->connect_cookie++;
1131 xprt->reestablish_timeout = 0; 1146 xprt->reestablish_timeout = 0;
1132 set_bit(XPRT_CLOSING, &xprt->state); 1147 set_bit(XPRT_CLOSING, &xprt->state);
1133 smp_mb__before_clear_bit(); 1148 smp_mb__before_clear_bit();
@@ -1140,6 +1155,7 @@ static void xs_tcp_state_change(struct sock *sk)
1140 set_bit(XPRT_CLOSING, &xprt->state); 1155 set_bit(XPRT_CLOSING, &xprt->state);
1141 xprt_force_disconnect(xprt); 1156 xprt_force_disconnect(xprt);
1142 case TCP_SYN_SENT: 1157 case TCP_SYN_SENT:
1158 xprt->connect_cookie++;
1143 case TCP_CLOSING: 1159 case TCP_CLOSING:
1144 /* 1160 /*
1145 * If the server closed down the connection, make sure that 1161 * If the server closed down the connection, make sure that
@@ -1186,9 +1202,11 @@ static void xs_udp_write_space(struct sock *sk)
1186 1202
1187 if (unlikely(!(sock = sk->sk_socket))) 1203 if (unlikely(!(sock = sk->sk_socket)))
1188 goto out; 1204 goto out;
1205 clear_bit(SOCK_NOSPACE, &sock->flags);
1206
1189 if (unlikely(!(xprt = xprt_from_sock(sk)))) 1207 if (unlikely(!(xprt = xprt_from_sock(sk))))
1190 goto out; 1208 goto out;
1191 if (unlikely(!test_and_clear_bit(SOCK_NOSPACE, &sock->flags))) 1209 if (test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags) == 0)
1192 goto out; 1210 goto out;
1193 1211
1194 xprt_write_space(xprt); 1212 xprt_write_space(xprt);
@@ -1219,9 +1237,11 @@ static void xs_tcp_write_space(struct sock *sk)
1219 1237
1220 if (unlikely(!(sock = sk->sk_socket))) 1238 if (unlikely(!(sock = sk->sk_socket)))
1221 goto out; 1239 goto out;
1240 clear_bit(SOCK_NOSPACE, &sock->flags);
1241
1222 if (unlikely(!(xprt = xprt_from_sock(sk)))) 1242 if (unlikely(!(xprt = xprt_from_sock(sk))))
1223 goto out; 1243 goto out;
1224 if (unlikely(!test_and_clear_bit(SOCK_NOSPACE, &sock->flags))) 1244 if (test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags) == 0)
1225 goto out; 1245 goto out;
1226 1246
1227 xprt_write_space(xprt); 1247 xprt_write_space(xprt);
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-20 04:26:23 -0400