9 files changed, 301 insertions, 235 deletions

diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c
index 8dc47f1d0001..880d0de3f50f 100644
--- a/net/sunrpc/auth.c
+++ b/net/sunrpc/auth.c
@@ -19,6 +19,15 @@
 # define RPCDBG_FACILITY        RPCDBG_AUTH
 #endif
 
+#define RPC_CREDCACHE_DEFAULT_HASHBITS  (4)
+struct rpc_cred_cache {
+        struct hlist_head       *hashtable;
+        unsigned int            hashbits;
+        spinlock_t              lock;
+};
+
+static unsigned int auth_hashbits = RPC_CREDCACHE_DEFAULT_HASHBITS;
+
 static DEFINE_SPINLOCK(rpc_authflavor_lock);
 static const struct rpc_authops *auth_flavors[RPC_AUTH_MAXFLAVOR] = {
         &authnull_ops,          /* AUTH_NULL */
@@ -29,6 +38,42 @@ static const struct rpc_authops *auth_flavors[RPC_AUTH_MAXFLAVOR] = {
 static LIST_HEAD(cred_unused);
 static unsigned long number_cred_unused;
 
+#define MAX_HASHTABLE_BITS (10) 
+static int param_set_hashtbl_sz(const char *val, struct kernel_param *kp)
+{
+        unsigned long num;
+        unsigned int nbits;
+        int ret;
+
+        if (!val)
+                goto out_inval;
+        ret = strict_strtoul(val, 0, &num);
+        if (ret == -EINVAL)
+                goto out_inval;
+        nbits = fls(num);
+        if (num > (1U << nbits))
+                nbits++;
+        if (nbits > MAX_HASHTABLE_BITS || nbits < 2)
+                goto out_inval;
+        *(unsigned int *)kp->arg = nbits;
+        return 0;
+out_inval:
+        return -EINVAL;
+}
+
+static int param_get_hashtbl_sz(char *buffer, struct kernel_param *kp)
+{
+        unsigned int nbits;
+
+        nbits = *(unsigned int *)kp->arg;
+        return sprintf(buffer, "%u", 1U << nbits);
+}
+
+#define param_check_hashtbl_sz(name, p) __param_check(name, p, unsigned int);
+
+module_param_named(auth_hashtable_size, auth_hashbits, hashtbl_sz, 0644);
+MODULE_PARM_DESC(auth_hashtable_size, "RPC credential cache hashtable size");
+
 static u32
 pseudoflavor_to_flavor(u32 flavor) {
         if (flavor >= RPC_AUTH_MAXFLAVOR)
@@ -145,16 +190,23 @@ int
 rpcauth_init_credcache(struct rpc_auth *auth)
 {
         struct rpc_cred_cache *new;
-        int i;
+        unsigned int hashsize;
 
         new = kmalloc(sizeof(*new), GFP_KERNEL);
         if (!new)
-                return -ENOMEM;
-        for (i = 0; i < RPC_CREDCACHE_NR; i++)
-                INIT_HLIST_HEAD(&new->hashtable[i]);
+                goto out_nocache;
+        new->hashbits = auth_hashbits;
+        hashsize = 1U << new->hashbits;
+        new->hashtable = kcalloc(hashsize, sizeof(new->hashtable[0]), GFP_KERNEL);
+        if (!new->hashtable)
+                goto out_nohashtbl;
         spin_lock_init(&new->lock);
         auth->au_credcache = new;
         return 0;
+out_nohashtbl:
+        kfree(new);
+out_nocache:
+        return -ENOMEM;
 }
 EXPORT_SYMBOL_GPL(rpcauth_init_credcache);
 
@@ -183,11 +235,12 @@ rpcauth_clear_credcache(struct rpc_cred_cache *cache)
         LIST_HEAD(free);
         struct hlist_head *head;
         struct rpc_cred *cred;
+        unsigned int hashsize = 1U << cache->hashbits;
         int             i;
 
         spin_lock(&rpc_credcache_lock);
         spin_lock(&cache->lock);
-        for (i = 0; i < RPC_CREDCACHE_NR; i++) {
+        for (i = 0; i < hashsize; i++) {
                 head = &cache->hashtable[i];
                 while (!hlist_empty(head)) {
                         cred = hlist_entry(head->first, struct rpc_cred, cr_hash);
@@ -216,6 +269,7 @@ rpcauth_destroy_credcache(struct rpc_auth *auth)
         if (cache) {
                 auth->au_credcache = NULL;
                 rpcauth_clear_credcache(cache);
+                kfree(cache->hashtable);
                 kfree(cache);
         }
 }
@@ -297,7 +351,7 @@ rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred,
                         *entry, *new;
         unsigned int nr;
 
-        nr = hash_long(acred->uid, RPC_CREDCACHE_HASHBITS);
+        nr = hash_long(acred->uid, cache->hashbits);
 
         rcu_read_lock();
         hlist_for_each_entry_rcu(entry, pos, &cache->hashtable[nr], cr_hash) {
@@ -390,16 +444,16 @@ rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred,
 }
 EXPORT_SYMBOL_GPL(rpcauth_init_cred);
 
-void
+struct rpc_cred *
 rpcauth_generic_bind_cred(struct rpc_task *task, struct rpc_cred *cred, int lookupflags)
 {
-        task->tk_msg.rpc_cred = get_rpccred(cred);
         dprintk("RPC: %5u holding %s cred %p\n", task->tk_pid,
                         cred->cr_auth->au_ops->au_name, cred);
+        return get_rpccred(cred);
 }
 EXPORT_SYMBOL_GPL(rpcauth_generic_bind_cred);
 
-static void
+static struct rpc_cred *
 rpcauth_bind_root_cred(struct rpc_task *task, int lookupflags)
 {
         struct rpc_auth *auth = task->tk_client->cl_auth;
@@ -407,45 +461,43 @@ rpcauth_bind_root_cred(struct rpc_task *task, int lookupflags)
                 .uid = 0,
                 .gid = 0,
         };
-        struct rpc_cred *ret;
 
         dprintk("RPC: %5u looking up %s cred\n",
                 task->tk_pid, task->tk_client->cl_auth->au_ops->au_name);
-        ret = auth->au_ops->lookup_cred(auth, &acred, lookupflags);
-        if (!IS_ERR(ret))
-                task->tk_msg.rpc_cred = ret;
-        else
-                task->tk_status = PTR_ERR(ret);
+        return auth->au_ops->lookup_cred(auth, &acred, lookupflags);
 }
 
-static void
+static struct rpc_cred *
 rpcauth_bind_new_cred(struct rpc_task *task, int lookupflags)
 {
         struct rpc_auth *auth = task->tk_client->cl_auth;
-        struct rpc_cred *ret;
 
         dprintk("RPC: %5u looking up %s cred\n",
                 task->tk_pid, auth->au_ops->au_name);
-        ret = rpcauth_lookupcred(auth, lookupflags);
-        if (!IS_ERR(ret))
-                task->tk_msg.rpc_cred = ret;
-        else
-                task->tk_status = PTR_ERR(ret);
+        return rpcauth_lookupcred(auth, lookupflags);
 }
 
-void
+static int
 rpcauth_bindcred(struct rpc_task *task, struct rpc_cred *cred, int flags)
 {
+        struct rpc_rqst *req = task->tk_rqstp;
+        struct rpc_cred *new;
         int lookupflags = 0;
 
         if (flags & RPC_TASK_ASYNC)
                 lookupflags |= RPCAUTH_LOOKUP_NEW;
         if (cred != NULL)
-                cred->cr_ops->crbind(task, cred, lookupflags);
+                new = cred->cr_ops->crbind(task, cred, lookupflags);
         else if (flags & RPC_TASK_ROOTCREDS)
-                rpcauth_bind_root_cred(task, lookupflags);
+                new = rpcauth_bind_root_cred(task, lookupflags);
         else
-                rpcauth_bind_new_cred(task, lookupflags);
+                new = rpcauth_bind_new_cred(task, lookupflags);
+        if (IS_ERR(new))
+                return PTR_ERR(new);
+        if (req->rq_cred != NULL)
+                put_rpccred(req->rq_cred);
+        req->rq_cred = new;
+        return 0;
 }
 
 void
@@ -484,22 +536,10 @@ out_nodestroy:
 }
 EXPORT_SYMBOL_GPL(put_rpccred);
 
-void
-rpcauth_unbindcred(struct rpc_task *task)
-{
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
-
-        dprintk("RPC: %5u releasing %s cred %p\n",
-                task->tk_pid, cred->cr_auth->au_ops->au_name, cred);
-
-        put_rpccred(cred);
-        task->tk_msg.rpc_cred = NULL;
-}
-
 __be32 *
 rpcauth_marshcred(struct rpc_task *task, __be32 *p)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 
         dprintk("RPC: %5u marshaling %s cred %p\n",
                 task->tk_pid, cred->cr_auth->au_ops->au_name, cred);
@@ -510,7 +550,7 @@ rpcauth_marshcred(struct rpc_task *task, __be32 *p)
 __be32 *
 rpcauth_checkverf(struct rpc_task *task, __be32 *p)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 
         dprintk("RPC: %5u validating %s cred %p\n",
                 task->tk_pid, cred->cr_auth->au_ops->au_name, cred);
@@ -522,7 +562,7 @@ int
 rpcauth_wrap_req(struct rpc_task *task, kxdrproc_t encode, void *rqstp,
                 __be32 *data, void *obj)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 
         dprintk("RPC: %5u using %s cred %p to wrap rpc data\n",
                         task->tk_pid, cred->cr_ops->cr_name, cred);
@@ -536,7 +576,7 @@ int
 rpcauth_unwrap_resp(struct rpc_task *task, kxdrproc_t decode, void *rqstp,
                 __be32 *data, void *obj)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 
         dprintk("RPC: %5u using %s cred %p to unwrap rpc data\n",
                         task->tk_pid, cred->cr_ops->cr_name, cred);
@@ -550,13 +590,21 @@ rpcauth_unwrap_resp(struct rpc_task *task, kxdrproc_t decode, void *rqstp,
 int
 rpcauth_refreshcred(struct rpc_task *task)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
         int err;
 
+        cred = task->tk_rqstp->rq_cred;
+        if (cred == NULL) {
+                err = rpcauth_bindcred(task, task->tk_msg.rpc_cred, task->tk_flags);
+                if (err < 0)
+                        goto out;
+                cred = task->tk_rqstp->rq_cred;
+        };
         dprintk("RPC: %5u refreshing %s cred %p\n",
                 task->tk_pid, cred->cr_auth->au_ops->au_name, cred);
 
         err = cred->cr_ops->crrefresh(task);
+out:
         if (err < 0)
                 task->tk_status = err;
         return err;
@@ -565,7 +613,7 @@ rpcauth_refreshcred(struct rpc_task *task)
 void
 rpcauth_invalcred(struct rpc_task *task)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 
         dprintk("RPC: %5u invalidating %s cred %p\n",
                 task->tk_pid, cred->cr_auth->au_ops->au_name, cred);
@@ -576,7 +624,7 @@ rpcauth_invalcred(struct rpc_task *task)
 int
 rpcauth_uptodatecred(struct rpc_task *task)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
 
         return cred == NULL ||
                 test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0;
@@ -587,14 +635,27 @@ static struct shrinker rpc_cred_shrinker = {
         .seeks = DEFAULT_SEEKS,
 };
 
-void __init rpcauth_init_module(void)
+int __init rpcauth_init_module(void)
 {
-        rpc_init_authunix();
-        rpc_init_generic_auth();
+        int err;
+
+        err = rpc_init_authunix();
+        if (err < 0)
+                goto out1;
+        err = rpc_init_generic_auth();
+        if (err < 0)
+                goto out2;
         register_shrinker(&rpc_cred_shrinker);
+        return 0;
+out2:
+        rpc_destroy_authunix();
+out1:
+        return err;
 }
 
 void __exit rpcauth_remove_module(void)
 {
+        rpc_destroy_authunix();
+        rpc_destroy_generic_auth();
         unregister_shrinker(&rpc_cred_shrinker);
 }
diff --git a/net/sunrpc/auth_generic.c b/net/sunrpc/auth_generic.c
index 8f623b0f03dd..43162bb3b78f 100644
--- a/net/sunrpc/auth_generic.c
+++ b/net/sunrpc/auth_generic.c
@@ -27,7 +27,6 @@ struct generic_cred {
 };
 
 static struct rpc_auth generic_auth;
-static struct rpc_cred_cache generic_cred_cache;
 static const struct rpc_credops generic_credops;
 
 /*
@@ -55,18 +54,13 @@ struct rpc_cred *rpc_lookup_machine_cred(void)
 }
 EXPORT_SYMBOL_GPL(rpc_lookup_machine_cred);
 
-static void
-generic_bind_cred(struct rpc_task *task, struct rpc_cred *cred, int lookupflags)
+static struct rpc_cred *generic_bind_cred(struct rpc_task *task,
+                struct rpc_cred *cred, int lookupflags)
 {
         struct rpc_auth *auth = task->tk_client->cl_auth;
         struct auth_cred *acred = &container_of(cred, struct generic_cred, gc_base)->acred;
-        struct rpc_cred *ret;
 
-        ret = auth->au_ops->lookup_cred(auth, acred, lookupflags);
-        if (!IS_ERR(ret))
-                task->tk_msg.rpc_cred = ret;
-        else
-                task->tk_status = PTR_ERR(ret);
+        return auth->au_ops->lookup_cred(auth, acred, lookupflags);
 }
 
 /*
@@ -159,20 +153,16 @@ out_nomatch:
         return 0;
 }
 
-void __init rpc_init_generic_auth(void)
+int __init rpc_init_generic_auth(void)
 {
-        spin_lock_init(&generic_cred_cache.lock);
+        return rpcauth_init_credcache(&generic_auth);
 }
 
 void __exit rpc_destroy_generic_auth(void)
 {
-        rpcauth_clear_credcache(&generic_cred_cache);
+        rpcauth_destroy_credcache(&generic_auth);
 }
 
-static struct rpc_cred_cache generic_cred_cache = {
-        {{ NULL, },},
-};
-
 static const struct rpc_authops generic_auth_ops = {
         .owner = THIS_MODULE,
         .au_name = "Generic",
@@ -183,7 +173,6 @@ static const struct rpc_authops generic_auth_ops = {
 static struct rpc_auth generic_auth = {
         .au_ops = &generic_auth_ops,
         .au_count = ATOMIC_INIT(0),
-        .au_credcache = &generic_cred_cache,
 };
 
 static const struct rpc_credops generic_credops = {
diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 8da2a0e68574..dcfc66bab2bb 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -373,7 +373,7 @@ gss_handle_downcall_result(struct gss_cred *gss_cred, struct gss_upcall_msg *gss
 static void
 gss_upcall_callback(struct rpc_task *task)
 {
-        struct gss_cred *gss_cred = container_of(task->tk_msg.rpc_cred,
+        struct gss_cred *gss_cred = container_of(task->tk_rqstp->rq_cred,
                         struct gss_cred, gc_base);
         struct gss_upcall_msg *gss_msg = gss_cred->gc_upcall;
         struct inode *inode = &gss_msg->inode->vfs_inode;
@@ -502,7 +502,7 @@ static void warn_gssd(void)
 static inline int
 gss_refresh_upcall(struct rpc_task *task)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
         struct gss_auth *gss_auth = container_of(cred->cr_auth,
                         struct gss_auth, rpc_auth);
         struct gss_cred *gss_cred = container_of(cred,
@@ -928,6 +928,7 @@ gss_do_free_ctx(struct gss_cl_ctx *ctx)
 {
         dprintk("RPC:       gss_free_ctx\n");
 
+        gss_delete_sec_context(&ctx->gc_gss_ctx);
         kfree(ctx->gc_wire_ctx.data);
         kfree(ctx);
 }
@@ -942,13 +943,7 @@ gss_free_ctx_callback(struct rcu_head *head)
 static void
 gss_free_ctx(struct gss_cl_ctx *ctx)
 {
-        struct gss_ctx *gc_gss_ctx;
-
-        gc_gss_ctx = rcu_dereference(ctx->gc_gss_ctx);
-        rcu_assign_pointer(ctx->gc_gss_ctx, NULL);
         call_rcu(&ctx->gc_rcu, gss_free_ctx_callback);
-        if (gc_gss_ctx)
-                gss_delete_sec_context(&gc_gss_ctx);
 }
 
 static void
@@ -1064,12 +1059,12 @@ out:
 static __be32 *
 gss_marshal(struct rpc_task *task, __be32 *p)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_rqst *req = task->tk_rqstp;
+        struct rpc_cred *cred = req->rq_cred;
         struct gss_cred *gss_cred = container_of(cred, struct gss_cred,
                                                  gc_base);
         struct gss_cl_ctx       *ctx = gss_cred_get_ctx(cred);
         __be32          *cred_len;
-        struct rpc_rqst *req = task->tk_rqstp;
         u32             maj_stat = 0;
         struct xdr_netobj mic;
         struct kvec     iov;
@@ -1119,7 +1114,7 @@ out_put_ctx:
 
 static int gss_renew_cred(struct rpc_task *task)
 {
-        struct rpc_cred *oldcred = task->tk_msg.rpc_cred;
+        struct rpc_cred *oldcred = task->tk_rqstp->rq_cred;
         struct gss_cred *gss_cred = container_of(oldcred,
                                                  struct gss_cred,
                                                  gc_base);
@@ -1133,7 +1128,7 @@ static int gss_renew_cred(struct rpc_task *task)
         new = gss_lookup_cred(auth, &acred, RPCAUTH_LOOKUP_NEW);
         if (IS_ERR(new))
                 return PTR_ERR(new);
-        task->tk_msg.rpc_cred = new;
+        task->tk_rqstp->rq_cred = new;
         put_rpccred(oldcred);
         return 0;
 }
@@ -1161,7 +1156,7 @@ static int gss_cred_is_negative_entry(struct rpc_cred *cred)
 static int
 gss_refresh(struct rpc_task *task)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
         int ret = 0;
 
         if (gss_cred_is_negative_entry(cred))
@@ -1172,7 +1167,7 @@ gss_refresh(struct rpc_task *task)
                 ret = gss_renew_cred(task);
                 if (ret < 0)
                         goto out;
-                cred = task->tk_msg.rpc_cred;
+                cred = task->tk_rqstp->rq_cred;
         }
 
         if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags))
@@ -1191,7 +1186,7 @@ gss_refresh_null(struct rpc_task *task)
 static __be32 *
 gss_validate(struct rpc_task *task, __be32 *p)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
         struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
         __be32          seq;
         struct kvec     iov;
@@ -1400,7 +1395,7 @@ static int
 gss_wrap_req(struct rpc_task *task,
              kxdrproc_t encode, void *rqstp, __be32 *p, void *obj)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
         struct gss_cred *gss_cred = container_of(cred, struct gss_cred,
                         gc_base);
         struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
@@ -1503,7 +1498,7 @@ static int
 gss_unwrap_resp(struct rpc_task *task,
                 kxdrproc_t decode, void *rqstp, __be32 *p, void *obj)
 {
-        struct rpc_cred *cred = task->tk_msg.rpc_cred;
+        struct rpc_cred *cred = task->tk_rqstp->rq_cred;
         struct gss_cred *gss_cred = container_of(cred, struct gss_cred,
                         gc_base);
         struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred);
diff --git a/net/sunrpc/auth_null.c b/net/sunrpc/auth_null.c
index 1db618f56ecb..a5c36c01707b 100644
--- a/net/sunrpc/auth_null.c
+++ b/net/sunrpc/auth_null.c
@@ -75,7 +75,7 @@ nul_marshal(struct rpc_task *task, __be32 *p)
 static int
 nul_refresh(struct rpc_task *task)
 {
-        set_bit(RPCAUTH_CRED_UPTODATE, &task->tk_msg.rpc_cred->cr_flags);
+        set_bit(RPCAUTH_CRED_UPTODATE, &task->tk_rqstp->rq_cred->cr_flags);
         return 0;
 }
 
diff --git a/net/sunrpc/auth_unix.c b/net/sunrpc/auth_unix.c
index aac2f8b4ee21..4cb70dc6e7ad 100644
--- a/net/sunrpc/auth_unix.c
+++ b/net/sunrpc/auth_unix.c
@@ -29,7 +29,6 @@ struct unx_cred {
 #endif
 
 static struct rpc_auth          unix_auth;
-static struct rpc_cred_cache    unix_cred_cache;
 static const struct rpc_credops unix_credops;
 
 static struct rpc_auth *
@@ -141,7 +140,7 @@ static __be32 *
 unx_marshal(struct rpc_task *task, __be32 *p)
 {
         struct rpc_clnt *clnt = task->tk_client;
-        struct unx_cred *cred = container_of(task->tk_msg.rpc_cred, struct unx_cred, uc_base);
+        struct unx_cred *cred = container_of(task->tk_rqstp->rq_cred, struct unx_cred, uc_base);
         __be32          *base, *hold;
         int             i;
 
@@ -174,7 +173,7 @@ unx_marshal(struct rpc_task *task, __be32 *p)
 static int
 unx_refresh(struct rpc_task *task)
 {
-        set_bit(RPCAUTH_CRED_UPTODATE, &task->tk_msg.rpc_cred->cr_flags);
+        set_bit(RPCAUTH_CRED_UPTODATE, &task->tk_rqstp->rq_cred->cr_flags);
         return 0;
 }
 
@@ -197,15 +196,20 @@ unx_validate(struct rpc_task *task, __be32 *p)
                 printk("RPC: giant verf size: %u\n", size);
                 return NULL;
         }
-        task->tk_msg.rpc_cred->cr_auth->au_rslack = (size >> 2) + 2;
+        task->tk_rqstp->rq_cred->cr_auth->au_rslack = (size >> 2) + 2;
         p += (size >> 2);
 
         return p;
 }
 
-void __init rpc_init_authunix(void)
+int __init rpc_init_authunix(void)
 {
-        spin_lock_init(&unix_cred_cache.lock);
+        return rpcauth_init_credcache(&unix_auth);
+}
+
+void rpc_destroy_authunix(void)
+{
+        rpcauth_destroy_credcache(&unix_auth);
 }
 
 const struct rpc_authops authunix_ops = {
@@ -219,17 +223,12 @@ const struct rpc_authops authunix_ops = {
 };
 
 static
-struct rpc_cred_cache   unix_cred_cache = {
-};
-
-static
 struct rpc_auth         unix_auth = {
         .au_cslack      = UNX_WRITESLACK,
         .au_rslack      = 2,                    /* assume AUTH_NULL verf */
         .au_ops         = &authunix_ops,
         .au_flavor      = RPC_AUTH_UNIX,
         .au_count       = ATOMIC_INIT(0),
-        .au_credcache   = &unix_cred_cache,
 };
 
 static
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 756fc324db9e..2388d83b68ff 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -414,6 +414,35 @@ out_no_clnt:
 EXPORT_SYMBOL_GPL(rpc_clone_client);
 
 /*
+ * Kill all tasks for the given client.
+ * XXX: kill their descendants as well?
+ */
+void rpc_killall_tasks(struct rpc_clnt *clnt)
+{
+        struct rpc_task *rovr;
+
+
+        if (list_empty(&clnt->cl_tasks))
+                return;
+        dprintk("RPC:       killing all tasks for client %p\n", clnt);
+        /*
+         * Spin lock all_tasks to prevent changes...
+         */
+        spin_lock(&clnt->cl_lock);
+        list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
+                if (!RPC_IS_ACTIVATED(rovr))
+                        continue;
+                if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
+                        rovr->tk_flags |= RPC_TASK_KILLED;
+                        rpc_exit(rovr, -EIO);
+                        rpc_wake_up_queued_task(rovr->tk_waitqueue, rovr);
+                }
+        }
+        spin_unlock(&clnt->cl_lock);
+}
+EXPORT_SYMBOL_GPL(rpc_killall_tasks);
+
+/*
  * Properly shut down an RPC client, terminating all outstanding
  * requests.
  */
@@ -538,6 +567,49 @@ out:
 }
 EXPORT_SYMBOL_GPL(rpc_bind_new_program);
 
+void rpc_task_release_client(struct rpc_task *task)
+{
+        struct rpc_clnt *clnt = task->tk_client;
+
+        if (clnt != NULL) {
+                /* Remove from client task list */
+                spin_lock(&clnt->cl_lock);
+                list_del(&task->tk_task);
+                spin_unlock(&clnt->cl_lock);
+                task->tk_client = NULL;
+
+                rpc_release_client(clnt);
+        }
+}
+
+static
+void rpc_task_set_client(struct rpc_task *task, struct rpc_clnt *clnt)
+{
+        if (clnt != NULL) {
+                rpc_task_release_client(task);
+                task->tk_client = clnt;
+                kref_get(&clnt->cl_kref);
+                if (clnt->cl_softrtry)
+                        task->tk_flags |= RPC_TASK_SOFT;
+                /* Add to the client's list of all tasks */
+                spin_lock(&clnt->cl_lock);
+                list_add_tail(&task->tk_task, &clnt->cl_tasks);
+                spin_unlock(&clnt->cl_lock);
+        }
+}
+
+static void
+rpc_task_set_rpc_message(struct rpc_task *task, const struct rpc_message *msg)
+{
+        if (msg != NULL) {
+                task->tk_msg.rpc_proc = msg->rpc_proc;
+                task->tk_msg.rpc_argp = msg->rpc_argp;
+                task->tk_msg.rpc_resp = msg->rpc_resp;
+                if (msg->rpc_cred != NULL)
+                        task->tk_msg.rpc_cred = get_rpccred(msg->rpc_cred);
+        }
+}
+
 /*
  * Default callback for async RPC calls
  */
@@ -562,6 +634,18 @@ struct rpc_task *rpc_run_task(const struct rpc_task_setup *task_setup_data)
         if (IS_ERR(task))
                 goto out;
 
+        rpc_task_set_client(task, task_setup_data->rpc_client);
+        rpc_task_set_rpc_message(task, task_setup_data->rpc_message);
+
+        if (task->tk_status != 0) {
+                int ret = task->tk_status;
+                rpc_put_task(task);
+                return ERR_PTR(ret);
+        }
+
+        if (task->tk_action == NULL)
+                rpc_call_start(task);
+
         atomic_inc(&task->tk_count);
         rpc_execute(task);
 out:
@@ -756,12 +840,13 @@ EXPORT_SYMBOL_GPL(rpc_force_rebind);
  * Restart an (async) RPC call from the call_prepare state.
  * Usually called from within the exit handler.
  */
-void
+int
 rpc_restart_call_prepare(struct rpc_task *task)
 {
         if (RPC_ASSASSINATED(task))
-                return;
+                return 0;
         task->tk_action = rpc_prepare_task;
+        return 1;
 }
 EXPORT_SYMBOL_GPL(rpc_restart_call_prepare);
 
@@ -769,13 +854,13 @@ EXPORT_SYMBOL_GPL(rpc_restart_call_prepare);
  * Restart an (async) RPC call. Usually called from within the
  * exit handler.
  */
-void
+int
 rpc_restart_call(struct rpc_task *task)
 {
         if (RPC_ASSASSINATED(task))
-                return;
-
+                return 0;
         task->tk_action = call_start;
+        return 1;
 }
 EXPORT_SYMBOL_GPL(rpc_restart_call);
 
@@ -824,11 +909,6 @@ call_reserve(struct rpc_task *task)
 {
         dprint_status(task);
 
-        if (!rpcauth_uptodatecred(task)) {
-                task->tk_action = call_refresh;
-                return;
-        }
-
         task->tk_status  = 0;
         task->tk_action  = call_reserveresult;
         xprt_reserve(task);
@@ -892,7 +972,7 @@ call_reserveresult(struct rpc_task *task)
 static void
 call_allocate(struct rpc_task *task)
 {
-        unsigned int slack = task->tk_msg.rpc_cred->cr_auth->au_cslack;
+        unsigned int slack = task->tk_client->cl_auth->au_cslack;
         struct rpc_rqst *req = task->tk_rqstp;
         struct rpc_xprt *xprt = task->tk_xprt;
         struct rpc_procinfo *proc = task->tk_msg.rpc_proc;
@@ -900,7 +980,7 @@ call_allocate(struct rpc_task *task)
         dprint_status(task);
 
         task->tk_status = 0;
-        task->tk_action = call_bind;
+        task->tk_action = call_refresh;
 
         if (req->rq_buffer)
                 return;
@@ -937,6 +1017,47 @@ call_allocate(struct rpc_task *task)
         rpc_exit(task, -ERESTARTSYS);
 }
 
+/*
+ * 2a.  Bind and/or refresh the credentials
+ */
+static void
+call_refresh(struct rpc_task *task)
+{
+        dprint_status(task);
+
+        task->tk_action = call_refreshresult;
+        task->tk_status = 0;
+        task->tk_client->cl_stats->rpcauthrefresh++;
+        rpcauth_refreshcred(task);
+}
+
+/*
+ * 2b.  Process the results of a credential refresh
+ */
+static void
+call_refreshresult(struct rpc_task *task)
+{
+        int status = task->tk_status;
+
+        dprint_status(task);
+
+        task->tk_status = 0;
+        task->tk_action = call_bind;
+        if (status >= 0 && rpcauth_uptodatecred(task))
+                return;
+        switch (status) {
+        case -EACCES:
+                rpc_exit(task, -EACCES);
+                return;
+        case -ENOMEM:
+                rpc_exit(task, -ENOMEM);
+                return;
+        case -ETIMEDOUT:
+                rpc_delay(task, 3*HZ);
+        }
+        task->tk_action = call_refresh;
+}
+
 static inline int
 rpc_task_need_encode(struct rpc_task *task)
 {
@@ -1472,43 +1593,6 @@ out_retry:
         }
 }
 
-/*
- * 8.   Refresh the credentials if rejected by the server
- */
-static void
-call_refresh(struct rpc_task *task)
-{
-        dprint_status(task);
-
-        task->tk_action = call_refreshresult;
-        task->tk_status = 0;
-        task->tk_client->cl_stats->rpcauthrefresh++;
-        rpcauth_refreshcred(task);
-}
-
-/*
- * 8a.  Process the results of a credential refresh
- */
-static void
-call_refreshresult(struct rpc_task *task)
-{
-        int status = task->tk_status;
-
-        dprint_status(task);
-
-        task->tk_status = 0;
-        task->tk_action = call_reserve;
-        if (status >= 0 && rpcauth_uptodatecred(task))
-                return;
-        if (status == -EACCES) {
-                rpc_exit(task, -EACCES);
-                return;
-        }
-        task->tk_action = call_refresh;
-        if (status != -ETIMEDOUT)
-                rpc_delay(task, 3*HZ);
-}
-
 static __be32 *
 rpc_encode_header(struct rpc_task *task)
 {
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index 4a843b883b89..cace6049e4a5 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -246,17 +246,8 @@ static inline void rpc_task_set_debuginfo(struct rpc_task *task)
 
 static void rpc_set_active(struct rpc_task *task)
 {
-        struct rpc_clnt *clnt;
-        if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
-                return;
         rpc_task_set_debuginfo(task);
-        /* Add to global list of all tasks */
-        clnt = task->tk_client;
-        if (clnt != NULL) {
-                spin_lock(&clnt->cl_lock);
-                list_add_tail(&task->tk_task, &clnt->cl_tasks);
-                spin_unlock(&clnt->cl_lock);
-        }
+        set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
 }
 
 /*
@@ -319,11 +310,6 @@ static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
         dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
                         task->tk_pid, rpc_qname(q), jiffies);
 
-        if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
-                printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
-                return;
-        }
-
         __rpc_add_wait_queue(q, task);
 
         BUG_ON(task->tk_callback != NULL);
@@ -334,8 +320,8 @@ static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
 void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
                                 rpc_action action)
 {
-        /* Mark the task as being activated if so needed */
-        rpc_set_active(task);
+        /* We shouldn't ever put an inactive task to sleep */
+        BUG_ON(!RPC_IS_ACTIVATED(task));
 
         /*
          * Protect the queue operations.
@@ -406,14 +392,6 @@ void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task
 EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
 
 /*
- * Wake up the specified task
- */
-static void rpc_wake_up_task(struct rpc_task *task)
-{
-        rpc_wake_up_queued_task(task->tk_waitqueue, task);
-}
-
-/*
  * Wake up the next task on a priority queue.
  */
 static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
@@ -600,7 +578,15 @@ void rpc_exit_task(struct rpc_task *task)
                 }
         }
 }
-EXPORT_SYMBOL_GPL(rpc_exit_task);
+
+void rpc_exit(struct rpc_task *task, int status)
+{
+        task->tk_status = status;
+        task->tk_action = rpc_exit_task;
+        if (RPC_IS_QUEUED(task))
+                rpc_wake_up_queued_task(task->tk_waitqueue, task);
+}
+EXPORT_SYMBOL_GPL(rpc_exit);
 
 void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
 {
@@ -690,7 +676,6 @@ static void __rpc_execute(struct rpc_task *task)
                         dprintk("RPC: %5u got signal\n", task->tk_pid);
                         task->tk_flags |= RPC_TASK_KILLED;
                         rpc_exit(task, -ERESTARTSYS);
-                        rpc_wake_up_task(task);
                 }
                 rpc_set_running(task);
                 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
@@ -714,8 +699,9 @@ static void __rpc_execute(struct rpc_task *task)
 void rpc_execute(struct rpc_task *task)
 {
         rpc_set_active(task);
-        rpc_set_running(task);
-        __rpc_execute(task);
+        rpc_make_runnable(task);
+        if (!RPC_IS_ASYNC(task))
+                __rpc_execute(task);
 }
 
 static void rpc_async_schedule(struct work_struct *work)
@@ -808,26 +794,9 @@ static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *ta
         /* Initialize workqueue for async tasks */
         task->tk_workqueue = task_setup_data->workqueue;
 
-        task->tk_client = task_setup_data->rpc_client;
-        if (task->tk_client != NULL) {
-                kref_get(&task->tk_client->cl_kref);
-                if (task->tk_client->cl_softrtry)
-                        task->tk_flags |= RPC_TASK_SOFT;
-        }
-
         if (task->tk_ops->rpc_call_prepare != NULL)
                 task->tk_action = rpc_prepare_task;
 
-        if (task_setup_data->rpc_message != NULL) {
-                task->tk_msg.rpc_proc = task_setup_data->rpc_message->rpc_proc;
-                task->tk_msg.rpc_argp = task_setup_data->rpc_message->rpc_argp;
-                task->tk_msg.rpc_resp = task_setup_data->rpc_message->rpc_resp;
-                /* Bind the user cred */
-                rpcauth_bindcred(task, task_setup_data->rpc_message->rpc_cred, task_setup_data->flags);
-                if (task->tk_action == NULL)
-                        rpc_call_start(task);
-        }
-
         /* starting timestamp */
         task->tk_start = ktime_get();
 
@@ -896,11 +865,8 @@ void rpc_put_task(struct rpc_task *task)
         if (task->tk_rqstp)
                 xprt_release(task);
         if (task->tk_msg.rpc_cred)
-                rpcauth_unbindcred(task);
-        if (task->tk_client) {
-                rpc_release_client(task->tk_client);
-                task->tk_client = NULL;
-        }
+                put_rpccred(task->tk_msg.rpc_cred);
+        rpc_task_release_client(task);
         if (task->tk_workqueue != NULL) {
                 INIT_WORK(&task->u.tk_work, rpc_async_release);
                 queue_work(task->tk_workqueue, &task->u.tk_work);
@@ -913,13 +879,6 @@ static void rpc_release_task(struct rpc_task *task)
 {
         dprintk("RPC: %5u release task\n", task->tk_pid);
 
-        if (!list_empty(&task->tk_task)) {
-                struct rpc_clnt *clnt = task->tk_client;
-                /* Remove from client task list */
-                spin_lock(&clnt->cl_lock);
-                list_del(&task->tk_task);
-                spin_unlock(&clnt->cl_lock);
-        }
         BUG_ON (RPC_IS_QUEUED(task));
 
         /* Wake up anyone who is waiting for task completion */
@@ -928,35 +887,6 @@ static void rpc_release_task(struct rpc_task *task)
         rpc_put_task(task);
 }
 
-/*
- * Kill all tasks for the given client.
- * XXX: kill their descendants as well?
- */
-void rpc_killall_tasks(struct rpc_clnt *clnt)
-{
-        struct rpc_task *rovr;
-
-
-        if (list_empty(&clnt->cl_tasks))
-                return;
-        dprintk("RPC:       killing all tasks for client %p\n", clnt);
-        /*
-         * Spin lock all_tasks to prevent changes...
-         */
-        spin_lock(&clnt->cl_lock);
-        list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
-                if (! RPC_IS_ACTIVATED(rovr))
-                        continue;
-                if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
-                        rovr->tk_flags |= RPC_TASK_KILLED;
-                        rpc_exit(rovr, -EIO);
-                        rpc_wake_up_task(rovr);
-                }
-        }
-        spin_unlock(&clnt->cl_lock);
-}
-EXPORT_SYMBOL_GPL(rpc_killall_tasks);
-
 int rpciod_up(void)
 {
         return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c
index f438347d817b..34b58f9e704a 100644
--- a/net/sunrpc/sunrpc_syms.c
+++ b/net/sunrpc/sunrpc_syms.c
@@ -33,10 +33,11 @@ init_sunrpc(void)
         if (err)
                 goto out;
         err = rpc_init_mempool();
-        if (err) {
-                unregister_rpc_pipefs();
-                goto out;
-        }
+        if (err)
+                goto out2;
+        err = rpcauth_init_module();
+        if (err)
+                goto out3;
 #ifdef RPC_DEBUG
         rpc_register_sysctl();
 #endif
@@ -47,7 +48,11 @@ init_sunrpc(void)
         cache_register(&unix_gid_cache);
         svc_init_xprt_sock();   /* svc sock transport */
         init_socket_xprt();     /* clnt sock transport */
-        rpcauth_init_module();
+        return 0;
+out3:
+        rpc_destroy_mempool();
+out2:
+        unregister_rpc_pipefs();
 out:
         return err;
 }
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index dcd0132396ba..970fb00f388c 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -1032,6 +1032,8 @@ void xprt_release(struct rpc_task *task)
         spin_unlock_bh(&xprt->transport_lock);
         if (req->rq_buffer)
                 xprt->ops->buf_free(req->rq_buffer);
+        if (req->rq_cred != NULL)
+                put_rpccred(req->rq_cred);
         task->tk_rqstp = NULL;
         if (req->rq_release_snd_buf)
                 req->rq_release_snd_buf(req);
@@ -1129,6 +1131,7 @@ static void xprt_destroy(struct kref *kref)
         rpc_destroy_wait_queue(&xprt->sending);
         rpc_destroy_wait_queue(&xprt->resend);
         rpc_destroy_wait_queue(&xprt->backlog);
+        cancel_work_sync(&xprt->task_cleanup);
         /*
          * Tear down transport state and free the rpc_xprt
          */