Merge tag 'nfsd-4.3' of git://linux-nfs.org/~bfields/linux

Pull nfsd updates from Bruce Fields:
 "Nothing major, but:

   - Add Jeff Layton as an nfsd co-maintainer: no change to existing
     practice, just an acknowledgement of the status quo.

   - Two patches ("nfsd: ensure that...") for a race overlooked by the
     state locking rewrite, causing a crash noticed by multiple users.

   - Lots of smaller bugfixes all over from Kinglong Mee.

   - From Jeff, some cleanup of server rpc code in preparation for
     possible shift of nfsd threads to workqueues"

* tag 'nfsd-4.3' of git://linux-nfs.org/~bfields/linux: (52 commits)
  nfsd: deal with DELEGRETURN racing with CB_RECALL
  nfsd: return CLID_INUSE for unexpected SETCLIENTID_CONFIRM case
  nfsd: ensure that delegation stateid hash references are only put once
  nfsd: ensure that the ol stateid hash reference is only put once
  net: sunrpc: fix tracepoint Warning: unknown op '->'
  nfsd: allow more than one laundry job to run at a time
  nfsd: don't WARN/backtrace for invalid container deployment.
  fs: fix fs/locks.c kernel-doc warning
  nfsd: Add Jeff Layton as co-maintainer
  NFSD: Return word2 bitmask if setting security label in OPEN/CREATE
  NFSD: Set the attributes used to store the verifier for EXCLUSIVE4_1
  nfsd: SUPPATTR_EXCLCREAT must be encoded before SECURITY_LABEL.
  nfsd: Fix an FS_LAYOUT_TYPES/LAYOUT_TYPES encode bug
  NFSD: Store parent's stat in a separate value
  nfsd: Fix two typos in comments
  lockd: NLM grace period shouldn't block NFSv4 opens
  nfsd: include linux/nfs4.h in export.h
  sunrpc: Switch to using hash list instead single list
  sunrpc/nfsd: Remove redundant code by exports seq_operations functions
  sunrpc: Store cache_detail in seq_file's private directly
  ...

6 files changed, 197 insertions, 150 deletions

diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 2928afffbb81..4a2340a54401 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -44,7 +44,7 @@ static void cache_revisit_request(struct cache_head *item);
 static void cache_init(struct cache_head *h)
 {
         time_t now = seconds_since_boot();
-        h->next = NULL;
+        INIT_HLIST_NODE(&h->cache_list);
         h->flags = 0;
         kref_init(&h->ref);
         h->expiry_time = now + CACHE_NEW_EXPIRY;
@@ -54,15 +54,14 @@ static void cache_init(struct cache_head *h)
 struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail,
                                        struct cache_head *key, int hash)
 {
-        struct cache_head **head,  **hp;
-        struct cache_head *new = NULL, *freeme = NULL;
+        struct cache_head *new = NULL, *freeme = NULL, *tmp = NULL;
+        struct hlist_head *head;
 
         head = &detail->hash_table[hash];
 
         read_lock(&detail->hash_lock);
 
-        for (hp=head; *hp != NULL ; hp = &(*hp)->next) {
-                struct cache_head *tmp = *hp;
+        hlist_for_each_entry(tmp, head, cache_list) {
                 if (detail->match(tmp, key)) {
                         if (cache_is_expired(detail, tmp))
                                 /* This entry is expired, we will discard it. */
@@ -88,12 +87,10 @@ struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail,
         write_lock(&detail->hash_lock);
 
         /* check if entry appeared while we slept */
-        for (hp=head; *hp != NULL ; hp = &(*hp)->next) {
-                struct cache_head *tmp = *hp;
+        hlist_for_each_entry(tmp, head, cache_list) {
                 if (detail->match(tmp, key)) {
                         if (cache_is_expired(detail, tmp)) {
-                                *hp = tmp->next;
-                                tmp->next = NULL;
+                                hlist_del_init(&tmp->cache_list);
                                 detail->entries --;
                                 freeme = tmp;
                                 break;
@@ -104,8 +101,8 @@ struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail,
                         return tmp;
                 }
         }
-        new->next = *head;
-        *head = new;
+
+        hlist_add_head(&new->cache_list, head);
         detail->entries++;
         cache_get(new);
         write_unlock(&detail->hash_lock);
@@ -143,7 +140,6 @@ struct cache_head *sunrpc_cache_update(struct cache_detail *detail,
          * If 'old' is not VALID, we update it directly,
          * otherwise we need to replace it
          */
-        struct cache_head **head;
         struct cache_head *tmp;
 
         if (!test_bit(CACHE_VALID, &old->flags)) {
@@ -168,15 +164,13 @@ struct cache_head *sunrpc_cache_update(struct cache_detail *detail,
         }
         cache_init(tmp);
         detail->init(tmp, old);
-        head = &detail->hash_table[hash];
 
         write_lock(&detail->hash_lock);
         if (test_bit(CACHE_NEGATIVE, &new->flags))
                 set_bit(CACHE_NEGATIVE, &tmp->flags);
         else
                 detail->update(tmp, new);
-        tmp->next = *head;
-        *head = tmp;
+        hlist_add_head(&tmp->cache_list, &detail->hash_table[hash]);
         detail->entries++;
         cache_get(tmp);
         cache_fresh_locked(tmp, new->expiry_time);
@@ -416,28 +410,29 @@ static int cache_clean(void)
         /* find a non-empty bucket in the table */
         while (current_detail &&
                current_index < current_detail->hash_size &&
-               current_detail->hash_table[current_index] == NULL)
+               hlist_empty(&current_detail->hash_table[current_index]))
                 current_index++;
 
         /* find a cleanable entry in the bucket and clean it, or set to next bucket */
 
         if (current_detail && current_index < current_detail->hash_size) {
-                struct cache_head *ch, **cp;
+                struct cache_head *ch = NULL;
                 struct cache_detail *d;
+                struct hlist_head *head;
+                struct hlist_node *tmp;
 
                 write_lock(&current_detail->hash_lock);
 
                 /* Ok, now to clean this strand */
 
-                cp = & current_detail->hash_table[current_index];
-                for (ch = *cp ; ch ; cp = & ch->next, ch = *cp) {
+                head = &current_detail->hash_table[current_index];
+                hlist_for_each_entry_safe(ch, tmp, head, cache_list) {
                         if (current_detail->nextcheck > ch->expiry_time)
                                 current_detail->nextcheck = ch->expiry_time+1;
                         if (!cache_is_expired(current_detail, ch))
                                 continue;
 
-                        *cp = ch->next;
-                        ch->next = NULL;
+                        hlist_del_init(&ch->cache_list);
                         current_detail->entries--;
                         rv = 1;
                         break;
@@ -1270,18 +1265,13 @@ EXPORT_SYMBOL_GPL(qword_get);
  * get a header, then pass each real item in the cache
  */
 
-struct handle {
-        struct cache_detail *cd;
-};
-
-static void *c_start(struct seq_file *m, loff_t *pos)
+void *cache_seq_start(struct seq_file *m, loff_t *pos)
         __acquires(cd->hash_lock)
 {
         loff_t n = *pos;
         unsigned int hash, entry;
         struct cache_head *ch;
-        struct cache_detail *cd = ((struct handle*)m->private)->cd;
-
+        struct cache_detail *cd = m->private;
 
         read_lock(&cd->hash_lock);
         if (!n--)
@@ -1289,7 +1279,7 @@ static void *c_start(struct seq_file *m, loff_t *pos)
         hash = n >> 32;
         entry = n & ((1LL<<32) - 1);
 
-        for (ch=cd->hash_table[hash]; ch; ch=ch->next)
+        hlist_for_each_entry(ch, &cd->hash_table[hash], cache_list)
                 if (!entry--)
                         return ch;
         n &= ~((1LL<<32) - 1);
@@ -1297,51 +1287,57 @@ static void *c_start(struct seq_file *m, loff_t *pos)
                 hash++;
                 n += 1LL<<32;
         } while(hash < cd->hash_size &&
-                cd->hash_table[hash]==NULL);
+                hlist_empty(&cd->hash_table[hash]));
         if (hash >= cd->hash_size)
                 return NULL;
         *pos = n+1;
-        return cd->hash_table[hash];
+        return hlist_entry_safe(cd->hash_table[hash].first,
+                                struct cache_head, cache_list);
 }
+EXPORT_SYMBOL_GPL(cache_seq_start);
 
-static void *c_next(struct seq_file *m, void *p, loff_t *pos)
+void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
 {
         struct cache_head *ch = p;
         int hash = (*pos >> 32);
-        struct cache_detail *cd = ((struct handle*)m->private)->cd;
+        struct cache_detail *cd = m->private;
 
         if (p == SEQ_START_TOKEN)
                 hash = 0;
-        else if (ch->next == NULL) {
+        else if (ch->cache_list.next == NULL) {
                 hash++;
                 *pos += 1LL<<32;
         } else {
                 ++*pos;
-                return ch->next;
+                return hlist_entry_safe(ch->cache_list.next,
+                                        struct cache_head, cache_list);
         }
         *pos &= ~((1LL<<32) - 1);
         while (hash < cd->hash_size &&
-               cd->hash_table[hash] == NULL) {
+               hlist_empty(&cd->hash_table[hash])) {
                 hash++;
                 *pos += 1LL<<32;
         }
         if (hash >= cd->hash_size)
                 return NULL;
         ++*pos;
-        return cd->hash_table[hash];
+        return hlist_entry_safe(cd->hash_table[hash].first,
+                                struct cache_head, cache_list);
 }
+EXPORT_SYMBOL_GPL(cache_seq_next);
 
-static void c_stop(struct seq_file *m, void *p)
+void cache_seq_stop(struct seq_file *m, void *p)
         __releases(cd->hash_lock)
 {
-        struct cache_detail *cd = ((struct handle*)m->private)->cd;
+        struct cache_detail *cd = m->private;
         read_unlock(&cd->hash_lock);
 }
+EXPORT_SYMBOL_GPL(cache_seq_stop);
 
 static int c_show(struct seq_file *m, void *p)
 {
         struct cache_head *cp = p;
-        struct cache_detail *cd = ((struct handle*)m->private)->cd;
+        struct cache_detail *cd = m->private;
 
         if (p == SEQ_START_TOKEN)
                 return cd->cache_show(m, cd, NULL);
@@ -1364,33 +1360,36 @@ static int c_show(struct seq_file *m, void *p)
 }
 
 static const struct seq_operations cache_content_op = {
-        .start  = c_start,
-        .next   = c_next,
-        .stop   = c_stop,
+        .start  = cache_seq_start,
+        .next   = cache_seq_next,
+        .stop   = cache_seq_stop,
         .show   = c_show,
 };
 
 static int content_open(struct inode *inode, struct file *file,
                         struct cache_detail *cd)
 {
-        struct handle *han;
+        struct seq_file *seq;
+        int err;
 
         if (!cd || !try_module_get(cd->owner))
                 return -EACCES;
-        han = __seq_open_private(file, &cache_content_op, sizeof(*han));
-        if (han == NULL) {
+
+        err = seq_open(file, &cache_content_op);
+        if (err) {
                 module_put(cd->owner);
-                return -ENOMEM;
+                return err;
         }
 
-        han->cd = cd;
+        seq = file->private_data;
+        seq->private = cd;
         return 0;
 }
 
 static int content_release(struct inode *inode, struct file *file,
                 struct cache_detail *cd)
 {
-        int ret = seq_release_private(inode, file);
+        int ret = seq_release(inode, file);
         module_put(cd->owner);
         return ret;
 }
@@ -1665,17 +1664,21 @@ EXPORT_SYMBOL_GPL(cache_unregister_net);
 struct cache_detail *cache_create_net(struct cache_detail *tmpl, struct net *net)
 {
         struct cache_detail *cd;
+        int i;
 
         cd = kmemdup(tmpl, sizeof(struct cache_detail), GFP_KERNEL);
         if (cd == NULL)
                 return ERR_PTR(-ENOMEM);
 
-        cd->hash_table = kzalloc(cd->hash_size * sizeof(struct cache_head *),
+        cd->hash_table = kzalloc(cd->hash_size * sizeof(struct hlist_head),
                                  GFP_KERNEL);
         if (cd->hash_table == NULL) {
                 kfree(cd);
                 return ERR_PTR(-ENOMEM);
         }
+
+        for (i = 0; i < cd->hash_size; i++)
+                INIT_HLIST_HEAD(&cd->hash_table[i]);
         cd->net = net;
         return cd;
 }
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 5a16d8d8c831..a8f579df14d8 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -34,36 +34,19 @@
 
 static void svc_unregister(const struct svc_serv *serv, struct net *net);
 
-#define svc_serv_is_pooled(serv)    ((serv)->sv_function)
+#define svc_serv_is_pooled(serv)    ((serv)->sv_ops->svo_function)
 
-/*
- * Mode for mapping cpus to pools.
- */
-enum {
-        SVC_POOL_AUTO = -1,     /* choose one of the others */
-        SVC_POOL_GLOBAL,        /* no mapping, just a single global pool
-                                 * (legacy & UP mode) */
-        SVC_POOL_PERCPU,        /* one pool per cpu */
-        SVC_POOL_PERNODE        /* one pool per numa node */
-};
 #define SVC_POOL_DEFAULT        SVC_POOL_GLOBAL
 
 /*
  * Structure for mapping cpus to pools and vice versa.
  * Setup once during sunrpc initialisation.
  */
-static struct svc_pool_map {
-        int count;                      /* How many svc_servs use us */
-        int mode;                       /* Note: int not enum to avoid
-                                         * warnings about "enumeration value
-                                         * not handled in switch" */
-        unsigned int npools;
-        unsigned int *pool_to;          /* maps pool id to cpu or node */
-        unsigned int *to_pool;          /* maps cpu or node to pool id */
-} svc_pool_map = {
-        .count = 0,
+struct svc_pool_map svc_pool_map = {
         .mode = SVC_POOL_DEFAULT
 };
+EXPORT_SYMBOL_GPL(svc_pool_map);
+
 static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */
 
 static int
@@ -236,7 +219,7 @@ svc_pool_map_init_pernode(struct svc_pool_map *m)
  * vice versa).  Initialise the map if we're the first user.
  * Returns the number of pools.
  */
-static unsigned int
+unsigned int
 svc_pool_map_get(void)
 {
         struct svc_pool_map *m = &svc_pool_map;
@@ -271,7 +254,7 @@ svc_pool_map_get(void)
         mutex_unlock(&svc_pool_map_mutex);
         return m->npools;
 }
-
+EXPORT_SYMBOL_GPL(svc_pool_map_get);
 
 /*
  * Drop a reference to the global map of cpus to pools.
@@ -280,7 +263,7 @@ svc_pool_map_get(void)
  * mode using the pool_mode module option without
  * rebooting or re-loading sunrpc.ko.
  */
-static void
+void
 svc_pool_map_put(void)
 {
         struct svc_pool_map *m = &svc_pool_map;
@@ -297,7 +280,7 @@ svc_pool_map_put(void)
 
         mutex_unlock(&svc_pool_map_mutex);
 }
-
+EXPORT_SYMBOL_GPL(svc_pool_map_put);
 
 static int svc_pool_map_get_node(unsigned int pidx)
 {
@@ -423,7 +406,7 @@ EXPORT_SYMBOL_GPL(svc_bind);
  */
 static struct svc_serv *
 __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
-             void (*shutdown)(struct svc_serv *serv, struct net *net))
+             struct svc_serv_ops *ops)
 {
         struct svc_serv *serv;
         unsigned int vers;
@@ -440,7 +423,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
                 bufsize = RPCSVC_MAXPAYLOAD;
         serv->sv_max_payload = bufsize? bufsize : 4096;
         serv->sv_max_mesg  = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
-        serv->sv_shutdown  = shutdown;
+        serv->sv_ops = ops;
         xdrsize = 0;
         while (prog) {
                 prog->pg_lovers = prog->pg_nvers-1;
@@ -486,26 +469,22 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
 
 struct svc_serv *
 svc_create(struct svc_program *prog, unsigned int bufsize,
-           void (*shutdown)(struct svc_serv *serv, struct net *net))
+           struct svc_serv_ops *ops)
 {
-        return __svc_create(prog, bufsize, /*npools*/1, shutdown);
+        return __svc_create(prog, bufsize, /*npools*/1, ops);
 }
 EXPORT_SYMBOL_GPL(svc_create);
 
 struct svc_serv *
 svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
-                  void (*shutdown)(struct svc_serv *serv, struct net *net),
-                  svc_thread_fn func, struct module *mod)
+                  struct svc_serv_ops *ops)
 {
         struct svc_serv *serv;
         unsigned int npools = svc_pool_map_get();
 
-        serv = __svc_create(prog, bufsize, npools, shutdown);
+        serv = __svc_create(prog, bufsize, npools, ops);
         if (!serv)
                 goto out_err;
-
-        serv->sv_function = func;
-        serv->sv_module = mod;
         return serv;
 out_err:
         svc_pool_map_put();
@@ -517,8 +496,8 @@ void svc_shutdown_net(struct svc_serv *serv, struct net *net)
 {
         svc_close_net(serv, net);
 
-        if (serv->sv_shutdown)
-                serv->sv_shutdown(serv, net);
+        if (serv->sv_ops->svo_shutdown)
+                serv->sv_ops->svo_shutdown(serv, net);
 }
 EXPORT_SYMBOL_GPL(svc_shutdown_net);
 
@@ -604,40 +583,52 @@ svc_release_buffer(struct svc_rqst *rqstp)
 }
 
 struct svc_rqst *
-svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
+svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)
 {
         struct svc_rqst *rqstp;
 
         rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
         if (!rqstp)
-                goto out_enomem;
+                return rqstp;
 
-        serv->sv_nrthreads++;
         __set_bit(RQ_BUSY, &rqstp->rq_flags);
         spin_lock_init(&rqstp->rq_lock);
         rqstp->rq_server = serv;
         rqstp->rq_pool = pool;
-        spin_lock_bh(&pool->sp_lock);
-        pool->sp_nrthreads++;
-        list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
-        spin_unlock_bh(&pool->sp_lock);
 
         rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
         if (!rqstp->rq_argp)
-                goto out_thread;
+                goto out_enomem;
 
         rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
         if (!rqstp->rq_resp)
-                goto out_thread;
+                goto out_enomem;
 
         if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
-                goto out_thread;
+                goto out_enomem;
 
         return rqstp;
-out_thread:
-        svc_exit_thread(rqstp);
 out_enomem:
-        return ERR_PTR(-ENOMEM);
+        svc_rqst_free(rqstp);
+        return NULL;
+}
+EXPORT_SYMBOL_GPL(svc_rqst_alloc);
+
+struct svc_rqst *
+svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
+{
+        struct svc_rqst *rqstp;
+
+        rqstp = svc_rqst_alloc(serv, pool, node);
+        if (!rqstp)
+                return ERR_PTR(-ENOMEM);
+
+        serv->sv_nrthreads++;
+        spin_lock_bh(&pool->sp_lock);
+        pool->sp_nrthreads++;
+        list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
+        spin_unlock_bh(&pool->sp_lock);
+        return rqstp;
 }
 EXPORT_SYMBOL_GPL(svc_prepare_thread);
 
@@ -739,12 +730,12 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
                         break;
                 }
 
-                __module_get(serv->sv_module);
-                task = kthread_create_on_node(serv->sv_function, rqstp,
+                __module_get(serv->sv_ops->svo_module);
+                task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,
                                               node, "%s", serv->sv_name);
                 if (IS_ERR(task)) {
                         error = PTR_ERR(task);
-                        module_put(serv->sv_module);
+                        module_put(serv->sv_ops->svo_module);
                         svc_exit_thread(rqstp);
                         break;
                 }
@@ -772,15 +763,21 @@ EXPORT_SYMBOL_GPL(svc_set_num_threads);
  * mutex" for the service.
  */
 void
-svc_exit_thread(struct svc_rqst *rqstp)
+svc_rqst_free(struct svc_rqst *rqstp)
 {
-        struct svc_serv *serv = rqstp->rq_server;
-        struct svc_pool *pool = rqstp->rq_pool;
-
         svc_release_buffer(rqstp);
         kfree(rqstp->rq_resp);
         kfree(rqstp->rq_argp);
         kfree(rqstp->rq_auth_data);
+        kfree_rcu(rqstp, rq_rcu_head);
+}
+EXPORT_SYMBOL_GPL(svc_rqst_free);
+
+void
+svc_exit_thread(struct svc_rqst *rqstp)
+{
+        struct svc_serv *serv = rqstp->rq_server;
+        struct svc_pool *pool = rqstp->rq_pool;
 
         spin_lock_bh(&pool->sp_lock);
         pool->sp_nrthreads--;
@@ -788,7 +785,7 @@ svc_exit_thread(struct svc_rqst *rqstp)
                 list_del_rcu(&rqstp->rq_all);
         spin_unlock_bh(&pool->sp_lock);
 
-        kfree_rcu(rqstp, rq_rcu_head);
+        svc_rqst_free(rqstp);
 
         /* Release the server */
         if (serv)
diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index 163ac45c3639..a6cbb2104667 100644
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -24,7 +24,6 @@ static int svc_deferred_recv(struct svc_rqst *rqstp);
 static struct cache_deferred_req *svc_defer(struct cache_req *req);
 static void svc_age_temp_xprts(unsigned long closure);
 static void svc_delete_xprt(struct svc_xprt *xprt);
-static void svc_xprt_do_enqueue(struct svc_xprt *xprt);
 
 /* apparently the "standard" is that clients close
  * idle connections after 5 minutes, servers after
@@ -225,12 +224,12 @@ static void svc_xprt_received(struct svc_xprt *xprt)
         }
 
         /* As soon as we clear busy, the xprt could be closed and
-         * 'put', so we need a reference to call svc_xprt_do_enqueue with:
+         * 'put', so we need a reference to call svc_enqueue_xprt with:
          */
         svc_xprt_get(xprt);
         smp_mb__before_atomic();
         clear_bit(XPT_BUSY, &xprt->xpt_flags);
-        svc_xprt_do_enqueue(xprt);
+        xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt);
         svc_xprt_put(xprt);
 }
 
@@ -320,7 +319,7 @@ static bool svc_xprt_has_something_to_do(struct svc_xprt *xprt)
         return false;
 }
 
-static void svc_xprt_do_enqueue(struct svc_xprt *xprt)
+void svc_xprt_do_enqueue(struct svc_xprt *xprt)
 {
         struct svc_pool *pool;
         struct svc_rqst *rqstp = NULL;
@@ -402,6 +401,7 @@ redo_search:
 out:
         trace_svc_xprt_do_enqueue(xprt, rqstp);
 }
+EXPORT_SYMBOL_GPL(svc_xprt_do_enqueue);
 
 /*
  * Queue up a transport with data pending. If there are idle nfsd
@@ -412,7 +412,7 @@ void svc_xprt_enqueue(struct svc_xprt *xprt)
 {
         if (test_bit(XPT_BUSY, &xprt->xpt_flags))
                 return;
-        svc_xprt_do_enqueue(xprt);
+        xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt);
 }
 EXPORT_SYMBOL_GPL(svc_xprt_enqueue);
 
diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index d25cd430f9ff..1dfae8317065 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -136,6 +136,79 @@ static dma_addr_t dma_map_xdr(struct svcxprt_rdma *xprt,
         return dma_addr;
 }
 
+/* Returns the address of the first read chunk or <nul> if no read chunk
+ * is present
+ */
+struct rpcrdma_read_chunk *
+svc_rdma_get_read_chunk(struct rpcrdma_msg *rmsgp)
+{
+        struct rpcrdma_read_chunk *ch =
+                (struct rpcrdma_read_chunk *)&rmsgp->rm_body.rm_chunks[0];
+
+        if (ch->rc_discrim == xdr_zero)
+                return NULL;
+        return ch;
+}
+
+/* Returns the address of the first read write array element or <nul>
+ * if no write array list is present
+ */
+static struct rpcrdma_write_array *
+svc_rdma_get_write_array(struct rpcrdma_msg *rmsgp)
+{
+        if (rmsgp->rm_body.rm_chunks[0] != xdr_zero ||
+            rmsgp->rm_body.rm_chunks[1] == xdr_zero)
+                return NULL;
+        return (struct rpcrdma_write_array *)&rmsgp->rm_body.rm_chunks[1];
+}
+
+/* Returns the address of the first reply array element or <nul> if no
+ * reply array is present
+ */
+static struct rpcrdma_write_array *
+svc_rdma_get_reply_array(struct rpcrdma_msg *rmsgp)
+{
+        struct rpcrdma_read_chunk *rch;
+        struct rpcrdma_write_array *wr_ary;
+        struct rpcrdma_write_array *rp_ary;
+
+        /* XXX: Need to fix when reply chunk may occur with read list
+         *      and/or write list.
+         */
+        if (rmsgp->rm_body.rm_chunks[0] != xdr_zero ||
+            rmsgp->rm_body.rm_chunks[1] != xdr_zero)
+                return NULL;
+
+        rch = svc_rdma_get_read_chunk(rmsgp);
+        if (rch) {
+                while (rch->rc_discrim != xdr_zero)
+                        rch++;
+
+                /* The reply chunk follows an empty write array located
+                 * at 'rc_position' here. The reply array is at rc_target.
+                 */
+                rp_ary = (struct rpcrdma_write_array *)&rch->rc_target;
+                goto found_it;
+        }
+
+        wr_ary = svc_rdma_get_write_array(rmsgp);
+        if (wr_ary) {
+                int chunk = be32_to_cpu(wr_ary->wc_nchunks);
+
+                rp_ary = (struct rpcrdma_write_array *)
+                         &wr_ary->wc_array[chunk].wc_target.rs_length;
+                goto found_it;
+        }
+
+        /* No read list, no write list */
+        rp_ary = (struct rpcrdma_write_array *)&rmsgp->rm_body.rm_chunks[2];
+
+ found_it:
+        if (rp_ary->wc_discrim == xdr_zero)
+                return NULL;
+        return rp_ary;
+}
+
 /* Assumptions:
  * - The specified write_len can be represented in sc_max_sge * PAGE_SIZE
  */
@@ -384,6 +457,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
                       int byte_count)
 {
         struct ib_send_wr send_wr;
+        u32 xdr_off;
         int sge_no;
         int sge_bytes;
         int page_no;
@@ -418,8 +492,8 @@ static int send_reply(struct svcxprt_rdma *rdma,
         ctxt->direction = DMA_TO_DEVICE;
 
         /* Map the payload indicated by 'byte_count' */
+        xdr_off = 0;
         for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) {
-                int xdr_off = 0;
                 sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count);
                 byte_count -= sge_bytes;
                 ctxt->sge[sge_no].addr =
@@ -457,6 +531,13 @@ static int send_reply(struct svcxprt_rdma *rdma,
         }
         rqstp->rq_next_page = rqstp->rq_respages + 1;
 
+        /* The loop above bumps sc_dma_used for each sge. The
+         * xdr_buf.tail gets a separate sge, but resides in the
+         * same page as xdr_buf.head. Don't count it twice.
+         */
+        if (sge_no > ctxt->count)
+                atomic_dec(&rdma->sc_dma_used);
+
         if (sge_no > rdma->sc_max_sge) {
                 pr_err("svcrdma: Too many sges (%d)\n", sge_no);
                 goto err;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 6b36279e4288..21e40365042c 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -91,7 +91,7 @@ struct svc_xprt_class svc_rdma_class = {
         .xcl_name = "rdma",
         .xcl_owner = THIS_MODULE,
         .xcl_ops = &svc_rdma_ops,
-        .xcl_max_payload = RPCRDMA_MAXPAYLOAD,
+        .xcl_max_payload = RPCSVC_MAXPAYLOAD_RDMA,
         .xcl_ident = XPRT_TRANSPORT_RDMA,
 };
 
@@ -659,6 +659,7 @@ static int rdma_cma_handler(struct rdma_cm_id *cma_id,
                 if (xprt) {
                         set_bit(XPT_CLOSE, &xprt->xpt_flags);
                         svc_xprt_enqueue(xprt);
+                        svc_xprt_put(xprt);
                 }
                 break;
         default:
@@ -1201,40 +1202,6 @@ static int svc_rdma_secure_port(struct svc_rqst *rqstp)
         return 1;
 }
 
-/*
- * Attempt to register the kvec representing the RPC memory with the
- * device.
- *
- * Returns:
- *  NULL : The device does not support fastreg or there were no more
- *         fastreg mr.
- *  frmr : The kvec register request was successfully posted.
- *    <0 : An error was encountered attempting to register the kvec.
- */
-int svc_rdma_fastreg(struct svcxprt_rdma *xprt,
-                     struct svc_rdma_fastreg_mr *frmr)
-{
-        struct ib_send_wr fastreg_wr;
-        u8 key;
-
-        /* Bump the key */
-        key = (u8)(frmr->mr->lkey & 0x000000FF);
-        ib_update_fast_reg_key(frmr->mr, ++key);
-
-        /* Prepare FASTREG WR */
-        memset(&fastreg_wr, 0, sizeof fastreg_wr);
-        fastreg_wr.opcode = IB_WR_FAST_REG_MR;
-        fastreg_wr.send_flags = IB_SEND_SIGNALED;
-        fastreg_wr.wr.fast_reg.iova_start = (unsigned long)frmr->kva;
-        fastreg_wr.wr.fast_reg.page_list = frmr->page_list;
-        fastreg_wr.wr.fast_reg.page_list_len = frmr->page_list_len;
-        fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
-        fastreg_wr.wr.fast_reg.length = frmr->map_len;
-        fastreg_wr.wr.fast_reg.access_flags = frmr->access_flags;
-        fastreg_wr.wr.fast_reg.rkey = frmr->mr->lkey;
-        return svc_rdma_send(xprt, &fastreg_wr);
-}
-
 int svc_rdma_send(struct svcxprt_rdma *xprt, struct ib_send_wr *wr)
 {
         struct ib_send_wr *bad_wr, *n_wr;
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index f49dd8b38122..e718d0959af3 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -51,7 +51,6 @@
 #include <linux/sunrpc/clnt.h>          /* rpc_xprt */
 #include <linux/sunrpc/rpc_rdma.h>      /* RPC/RDMA protocol */
 #include <linux/sunrpc/xprtrdma.h>      /* xprt parameters */
-#include <linux/sunrpc/svc.h>           /* RPCSVC_MAXPAYLOAD */
 
 #define RDMA_RESOLVE_TIMEOUT    (5000)  /* 5 seconds */
 #define RDMA_CONNECT_RETRY_MAX  (2)     /* retries if no listener backlog */