3 files changed, 58 insertions, 21 deletions

diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c
index 175b5135bdcf..e317583fcc73 100644
--- a/net/9p/trans_virtio.c
+++ b/net/9p/trans_virtio.c
@@ -263,7 +263,6 @@ p9_virtio_request(struct p9_client *client, struct p9_req_t *req)
 {
         int in, out, inp, outp;
         struct virtio_chan *chan = client->trans;
-        char *rdata = (char *)req->rc+sizeof(struct p9_fcall);
         unsigned long flags;
         size_t pdata_off = 0;
         struct trans_rpage_info *rpinfo = NULL;
@@ -346,7 +345,8 @@ req_retry_pinned:
                  * Arrange in such a way that server places header in the
                  * alloced memory and payload onto the user buffer.
                  */
-                inp = pack_sg_list(chan->sg, out, VIRTQUEUE_NUM, rdata, 11);
+                inp = pack_sg_list(chan->sg, out,
+                                   VIRTQUEUE_NUM, req->rc->sdata, 11);
                 /*
                  * Running executables in the filesystem may result in
                  * a read request with kernel buffer as opposed to user buffer.
@@ -366,8 +366,8 @@ req_retry_pinned:
                 }
                 in += inp;
         } else {
-                in = pack_sg_list(chan->sg, out, VIRTQUEUE_NUM, rdata,
-                                req->rc->capacity);
+                in = pack_sg_list(chan->sg, out, VIRTQUEUE_NUM,
+                                  req->rc->sdata, req->rc->capacity);
         }
 
         err = virtqueue_add_buf(chan->vq, chan->sg, out, in, req->tc);
@@ -592,7 +592,14 @@ static struct p9_trans_module p9_virtio_trans = {
         .close = p9_virtio_close,
         .request = p9_virtio_request,
         .cancel = p9_virtio_cancel,
-        .maxsize = PAGE_SIZE*VIRTQUEUE_NUM,
+
+        /*
+         * We leave one entry for input and one entry for response
+         * headers. We also skip one more entry to accomodate, address
+         * that are not at page boundary, that can result in an extra
+         * page in zero copy.
+         */
+        .maxsize = PAGE_SIZE * (VIRTQUEUE_NUM - 3),
         .pref = P9_TRANS_PREF_PAYLOAD_SEP,
         .def = 0,
         .owner = THIS_MODULE,
diff --git a/net/ceph/msgpool.c b/net/ceph/msgpool.c
index d5f2d97ac05c..1f4cb30a42c5 100644
--- a/net/ceph/msgpool.c
+++ b/net/ceph/msgpool.c
@@ -7,27 +7,37 @@
 
 #include <linux/ceph/msgpool.h>
 
-static void *alloc_fn(gfp_t gfp_mask, void *arg)
+static void *msgpool_alloc(gfp_t gfp_mask, void *arg)
 {
         struct ceph_msgpool *pool = arg;
-        void *p;
+        struct ceph_msg *msg;
 
-        p = ceph_msg_new(0, pool->front_len, gfp_mask);
-        if (!p)
-                pr_err("msgpool %s alloc failed\n", pool->name);
-        return p;
+        msg = ceph_msg_new(0, pool->front_len, gfp_mask);
+        if (!msg) {
+                dout("msgpool_alloc %s failed\n", pool->name);
+        } else {
+                dout("msgpool_alloc %s %p\n", pool->name, msg);
+                msg->pool = pool;
+        }
+        return msg;
 }
 
-static void free_fn(void *element, void *arg)
+static void msgpool_free(void *element, void *arg)
 {
-        ceph_msg_put(element);
+        struct ceph_msgpool *pool = arg;
+        struct ceph_msg *msg = element;
+
+        dout("msgpool_release %s %p\n", pool->name, msg);
+        msg->pool = NULL;
+        ceph_msg_put(msg);
 }
 
 int ceph_msgpool_init(struct ceph_msgpool *pool,
                       int front_len, int size, bool blocking, const char *name)
 {
+        dout("msgpool %s init\n", name);
         pool->front_len = front_len;
-        pool->pool = mempool_create(size, alloc_fn, free_fn, pool);
+        pool->pool = mempool_create(size, msgpool_alloc, msgpool_free, pool);
         if (!pool->pool)
                 return -ENOMEM;
         pool->name = name;
@@ -36,14 +46,17 @@ int ceph_msgpool_init(struct ceph_msgpool *pool,
 
 void ceph_msgpool_destroy(struct ceph_msgpool *pool)
 {
+        dout("msgpool %s destroy\n", pool->name);
         mempool_destroy(pool->pool);
 }
 
 struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool,
                                   int front_len)
 {
+        struct ceph_msg *msg;
+
         if (front_len > pool->front_len) {
-                pr_err("msgpool_get pool %s need front %d, pool size is %d\n",
+                dout("msgpool_get %s need front %d, pool size is %d\n",
                        pool->name, front_len, pool->front_len);
                 WARN_ON(1);
 
@@ -51,14 +64,19 @@ struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool,
                 return ceph_msg_new(0, front_len, GFP_NOFS);
         }
 
-        return mempool_alloc(pool->pool, GFP_NOFS);
+        msg = mempool_alloc(pool->pool, GFP_NOFS);
+        dout("msgpool_get %s %p\n", pool->name, msg);
+        return msg;
 }
 
 void ceph_msgpool_put(struct ceph_msgpool *pool, struct ceph_msg *msg)
 {
+        dout("msgpool_put %s %p\n", pool->name, msg);
+
         /* reset msg front_len; user may have changed it */
         msg->front.iov_len = pool->front_len;
         msg->hdr.front_len = cpu_to_le32(pool->front_len);
 
         kref_init(&msg->kref);  /* retake single ref */
+        mempool_free(msg, pool->pool);
 }
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index ce310eee708d..16836a7df7a6 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -685,6 +685,18 @@ static void __remove_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd)
         put_osd(osd);
 }
 
+static void remove_all_osds(struct ceph_osd_client *osdc)
+{
+        dout("__remove_old_osds %p\n", osdc);
+        mutex_lock(&osdc->request_mutex);
+        while (!RB_EMPTY_ROOT(&osdc->osds)) {
+                struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds),
+                                                struct ceph_osd, o_node);
+                __remove_osd(osdc, osd);
+        }
+        mutex_unlock(&osdc->request_mutex);
+}
+
 static void __move_osd_to_lru(struct ceph_osd_client *osdc,
                               struct ceph_osd *osd)
 {
@@ -701,14 +713,14 @@ static void __remove_osd_from_lru(struct ceph_osd *osd)
                 list_del_init(&osd->o_osd_lru);
 }
 
-static void remove_old_osds(struct ceph_osd_client *osdc, int remove_all)
+static void remove_old_osds(struct ceph_osd_client *osdc)
 {
         struct ceph_osd *osd, *nosd;
 
         dout("__remove_old_osds %p\n", osdc);
         mutex_lock(&osdc->request_mutex);
         list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) {
-                if (!remove_all && time_before(jiffies, osd->lru_ttl))
+                if (time_before(jiffies, osd->lru_ttl))
                         break;
                 __remove_osd(osdc, osd);
         }
@@ -751,6 +763,7 @@ static void __insert_osd(struct ceph_osd_client *osdc, struct ceph_osd *new)
         struct rb_node *parent = NULL;
         struct ceph_osd *osd = NULL;
 
+        dout("__insert_osd %p osd%d\n", new, new->o_osd);
         while (*p) {
                 parent = *p;
                 osd = rb_entry(parent, struct ceph_osd, o_node);
@@ -1144,7 +1157,7 @@ static void handle_osds_timeout(struct work_struct *work)
 
         dout("osds timeout\n");
         down_read(&osdc->map_sem);
-        remove_old_osds(osdc, 0);
+        remove_old_osds(osdc);
         up_read(&osdc->map_sem);
 
         schedule_delayed_work(&osdc->osds_timeout_work,
@@ -1862,8 +1875,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc)
                 ceph_osdmap_destroy(osdc->osdmap);
                 osdc->osdmap = NULL;
         }
-        remove_old_osds(osdc, 1);
-        WARN_ON(!RB_EMPTY_ROOT(&osdc->osds));
+        remove_all_osds(osdc);
         mempool_destroy(osdc->req_mempool);
         ceph_msgpool_destroy(&osdc->msgpool_op);
         ceph_msgpool_destroy(&osdc->msgpool_op_reply);