bcache: Move sector allocator to alloc.c

Just reorganizing things a bit.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>

4 files changed, 189 insertions, 186 deletions

diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c
index b9bd5866055d..4970ddc6a7f6 100644
--- a/drivers/md/bcache/alloc.c
+++ b/drivers/md/bcache/alloc.c
@@ -487,8 +487,188 @@ int bch_bucket_alloc_set(struct cache_set *c, unsigned watermark,
         return ret;
 }
 
+/* Sector allocator */
+
+struct open_bucket {
+        struct list_head        list;
+        unsigned                last_write_point;
+        unsigned                sectors_free;
+        BKEY_PADDED(key);
+};
+
+/*
+ * We keep multiple buckets open for writes, and try to segregate different
+ * write streams for better cache utilization: first we look for a bucket where
+ * the last write to it was sequential with the current write, and failing that
+ * we look for a bucket that was last used by the same task.
+ *
+ * The ideas is if you've got multiple tasks pulling data into the cache at the
+ * same time, you'll get better cache utilization if you try to segregate their
+ * data and preserve locality.
+ *
+ * For example, say you've starting Firefox at the same time you're copying a
+ * bunch of files. Firefox will likely end up being fairly hot and stay in the
+ * cache awhile, but the data you copied might not be; if you wrote all that
+ * data to the same buckets it'd get invalidated at the same time.
+ *
+ * Both of those tasks will be doing fairly random IO so we can't rely on
+ * detecting sequential IO to segregate their data, but going off of the task
+ * should be a sane heuristic.
+ */
+static struct open_bucket *pick_data_bucket(struct cache_set *c,
+                                            const struct bkey *search,
+                                            unsigned write_point,
+                                            struct bkey *alloc)
+{
+        struct open_bucket *ret, *ret_task = NULL;
+
+        list_for_each_entry_reverse(ret, &c->data_buckets, list)
+                if (!bkey_cmp(&ret->key, search))
+                        goto found;
+                else if (ret->last_write_point == write_point)
+                        ret_task = ret;
+
+        ret = ret_task ?: list_first_entry(&c->data_buckets,
+                                           struct open_bucket, list);
+found:
+        if (!ret->sectors_free && KEY_PTRS(alloc)) {
+                ret->sectors_free = c->sb.bucket_size;
+                bkey_copy(&ret->key, alloc);
+                bkey_init(alloc);
+        }
+
+        if (!ret->sectors_free)
+                ret = NULL;
+
+        return ret;
+}
+
+/*
+ * Allocates some space in the cache to write to, and k to point to the newly
+ * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
+ * end of the newly allocated space).
+ *
+ * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
+ * sectors were actually allocated.
+ *
+ * If s->writeback is true, will not fail.
+ */
+bool bch_alloc_sectors(struct cache_set *c, struct bkey *k, unsigned sectors,
+                       unsigned write_point, unsigned write_prio, bool wait)
+{
+        struct open_bucket *b;
+        BKEY_PADDED(key) alloc;
+        unsigned i;
+
+        /*
+         * We might have to allocate a new bucket, which we can't do with a
+         * spinlock held. So if we have to allocate, we drop the lock, allocate
+         * and then retry. KEY_PTRS() indicates whether alloc points to
+         * allocated bucket(s).
+         */
+
+        bkey_init(&alloc.key);
+        spin_lock(&c->data_bucket_lock);
+
+        while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
+                unsigned watermark = write_prio
+                        ? WATERMARK_MOVINGGC
+                        : WATERMARK_NONE;
+
+                spin_unlock(&c->data_bucket_lock);
+
+                if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
+                        return false;
+
+                spin_lock(&c->data_bucket_lock);
+        }
+
+        /*
+         * If we had to allocate, we might race and not need to allocate the
+         * second time we call find_data_bucket(). If we allocated a bucket but
+         * didn't use it, drop the refcount bch_bucket_alloc_set() took:
+         */
+        if (KEY_PTRS(&alloc.key))
+                __bkey_put(c, &alloc.key);
+
+        for (i = 0; i < KEY_PTRS(&b->key); i++)
+                EBUG_ON(ptr_stale(c, &b->key, i));
+
+        /* Set up the pointer to the space we're allocating: */
+
+        for (i = 0; i < KEY_PTRS(&b->key); i++)
+                k->ptr[i] = b->key.ptr[i];
+
+        sectors = min(sectors, b->sectors_free);
+
+        SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
+        SET_KEY_SIZE(k, sectors);
+        SET_KEY_PTRS(k, KEY_PTRS(&b->key));
+
+        /*
+         * Move b to the end of the lru, and keep track of what this bucket was
+         * last used for:
+         */
+        list_move_tail(&b->list, &c->data_buckets);
+        bkey_copy_key(&b->key, k);
+        b->last_write_point = write_point;
+
+        b->sectors_free -= sectors;
+
+        for (i = 0; i < KEY_PTRS(&b->key); i++) {
+                SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
+
+                atomic_long_add(sectors,
+                                &PTR_CACHE(c, &b->key, i)->sectors_written);
+        }
+
+        if (b->sectors_free < c->sb.block_size)
+                b->sectors_free = 0;
+
+        /*
+         * k takes refcounts on the buckets it points to until it's inserted
+         * into the btree, but if we're done with this bucket we just transfer
+         * get_data_bucket()'s refcount.
+         */
+        if (b->sectors_free)
+                for (i = 0; i < KEY_PTRS(&b->key); i++)
+                        atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
+
+        spin_unlock(&c->data_bucket_lock);
+        return true;
+}
+
 /* Init */
 
+void bch_open_buckets_free(struct cache_set *c)
+{
+        struct open_bucket *b;
+
+        while (!list_empty(&c->data_buckets)) {
+                b = list_first_entry(&c->data_buckets,
+                                     struct open_bucket, list);
+                list_del(&b->list);
+                kfree(b);
+        }
+}
+
+int bch_open_buckets_alloc(struct cache_set *c)
+{
+        int i;
+
+        spin_lock_init(&c->data_bucket_lock);
+
+        for (i = 0; i < 6; i++) {
+                struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
+                if (!b)
+                        return -ENOMEM;
+
+                list_add(&b->list, &c->data_buckets);
+        }
+
+        return 0;
+}
+
 int bch_cache_allocator_start(struct cache *ca)
 {
         struct task_struct *k = kthread_run(bch_allocator_thread,
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 20fe96c121d9..e32f6fd91755 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -1170,6 +1170,8 @@ int __bch_bucket_alloc_set(struct cache_set *, unsigned,
                            struct bkey *, int, bool);
 int bch_bucket_alloc_set(struct cache_set *, unsigned,
                          struct bkey *, int, bool);
+bool bch_alloc_sectors(struct cache_set *, struct bkey *, unsigned,
+                       unsigned, unsigned, bool);
 
 __printf(2, 3)
 bool bch_cache_set_error(struct cache_set *, const char *, ...);
@@ -1210,6 +1212,8 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *);
 void bch_btree_cache_free(struct cache_set *);
 int bch_btree_cache_alloc(struct cache_set *);
 void bch_moving_init_cache_set(struct cache_set *);
+int bch_open_buckets_alloc(struct cache_set *);
+void bch_open_buckets_free(struct cache_set *);
 
 int bch_cache_allocator_start(struct cache *ca);
 int bch_cache_allocator_init(struct cache *ca);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 05c7c216f65e..cf7850a7592c 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -255,186 +255,6 @@ static void bch_data_insert_keys(struct closure *cl)
         closure_return(cl);
 }
 
-struct open_bucket {
-        struct list_head        list;
-        struct task_struct      *last;
-        unsigned                sectors_free;
-        BKEY_PADDED(key);
-};
-
-void bch_open_buckets_free(struct cache_set *c)
-{
-        struct open_bucket *b;
-
-        while (!list_empty(&c->data_buckets)) {
-                b = list_first_entry(&c->data_buckets,
-                                     struct open_bucket, list);
-                list_del(&b->list);
-                kfree(b);
-        }
-}
-
-int bch_open_buckets_alloc(struct cache_set *c)
-{
-        int i;
-
-        spin_lock_init(&c->data_bucket_lock);
-
-        for (i = 0; i < 6; i++) {
-                struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
-                if (!b)
-                        return -ENOMEM;
-
-                list_add(&b->list, &c->data_buckets);
-        }
-
-        return 0;
-}
-
-/*
- * We keep multiple buckets open for writes, and try to segregate different
- * write streams for better cache utilization: first we look for a bucket where
- * the last write to it was sequential with the current write, and failing that
- * we look for a bucket that was last used by the same task.
- *
- * The ideas is if you've got multiple tasks pulling data into the cache at the
- * same time, you'll get better cache utilization if you try to segregate their
- * data and preserve locality.
- *
- * For example, say you've starting Firefox at the same time you're copying a
- * bunch of files. Firefox will likely end up being fairly hot and stay in the
- * cache awhile, but the data you copied might not be; if you wrote all that
- * data to the same buckets it'd get invalidated at the same time.
- *
- * Both of those tasks will be doing fairly random IO so we can't rely on
- * detecting sequential IO to segregate their data, but going off of the task
- * should be a sane heuristic.
- */
-static struct open_bucket *pick_data_bucket(struct cache_set *c,
-                                            const struct bkey *search,
-                                            struct task_struct *task,
-                                            struct bkey *alloc)
-{
-        struct open_bucket *ret, *ret_task = NULL;
-
-        list_for_each_entry_reverse(ret, &c->data_buckets, list)
-                if (!bkey_cmp(&ret->key, search))
-                        goto found;
-                else if (ret->last == task)
-                        ret_task = ret;
-
-        ret = ret_task ?: list_first_entry(&c->data_buckets,
-                                           struct open_bucket, list);
-found:
-        if (!ret->sectors_free && KEY_PTRS(alloc)) {
-                ret->sectors_free = c->sb.bucket_size;
-                bkey_copy(&ret->key, alloc);
-                bkey_init(alloc);
-        }
-
-        if (!ret->sectors_free)
-                ret = NULL;
-
-        return ret;
-}
-
-/*
- * Allocates some space in the cache to write to, and k to point to the newly
- * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
- * end of the newly allocated space).
- *
- * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
- * sectors were actually allocated.
- *
- * If s->writeback is true, will not fail.
- */
-static bool bch_alloc_sectors(struct data_insert_op *op,
-                              struct bkey *k, unsigned sectors)
-{
-        struct cache_set *c = op->c;
-        struct open_bucket *b;
-        BKEY_PADDED(key) alloc;
-        unsigned i;
-
-        /*
-         * We might have to allocate a new bucket, which we can't do with a
-         * spinlock held. So if we have to allocate, we drop the lock, allocate
-         * and then retry. KEY_PTRS() indicates whether alloc points to
-         * allocated bucket(s).
-         */
-
-        bkey_init(&alloc.key);
-        spin_lock(&c->data_bucket_lock);
-
-        while (!(b = pick_data_bucket(c, k, op->task, &alloc.key))) {
-                unsigned watermark = op->write_prio
-                        ? WATERMARK_MOVINGGC
-                        : WATERMARK_NONE;
-
-                spin_unlock(&c->data_bucket_lock);
-
-                if (bch_bucket_alloc_set(c, watermark, &alloc.key,
-                                         1, op->writeback))
-                        return false;
-
-                spin_lock(&c->data_bucket_lock);
-        }
-
-        /*
-         * If we had to allocate, we might race and not need to allocate the
-         * second time we call find_data_bucket(). If we allocated a bucket but
-         * didn't use it, drop the refcount bch_bucket_alloc_set() took:
-         */
-        if (KEY_PTRS(&alloc.key))
-                __bkey_put(c, &alloc.key);
-
-        for (i = 0; i < KEY_PTRS(&b->key); i++)
-                EBUG_ON(ptr_stale(c, &b->key, i));
-
-        /* Set up the pointer to the space we're allocating: */
-
-        for (i = 0; i < KEY_PTRS(&b->key); i++)
-                k->ptr[i] = b->key.ptr[i];
-
-        sectors = min(sectors, b->sectors_free);
-
-        SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
-        SET_KEY_SIZE(k, sectors);
-        SET_KEY_PTRS(k, KEY_PTRS(&b->key));
-
-        /*
-         * Move b to the end of the lru, and keep track of what this bucket was
-         * last used for:
-         */
-        list_move_tail(&b->list, &c->data_buckets);
-        bkey_copy_key(&b->key, k);
-        b->last = op->task;
-
-        b->sectors_free -= sectors;
-
-        for (i = 0; i < KEY_PTRS(&b->key); i++) {
-                SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
-
-                atomic_long_add(sectors,
-                                &PTR_CACHE(c, &b->key, i)->sectors_written);
-        }
-
-        if (b->sectors_free < c->sb.block_size)
-                b->sectors_free = 0;
-
-        /*
-         * k takes refcounts on the buckets it points to until it's inserted
-         * into the btree, but if we're done with this bucket we just transfer
-         * get_data_bucket()'s refcount.
-         */
-        if (b->sectors_free)
-                for (i = 0; i < KEY_PTRS(&b->key); i++)
-                        atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
-
-        spin_unlock(&c->data_bucket_lock);
-        return true;
-}
-
 static void bch_data_invalidate(struct closure *cl)
 {
         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
@@ -545,7 +365,9 @@ static void bch_data_insert_start(struct closure *cl)
                 SET_KEY_INODE(k, op->inode);
                 SET_KEY_OFFSET(k, bio->bi_sector);
 
-                if (!bch_alloc_sectors(op, k, bio_sectors(bio)))
+                if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
+                                       op->write_point, op->write_prio,
+                                       op->writeback))
                         goto err;
 
                 n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
@@ -968,7 +790,7 @@ static struct search *search_alloc(struct bio *bio, struct bcache_device *d)
         s->iop.c                = d->c;
         s->d                    = d;
         s->op.lock              = -1;
-        s->iop.task             = current;
+        s->iop.write_point      = hash_long((unsigned long) current, 16);
         s->orig_bio             = bio;
         s->write                = (bio->bi_rw & REQ_WRITE) != 0;
         s->iop.flush_journal    = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0;
diff --git a/drivers/md/bcache/request.h b/drivers/md/bcache/request.h
index 54d7de27356f..2cd65bf073c2 100644
--- a/drivers/md/bcache/request.h
+++ b/drivers/md/bcache/request.h
@@ -6,10 +6,10 @@
 struct data_insert_op {
         struct closure          cl;
         struct cache_set        *c;
-        struct task_struct      *task;
         struct bio              *bio;
 
         unsigned                inode;
+        uint16_t                write_point;
         uint16_t                write_prio;
         short                   error;
 
@@ -31,9 +31,6 @@ struct data_insert_op {
 unsigned bch_get_congested(struct cache_set *);
 void bch_data_insert(struct closure *cl);
 
-void bch_open_buckets_free(struct cache_set *);
-int bch_open_buckets_alloc(struct cache_set *);
-
 void bch_cached_dev_request_init(struct cached_dev *dc);
 void bch_flash_dev_request_init(struct bcache_device *d);