1 files changed, 405 insertions, 0 deletions

diff --git a/drivers/md/bcache/btree.h b/drivers/md/bcache/btree.h
new file mode 100644
index 000000000000..af4a7092a28c
--- /dev/null
+++ b/drivers/md/bcache/btree.h
@@ -0,0 +1,405 @@
+#ifndef _BCACHE_BTREE_H
+#define _BCACHE_BTREE_H
+
+/*
+ * THE BTREE:
+ *
+ * At a high level, bcache's btree is relatively standard b+ tree. All keys and
+ * pointers are in the leaves; interior nodes only have pointers to the child
+ * nodes.
+ *
+ * In the interior nodes, a struct bkey always points to a child btree node, and
+ * the key is the highest key in the child node - except that the highest key in
+ * an interior node is always MAX_KEY. The size field refers to the size on disk
+ * of the child node - this would allow us to have variable sized btree nodes
+ * (handy for keeping the depth of the btree 1 by expanding just the root).
+ *
+ * Btree nodes are themselves log structured, but this is hidden fairly
+ * thoroughly. Btree nodes on disk will in practice have extents that overlap
+ * (because they were written at different times), but in memory we never have
+ * overlapping extents - when we read in a btree node from disk, the first thing
+ * we do is resort all the sets of keys with a mergesort, and in the same pass
+ * we check for overlapping extents and adjust them appropriately.
+ *
+ * struct btree_op is a central interface to the btree code. It's used for
+ * specifying read vs. write locking, and the embedded closure is used for
+ * waiting on IO or reserve memory.
+ *
+ * BTREE CACHE:
+ *
+ * Btree nodes are cached in memory; traversing the btree might require reading
+ * in btree nodes which is handled mostly transparently.
+ *
+ * bch_btree_node_get() looks up a btree node in the cache and reads it in from
+ * disk if necessary. This function is almost never called directly though - the
+ * btree() macro is used to get a btree node, call some function on it, and
+ * unlock the node after the function returns.
+ *
+ * The root is special cased - it's taken out of the cache's lru (thus pinning
+ * it in memory), so we can find the root of the btree by just dereferencing a
+ * pointer instead of looking it up in the cache. This makes locking a bit
+ * tricky, since the root pointer is protected by the lock in the btree node it
+ * points to - the btree_root() macro handles this.
+ *
+ * In various places we must be able to allocate memory for multiple btree nodes
+ * in order to make forward progress. To do this we use the btree cache itself
+ * as a reserve; if __get_free_pages() fails, we'll find a node in the btree
+ * cache we can reuse. We can't allow more than one thread to be doing this at a
+ * time, so there's a lock, implemented by a pointer to the btree_op closure -
+ * this allows the btree_root() macro to implicitly release this lock.
+ *
+ * BTREE IO:
+ *
+ * Btree nodes never have to be explicitly read in; bch_btree_node_get() handles
+ * this.
+ *
+ * For writing, we have two btree_write structs embeddded in struct btree - one
+ * write in flight, and one being set up, and we toggle between them.
+ *
+ * Writing is done with a single function -  bch_btree_write() really serves two
+ * different purposes and should be broken up into two different functions. When
+ * passing now = false, it merely indicates that the node is now dirty - calling
+ * it ensures that the dirty keys will be written at some point in the future.
+ *
+ * When passing now = true, bch_btree_write() causes a write to happen
+ * "immediately" (if there was already a write in flight, it'll cause the write
+ * to happen as soon as the previous write completes). It returns immediately
+ * though - but it takes a refcount on the closure in struct btree_op you passed
+ * to it, so a closure_sync() later can be used to wait for the write to
+ * complete.
+ *
+ * This is handy because btree_split() and garbage collection can issue writes
+ * in parallel, reducing the amount of time they have to hold write locks.
+ *
+ * LOCKING:
+ *
+ * When traversing the btree, we may need write locks starting at some level -
+ * inserting a key into the btree will typically only require a write lock on
+ * the leaf node.
+ *
+ * This is specified with the lock field in struct btree_op; lock = 0 means we
+ * take write locks at level <= 0, i.e. only leaf nodes. bch_btree_node_get()
+ * checks this field and returns the node with the appropriate lock held.
+ *
+ * If, after traversing the btree, the insertion code discovers it has to split
+ * then it must restart from the root and take new locks - to do this it changes
+ * the lock field and returns -EINTR, which causes the btree_root() macro to
+ * loop.
+ *
+ * Handling cache misses require a different mechanism for upgrading to a write
+ * lock. We do cache lookups with only a read lock held, but if we get a cache
+ * miss and we wish to insert this data into the cache, we have to insert a
+ * placeholder key to detect races - otherwise, we could race with a write and
+ * overwrite the data that was just written to the cache with stale data from
+ * the backing device.
+ *
+ * For this we use a sequence number that write locks and unlocks increment - to
+ * insert the check key it unlocks the btree node and then takes a write lock,
+ * and fails if the sequence number doesn't match.
+ */
+
+#include "bset.h"
+#include "debug.h"
+
+struct btree_write {
+        struct closure          *owner;
+        atomic_t                *journal;
+
+        /* If btree_split() frees a btree node, it writes a new pointer to that
+         * btree node indicating it was freed; it takes a refcount on
+         * c->prio_blocked because we can't write the gens until the new
+         * pointer is on disk. This allows btree_write_endio() to release the
+         * refcount that btree_split() took.
+         */
+        int                     prio_blocked;
+};
+
+struct btree {
+        /* Hottest entries first */
+        struct hlist_node       hash;
+
+        /* Key/pointer for this btree node */
+        BKEY_PADDED(key);
+
+        /* Single bit - set when accessed, cleared by shrinker */
+        unsigned long           accessed;
+        unsigned long           seq;
+        struct rw_semaphore     lock;
+        struct cache_set        *c;
+
+        unsigned long           flags;
+        uint16_t                written;        /* would be nice to kill */
+        uint8_t                 level;
+        uint8_t                 nsets;
+        uint8_t                 page_order;
+
+        /*
+         * Set of sorted keys - the real btree node - plus a binary search tree
+         *
+         * sets[0] is special; set[0]->tree, set[0]->prev and set[0]->data point
+         * to the memory we have allocated for this btree node. Additionally,
+         * set[0]->data points to the entire btree node as it exists on disk.
+         */
+        struct bset_tree        sets[MAX_BSETS];
+
+        /* Used to refcount bio splits, also protects b->bio */
+        struct closure_with_waitlist    io;
+
+        /* Gets transferred to w->prio_blocked - see the comment there */
+        int                     prio_blocked;
+
+        struct list_head        list;
+        struct delayed_work     work;
+
+        uint64_t                io_start_time;
+        struct btree_write      writes[2];
+        struct bio              *bio;
+};
+
+#define BTREE_FLAG(flag)                                                \
+static inline bool btree_node_ ## flag(struct btree *b)                 \
+{       return test_bit(BTREE_NODE_ ## flag, &b->flags); }              \
+                                                                        \
+static inline void set_btree_node_ ## flag(struct btree *b)             \
+{       set_bit(BTREE_NODE_ ## flag, &b->flags); }                      \
+
+enum btree_flags {
+        BTREE_NODE_read_done,
+        BTREE_NODE_io_error,
+        BTREE_NODE_dirty,
+        BTREE_NODE_write_idx,
+};
+
+BTREE_FLAG(read_done);
+BTREE_FLAG(io_error);
+BTREE_FLAG(dirty);
+BTREE_FLAG(write_idx);
+
+static inline struct btree_write *btree_current_write(struct btree *b)
+{
+        return b->writes + btree_node_write_idx(b);
+}
+
+static inline struct btree_write *btree_prev_write(struct btree *b)
+{
+        return b->writes + (btree_node_write_idx(b) ^ 1);
+}
+
+static inline unsigned bset_offset(struct btree *b, struct bset *i)
+{
+        return (((size_t) i) - ((size_t) b->sets->data)) >> 9;
+}
+
+static inline struct bset *write_block(struct btree *b)
+{
+        return ((void *) b->sets[0].data) + b->written * block_bytes(b->c);
+}
+
+static inline bool bset_written(struct btree *b, struct bset_tree *t)
+{
+        return t->data < write_block(b);
+}
+
+static inline bool bkey_written(struct btree *b, struct bkey *k)
+{
+        return k < write_block(b)->start;
+}
+
+static inline void set_gc_sectors(struct cache_set *c)
+{
+        atomic_set(&c->sectors_to_gc, c->sb.bucket_size * c->nbuckets / 8);
+}
+
+static inline bool bch_ptr_invalid(struct btree *b, const struct bkey *k)
+{
+        return __bch_ptr_invalid(b->c, b->level, k);
+}
+
+static inline struct bkey *bch_btree_iter_init(struct btree *b,
+                                               struct btree_iter *iter,
+                                               struct bkey *search)
+{
+        return __bch_btree_iter_init(b, iter, search, b->sets);
+}
+
+/* Looping macros */
+
+#define for_each_cached_btree(b, c, iter)                               \
+        for (iter = 0;                                                  \
+             iter < ARRAY_SIZE((c)->bucket_hash);                       \
+             iter++)                                                    \
+                hlist_for_each_entry_rcu((b), (c)->bucket_hash + iter, hash)
+
+#define for_each_key_filter(b, k, iter, filter)                         \
+        for (bch_btree_iter_init((b), (iter), NULL);                    \
+             ((k) = bch_btree_iter_next_filter((iter), b, filter));)
+
+#define for_each_key(b, k, iter)                                        \
+        for (bch_btree_iter_init((b), (iter), NULL);                    \
+             ((k) = bch_btree_iter_next(iter));)
+
+/* Recursing down the btree */
+
+struct btree_op {
+        struct closure          cl;
+        struct cache_set        *c;
+
+        /* Journal entry we have a refcount on */
+        atomic_t                *journal;
+
+        /* Bio to be inserted into the cache */
+        struct bio              *cache_bio;
+
+        unsigned                inode;
+
+        uint16_t                write_prio;
+
+        /* Btree level at which we start taking write locks */
+        short                   lock;
+
+        /* Btree insertion type */
+        enum {
+                BTREE_INSERT,
+                BTREE_REPLACE
+        } type:8;
+
+        unsigned                csum:1;
+        unsigned                skip:1;
+        unsigned                flush_journal:1;
+
+        unsigned                insert_data_done:1;
+        unsigned                lookup_done:1;
+        unsigned                insert_collision:1;
+
+        /* Anything after this point won't get zeroed in do_bio_hook() */
+
+        /* Keys to be inserted */
+        struct keylist          keys;
+        BKEY_PADDED(replace);
+};
+
+void bch_btree_op_init_stack(struct btree_op *);
+
+static inline void rw_lock(bool w, struct btree *b, int level)
+{
+        w ? down_write_nested(&b->lock, level + 1)
+          : down_read_nested(&b->lock, level + 1);
+        if (w)
+                b->seq++;
+}
+
+static inline void rw_unlock(bool w, struct btree *b)
+{
+#ifdef CONFIG_BCACHE_EDEBUG
+        unsigned i;
+
+        if (w &&
+            b->key.ptr[0] &&
+            btree_node_read_done(b))
+                for (i = 0; i <= b->nsets; i++)
+                        bch_check_key_order(b, b->sets[i].data);
+#endif
+
+        if (w)
+                b->seq++;
+        (w ? up_write : up_read)(&b->lock);
+}
+
+#define insert_lock(s, b)       ((b)->level <= (s)->lock)
+
+/*
+ * These macros are for recursing down the btree - they handle the details of
+ * locking and looking up nodes in the cache for you. They're best treated as
+ * mere syntax when reading code that uses them.
+ *
+ * op->lock determines whether we take a read or a write lock at a given depth.
+ * If you've got a read lock and find that you need a write lock (i.e. you're
+ * going to have to split), set op->lock and return -EINTR; btree_root() will
+ * call you again and you'll have the correct lock.
+ */
+
+/**
+ * btree - recurse down the btree on a specified key
+ * @fn:         function to call, which will be passed the child node
+ * @key:        key to recurse on
+ * @b:          parent btree node
+ * @op:         pointer to struct btree_op
+ */
+#define btree(fn, key, b, op, ...)                                      \
+({                                                                      \
+        int _r, l = (b)->level - 1;                                     \
+        bool _w = l <= (op)->lock;                                      \
+        struct btree *_b = bch_btree_node_get((b)->c, key, l, op);      \
+        if (!IS_ERR(_b)) {                                              \
+                _r = bch_btree_ ## fn(_b, op, ##__VA_ARGS__);           \
+                rw_unlock(_w, _b);                                      \
+        } else                                                          \
+                _r = PTR_ERR(_b);                                       \
+        _r;                                                             \
+})
+
+/**
+ * btree_root - call a function on the root of the btree
+ * @fn:         function to call, which will be passed the child node
+ * @c:          cache set
+ * @op:         pointer to struct btree_op
+ */
+#define btree_root(fn, c, op, ...)                                      \
+({                                                                      \
+        int _r = -EINTR;                                                \
+        do {                                                            \
+                struct btree *_b = (c)->root;                           \
+                bool _w = insert_lock(op, _b);                          \
+                rw_lock(_w, _b, _b->level);                             \
+                if (_b == (c)->root &&                                  \
+                    _w == insert_lock(op, _b))                          \
+                        _r = bch_btree_ ## fn(_b, op, ##__VA_ARGS__);   \
+                rw_unlock(_w, _b);                                      \
+                bch_cannibalize_unlock(c, &(op)->cl);           \
+        } while (_r == -EINTR);                                         \
+                                                                        \
+        _r;                                                             \
+})
+
+static inline bool should_split(struct btree *b)
+{
+        struct bset *i = write_block(b);
+        return b->written >= btree_blocks(b) ||
+                (i->seq == b->sets[0].data->seq &&
+                 b->written + __set_blocks(i, i->keys + 15, b->c)
+                 > btree_blocks(b));
+}
+
+void bch_btree_read_done(struct closure *);
+void bch_btree_read(struct btree *);
+void bch_btree_write(struct btree *b, bool now, struct btree_op *op);
+
+void bch_cannibalize_unlock(struct cache_set *, struct closure *);
+void bch_btree_set_root(struct btree *);
+struct btree *bch_btree_node_alloc(struct cache_set *, int, struct closure *);
+struct btree *bch_btree_node_get(struct cache_set *, struct bkey *,
+                                int, struct btree_op *);
+
+bool bch_btree_insert_keys(struct btree *, struct btree_op *);
+bool bch_btree_insert_check_key(struct btree *, struct btree_op *,
+                                   struct bio *);
+int bch_btree_insert(struct btree_op *, struct cache_set *);
+
+int bch_btree_search_recurse(struct btree *, struct btree_op *);
+
+void bch_queue_gc(struct cache_set *);
+size_t bch_btree_gc_finish(struct cache_set *);
+void bch_moving_gc(struct closure *);
+int bch_btree_check(struct cache_set *, struct btree_op *);
+uint8_t __bch_btree_mark_key(struct cache_set *, int, struct bkey *);
+
+void bch_keybuf_init(struct keybuf *, keybuf_pred_fn *);
+void bch_refill_keybuf(struct cache_set *, struct keybuf *, struct bkey *);
+bool bch_keybuf_check_overlapping(struct keybuf *, struct bkey *,
+                                  struct bkey *);
+void bch_keybuf_del(struct keybuf *, struct keybuf_key *);
+struct keybuf_key *bch_keybuf_next(struct keybuf *);
+struct keybuf_key *bch_keybuf_next_rescan(struct cache_set *,
+                                          struct keybuf *, struct bkey *);
+
+#endif