FS-Cache: Provide proper invalidation

Provide a proper invalidation method rather than relying on the netfs retiring
the cookie it has and getting a new one.  The problem with this is that isn't
easy for the netfs to make sure that it has completed/cancelled all its
outstanding storage and retrieval operations on the cookie it is retiring.

Instead, have the cache provide an invalidation method that will cancel or wait
for all currently outstanding operations before invalidating the cache, and
will cause new operations to queue up behind that.  Whilst invalidation is in
progress, some requests will be rejected until the cache can stack a barrier on
the operation queue to cause new operations to be deferred behind it.

Signed-off-by: David Howells <dhowells@redhat.com>

6 files changed, 235 insertions, 1 deletions

diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c
index 66be9eccede0..8dcb114758e3 100644
--- a/fs/fscache/cookie.c
+++ b/fs/fscache/cookie.c
@@ -370,6 +370,66 @@ cant_attach_object:
 }
 
 /*
+ * Invalidate an object.  Callable with spinlocks held.
+ */
+void __fscache_invalidate(struct fscache_cookie *cookie)
+{
+        struct fscache_object *object;
+
+        _enter("{%s}", cookie->def->name);
+
+        fscache_stat(&fscache_n_invalidates);
+
+        /* Only permit invalidation of data files.  Invalidating an index will
+         * require the caller to release all its attachments to the tree rooted
+         * there, and if it's doing that, it may as well just retire the
+         * cookie.
+         */
+        ASSERTCMP(cookie->def->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
+
+        /* We will be updating the cookie too. */
+        BUG_ON(!cookie->def->get_aux);
+
+        /* If there's an object, we tell the object state machine to handle the
+         * invalidation on our behalf, otherwise there's nothing to do.
+         */
+        if (!hlist_empty(&cookie->backing_objects)) {
+                spin_lock(&cookie->lock);
+
+                if (!hlist_empty(&cookie->backing_objects) &&
+                    !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
+                                      &cookie->flags)) {
+                        object = hlist_entry(cookie->backing_objects.first,
+                                             struct fscache_object,
+                                             cookie_link);
+                        if (object->state < FSCACHE_OBJECT_DYING)
+                                fscache_raise_event(
+                                        object, FSCACHE_OBJECT_EV_INVALIDATE);
+                }
+
+                spin_unlock(&cookie->lock);
+        }
+
+        _leave("");
+}
+EXPORT_SYMBOL(__fscache_invalidate);
+
+/*
+ * Wait for object invalidation to complete.
+ */
+void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
+{
+        _enter("%p", cookie);
+
+        wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
+                    fscache_wait_bit_interruptible,
+                    TASK_UNINTERRUPTIBLE);
+
+        _leave("");
+}
+EXPORT_SYMBOL(__fscache_wait_on_invalidate);
+
+/*
  * update the index entries backing a cookie
  */
 void __fscache_update_cookie(struct fscache_cookie *cookie)
diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h
index f6aad48d38a8..c81179303930 100644
--- a/fs/fscache/internal.h
+++ b/fs/fscache/internal.h
@@ -122,11 +122,17 @@ extern int fscache_submit_exclusive_op(struct fscache_object *,
 extern int fscache_submit_op(struct fscache_object *,
                              struct fscache_operation *);
 extern int fscache_cancel_op(struct fscache_operation *);
+extern void fscache_cancel_all_ops(struct fscache_object *);
 extern void fscache_abort_object(struct fscache_object *);
 extern void fscache_start_operations(struct fscache_object *);
 extern void fscache_operation_gc(struct work_struct *);
 
 /*
+ * page.c
+ */
+extern void fscache_invalidate_writes(struct fscache_cookie *);
+
+/*
  * proc.c
  */
 #ifdef CONFIG_PROC_FS
@@ -205,6 +211,9 @@ extern atomic_t fscache_n_acquires_ok;
 extern atomic_t fscache_n_acquires_nobufs;
 extern atomic_t fscache_n_acquires_oom;
 
+extern atomic_t fscache_n_invalidates;
+extern atomic_t fscache_n_invalidates_run;
+
 extern atomic_t fscache_n_updates;
 extern atomic_t fscache_n_updates_null;
 extern atomic_t fscache_n_updates_run;
@@ -237,6 +246,7 @@ extern atomic_t fscache_n_cop_alloc_object;
 extern atomic_t fscache_n_cop_lookup_object;
 extern atomic_t fscache_n_cop_lookup_complete;
 extern atomic_t fscache_n_cop_grab_object;
+extern atomic_t fscache_n_cop_invalidate_object;
 extern atomic_t fscache_n_cop_update_object;
 extern atomic_t fscache_n_cop_drop_object;
 extern atomic_t fscache_n_cop_put_object;
diff --git a/fs/fscache/object.c b/fs/fscache/object.c
index 773bc798a416..80b549141ea6 100644
--- a/fs/fscache/object.c
+++ b/fs/fscache/object.c
@@ -14,6 +14,7 @@
 
 #define FSCACHE_DEBUG_LEVEL COOKIE
 #include <linux/module.h>
+#include <linux/slab.h>
 #include "internal.h"
 
 const char *fscache_object_states[FSCACHE_OBJECT__NSTATES] = {
@@ -22,6 +23,7 @@ const char *fscache_object_states[FSCACHE_OBJECT__NSTATES] = {
         [FSCACHE_OBJECT_CREATING]       = "OBJECT_CREATING",
         [FSCACHE_OBJECT_AVAILABLE]      = "OBJECT_AVAILABLE",
         [FSCACHE_OBJECT_ACTIVE]         = "OBJECT_ACTIVE",
+        [FSCACHE_OBJECT_INVALIDATING]   = "OBJECT_INVALIDATING",
         [FSCACHE_OBJECT_UPDATING]       = "OBJECT_UPDATING",
         [FSCACHE_OBJECT_DYING]          = "OBJECT_DYING",
         [FSCACHE_OBJECT_LC_DYING]       = "OBJECT_LC_DYING",
@@ -39,6 +41,7 @@ const char fscache_object_states_short[FSCACHE_OBJECT__NSTATES][5] = {
         [FSCACHE_OBJECT_CREATING]       = "CRTN",
         [FSCACHE_OBJECT_AVAILABLE]      = "AVBL",
         [FSCACHE_OBJECT_ACTIVE]         = "ACTV",
+        [FSCACHE_OBJECT_INVALIDATING]   = "INVL",
         [FSCACHE_OBJECT_UPDATING]       = "UPDT",
         [FSCACHE_OBJECT_DYING]          = "DYNG",
         [FSCACHE_OBJECT_LC_DYING]       = "LCDY",
@@ -54,6 +57,7 @@ static void fscache_put_object(struct fscache_object *);
 static void fscache_initialise_object(struct fscache_object *);
 static void fscache_lookup_object(struct fscache_object *);
 static void fscache_object_available(struct fscache_object *);
+static void fscache_invalidate_object(struct fscache_object *);
 static void fscache_release_object(struct fscache_object *);
 static void fscache_withdraw_object(struct fscache_object *);
 static void fscache_enqueue_dependents(struct fscache_object *);
@@ -79,6 +83,15 @@ static inline void fscache_done_parent_op(struct fscache_object *object)
 }
 
 /*
+ * Notify netfs of invalidation completion.
+ */
+static inline void fscache_invalidation_complete(struct fscache_cookie *cookie)
+{
+        if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
+                wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
+}
+
+/*
  * process events that have been sent to an object's state machine
  * - initiates parent lookup
  * - does object lookup
@@ -125,6 +138,16 @@ static void fscache_object_state_machine(struct fscache_object *object)
         case FSCACHE_OBJECT_ACTIVE:
                 goto active_transit;
 
+                /* Invalidate an object on disk */
+        case FSCACHE_OBJECT_INVALIDATING:
+                clear_bit(FSCACHE_OBJECT_EV_INVALIDATE, &object->events);
+                fscache_stat(&fscache_n_invalidates_run);
+                fscache_stat(&fscache_n_cop_invalidate_object);
+                fscache_invalidate_object(object);
+                fscache_stat_d(&fscache_n_cop_invalidate_object);
+                fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
+                goto active_transit;
+
                 /* update the object metadata on disk */
         case FSCACHE_OBJECT_UPDATING:
                 clear_bit(FSCACHE_OBJECT_EV_UPDATE, &object->events);
@@ -275,6 +298,9 @@ active_transit:
         case FSCACHE_OBJECT_EV_ERROR:
                 new_state = FSCACHE_OBJECT_DYING;
                 goto change_state;
+        case FSCACHE_OBJECT_EV_INVALIDATE:
+                new_state = FSCACHE_OBJECT_INVALIDATING;
+                goto change_state;
         case FSCACHE_OBJECT_EV_UPDATE:
                 new_state = FSCACHE_OBJECT_UPDATING;
                 goto change_state;
@@ -679,6 +705,7 @@ static void fscache_withdraw_object(struct fscache_object *object)
                 if (object->cookie == cookie) {
                         hlist_del_init(&object->cookie_link);
                         object->cookie = NULL;
+                        fscache_invalidation_complete(cookie);
                         detached = true;
                 }
                 spin_unlock(&cookie->lock);
@@ -888,3 +915,48 @@ enum fscache_checkaux fscache_check_aux(struct fscache_object *object,
         return result;
 }
 EXPORT_SYMBOL(fscache_check_aux);
+
+/*
+ * Asynchronously invalidate an object.
+ */
+static void fscache_invalidate_object(struct fscache_object *object)
+{
+        struct fscache_operation *op;
+        struct fscache_cookie *cookie = object->cookie;
+
+        _enter("{OBJ%x}", object->debug_id);
+
+        /* Reject any new read/write ops and abort any that are pending. */
+        fscache_invalidate_writes(cookie);
+        clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
+        fscache_cancel_all_ops(object);
+
+        /* Now we have to wait for in-progress reads and writes */
+        op = kzalloc(sizeof(*op), GFP_KERNEL);
+        if (!op) {
+                fscache_raise_event(object, FSCACHE_OBJECT_EV_ERROR);
+                _leave(" [ENOMEM]");
+                return;
+        }
+
+        fscache_operation_init(op, object->cache->ops->invalidate_object, NULL);
+        op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE);
+
+        spin_lock(&cookie->lock);
+        if (fscache_submit_exclusive_op(object, op) < 0)
+                BUG();
+        spin_unlock(&cookie->lock);
+        fscache_put_operation(op);
+
+        /* Once we've completed the invalidation, we know there will be no data
+         * stored in the cache and thus we can reinstate the data-check-skip
+         * optimisation.
+         */
+        set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
+
+        /* We can allow read and write requests to come in once again.  They'll
+         * queue up behind our exclusive invalidation operation.
+         */
+        fscache_invalidation_complete(cookie);
+        _leave("");
+}
diff --git a/fs/fscache/operation.c b/fs/fscache/operation.c
index 748f9553c2cb..c58dbe613266 100644
--- a/fs/fscache/operation.c
+++ b/fs/fscache/operation.c
@@ -324,6 +324,38 @@ int fscache_cancel_op(struct fscache_operation *op)
 }
 
 /*
+ * Cancel all pending operations on an object
+ */
+void fscache_cancel_all_ops(struct fscache_object *object)
+{
+        struct fscache_operation *op;
+
+        _enter("OBJ%x", object->debug_id);
+
+        spin_lock(&object->lock);
+
+        while (!list_empty(&object->pending_ops)) {
+                op = list_entry(object->pending_ops.next,
+                                struct fscache_operation, pend_link);
+                fscache_stat(&fscache_n_op_cancelled);
+                list_del_init(&op->pend_link);
+
+                ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING);
+                op->state = FSCACHE_OP_ST_CANCELLED;
+
+                if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
+                        object->n_exclusive--;
+                if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags))
+                        wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
+                fscache_put_operation(op);
+                cond_resched_lock(&object->lock);
+        }
+
+        spin_unlock(&object->lock);
+        _leave("");
+}
+
+/*
  * Record the completion of an in-progress operation.
  */
 void fscache_op_complete(struct fscache_operation *op)
diff --git a/fs/fscache/page.c b/fs/fscache/page.c
index b38b13d2a555..7bf9d2557052 100644
--- a/fs/fscache/page.c
+++ b/fs/fscache/page.c
@@ -361,6 +361,11 @@ int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
         if (hlist_empty(&cookie->backing_objects))
                 goto nobufs;
 
+        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
+                _leave(" = -ENOBUFS [invalidating]");
+                return -ENOBUFS;
+        }
+
         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
         ASSERTCMP(page, !=, NULL);
 
@@ -483,6 +488,11 @@ int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
         if (hlist_empty(&cookie->backing_objects))
                 goto nobufs;
 
+        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
+                _leave(" = -ENOBUFS [invalidating]");
+                return -ENOBUFS;
+        }
+
         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
         ASSERTCMP(*nr_pages, >, 0);
         ASSERT(!list_empty(pages));
@@ -591,6 +601,11 @@ int __fscache_alloc_page(struct fscache_cookie *cookie,
         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
         ASSERTCMP(page, !=, NULL);
 
+        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
+                _leave(" = -ENOBUFS [invalidating]");
+                return -ENOBUFS;
+        }
+
         if (fscache_wait_for_deferred_lookup(cookie) < 0)
                 return -ERESTARTSYS;
 
@@ -731,6 +746,37 @@ superseded:
 }
 
 /*
+ * Clear the pages pending writing for invalidation
+ */
+void fscache_invalidate_writes(struct fscache_cookie *cookie)
+{
+        struct page *page;
+        void *results[16];
+        int n, i;
+
+        _enter("");
+
+        while (spin_lock(&cookie->stores_lock),
+               n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
+                                              ARRAY_SIZE(results),
+                                              FSCACHE_COOKIE_PENDING_TAG),
+               n > 0) {
+                for (i = n - 1; i >= 0; i--) {
+                        page = results[i];
+                        radix_tree_delete(&cookie->stores, page->index);
+                }
+
+                spin_unlock(&cookie->stores_lock);
+
+                for (i = n - 1; i >= 0; i--)
+                        page_cache_release(results[i]);
+        }
+
+        spin_unlock(&cookie->stores_lock);
+        _leave("");
+}
+
+/*
  * request a page be stored in the cache
  * - returns:
  *   -ENOMEM    - out of memory, nothing done
@@ -776,6 +822,11 @@ int __fscache_write_page(struct fscache_cookie *cookie,
 
         fscache_stat(&fscache_n_stores);
 
+        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
+                _leave(" = -ENOBUFS [invalidating]");
+                return -ENOBUFS;
+        }
+
         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
         if (!op)
                 goto nomem;
diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c
index 4765190d537f..51cdaee14109 100644
--- a/fs/fscache/stats.c
+++ b/fs/fscache/stats.c
@@ -80,6 +80,9 @@ atomic_t fscache_n_acquires_ok;
 atomic_t fscache_n_acquires_nobufs;
 atomic_t fscache_n_acquires_oom;
 
+atomic_t fscache_n_invalidates;
+atomic_t fscache_n_invalidates_run;
+
 atomic_t fscache_n_updates;
 atomic_t fscache_n_updates_null;
 atomic_t fscache_n_updates_run;
@@ -112,6 +115,7 @@ atomic_t fscache_n_cop_alloc_object;
 atomic_t fscache_n_cop_lookup_object;
 atomic_t fscache_n_cop_lookup_complete;
 atomic_t fscache_n_cop_grab_object;
+atomic_t fscache_n_cop_invalidate_object;
 atomic_t fscache_n_cop_update_object;
 atomic_t fscache_n_cop_drop_object;
 atomic_t fscache_n_cop_put_object;
@@ -168,6 +172,10 @@ static int fscache_stats_show(struct seq_file *m, void *v)
                    atomic_read(&fscache_n_object_created),
                    atomic_read(&fscache_n_object_lookups_timed_out));
 
+        seq_printf(m, "Invals : n=%u run=%u\n",
+                   atomic_read(&fscache_n_invalidates),
+                   atomic_read(&fscache_n_invalidates_run));
+
         seq_printf(m, "Updates: n=%u nul=%u run=%u\n",
                    atomic_read(&fscache_n_updates),
                    atomic_read(&fscache_n_updates_null),
@@ -246,7 +254,8 @@ static int fscache_stats_show(struct seq_file *m, void *v)
                    atomic_read(&fscache_n_cop_lookup_object),
                    atomic_read(&fscache_n_cop_lookup_complete),
                    atomic_read(&fscache_n_cop_grab_object));
-        seq_printf(m, "CacheOp: upo=%d dro=%d pto=%d atc=%d syn=%d\n",
+        seq_printf(m, "CacheOp: inv=%d upo=%d dro=%d pto=%d atc=%d syn=%d\n",
+                   atomic_read(&fscache_n_cop_invalidate_object),
                    atomic_read(&fscache_n_cop_update_object),
                    atomic_read(&fscache_n_cop_drop_object),
                    atomic_read(&fscache_n_cop_put_object),