1 files changed, 102 insertions, 2 deletions

diff --git a/mm/z3fold.c b/mm/z3fold.c
index 1a029a7432ee..75b7962439ff 100644
--- a/mm/z3fold.c
+++ b/mm/z3fold.c
@@ -41,6 +41,7 @@
 #include <linux/workqueue.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/wait.h>
 #include <linux/zpool.h>
 #include <linux/magic.h>
 
@@ -145,6 +146,8 @@ struct z3fold_header {
  * @release_wq: workqueue for safe page release
  * @work:       work_struct for safe page release
  * @inode:      inode for z3fold pseudo filesystem
+ * @destroying: bool to stop migration once we start destruction
+ * @isolated: int to count the number of pages currently in isolation
  *
  * This structure is allocated at pool creation time and maintains metadata
  * pertaining to a particular z3fold pool.
@@ -163,8 +166,11 @@ struct z3fold_pool {
         const struct zpool_ops *zpool_ops;
         struct workqueue_struct *compact_wq;
         struct workqueue_struct *release_wq;
+        struct wait_queue_head isolate_wait;
         struct work_struct work;
         struct inode *inode;
+        bool destroying;
+        int isolated;
 };
 
 /*
@@ -769,6 +775,7 @@ static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp,
                 goto out_c;
         spin_lock_init(&pool->lock);
         spin_lock_init(&pool->stale_lock);
+        init_waitqueue_head(&pool->isolate_wait);
         pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2);
         if (!pool->unbuddied)
                 goto out_pool;
@@ -808,6 +815,15 @@ out:
         return NULL;
 }
 
+static bool pool_isolated_are_drained(struct z3fold_pool *pool)
+{
+        bool ret;
+
+        spin_lock(&pool->lock);
+        ret = pool->isolated == 0;
+        spin_unlock(&pool->lock);
+        return ret;
+}
 /**
  * z3fold_destroy_pool() - destroys an existing z3fold pool
  * @pool:       the z3fold pool to be destroyed
@@ -817,9 +833,35 @@ out:
 static void z3fold_destroy_pool(struct z3fold_pool *pool)
 {
         kmem_cache_destroy(pool->c_handle);
-        z3fold_unregister_migration(pool);
-        destroy_workqueue(pool->release_wq);
+        /*
+         * We set pool-> destroying under lock to ensure that
+         * z3fold_page_isolate() sees any changes to destroying. This way we
+         * avoid the need for any memory barriers.
+         */
+
+        spin_lock(&pool->lock);
+        pool->destroying = true;
+        spin_unlock(&pool->lock);
+
+        /*
+         * We need to ensure that no pages are being migrated while we destroy
+         * these workqueues, as migration can queue work on either of the
+         * workqueues.
+         */
+        wait_event(pool->isolate_wait, !pool_isolated_are_drained(pool));
+
+        /*
+         * We need to destroy pool->compact_wq before pool->release_wq,
+         * as any pending work on pool->compact_wq will call
+         * queue_work(pool->release_wq, &pool->work).
+         *
+         * There are still outstanding pages until both workqueues are drained,
+         * so we cannot unregister migration until then.
+         */
+
         destroy_workqueue(pool->compact_wq);
+        destroy_workqueue(pool->release_wq);
+        z3fold_unregister_migration(pool);
         kfree(pool);
 }
 
@@ -1297,6 +1339,28 @@ static u64 z3fold_get_pool_size(struct z3fold_pool *pool)
         return atomic64_read(&pool->pages_nr);
 }
 
+/*
+ * z3fold_dec_isolated() expects to be called while pool->lock is held.
+ */
+static void z3fold_dec_isolated(struct z3fold_pool *pool)
+{
+        assert_spin_locked(&pool->lock);
+        VM_BUG_ON(pool->isolated <= 0);
+        pool->isolated--;
+
+        /*
+         * If we have no more isolated pages, we have to see if
+         * z3fold_destroy_pool() is waiting for a signal.
+         */
+        if (pool->isolated == 0 && waitqueue_active(&pool->isolate_wait))
+                wake_up_all(&pool->isolate_wait);
+}
+
+static void z3fold_inc_isolated(struct z3fold_pool *pool)
+{
+        pool->isolated++;
+}
+
 static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)
 {
         struct z3fold_header *zhdr;
@@ -1323,6 +1387,34 @@ static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)
                 spin_lock(&pool->lock);
                 if (!list_empty(&page->lru))
                         list_del(&page->lru);
+                /*
+                 * We need to check for destruction while holding pool->lock, as
+                 * otherwise destruction could see 0 isolated pages, and
+                 * proceed.
+                 */
+                if (unlikely(pool->destroying)) {
+                        spin_unlock(&pool->lock);
+                        /*
+                         * If this page isn't stale, somebody else holds a
+                         * reference to it. Let't drop our refcount so that they
+                         * can call the release logic.
+                         */
+                        if (unlikely(kref_put(&zhdr->refcount,
+                                              release_z3fold_page_locked))) {
+                                /*
+                                 * If we get here we have kref problems, so we
+                                 * should freak out.
+                                 */
+                                WARN(1, "Z3fold is experiencing kref problems\n");
+                                z3fold_page_unlock(zhdr);
+                                return false;
+                        }
+                        z3fold_page_unlock(zhdr);
+                        return false;
+                }
+
+
+                z3fold_inc_isolated(pool);
                 spin_unlock(&pool->lock);
                 z3fold_page_unlock(zhdr);
                 return true;
@@ -1391,6 +1483,10 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa
 
         queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work);
 
+        spin_lock(&pool->lock);
+        z3fold_dec_isolated(pool);
+        spin_unlock(&pool->lock);
+
         page_mapcount_reset(page);
         put_page(page);
         return 0;
@@ -1410,10 +1506,14 @@ static void z3fold_page_putback(struct page *page)
         INIT_LIST_HEAD(&page->lru);
         if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {
                 atomic64_dec(&pool->pages_nr);
+                spin_lock(&pool->lock);
+                z3fold_dec_isolated(pool);
+                spin_unlock(&pool->lock);
                 return;
         }
         spin_lock(&pool->lock);
         list_add(&page->lru, &pool->lru);
+        z3fold_dec_isolated(pool);
         spin_unlock(&pool->lock);
         z3fold_page_unlock(zhdr);
 }