dm: allow remove to be deferred

This patch allows the removal of an open device to be deferred until
it is closed.  (Previously such a removal attempt would fail.)

The deferred remove functionality is enabled by setting the flag
DM_DEFERRED_REMOVE in the ioctl structure on DM_DEV_REMOVE or
DM_REMOVE_ALL ioctl.

On return from DM_DEV_REMOVE, the flag DM_DEFERRED_REMOVE indicates if
the device was removed immediately or flagged to be removed on close -
if the flag is clear, the device was removed.

On return from DM_DEV_STATUS and other ioctls, the flag
DM_DEFERRED_REMOVE is set if the device is scheduled to be removed on
closure.

A device that is scheduled to be deleted can be revived using the
message "@cancel_deferred_remove". This message clears the
DMF_DEFERRED_REMOVE flag so that the device won't be deleted on close.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Alasdair G Kergon <agk@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@redhat.com>

4 files changed, 99 insertions, 12 deletions

diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index afe08146f73e..51521429fb59 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -57,7 +57,7 @@ struct vers_iter {
 static struct list_head _name_buckets[NUM_BUCKETS];
 static struct list_head _uuid_buckets[NUM_BUCKETS];
 
-static void dm_hash_remove_all(int keep_open_devices);
+static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool only_deferred);
 
 /*
  * Guards access to both hash tables.
@@ -86,7 +86,7 @@ static int dm_hash_init(void)
 
 static void dm_hash_exit(void)
 {
-        dm_hash_remove_all(0);
+        dm_hash_remove_all(false, false, false);
 }
 
 /*-----------------------------------------------------------------
@@ -276,7 +276,7 @@ static struct dm_table *__hash_remove(struct hash_cell *hc)
         return table;
 }
 
-static void dm_hash_remove_all(int keep_open_devices)
+static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool only_deferred)
 {
         int i, dev_skipped;
         struct hash_cell *hc;
@@ -293,7 +293,8 @@ retry:
                         md = hc->md;
                         dm_get(md);
 
-                        if (keep_open_devices && dm_lock_for_deletion(md)) {
+                        if (keep_open_devices &&
+                            dm_lock_for_deletion(md, mark_deferred, only_deferred)) {
                                 dm_put(md);
                                 dev_skipped++;
                                 continue;
@@ -450,6 +451,11 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
         return md;
 }
 
+void dm_deferred_remove(void)
+{
+        dm_hash_remove_all(true, false, true);
+}
+
 /*-----------------------------------------------------------------
  * Implementation of the ioctl commands
  *---------------------------------------------------------------*/
@@ -461,7 +467,7 @@ typedef int (*ioctl_fn)(struct dm_ioctl *param, size_t param_size);
 
 static int remove_all(struct dm_ioctl *param, size_t param_size)
 {
-        dm_hash_remove_all(1);
+        dm_hash_remove_all(true, !!(param->flags & DM_DEFERRED_REMOVE), false);
         param->data_size = 0;
         return 0;
 }
@@ -683,6 +689,9 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param)
         if (dm_suspended_md(md))
                 param->flags |= DM_SUSPEND_FLAG;
 
+        if (dm_test_deferred_remove_flag(md))
+                param->flags |= DM_DEFERRED_REMOVE;
+
         param->dev = huge_encode_dev(disk_devt(disk));
 
         /*
@@ -832,8 +841,13 @@ static int dev_remove(struct dm_ioctl *param, size_t param_size)
         /*
          * Ensure the device is not open and nothing further can open it.
          */
-        r = dm_lock_for_deletion(md);
+        r = dm_lock_for_deletion(md, !!(param->flags & DM_DEFERRED_REMOVE), false);
         if (r) {
+                if (r == -EBUSY && param->flags & DM_DEFERRED_REMOVE) {
+                        up_write(&_hash_lock);
+                        dm_put(md);
+                        return 0;
+                }
                 DMDEBUG_LIMIT("unable to remove open device %s", hc->name);
                 up_write(&_hash_lock);
                 dm_put(md);
@@ -848,6 +862,8 @@ static int dev_remove(struct dm_ioctl *param, size_t param_size)
                 dm_table_destroy(t);
         }
 
+        param->flags &= ~DM_DEFERRED_REMOVE;
+
         if (!dm_kobject_uevent(md, KOBJ_REMOVE, param->event_nr))
                 param->flags |= DM_UEVENT_GENERATED_FLAG;
 
@@ -1469,6 +1485,14 @@ static int message_for_md(struct mapped_device *md, unsigned argc, char **argv,
         if (**argv != '@')
                 return 2; /* no '@' prefix, deliver to target */
 
+        if (!strcasecmp(argv[0], "@cancel_deferred_remove")) {
+                if (argc != 1) {
+                        DMERR("Invalid arguments for @cancel_deferred_remove");
+                        return -EINVAL;
+                }
+                return dm_cancel_deferred_remove(md);
+        }
+
         r = dm_stats_message(md, argc, argv, result, maxlen);
         if (r < 2)
                 return r;
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index b3e26c7d1417..0704c523a76b 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -49,6 +49,11 @@ static unsigned int _major = 0;
 static DEFINE_IDR(_minor_idr);
 
 static DEFINE_SPINLOCK(_minor_lock);
+
+static void do_deferred_remove(struct work_struct *w);
+
+static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
+
 /*
  * For bio-based dm.
  * One of these is allocated per bio.
@@ -116,6 +121,7 @@ EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
 #define DMF_DELETING 4
 #define DMF_NOFLUSH_SUSPENDING 5
 #define DMF_MERGE_IS_OPTIONAL 6
+#define DMF_DEFERRED_REMOVE 7
 
 /*
  * A dummy definition to make RCU happy.
@@ -299,6 +305,8 @@ out_free_io_cache:
 
 static void local_exit(void)
 {
+        flush_scheduled_work();
+
         kmem_cache_destroy(_rq_tio_cache);
         kmem_cache_destroy(_io_cache);
         unregister_blkdev(_major, _name);
@@ -404,7 +412,10 @@ static void dm_blk_close(struct gendisk *disk, fmode_t mode)
 
         spin_lock(&_minor_lock);
 
-        atomic_dec(&md->open_count);
+        if (atomic_dec_and_test(&md->open_count) &&
+            (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
+                schedule_work(&deferred_remove_work);
+
         dm_put(md);
 
         spin_unlock(&_minor_lock);
@@ -418,14 +429,18 @@ int dm_open_count(struct mapped_device *md)
 /*
  * Guarantees nothing is using the device before it's deleted.
  */
-int dm_lock_for_deletion(struct mapped_device *md)
+int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
 {
         int r = 0;
 
         spin_lock(&_minor_lock);
 
-        if (dm_open_count(md))
+        if (dm_open_count(md)) {
                 r = -EBUSY;
+                if (mark_deferred)
+                        set_bit(DMF_DEFERRED_REMOVE, &md->flags);
+        } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
+                r = -EEXIST;
         else
                 set_bit(DMF_DELETING, &md->flags);
 
@@ -434,6 +449,27 @@ int dm_lock_for_deletion(struct mapped_device *md)
         return r;
 }
 
+int dm_cancel_deferred_remove(struct mapped_device *md)
+{
+        int r = 0;
+
+        spin_lock(&_minor_lock);
+
+        if (test_bit(DMF_DELETING, &md->flags))
+                r = -EBUSY;
+        else
+                clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
+
+        spin_unlock(&_minor_lock);
+
+        return r;
+}
+
+static void do_deferred_remove(struct work_struct *w)
+{
+        dm_deferred_remove();
+}
+
 sector_t dm_get_size(struct mapped_device *md)
 {
         return get_capacity(md->disk);
@@ -2894,6 +2930,11 @@ int dm_suspended_md(struct mapped_device *md)
         return test_bit(DMF_SUSPENDED, &md->flags);
 }
 
+int dm_test_deferred_remove_flag(struct mapped_device *md)
+{
+        return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
+}
+
 int dm_suspended(struct dm_target *ti)
 {
         return dm_suspended_md(dm_table_get_md(ti->table));
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 1d1ad7b7e527..c57ba550f69e 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -129,6 +129,16 @@ int dm_deleting_md(struct mapped_device *md);
 int dm_suspended_md(struct mapped_device *md);
 
 /*
+ * Test if the device is scheduled for deferred remove.
+ */
+int dm_test_deferred_remove_flag(struct mapped_device *md);
+
+/*
+ * Try to remove devices marked for deferred removal.
+ */
+void dm_deferred_remove(void);
+
+/*
  * The device-mapper can be driven through one of two interfaces;
  * ioctl or filesystem, depending which patch you have applied.
  */
@@ -158,7 +168,8 @@ void dm_stripe_exit(void);
 void dm_destroy(struct mapped_device *md);
 void dm_destroy_immediate(struct mapped_device *md);
 int dm_open_count(struct mapped_device *md);
-int dm_lock_for_deletion(struct mapped_device *md);
+int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred);
+int dm_cancel_deferred_remove(struct mapped_device *md);
 int dm_request_based(struct mapped_device *md);
 sector_t dm_get_size(struct mapped_device *md);
 struct dm_stats *dm_get_stats(struct mapped_device *md);
diff --git a/include/uapi/linux/dm-ioctl.h b/include/uapi/linux/dm-ioctl.h
index f1e12bd40b3b..c8a4302093a3 100644
--- a/include/uapi/linux/dm-ioctl.h
+++ b/include/uapi/linux/dm-ioctl.h
@@ -267,9 +267,9 @@ enum {
 #define DM_DEV_SET_GEOMETRY     _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)
 
 #define DM_VERSION_MAJOR        4
-#define DM_VERSION_MINOR        26
+#define DM_VERSION_MINOR        27
 #define DM_VERSION_PATCHLEVEL   0
-#define DM_VERSION_EXTRA        "-ioctl (2013-08-15)"
+#define DM_VERSION_EXTRA        "-ioctl (2013-10-30)"
 
 /* Status bits */
 #define DM_READONLY_FLAG        (1 << 0) /* In/Out */
@@ -341,4 +341,15 @@ enum {
  */
 #define DM_DATA_OUT_FLAG                (1 << 16) /* Out */
 
+/*
+ * If set with DM_DEV_REMOVE or DM_REMOVE_ALL this indicates that if
+ * the device cannot be removed immediately because it is still in use
+ * it should instead be scheduled for removal when it gets closed.
+ *
+ * On return from DM_DEV_REMOVE, DM_DEV_STATUS or other ioctls, this
+ * flag indicates that the device is scheduled to be removed when it
+ * gets closed.
+ */
+#define DM_DEFERRED_REMOVE              (1 << 17) /* In/Out */
+
 #endif                          /* _LINUX_DM_IOCTL_H */