dm: optimize use SRCU and RCU

This patch removes "io_lock" and "map_lock" in struct mapped_device and
"holders" in struct dm_table and replaces these mechanisms with
sleepable-rcu.

Previously, the code would call "dm_get_live_table" and "dm_table_put" to
get and release table. Now, the code is changed to call "dm_get_live_table"
and "dm_put_live_table". dm_get_live_table locks sleepable-rcu and
dm_put_live_table unlocks it.

dm_get_live_table_fast/dm_put_live_table_fast can be used instead of
dm_get_live_table/dm_put_live_table. These *_fast functions use
non-sleepable RCU, so the caller must not block between them.

If the code changes active or inactive dm table, it must call
dm_sync_table before destroying the old table.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jun'ichi Nomura <j-nomura@ce.jp.nec.com>
Signed-off-by: Alasdair G Kergon <agk@redhat.com>

5 files changed, 180 insertions, 147 deletions

diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 12f04868e899..f1b758675ec7 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -36,6 +36,14 @@ struct hash_cell {
         struct dm_table *new_map;
 };
 
+/*
+ * A dummy definition to make RCU happy.
+ * struct dm_table should never be dereferenced in this file.
+ */
+struct dm_table {
+        int undefined__;
+};
+
 struct vers_iter {
     size_t param_size;
     struct dm_target_versions *vers, *old_vers;
@@ -242,9 +250,10 @@ static int dm_hash_insert(const char *name, const char *uuid, struct mapped_devi
         return -EBUSY;
 }
 
-static void __hash_remove(struct hash_cell *hc)
+static struct dm_table *__hash_remove(struct hash_cell *hc)
 {
         struct dm_table *table;
+        int srcu_idx;
 
         /* remove from the dev hash */
         list_del(&hc->uuid_list);
@@ -253,16 +262,18 @@ static void __hash_remove(struct hash_cell *hc)
         dm_set_mdptr(hc->md, NULL);
         mutex_unlock(&dm_hash_cells_mutex);
 
-        table = dm_get_live_table(hc->md);
-        if (table) {
+        table = dm_get_live_table(hc->md, &srcu_idx);
+        if (table)
                 dm_table_event(table);
-                dm_table_put(table);
-        }
+        dm_put_live_table(hc->md, srcu_idx);
 
+        table = NULL;
         if (hc->new_map)
-                dm_table_destroy(hc->new_map);
+                table = hc->new_map;
         dm_put(hc->md);
         free_cell(hc);
+
+        return table;
 }
 
 static void dm_hash_remove_all(int keep_open_devices)
@@ -270,6 +281,7 @@ static void dm_hash_remove_all(int keep_open_devices)
         int i, dev_skipped;
         struct hash_cell *hc;
         struct mapped_device *md;
+        struct dm_table *t;
 
 retry:
         dev_skipped = 0;
@@ -287,10 +299,14 @@ retry:
                                 continue;
                         }
 
-                        __hash_remove(hc);
+                        t = __hash_remove(hc);
 
                         up_write(&_hash_lock);
 
+                        if (t) {
+                                dm_sync_table(md);
+                                dm_table_destroy(t);
+                        }
                         dm_put(md);
                         if (likely(keep_open_devices))
                                 dm_destroy(md);
@@ -356,6 +372,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
         struct dm_table *table;
         struct mapped_device *md;
         unsigned change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0;
+        int srcu_idx;
 
         /*
          * duplicate new.
@@ -418,11 +435,10 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
         /*
          * Wake up any dm event waiters.
          */
-        table = dm_get_live_table(hc->md);
-        if (table) {
+        table = dm_get_live_table(hc->md, &srcu_idx);
+        if (table)
                 dm_table_event(table);
-                dm_table_put(table);
-        }
+        dm_put_live_table(hc->md, srcu_idx);
 
         if (!dm_kobject_uevent(hc->md, KOBJ_CHANGE, param->event_nr))
                 param->flags |= DM_UEVENT_GENERATED_FLAG;
@@ -620,11 +636,14 @@ static int check_name(const char *name)
  * _hash_lock without first calling dm_table_put, because dm_table_destroy
  * waits for this dm_table_put and could be called under this lock.
  */
-static struct dm_table *dm_get_inactive_table(struct mapped_device *md)
+static struct dm_table *dm_get_inactive_table(struct mapped_device *md, int *srcu_idx)
 {
         struct hash_cell *hc;
         struct dm_table *table = NULL;
 
+        /* increment rcu count, we don't care about the table pointer */
+        dm_get_live_table(md, srcu_idx);
+
         down_read(&_hash_lock);
         hc = dm_get_mdptr(md);
         if (!hc || hc->md != md) {
@@ -633,8 +652,6 @@ static struct dm_table *dm_get_inactive_table(struct mapped_device *md)
         }
 
         table = hc->new_map;
-        if (table)
-                dm_table_get(table);
 
 out:
         up_read(&_hash_lock);
@@ -643,10 +660,11 @@ out:
 }
 
 static struct dm_table *dm_get_live_or_inactive_table(struct mapped_device *md,
-                                                      struct dm_ioctl *param)
+                                                      struct dm_ioctl *param,
+                                                      int *srcu_idx)
 {
         return (param->flags & DM_QUERY_INACTIVE_TABLE_FLAG) ?
-                dm_get_inactive_table(md) : dm_get_live_table(md);
+                dm_get_inactive_table(md, srcu_idx) : dm_get_live_table(md, srcu_idx);
 }
 
 /*
@@ -657,6 +675,7 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param)
 {
         struct gendisk *disk = dm_disk(md);
         struct dm_table *table;
+        int srcu_idx;
 
         param->flags &= ~(DM_SUSPEND_FLAG | DM_READONLY_FLAG |
                           DM_ACTIVE_PRESENT_FLAG);
@@ -676,26 +695,27 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param)
         param->event_nr = dm_get_event_nr(md);
         param->target_count = 0;
 
-        table = dm_get_live_table(md);
+        table = dm_get_live_table(md, &srcu_idx);
         if (table) {
                 if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) {
                         if (get_disk_ro(disk))
                                 param->flags |= DM_READONLY_FLAG;