6 files changed, 84 insertions, 33 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 5d0d7441a443..77807a5d7f9e 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -520,6 +520,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         mutex_init(&q->blk_trace_mutex);
 #endif
         mutex_init(&q->sysfs_lock);
+        mutex_init(&q->sysfs_dir_lock);
         spin_lock_init(&q->queue_lock);
 
         init_waitqueue_head(&q->mq_freeze_wq);
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index 6ddde3774ebe..a0d3ce30fa08 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -270,7 +270,7 @@ void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
         struct blk_mq_hw_ctx *hctx;
         int i;
 
-        lockdep_assert_held(&q->sysfs_lock);
+        lockdep_assert_held(&q->sysfs_dir_lock);
 
         queue_for_each_hw_ctx(q, hctx, i)
                 blk_mq_unregister_hctx(hctx);
@@ -320,7 +320,7 @@ int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
         int ret, i;
 
         WARN_ON_ONCE(!q->kobj.parent);
-        lockdep_assert_held(&q->sysfs_lock);
+        lockdep_assert_held(&q->sysfs_dir_lock);
 
         ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
         if (ret < 0)
@@ -354,7 +354,7 @@ void blk_mq_sysfs_unregister(struct request_queue *q)
         struct blk_mq_hw_ctx *hctx;
         int i;
 
-        mutex_lock(&q->sysfs_lock);
+        mutex_lock(&q->sysfs_dir_lock);
         if (!q->mq_sysfs_init_done)
                 goto unlock;
 
@@ -362,7 +362,7 @@ void blk_mq_sysfs_unregister(struct request_queue *q)
                 blk_mq_unregister_hctx(hctx);
 
 unlock:
-        mutex_unlock(&q->sysfs_lock);
+        mutex_unlock(&q->sysfs_dir_lock);
 }
 
 int blk_mq_sysfs_register(struct request_queue *q)
@@ -370,7 +370,7 @@ int blk_mq_sysfs_register(struct request_queue *q)
         struct blk_mq_hw_ctx *hctx;
         int i, ret = 0;
 
-        mutex_lock(&q->sysfs_lock);
+        mutex_lock(&q->sysfs_dir_lock);
         if (!q->mq_sysfs_init_done)
                 goto unlock;
 
@@ -381,7 +381,7 @@ int blk_mq_sysfs_register(struct request_queue *q)
         }
 
 unlock:
-        mutex_unlock(&q->sysfs_lock);
+        mutex_unlock(&q->sysfs_dir_lock);
 
         return ret;
 }
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 5b0b5224cfd4..107513495220 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -938,6 +938,7 @@ int blk_register_queue(struct gendisk *disk)
         int ret;
         struct device *dev = disk_to_dev(disk);
         struct request_queue *q = disk->queue;
+        bool has_elevator = false;
 
         if (WARN_ON(!q))
                 return -ENXIO;
@@ -945,7 +946,6 @@ int blk_register_queue(struct gendisk *disk)
         WARN_ONCE(blk_queue_registered(q),
                   "%s is registering an already registered queue\n",
                   kobject_name(&dev->kobj));
-        blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
 
         /*
          * SCSI probing may synchronously create and destroy a lot of
@@ -965,8 +965,7 @@ int blk_register_queue(struct gendisk *disk)
         if (ret)
                 return ret;
 
-        /* Prevent changes through sysfs until registration is completed. */
-        mutex_lock(&q->sysfs_lock);
+        mutex_lock(&q->sysfs_dir_lock);
 
         ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
         if (ret < 0) {
@@ -987,26 +986,36 @@ int blk_register_queue(struct gendisk *disk)
                 blk_mq_debugfs_register(q);
         }
 
-        kobject_uevent(&q->kobj, KOBJ_ADD);
-
-        wbt_enable_default(q);
-
-        blk_throtl_register_queue(q);
-
+        /*
+         * The flag of QUEUE_FLAG_REGISTERED isn't set yet, so elevator
+         * switch won't happen at all.
+         */
         if (q->elevator) {
-                ret = elv_register_queue(q);
+                ret = elv_register_queue(q, false);
                 if (ret) {
-                        mutex_unlock(&q->sysfs_lock);
-                        kobject_uevent(&q->kobj, KOBJ_REMOVE);
+                        mutex_unlock(&q->sysfs_dir_lock);
                         kobject_del(&q->kobj);
                         blk_trace_remove_sysfs(dev);
                         kobject_put(&dev->kobj);
                         return ret;
                 }
+                has_elevator = true;
         }
+
+        mutex_lock(&q->sysfs_lock);
+        blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
+        wbt_enable_default(q);
+        blk_throtl_register_queue(q);
+
+        /* Now everything is ready and send out KOBJ_ADD uevent */
+        kobject_uevent(&q->kobj, KOBJ_ADD);
+        if (has_elevator)
+                kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
+        mutex_unlock(&q->sysfs_lock);
+
         ret = 0;
 unlock:
-        mutex_unlock(&q->sysfs_lock);
+        mutex_unlock(&q->sysfs_dir_lock);
         return ret;
 }
 EXPORT_SYMBOL_GPL(blk_register_queue);
@@ -1021,6 +1030,7 @@ EXPORT_SYMBOL_GPL(blk_register_queue);
 void blk_unregister_queue(struct gendisk *disk)
 {
         struct request_queue *q = disk->queue;
+        bool has_elevator;
 
         if (WARN_ON(!q))
                 return;
@@ -1035,25 +1045,25 @@ void blk_unregister_queue(struct gendisk *disk)
          * concurrent elv_iosched_store() calls.
          */
         mutex_lock(&q->sysfs_lock);
-
         blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
+        has_elevator = !!q->elevator;
+        mutex_unlock(&q->sysfs_lock);
 
+        mutex_lock(&q->sysfs_dir_lock);
         /*
          * Remove the sysfs attributes before unregistering the queue data
          * structures that can be modified through sysfs.
          */
         if (queue_is_mq(q))
                 blk_mq_unregister_dev(disk_to_dev(disk), q);
-        mutex_unlock(&q->sysfs_lock);
 
         kobject_uevent(&q->kobj, KOBJ_REMOVE);
         kobject_del(&q->kobj);
         blk_trace_remove_sysfs(disk_to_dev(disk));
 
-        mutex_lock(&q->sysfs_lock);
-        if (q->elevator)
+        if (has_elevator)
                 elv_unregister_queue(q);
-        mutex_unlock(&q->sysfs_lock);
+        mutex_unlock(&q->sysfs_dir_lock);
 
         kobject_put(&disk_to_dev(disk)->kobj);
 }
diff --git a/block/blk.h b/block/blk.h
index de6b2e146d6e..e4619fc5c99a 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -188,7 +188,7 @@ int elevator_init_mq(struct request_queue *q);
 int elevator_switch_mq(struct request_queue *q,
                               struct elevator_type *new_e);
 void __elevator_exit(struct request_queue *, struct elevator_queue *);
-int elv_register_queue(struct request_queue *q);
+int elv_register_queue(struct request_queue *q, bool uevent);
 void elv_unregister_queue(struct request_queue *q);
 
 static inline void elevator_exit(struct request_queue *q,
diff --git a/block/elevator.c b/block/elevator.c
index 03d923196569..4781c4205a5d 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -470,13 +470,16 @@ static struct kobj_type elv_ktype = {
         .release        = elevator_release,
 };
 
-int elv_register_queue(struct request_queue *q)
+/*
+ * elv_register_queue is called from either blk_register_queue or
+ * elevator_switch, elevator switch is prevented from being happen
+ * in the two paths, so it is safe to not hold q->sysfs_lock.
+ */
+int elv_register_queue(struct request_queue *q, bool uevent)
 {
         struct elevator_queue *e = q->elevator;
         int error;
 
-        lockdep_assert_held(&q->sysfs_lock);
-
         error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
         if (!error) {
                 struct elv_fs_entry *attr = e->type->elevator_attrs;
@@ -487,24 +490,34 @@ int elv_register_queue(struct request_queue *q)
                                 attr++;
                         }
                 }
-                kobject_uevent(&e->kobj, KOBJ_ADD);
+                if (uevent)
+                        kobject_uevent(&e->kobj, KOBJ_ADD);
+
+                mutex_lock(&q->sysfs_lock);
                 e->registered = 1;
+                mutex_unlock(&q->sysfs_lock);
         }
         return error;
 }
 
+/*
+ * elv_unregister_queue is called from either blk_unregister_queue or
+ * elevator_switch, elevator switch is prevented from being happen
+ * in the two paths, so it is safe to not hold q->sysfs_lock.
+ */
 void elv_unregister_queue(struct request_queue *q)
 {
-        lockdep_assert_held(&q->sysfs_lock);
-
         if (q) {
                 struct elevator_queue *e = q->elevator;
 
                 kobject_uevent(&e->kobj, KOBJ_REMOVE);
                 kobject_del(&e->kobj);
+
+                mutex_lock(&q->sysfs_lock);
                 e->registered = 0;
                 /* Re-enable throttling in case elevator disabled it */
                 wbt_enable_default(q);
+                mutex_unlock(&q->sysfs_lock);
         }
 }
 
@@ -567,10 +580,32 @@ int elevator_switch_mq(struct request_queue *q,
         lockdep_assert_held(&q->sysfs_lock);
 
         if (q->elevator) {
-                if (q->elevator->registered)
+                if (q->elevator->registered) {
+                        mutex_unlock(&q->sysfs_lock);
+
+                        /*
+                         * Concurrent elevator switch can't happen becasue
+                         * sysfs write is always exclusively on same file.
+                         *
+                         * Also the elevator queue won't be freed after
+                         * sysfs_lock is released becasue kobject_del() in
+                         * blk_unregister_queue() waits for completion of
+                         * .store & .show on its attributes.
+                         */
                         elv_unregister_queue(q);
+
+                        mutex_lock(&q->sysfs_lock);
+                }
                 ioc_clear_queue(q);
                 elevator_exit(q, q->elevator);
+
+                /*
+                 * sysfs_lock may be dropped, so re-check if queue is
+                 * unregistered. If yes, don't switch to new elevator
+                 * any more
+                 */
+                if (!blk_queue_registered(q))
+                        return 0;
         }
 
         ret = blk_mq_init_sched(q, new_e);
@@ -578,7 +613,11 @@ int elevator_switch_mq(struct request_queue *q,
                 goto out;
 
         if (new_e) {
-                ret = elv_register_queue(q);
+                mutex_unlock(&q->sysfs_lock);
+
+                ret = elv_register_queue(q, true);
+
+                mutex_lock(&q->sysfs_lock);
                 if (ret) {
                         elevator_exit(q, q->elevator);
                         goto out;
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index d5077f3fdfd6..1ac790178787 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -535,6 +535,7 @@ struct request_queue {
         struct delayed_work     requeue_work;
 
         struct mutex            sysfs_lock;
+        struct mutex            sysfs_dir_lock;
 
         /*
          * for reusing dead hctx instance in case of updating