rq-qos: use a mb for got_token

Oleg noticed that our checking of data.got_token is unsafe in the
cleanup case, and should really use a memory barrier.  Use a wmb on the
write side, and a rmb() on the read side.  We don't need one in the main
loop since we're saved by set_current_state().

Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

1 files changed, 3 insertions, 0 deletions

diff --git a/block/blk-rq-qos.c b/block/blk-rq-qos.c
index c450b8952eae..3954c0dc1443 100644
--- a/block/blk-rq-qos.c
+++ b/block/blk-rq-qos.c
@@ -202,6 +202,7 @@ static int rq_qos_wake_function(struct wait_queue_entry *curr,
                 return -1;
 
         data->got_token = true;
+        smp_wmb();
         list_del_init(&curr->entry);
         wake_up_process(data->task);
         return 1;
@@ -246,6 +247,7 @@ void rq_qos_wait(struct rq_wait *rqw, void *private_data,
         prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);
         has_sleeper = !wq_has_single_sleeper(&rqw->wait);
         do {
+                /* The memory barrier in set_task_state saves us here. */
                 if (data.got_token)
                         break;
                 if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {
@@ -256,6 +258,7 @@ void rq_qos_wait(struct rq_wait *rqw, void *private_data,
                          * which means we now have two. Put our local token
                          * and wake anyone else potentially waiting for one.
                          */
+                        smp_rmb();
                         if (data.got_token)
                                 cleanup_cb(rqw, private_data);
                         break;