bfq: Split shared queues on move between cgroups

commit 3bc5e683c67d94bd839a1da2e796c15847b51b69 upstream.

When bfqq is shared by multiple processes it can happen that one of the
processes gets moved to a different cgroup (or just starts submitting IO
for different cgroup). In case that happens we need to split the merged
bfqq as otherwise we will have IO for multiple cgroups in one bfqq and
we will just account IO time to wrong entities etc.

Similarly if the bfqq is scheduled to merge with another bfqq but the
merge didn't happen yet, cancel the merge as it need not be valid
anymore.

CC: stable@vger.kernel.org
Fixes: e21b7a0b9887 ("block, bfq: add full hierarchical scheduling and cgroups support")
Tested-by: "yukuai (C)" <yukuai3@huawei.com>
Signed-off-by: Jan Kara <jack@suse.cz>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/20220401102752.8599-3-jack@suse.cz
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index 809bc612d96b315e2783548dcf2f99be63ba48c9..6a7f2b7822a542e1681f0d5341d0ec241ce00bba 100644 (file)
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -735,9 +735,39 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
        }
 
        if (sync_bfqq) {
-               entity = &sync_bfqq->entity;
-               if (entity->sched_data != &bfqg->sched_data)
-                       bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
+               if (!sync_bfqq->new_bfqq && !bfq_bfqq_coop(sync_bfqq)) {
+                       /* We are the only user of this bfqq, just move it */
+                       if (sync_bfqq->entity.sched_data != &bfqg->sched_data)
+                               bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
+               } else {
+                       struct bfq_queue *bfqq;
+
+                       /*
+                        * The queue was merged to a different queue. Check
+                        * that the merge chain still belongs to the same
+                        * cgroup.
+                        */
+                       for (bfqq = sync_bfqq; bfqq; bfqq = bfqq->new_bfqq)
+                               if (bfqq->entity.sched_data !=
+                                   &bfqg->sched_data)
+                                       break;
+                       if (bfqq) {
+                               /*
+                                * Some queue changed cgroup so the merge is
+                                * not valid anymore. We cannot easily just
+                                * cancel the merge (by clearing new_bfqq) as
+                                * there may be other processes using this
+                                * queue and holding refs to all queues below
+                                * sync_bfqq->new_bfqq. Similarly if the merge
+                                * already happened, we need to detach from
+                                * bfqq now so that we cannot merge bio to a
+                                * request from the old cgroup.
+                                */
+                               bfq_put_cooperator(sync_bfqq);
+                               bfq_release_process_ref(bfqd, sync_bfqq);
+                               bic_set_bfqq(bic, NULL, 1);
+                       }
+               }
        }
 
        return bfqg;

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index b8907f1cd07f5fe2091e93265060daafbf71cdea..6d7c7c869b5d4a912b95b3d9cb82bba9b1168f3d 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -5319,7 +5319,7 @@ static void bfq_put_stable_ref(struct bfq_queue *bfqq)
        bfq_put_queue(bfqq);
 }
 
-static void bfq_put_cooperator(struct bfq_queue *bfqq)
+void bfq_put_cooperator(struct bfq_queue *bfqq)
 {
        struct bfq_queue *__bfqq, *next;
 

diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index a1ec70af64c8abb2e738fd544b860ecf34096843..3773401623ad28a1e7a9c46e935532e5ac4373c0 100644 (file)
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -981,6 +981,7 @@ void bfq_weights_tree_remove(struct bfq_data *bfqd,
 void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
                     bool compensate, enum bfqq_expiration reason);
 void bfq_put_queue(struct bfq_queue *bfqq);
+void bfq_put_cooperator(struct bfq_queue *bfqq);
 void bfq_end_wr_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);
 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq);
 void bfq_schedule_dispatch(struct bfq_data *bfqd);