litmus-rt.git - The LITMUS^RT kernel.

diff options

author	Dave Airlie <airlied@redhat.com>	2010-02-24 22:44:04 -0500
committer	Dave Airlie <airlied@redhat.com>	2010-02-26 01:23:23 -0500
commit	eb6b6d7cdd5548fa03a919d14615195600013be2 (patch)
tree	c499472ef28a903f94b09da9a24f66b021412813 /drivers/gpu/drm/radeon/atombios_dp.c
parent	30d6c72c4a760cfc9069ee100786e4d6cf6de59d (diff)
parent	383be5d1789d9a7a2e77dca1cb0aca89507d069e (diff)

Merge remote branch 'korg/drm-radeon-testing' into drm-next-stage

* korg/drm-radeon-testing: (62 commits) drm/radeon/kms: update new pll algo drm/radeon/kms: add support for square microtiles on r3xx-r5xx drm/radeon/kms: force pinning buffer into visible VRAM drm/radeon/kms/evergreen: fix typo in cursor code drm/radeon/kms: implement reading active PCIE lanes on R600+ drm/radeon/kms: for downclocking non-mobility check PERFORMANCE state drm/radeon/kms: simplify storing current and requested PM mode drm/radeon: fixes for r6xx/r7xx gfx init drm/radeon/rv740: fix backend setup drm/radeon/kms: fix R3XX/R4XX memory controller initialization [rfc] drm/radeon/kms: pm debugging check for vbl. drm/radeon: Fix memory allocation failures in the preKMS command stream checking. drm: Add generic multipart buffer. drm/radeon/kms: simplify memory controller setup V2 drm/radeon: Add asic hook for dma copy to r200 cards. drm/radeon/kms: Create asic structure for r300 pcie cards. drm/radeon/kms: remove unused r600_gart_clear_page drm/radeon/kms: remove HDP flushes from fence emit (v2) drm/radeon/kms: add LVDS pll quirk for Dell Studio 15 drm/radeon/kms: simplify picking power state ... Conflicts: drivers/gpu/drm/radeon/atom.c drivers/gpu/drm/radeon/atombios.h drivers/gpu/drm/radeon/atombios_dp.c drivers/gpu/drm/radeon/r600.c drivers/gpu/drm/radeon/r600_audio.c drivers/gpu/drm/radeon/r600_cp.c drivers/gpu/drm/radeon/radeon.h drivers/gpu/drm/radeon/radeon_connectors.c drivers/gpu/drm/radeon/radeon_ring.c drivers/gpu/drm/radeon/rv770.c

Diffstat (limited to 'drivers/gpu/drm/radeon/atombios_dp.c')

-rw-r--r--

drivers/gpu/drm/radeon/atombios_dp.c

1 files changed, 42 insertions, 22 deletions

                 train_set[lane] = v | p;
 }
+union aux_channel_transaction {
+        PROCESS_AUX_CHANNEL_TRANSACTION_PS_ALLOCATION v1;
+        PROCESS_AUX_CHANNEL_TRANSACTION_PARAMETERS_V2 v2;
+};
 /* radeon aux chan functions */
 bool radeon_process_aux_ch(struct radeon_i2c_chan *chan, u8 *req_bytes,
 {
         struct drm_device *dev = chan->dev;
         struct radeon_device *rdev = dev->dev_private;
-        PROCESS_AUX_CHANNEL_TRANSACTION_PS_ALLOCATION args;
+        union aux_channel_transaction args;
         int index = GetIndexIntoMasterTable(COMMAND, ProcessAuxChannelTransaction);
         unsigned char *base;
         int retry_count = 0;
 retry:
         memcpy(base, req_bytes, num_bytes);
-        args.lpAuxRequest = 0;
+        args.v1.lpAuxRequest = 0;
-        args.lpDataOut = 16;
+        args.v1.lpDataOut = 16;
-        args.ucDataOutLen = 0;
+        args.v1.ucDataOutLen = 0;
-        args.ucChannelID = chan->rec.i2c_id;
+        args.v1.ucChannelID = chan->rec.i2c_id;
-        args.ucDelay = delay / 10;
+        args.v1.ucDelay = delay / 10;
+        if (ASIC_IS_DCE4(rdev))
+                args.v2.ucHPD_ID = chan->rec.hpd_id;
         atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
-        if (args.ucReplyStatus && !args.ucDataOutLen) {
+        if (args.v1.ucReplyStatus && !args.v1.ucDataOutLen) {
-                if (args.ucReplyStatus == 0x20 && retry_count++ < 10)
+                if (args.v1.ucReplyStatus == 0x20 && retry_count++ < 10)
                         goto retry;
                 DRM_DEBUG("failed to get auxch %02x%02x %02x %02x 0x%02x %02x after %d retries\n",
                           req_bytes[1], req_bytes[0], req_bytes[2], req_bytes[3],
-                          chan->rec.i2c_id, args.ucReplyStatus, retry_count);
+                          chan->rec.i2c_id, args.v1.ucReplyStatus, retry_count);
                 return false;
         }
-        if (args.ucDataOutLen && read_byte && read_buf_len) {
+        if (args.v1.ucDataOutLen && read_byte && read_buf_len) {
-                if (read_buf_len < args.ucDataOutLen) {
+                if (read_buf_len < args.v1.ucDataOutLen) {
                         DRM_ERROR("Buffer to small for return answer %d %d\n",
-                                  read_buf_len, args.ucDataOutLen);
+                                  read_buf_len, args.v1.ucDataOutLen);
                         return false;
                 }
                 {
-                        int len = min(read_buf_len, args.ucDataOutLen);
+                        int len = min(read_buf_len, args.v1.ucDataOutLen);
                         memcpy(read_byte, base + 16, len);
                 }
         }
         dp_set_link_bw_lanes(radeon_connector, link_configuration);
         /* disable downspread on the sink */
         dp_set_downspread(radeon_connector, 0);
-        /* start training on the source */
+        if (ASIC_IS_DCE4(rdev)) {
-        radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_START,
+                /* start training on the source */
-                                  dig_connector->dp_clock, enc_id, 0);
+                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_START);
-        /* set training pattern 1 on the source */
+                /* set training pattern 1 on the source */
-        radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
+                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1);
-                                  dig_connector->dp_clock, enc_id, 0);
+        } else {
+                /* start training on the source */
+                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_START,
+                                          dig_connector->dp_clock, enc_id, 0);
+                /* set training pattern 1 on the source */
+                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
+                                          dig_connector->dp_clock, enc_id, 0);
+        }
         /* set initial vs/emph */
         memset(train_set, 0, 4);
         /* set training pattern 2 on the sink */
         dp_set_training(radeon_connector, DP_TRAINING_PATTERN_2);
         /* set training pattern 2 on the source */
-        radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
+        if (ASIC_IS_DCE4(rdev))
-                                  dig_connector->dp_clock, enc_id, 1);
+                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2);
+        else
+                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_PATTERN_SEL,
+                                          dig_connector->dp_clock, enc_id, 1);
         /* channel equalization loop */
         tries = 0;
                           >> DP_TRAIN_PRE_EMPHASIS_SHIFT);
         /* disable the training pattern on the sink */
-        dp_set_training(radeon_connector, DP_TRAINING_PATTERN_DISABLE);
+        if (ASIC_IS_DCE4(rdev))
+                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE);
+        else
+                radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_COMPLETE,
+                                          dig_connector->dp_clock, enc_id, 0);
         radeon_dp_encoder_service(rdev, ATOM_DP_ACTION_TRAINING_COMPLETE,
                                   dig_connector->dp_clock, enc_id, 0);

311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364

437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527

3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393

// SPDX-License-Identifier: GPL-2.0-or-later /* * Budget Fair Queueing (BFQ) I/O scheduler. * * Based on ideas and code from CFQ: * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> * * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> * Paolo Valente <paolo.valente@unimore.it> * * Copyright (C) 2010 Paolo Valente <paolo.valente@unimore.it> * Arianna Avanzini <avanzini@google.com> * * Copyright (C) 2017 Paolo Valente <paolo.valente@linaro.org> * * BFQ is a proportional-share I/O scheduler, with some extra * low-latency capabilities. BFQ also supports full hierarchical * scheduling through cgroups. Next paragraphs provide an introduction * on BFQ inner workings. Details on BFQ benefits, usage and * limitations can be found in Documentation/block/bfq-iosched.rst. * * BFQ is a proportional-share storage-I/O scheduling algorithm based * on the slice-by-slice service scheme of CFQ. But BFQ assigns * budgets, measured in number of sectors, to processes instead of * time slices. The device is not granted to the in-service process * for a given time slice, but until it has exhausted its assigned * budget. This change from the time to the service domain enables BFQ * to distribute the device throughput among processes as desired, * without any distortion due to throughput fluctuations, or to device * internal queueing. BFQ uses an ad hoc internal scheduler, called * B-WF2Q+, to schedule processes according to their budgets. More * precisely, BFQ schedules queues associated with processes. Each * process/queue is assigned a user-configurable weight, and B-WF2Q+ * guarantees that each queue receives a fraction of the throughput * proportional to its weight. Thanks to the accurate policy of * B-WF2Q+, BFQ can afford to assign high budgets to I/O-bound * processes issuing sequential requests (to boost the throughput), * and yet guarantee a low latency to interactive and soft real-time * applications. * * In particular, to provide these low-latency guarantees, BFQ * explicitly privileges the I/O of two classes of time-sensitive * applications: interactive and soft real-time. In more detail, BFQ * behaves this way if the low_latency parameter is set (default * configuration). This feature enables BFQ to provide applications in * these classes with a very low latency. * * To implement this feature, BFQ constantly tries to detect whether * the I/O requests in a bfq_queue come from an interactive or a soft * real-time application. For brevity, in these cases, the queue is * said to be interactive or soft real-time. In both cases, BFQ * privileges the service of the queue, over that of non-interactive * and non-soft-real-time queues. This privileging is performed, * mainly, by raising the weight of the queue. So, for brevity, we * call just weight-raising periods the time periods during which a * queue is privileged, because deemed interactive or soft real-time. * * The detection of soft real-time queues/applications is described in * detail in the comments on the function * bfq_bfqq_softrt_next_start. On the other hand, the detection of an * interactive queue works as follows: a queue is deemed interactive * if it is constantly non empty only for a limited time interval, * after which it does become empty. The queue may be deemed * interactive again (for a limited time), if it restarts being * constantly non empty, provided that this happens only after the * queue has remained empty for a given minimum idle time. * * By default, BFQ computes automatically the above maximum time * interval, i.e., the time interval after which a constantly * non-empty queue stops being deemed interactive. Since a queue is * weight-raised while it is deemed interactive, this maximum time * interval happens to coincide with the (maximum) duration of the * weight-raising for interactive queues. * * Finally, BFQ also features additional heuristics for * preserving both a low latency and a high throughput on NCQ-capable, * rotational or flash-based devices, and to get the job done quickly * for applications consisting in many I/O-bound processes. * * NOTE: if the main or only goal, with a given device, is to achieve * the maximum-possible throughput at all times, then do switch off * all low-latency heuristics for that device, by setting low_latency * to 0. * * BFQ is described in [1], where also a reference to the initial, * more theoretical paper on BFQ can be found. The interested reader * can find in the latter paper full details on the main algorithm, as * well as formulas of the guarantees and formal proofs of all the * properties. With respect to the version of BFQ presented in these * papers, this implementation adds a few more heuristics, such as the * ones that guarantee a low latency to interactive and soft real-time * applications, and a hierarchical extension based on H-WF2Q+. * * B-WF2Q+ is based on WF2Q+, which is described in [2], together with * H-WF2Q+, while the augmented tree used here to implement B-WF2Q+ * with O(log N) complexity derives from the one introduced with EEVDF * in [3]. * * [1] P. Valente, A. Avanzini, "Evolution of the BFQ Storage I/O * Scheduler", Proceedings of the First Workshop on Mobile System * Technologies (MST-2015), May 2015. * http://algogroup.unimore.it/people/paolo/disk_sched/mst-2015.pdf * * [2] Jon C.R. Bennett and H. Zhang, "Hierarchical Packet Fair Queueing * Algorithms", IEEE/ACM Transactions on Networking, 5(5):675-689, * Oct 1997. * * http://www.cs.cmu.edu/~hzhang/papers/TON-97-Oct.ps.gz * * [3] I. Stoica and H. Abdel-Wahab, "Earliest Eligible Virtual Deadline * First: A Flexible and Accurate Mechanism for Proportional Share * Resource Allocation", technical report. * * http://www.cs.berkeley.edu/~istoica/papers/eevdf-tr-95.pdf */ #include <linux/module.h> #include <linux/slab.h> #include <linux/blkdev.h> #include <linux/cgroup.h> #include <linux/elevator.h> #include <linux/ktime.h> #include <linux/rbtree.h> #include <linux/ioprio.h> #include <linux/sbitmap.h> #include <linux/delay.h> #include "blk.h" #include "blk-mq.h" #include "blk-mq-tag.h" #include "blk-mq-sched.h" #include "bfq-iosched.h" #include "blk-wbt.h" #define BFQ_BFQQ_FNS(name) \ void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \ { \ __set_bit(BFQQF_##name, &(bfqq)->flags); \ } \ void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \ { \ __clear_bit(BFQQF_##name, &(bfqq)->flags); \ } \ int bfq_bfqq_##name(const struct bfq_queue *bfqq) \ { \ return test_bit(BFQQF_##name, &(bfqq)->flags); \ } BFQ_BFQQ_FNS(just_created); BFQ_BFQQ_FNS(busy); BFQ_BFQQ_FNS(wait_request); BFQ_BFQQ_FNS(non_blocking_wait_rq); BFQ_BFQQ_FNS(fifo_expire); BFQ_BFQQ_FNS(has_short_ttime); BFQ_BFQQ_FNS(sync); BFQ_BFQQ_FNS(IO_bound); BFQ_BFQQ_FNS(in_large_burst); BFQ_BFQQ_FNS(coop); BFQ_BFQQ_FNS(split_coop); BFQ_BFQQ_FNS(softrt_update); BFQ_BFQQ_FNS(has_waker); #undef BFQ_BFQQ_FNS \ /* Expiration time of sync (0) and async (1) requests, in ns. */ static const u64 bfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 }; /* Maximum backwards seek (magic number lifted from CFQ), in KiB. */ static const int bfq_back_max = 16 * 1024; /* Penalty of a backwards seek, in number of sectors. */ static const int bfq_back_penalty = 2; /* Idling period duration, in ns. */ static u64 bfq_slice_idle = NSEC_PER_SEC / 125; /* Minimum number of assigned budgets for which stats are safe to compute. */ static const int bfq_stats_min_budgets = 194; /* Default maximum budget values, in sectors and number of requests. */ static const int bfq_default_max_budget = 16 * 1024; /* * When a sync request is dispatched, the queue that contains that * request, and all the ancestor entities of that queue, are charged * with the number of sectors of the request. In contrast, if the * request is async, then the queue and its ancestor entities are * charged with the number of sectors of the request, multiplied by * the factor below. This throttles the bandwidth for async I/O, * w.r.t. to sync I/O, and it is done to counter the tendency of async * writes to steal I/O throughput to reads. * * The current value of this parameter is the result of a tuning with * several hardware and software configurations. We tried to find the * lowest value for which writes do not cause noticeable problems to * reads. In fact, the lower this parameter, the stabler I/O control, * in the following respect. The lower this parameter is, the less * the bandwidth enjoyed by a group decreases * - when the group does writes, w.r.t. to when it does reads; * - when other groups do reads, w.r.t. to when they do writes. */ static const int bfq_async_charge_factor = 3; /* Default timeout values, in jiffies, approximating CFQ defaults. */ const int bfq_timeout = HZ / 8; /* * Time limit for merging (see comments in bfq_setup_cooperator). Set * to the slowest value that, in our tests, proved to be effective in * removing false positives, while not causing true positives to miss * queue merging. * * As can be deduced from the low time limit below, queue merging, if * successful, happens at the very beginning of the I/O of the involved * cooperating processes, as a consequence of the arrival of the very * first requests from each cooperator. After that, there is very * little chance to find cooperators. */ static const unsigned long bfq_merge_time_limit = HZ/10; static struct kmem_cache *bfq_pool; /* Below this threshold (in ns), we consider thinktime immediate. */ #define BFQ_MIN_TT (2 * NSEC_PER_MSEC) /* hw_tag detection: parallel requests threshold and min samples needed. */ #define BFQ_HW_QUEUE_THRESHOLD 3 #define BFQ_HW_QUEUE_SAMPLES 32 #define BFQQ_SEEK_THR (sector_t)(8 * 100) #define BFQQ_SECT_THR_NONROT (sector_t)(2 * 32) #define BFQ_RQ_SEEKY(bfqd, last_pos, rq) \ (get_sdist(last_pos, rq) > \ BFQQ_SEEK_THR && \ (!blk_queue_nonrot(bfqd->queue) || \ blk_rq_sectors(rq) < BFQQ_SECT_THR_NONROT)) #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) /* * Sync random I/O is likely to be confused with soft real-time I/O, * because it is characterized by limited throughput and apparently * isochronous arrival pattern. To avoid false positives, queues * containing only random (seeky) I/O are prevented from being tagged * as soft real-time. */ #define BFQQ_TOTALLY_SEEKY(bfqq) (bfqq->seek_history == -1) /* Min number of samples required to perform peak-rate update */ #define BFQ_RATE_MIN_SAMPLES 32 /* Min observation time interval required to perform a peak-rate update (ns) */ #define BFQ_RATE_MIN_INTERVAL (300*NSEC_PER_MSEC) /* Target observation time interval for a peak-rate update (ns) */ #define BFQ_RATE_REF_INTERVAL NSEC_PER_SEC /* * Shift used for peak-rate fixed precision calculations. * With * - the current shift: 16 positions * - the current type used to store rate: u32 * - the current unit of measure for rate: [sectors/usec], or, more precisely, * [(sectors/usec) / 2^BFQ_RATE_SHIFT] to take into account the shift, * the range of rates that can be stored is * [1 / 2^BFQ_RATE_SHIFT, 2^(32 - BFQ_RATE_SHIFT)] sectors/usec = * [1 / 2^16, 2^16] sectors/usec = [15e-6, 65536] sectors/usec = * [15, 65G] sectors/sec * Which, assuming a sector size of 512B, corresponds to a range of * [7.5K, 33T] B/sec */ #define BFQ_RATE_SHIFT 16 /* * When configured for computing the duration of the weight-raising * for interactive queues automatically (see the comments at the * beginning of this file), BFQ does it using the following formula: * duration = (ref_rate / r) * ref_wr_duration, * where r is the peak rate of the device, and ref_rate and * ref_wr_duration are two reference parameters. In particular, * ref_rate is the peak rate of the reference storage device (see * below), and ref_wr_duration is about the maximum time needed, with * BFQ and while reading two files in parallel, to load typical large * applications on the reference device (see the comments on * max_service_from_wr below, for more details on how ref_wr_duration * is obtained). In practice, the slower/faster the device at hand * is, the more/less it takes to load applications with respect to the * reference device. Accordingly, the longer/shorter BFQ grants * weight raising to interactive applications. * * BFQ uses two different reference pairs (ref_rate, ref_wr_duration), * depending on whether the device is rotational or non-rotational. * * In the following definitions, ref_rate[0] and ref_wr_duration[0] * are the reference values for a rotational device, whereas * ref_rate[1] and ref_wr_duration[1] are the reference values for a * non-rotational device. The reference rates are not the actual peak * rates of the devices used as a reference, but slightly lower * values. The reason for using slightly lower values is that the * peak-rate estimator tends to yield slightly lower values than the * actual peak rate (it can yield the actual peak rate only if there * is only one process doing I/O, and the process does sequential * I/O). * * The reference peak rates are measured in sectors/usec, left-shifted * by BFQ_RATE_SHIFT. */ static int ref_rate[2] = {14000, 33000}; /* * To improve readability, a conversion function is used to initialize * the following array, which entails that the array can be * initialized only in a function. */ static int ref_wr_duration[2]; /* * BFQ uses the above-detailed, time-based weight-raising mechanism to * privilege interactive tasks. This mechanism is vulnerable to the * following false positives: I/O-bound applications that will go on * doing I/O for much longer than the duration of weight * raising. These applications have basically no benefit from being * weight-raised at the beginning of their I/O. On the opposite end, * while being weight-raised, these applications * a) unjustly steal throughput to applications that may actually need * low latency; * b) make BFQ uselessly perform device idling; device idling results * in loss of device throughput with most flash-based storage, and may * increase latencies when used purposelessly. * * BFQ tries to reduce these problems, by adopting the following * countermeasure. To introduce this countermeasure, we need first to * finish explaining how the duration of weight-raising for * interactive tasks is computed. * * For a bfq_queue deemed as interactive, the duration of weight * raising is dynamically adjusted, as a function of the estimated * peak rate of the device, so as to be equal to the time needed to * execute the 'largest' interactive task we benchmarked so far. By * largest task, we mean the task for which each involved process has * to do more I/O than for any of the other tasks we benchmarked. This * reference interactive task is the start-up of LibreOffice Writer, * and in this task each process/bfq_queue needs to have at most ~110K * sectors transferred. * * This last piece of information enables BFQ to reduce the actual * duration of weight-raising for at least one class of I/O-bound * applications: those doing sequential or quasi-sequential I/O. An * example is file copy. In fact, once started, the main I/O-bound * processes of these applications usually consume the above 110K * sectors in much less time than the processes of an application that * is starting, because these I/O-bound processes will greedily devote * almost all their CPU cycles only to their target, * throughput-friendly I/O operations. This is even more true if BFQ * happens to be underestimating the device peak rate, and thus * overestimating the duration of weight raising. But, according to * our measurements, once transferred 110K sectors, these processes * have no right to be weight-raised any longer. * * Basing on the last consideration, BFQ ends weight-raising for a * bfq_queue if the latter happens to have received an amount of * service at least equal to the following constant. The constant is * set to slightly more than 110K, to have a minimum safety margin. * * This early ending of weight-raising reduces the amount of time * during which interactive false positives cause the two problems * described at the beginning of these comments. */ static const unsigned long max_service_from_wr = 120000; #define RQ_BIC(rq) icq_to_bic((rq)->elv.priv[0]) #define RQ_BFQQ(rq) ((rq)->elv.priv[1]) struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync) { return bic->bfqq[is_sync]; } void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync) { bic->bfqq[is_sync] = bfqq; } struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic) { return bic->icq.q->elevator->elevator_data; } /** * icq_to_bic - convert iocontext queue structure to bfq_io_cq. * @icq: the iocontext queue. */ static struct bfq_io_cq *icq_to_bic(struct io_cq *icq) { /* bic->icq is the first member, %NULL will convert to %NULL */ return container_of(icq, struct bfq_io_cq, icq); } /** * bfq_bic_lookup - search into @ioc a bic associated to @bfqd. * @bfqd: the lookup key. * @ioc: the io_context of the process doing I/O. * @q: the request queue. */ static struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, struct io_context *ioc, struct request_queue *q) { if (ioc) { unsigned long flags; struct bfq_io_cq *icq; spin_lock_irqsave(&q->queue_lock, flags); icq = icq_to_bic(ioc_lookup_icq(ioc, q)); spin_unlock_irqrestore(&q->queue_lock, flags); return icq; } return NULL; } /* * Scheduler run of queue, if there are requests pending and no one in the * driver that will restart queueing. */ void bfq_schedule_dispatch(struct bfq_data *bfqd) { if (bfqd->queued != 0) { bfq_log(bfqd, "schedule dispatch"); blk_mq_run_hw_queues(bfqd->queue, true); } } #define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) #define bfq_class_rt(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_RT) #define bfq_sample_valid(samples) ((samples) > 80) /* * Lifted from AS - choose which of rq1 and rq2 that is best served now. * We choose the request that is closer to the head right now. Distance * behind the head is penalized and only allowed to a certain extent. */ static struct request *bfq_choose_req(struct bfq_data *bfqd, struct request *rq1, struct request *rq2, sector_t last) { sector_t s1, s2, d1 = 0, d2 = 0; unsigned long back_max; #define BFQ_RQ1_WRAP 0x01 /* request 1 wraps */ #define BFQ_RQ2_WRAP 0x02 /* request 2 wraps */ unsigned int wrap = 0; /* bit mask: requests behind the disk head? */ if (!rq1 || rq1 == rq2) return rq2; if (!rq2) return rq1; if (rq_is_sync(rq1) && !rq_is_sync(rq2)) return rq1; else if (rq_is_sync(rq2) && !rq_is_sync(rq1)) return rq2; if ((rq1->cmd_flags & REQ_META) && !(rq2->cmd_flags & REQ_META)) return rq1; else if ((rq2->cmd_flags & REQ_META) && !(rq1->cmd_flags & REQ_META)) return rq2; s1 = blk_rq_pos(rq1); s2 = blk_rq_pos(rq2); /* * By definition, 1KiB is 2 sectors. */ back_max = bfqd->bfq_back_max * 2; /* * Strict one way elevator _except_ in the case where we allow * short backward seeks which are biased as twice the cost of a * similar forward seek. */ if (s1 >= last) d1 = s1 - last; else if (s1 + back_max >= last) d1 = (last - s1) * bfqd->bfq_back_penalty; else wrap |= BFQ_RQ1_WRAP; if (s2 >= last) d2 = s2 - last; else if (s2 + back_max >= last) d2 = (last - s2) * bfqd->bfq_back_penalty; else wrap |= BFQ_RQ2_WRAP; /* Found required data */ /* * By doing switch() on the bit mask "wrap" we avoid having to * check two variables for all permutations: --> faster! */ switch (wrap) { case 0: /* common case for CFQ: rq1 and rq2 not wrapped */ if (d1 < d2) return rq1; else if (d2 < d1) return rq2; if (s1 >= s2) return rq1; else return rq2; case BFQ_RQ2_WRAP: return rq1; case BFQ_RQ1_WRAP: return rq2; case BFQ_RQ1_WRAP|BFQ_RQ2_WRAP: /* both rqs wrapped */ default: /* * Since both rqs are wrapped, * start with the one that's further behind head * (--> only *one* back seek required), * since back seek takes more time than forward. */ if (s1 <= s2) return rq1; else return rq2; } } /* * Async I/O can easily starve sync I/O (both sync reads and sync * writes), by consuming all tags. Similarly, storms of sync writes, * such as those that sync(2) may trigger, can starve sync reads. * Limit depths of async I/O and sync writes so as to counter both * problems. */ static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data) { struct bfq_data *bfqd = data->q->elevator->elevator_data; if (op_is_sync(op) && !op_is_write(op)) return; data->shallow_depth = bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)]; bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", __func__, bfqd->wr_busy_queues, op_is_sync(op), data->shallow_depth); } static struct bfq_queue * bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, sector_t sector, struct rb_node **ret_parent, struct rb_node ***rb_link) { struct rb_node **p, *parent; struct bfq_queue *bfqq = NULL; parent = NULL; p = &root->rb_node; while (*p) { struct rb_node **n; parent = *p; bfqq = rb_entry(parent, struct bfq_queue, pos_node); /* * Sort strictly based on sector. Smallest to the left, * largest to the right. */ if (sector > blk_rq_pos(bfqq->next_rq)) n = &(*p)->rb_right; else if (sector < blk_rq_pos(bfqq->next_rq)) n = &(*p)->rb_left; else break; p = n; bfqq = NULL; } *ret_parent = parent; if (rb_link) *rb_link = p; bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", (unsigned long long)sector, bfqq ? bfqq->pid : 0); return bfqq; } static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) { return bfqq->service_from_backlogged > 0 && time_is_before_jiffies(bfqq->first_IO_time + bfq_merge_time_limit); } /* * The following function is not marked as __cold because it is * actually cold, but for the same performance goal described in the * comments on the likely() at the beginning of * bfq_setup_cooperator(). Unexpectedly, to reach an even lower * execution time for the case where this function is not invoked, we * had to add an unlikely() in each involved if(). */ void __cold bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct rb_node **p, *parent; struct bfq_queue *__bfqq; if (bfqq->pos_root) { rb_erase(&bfqq->pos_node, bfqq->pos_root); bfqq->pos_root = NULL; } /* * bfqq cannot be merged any longer (see comments in * bfq_setup_cooperator): no point in adding bfqq into the * position tree. */ if (bfq_too_late_for_merging(bfqq)) return; if (bfq_class_idle(bfqq)) return; if (!bfqq->next_rq) return; bfqq->pos_root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, blk_rq_pos(bfqq->next_rq), &parent, &p); if (!__bfqq) { rb_link_node(&bfqq->pos_node, parent, p); rb_insert_color(&bfqq->pos_node, bfqq->pos_root); } else bfqq->pos_root = NULL; } /* * The following function returns false either if every active queue * must receive the same share of the throughput (symmetric scenario), * or, as a special case, if bfqq must receive a share of the * throughput lower than or equal to the share that every other active * queue must receive. If bfqq does sync I/O, then these are the only * two cases where bfqq happens to be guaranteed its share of the * throughput even if I/O dispatching is not plugged when bfqq remains * temporarily empty (for more details, see the comments in the * function bfq_better_to_idle()). For this reason, the return value * of this function is used to check whether I/O-dispatch plugging can * be avoided. * * The above first case (symmetric scenario) occurs when: * 1) all active queues have the same weight, * 2) all active queues belong to the same I/O-priority class, * 3) all active groups at the same level in the groups tree have the same * weight, * 4) all active groups at the same level in the groups tree have the same * number of children. * * Unfortunately, keeping the necessary state for evaluating exactly * the last two symmetry sub-conditions above would be quite complex * and time consuming. Therefore this function evaluates, instead, * only the following stronger three sub-conditions, for which it is * much easier to maintain the needed state: * 1) all active queues have the same weight, * 2) all active queues belong to the same I/O-priority class, * 3) there are no active groups. * In particular, the last condition is always true if hierarchical * support or the cgroups interface are not enabled, thus no state * needs to be maintained in this case. */ static bool bfq_asymmetric_scenario(struct bfq_data *bfqd, struct bfq_queue *bfqq) { bool smallest_weight = bfqq && bfqq->weight_counter && bfqq->weight_counter == container_of( rb_first_cached(&bfqd->queue_weights_tree), struct bfq_weight_counter, weights_node); /* * For queue weights to differ, queue_weights_tree must contain * at least two nodes. */ bool varied_queue_weights = !smallest_weight && !RB_EMPTY_ROOT(&bfqd->queue_weights_tree.rb_root) && (bfqd->queue_weights_tree.rb_root.rb_node->rb_left || bfqd->queue_weights_tree.rb_root.rb_node->rb_right); bool multiple_classes_busy = (bfqd->busy_queues[0] && bfqd->busy_queues[1]) || (bfqd->busy_queues[0] && bfqd->busy_queues[2]) || (bfqd->busy_queues[1] && bfqd->busy_queues[2]); return varied_queue_weights || multiple_classes_busy #ifdef CONFIG_BFQ_GROUP_IOSCHED || bfqd->num_groups_with_pending_reqs > 0 #endif ; } /* * If the weight-counter tree passed as input contains no counter for * the weight of the input queue, then add that counter; otherwise just * increment the existing counter. * * Note that weight-counter trees contain few nodes in mostly symmetric * scenarios. For example, if all queues have the same weight, then the * weight-counter tree for the queues may contain at most one node. * This holds even if low_latency is on, because weight-raised queues * are not inserted in the tree. * In most scenarios, the rate at which nodes are created/destroyed * should be low too. */ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct rb_root_cached *root) { struct bfq_entity *entity = &bfqq->entity; struct rb_node **new = &(root->rb_root.rb_node), *parent = NULL; bool leftmost = true; /* * Do not insert if the queue is already associated with a * counter, which happens if: * 1) a request arrival has caused the queue to become both * non-weight-raised, and hence change its weight, and * backlogged; in this respect, each of the two events * causes an invocation of this function, * 2) this is the invocation of this function caused by the * second event. This second invocation is actually useless, * and we handle this fact by exiting immediately. More * efficient or clearer solutions might possibly be adopted. */ if (bfqq->weight_counter) return; while (*new) { struct bfq_weight_counter *__counter = container_of(*new, struct bfq_weight_counter, weights_node); parent = *new; if (entity->weight == __counter->weight) { bfqq->weight_counter = __counter; goto inc_counter; } if (entity->weight < __counter->weight) new = &((*new)->rb_left); else { new = &((*new)->rb_right); leftmost = false; } } bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), GFP_ATOMIC); /* * In the unlucky event of an allocation failure, we just * exit. This will cause the weight of queue to not be * considered in bfq_asymmetric_scenario, which, in its turn, * causes the scenario to be deemed wrongly symmetric in case * bfqq's weight would have been the only weight making the * scenario asymmetric. On the bright side, no unbalance will * however occur when bfqq becomes inactive again (the * invocation of this function is triggered by an activation * of queue). In fact, bfq_weights_tree_remove does nothing * if !bfqq->weight_counter. */ if (unlikely(!bfqq->weight_counter)) return; bfqq->weight_counter->weight = entity->weight; rb_link_node(&bfqq->weight_counter->weights_node, parent, new); rb_insert_color_cached(&bfqq->weight_counter->weights_node, root, leftmost); inc_counter: bfqq->weight_counter->num_active++; bfqq->ref++; } /* * Decrement the weight counter associated with the queue, and, if the * counter reaches 0, remove the counter from the tree. * See the comments to the function bfq_weights_tree_add() for considerations * about overhead. */ void __bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct rb_root_cached *root) { if (!bfqq->weight_counter) return; bfqq->weight_counter->num_active--; if (bfqq->weight_counter->num_active > 0) goto reset_entity_pointer; rb_erase_cached(&bfqq->weight_counter->weights_node, root); kfree(bfqq->weight_counter); reset_entity_pointer: bfqq->weight_counter = NULL; bfq_put_queue(bfqq); } /* * Invoke __bfq_weights_tree_remove on bfqq and decrement the number * of active groups for each queue's inactive parent entity. */ void bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct bfq_entity *entity = bfqq->entity.parent; for_each_entity(entity) { struct bfq_sched_data *sd = entity->my_sched_data; if (sd->next_in_service || sd->in_service_entity) { /* * entity is still active, because either * next_in_service or in_service_entity is not * NULL (see the comments on the definition of * next_in_service for details on why * in_service_entity must be checked too). * * As a consequence, its parent entities are * active as well, and thus this loop must * stop here. */ break; } /* * The decrement of num_groups_with_pending_reqs is * not performed immediately upon the deactivation of * entity, but it is delayed to when it also happens * that the first leaf descendant bfqq of entity gets * all its pending requests completed. The following * instructions perform this delayed decrement, if * needed. See the comments on * num_groups_with_pending_reqs for details. */ if (entity->in_groups_with_pending_reqs) { entity->in_groups_with_pending_reqs = false; bfqd->num_groups_with_pending_reqs--; } } /* * Next function is invoked last, because it causes bfqq to be * freed if the following holds: bfqq is not in service and * has no dispatched request. DO NOT use bfqq after the next * function invocation. */ __bfq_weights_tree_remove(bfqd, bfqq, &bfqd->queue_weights_tree); } /* * Return expired entry, or NULL to just start from scratch in rbtree. */ static struct request *bfq_check_fifo(struct bfq_queue *bfqq, struct request *last) { struct request *rq; if (bfq_bfqq_fifo_expire(bfqq)) return NULL; bfq_mark_bfqq_fifo_expire(bfqq); rq = rq_entry_fifo(bfqq->fifo.next); if (rq == last || ktime_get_ns() < rq->fifo_time) return NULL; bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); return rq; } static struct request *bfq_find_next_rq(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct request *last) { struct rb_node *rbnext = rb_next(&last->rb_node); struct rb_node *rbprev = rb_prev(&last->rb_node); struct request *next, *prev = NULL; /* Follow expired path, else get first next available. */ next = bfq_check_fifo(bfqq, last); if (next) return next; if (rbprev) prev = rb_entry_rq(rbprev); if (rbnext) next = rb_entry_rq(rbnext); else { rbnext = rb_first(&bfqq->sort_list); if (rbnext && rbnext != &last->rb_node) next = rb_entry_rq(rbnext); } return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last)); } /* see the definition of bfq_async_charge_factor for details */ static unsigned long bfq_serv_to_charge(struct request *rq, struct bfq_queue *bfqq) { if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 || bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) return blk_rq_sectors(rq); return blk_rq_sectors(rq) * bfq_async_charge_factor; } /** * bfq_updated_next_req - update the queue after a new next_rq selection. * @bfqd: the device data the queue belongs to. * @bfqq: the queue to update. * * If the first request of a queue changes we make sure that the queue * has enough budget to serve at least its first request (if the * request has grown). We do this because if the queue has not enough * budget for its first request, it has to go through two dispatch * rounds to actually get it dispatched. */ static void bfq_updated_next_req(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; struct request *next_rq = bfqq->next_rq; unsigned long new_budget; if (!next_rq) return; if (bfqq == bfqd->in_service_queue) /* * In order not to break guarantees, budgets cannot be * changed after an entity has been selected. */ return; new_budget = max_t(unsigned long, max_t(unsigned long, bfqq->max_budget, bfq_serv_to_charge(next_rq, bfqq)), entity->service); if (entity->budget != new_budget) { entity->budget = new_budget; bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", new_budget); bfq_requeue_bfqq(bfqd, bfqq, false); } } static unsigned int bfq_wr_duration(struct bfq_data *bfqd) { u64 dur; if (bfqd->bfq_wr_max_time > 0) return bfqd->bfq_wr_max_time; dur = bfqd->rate_dur_prod; do_div(dur, bfqd->peak_rate); /* * Limit duration between 3 and 25 seconds. The upper limit * has been conservatively set after the following worst case: * on a QEMU/KVM virtual machine * - running in a slow PC * - with a virtual disk stacked on a slow low-end 5400rpm HDD * - serving a heavy I/O workload, such as the sequential reading * of several files * mplayer took 23 seconds to start, if constantly weight-raised. * * As for higher values than that accommodating the above bad * scenario, tests show that higher values would often yield * the opposite of the desired result, i.e., would worsen * responsiveness by allowing non-interactive applications to * preserve weight raising for too long. * * On the other end, lower values than 3 seconds make it * difficult for most interactive tasks to complete their jobs * before weight-raising finishes. */ return clamp_val(dur, msecs_to_jiffies(3000), msecs_to_jiffies(25000)); } /* switch back from soft real-time to interactive weight raising */ static void switch_back_to_interactive_wr(struct bfq_queue *bfqq, struct bfq_data *bfqd) { bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); bfqq->last_wr_start_finish = bfqq->wr_start_at_switch_to_srt; } static void bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, struct bfq_io_cq *bic, bool bfq_already_existing) { unsigned int old_wr_coeff = bfqq->wr_coeff; bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq); if (bic->saved_has_short_ttime) bfq_mark_bfqq_has_short_ttime(bfqq); else bfq_clear_bfqq_has_short_ttime(bfqq); if (bic->saved_IO_bound) bfq_mark_bfqq_IO_bound(bfqq); else bfq_clear_bfqq_IO_bound(bfqq); bfqq->entity.new_weight = bic->saved_weight; bfqq->ttime = bic->saved_ttime; bfqq->wr_coeff = bic->saved_wr_coeff; bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time; if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || time_is_before_jiffies(bfqq->last_wr_start_finish + bfqq->wr_cur_max_time))) { if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && !bfq_bfqq_in_large_burst(bfqq) && time_is_after_eq_jiffies(bfqq->wr_start_at_switch_to_srt + bfq_wr_duration(bfqd))) { switch_back_to_interactive_wr(bfqq, bfqd); } else { bfqq->wr_coeff = 1; bfq_log_bfqq(bfqq->bfqd, bfqq, "resume state: switching off wr"); } } /* make sure weight will be updated, however we got here */ bfqq->entity.prio_changed = 1; if (likely(!busy)) return; if (old_wr_coeff == 1 && bfqq->wr_coeff > 1) bfqd->wr_busy_queues++; else if (old_wr_coeff > 1 && bfqq->wr_coeff == 1) bfqd->wr_busy_queues--; } static int bfqq_process_refs(struct bfq_queue *bfqq) { return bfqq->ref - bfqq->allocated - bfqq->entity.on_st - (bfqq->weight_counter != NULL); } /* Empty burst list and add just bfqq (see comments on bfq_handle_burst) */ static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct bfq_queue *item; struct hlist_node *n; hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) hlist_del_init(&item->burst_list_node); /* * Start the creation of a new burst list only if there is no * active queue. See comments on the conditional invocation of * bfq_handle_burst(). */ if (bfq_tot_busy_queues(bfqd) == 0) { hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); bfqd->burst_size = 1; } else bfqd->burst_size = 0; bfqd->burst_parent_entity = bfqq->entity.parent; } /* Add bfqq to the list of queues in current burst (see bfq_handle_burst) */ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) { /* Increment burst size to take into account also bfqq */ bfqd->burst_size++; if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) { struct bfq_queue *pos, *bfqq_item; struct hlist_node *n; /* * Enough queues have been activated shortly after each * other to consider this burst as large. */ bfqd->large_burst = true; /* * We can now mark all queues in the burst list as * belonging to a large burst. */ hlist_for_each_entry(bfqq_item, &bfqd->burst_list, burst_list_node) bfq_mark_bfqq_in_large_burst(bfqq_item); bfq_mark_bfqq_in_large_burst(bfqq); /* * From now on, and until the current burst finishes, any * new queue being activated shortly after the last queue * was inserted in the burst can be immediately marked as * belonging to a large burst. So the burst list is not * needed any more. Remove it. */ hlist_for_each_entry_safe(pos, n, &bfqd->burst_list, burst_list_node) hlist_del_init(&pos->burst_list_node); } else /* * Burst not yet large: add bfqq to the burst list. Do * not increment the ref counter for bfqq, because bfqq * is removed from the burst list before freeing bfqq * in put_queue. */ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); } /* * If many queues belonging to the same group happen to be created * shortly after each other, then the processes associated with these * queues have typically a common goal. In particular, bursts of queue * creations are usually caused by services or applications that spawn * many parallel threads/processes. Examples are systemd during boot, * or git grep. To help these processes get their job done as soon as * possible, it is usually better to not grant either weight-raising * or device idling to their queues, unless these queues must be * protected from the I/O flowing through other active queues. * * In this comment we describe, firstly, the reasons why this fact * holds, and, secondly, the next function, which implements the main * steps needed to properly mark these queues so that they can then be * treated in a different way. * * The above services or applications benefit mostly from a high * throughput: the quicker the requests of the activated queues are * cumulatively served, the sooner the target job of these queues gets * completed. As a consequence, weight-raising any of these queues, * which also implies idling the device for it, is almost always * counterproductive, unless there are other active queues to isolate * these new queues from. If there no other active queues, then * weight-raising these new queues just lowers throughput in most * cases. * * On the other hand, a burst of queue creations may be caused also by * the start of an application that does not consist of a lot of * parallel I/O-bound threads. In fact, with a complex application, * several short processes may need to be executed to start-up the * application. In this respect, to start an application as quickly as * possible, the best thing to do is in any case to privilege the I/O * related to the application with respect to all other * I/O. Therefore, the best strategy to start as quickly as possible * an application that causes a burst of queue creations is to * weight-raise all the queues created during the burst. This is the * exact opposite of the best strategy for the other type of bursts. * * In the end, to take the best action for each of the two cases, the * two types of bursts need to be distinguished. Fortunately, this * seems relatively easy, by looking at the sizes of the bursts. In * particular, we found a threshold such that only bursts with a * larger size than that threshold are apparently caused by * services or commands such as systemd or git grep. For brevity, * hereafter we call just 'large' these bursts. BFQ *does not* * weight-raise queues whose creation occurs in a large burst. In * addition, for each of these queues BFQ performs or does not perform * idling depending on which choice boosts the throughput more. The * exact choice depends on the device and request pattern at * hand. * * Unfortunately, false positives may occur while an interactive task * is starting (e.g., an application is being started). The * consequence is that the queues associated with the task do not * enjoy weight raising as expected. Fortunately these false positives * are very rare. They typically occur if some service happens to * start doing I/O exactly when the interactive task starts. * * Turning back to the next function, it is invoked only if there are * no active queues (apart from active queues that would belong to the * same, possible burst bfqq would belong to), and it implements all * the steps needed to detect the occurrence of a large burst and to * properly mark all the queues belonging to it (so that they can then * be treated in a different way). This goal is achieved by * maintaining a "burst list" that holds, temporarily, the queues that * belong to the burst in progress. The list is then used to mark * these queues as belonging to a large burst if the burst does become * large. The main steps are the following. * * . when the very first queue is created, the queue is inserted into the * list (as it could be the first queue in a possible burst) * * . if the current burst has not yet become large, and a queue Q that does * not yet belong to the burst is activated shortly after the last time * at which a new queue entered the burst list, then the function appends * Q to the burst list * * . if, as a consequence of the previous step, the burst size reaches * the large-burst threshold, then * * . all the queues in the burst list are marked as belonging to a * large burst * * . the burst list is deleted; in fact, the burst list already served * its purpose (keeping temporarily track of the queues in a burst, * so as to be able to mark them as belonging to a large burst in the * previous sub-step), and now is not needed any more * * . the device enters a large-burst mode * * . if a queue Q that does not belong to the burst is created while * the device is in large-burst mode and shortly after the last time * at which a queue either entered the burst list or was marked as * belonging to the current large burst, then Q is immediately marked * as belonging to a large burst. * * . if a queue Q that does not belong to the burst is created a while * later, i.e., not shortly after, than the last time at which a queue * either entered the burst list or was marked as belonging to the * current large burst, then the current burst is deemed as finished and: * * . the large-burst mode is reset if set * * . the burst list is emptied * * . Q is inserted in the burst list, as Q may be the first queue * in a possible new burst (then the burst list contains just Q * after this step). */ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) { /* * If bfqq is already in the burst list or is part of a large * burst, or finally has just been split, then there is * nothing else to do. */ if (!hlist_unhashed(&bfqq->burst_list_node) || bfq_bfqq_in_large_burst(bfqq) || time_is_after_eq_jiffies(bfqq->split_time + msecs_to_jiffies(10))) return; /* * If bfqq's creation happens late enough, or bfqq belongs to * a different group than the burst group, then the current * burst is finished, and related data structures must be * reset. * * In this respect, consider the special case where bfqq is * the very first queue created after BFQ is selected for this * device. In this case, last_ins_in_burst and * burst_parent_entity are not yet significant when we get * here. But it is easy to verify that, whether or not the * following condition is true, bfqq will end up being * inserted into the burst list. In particular the list will * happen to contain only bfqq. And this is exactly what has * to happen, as bfqq may be the first queue of the first * burst. */ if (time_is_before_jiffies(bfqd->last_ins_in_burst + bfqd->bfq_burst_interval) || bfqq->entity.parent != bfqd->burst_parent_entity) { bfqd->large_burst = false; bfq_reset_burst_list(bfqd, bfqq); goto end; } /* * If we get here, then bfqq is being activated shortly after the * last queue. So, if the current burst is also large, we can mark * bfqq as belonging to this large burst immediately. */ if (bfqd->large_burst) { bfq_mark_bfqq_in_large_burst(bfqq); goto end; } /* * If we get here, then a large-burst state has not yet been * reached, but bfqq is being activated shortly after the last * queue. Then we add bfqq to the burst. */ bfq_add_to_burst(bfqd, bfqq); end: /* * At this point, bfqq either has been added to the current * burst or has caused the current burst to terminate and a * possible new burst to start. In particular, in the second * case, bfqq has become the first queue in the possible new * burst. In both cases last_ins_in_burst needs to be moved * forward. */ bfqd->last_ins_in_burst = jiffies; } static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) { struct bfq_entity *entity = &bfqq->entity; return entity->budget - entity->service; } /* * If enough samples have been computed, return the current max budget * stored in bfqd, which is dynamically updated according to the * estimated disk peak rate; otherwise return the default max budget */ static int bfq_max_budget(struct bfq_data *bfqd) { if (bfqd->budgets_assigned < bfq_stats_min_budgets) return bfq_default_max_budget; else return bfqd->bfq_max_budget; } /* * Return min budget, which is a fraction of the current or default * max budget (trying with 1/32) */ static int bfq_min_budget(struct bfq_data *bfqd) { if (bfqd->budgets_assigned < bfq_stats_min_budgets) return bfq_default_max_budget / 32; else return bfqd->bfq_max_budget / 32; } /* * The next function, invoked after the input queue bfqq switches from * idle to busy, updates the budget of bfqq. The function also tells * whether the in-service queue should be expired, by returning * true. The purpose of expiring the in-service queue is to give bfqq * the chance to possibly preempt the in-service queue, and the reason * for preempting the in-service queue is to achieve one of the two * goals below. * * 1. Guarantee to bfqq its reserved bandwidth even if bfqq has * expired because it has remained idle. In particular, bfqq may have * expired for one of the following two reasons: * * - BFQQE_NO_MORE_REQUESTS bfqq did not enjoy any device idling * and did not make it to issue a new request before its last * request was served; * * - BFQQE_TOO_IDLE bfqq did enjoy device idling, but did not issue * a new request before the expiration of the idling-time. * * Even if bfqq has expired for one of the above reasons, the process * associated with the queue may be however issuing requests greedily, * and thus be sensitive to the bandwidth it receives (bfqq may have * remained idle for other reasons: CPU high load, bfqq not enjoying * idling, I/O throttling somewhere in the path from the process to * the I/O scheduler, ...). But if, after every expiration for one of * the above two reasons, bfqq has to wait for the service of at least * one full budget of another queue before being served again, then * bfqq is likely to get a much lower bandwidth or resource time than * its reserved ones. To address this issue, two countermeasures need * to be taken. * * First, the budget and the timestamps of bfqq need to be updated in * a special way on bfqq reactivation: they need to be updated as if * bfqq did not remain idle and did not expire. In fact, if they are * computed as if bfqq expired and remained idle until reactivation, * then the process associated with bfqq is treated as if, instead of * being greedy, it stopped issuing requests when bfqq remained idle, * and restarts issuing requests only on this reactivation. In other * words, the scheduler does not help the process recover the "service * hole" between bfqq expiration and reactivation. As a consequence, * the process receives a lower bandwidth than its reserved one. In * contrast, to recover this hole, the budget must be updated as if * bfqq was not expired at all before this reactivation, i.e., it must * be set to the value of the remaining budget when bfqq was * expired. Along the same line, timestamps need to be assigned the * value they had the last time bfqq was selected for service, i.e., * before last expiration. Thus timestamps need to be back-shifted * with respect to their normal computation (see [1] for more details * on this tricky aspect). * * Secondly, to allow the process to recover the hole, the in-service * queue must be expired too, to give bfqq the chance to preempt it * immediately. In fact, if bfqq has to wait for a full budget of the * in-service queue to be completed, then it may become impossible to * let the process recover the hole, even if the back-shifted * timestamps of bfqq are lower than those of the in-service queue. If * this happens for most or all of the holes, then the process may not * receive its reserved bandwidth. In this respect, it is worth noting * that, being the service of outstanding requests unpreemptible, a * little fraction of the holes may however be unrecoverable, thereby * causing a little loss of bandwidth. * * The last important point is detecting whether bfqq does need this * bandwidth recovery. In this respect, the next function deems the * process associated with bfqq greedy, and thus allows it to recover * the hole, if: 1) the process is waiting for the arrival of a new * request (which implies that bfqq expired for one of the above two * reasons), and 2) such a request has arrived soon. The first * condition is controlled through the flag non_blocking_wait_rq, * while the second through the flag arrived_in_time. If both * conditions hold, then the function computes the budget in the * above-described special way, and signals that the in-service queue * should be expired. Timestamp back-shifting is done later in * __bfq_activate_entity. * * 2. Reduce latency. Even if timestamps are not backshifted to let * the process associated with bfqq recover a service hole, bfqq may * however happen to have, after being (re)activated, a lower finish * timestamp than the in-service queue. That is, the next budget of * bfqq may have to be completed before the one of the in-service * queue. If this is the case, then preempting the in-service queue * allows this goal to be achieved, apart from the unpreemptible, * outstanding requests mentioned above. * * Unfortunately, regardless of which of the above two goals one wants * to achieve, service trees need first to be updated to know whether * the in-service queue must be preempted. To have service trees * correctly updated, the in-service queue must be expired and * rescheduled, and bfqq must be scheduled too. This is one of the * most costly operations (in future versions, the scheduling * mechanism may be re-designed in such a way to make it possible to * know whether preemption is needed without needing to update service * trees). In addition, queue preemptions almost always cause random * I/O, which may in turn cause loss of throughput. Finally, there may * even be no in-service queue when the next function is invoked (so, * no queue to compare timestamps with). Because of these facts, the * next function adopts the following simple scheme to avoid costly * operations, too frequent preemptions and too many dependencies on * the state of the scheduler: it requests the expiration of the * in-service queue (unconditionally) only for queues that need to * recover a hole. Then it delegates to other parts of the code the * responsibility of handling the above case 2. */ static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd, struct bfq_queue *bfqq, bool arrived_in_time) { struct bfq_entity *entity = &bfqq->entity; /* * In the next compound condition, we check also whether there * is some budget left, because otherwise there is no point in * trying to go on serving bfqq with this same budget: bfqq * would be expired immediately after being selected for * service. This would only cause useless overhead. */ if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time && bfq_bfqq_budget_left(bfqq) > 0) { /* * We do not clear the flag non_blocking_wait_rq here, as * the latter is used in bfq_activate_bfqq to signal * that timestamps need to be back-shifted (and is * cleared right after). */ /* * In next assignment we rely on that either * entity->service or entity->budget are not updated * on expiration if bfqq is empty (see * __bfq_bfqq_recalc_budget). Thus both quantities * remain unchanged after such an expiration, and the * following statement therefore assigns to * entity->budget the remaining budget on such an * expiration. */ entity->budget = min_t(unsigned long, bfq_bfqq_budget_left(bfqq), bfqq->max_budget); /* * At this point, we have used entity->service to get * the budget left (needed for updating * entity->budget). Thus we finally can, and have to, * reset entity->service. The latter must be reset * because bfqq would otherwise be charged again for * the service it has received during its previous * service slot(s). */ entity->service = 0; return true; } /* * We can finally complete expiration, by setting service to 0. */ entity->service = 0; entity->budget = max_t(unsigned long, bfqq->max_budget, bfq_serv_to_charge(bfqq->next_rq, bfqq)); bfq_clear_bfqq_non_blocking_wait_rq(bfqq); return false; } /* * Return the farthest past time instant according to jiffies * macros. */ static unsigned long bfq_smallest_from_now(void) { return jiffies - MAX_JIFFY_OFFSET; } static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, struct bfq_queue *bfqq, unsigned int old_wr_coeff, bool wr_or_deserves_wr, bool interactive, bool in_burst, bool soft_rt) { if (old_wr_coeff == 1 && wr_or_deserves_wr) { /* start a weight-raising period */ if (interactive) { bfqq->service_from_wr = 0; bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); } else { /* * No interactive weight raising in progress * here: assign minus infinity to * wr_start_at_switch_to_srt, to make sure * that, at the end of the soft-real-time * weight raising periods that is starting * now, no interactive weight-raising period * may be wrongly considered as still in * progress (and thus actually started by * mistake). */ bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now(); bfqq->wr_coeff = bfqd->bfq_wr_coeff * BFQ_SOFTRT_WEIGHT_FACTOR; bfqq->wr_cur_max_time = bfqd->bfq_wr_rt_max_time; } /* * If needed, further reduce budget to make sure it is * close to bfqq's backlog, so as to reduce the * scheduling-error component due to a too large * budget. Do not care about throughput consequences, * but only about latency. Finally, do not assign a * too small budget either, to avoid increasing * latency by causing too frequent expirations. */ bfqq->entity.budget = min_t(unsigned long, bfqq->entity.budget, 2 * bfq_min_budget(bfqd)); } else if (old_wr_coeff > 1) { if (interactive) { /* update wr coeff and duration */ bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); } else if (in_burst) bfqq->wr_coeff = 1; else if (soft_rt) { /* * The application is now or still meeting the * requirements for being deemed soft rt. We * can then correctly and safely (re)charge * the weight-raising duration for the * application with the weight-raising * duration for soft rt applications. * * In particular, doing this recharge now, i.e., * before the weight-raising period for the * application finishes, reduces the probability * of the following negative scenario: * 1) the weight of a soft rt application is * raised at startup (as for any newly * created application), * 2) since the application is not interactive, * at a certain time weight-raising is * stopped for the application, * 3) at that time the application happens to * still have pending requests, and hence * is destined to not have a chance to be * deemed soft rt before these requests are * completed (see the comments to the * function bfq_bfqq_softrt_next_start() * for details on soft rt detection), * 4) these pending requests experience a high * latency because the application is not * weight-raised while they are pending. */ if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time) { bfqq->wr_start_at_switch_to_srt = bfqq->last_wr_start_finish; bfqq->wr_cur_max_time = bfqd->bfq_wr_rt_max_time; bfqq->wr_coeff = bfqd->bfq_wr_coeff * BFQ_SOFTRT_WEIGHT_FACTOR; } bfqq->last_wr_start_finish = jiffies; } } } static bool bfq_bfqq_idle_for_long_time(struct bfq_data *bfqd, struct bfq_queue *bfqq) { return bfqq->dispatched == 0 && time_is_before_jiffies( bfqq->budget_timeout + bfqd->bfq_wr_min_idle_time); } /* * Return true if bfqq is in a higher priority class, or has a higher * weight than the in-service queue. */ static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq, struct bfq_queue *in_serv_bfqq) { int bfqq_weight, in_serv_weight; if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class) return true; if (in_serv_bfqq->entity.parent == bfqq->entity.parent) { bfqq_weight = bfqq->entity.weight; in_serv_weight = in_serv_bfqq->entity.weight; } else { if (bfqq->entity.parent) bfqq_weight = bfqq->entity.parent->weight; else bfqq_weight = bfqq->entity.weight; if (in_serv_bfqq->entity.parent) in_serv_weight = in_serv_bfqq->entity.parent->weight; else in_serv_weight = in_serv_bfqq->entity.weight; } return bfqq_weight > in_serv_weight; } static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, struct bfq_queue *bfqq, int old_wr_coeff, struct request *rq, bool *interactive) { bool soft_rt, in_burst, wr_or_deserves_wr, bfqq_wants_to_preempt, idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), /* * See the comments on * bfq_bfqq_update_budg_for_activation for * details on the usage of the next variable. */ arrived_in_time = ktime_get_ns() <= bfqq->ttime.last_end_request + bfqd->bfq_slice_idle * 3; /* * bfqq deserves to be weight-raised if: * - it is sync, * - it does not belong to a large burst, * - it has been idle for enough time or is soft real-time, * - is linked to a bfq_io_cq (it is not shared in any sense). */ in_burst = bfq_bfqq_in_large_burst(bfqq); soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && !BFQQ_TOTALLY_SEEKY(bfqq) && !in_burst && time_is_before_jiffies(bfqq->soft_rt_next_start) && bfqq->dispatched == 0; *interactive = !in_burst && idle_for_long_time; wr_or_deserves_wr = bfqd->low_latency && (bfqq->wr_coeff > 1 || (bfq_bfqq_sync(bfqq) && bfqq->bic && (*interactive || soft_rt))); /* * Using the last flag, update budget and check whether bfqq * may want to preempt the in-service queue. */ bfqq_wants_to_preempt = bfq_bfqq_update_budg_for_activation(bfqd, bfqq, arrived_in_time); /* * If bfqq happened to be activated in a burst, but has been * idle for much more than an interactive queue, then we * assume that, in the overall I/O initiated in the burst, the * I/O associated with bfqq is finished. So bfqq does not need * to be treated as a queue belonging to a burst * anymore. Accordingly, we reset bfqq's in_large_burst flag * if set, and remove bfqq from the burst list if it's * there. We do not decrement burst_size, because the fact * that bfqq does not need to belong to the burst list any * more does not invalidate the fact that bfqq was created in * a burst. */ if (likely(!bfq_bfqq_just_created(bfqq)) && idle_for_long_time && time_is_before_jiffies( bfqq->budget_timeout + msecs_to_jiffies(10000))) { hlist_del_init(&bfqq->burst_list_node); bfq_clear_bfqq_in_large_burst(bfqq); } bfq_clear_bfqq_just_created(bfqq); if (!bfq_bfqq_IO_bound(bfqq)) { if (arrived_in_time) { bfqq->requests_within_timer++; if (bfqq->requests_within_timer >= bfqd->bfq_requests_within_timer) bfq_mark_bfqq_IO_bound(bfqq); } else bfqq->requests_within_timer = 0; } if (bfqd->low_latency) { if (unlikely(time_is_after_jiffies(bfqq->split_time))) /* wraparound */ bfqq->split_time = jiffies - bfqd->bfq_wr_min_idle_time - 1; if (time_is_before_jiffies(bfqq->split_time + bfqd->bfq_wr_min_idle_time)) { bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, old_wr_coeff, wr_or_deserves_wr, *interactive, in_burst, soft_rt); if (old_wr_coeff != bfqq->wr_coeff) bfqq->entity.prio_changed = 1; } } bfqq->last_idle_bklogged = jiffies; bfqq->service_from_backlogged = 0; bfq_clear_bfqq_softrt_update(bfqq); bfq_add_bfqq_busy(bfqd, bfqq); /* * Expire in-service queue only if preemption may be needed * for guarantees. In particular, we care only about two * cases. The first is that bfqq has to recover a service * hole, as explained in the comments on * bfq_bfqq_update_budg_for_activation(), i.e., that * bfqq_wants_to_preempt is true. However, if bfqq does not * carry time-critical I/O, then bfqq's bandwidth is less * important than that of queues that carry time-critical I/O. * So, as a further constraint, we consider this case only if * bfqq is at least as weight-raised, i.e., at least as time * critical, as the in-service queue. * * The second case is that bfqq is in a higher priority class, * or has a higher weight than the in-service queue. If this * condition does not hold, we don't care because, even if * bfqq does not start to be served immediately, the resulting * delay for bfqq's I/O is however lower or much lower than * the ideal completion time to be guaranteed to bfqq's I/O. * * In both cases, preemption is needed only if, according to * the timestamps of both bfqq and of the in-service queue, * bfqq actually is the next queue to serve. So, to reduce * useless preemptions, the return value of * next_queue_may_preempt() is considered in the next compound * condition too. Yet next_queue_may_preempt() just checks a * simple, necessary condition for bfqq to be the next queue * to serve. In fact, to evaluate a sufficient condition, the * timestamps of the in-service queue would need to be * updated, and this operation is quite costly (see the * comments on bfq_bfqq_update_budg_for_activation()). */ if (bfqd->in_service_queue && ((bfqq_wants_to_preempt && bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) || bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) && next_queue_may_preempt(bfqd)) bfq_bfqq_expire(bfqd, bfqd->in_service_queue, false, BFQQE_PREEMPTED); } static void bfq_reset_inject_limit(struct bfq_data *bfqd, struct bfq_queue *bfqq) { /* invalidate baseline total service time */ bfqq->last_serv_time_ns = 0; /* * Reset pointer in case we are waiting for * some request completion. */ bfqd->waited_rq = NULL; /* * If bfqq has a short think time, then start by setting the * inject limit to 0 prudentially, because the service time of * an injected I/O request may be higher than the think time * of bfqq, and therefore, if one request was injected when * bfqq remains empty, this injected request might delay the * service of the next I/O request for bfqq significantly. In * case bfqq can actually tolerate some injection, then the * adaptive update will however raise the limit soon. This * lucky circumstance holds exactly because bfqq has a short * think time, and thus, after remaining empty, is likely to * get new I/O enqueued---and then completed---before being * expired. This is the very pattern that gives the * limit-update algorithm the chance to measure the effect of * injection on request service times, and then to update the * limit accordingly. * * However, in the following special case, the inject limit is * left to 1 even if the think time is short: bfqq's I/O is * synchronized with that of some other queue, i.e., bfqq may * receive new I/O only after the I/O of the other queue is * completed. Keeping the inject limit to 1 allows the * blocking I/O to be served while bfqq is in service. And * this is very convenient both for bfqq and for overall * throughput, as explained in detail in the comments in * bfq_update_has_short_ttime(). * * On the opposite end, if bfqq has a long think time, then * start directly by 1, because: * a) on the bright side, keeping at most one request in * service in the drive is unlikely to cause any harm to the * latency of bfqq's requests, as the service time of a single * request is likely to be lower than the think time of bfqq; * b) on the downside, after becoming empty, bfqq is likely to * expire before getting its next request. With this request * arrival pattern, it is very hard to sample total service * times and update the inject limit accordingly (see comments * on bfq_update_inject_limit()). So the limit is likely to be * never, or at least seldom, updated. As a consequence, by * setting the limit to 1, we avoid that no injection ever * occurs with bfqq. On the downside, this proactive step * further reduces chances to actually compute the baseline * total service time. Thus it reduces chances to execute the * limit-update algorithm and possibly raise the limit to more * than 1. */ if (bfq_bfqq_has_short_ttime(bfqq)) bfqq->inject_limit = 0; else bfqq->inject_limit = 1; bfqq->decrease_time_jif = jiffies; } static void bfq_add_request(struct request *rq) { struct bfq_queue *bfqq = RQ_BFQQ(rq); struct bfq_data *bfqd = bfqq->bfqd; struct request *next_rq, *prev; unsigned int old_wr_coeff = bfqq->wr_coeff; bool interactive = false; bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); bfqq->queued[rq_is_sync(rq)]++; bfqd->queued++; if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) { /* * Detect whether bfqq's I/O seems synchronized with * that of some other queue, i.e., whether bfqq, after * remaining empty, happens to receive new I/O only * right after some I/O request of the other queue has * been completed. We call waker queue the other * queue, and we assume, for simplicity, that bfqq may * have at most one waker queue. * * A remarkable throughput boost can be reached by * unconditionally injecting the I/O of the waker * queue, every time a new bfq_dispatch_request * happens to be invoked while I/O is being plugged * for bfqq. In addition to boosting throughput, this * unblocks bfqq's I/O, thereby improving bandwidth * and latency for bfqq. Note that these same results * may be achieved with the general injection * mechanism, but less effectively. For details on * this aspect, see the comments on the choice of the * queue for injection in bfq_select_queue(). * * Turning back to the detection of a waker queue, a * queue Q is deemed as a waker queue for bfqq if, for * two consecutive times, bfqq happens to become non * empty right after a request of Q has been * completed. In particular, on the first time, Q is * tentatively set as a candidate waker queue, while * on the second time, the flag * bfq_bfqq_has_waker(bfqq) is set to confirm that Q * is a waker queue for bfqq. These detection steps * are performed only if bfqq has a long think time, * so as to make it more likely that bfqq's I/O is * actually being blocked by a synchronization. This * last filter, plus the above two-times requirement, * make false positives less likely. * * NOTE * * The sooner a waker queue is detected, the sooner * throughput can be boosted by injecting I/O from the * waker queue. Fortunately, detection is likely to be * actually fast, for the following reasons. While * blocked by synchronization, bfqq has a long think * time. This implies that bfqq's inject limit is at * least equal to 1 (see the comments in * bfq_update_inject_limit()). So, thanks to * injection, the waker queue is likely to be served * during the very first I/O-plugging time interval * for bfqq. This triggers the first step of the * detection mechanism. Thanks again to injection, the * candidate waker queue is then likely to be * confirmed no later than during the next * I/O-plugging interval for bfqq. */ if (!bfq_bfqq_has_short_ttime(bfqq) && ktime_get_ns() - bfqd->last_completion < 200 * NSEC_PER_USEC) { if (bfqd->last_completed_rq_bfqq != bfqq && bfqd->last_completed_rq_bfqq != bfqq->waker_bfqq) { /* * First synchronization detected with * a candidate waker queue, or with a * different candidate waker queue * from the current one. */ bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq; /* * If the waker queue disappears, then * bfqq->waker_bfqq must be reset. To * this goal, we maintain in each * waker queue a list, woken_list, of * all the queues that reference the * waker queue through their * waker_bfqq pointer. When the waker * queue exits, the waker_bfqq pointer * of all the queues in the woken_list * is reset. * * In addition, if bfqq is already in * the woken_list of a waker queue, * then, before being inserted into * the woken_list of a new waker * queue, bfqq must be removed from * the woken_list of the old waker * queue. */ if (!hlist_unhashed(&bfqq->woken_list_node)) hlist_del_init(&bfqq->woken_list_node); hlist_add_head(&bfqq->woken_list_node, &bfqd->last_completed_rq_bfqq->woken_list); bfq_clear_bfqq_has_waker(bfqq); } else if (bfqd->last_completed_rq_bfqq == bfqq->waker_bfqq && !bfq_bfqq_has_waker(bfqq)) { /* * synchronization with waker_bfqq * seen for the second time */ bfq_mark_bfqq_has_waker(bfqq); } } /* * Periodically reset inject limit, to make sure that * the latter eventually drops in case workload * changes, see step (3) in the comments on * bfq_update_inject_limit(). */ if (time_is_before_eq_jiffies(bfqq->decrease_time_jif + msecs_to_jiffies(1000))) bfq_reset_inject_limit(bfqd, bfqq); /* * The following conditions must hold to setup a new * sampling of total service time, and then a new * update of the inject limit: * - bfqq is in service, because the total service * time is evaluated only for the I/O requests of * the queues in service; * - this is the right occasion to compute or to * lower the baseline total service time, because * there are actually no requests in the drive, * or * the baseline total service time is available, and * this is the right occasion to compute the other * quantity needed to update the inject limit, i.e., * the total service time caused by the amount of * injection allowed by the current value of the * limit. It is the right occasion because injection * has actually been performed during the service * hole, and there are still in-flight requests, * which are very likely to be exactly the injected * requests, or part of them; * - the minimum interval for sampling the total * service time and updating the inject limit has * elapsed. */ if (bfqq == bfqd->in_service_queue && (bfqd->rq_in_driver == 0 || (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected && bfqd->rq_in_driver > 0)) && time_is_before_eq_jiffies(bfqq->decrease_time_jif + msecs_to_jiffies(100))) { bfqd->last_empty_occupied_ns = ktime_get_ns(); /* * Start the state machine for measuring the * total service time of rq: setting * wait_dispatch will cause bfqd->waited_rq to * be set when rq will be dispatched. */ bfqd->wait_dispatch = true; bfqd->rqs_injected = false; } } elv_rb_add(&bfqq->sort_list, rq); /* * Check if this request is a better next-serve candidate. */ prev = bfqq->next_rq; next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); bfqq->next_rq = next_rq; /* * Adjust priority tree position, if next_rq changes. * See comments on bfq_pos_tree_add_move() for the unlikely(). */ if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) bfq_pos_tree_add_move(bfqd, bfqq); if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, rq, &interactive); else { if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) && time_is_before_jiffies( bfqq->last_wr_start_finish + bfqd->bfq_wr_min_inter_arr_async)) { bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); bfqd->wr_busy_queues++; bfqq->entity.prio_changed = 1; } if (prev != bfqq->next_rq) bfq_updated_next_req(bfqd, bfqq); } /* * Assign jiffies to last_wr_start_finish in the following * cases: * * . if bfqq is not going to be weight-raised, because, for * non weight-raised queues, last_wr_start_finish stores the * arrival time of the last request; as of now, this piece * of information is used only for deciding whether to * weight-raise async queues * * . if bfqq is not weight-raised, because, if bfqq is now * switching to weight-raised, then last_wr_start_finish * stores the time when weight-raising starts * * . if bfqq is interactive, because, regardless of whether * bfqq is currently weight-raised, the weight-raising * period must start or restart (this case is considered * separately because it is not detected by the above * conditions, if bfqq is already weight-raised) * * last_wr_start_finish has to be updated also if bfqq is soft * real-time, because the weight-raising period is constantly * restarted on idle-to-busy transitions for these queues, but * this is already done in bfq_bfqq_handle_idle_busy_switch if * needed. */ if (bfqd->low_latency && (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) bfqq->last_wr_start_finish = jiffies; } static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, struct bio *bio, struct request_queue *q) { struct bfq_queue *bfqq = bfqd->bio_bfqq; if (bfqq) return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); return NULL; } static sector_t get_sdist(sector_t last_pos, struct request *rq) { if (last_pos) return abs(blk_rq_pos(rq) - last_pos); return 0; } #if 0 /* Still not clear if we can do without next two functions */ static void bfq_activate_request(struct request_queue *q, struct request *rq) { struct bfq_data *bfqd = q->elevator->elevator_data; bfqd->rq_in_driver++; } static void bfq_deactivate_request(struct request_queue *q, struct request *rq) { struct bfq_data *bfqd = q->elevator->elevator_data; bfqd->rq_in_driver--; } #endif static void bfq_remove_request(struct request_queue *q, struct request *rq) { struct bfq_queue *bfqq = RQ_BFQQ(rq); struct bfq_data *bfqd = bfqq->bfqd; const int sync = rq_is_sync(rq); if (bfqq->next_rq == rq) { bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); bfq_updated_next_req(bfqd, bfqq); } if (rq->queuelist.prev != &rq->queuelist) list_del_init(&rq->queuelist); bfqq->queued[sync]--; bfqd->queued--; elv_rb_del(&bfqq->sort_list, rq); elv_rqhash_del(q, rq); if (q->last_merge == rq) q->last_merge = NULL; if (RB_EMPTY_ROOT(&bfqq->sort_list)) { bfqq->next_rq = NULL; if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { bfq_del_bfqq_busy(bfqd, bfqq, false); /* * bfqq emptied. In normal operation, when * bfqq is empty, bfqq->entity.service and * bfqq->entity.budget must contain, * respectively, the service received and the * budget used last time bfqq emptied. These * facts do not hold in this case, as at least * this last removal occurred while bfqq is * not in service. To avoid inconsistencies, * reset both bfqq->entity.service and * bfqq->entity.budget, if bfqq has still a * process that may issue I/O requests to it. */ bfqq->entity.budget = bfqq->entity.service = 0; } /* * Remove queue from request-position tree as it is empty. */ if (bfqq->pos_root) { rb_erase(&bfqq->pos_node, bfqq->pos_root); bfqq->pos_root = NULL; } } else { /* see comments on bfq_pos_tree_add_move() for the unlikely() */ if (unlikely(!bfqd->nonrot_with_queueing)) bfq_pos_tree_add_move(bfqd, bfqq); } if (rq->cmd_flags & REQ_META) bfqq->meta_pending--; } static bool bfq_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio, unsigned int nr_segs) { struct request_queue *q = hctx->queue; struct bfq_data *bfqd = q->elevator->elevator_data; struct request *free = NULL; /* * bfq_bic_lookup grabs the queue_lock: invoke it now and * store its return value for later use, to avoid nesting * queue_lock inside the bfqd->lock. We assume that the bic * returned by bfq_bic_lookup does not go away before * bfqd->lock is taken. */ struct bfq_io_cq *bic = bfq_bic_lookup(bfqd, current->io_context, q); bool ret; spin_lock_irq(&bfqd->lock); if (bic) bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf)); else bfqd->bio_bfqq = NULL; bfqd->bio_bic = bic; ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); if (free) blk_mq_free_request(free); spin_unlock_irq(&bfqd->lock); return ret; } static int bfq_request_merge(struct request_queue *q, struct request **req, struct bio *bio) { struct bfq_data *bfqd = q->elevator->elevator_data; struct request *__rq; __rq = bfq_find_rq_fmerge(bfqd, bio, q); if (__rq && elv_bio_merge_ok(__rq, bio)) { *req = __rq; return ELEVATOR_FRONT_MERGE; } return ELEVATOR_NO_MERGE; } static struct bfq_queue *bfq_init_rq(struct request *rq); static void bfq_request_merged(struct request_queue *q, struct request *req, enum elv_merge type) { if (type == ELEVATOR_FRONT_MERGE && rb_prev(&req->rb_node) && blk_rq_pos(req) < blk_rq_pos(container_of(rb_prev(&req->rb_node), struct request, rb_node))) { struct bfq_queue *bfqq = bfq_init_rq(req); struct bfq_data *bfqd = bfqq->bfqd; struct request *prev, *next_rq; /* Reposition request in its sort_list */ elv_rb_del(&bfqq->sort_list, req); elv_rb_add(&bfqq->sort_list, req); /* Choose next request to be served for bfqq */ prev = bfqq->next_rq; next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, bfqd->last_position); bfqq->next_rq = next_rq; /* * If next_rq changes, update both the queue's budget to * fit the new request and the queue's position in its * rq_pos_tree. */ if (prev != bfqq->next_rq) { bfq_updated_next_req(bfqd, bfqq); /* * See comments on bfq_pos_tree_add_move() for * the unlikely(). */ if (unlikely(!bfqd->nonrot_with_queueing)) bfq_pos_tree_add_move(bfqd, bfqq); } } } /* * This function is called to notify the scheduler that the requests * rq and 'next' have been merged, with 'next' going away. BFQ * exploits this hook to address the following issue: if 'next' has a * fifo_time lower that rq, then the fifo_time of rq must be set to * the value of 'next', to not forget the greater age of 'next'. * * NOTE: in this function we assume that rq is in a bfq_queue, basing * on that rq is picked from the hash table q->elevator->hash, which, * in its turn, is filled only with I/O requests present in * bfq_queues, while BFQ is in use for the request queue q. In fact, * the function that fills this hash table (elv_rqhash_add) is called * only by bfq_insert_request. */ static void bfq_requests_merged(struct request_queue *q, struct request *rq, struct request *next) { struct bfq_queue *bfqq = bfq_init_rq(rq), *next_bfqq = bfq_init_rq(next); /* * If next and rq belong to the same bfq_queue and next is older * than rq, then reposition rq in the fifo (by substituting next * with rq). Otherwise, if next and rq belong to different * bfq_queues, never reposition rq: in fact, we would have to * reposition it with respect to next's position in its own fifo, * which would most certainly be too expensive with respect to * the benefits. */ if (bfqq == next_bfqq && !list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && next->fifo_time < rq->fifo_time) { list_del_init(&rq->queuelist); list_replace_init(&next->queuelist, &rq->queuelist); rq->fifo_time = next->fifo_time; } if (bfqq->next_rq == next) bfqq->next_rq = rq; bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags); } /* Must be called with bfqq != NULL */ static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) { if (bfq_bfqq_busy(bfqq)) bfqq->bfqd->wr_busy_queues--; bfqq->wr_coeff = 1; bfqq->wr_cur_max_time = 0; bfqq->last_wr_start_finish = jiffies; /* * Trigger a weight change on the next invocation of * __bfq_entity_update_weight_prio. */ bfqq->entity.prio_changed = 1; } void bfq_end_wr_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) { int i, j; for (i = 0; i < 2; i++) for (j = 0; j < IOPRIO_BE_NR; j++) if (bfqg->async_bfqq[i][j]) bfq_bfqq_end_wr(bfqg->async_bfqq[i][j]); if (bfqg->async_idle_bfqq) bfq_bfqq_end_wr(bfqg->async_idle_bfqq); } static void bfq_end_wr(struct bfq_data *bfqd) { struct bfq_queue *bfqq; spin_lock_irq(&bfqd->lock); list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) bfq_bfqq_end_wr(bfqq); list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) bfq_bfqq_end_wr(bfqq); bfq_end_wr_async(bfqd); spin_unlock_irq(&bfqd->lock); } static sector_t bfq_io_struct_pos(void *io_struct, bool request) { if (request) return blk_rq_pos(io_struct); else return ((struct bio *)io_struct)->bi_iter.bi_sector; } static int bfq_rq_close_to_sector(void *io_struct, bool request, sector_t sector) { return abs(bfq_io_struct_pos(io_struct, request) - sector) <= BFQQ_CLOSE_THR; } static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd, struct bfq_queue *bfqq, sector_t sector) { struct rb_root *root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; struct rb_node *parent, *node; struct bfq_queue *__bfqq; if (RB_EMPTY_ROOT(root)) return NULL; /* * First, if we find a request starting at the end of the last * request, choose it. */ __bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL); if (__bfqq) return __bfqq; /* * If the exact sector wasn't found, the parent of the NULL leaf * will contain the closest sector (rq_pos_tree sorted by * next_request position). */ __bfqq = rb_entry(parent, struct bfq_queue, pos_node); if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) return __bfqq; if (blk_rq_pos(__bfqq->next_rq) < sector) node = rb_next(&__bfqq->pos_node); else node = rb_prev(&__bfqq->pos_node); if (!node) return NULL; __bfqq = rb_entry(node, struct bfq_queue, pos_node); if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) return __bfqq; return NULL; } static struct bfq_queue *bfq_find_close_cooperator(struct bfq_data *bfqd, struct bfq_queue *cur_bfqq, sector_t sector) { struct bfq_queue *bfqq; /* * We shall notice if some of the queues are cooperating, * e.g., working closely on the same area of the device. In * that case, we can group them together and: 1) don't waste * time idling, and 2) serve the union of their requests in * the best possible order for throughput. */ bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); if (!bfqq || bfqq == cur_bfqq) return NULL; return bfqq; } static struct bfq_queue * bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) { int process_refs, new_process_refs; struct bfq_queue *__bfqq; /* * If there are no process references on the new_bfqq, then it is * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain * may have dropped their last reference (not just their last process * reference). */ if (!bfqq_process_refs(new_bfqq)) return NULL; /* Avoid a circular list and skip interim queue merges. */ while ((__bfqq = new_bfqq->new_bfqq)) { if (__bfqq == bfqq) return NULL; new_bfqq = __bfqq; } process_refs = bfqq_process_refs(bfqq); new_process_refs = bfqq_process_refs(new_bfqq); /* * If the process for the bfqq has gone away, there is no * sense in merging the queues. */ if (process_refs == 0 || new_process_refs == 0) return NULL; bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", new_bfqq->pid); /* * Merging is just a redirection: the requests of the process * owning one of the two queues are redirected to the other queue. * The latter queue, in its turn, is set as shared if this is the * first time that the requests of some process are redirected to * it. * * We redirect bfqq to new_bfqq and not the opposite, because * we are in the context of the process owning bfqq, thus we * have the io_cq of this process. So we can immediately * configure this io_cq to redirect the requests of the * process to new_bfqq. In contrast, the io_cq of new_bfqq is * not available any more (new_bfqq->bic == NULL). * * Anyway, even in case new_bfqq coincides with the in-service * queue, redirecting requests the in-service queue is the * best option, as we feed the in-service queue with new * requests close to the last request served and, by doing so, * are likely to increase the throughput. */ bfqq->new_bfqq = new_bfqq; new_bfqq->ref += process_refs; return new_bfqq; } static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) { if (bfq_too_late_for_merging(new_bfqq)) return false; if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || (bfqq->ioprio_class != new_bfqq->ioprio_class)) return false; /* * If either of the queues has already been detected as seeky, * then merging it with the other queue is unlikely to lead to * sequential I/O. */ if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq)) return false; /* * Interleaved I/O is known to be done by (some) applications * only for reads, so it does not make sense to merge async * queues. */ if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq)) return false; return true; } /* * Attempt to schedule a merge of bfqq with the currently in-service * queue or with a close queue among the scheduled queues. Return * NULL if no merge was scheduled, a pointer to the shared bfq_queue * structure otherwise. * * The OOM queue is not allowed to participate to cooperation: in fact, since * the requests temporarily redirected to the OOM queue could be redirected * again to dedicated queues at any time, the state needed to correctly * handle merging with the OOM queue would be quite complex and expensive * to maintain. Besides, in such a critical condition as an out of memory, * the benefits of queue merging may be little relevant, or even negligible. * * WARNING: queue merging may impair fairness among non-weight raised * queues, for at least two reasons: 1) the original weight of a * merged queue may change during the merged state, 2) even being the * weight the same, a merged queue may be bloated with many more * requests than the ones produced by its originally-associated * process. */ static struct bfq_queue * bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, void *io_struct, bool request) { struct bfq_queue *in_service_bfqq, *new_bfqq; /* * Do not perform queue merging if the device is non * rotational and performs internal queueing. In fact, such a * device reaches a high speed through internal parallelism * and pipelining. This means that, to reach a high * throughput, it must have many requests enqueued at the same * time. But, in this configuration, the internal scheduling * algorithm of the device does exactly the job of queue * merging: it reorders requests so as to obtain as much as * possible a sequential I/O pattern. As a consequence, with * the workload generated by processes doing interleaved I/O, * the throughput reached by the device is likely to be the * same, with and without queue merging. * * Disabling merging also provides a remarkable benefit in * terms of throughput. Merging tends to make many workloads * artificially more uneven, because of shared queues * remaining non empty for incomparably more time than * non-merged queues. This may accentuate workload * asymmetries. For example, if one of the queues in a set of * merged queues has a higher weight than a normal queue, then * the shared queue may inherit such a high weight and, by * staying almost always active, may force BFQ to perform I/O * plugging most of the time. This evidently makes it harder * for BFQ to let the device reach a high throughput. * * Finally, the likely() macro below is not used because one * of the two branches is more likely than the other, but to * have the code path after the following if() executed as * fast as possible for the case of a non rotational device * with queueing. We want it because this is the fastest kind * of device. On the opposite end, the likely() may lengthen * the execution time of BFQ for the case of slower devices * (rotational or at least without queueing). But in this case * the execution time of BFQ matters very little, if not at * all. */ if (likely(bfqd->nonrot_with_queueing)) return NULL; /* * Prevent bfqq from being merged if it has been created too * long ago. The idea is that true cooperating processes, and * thus their associated bfq_queues, are supposed to be * created shortly after each other. This is the case, e.g., * for KVM/QEMU and dump I/O threads. Basing on this * assumption, the following filtering greatly reduces the * probability that two non-cooperating processes, which just * happen to do close I/O for some short time interval, have * their queues merged by mistake. */ if (bfq_too_late_for_merging(bfqq)) return NULL; if (bfqq->new_bfqq) return bfqq->new_bfqq; if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) return NULL; /* If there is only one backlogged queue, don't search. */ if (bfq_tot_busy_queues(bfqd) == 1) return NULL; in_service_bfqq = bfqd->in_service_queue; if (in_service_bfqq && in_service_bfqq != bfqq && likely(in_service_bfqq != &bfqd->oom_bfqq) && bfq_rq_close_to_sector(io_struct, request, bfqd->in_serv_last_pos) && bfqq->entity.parent == in_service_bfqq->entity.parent && bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); if (new_bfqq) return new_bfqq; } /* * Check whether there is a cooperator among currently scheduled * queues. The only thing we need is that the bio/request is not * NULL, as we need it to establish whether a cooperator exists. */ new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, bfq_io_struct_pos(io_struct, request)); if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq) && bfq_may_be_close_cooperator(bfqq, new_bfqq)) return bfq_setup_merge(bfqq, new_bfqq); return NULL; } static void bfq_bfqq_save_state(struct bfq_queue *bfqq) { struct bfq_io_cq *bic = bfqq->bic; /* * If !bfqq->bic, the queue is already shared or its requests * have already been redirected to a shared queue; both idle window * and weight raising state have already been saved. Do nothing. */ if (!bic) return; bic->saved_weight = bfqq->entity.orig_weight; bic->saved_ttime = bfqq->ttime; bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq); bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); if (unlikely(bfq_bfqq_just_created(bfqq) && !bfq_bfqq_in_large_burst(bfqq) && bfqq->bfqd->low_latency)) { /* * bfqq being merged right after being created: bfqq * would have deserved interactive weight raising, but * did not make it to be set in a weight-raised state, * because of this early merge. Store directly the * weight-raising state that would have been assigned * to bfqq, so that to avoid that bfqq unjustly fails * to enjoy weight raising if split soon. */ bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; bic->saved_wr_start_at_switch_to_srt = bfq_smallest_from_now(); bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd); bic->saved_last_wr_start_finish = jiffies; } else { bic->saved_wr_coeff = bfqq->wr_coeff; bic->saved_wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt; bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish; bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time; } } static void bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) { bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", (unsigned long)new_bfqq->pid); /* Save weight raising and idle window of the merged queues */ bfq_bfqq_save_state(bfqq); bfq_bfqq_save_state(new_bfqq); if (bfq_bfqq_IO_bound(bfqq)) bfq_mark_bfqq_IO_bound(new_bfqq); bfq_clear_bfqq_IO_bound(bfqq); /* * If bfqq is weight-raised, then let new_bfqq inherit * weight-raising. To reduce false positives, neglect the case * where bfqq has just been created, but has not yet made it * to be weight-raised (which may happen because EQM may merge * bfqq even before bfq_add_request is executed for the first * time for bfqq). Handling this case would however be very * easy, thanks to the flag just_created. */ if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) { new_bfqq->wr_coeff = bfqq->wr_coeff; new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; new_bfqq->wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt; if (bfq_bfqq_busy(new_bfqq)) bfqd->wr_busy_queues++; new_bfqq->entity.prio_changed = 1; } if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */ bfqq->wr_coeff = 1; bfqq->entity.prio_changed = 1; if (bfq_bfqq_busy(bfqq)) bfqd->wr_busy_queues--; } bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", bfqd->wr_busy_queues); /* * Merge queues (that is, let bic redirect its requests to new_bfqq) */ bic_set_bfqq(bic, new_bfqq, 1); bfq_mark_bfqq_coop(new_bfqq); /* * new_bfqq now belongs to at least two bics (it is a shared queue): * set new_bfqq->bic to NULL. bfqq either: * - does not belong to any bic any more, and hence bfqq->bic must * be set to NULL, or * - is a queue whose owning bics have already been redirected to a * different queue, hence the queue is destined to not belong to * any bic soon and bfqq->bic is already NULL (therefore the next * assignment causes no harm). */ new_bfqq->bic = NULL; /* * If the queue is shared, the pid is the pid of one of the associated * processes. Which pid depends on the exact sequence of merge events * the queue underwent. So printing such a pid is useless and confusing * because it reports a random pid between those of the associated * processes. * We mark such a queue with a pid -1, and then print SHARED instead of * a pid in logging messages. */ new_bfqq->pid = -1; bfqq->bic = NULL; /* release process reference to bfqq */ bfq_put_queue(bfqq); } static bool bfq_allow_bio_merge(struct request_queue *q, struct request *rq, struct bio *bio) { struct bfq_data *bfqd = q->elevator->elevator_data; bool is_sync = op_is_sync(bio->bi_opf); struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; /* * Disallow merge of a sync bio into an async request. */ if (is_sync && !rq_is_sync(rq)) return false; /* * Lookup the bfqq that this bio will be queued with. Allow * merge only if rq is queued there. */ if (!bfqq) return false; /* * We take advantage of this function to perform an early merge * of the queues of possible cooperating processes. */ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); if (new_bfqq) { /* * bic still points to bfqq, then it has not yet been * redirected to some other bfq_queue, and a queue * merge between bfqq and new_bfqq can be safely * fulfilled, i.e., bic can be redirected to new_bfqq * and bfqq can be put. */ bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq, new_bfqq); /* * If we get here, bio will be queued into new_queue, * so use new_bfqq to decide whether bio and rq can be * merged. */ bfqq = new_bfqq; /* * Change also bqfd->bio_bfqq, as * bfqd->bio_bic now points to new_bfqq, and * this function may be invoked again (and then may * use again bqfd->bio_bfqq). */ bfqd->bio_bfqq = bfqq; } return bfqq == RQ_BFQQ(rq); } /* * Set the maximum time for the in-service queue to consume its * budget. This prevents seeky processes from lowering the throughput. * In practice, a time-slice service scheme is used with seeky * processes. */ static void bfq_set_budget_timeout(struct bfq_data *bfqd, struct bfq_queue *bfqq) { unsigned int timeout_coeff; if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) timeout_coeff = 1; else timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; bfqd->last_budget_start = ktime_get(); bfqq->budget_timeout = jiffies + bfqd->bfq_timeout * timeout_coeff; } static void __bfq_set_in_service_queue(struct bfq_data *bfqd, struct bfq_queue *bfqq) { if (bfqq) { bfq_clear_bfqq_fifo_expire(bfqq); bfqd->budgets_assigned = (bfqd->budgets_assigned * 7 + 256) / 8; if (time_is_before_jiffies(bfqq->last_wr_start_finish) && bfqq->wr_coeff > 1 && bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && time_is_before_jiffies(bfqq->budget_timeout)) { /* * For soft real-time queues, move the start * of the weight-raising period forward by the * time the queue has not received any * service. Otherwise, a relatively long * service delay is likely to cause the * weight-raising period of the queue to end, * because of the short duration of the * weight-raising period of a soft real-time * queue. It is worth noting that this move * is not so dangerous for the other queues, * because soft real-time queues are not * greedy. * * To not add a further variable, we use the * overloaded field budget_timeout to * determine for how long the queue has not * received service, i.e., how much time has * elapsed since the queue expired. However, * this is a little imprecise, because * budget_timeout is set to jiffies if bfqq * not only expires, but also remains with no * request. */ if (time_after(bfqq->budget_timeout, bfqq->last_wr_start_finish)) bfqq->last_wr_start_finish += jiffies - bfqq->budget_timeout; else bfqq->last_wr_start_finish = jiffies; } bfq_set_budget_timeout(bfqd, bfqq); bfq_log_bfqq(bfqd, bfqq, "set_in_service_queue, cur-budget = %d", bfqq->entity.budget); } bfqd->in_service_queue = bfqq; } /* * Get and set a new queue for service. */ static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) { struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); __bfq_set_in_service_queue(bfqd, bfqq); return bfqq; } static void bfq_arm_slice_timer(struct bfq_data *bfqd) { struct bfq_queue *bfqq = bfqd->in_service_queue; u32 sl; bfq_mark_bfqq_wait_request(bfqq); /* * We don't want to idle for seeks, but we do want to allow * fair distribution of slice time for a process doing back-to-back * seeks. So allow a little bit of time for him to submit a new rq. */ sl = bfqd->bfq_slice_idle; /* * Unless the queue is being weight-raised or the scenario is * asymmetric, grant only minimum idle time if the queue * is seeky. A long idling is preserved for a weight-raised * queue, or, more in general, in an asymmetric scenario, * because a long idling is needed for guaranteeing to a queue * its reserved share of the throughput (in particular, it is * needed if the queue has a higher weight than some other * queue). */ if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && !bfq_asymmetric_scenario(bfqd, bfqq)) sl = min_t(u64, sl, BFQ_MIN_TT); else if (bfqq->wr_coeff > 1) sl = max_t(u32, sl, 20ULL * NSEC_PER_MSEC); bfqd->last_idling_start = ktime_get(); bfqd->last_idling_start_jiffies = jiffies; hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), HRTIMER_MODE_REL); bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); } /* * In autotuning mode, max_budget is dynamically recomputed as the * amount of sectors transferred in timeout at the estimated peak * rate. This enables BFQ to utilize a full timeslice with a full * budget, even if the in-service queue is served at peak rate. And * this maximises throughput with sequential workloads. */ static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd) { return (u64)bfqd->peak_rate * USEC_PER_MSEC * jiffies_to_msecs(bfqd->bfq_timeout)>>BFQ_RATE_SHIFT; } /* * Update parameters related to throughput and responsiveness, as a * function of the estimated peak rate. See comments on * bfq_calc_max_budget(), and on the ref_wr_duration array. */ static void update_thr_responsiveness_params(struct bfq_data *bfqd) { if (bfqd->bfq_user_max_budget == 0) { bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); bfq_log(bfqd, "new max_budget = %d", bfqd->bfq_max_budget); } } static void bfq_reset_rate_computation(struct bfq_data *bfqd, struct request *rq) { if (rq != NULL) { /* new rq dispatch now, reset accordingly */ bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns(); bfqd->peak_rate_samples = 1; bfqd->sequential_samples = 0; bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size = blk_rq_sectors(rq); } else /* no new rq dispatched, just reset the number of samples */ bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ bfq_log(bfqd, "reset_rate_computation at end, sample %u/%u tot_sects %llu", bfqd->peak_rate_samples, bfqd->sequential_samples, bfqd->tot_sectors_dispatched); } static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) { u32 rate, weight, divisor; /* * For the convergence property to hold (see comments on * bfq_update_peak_rate()) and for the assessment to be * reliable, a minimum number of samples must be present, and * a minimum amount of time must have elapsed. If not so, do * not compute new rate. Just reset parameters, to get ready * for a new evaluation attempt. */ if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) goto reset_computation; /* * If a new request completion has occurred after last * dispatch, then, to approximate the rate at which requests * have been served by the device, it is more precise to * extend the observation interval to the last completion. */ bfqd->delta_from_first = max_t(u64, bfqd->delta_from_first, bfqd->last_completion - bfqd->first_dispatch); /* * Rate computed in sects/usec, and not sects/nsec, for * precision issues. */ rate = div64_ul(bfqd->tot_sectors_dispatched<<BFQ_RATE_SHIFT, div_u64(bfqd->delta_from_first, NSEC_PER_USEC)); /* * Peak rate not updated if: * - the percentage of sequential dispatches is below 3/4 of the * total, and rate is below the current estimated peak rate * - rate is unreasonably high (> 20M sectors/sec) */ if ((bfqd->sequential_samples < (3 * bfqd->peak_rate_samples)>>2 && rate <= bfqd->peak_rate) || rate > 20<<BFQ_RATE_SHIFT) goto reset_computation; /* * We have to update the peak rate, at last! To this purpose, * we use a low-pass filter. We compute the smoothing constant * of the filter as a function of the 'weight' of the new * measured rate. * * As can be seen in next formulas, we define this weight as a * quantity proportional to how sequential the workload is, * and to how long the observation time interval is. * * The weight runs from 0 to 8. The maximum value of the * weight, 8, yields the minimum value for the smoothing * constant. At this minimum value for the smoothing constant, * the measured rate contributes for half of the next value of * the estimated peak rate. * * So, the first step is to compute the weight as a function * of how sequential the workload is. Note that the weight * cannot reach 9, because bfqd->sequential_samples cannot * become equal to bfqd->peak_rate_samples, which, in its * turn, holds true because bfqd->sequential_samples is not * incremented for the first sample. */ weight = (9 * bfqd->sequential_samples) / bfqd->peak_rate_samples; /* * Second step: further refine the weight as a function of the * duration of the observation interval. */ weight = min_t(u32, 8, div_u64(weight * bfqd->delta_from_first, BFQ_RATE_REF_INTERVAL)); /* * Divisor ranging from 10, for minimum weight, to 2, for * maximum weight. */ divisor = 10 - weight; /* * Finally, update peak rate: * * peak_rate = peak_rate * (divisor-1) / divisor + rate / divisor */ bfqd->peak_rate *= divisor-1; bfqd->peak_rate /= divisor; rate /= divisor; /* smoothing constant alpha = 1/divisor */ bfqd->peak_rate += rate; /* * For a very slow device, bfqd->peak_rate can reach 0 (see * the minimum representable values reported in the comments * on BFQ_RATE_SHIFT). Push to 1 if this happens, to avoid * divisions by zero where bfqd->peak_rate is used as a * divisor. */ bfqd->peak_rate = max_t(u32, 1, bfqd->peak_rate); update_thr_responsiveness_params(bfqd); reset_computation: bfq_reset_rate_computation(bfqd, rq); } /* * Update the read/write peak rate (the main quantity used for * auto-tuning, see update_thr_responsiveness_params()). * * It is not trivial to estimate the peak rate (correctly): because of * the presence of sw and hw queues between the scheduler and the * device components that finally serve I/O requests, it is hard to * say exactly when a given dispatched request is served inside the * device, and for how long. As a consequence, it is hard to know * precisely at what rate a given set of requests is actually served * by the device. * * On the opposite end, the dispatch time of any request is trivially * available, and, from this piece of information, the "dispatch rate" * of requests can be immediately computed. So, the idea in the next * function is to use what is known, namely request dispatch times * (plus, when useful, request completion times), to estimate what is * unknown, namely in-device request service rate. * * The main issue is that, because of the above facts, the rate at * which a certain set of requests is dispatched over a certain time * interval can vary greatly with respect to the rate at which the * same requests are then served. But, since the size of any * intermediate queue is limited, and the service scheme is lossless * (no request is silently dropped), the following obvious convergence * property holds: the number of requests dispatched MUST become * closer and closer to the number of requests completed as the * observation interval grows. This is the key property used in * the next function to estimate the peak service rate as a function * of the observed dispatch rate. The function assumes to be invoked * on every request dispatch. */ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) { u64 now_ns = ktime_get_ns(); if (bfqd->peak_rate_samples == 0) { /* first dispatch */ bfq_log(bfqd, "update_peak_rate: goto reset, samples %d", bfqd->peak_rate_samples); bfq_reset_rate_computation(bfqd, rq); goto update_last_values; /* will add one sample */ } /* * Device idle for very long: the observation interval lasting * up to this dispatch cannot be a valid observation interval * for computing a new peak rate (similarly to the late- * completion event in bfq_completed_request()). Go to * update_rate_and_reset to have the following three steps * taken: * - close the observation interval at the last (previous) * request dispatch or completion * - compute rate, if possible, for that observation interval * - start a new observation interval with this dispatch */ if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && bfqd->rq_in_driver == 0) goto update_rate_and_reset; /* Update sampling information */ bfqd->peak_rate_samples++; if ((bfqd->rq_in_driver > 0 || now_ns - bfqd->last_completion < BFQ_MIN_TT) && !BFQ_RQ_SEEKY(bfqd, bfqd->last_position, rq)) bfqd->sequential_samples++; bfqd->tot_sectors_dispatched += blk_rq_sectors(rq); /* Reset max observed rq size every 32 dispatches */ if (likely(bfqd->peak_rate_samples % 32)) bfqd->last_rq_max_size = max_t(u32, blk_rq_sectors(rq), bfqd->last_rq_max_size); else bfqd->last_rq_max_size = blk_rq_sectors(rq); bfqd->delta_from_first = now_ns - bfqd->first_dispatch; /* Target observation interval not yet reached, go on sampling */ if (bfqd->delta_from_first < BFQ_RATE_REF_INTERVAL) goto update_last_values; update_rate_and_reset: bfq_update_rate_reset(bfqd, rq); update_last_values: bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); if (RQ_BFQQ(rq) == bfqd->in_service_queue) bfqd->in_serv_last_pos = bfqd->last_position; bfqd->last_dispatch = now_ns; } /* * Remove request from internal lists. */ static void bfq_dispatch_remove(struct request_queue *q, struct request *rq) { struct bfq_queue *bfqq = RQ_BFQQ(rq); /* * For consistency, the next instruction should have been * executed after removing the request from the queue and * dispatching it. We execute instead this instruction before * bfq_remove_request() (and hence introduce a temporary * inconsistency), for efficiency. In fact, should this * dispatch occur for a non in-service bfqq, this anticipated * increment prevents two counters related to bfqq->dispatched * from risking to be, first, uselessly decremented, and then * incremented again when the (new) value of bfqq->dispatched * happens to be taken into account. */ bfqq->dispatched++; bfq_update_peak_rate(q->elevator->elevator_data, rq); bfq_remove_request(q, rq); } /* * There is a case where idling does not have to be performed for * throughput concerns, but to preserve the throughput share of * the process associated with bfqq. * * To introduce this case, we can note that allowing the drive * to enqueue more than one request at a time, and hence * delegating de facto final scheduling decisions to the * drive's internal scheduler, entails loss of control on the * actual request service order. In particular, the critical * situation is when requests from different processes happen * to be present, at the same time, in the internal queue(s) * of the drive. In such a situation, the drive, by deciding * the service order of the internally-queued requests, does * determine also the actual throughput distribution among * these processes. But the drive typically has no notion or * concern about per-process throughput distribution, and * makes its decisions only on a per-request basis. Therefore, * the service distribution enforced by the drive's internal * scheduler is likely to coincide with the desired throughput * distribution only in a completely symmetric, or favorably * skewed scenario where: * (i-a) each of these processes must get the same throughput as * the others, * (i-b) in case (i-a) does not hold, it holds that the process * associated with bfqq must receive a lower or equal * throughput than any of the other processes; * (ii) the I/O of each process has the same properties, in * terms of locality (sequential or random), direction * (reads or writes), request sizes, greediness * (from I/O-bound to sporadic), and so on; * In fact, in such a scenario, the drive tends to treat the requests * of each process in about the same way as the requests of the * others, and thus to provide each of these processes with about the * same throughput. This is exactly the desired throughput * distribution if (i-a) holds, or, if (i-b) holds instead, this is an * even more convenient distribution for (the process associated with) * bfqq. * * In contrast, in any asymmetric or unfavorable scenario, device * idling (I/O-dispatch plugging) is certainly needed to guarantee * that bfqq receives its assigned fraction of the device throughput * (see [1] for details). * * The problem is that idling may significantly reduce throughput with * certain combinations of types of I/O and devices. An important * example is sync random I/O on flash storage with command * queueing. So, unless bfqq falls in cases where idling also boosts * throughput, it is important to check conditions (i-a), i(-b) and * (ii) accurately, so as to avoid idling when not strictly needed for * service guarantees. * * Unfortunately, it is extremely difficult to thoroughly check * condition (ii). And, in case there are active groups, it becomes * very difficult to check conditions (i-a) and (i-b) too. In fact, * if there are active groups, then, for conditions (i-a) or (i-b) to * become false 'indirectly', it is enough that an active group * contains more active processes or sub-groups than some other active * group. More precisely, for conditions (i-a) or (i-b) to become * false because of such a group, it is not even necessary that the * group is (still) active: it is sufficient that, even if the group * has become inactive, some of its descendant processes still have * some request already dispatched but still waiting for * completion. In fact, requests have still to be guaranteed their * share of the throughput even after being dispatched. In this * respect, it is easy to show that, if a group frequently becomes * inactive while still having in-flight requests, and if, when this * happens, the group is not considered in the calculation of whether * the scenario is asymmetric, then the group may fail to be * guaranteed its fair share of the throughput (basically because * idling may not be performed for the descendant processes of the * group, but it had to be). We address this issue with the following * bi-modal behavior, implemented in the function * bfq_asymmetric_scenario(). * * If there are groups with requests waiting for completion * (as commented above, some of these groups may even be * already inactive), then the scenario is tagged as * asymmetric, conservatively, without checking any of the * conditions (i-a), (i-b) or (ii). So the device is idled for bfqq. * This behavior matches also the fact that groups are created * exactly if controlling I/O is a primary concern (to * preserve bandwidth and latency guarantees). * * On the opposite end, if there are no groups with requests waiting * for completion, then only conditions (i-a) and (i-b) are actually * controlled, i.e., provided that conditions (i-a) or (i-b) holds, * idling is not performed, regardless of whether condition (ii) * holds. In other words, only if conditions (i-a) and (i-b) do not * hold, then idling is allowed, and the device tends to be prevented * from queueing many requests, possibly of several processes. Since * there are no groups with requests waiting for completion, then, to * control conditions (i-a) and (i-b) it is enough to check just * whether all the queues with requests waiting for completion also * have the same weight. * * Not checking condition (ii) evidently exposes bfqq to the * risk of getting less throughput than its fair share. * However, for queues with the same weight, a further * mechanism, preemption, mitigates or even eliminates this * problem. And it does so without consequences on overall * throughput. This mechanism and its benefits are explained * in the next three paragraphs. * * Even if a queue, say Q, is expired when it remains idle, Q * can still preempt the new in-service queue if the next * request of Q arrives soon (see the comments on * bfq_bfqq_update_budg_for_activation). If all queues and * groups have the same weight, this form of preemption, * combined with the hole-recovery heuristic described in the * comments on function bfq_bfqq_update_budg_for_activation, * are enough to preserve a correct bandwidth distribution in * the mid term, even without idling. In fact, even if not * idling allows the internal queues of the device to contain * many requests, and thus to reorder requests, we can rather * safely assume that the internal scheduler still preserves a * minimum of mid-term fairness. * * More precisely, this preemption-based, idleless approach * provides fairness in terms of IOPS, and not sectors per * second. This can be seen with a simple example. Suppose * that there are two queues with the same weight, but that * the first queue receives requests of 8 sectors, while the * second queue receives requests of 1024 sectors. In * addition, suppose that each of the two queues contains at * most one request at a time, which implies that each queue * always remains idle after it is served. Finally, after * remaining idle, each queue receives very quickly a new * request. It follows that the two queues are served * alternatively, preempting each other if needed. This * implies that, although both queues have the same weight, * the queue with large requests receives a service that is * 1024/8 times as high as the service received by the other * queue. * * The motivation for using preemption instead of idling (for * queues with the same weight) is that, by not idling, * service guarantees are preserved (completely or at least in * part) without minimally sacrificing throughput. And, if * there is no active group, then the primary expectation for * this device is probably a high throughput. * * We are now left only with explaining the additional * compound condition that is checked below for deciding * whether the scenario is asymmetric. To explain this * compound condition, we need to add that the function * bfq_asymmetric_scenario checks the weights of only * non-weight-raised queues, for efficiency reasons (see * comments on bfq_weights_tree_add()). Then the fact that * bfqq is weight-raised is checked explicitly here. More * precisely, the compound condition below takes into account * also the fact that, even if bfqq is being weight-raised,


context:
space:
mode: