diff options
| -rw-r--r-- | kernel/sched.c | 28 | ||||
| -rw-r--r-- | kernel/sched_rt.c | 46 |
2 files changed, 37 insertions, 37 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index 3dbe6178ebfd..01c5016e57f1 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -141,7 +141,7 @@ struct rt_prio_array { | |||
| 141 | 141 | ||
| 142 | struct rt_bandwidth { | 142 | struct rt_bandwidth { |
| 143 | /* nests inside the rq lock: */ | 143 | /* nests inside the rq lock: */ |
| 144 | spinlock_t rt_runtime_lock; | 144 | raw_spinlock_t rt_runtime_lock; |
| 145 | ktime_t rt_period; | 145 | ktime_t rt_period; |
| 146 | u64 rt_runtime; | 146 | u64 rt_runtime; |
| 147 | struct hrtimer rt_period_timer; | 147 | struct hrtimer rt_period_timer; |
| @@ -178,7 +178,7 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) | |||
| 178 | rt_b->rt_period = ns_to_ktime(period); | 178 | rt_b->rt_period = ns_to_ktime(period); |
| 179 | rt_b->rt_runtime = runtime; | 179 | rt_b->rt_runtime = runtime; |
| 180 | 180 | ||
| 181 | spin_lock_init(&rt_b->rt_runtime_lock); | 181 | raw_spin_lock_init(&rt_b->rt_runtime_lock); |
| 182 | 182 | ||
| 183 | hrtimer_init(&rt_b->rt_period_timer, | 183 | hrtimer_init(&rt_b->rt_period_timer, |
| 184 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 184 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| @@ -200,7 +200,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | |||
| 200 | if (hrtimer_active(&rt_b->rt_period_timer)) | 200 | if (hrtimer_active(&rt_b->rt_period_timer)) |
| 201 | return; | 201 | return; |
| 202 | 202 | ||
| 203 | spin_lock(&rt_b->rt_runtime_lock); | 203 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 204 | for (;;) { | 204 | for (;;) { |
| 205 | unsigned long delta; | 205 | unsigned long delta; |
| 206 | ktime_t soft, hard; | 206 | ktime_t soft, hard; |
| @@ -217,7 +217,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | |||
| 217 | __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta, | 217 | __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta, |
| 218 | HRTIMER_MODE_ABS_PINNED, 0); | 218 | HRTIMER_MODE_ABS_PINNED, 0); |
| 219 | } | 219 | } |
| 220 | spin_unlock(&rt_b->rt_runtime_lock); | 220 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 221 | } | 221 | } |
| 222 | 222 | ||
| 223 | #ifdef CONFIG_RT_GROUP_SCHED | 223 | #ifdef CONFIG_RT_GROUP_SCHED |
| @@ -470,7 +470,7 @@ struct rt_rq { | |||
| 470 | u64 rt_time; | 470 | u64 rt_time; |
| 471 | u64 rt_runtime; | 471 | u64 rt_runtime; |
| 472 | /* Nests inside the rq lock: */ | 472 | /* Nests inside the rq lock: */ |
| 473 | spinlock_t rt_runtime_lock; | 473 | raw_spinlock_t rt_runtime_lock; |
| 474 | 474 | ||
| 475 | #ifdef CONFIG_RT_GROUP_SCHED | 475 | #ifdef CONFIG_RT_GROUP_SCHED |
| 476 | unsigned long rt_nr_boosted; | 476 | unsigned long rt_nr_boosted; |
| @@ -9366,7 +9366,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) | |||
| 9366 | rt_rq->rt_time = 0; | 9366 | rt_rq->rt_time = 0; |
| 9367 | rt_rq->rt_throttled = 0; | 9367 | rt_rq->rt_throttled = 0; |
| 9368 | rt_rq->rt_runtime = 0; | 9368 | rt_rq->rt_runtime = 0; |
| 9369 | spin_lock_init(&rt_rq->rt_runtime_lock); | 9369 | raw_spin_lock_init(&rt_rq->rt_runtime_lock); |
| 9370 | 9370 | ||
| 9371 | #ifdef CONFIG_RT_GROUP_SCHED | 9371 | #ifdef CONFIG_RT_GROUP_SCHED |
| 9372 | rt_rq->rt_nr_boosted = 0; | 9372 | rt_rq->rt_nr_boosted = 0; |
| @@ -10305,18 +10305,18 @@ static int tg_set_bandwidth(struct task_group *tg, | |||
| 10305 | if (err) | 10305 | if (err) |
| 10306 | goto unlock; | 10306 | goto unlock; |
| 10307 | 10307 | ||
| 10308 | spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); | 10308 | raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); |
| 10309 | tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); | 10309 | tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); |
| 10310 | tg->rt_bandwidth.rt_runtime = rt_runtime; | 10310 | tg->rt_bandwidth.rt_runtime = rt_runtime; |
| 10311 | 10311 | ||
| 10312 | for_each_possible_cpu(i) { | 10312 | for_each_possible_cpu(i) { |
| 10313 | struct rt_rq *rt_rq = tg->rt_rq[i]; | 10313 | struct rt_rq *rt_rq = tg->rt_rq[i]; |
| 10314 | 10314 | ||
| 10315 | spin_lock(&rt_rq->rt_runtime_lock); | 10315 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 10316 | rt_rq->rt_runtime = rt_runtime; | 10316 | rt_rq->rt_runtime = rt_runtime; |
| 10317 | spin_unlock(&rt_rq->rt_runtime_lock); | 10317 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 10318 | } | 10318 | } |
| 10319 | spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); | 10319 | raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); |
| 10320 | unlock: | 10320 | unlock: |
| 10321 | read_unlock(&tasklist_lock); | 10321 | read_unlock(&tasklist_lock); |
| 10322 | mutex_unlock(&rt_constraints_mutex); | 10322 | mutex_unlock(&rt_constraints_mutex); |
| @@ -10421,15 +10421,15 @@ static int sched_rt_global_constraints(void) | |||
| 10421 | if (sysctl_sched_rt_runtime == 0) | 10421 | if (sysctl_sched_rt_runtime == 0) |
| 10422 | return -EBUSY; | 10422 | return -EBUSY; |
| 10423 | 10423 | ||
| 10424 | spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); | 10424 | raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); |
| 10425 | for_each_possible_cpu(i) { | 10425 | for_each_possible_cpu(i) { |
| 10426 | struct rt_rq *rt_rq = &cpu_rq(i)->rt; | 10426 | struct rt_rq *rt_rq = &cpu_rq(i)->rt; |
| 10427 | 10427 | ||
| 10428 | spin_lock(&rt_rq->rt_runtime_lock); | 10428 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 10429 | rt_rq->rt_runtime = global_rt_runtime(); | 10429 | rt_rq->rt_runtime = global_rt_runtime(); |
| 10430 | spin_unlock(&rt_rq->rt_runtime_lock); | 10430 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 10431 | } | 10431 | } |
| 10432 | spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); | 10432 | raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); |
| 10433 | 10433 | ||
| 10434 | return 0; | 10434 | return 0; |
| 10435 | } | 10435 | } |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index a8325a7ff94c..d2ea2828164e 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
| @@ -327,7 +327,7 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 327 | 327 | ||
| 328 | weight = cpumask_weight(rd->span); | 328 | weight = cpumask_weight(rd->span); |
| 329 | 329 | ||
| 330 | spin_lock(&rt_b->rt_runtime_lock); | 330 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 331 | rt_period = ktime_to_ns(rt_b->rt_period); | 331 | rt_period = ktime_to_ns(rt_b->rt_period); |
| 332 | for_each_cpu(i, rd->span) { | 332 | for_each_cpu(i, rd->span) { |
| 333 | struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); | 333 | struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); |
| @@ -336,7 +336,7 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 336 | if (iter == rt_rq) | 336 | if (iter == rt_rq) |
| 337 | continue; | 337 | continue; |
| 338 | 338 | ||
| 339 | spin_lock(&iter->rt_runtime_lock); | 339 | raw_spin_lock(&iter->rt_runtime_lock); |
| 340 | /* | 340 | /* |
| 341 | * Either all rqs have inf runtime and there's nothing to steal | 341 | * Either all rqs have inf runtime and there's nothing to steal |
| 342 | * or __disable_runtime() below sets a specific rq to inf to | 342 | * or __disable_runtime() below sets a specific rq to inf to |
| @@ -358,14 +358,14 @@ static int do_balance_runtime(struct rt_rq *rt_rq) | |||
| 358 | rt_rq->rt_runtime += diff; | 358 | rt_rq->rt_runtime += diff; |
| 359 | more = 1; | 359 | more = 1; |
| 360 | if (rt_rq->rt_runtime == rt_period) { | 360 | if (rt_rq->rt_runtime == rt_period) { |
| 361 | spin_unlock(&iter->rt_runtime_lock); | 361 | raw_spin_unlock(&iter->rt_runtime_lock); |
| 362 | break; | 362 | break; |
| 363 | } | 363 | } |
| 364 | } | 364 | } |
| 365 | next: | 365 | next: |
| 366 | spin_unlock(&iter->rt_runtime_lock); | 366 | raw_spin_unlock(&iter->rt_runtime_lock); |
| 367 | } | 367 | } |
| 368 | spin_unlock(&rt_b->rt_runtime_lock); | 368 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 369 | 369 | ||
| 370 | return more; | 370 | return more; |
| 371 | } | 371 | } |
| @@ -386,8 +386,8 @@ static void __disable_runtime(struct rq *rq) | |||
| 386 | s64 want; | 386 | s64 want; |
| 387 | int i; | 387 | int i; |
| 388 | 388 | ||
| 389 | spin_lock(&rt_b->rt_runtime_lock); | 389 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 390 | spin_lock(&rt_rq->rt_runtime_lock); | 390 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 391 | /* | 391 | /* |
| 392 | * Either we're all inf and nobody needs to borrow, or we're | 392 | * Either we're all inf and nobody needs to borrow, or we're |
| 393 | * already disabled and thus have nothing to do, or we have | 393 | * already disabled and thus have nothing to do, or we have |
| @@ -396,7 +396,7 @@ static void __disable_runtime(struct rq *rq) | |||
| 396 | if (rt_rq->rt_runtime == RUNTIME_INF || | 396 | if (rt_rq->rt_runtime == RUNTIME_INF || |
| 397 | rt_rq->rt_runtime == rt_b->rt_runtime) | 397 | rt_rq->rt_runtime == rt_b->rt_runtime) |
| 398 | goto balanced; | 398 | goto balanced; |
| 399 | spin_unlock(&rt_rq->rt_runtime_lock); | 399 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 400 | 400 | ||
| 401 | /* | 401 | /* |
| 402 | * Calculate the difference between what we started out with | 402 | * Calculate the difference between what we started out with |
| @@ -418,7 +418,7 @@ static void __disable_runtime(struct rq *rq) | |||
| 418 | if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) | 418 | if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) |
| 419 | continue; | 419 | continue; |
| 420 | 420 | ||
| 421 | spin_lock(&iter->rt_runtime_lock); | 421 | raw_spin_lock(&iter->rt_runtime_lock); |
| 422 | if (want > 0) { | 422 | if (want > 0) { |
| 423 | diff = min_t(s64, iter->rt_runtime, want); | 423 | diff = min_t(s64, iter->rt_runtime, want); |
| 424 | iter->rt_runtime -= diff; | 424 | iter->rt_runtime -= diff; |
| @@ -427,13 +427,13 @@ static void __disable_runtime(struct rq *rq) | |||
| 427 | iter->rt_runtime -= want; | 427 | iter->rt_runtime -= want; |
| 428 | want -= want; | 428 | want -= want; |
| 429 | } | 429 | } |
| 430 | spin_unlock(&iter->rt_runtime_lock); | 430 | raw_spin_unlock(&iter->rt_runtime_lock); |
| 431 | 431 | ||
| 432 | if (!want) | 432 | if (!want) |
| 433 | break; | 433 | break; |
| 434 | } | 434 | } |
| 435 | 435 | ||
| 436 | spin_lock(&rt_rq->rt_runtime_lock); | 436 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 437 | /* | 437 | /* |
| 438 | * We cannot be left wanting - that would mean some runtime | 438 | * We cannot be left wanting - that would mean some runtime |
| 439 | * leaked out of the system. | 439 | * leaked out of the system. |
| @@ -445,8 +445,8 @@ balanced: | |||
| 445 | * runtime - in which case borrowing doesn't make sense. | 445 | * runtime - in which case borrowing doesn't make sense. |
| 446 | */ | 446 | */ |
| 447 | rt_rq->rt_runtime = RUNTIME_INF; | 447 | rt_rq->rt_runtime = RUNTIME_INF; |
| 448 | spin_unlock(&rt_rq->rt_runtime_lock); | 448 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 449 | spin_unlock(&rt_b->rt_runtime_lock); | 449 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 450 | } | 450 | } |
| 451 | } | 451 | } |
| 452 | 452 | ||
| @@ -472,13 +472,13 @@ static void __enable_runtime(struct rq *rq) | |||
| 472 | for_each_leaf_rt_rq(rt_rq, rq) { | 472 | for_each_leaf_rt_rq(rt_rq, rq) { |
| 473 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); | 473 | struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); |
| 474 | 474 | ||
| 475 | spin_lock(&rt_b->rt_runtime_lock); | 475 | raw_spin_lock(&rt_b->rt_runtime_lock); |
| 476 | spin_lock(&rt_rq->rt_runtime_lock); | 476 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 477 | rt_rq->rt_runtime = rt_b->rt_runtime; | 477 | rt_rq->rt_runtime = rt_b->rt_runtime; |
| 478 | rt_rq->rt_time = 0; | 478 | rt_rq->rt_time = 0; |
| 479 | rt_rq->rt_throttled = 0; | 479 | rt_rq->rt_throttled = 0; |
| 480 | spin_unlock(&rt_rq->rt_runtime_lock); | 480 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 481 | spin_unlock(&rt_b->rt_runtime_lock); | 481 | raw_spin_unlock(&rt_b->rt_runtime_lock); |
| 482 | } | 482 | } |
| 483 | } | 483 | } |
| 484 | 484 | ||
| @@ -496,9 +496,9 @@ static int balance_runtime(struct rt_rq *rt_rq) | |||
| 496 | int more = 0; | 496 | int more = 0; |
| 497 | 497 | ||
| 498 | if (rt_rq->rt_time > rt_rq->rt_runtime) { | 498 | if (rt_rq->rt_time > rt_rq->rt_runtime) { |
| 499 | spin_unlock(&rt_rq->rt_runtime_lock); | 499 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 500 | more = do_balance_runtime(rt_rq); | 500 | more = do_balance_runtime(rt_rq); |
| 501 | spin_lock(&rt_rq->rt_runtime_lock); | 501 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 502 | } | 502 | } |
| 503 | 503 | ||
| 504 | return more; | 504 | return more; |
| @@ -528,7 +528,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) | |||
| 528 | if (rt_rq->rt_time) { | 528 | if (rt_rq->rt_time) { |
| 529 | u64 runtime; | 529 | u64 runtime; |
| 530 | 530 | ||
| 531 | spin_lock(&rt_rq->rt_runtime_lock); | 531 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 532 | if (rt_rq->rt_throttled) | 532 | if (rt_rq->rt_throttled) |
| 533 | balance_runtime(rt_rq); | 533 | balance_runtime(rt_rq); |
| 534 | runtime = rt_rq->rt_runtime; | 534 | runtime = rt_rq->rt_runtime; |
| @@ -539,7 +539,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) | |||
| 539 | } | 539 | } |
| 540 | if (rt_rq->rt_time || rt_rq->rt_nr_running) | 540 | if (rt_rq->rt_time || rt_rq->rt_nr_running) |
| 541 | idle = 0; | 541 | idle = 0; |
| 542 | spin_unlock(&rt_rq->rt_runtime_lock); | 542 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 543 | } else if (rt_rq->rt_nr_running) | 543 | } else if (rt_rq->rt_nr_running) |
| 544 | idle = 0; | 544 | idle = 0; |
| 545 | 545 | ||
| @@ -624,11 +624,11 @@ static void update_curr_rt(struct rq *rq) | |||
| 624 | rt_rq = rt_rq_of_se(rt_se); | 624 | rt_rq = rt_rq_of_se(rt_se); |
| 625 | 625 | ||
| 626 | if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { | 626 | if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { |
| 627 | spin_lock(&rt_rq->rt_runtime_lock); | 627 | raw_spin_lock(&rt_rq->rt_runtime_lock); |
| 628 | rt_rq->rt_time += delta_exec; | 628 | rt_rq->rt_time += delta_exec; |
| 629 | if (sched_rt_runtime_exceeded(rt_rq)) | 629 | if (sched_rt_runtime_exceeded(rt_rq)) |
| 630 | resched_task(curr); | 630 | resched_task(curr); |
| 631 | spin_unlock(&rt_rq->rt_runtime_lock); | 631 | raw_spin_unlock(&rt_rq->rt_runtime_lock); |
| 632 | } | 632 | } |
| 633 | } | 633 | } |
| 634 | } | 634 | } |
