aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Zijlstra <peterz@infradead.org>2017-09-20 13:00:17 -0400
committerThomas Gleixner <tglx@linutronix.de>2017-09-25 16:11:42 -0400
commit4dddfb5faa6118564b0c54a163353d13882299d8 (patch)
tree17aaae90bec3f102a752c07cf6791e0ff43c5424
parent96abb968549cdefd0964d1f7af0a79f4e6e7f897 (diff)
smp/hotplug: Rewrite AP state machine core
There is currently no explicit state change on rollback. That is, st->bringup, st->rollback and st->target are not consistent when doing the rollback. Rework the AP state handling to be more coherent. This does mean we have to do a second AP kick-and-wait for rollback, but since rollback is the slow path of a slowpath, this really should not matter. Take this opportunity to simplify the AP thread function to only run a single callback per invocation. This unifies the three single/up/down modes is supports. The looping it used to do for up/down are achieved by retaining should_run and relying on the main smpboot_thread_fn() loop. (I have most of a patch that does the same for the BP state handling, but that's not critical and gets a little complicated because CPUHP_BRINGUP_CPU does the AP handoff from a callback, which gets recursive @st usage, I still have de-fugly that.) [ tglx: Move cpuhp_down_callbacks() et al. into the HOTPLUG_CPU section to avoid gcc complaining about unused functions. Make the HOTPLUG_CPU one piece instead of having two consecutive ifdef sections of the same type. ] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: bigeasy@linutronix.de Cc: efault@gmx.de Cc: rostedt@goodmis.org Cc: max.byungchul.park@gmail.com Link: https://lkml.kernel.org/r/20170920170546.769658088@infradead.org
-rw-r--r--kernel/cpu.c321
1 files changed, 206 insertions, 115 deletions
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 323b71050b54..1139063de5af 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -58,6 +58,7 @@ struct cpuhp_cpu_state {
58 bool single; 58 bool single;
59 bool bringup; 59 bool bringup;
60 struct hlist_node *node; 60 struct hlist_node *node;
61 struct hlist_node *last;
61 enum cpuhp_state cb_state; 62 enum cpuhp_state cb_state;
62 int result; 63 int result;
63 struct completion done; 64 struct completion done;
@@ -112,6 +113,14 @@ static bool cpuhp_is_ap_state(enum cpuhp_state state)
112 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 113 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
113} 114}
114 115
116/*
117 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
118 */
119static bool cpuhp_is_atomic_state(enum cpuhp_state state)
120{
121 return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
122}
123
115static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 124static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
116{ 125{
117 struct cpuhp_step *sp; 126 struct cpuhp_step *sp;
@@ -286,7 +295,72 @@ void cpu_hotplug_enable(void)
286EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 295EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
287#endif /* CONFIG_HOTPLUG_CPU */ 296#endif /* CONFIG_HOTPLUG_CPU */
288 297
289static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st); 298static inline enum cpuhp_state
299cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
300{
301 enum cpuhp_state prev_state = st->state;
302
303 st->rollback = false;
304 st->last = NULL;
305
306 st->target = target;
307 st->single = false;
308 st->bringup = st->state < target;
309
310 return prev_state;
311}
312
313static inline void
314cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
315{
316 st->rollback = true;
317
318 /*
319 * If we have st->last we need to undo partial multi_instance of this
320 * state first. Otherwise start undo at the previous state.
321 */
322 if (!st->last) {
323 if (st->bringup)
324 st->state--;
325 else
326 st->state++;
327 }
328
329 st->target = prev_state;
330 st->bringup = !st->bringup;
331}
332
333/* Regular hotplug invocation of the AP hotplug thread */
334static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
335{
336 if (!st->single && st->state == st->target)
337 return;
338
339 st->result = 0;
340 /*
341 * Make sure the above stores are visible before should_run becomes
342 * true. Paired with the mb() above in cpuhp_thread_fun()
343 */
344 smp_mb();
345 st->should_run = true;
346 wake_up_process(st->thread);
347 wait_for_completion(&st->done);
348}
349
350static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
351{
352 enum cpuhp_state prev_state;
353 int ret;
354
355 prev_state = cpuhp_set_state(st, target);
356 __cpuhp_kick_ap(st);
357 if ((ret = st->result)) {
358 cpuhp_reset_state(st, prev_state);
359 __cpuhp_kick_ap(st);
360 }
361
362 return ret;
363}
290 364
291static int bringup_wait_for_ap(unsigned int cpu) 365static int bringup_wait_for_ap(unsigned int cpu)
292{ 366{
@@ -301,12 +375,10 @@ static int bringup_wait_for_ap(unsigned int cpu)
301 stop_machine_unpark(cpu); 375 stop_machine_unpark(cpu);
302 kthread_unpark(st->thread); 376 kthread_unpark(st->thread);
303 377
304 /* Should we go further up ? */ 378 if (st->target <= CPUHP_AP_ONLINE_IDLE)
305 if (st->target > CPUHP_AP_ONLINE_IDLE) { 379 return 0;
306 __cpuhp_kick_ap_work(st); 380
307 wait_for_completion(&st->done); 381 return cpuhp_kick_ap(st, st->target);
308 }
309 return st->result;
310} 382}
311 383
312static int bringup_cpu(unsigned int cpu) 384static int bringup_cpu(unsigned int cpu)
@@ -332,32 +404,6 @@ static int bringup_cpu(unsigned int cpu)
332/* 404/*
333 * Hotplug state machine related functions 405 * Hotplug state machine related functions
334 */ 406 */
335static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
336{
337 for (st->state++; st->state < st->target; st->state++) {
338 struct cpuhp_step *step = cpuhp_get_step(st->state);
339
340 if (!step->skip_onerr)
341 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
342 }
343}
344
345static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
346 enum cpuhp_state target)
347{
348 enum cpuhp_state prev_state = st->state;
349 int ret = 0;
350
351 for (; st->state > target; st->state--) {
352 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
353 if (ret) {
354 st->target = prev_state;
355 undo_cpu_down(cpu, st);
356 break;
357 }
358 }
359 return ret;
360}
361 407
362static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) 408static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
363{ 409{
@@ -404,71 +450,90 @@ static int cpuhp_should_run(unsigned int cpu)
404 return st->should_run; 450 return st->should_run;
405} 451}
406 452
407/* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */
408static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
409{
410 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
411
412 return cpuhp_down_callbacks(cpu, st, target);
413}
414
415/* Execute the online startup callbacks. Used to be CPU_ONLINE */
416static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
417{
418 return cpuhp_up_callbacks(cpu, st, st->target);
419}
420
421/* 453/*
422 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 454 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
423 * callbacks when a state gets [un]installed at runtime. 455 * callbacks when a state gets [un]installed at runtime.
456 *
457 * Each invocation of this function by the smpboot thread does a single AP
458 * state callback.
459 *
460 * It has 3 modes of operation:
461 * - single: runs st->cb_state
462 * - up: runs ++st->state, while st->state < st->target
463 * - down: runs st->state--, while st->state > st->target
464 *
465 * When complete or on error, should_run is cleared and the completion is fired.
424 */ 466 */
425static void cpuhp_thread_fun(unsigned int cpu) 467static void cpuhp_thread_fun(unsigned int cpu)
426{ 468{
427 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 469 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
428 int ret = 0; 470 bool bringup = st->bringup;
471 enum cpuhp_state state;
429 472
430 /* 473 /*
431 * Paired with the mb() in cpuhp_kick_ap_work and 474 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
432 * cpuhp_invoke_ap_callback, so the work set is consistent visible. 475 * that if we see ->should_run we also see the rest of the state.
433 */ 476 */
434 smp_mb(); 477 smp_mb();
435 if (!st->should_run)
436 return;
437 478
438 st->should_run = false; 479 if (WARN_ON_ONCE(!st->should_run))
480 return;
439 481
440 lock_map_acquire(&cpuhp_state_lock_map); 482 lock_map_acquire(&cpuhp_state_lock_map);
441 /* Single callback invocation for [un]install ? */ 483
442 if (st->single) { 484 if (st->single) {
443 if (st->cb_state < CPUHP_AP_ONLINE) { 485 state = st->cb_state;
444 local_irq_disable(); 486 st->should_run = false;
445 ret = cpuhp_invoke_callback(cpu, st->cb_state, 487 } else {
446 st->bringup, st->node, 488 if (bringup) {
447 NULL); 489 st->state++;
448 local_irq_enable(); 490 state = st->state;
491 st->should_run = (st->state < st->target);
492 WARN_ON_ONCE(st->state > st->target);
449 } else { 493 } else {
450 ret = cpuhp_invoke_callback(cpu, st->cb_state, 494 state = st->state;
451 st->bringup, st->node, 495 st->state--;
452 NULL); 496 st->should_run = (st->state > st->target);
497 WARN_ON_ONCE(st->state < st->target);
453 } 498 }
454 } else if (st->rollback) { 499 }
455 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 500
501 WARN_ON_ONCE(!cpuhp_is_ap_state(state));
502
503 if (st->rollback) {
504 struct cpuhp_step *step = cpuhp_get_step(state);
505 if (step->skip_onerr)
506 goto next;
507 }
508
509 if (cpuhp_is_atomic_state(state)) {
510 local_irq_disable();
511 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
512 local_irq_enable();
456 513
457 undo_cpu_down(cpu, st); 514 /*
458 st->rollback = false; 515 * STARTING/DYING must not fail!
516 */
517 WARN_ON_ONCE(st->result);
459 } else { 518 } else {
460 /* Cannot happen .... */ 519 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
461 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 520 }
462 521
463 /* Regular hotplug work */ 522 if (st->result) {
464 if (st->state < st->target) 523 /*
465 ret = cpuhp_ap_online(cpu, st); 524 * If we fail on a rollback, we're up a creek without no
466 else if (st->state > st->target) 525 * paddle, no way forward, no way back. We loose, thanks for
467 ret = cpuhp_ap_offline(cpu, st); 526 * playing.
527 */
528 WARN_ON_ONCE(st->rollback);
529 st->should_run = false;
468 } 530 }
531
532next:
469 lock_map_release(&cpuhp_state_lock_map); 533 lock_map_release(&cpuhp_state_lock_map);
470 st->result = ret; 534
471 complete(&st->done); 535 if (!st->should_run)
536 complete(&st->done);
472} 537}
473 538
474/* Invoke a single callback on a remote cpu */ 539/* Invoke a single callback on a remote cpu */
@@ -477,6 +542,7 @@ cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
477 struct hlist_node *node) 542 struct hlist_node *node)
478{ 543{
479 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 544 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
545 int ret;
480 546
481 if (!cpu_online(cpu)) 547 if (!cpu_online(cpu))
482 return 0; 548 return 0;
@@ -491,48 +557,43 @@ cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
491 if (!st->thread) 557 if (!st->thread)
492 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL); 558 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
493 559
560 st->rollback = false;
561 st->last = NULL;
562
563 st->node = node;
564 st->bringup = bringup;
494 st->cb_state = state; 565 st->cb_state = state;
495 st->single = true; 566 st->single = true;
496 st->bringup = bringup;
497 st->node = node;
498 567
499 /* 568 __cpuhp_kick_ap(st);
500 * Make sure the above stores are visible before should_run becomes
501 * true. Paired with the mb() above in cpuhp_thread_fun()
502 */
503 smp_mb();
504 st->should_run = true;
505 wake_up_process(st->thread);
506 wait_for_completion(&st->done);
507 return st->result;
508}
509 569
510/* Regular hotplug invocation of the AP hotplug thread */
511static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
512{
513 st->result = 0;
514 st->single = false;
515 /* 570 /*
516 * Make sure the above stores are visible before should_run becomes 571 * If we failed and did a partial, do a rollback.
517 * true. Paired with the mb() above in cpuhp_thread_fun()
518 */ 572 */
519 smp_mb(); 573 if ((ret = st->result) && st->last) {
520 st->should_run = true; 574 st->rollback = true;
521 wake_up_process(st->thread); 575 st->bringup = !bringup;
576
577 __cpuhp_kick_ap(st);
578 }
579
580 return ret;
522} 581}
523 582
524static int cpuhp_kick_ap_work(unsigned int cpu) 583static int cpuhp_kick_ap_work(unsigned int cpu)
525{ 584{
526 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 585 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
527 enum cpuhp_state state = st->state; 586 enum cpuhp_state prev_state = st->state;
587 int ret;
528 588
529 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work);
530 lock_map_acquire(&cpuhp_state_lock_map); 589 lock_map_acquire(&cpuhp_state_lock_map);
531 lock_map_release(&cpuhp_state_lock_map); 590 lock_map_release(&cpuhp_state_lock_map);
532 __cpuhp_kick_ap_work(st); 591
533 wait_for_completion(&st->done); 592 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
534 trace_cpuhp_exit(cpu, st->state, state, st->result); 593 ret = cpuhp_kick_ap(st, st->target);
535 return st->result; 594 trace_cpuhp_exit(cpu, st->state, prev_state, ret);
595
596 return ret;
536} 597}
537 598
538static struct smp_hotplug_thread cpuhp_threads = { 599static struct smp_hotplug_thread cpuhp_threads = {
@@ -693,11 +754,32 @@ void cpuhp_report_idle_dead(void)
693 cpuhp_complete_idle_dead, st, 0); 754 cpuhp_complete_idle_dead, st, 0);
694} 755}
695 756
696#else 757static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
697#define takedown_cpu NULL 758{
698#endif 759 for (st->state++; st->state < st->target; st->state++) {
760 struct cpuhp_step *step = cpuhp_get_step(st->state);
699 761
700#ifdef CONFIG_HOTPLUG_CPU 762 if (!step->skip_onerr)
763 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
764 }
765}
766
767static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
768 enum cpuhp_state target)
769{
770 enum cpuhp_state prev_state = st->state;
771 int ret = 0;
772
773 for (; st->state > target; st->state--) {
774 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
775 if (ret) {
776 st->target = prev_state;
777 undo_cpu_down(cpu, st);
778 break;
779 }
780 }
781 return ret;
782}
701 783
702/* Requires cpu_add_remove_lock to be held */ 784/* Requires cpu_add_remove_lock to be held */
703static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 785static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
@@ -716,13 +798,13 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
716 798
717 cpuhp_tasks_frozen = tasks_frozen; 799 cpuhp_tasks_frozen = tasks_frozen;
718 800
719 prev_state = st->state; 801 prev_state = cpuhp_set_state(st, target);
720 st->target = target;
721 /* 802 /*
722 * If the current CPU state is in the range of the AP hotplug thread, 803 * If the current CPU state is in the range of the AP hotplug thread,
723 * then we need to kick the thread. 804 * then we need to kick the thread.
724 */ 805 */
725 if (st->state > CPUHP_TEARDOWN_CPU) { 806 if (st->state > CPUHP_TEARDOWN_CPU) {
807 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
726 ret = cpuhp_kick_ap_work(cpu); 808 ret = cpuhp_kick_ap_work(cpu);
727 /* 809 /*
728 * The AP side has done the error rollback already. Just 810 * The AP side has done the error rollback already. Just
@@ -737,6 +819,8 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
737 */ 819 */
738 if (st->state > CPUHP_TEARDOWN_CPU) 820 if (st->state > CPUHP_TEARDOWN_CPU)
739 goto out; 821 goto out;
822
823 st->target = target;
740 } 824 }
741 /* 825 /*
742 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 826 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
@@ -744,9 +828,8 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
744 */ 828 */
745 ret = cpuhp_down_callbacks(cpu, st, target); 829 ret = cpuhp_down_callbacks(cpu, st, target);
746 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) { 830 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
747 st->target = prev_state; 831 cpuhp_reset_state(st, prev_state);
748 st->rollback = true; 832 __cpuhp_kick_ap(st);
749 cpuhp_kick_ap_work(cpu);
750 } 833 }
751 834
752out: 835out:
@@ -771,11 +854,15 @@ out:
771 cpu_maps_update_done(); 854 cpu_maps_update_done();
772 return err; 855 return err;
773} 856}
857
774int cpu_down(unsigned int cpu) 858int cpu_down(unsigned int cpu)
775{ 859{
776 return do_cpu_down(cpu, CPUHP_OFFLINE); 860 return do_cpu_down(cpu, CPUHP_OFFLINE);
777} 861}
778EXPORT_SYMBOL(cpu_down); 862EXPORT_SYMBOL(cpu_down);
863
864#else
865#define takedown_cpu NULL
779#endif /*CONFIG_HOTPLUG_CPU*/ 866#endif /*CONFIG_HOTPLUG_CPU*/
780 867
781/** 868/**
@@ -846,7 +933,7 @@ static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
846 933
847 cpuhp_tasks_frozen = tasks_frozen; 934 cpuhp_tasks_frozen = tasks_frozen;
848 935
849 st->target = target; 936 cpuhp_set_state(st, target);
850 /* 937 /*
851 * If the current CPU state is in the range of the AP hotplug thread, 938 * If the current CPU state is in the range of the AP hotplug thread,
852 * then we need to kick the thread once more. 939 * then we need to kick the thread once more.
@@ -1313,6 +1400,10 @@ static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1313 struct cpuhp_step *sp = cpuhp_get_step(state); 1400 struct cpuhp_step *sp = cpuhp_get_step(state);
1314 int ret; 1401 int ret;
1315 1402
1403 /*
1404 * If there's nothing to do, we done.
1405 * Relies on the union for multi_instance.
1406 */
1316 if ((bringup && !sp->startup.single) || 1407 if ((bringup && !sp->startup.single) ||
1317 (!bringup && !sp->teardown.single)) 1408 (!bringup && !sp->teardown.single))
1318 return 0; 1409 return 0;