diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/locking/locktorture.c | 5 | ||||
| -rw-r--r-- | kernel/rcu/rcuperf.c | 5 | ||||
| -rw-r--r-- | kernel/rcu/rcutorture.c | 5 | ||||
| -rw-r--r-- | kernel/rcu/srcutree.c | 26 | ||||
| -rw-r--r-- | kernel/rcu/tree.c | 100 | ||||
| -rw-r--r-- | kernel/rcu/tree.h | 4 | ||||
| -rw-r--r-- | kernel/rcu/tree_plugin.h | 38 | ||||
| -rw-r--r-- | kernel/torture.c | 15 |
8 files changed, 157 insertions, 41 deletions
diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 8402b3349dca..57bef4fbfb31 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c | |||
| @@ -21,6 +21,9 @@ | |||
| 21 | * Davidlohr Bueso <dave@stgolabs.net> | 21 | * Davidlohr Bueso <dave@stgolabs.net> |
| 22 | * Based on kernel/rcu/torture.c. | 22 | * Based on kernel/rcu/torture.c. |
| 23 | */ | 23 | */ |
| 24 | |||
| 25 | #define pr_fmt(fmt) fmt | ||
| 26 | |||
| 24 | #include <linux/kernel.h> | 27 | #include <linux/kernel.h> |
| 25 | #include <linux/module.h> | 28 | #include <linux/module.h> |
| 26 | #include <linux/kthread.h> | 29 | #include <linux/kthread.h> |
| @@ -57,7 +60,7 @@ torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); | |||
| 57 | torture_param(int, stat_interval, 60, | 60 | torture_param(int, stat_interval, 60, |
| 58 | "Number of seconds between stats printk()s"); | 61 | "Number of seconds between stats printk()s"); |
| 59 | torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); | 62 | torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); |
| 60 | torture_param(bool, verbose, true, | 63 | torture_param(int, verbose, 1, |
| 61 | "Enable verbose debugging printk()s"); | 64 | "Enable verbose debugging printk()s"); |
| 62 | 65 | ||
| 63 | static char *torture_type = "spin_lock"; | 66 | static char *torture_type = "spin_lock"; |
diff --git a/kernel/rcu/rcuperf.c b/kernel/rcu/rcuperf.c index e232846516b3..df29119b2013 100644 --- a/kernel/rcu/rcuperf.c +++ b/kernel/rcu/rcuperf.c | |||
| @@ -19,6 +19,9 @@ | |||
| 19 | * | 19 | * |
| 20 | * Authors: Paul E. McKenney <paulmck@us.ibm.com> | 20 | * Authors: Paul E. McKenney <paulmck@us.ibm.com> |
| 21 | */ | 21 | */ |
| 22 | |||
| 23 | #define pr_fmt(fmt) fmt | ||
| 24 | |||
| 22 | #include <linux/types.h> | 25 | #include <linux/types.h> |
| 23 | #include <linux/kernel.h> | 26 | #include <linux/kernel.h> |
| 24 | #include <linux/init.h> | 27 | #include <linux/init.h> |
| @@ -88,7 +91,7 @@ torture_param(int, nreaders, -1, "Number of RCU reader threads"); | |||
| 88 | torture_param(int, nwriters, -1, "Number of RCU updater threads"); | 91 | torture_param(int, nwriters, -1, "Number of RCU updater threads"); |
| 89 | torture_param(bool, shutdown, !IS_ENABLED(MODULE), | 92 | torture_param(bool, shutdown, !IS_ENABLED(MODULE), |
| 90 | "Shutdown at end of performance tests."); | 93 | "Shutdown at end of performance tests."); |
| 91 | torture_param(bool, verbose, true, "Enable verbose debugging printk()s"); | 94 | torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); |
| 92 | torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); | 95 | torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); |
| 93 | 96 | ||
| 94 | static char *perf_type = "rcu"; | 97 | static char *perf_type = "rcu"; |
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 42fcb7f05fac..5604bfac8df4 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c | |||
| @@ -22,6 +22,9 @@ | |||
| 22 | * | 22 | * |
| 23 | * See also: Documentation/RCU/torture.txt | 23 | * See also: Documentation/RCU/torture.txt |
| 24 | */ | 24 | */ |
| 25 | |||
| 26 | #define pr_fmt(fmt) fmt | ||
| 27 | |||
| 25 | #include <linux/types.h> | 28 | #include <linux/types.h> |
| 26 | #include <linux/kernel.h> | 29 | #include <linux/kernel.h> |
| 27 | #include <linux/init.h> | 30 | #include <linux/init.h> |
| @@ -101,7 +104,7 @@ torture_param(int, test_boost_interval, 7, | |||
| 101 | "Interval between boost tests, seconds."); | 104 | "Interval between boost tests, seconds."); |
| 102 | torture_param(bool, test_no_idle_hz, true, | 105 | torture_param(bool, test_no_idle_hz, true, |
| 103 | "Test support for tickless idle CPUs"); | 106 | "Test support for tickless idle CPUs"); |
| 104 | torture_param(bool, verbose, true, | 107 | torture_param(int, verbose, 1, |
| 105 | "Enable verbose debugging printk()s"); | 108 | "Enable verbose debugging printk()s"); |
| 106 | 109 | ||
| 107 | static char *torture_type = "rcu"; | 110 | static char *torture_type = "rcu"; |
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index b4123d7a2cec..5a1a9a07b407 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c | |||
| @@ -641,6 +641,9 @@ static void srcu_funnel_exp_start(struct srcu_struct *sp, struct srcu_node *snp, | |||
| 641 | * period s. Losers must either ensure that their desired grace-period | 641 | * period s. Losers must either ensure that their desired grace-period |
| 642 | * number is recorded on at least their leaf srcu_node structure, or they | 642 | * number is recorded on at least their leaf srcu_node structure, or they |
| 643 | * must take steps to invoke their own callbacks. | 643 | * must take steps to invoke their own callbacks. |
| 644 | * | ||
| 645 | * Note that this function also does the work of srcu_funnel_exp_start(), | ||
| 646 | * in some cases by directly invoking it. | ||
| 644 | */ | 647 | */ |
| 645 | static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, | 648 | static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, |
| 646 | unsigned long s, bool do_norm) | 649 | unsigned long s, bool do_norm) |
| @@ -823,17 +826,17 @@ static void srcu_leak_callback(struct rcu_head *rhp) | |||
| 823 | * more than one CPU, this means that when "func()" is invoked, each CPU | 826 | * more than one CPU, this means that when "func()" is invoked, each CPU |
| 824 | * is guaranteed to have executed a full memory barrier since the end of | 827 | * is guaranteed to have executed a full memory barrier since the end of |
| 825 | * its last corresponding SRCU read-side critical section whose beginning | 828 | * its last corresponding SRCU read-side critical section whose beginning |
| 826 | * preceded the call to call_rcu(). It also means that each CPU executing | 829 | * preceded the call to call_srcu(). It also means that each CPU executing |
| 827 | * an SRCU read-side critical section that continues beyond the start of | 830 | * an SRCU read-side critical section that continues beyond the start of |
| 828 | * "func()" must have executed a memory barrier after the call_rcu() | 831 | * "func()" must have executed a memory barrier after the call_srcu() |
| 829 | * but before the beginning of that SRCU read-side critical section. | 832 | * but before the beginning of that SRCU read-side critical section. |
| 830 | * Note that these guarantees include CPUs that are offline, idle, or | 833 | * Note that these guarantees include CPUs that are offline, idle, or |
| 831 | * executing in user mode, as well as CPUs that are executing in the kernel. | 834 | * executing in user mode, as well as CPUs that are executing in the kernel. |
| 832 | * | 835 | * |
| 833 | * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the | 836 | * Furthermore, if CPU A invoked call_srcu() and CPU B invoked the |
| 834 | * resulting SRCU callback function "func()", then both CPU A and CPU | 837 | * resulting SRCU callback function "func()", then both CPU A and CPU |
| 835 | * B are guaranteed to execute a full memory barrier during the time | 838 | * B are guaranteed to execute a full memory barrier during the time |
| 836 | * interval between the call to call_rcu() and the invocation of "func()". | 839 | * interval between the call to call_srcu() and the invocation of "func()". |
| 837 | * This guarantee applies even if CPU A and CPU B are the same CPU (but | 840 | * This guarantee applies even if CPU A and CPU B are the same CPU (but |
| 838 | * again only if the system has more than one CPU). | 841 | * again only if the system has more than one CPU). |
| 839 | * | 842 | * |
| @@ -1268,11 +1271,11 @@ void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf) | |||
| 1268 | unsigned long l0, l1; | 1271 | unsigned long l0, l1; |
| 1269 | unsigned long u0, u1; | 1272 | unsigned long u0, u1; |
| 1270 | long c0, c1; | 1273 | long c0, c1; |
| 1271 | struct srcu_data *counts; | 1274 | struct srcu_data *sdp; |
| 1272 | 1275 | ||
| 1273 | counts = per_cpu_ptr(sp->sda, cpu); | 1276 | sdp = per_cpu_ptr(sp->sda, cpu); |
| 1274 | u0 = counts->srcu_unlock_count[!idx]; | 1277 | u0 = sdp->srcu_unlock_count[!idx]; |
| 1275 | u1 = counts->srcu_unlock_count[idx]; | 1278 | u1 = sdp->srcu_unlock_count[idx]; |
| 1276 | 1279 | ||
| 1277 | /* | 1280 | /* |
| 1278 | * Make sure that a lock is always counted if the corresponding | 1281 | * Make sure that a lock is always counted if the corresponding |
| @@ -1280,12 +1283,13 @@ void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf) | |||
| 1280 | */ | 1283 | */ |
| 1281 | smp_rmb(); | 1284 | smp_rmb(); |
| 1282 | 1285 | ||
| 1283 | l0 = counts->srcu_lock_count[!idx]; | 1286 | l0 = sdp->srcu_lock_count[!idx]; |
| 1284 | l1 = counts->srcu_lock_count[idx]; | 1287 | l1 = sdp->srcu_lock_count[idx]; |
| 1285 | 1288 | ||
| 1286 | c0 = l0 - u0; | 1289 | c0 = l0 - u0; |
| 1287 | c1 = l1 - u1; | 1290 | c1 = l1 - u1; |
| 1288 | pr_cont(" %d(%ld,%ld)", cpu, c0, c1); | 1291 | pr_cont(" %d(%ld,%ld %1p)", |
| 1292 | cpu, c0, c1, rcu_segcblist_head(&sdp->srcu_cblist)); | ||
| 1289 | s0 += c0; | 1293 | s0 += c0; |
| 1290 | s1 += c1; | 1294 | s1 += c1; |
| 1291 | } | 1295 | } |
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index aa7cade1b9f3..a9a4a260ea7d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
| @@ -1368,7 +1368,6 @@ static inline void panic_on_rcu_stall(void) | |||
| 1368 | static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) | 1368 | static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) |
| 1369 | { | 1369 | { |
| 1370 | int cpu; | 1370 | int cpu; |
| 1371 | long delta; | ||
| 1372 | unsigned long flags; | 1371 | unsigned long flags; |
| 1373 | unsigned long gpa; | 1372 | unsigned long gpa; |
| 1374 | unsigned long j; | 1373 | unsigned long j; |
| @@ -1381,18 +1380,6 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) | |||
| 1381 | if (rcu_cpu_stall_suppress) | 1380 | if (rcu_cpu_stall_suppress) |
| 1382 | return; | 1381 | return; |
| 1383 | 1382 | ||
| 1384 | /* Only let one CPU complain about others per time interval. */ | ||
| 1385 | |||
| 1386 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | ||
| 1387 | delta = jiffies - READ_ONCE(rsp->jiffies_stall); | ||
| 1388 | if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { | ||
| 1389 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
| 1390 | return; | ||
| 1391 | } | ||
| 1392 | WRITE_ONCE(rsp->jiffies_stall, | ||
| 1393 | jiffies + 3 * rcu_jiffies_till_stall_check() + 3); | ||
| 1394 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
| 1395 | |||
| 1396 | /* | 1383 | /* |
| 1397 | * OK, time to rat on our buddy... | 1384 | * OK, time to rat on our buddy... |
| 1398 | * See Documentation/RCU/stallwarn.txt for info on how to debug | 1385 | * See Documentation/RCU/stallwarn.txt for info on how to debug |
| @@ -1441,6 +1428,10 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) | |||
| 1441 | sched_show_task(current); | 1428 | sched_show_task(current); |
| 1442 | } | 1429 | } |
| 1443 | } | 1430 | } |
| 1431 | /* Rewrite if needed in case of slow consoles. */ | ||
| 1432 | if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) | ||
| 1433 | WRITE_ONCE(rsp->jiffies_stall, | ||
| 1434 | jiffies + 3 * rcu_jiffies_till_stall_check() + 3); | ||
| 1444 | 1435 | ||
| 1445 | rcu_check_gp_kthread_starvation(rsp); | 1436 | rcu_check_gp_kthread_starvation(rsp); |
| 1446 | 1437 | ||
| @@ -1485,6 +1476,7 @@ static void print_cpu_stall(struct rcu_state *rsp) | |||
| 1485 | rcu_dump_cpu_stacks(rsp); | 1476 | rcu_dump_cpu_stacks(rsp); |
| 1486 | 1477 | ||
| 1487 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | 1478 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 1479 | /* Rewrite if needed in case of slow consoles. */ | ||
| 1488 | if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) | 1480 | if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) |
| 1489 | WRITE_ONCE(rsp->jiffies_stall, | 1481 | WRITE_ONCE(rsp->jiffies_stall, |
| 1490 | jiffies + 3 * rcu_jiffies_till_stall_check() + 3); | 1482 | jiffies + 3 * rcu_jiffies_till_stall_check() + 3); |
| @@ -1508,6 +1500,7 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) | |||
| 1508 | unsigned long gpnum; | 1500 | unsigned long gpnum; |
| 1509 | unsigned long gps; | 1501 | unsigned long gps; |
| 1510 | unsigned long j; | 1502 | unsigned long j; |
| 1503 | unsigned long jn; | ||
| 1511 | unsigned long js; | 1504 | unsigned long js; |
| 1512 | struct rcu_node *rnp; | 1505 | struct rcu_node *rnp; |
| 1513 | 1506 | ||
| @@ -1546,14 +1539,17 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) | |||
| 1546 | ULONG_CMP_GE(gps, js)) | 1539 | ULONG_CMP_GE(gps, js)) |
| 1547 | return; /* No stall or GP completed since entering function. */ | 1540 | return; /* No stall or GP completed since entering function. */ |
| 1548 | rnp = rdp->mynode; | 1541 | rnp = rdp->mynode; |
| 1542 | jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; | ||
| 1549 | if (rcu_gp_in_progress(rsp) && | 1543 | if (rcu_gp_in_progress(rsp) && |
| 1550 | (READ_ONCE(rnp->qsmask) & rdp->grpmask)) { | 1544 | (READ_ONCE(rnp->qsmask) & rdp->grpmask) && |
| 1545 | cmpxchg(&rsp->jiffies_stall, js, jn) == js) { | ||
| 1551 | 1546 | ||
| 1552 | /* We haven't checked in, so go dump stack. */ | 1547 | /* We haven't checked in, so go dump stack. */ |
| 1553 | print_cpu_stall(rsp); | 1548 | print_cpu_stall(rsp); |
| 1554 | 1549 | ||
| 1555 | } else if (rcu_gp_in_progress(rsp) && | 1550 | } else if (rcu_gp_in_progress(rsp) && |
| 1556 | ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) { | 1551 | ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) && |
| 1552 | cmpxchg(&rsp->jiffies_stall, js, jn) == js) { | ||
| 1557 | 1553 | ||
| 1558 | /* They had a few time units to dump stack, so complain. */ | 1554 | /* They had a few time units to dump stack, so complain. */ |
| 1559 | print_other_cpu_stall(rsp, gpnum); | 1555 | print_other_cpu_stall(rsp, gpnum); |
| @@ -1685,6 +1681,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, | |||
| 1685 | } | 1681 | } |
| 1686 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedroot")); | 1682 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedroot")); |
| 1687 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT); | 1683 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT); |
| 1684 | rsp->gp_req_activity = jiffies; | ||
| 1688 | if (!rsp->gp_kthread) { | 1685 | if (!rsp->gp_kthread) { |
| 1689 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread")); | 1686 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread")); |
| 1690 | goto unlock_out; | 1687 | goto unlock_out; |
| @@ -2084,7 +2081,8 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) | |||
| 2084 | */ | 2081 | */ |
| 2085 | rcu_for_each_node_breadth_first(rsp, rnp) { | 2082 | rcu_for_each_node_breadth_first(rsp, rnp) { |
| 2086 | raw_spin_lock_irq_rcu_node(rnp); | 2083 | raw_spin_lock_irq_rcu_node(rnp); |
| 2087 | WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); | 2084 | if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp))) |
| 2085 | dump_blkd_tasks(rnp, 10); | ||
| 2088 | WARN_ON_ONCE(rnp->qsmask); | 2086 | WARN_ON_ONCE(rnp->qsmask); |
| 2089 | WRITE_ONCE(rnp->completed, rsp->gpnum); | 2087 | WRITE_ONCE(rnp->completed, rsp->gpnum); |
| 2090 | rdp = this_cpu_ptr(rsp->rda); | 2088 | rdp = this_cpu_ptr(rsp->rda); |
| @@ -2116,10 +2114,12 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) | |||
| 2116 | /* Advance CBs to reduce false positives below. */ | 2114 | /* Advance CBs to reduce false positives below. */ |
| 2117 | if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) { | 2115 | if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) { |
| 2118 | WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); | 2116 | WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); |
| 2117 | rsp->gp_req_activity = jiffies; | ||
| 2119 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), | 2118 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), |
| 2120 | TPS("newreq")); | 2119 | TPS("newreq")); |
| 2120 | } else { | ||
| 2121 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT); | ||
| 2121 | } | 2122 | } |
| 2122 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT); | ||
| 2123 | raw_spin_unlock_irq_rcu_node(rnp); | 2123 | raw_spin_unlock_irq_rcu_node(rnp); |
| 2124 | } | 2124 | } |
| 2125 | 2125 | ||
| @@ -2294,6 +2294,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, | |||
| 2294 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | 2294 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| 2295 | return; | 2295 | return; |
| 2296 | } | 2296 | } |
| 2297 | rnp->completedqs = rnp->gpnum; | ||
| 2297 | mask = rnp->grpmask; | 2298 | mask = rnp->grpmask; |
| 2298 | if (rnp->parent == NULL) { | 2299 | if (rnp->parent == NULL) { |
| 2299 | 2300 | ||
| @@ -2747,6 +2748,65 @@ static void force_quiescent_state(struct rcu_state *rsp) | |||
| 2747 | } | 2748 | } |
| 2748 | 2749 | ||
| 2749 | /* | 2750 | /* |
| 2751 | * This function checks for grace-period requests that fail to motivate | ||
| 2752 | * RCU to come out of its idle mode. | ||
| 2753 | */ | ||
| 2754 | static void | ||
| 2755 | rcu_check_gp_start_stall(struct rcu_state *rsp, struct rcu_node *rnp, | ||
| 2756 | struct rcu_data *rdp) | ||
| 2757 | { | ||
| 2758 | unsigned long flags; | ||
| 2759 | unsigned long j; | ||
| 2760 | struct rcu_node *rnp_root = rcu_get_root(rsp); | ||
| 2761 | static atomic_t warned = ATOMIC_INIT(0); | ||
| 2762 | |||
| 2763 | if (!IS_ENABLED(CONFIG_PROVE_RCU) || | ||
| 2764 | rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp))) | ||
| 2765 | return; | ||
| 2766 | j = jiffies; /* Expensive access, and in common case don't get here. */ | ||
| 2767 | if (time_before(j, READ_ONCE(rsp->gp_req_activity) + HZ) || | ||
| 2768 | time_before(j, READ_ONCE(rsp->gp_activity) + HZ) || | ||
| 2769 | atomic_read(&warned)) | ||
| 2770 | return; | ||
| 2771 | |||
| 2772 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | ||
| 2773 | j = jiffies; | ||
| 2774 | if (rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)) || | ||
| 2775 | time_before(j, READ_ONCE(rsp->gp_req_activity) + HZ) || | ||
| 2776 | time_before(j, READ_ONCE(rsp->gp_activity) + HZ) || | ||
| 2777 | atomic_read(&warned)) { | ||
| 2778 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
| 2779 | return; | ||
| 2780 | } | ||
| 2781 | /* Hold onto the leaf lock to make others see warned==1. */ | ||
| 2782 | |||
| 2783 | if (rnp_root != rnp) | ||
| 2784 | raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */ | ||
| 2785 | j = jiffies; | ||
| 2786 | if (rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)) || | ||
| 2787 | time_before(j, rsp->gp_req_activity + HZ) || | ||
| 2788 | time_before(j, rsp->gp_activity + HZ) || | ||
| 2789 | atomic_xchg(&warned, 1)) { | ||
| 2790 | raw_spin_unlock_rcu_node(rnp_root); /* irqs remain disabled. */ | ||
| 2791 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
| 2792 | return; | ||
| 2793 | } | ||
| 2794 | pr_alert("%s: g%lu %d%d%d%d gar:%lu ga:%lu f%#x %s->state:%#lx\n", | ||
| 2795 | __func__, READ_ONCE(rsp->gpnum), | ||
| 2796 | need_future_gp_element(rcu_get_root(rsp), 0), | ||
| 2797 | need_future_gp_element(rcu_get_root(rsp), 1), | ||
| 2798 | need_future_gp_element(rcu_get_root(rsp), 2), | ||
| 2799 | need_future_gp_element(rcu_get_root(rsp), 3), | ||
| 2800 | j - rsp->gp_req_activity, j - rsp->gp_activity, | ||
| 2801 | rsp->gp_flags, rsp->name, | ||
| 2802 | rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL); | ||
| 2803 | WARN_ON(1); | ||
| 2804 | if (rnp_root != rnp) | ||
| 2805 | raw_spin_unlock_rcu_node(rnp_root); | ||
| 2806 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
| 2807 | } | ||
| 2808 | |||
| 2809 | /* | ||
| 2750 | * This does the RCU core processing work for the specified rcu_state | 2810 | * This does the RCU core processing work for the specified rcu_state |
| 2751 | * and rcu_data structures. This may be called only from the CPU to | 2811 | * and rcu_data structures. This may be called only from the CPU to |
| 2752 | * whom the rdp belongs. | 2812 | * whom the rdp belongs. |
| @@ -2757,7 +2817,7 @@ __rcu_process_callbacks(struct rcu_state *rsp) | |||
| 2757 | unsigned long flags; | 2817 | unsigned long flags; |
| 2758 | bool needwake; | 2818 | bool needwake; |
| 2759 | struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); | 2819 | struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); |
| 2760 | struct rcu_node *rnp; | 2820 | struct rcu_node *rnp = rdp->mynode; |
| 2761 | 2821 | ||
| 2762 | WARN_ON_ONCE(!rdp->beenonline); | 2822 | WARN_ON_ONCE(!rdp->beenonline); |
| 2763 | 2823 | ||
| @@ -2771,7 +2831,6 @@ __rcu_process_callbacks(struct rcu_state *rsp) | |||
| 2771 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) { | 2831 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) { |
| 2772 | local_irq_restore(flags); | 2832 | local_irq_restore(flags); |
| 2773 | } else { | 2833 | } else { |
| 2774 | rnp = rdp->mynode; | ||
| 2775 | raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ | 2834 | raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ |
| 2776 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); | 2835 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); |
| 2777 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | 2836 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
| @@ -2780,6 +2839,8 @@ __rcu_process_callbacks(struct rcu_state *rsp) | |||
| 2780 | } | 2839 | } |
| 2781 | } | 2840 | } |
| 2782 | 2841 | ||
| 2842 | rcu_check_gp_start_stall(rsp, rnp, rdp); | ||
| 2843 | |||
| 2783 | /* If there are callbacks ready, invoke them. */ | 2844 | /* If there are callbacks ready, invoke them. */ |
| 2784 | if (rcu_segcblist_ready_cbs(&rdp->cblist)) | 2845 | if (rcu_segcblist_ready_cbs(&rdp->cblist)) |
| 2785 | invoke_rcu_callbacks(rsp, rdp); | 2846 | invoke_rcu_callbacks(rsp, rdp); |
| @@ -3930,6 +3991,7 @@ static void __init rcu_init_one(struct rcu_state *rsp) | |||
| 3930 | &rcu_fqs_class[i], fqs[i]); | 3991 | &rcu_fqs_class[i], fqs[i]); |
| 3931 | rnp->gpnum = rsp->gpnum; | 3992 | rnp->gpnum = rsp->gpnum; |
| 3932 | rnp->completed = rsp->completed; | 3993 | rnp->completed = rsp->completed; |
| 3994 | rnp->completedqs = rsp->completed; | ||
| 3933 | rnp->qsmask = 0; | 3995 | rnp->qsmask = 0; |
| 3934 | rnp->qsmaskinit = 0; | 3996 | rnp->qsmaskinit = 0; |
| 3935 | rnp->grplo = j * cpustride; | 3997 | rnp->grplo = j * cpustride; |
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 78e051dffc5b..3c1942174c56 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h | |||
| @@ -87,6 +87,7 @@ struct rcu_node { | |||
| 87 | unsigned long completed; /* Last GP completed for this node. */ | 87 | unsigned long completed; /* Last GP completed for this node. */ |
| 88 | /* This will either be equal to or one */ | 88 | /* This will either be equal to or one */ |
| 89 | /* behind the root rcu_node's gpnum. */ | 89 | /* behind the root rcu_node's gpnum. */ |
| 90 | unsigned long completedqs; /* All QSes done for this node. */ | ||
| 90 | unsigned long qsmask; /* CPUs or groups that need to switch in */ | 91 | unsigned long qsmask; /* CPUs or groups that need to switch in */ |
| 91 | /* order for current grace period to proceed.*/ | 92 | /* order for current grace period to proceed.*/ |
| 92 | /* In leaf rcu_node, each bit corresponds to */ | 93 | /* In leaf rcu_node, each bit corresponds to */ |
| @@ -373,6 +374,8 @@ struct rcu_state { | |||
| 373 | /* but in jiffies. */ | 374 | /* but in jiffies. */ |
| 374 | unsigned long gp_activity; /* Time of last GP kthread */ | 375 | unsigned long gp_activity; /* Time of last GP kthread */ |
| 375 | /* activity in jiffies. */ | 376 | /* activity in jiffies. */ |
| 377 | unsigned long gp_req_activity; /* Time of last GP request */ | ||
| 378 | /* in jiffies. */ | ||
| 376 | unsigned long jiffies_stall; /* Time at which to check */ | 379 | unsigned long jiffies_stall; /* Time at which to check */ |
| 377 | /* for CPU stalls. */ | 380 | /* for CPU stalls. */ |
| 378 | unsigned long jiffies_resched; /* Time at which to resched */ | 381 | unsigned long jiffies_resched; /* Time at which to resched */ |
| @@ -453,6 +456,7 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); | |||
| 453 | static void rcu_preempt_check_callbacks(void); | 456 | static void rcu_preempt_check_callbacks(void); |
| 454 | void call_rcu(struct rcu_head *head, rcu_callback_t func); | 457 | void call_rcu(struct rcu_head *head, rcu_callback_t func); |
| 455 | static void __init __rcu_init_preempt(void); | 458 | static void __init __rcu_init_preempt(void); |
| 459 | static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); | ||
| 456 | static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); | 460 | static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); |
| 457 | static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); | 461 | static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); |
| 458 | static void invoke_rcu_callbacks_kthread(void); | 462 | static void invoke_rcu_callbacks_kthread(void); |
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 7fd12039e512..e387ea712758 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
| @@ -260,8 +260,10 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) | |||
| 260 | * ->exp_tasks pointers, respectively, to reference the newly | 260 | * ->exp_tasks pointers, respectively, to reference the newly |
| 261 | * blocked tasks. | 261 | * blocked tasks. |
| 262 | */ | 262 | */ |
| 263 | if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) | 263 | if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) { |
| 264 | rnp->gp_tasks = &t->rcu_node_entry; | 264 | rnp->gp_tasks = &t->rcu_node_entry; |
| 265 | WARN_ON_ONCE(rnp->completedqs == rnp->gpnum); | ||
| 266 | } | ||
| 265 | if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) | 267 | if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) |
| 266 | rnp->exp_tasks = &t->rcu_node_entry; | 268 | rnp->exp_tasks = &t->rcu_node_entry; |
| 267 | WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) != | 269 | WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) != |
| @@ -535,6 +537,8 @@ void rcu_read_unlock_special(struct task_struct *t) | |||
| 535 | WARN_ON_ONCE(rnp != t->rcu_blocked_node); | 537 | WARN_ON_ONCE(rnp != t->rcu_blocked_node); |
| 536 | WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); | 538 | WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); |
| 537 | empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); | 539 | empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); |
| 540 | WARN_ON_ONCE(rnp->completedqs == rnp->gpnum && | ||
| 541 | (!empty_norm || rnp->qsmask)); | ||
| 538 | empty_exp = sync_rcu_preempt_exp_done(rnp); | 542 | empty_exp = sync_rcu_preempt_exp_done(rnp); |
| 539 | smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ | 543 | smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ |
| 540 | np = rcu_next_node_entry(t, rnp); | 544 | np = rcu_next_node_entry(t, rnp); |
| @@ -697,7 +701,8 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) | |||
| 697 | struct task_struct *t; | 701 | struct task_struct *t; |
| 698 | 702 | ||
| 699 | RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n"); | 703 | RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n"); |
| 700 | WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); | 704 | if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp))) |
| 705 | dump_blkd_tasks(rnp, 10); | ||
| 701 | if (rcu_preempt_has_tasks(rnp)) { | 706 | if (rcu_preempt_has_tasks(rnp)) { |
| 702 | rnp->gp_tasks = rnp->blkd_tasks.next; | 707 | rnp->gp_tasks = rnp->blkd_tasks.next; |
| 703 | t = container_of(rnp->gp_tasks, struct task_struct, | 708 | t = container_of(rnp->gp_tasks, struct task_struct, |
| @@ -841,6 +846,27 @@ void exit_rcu(void) | |||
| 841 | __rcu_read_unlock(); | 846 | __rcu_read_unlock(); |
| 842 | } | 847 | } |
| 843 | 848 | ||
| 849 | /* | ||
| 850 | * Dump the blocked-tasks state, but limit the list dump to the | ||
| 851 | * specified number of elements. | ||
| 852 | */ | ||
| 853 | static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck) | ||
| 854 | { | ||
| 855 | int i; | ||
| 856 | struct list_head *lhp; | ||
| 857 | |||
| 858 | raw_lockdep_assert_held_rcu_node(rnp); | ||
| 859 | pr_info("%s: grp: %d-%d level: %d ->qamask %#lx ->gp_tasks %p ->boost_tasks %p ->exp_tasks %p &->blkd_tasks: %p offset: %u\n", __func__, rnp->grplo, rnp->grphi, rnp->level, rnp->qsmask, rnp->gp_tasks, rnp->boost_tasks, rnp->exp_tasks, &rnp->blkd_tasks, (unsigned int)offsetof(typeof(*rnp), blkd_tasks)); | ||
| 860 | pr_cont("\t->blkd_tasks"); | ||
| 861 | i = 0; | ||
| 862 | list_for_each(lhp, &rnp->blkd_tasks) { | ||
| 863 | pr_cont(" %p", lhp); | ||
| 864 | if (++i >= 10) | ||
| 865 | break; | ||
| 866 | } | ||
| 867 | pr_cont("\n"); | ||
| 868 | } | ||
| 869 | |||
| 844 | #else /* #ifdef CONFIG_PREEMPT_RCU */ | 870 | #else /* #ifdef CONFIG_PREEMPT_RCU */ |
| 845 | 871 | ||
| 846 | static struct rcu_state *const rcu_state_p = &rcu_sched_state; | 872 | static struct rcu_state *const rcu_state_p = &rcu_sched_state; |
| @@ -949,6 +975,14 @@ void exit_rcu(void) | |||
| 949 | { | 975 | { |
| 950 | } | 976 | } |
| 951 | 977 | ||
| 978 | /* | ||
| 979 | * Dump the guaranteed-empty blocked-tasks state. Trust but verify. | ||
| 980 | */ | ||
| 981 | static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck) | ||
| 982 | { | ||
| 983 | WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks)); | ||
| 984 | } | ||
| 985 | |||
| 952 | #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ | 986 | #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ |
| 953 | 987 | ||
| 954 | #ifdef CONFIG_RCU_BOOST | 988 | #ifdef CONFIG_RCU_BOOST |
diff --git a/kernel/torture.c b/kernel/torture.c index 3de1efbecd6a..1ac24a826589 100644 --- a/kernel/torture.c +++ b/kernel/torture.c | |||
| @@ -20,6 +20,9 @@ | |||
| 20 | * Author: Paul E. McKenney <paulmck@us.ibm.com> | 20 | * Author: Paul E. McKenney <paulmck@us.ibm.com> |
| 21 | * Based on kernel/rcu/torture.c. | 21 | * Based on kernel/rcu/torture.c. |
| 22 | */ | 22 | */ |
| 23 | |||
| 24 | #define pr_fmt(fmt) fmt | ||
| 25 | |||
| 23 | #include <linux/types.h> | 26 | #include <linux/types.h> |
| 24 | #include <linux/kernel.h> | 27 | #include <linux/kernel.h> |
| 25 | #include <linux/init.h> | 28 | #include <linux/init.h> |
| @@ -53,7 +56,7 @@ MODULE_LICENSE("GPL"); | |||
| 53 | MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>"); | 56 | MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>"); |
| 54 | 57 | ||
| 55 | static char *torture_type; | 58 | static char *torture_type; |
| 56 | static bool verbose; | 59 | static int verbose; |
| 57 | 60 | ||
| 58 | /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ | 61 | /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ |
| 59 | #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ | 62 | #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ |
| @@ -98,7 +101,7 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, | |||
| 98 | if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) | 101 | if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) |
| 99 | return false; | 102 | return false; |
| 100 | 103 | ||
| 101 | if (verbose) | 104 | if (verbose > 1) |
| 102 | pr_alert("%s" TORTURE_FLAG | 105 | pr_alert("%s" TORTURE_FLAG |
| 103 | "torture_onoff task: offlining %d\n", | 106 | "torture_onoff task: offlining %d\n", |
| 104 | torture_type, cpu); | 107 | torture_type, cpu); |
| @@ -111,7 +114,7 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, | |||
| 111 | "torture_onoff task: offline %d failed: errno %d\n", | 114 | "torture_onoff task: offline %d failed: errno %d\n", |
| 112 | torture_type, cpu, ret); | 115 | torture_type, cpu, ret); |
| 113 | } else { | 116 | } else { |
| 114 | if (verbose) | 117 | if (verbose > 1) |
| 115 | pr_alert("%s" TORTURE_FLAG | 118 | pr_alert("%s" TORTURE_FLAG |
| 116 | "torture_onoff task: offlined %d\n", | 119 | "torture_onoff task: offlined %d\n", |
| 117 | torture_type, cpu); | 120 | torture_type, cpu); |
| @@ -147,7 +150,7 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, | |||
| 147 | if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) | 150 | if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) |
| 148 | return false; | 151 | return false; |
| 149 | 152 | ||
| 150 | if (verbose) | 153 | if (verbose > 1) |
| 151 | pr_alert("%s" TORTURE_FLAG | 154 | pr_alert("%s" TORTURE_FLAG |
| 152 | "torture_onoff task: onlining %d\n", | 155 | "torture_onoff task: onlining %d\n", |
| 153 | torture_type, cpu); | 156 | torture_type, cpu); |
| @@ -160,7 +163,7 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, | |||
| 160 | "torture_onoff task: online %d failed: errno %d\n", | 163 | "torture_onoff task: online %d failed: errno %d\n", |
| 161 | torture_type, cpu, ret); | 164 | torture_type, cpu, ret); |
| 162 | } else { | 165 | } else { |
| 163 | if (verbose) | 166 | if (verbose > 1) |
| 164 | pr_alert("%s" TORTURE_FLAG | 167 | pr_alert("%s" TORTURE_FLAG |
| 165 | "torture_onoff task: onlined %d\n", | 168 | "torture_onoff task: onlined %d\n", |
| 166 | torture_type, cpu); | 169 | torture_type, cpu); |
| @@ -647,7 +650,7 @@ static void torture_stutter_cleanup(void) | |||
| 647 | * The runnable parameter points to a flag that controls whether or not | 650 | * The runnable parameter points to a flag that controls whether or not |
| 648 | * the test is currently runnable. If there is no such flag, pass in NULL. | 651 | * the test is currently runnable. If there is no such flag, pass in NULL. |
| 649 | */ | 652 | */ |
| 650 | bool torture_init_begin(char *ttype, bool v) | 653 | bool torture_init_begin(char *ttype, int v) |
| 651 | { | 654 | { |
| 652 | mutex_lock(&fullstop_mutex); | 655 | mutex_lock(&fullstop_mutex); |
| 653 | if (torture_type != NULL) { | 656 | if (torture_type != NULL) { |
