aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/locking/locktorture.c5
-rw-r--r--kernel/rcu/rcuperf.c5
-rw-r--r--kernel/rcu/rcutorture.c5
-rw-r--r--kernel/rcu/srcutree.c26
-rw-r--r--kernel/rcu/tree.c100
-rw-r--r--kernel/rcu/tree.h4
-rw-r--r--kernel/rcu/tree_plugin.h38
-rw-r--r--kernel/torture.c15
8 files changed, 157 insertions, 41 deletions
diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c
index 8402b3349dca..57bef4fbfb31 100644
--- a/kernel/locking/locktorture.c
+++ b/kernel/locking/locktorture.c
@@ -21,6 +21,9 @@
21 * Davidlohr Bueso <dave@stgolabs.net> 21 * Davidlohr Bueso <dave@stgolabs.net>
22 * Based on kernel/rcu/torture.c. 22 * Based on kernel/rcu/torture.c.
23 */ 23 */
24
25#define pr_fmt(fmt) fmt
26
24#include <linux/kernel.h> 27#include <linux/kernel.h>
25#include <linux/module.h> 28#include <linux/module.h>
26#include <linux/kthread.h> 29#include <linux/kthread.h>
@@ -57,7 +60,7 @@ torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
57torture_param(int, stat_interval, 60, 60torture_param(int, stat_interval, 60,
58 "Number of seconds between stats printk()s"); 61 "Number of seconds between stats printk()s");
59torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); 62torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
60torture_param(bool, verbose, true, 63torture_param(int, verbose, 1,
61 "Enable verbose debugging printk()s"); 64 "Enable verbose debugging printk()s");
62 65
63static char *torture_type = "spin_lock"; 66static char *torture_type = "spin_lock";
diff --git a/kernel/rcu/rcuperf.c b/kernel/rcu/rcuperf.c
index e232846516b3..df29119b2013 100644
--- a/kernel/rcu/rcuperf.c
+++ b/kernel/rcu/rcuperf.c
@@ -19,6 +19,9 @@
19 * 19 *
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
21 */ 21 */
22
23#define pr_fmt(fmt) fmt
24
22#include <linux/types.h> 25#include <linux/types.h>
23#include <linux/kernel.h> 26#include <linux/kernel.h>
24#include <linux/init.h> 27#include <linux/init.h>
@@ -88,7 +91,7 @@ torture_param(int, nreaders, -1, "Number of RCU reader threads");
88torture_param(int, nwriters, -1, "Number of RCU updater threads"); 91torture_param(int, nwriters, -1, "Number of RCU updater threads");
89torture_param(bool, shutdown, !IS_ENABLED(MODULE), 92torture_param(bool, shutdown, !IS_ENABLED(MODULE),
90 "Shutdown at end of performance tests."); 93 "Shutdown at end of performance tests.");
91torture_param(bool, verbose, true, "Enable verbose debugging printk()s"); 94torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
92torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); 95torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable");
93 96
94static char *perf_type = "rcu"; 97static char *perf_type = "rcu";
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 42fcb7f05fac..5604bfac8df4 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -22,6 +22,9 @@
22 * 22 *
23 * See also: Documentation/RCU/torture.txt 23 * See also: Documentation/RCU/torture.txt
24 */ 24 */
25
26#define pr_fmt(fmt) fmt
27
25#include <linux/types.h> 28#include <linux/types.h>
26#include <linux/kernel.h> 29#include <linux/kernel.h>
27#include <linux/init.h> 30#include <linux/init.h>
@@ -101,7 +104,7 @@ torture_param(int, test_boost_interval, 7,
101 "Interval between boost tests, seconds."); 104 "Interval between boost tests, seconds.");
102torture_param(bool, test_no_idle_hz, true, 105torture_param(bool, test_no_idle_hz, true,
103 "Test support for tickless idle CPUs"); 106 "Test support for tickless idle CPUs");
104torture_param(bool, verbose, true, 107torture_param(int, verbose, 1,
105 "Enable verbose debugging printk()s"); 108 "Enable verbose debugging printk()s");
106 109
107static char *torture_type = "rcu"; 110static char *torture_type = "rcu";
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index b4123d7a2cec..5a1a9a07b407 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -641,6 +641,9 @@ static void srcu_funnel_exp_start(struct srcu_struct *sp, struct srcu_node *snp,
641 * period s. Losers must either ensure that their desired grace-period 641 * period s. Losers must either ensure that their desired grace-period
642 * number is recorded on at least their leaf srcu_node structure, or they 642 * number is recorded on at least their leaf srcu_node structure, or they
643 * must take steps to invoke their own callbacks. 643 * must take steps to invoke their own callbacks.
644 *
645 * Note that this function also does the work of srcu_funnel_exp_start(),
646 * in some cases by directly invoking it.
644 */ 647 */
645static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, 648static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,
646 unsigned long s, bool do_norm) 649 unsigned long s, bool do_norm)
@@ -823,17 +826,17 @@ static void srcu_leak_callback(struct rcu_head *rhp)
823 * more than one CPU, this means that when "func()" is invoked, each CPU 826 * more than one CPU, this means that when "func()" is invoked, each CPU
824 * is guaranteed to have executed a full memory barrier since the end of 827 * is guaranteed to have executed a full memory barrier since the end of
825 * its last corresponding SRCU read-side critical section whose beginning 828 * its last corresponding SRCU read-side critical section whose beginning
826 * preceded the call to call_rcu(). It also means that each CPU executing 829 * preceded the call to call_srcu(). It also means that each CPU executing
827 * an SRCU read-side critical section that continues beyond the start of 830 * an SRCU read-side critical section that continues beyond the start of
828 * "func()" must have executed a memory barrier after the call_rcu() 831 * "func()" must have executed a memory barrier after the call_srcu()
829 * but before the beginning of that SRCU read-side critical section. 832 * but before the beginning of that SRCU read-side critical section.
830 * Note that these guarantees include CPUs that are offline, idle, or 833 * Note that these guarantees include CPUs that are offline, idle, or
831 * executing in user mode, as well as CPUs that are executing in the kernel. 834 * executing in user mode, as well as CPUs that are executing in the kernel.
832 * 835 *
833 * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the 836 * Furthermore, if CPU A invoked call_srcu() and CPU B invoked the
834 * resulting SRCU callback function "func()", then both CPU A and CPU 837 * resulting SRCU callback function "func()", then both CPU A and CPU
835 * B are guaranteed to execute a full memory barrier during the time 838 * B are guaranteed to execute a full memory barrier during the time
836 * interval between the call to call_rcu() and the invocation of "func()". 839 * interval between the call to call_srcu() and the invocation of "func()".
837 * This guarantee applies even if CPU A and CPU B are the same CPU (but 840 * This guarantee applies even if CPU A and CPU B are the same CPU (but
838 * again only if the system has more than one CPU). 841 * again only if the system has more than one CPU).
839 * 842 *
@@ -1268,11 +1271,11 @@ void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf)
1268 unsigned long l0, l1; 1271 unsigned long l0, l1;
1269 unsigned long u0, u1; 1272 unsigned long u0, u1;
1270 long c0, c1; 1273 long c0, c1;
1271 struct srcu_data *counts; 1274 struct srcu_data *sdp;
1272 1275
1273 counts = per_cpu_ptr(sp->sda, cpu); 1276 sdp = per_cpu_ptr(sp->sda, cpu);
1274 u0 = counts->srcu_unlock_count[!idx]; 1277 u0 = sdp->srcu_unlock_count[!idx];
1275 u1 = counts->srcu_unlock_count[idx]; 1278 u1 = sdp->srcu_unlock_count[idx];
1276 1279
1277 /* 1280 /*
1278 * Make sure that a lock is always counted if the corresponding 1281 * Make sure that a lock is always counted if the corresponding
@@ -1280,12 +1283,13 @@ void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf)
1280 */ 1283 */
1281 smp_rmb(); 1284 smp_rmb();
1282 1285
1283 l0 = counts->srcu_lock_count[!idx]; 1286 l0 = sdp->srcu_lock_count[!idx];
1284 l1 = counts->srcu_lock_count[idx]; 1287 l1 = sdp->srcu_lock_count[idx];
1285 1288
1286 c0 = l0 - u0; 1289 c0 = l0 - u0;
1287 c1 = l1 - u1; 1290 c1 = l1 - u1;
1288 pr_cont(" %d(%ld,%ld)", cpu, c0, c1); 1291 pr_cont(" %d(%ld,%ld %1p)",
1292 cpu, c0, c1, rcu_segcblist_head(&sdp->srcu_cblist));
1289 s0 += c0; 1293 s0 += c0;
1290 s1 += c1; 1294 s1 += c1;
1291 } 1295 }
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index aa7cade1b9f3..a9a4a260ea7d 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1368,7 +1368,6 @@ static inline void panic_on_rcu_stall(void)
1368static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum) 1368static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
1369{ 1369{
1370 int cpu; 1370 int cpu;
1371 long delta;
1372 unsigned long flags; 1371 unsigned long flags;
1373 unsigned long gpa; 1372 unsigned long gpa;
1374 unsigned long j; 1373 unsigned long j;
@@ -1381,18 +1380,6 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
1381 if (rcu_cpu_stall_suppress) 1380 if (rcu_cpu_stall_suppress)
1382 return; 1381 return;
1383 1382
1384 /* Only let one CPU complain about others per time interval. */
1385
1386 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1387 delta = jiffies - READ_ONCE(rsp->jiffies_stall);
1388 if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) {
1389 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1390 return;
1391 }
1392 WRITE_ONCE(rsp->jiffies_stall,
1393 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1394 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1395
1396 /* 1383 /*
1397 * OK, time to rat on our buddy... 1384 * OK, time to rat on our buddy...
1398 * See Documentation/RCU/stallwarn.txt for info on how to debug 1385 * See Documentation/RCU/stallwarn.txt for info on how to debug
@@ -1441,6 +1428,10 @@ static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
1441 sched_show_task(current); 1428 sched_show_task(current);
1442 } 1429 }
1443 } 1430 }
1431 /* Rewrite if needed in case of slow consoles. */
1432 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1433 WRITE_ONCE(rsp->jiffies_stall,
1434 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1444 1435
1445 rcu_check_gp_kthread_starvation(rsp); 1436 rcu_check_gp_kthread_starvation(rsp);
1446 1437
@@ -1485,6 +1476,7 @@ static void print_cpu_stall(struct rcu_state *rsp)
1485 rcu_dump_cpu_stacks(rsp); 1476 rcu_dump_cpu_stacks(rsp);
1486 1477
1487 raw_spin_lock_irqsave_rcu_node(rnp, flags); 1478 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1479 /* Rewrite if needed in case of slow consoles. */
1488 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) 1480 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1489 WRITE_ONCE(rsp->jiffies_stall, 1481 WRITE_ONCE(rsp->jiffies_stall,
1490 jiffies + 3 * rcu_jiffies_till_stall_check() + 3); 1482 jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
@@ -1508,6 +1500,7 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
1508 unsigned long gpnum; 1500 unsigned long gpnum;
1509 unsigned long gps; 1501 unsigned long gps;
1510 unsigned long j; 1502 unsigned long j;
1503 unsigned long jn;
1511 unsigned long js; 1504 unsigned long js;
1512 struct rcu_node *rnp; 1505 struct rcu_node *rnp;
1513 1506
@@ -1546,14 +1539,17 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
1546 ULONG_CMP_GE(gps, js)) 1539 ULONG_CMP_GE(gps, js))
1547 return; /* No stall or GP completed since entering function. */ 1540 return; /* No stall or GP completed since entering function. */
1548 rnp = rdp->mynode; 1541 rnp = rdp->mynode;
1542 jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
1549 if (rcu_gp_in_progress(rsp) && 1543 if (rcu_gp_in_progress(rsp) &&
1550 (READ_ONCE(rnp->qsmask) & rdp->grpmask)) { 1544 (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
1545 cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1551 1546
1552 /* We haven't checked in, so go dump stack. */ 1547 /* We haven't checked in, so go dump stack. */
1553 print_cpu_stall(rsp); 1548 print_cpu_stall(rsp);
1554 1549
1555 } else if (rcu_gp_in_progress(rsp) && 1550 } else if (rcu_gp_in_progress(rsp) &&
1556 ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) { 1551 ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
1552 cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1557 1553
1558 /* They had a few time units to dump stack, so complain. */ 1554 /* They had a few time units to dump stack, so complain. */
1559 print_other_cpu_stall(rsp, gpnum); 1555 print_other_cpu_stall(rsp, gpnum);
@@ -1685,6 +1681,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
1685 } 1681 }
1686 trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedroot")); 1682 trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedroot"));
1687 WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT); 1683 WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT);
1684 rsp->gp_req_activity = jiffies;
1688 if (!rsp->gp_kthread) { 1685 if (!rsp->gp_kthread) {
1689 trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread")); 1686 trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread"));
1690 goto unlock_out; 1687 goto unlock_out;
@@ -2084,7 +2081,8 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
2084 */ 2081 */
2085 rcu_for_each_node_breadth_first(rsp, rnp) { 2082 rcu_for_each_node_breadth_first(rsp, rnp) {
2086 raw_spin_lock_irq_rcu_node(rnp); 2083 raw_spin_lock_irq_rcu_node(rnp);
2087 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); 2084 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
2085 dump_blkd_tasks(rnp, 10);
2088 WARN_ON_ONCE(rnp->qsmask); 2086 WARN_ON_ONCE(rnp->qsmask);
2089 WRITE_ONCE(rnp->completed, rsp->gpnum); 2087 WRITE_ONCE(rnp->completed, rsp->gpnum);
2090 rdp = this_cpu_ptr(rsp->rda); 2088 rdp = this_cpu_ptr(rsp->rda);
@@ -2116,10 +2114,12 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
2116 /* Advance CBs to reduce false positives below. */ 2114 /* Advance CBs to reduce false positives below. */
2117 if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) { 2115 if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
2118 WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); 2116 WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
2117 rsp->gp_req_activity = jiffies;
2119 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), 2118 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),
2120 TPS("newreq")); 2119 TPS("newreq"));
2120 } else {
2121 WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
2121 } 2122 }
2122 WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
2123 raw_spin_unlock_irq_rcu_node(rnp); 2123 raw_spin_unlock_irq_rcu_node(rnp);
2124} 2124}
2125 2125
@@ -2294,6 +2294,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
2294 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 2294 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2295 return; 2295 return;
2296 } 2296 }
2297 rnp->completedqs = rnp->gpnum;
2297 mask = rnp->grpmask; 2298 mask = rnp->grpmask;
2298 if (rnp->parent == NULL) { 2299 if (rnp->parent == NULL) {
2299 2300
@@ -2747,6 +2748,65 @@ static void force_quiescent_state(struct rcu_state *rsp)
2747} 2748}
2748 2749
2749/* 2750/*
2751 * This function checks for grace-period requests that fail to motivate
2752 * RCU to come out of its idle mode.
2753 */
2754static void
2755rcu_check_gp_start_stall(struct rcu_state *rsp, struct rcu_node *rnp,
2756 struct rcu_data *rdp)
2757{
2758 unsigned long flags;
2759 unsigned long j;
2760 struct rcu_node *rnp_root = rcu_get_root(rsp);
2761 static atomic_t warned = ATOMIC_INIT(0);
2762
2763 if (!IS_ENABLED(CONFIG_PROVE_RCU) ||
2764 rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)))
2765 return;
2766 j = jiffies; /* Expensive access, and in common case don't get here. */
2767 if (time_before(j, READ_ONCE(rsp->gp_req_activity) + HZ) ||
2768 time_before(j, READ_ONCE(rsp->gp_activity) + HZ) ||
2769 atomic_read(&warned))
2770 return;
2771
2772 raw_spin_lock_irqsave_rcu_node(rnp, flags);
2773 j = jiffies;
2774 if (rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)) ||
2775 time_before(j, READ_ONCE(rsp->gp_req_activity) + HZ) ||
2776 time_before(j, READ_ONCE(rsp->gp_activity) + HZ) ||
2777 atomic_read(&warned)) {
2778 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2779 return;
2780 }
2781 /* Hold onto the leaf lock to make others see warned==1. */
2782
2783 if (rnp_root != rnp)
2784 raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
2785 j = jiffies;
2786 if (rcu_gp_in_progress(rsp) || !need_any_future_gp(rcu_get_root(rsp)) ||
2787 time_before(j, rsp->gp_req_activity + HZ) ||
2788 time_before(j, rsp->gp_activity + HZ) ||
2789 atomic_xchg(&warned, 1)) {
2790 raw_spin_unlock_rcu_node(rnp_root); /* irqs remain disabled. */
2791 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2792 return;
2793 }
2794 pr_alert("%s: g%lu %d%d%d%d gar:%lu ga:%lu f%#x %s->state:%#lx\n",
2795 __func__, READ_ONCE(rsp->gpnum),
2796 need_future_gp_element(rcu_get_root(rsp), 0),
2797 need_future_gp_element(rcu_get_root(rsp), 1),
2798 need_future_gp_element(rcu_get_root(rsp), 2),
2799 need_future_gp_element(rcu_get_root(rsp), 3),
2800 j - rsp->gp_req_activity, j - rsp->gp_activity,
2801 rsp->gp_flags, rsp->name,
2802 rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL);
2803 WARN_ON(1);
2804 if (rnp_root != rnp)
2805 raw_spin_unlock_rcu_node(rnp_root);
2806 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2807}
2808
2809/*
2750 * This does the RCU core processing work for the specified rcu_state 2810 * This does the RCU core processing work for the specified rcu_state
2751 * and rcu_data structures. This may be called only from the CPU to 2811 * and rcu_data structures. This may be called only from the CPU to
2752 * whom the rdp belongs. 2812 * whom the rdp belongs.
@@ -2757,7 +2817,7 @@ __rcu_process_callbacks(struct rcu_state *rsp)
2757 unsigned long flags; 2817 unsigned long flags;
2758 bool needwake; 2818 bool needwake;
2759 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); 2819 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
2760 struct rcu_node *rnp; 2820 struct rcu_node *rnp = rdp->mynode;
2761 2821
2762 WARN_ON_ONCE(!rdp->beenonline); 2822 WARN_ON_ONCE(!rdp->beenonline);
2763 2823
@@ -2771,7 +2831,6 @@ __rcu_process_callbacks(struct rcu_state *rsp)
2771 if (rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) { 2831 if (rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) {
2772 local_irq_restore(flags); 2832 local_irq_restore(flags);
2773 } else { 2833 } else {
2774 rnp = rdp->mynode;
2775 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ 2834 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
2776 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); 2835 needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
2777 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 2836 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
@@ -2780,6 +2839,8 @@ __rcu_process_callbacks(struct rcu_state *rsp)
2780 } 2839 }
2781 } 2840 }
2782 2841
2842 rcu_check_gp_start_stall(rsp, rnp, rdp);
2843
2783 /* If there are callbacks ready, invoke them. */ 2844 /* If there are callbacks ready, invoke them. */
2784 if (rcu_segcblist_ready_cbs(&rdp->cblist)) 2845 if (rcu_segcblist_ready_cbs(&rdp->cblist))
2785 invoke_rcu_callbacks(rsp, rdp); 2846 invoke_rcu_callbacks(rsp, rdp);
@@ -3930,6 +3991,7 @@ static void __init rcu_init_one(struct rcu_state *rsp)
3930 &rcu_fqs_class[i], fqs[i]); 3991 &rcu_fqs_class[i], fqs[i]);
3931 rnp->gpnum = rsp->gpnum; 3992 rnp->gpnum = rsp->gpnum;
3932 rnp->completed = rsp->completed; 3993 rnp->completed = rsp->completed;
3994 rnp->completedqs = rsp->completed;
3933 rnp->qsmask = 0; 3995 rnp->qsmask = 0;
3934 rnp->qsmaskinit = 0; 3996 rnp->qsmaskinit = 0;
3935 rnp->grplo = j * cpustride; 3997 rnp->grplo = j * cpustride;
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 78e051dffc5b..3c1942174c56 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -87,6 +87,7 @@ struct rcu_node {
87 unsigned long completed; /* Last GP completed for this node. */ 87 unsigned long completed; /* Last GP completed for this node. */
88 /* This will either be equal to or one */ 88 /* This will either be equal to or one */
89 /* behind the root rcu_node's gpnum. */ 89 /* behind the root rcu_node's gpnum. */
90 unsigned long completedqs; /* All QSes done for this node. */
90 unsigned long qsmask; /* CPUs or groups that need to switch in */ 91 unsigned long qsmask; /* CPUs or groups that need to switch in */
91 /* order for current grace period to proceed.*/ 92 /* order for current grace period to proceed.*/
92 /* In leaf rcu_node, each bit corresponds to */ 93 /* In leaf rcu_node, each bit corresponds to */
@@ -373,6 +374,8 @@ struct rcu_state {
373 /* but in jiffies. */ 374 /* but in jiffies. */
374 unsigned long gp_activity; /* Time of last GP kthread */ 375 unsigned long gp_activity; /* Time of last GP kthread */
375 /* activity in jiffies. */ 376 /* activity in jiffies. */
377 unsigned long gp_req_activity; /* Time of last GP request */
378 /* in jiffies. */
376 unsigned long jiffies_stall; /* Time at which to check */ 379 unsigned long jiffies_stall; /* Time at which to check */
377 /* for CPU stalls. */ 380 /* for CPU stalls. */
378 unsigned long jiffies_resched; /* Time at which to resched */ 381 unsigned long jiffies_resched; /* Time at which to resched */
@@ -453,6 +456,7 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
453static void rcu_preempt_check_callbacks(void); 456static void rcu_preempt_check_callbacks(void);
454void call_rcu(struct rcu_head *head, rcu_callback_t func); 457void call_rcu(struct rcu_head *head, rcu_callback_t func);
455static void __init __rcu_init_preempt(void); 458static void __init __rcu_init_preempt(void);
459static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck);
456static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); 460static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
457static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); 461static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
458static void invoke_rcu_callbacks_kthread(void); 462static void invoke_rcu_callbacks_kthread(void);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 7fd12039e512..e387ea712758 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -260,8 +260,10 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
260 * ->exp_tasks pointers, respectively, to reference the newly 260 * ->exp_tasks pointers, respectively, to reference the newly
261 * blocked tasks. 261 * blocked tasks.
262 */ 262 */
263 if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) 263 if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) {
264 rnp->gp_tasks = &t->rcu_node_entry; 264 rnp->gp_tasks = &t->rcu_node_entry;
265 WARN_ON_ONCE(rnp->completedqs == rnp->gpnum);
266 }
265 if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) 267 if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
266 rnp->exp_tasks = &t->rcu_node_entry; 268 rnp->exp_tasks = &t->rcu_node_entry;
267 WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) != 269 WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) !=
@@ -535,6 +537,8 @@ void rcu_read_unlock_special(struct task_struct *t)
535 WARN_ON_ONCE(rnp != t->rcu_blocked_node); 537 WARN_ON_ONCE(rnp != t->rcu_blocked_node);
536 WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); 538 WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
537 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); 539 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
540 WARN_ON_ONCE(rnp->completedqs == rnp->gpnum &&
541 (!empty_norm || rnp->qsmask));
538 empty_exp = sync_rcu_preempt_exp_done(rnp); 542 empty_exp = sync_rcu_preempt_exp_done(rnp);
539 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ 543 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
540 np = rcu_next_node_entry(t, rnp); 544 np = rcu_next_node_entry(t, rnp);
@@ -697,7 +701,8 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
697 struct task_struct *t; 701 struct task_struct *t;
698 702
699 RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n"); 703 RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n");
700 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); 704 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
705 dump_blkd_tasks(rnp, 10);
701 if (rcu_preempt_has_tasks(rnp)) { 706 if (rcu_preempt_has_tasks(rnp)) {
702 rnp->gp_tasks = rnp->blkd_tasks.next; 707 rnp->gp_tasks = rnp->blkd_tasks.next;
703 t = container_of(rnp->gp_tasks, struct task_struct, 708 t = container_of(rnp->gp_tasks, struct task_struct,
@@ -841,6 +846,27 @@ void exit_rcu(void)
841 __rcu_read_unlock(); 846 __rcu_read_unlock();
842} 847}
843 848
849/*
850 * Dump the blocked-tasks state, but limit the list dump to the
851 * specified number of elements.
852 */
853static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
854{
855 int i;
856 struct list_head *lhp;
857
858 raw_lockdep_assert_held_rcu_node(rnp);
859 pr_info("%s: grp: %d-%d level: %d ->qamask %#lx ->gp_tasks %p ->boost_tasks %p ->exp_tasks %p &->blkd_tasks: %p offset: %u\n", __func__, rnp->grplo, rnp->grphi, rnp->level, rnp->qsmask, rnp->gp_tasks, rnp->boost_tasks, rnp->exp_tasks, &rnp->blkd_tasks, (unsigned int)offsetof(typeof(*rnp), blkd_tasks));
860 pr_cont("\t->blkd_tasks");
861 i = 0;
862 list_for_each(lhp, &rnp->blkd_tasks) {
863 pr_cont(" %p", lhp);
864 if (++i >= 10)
865 break;
866 }
867 pr_cont("\n");
868}
869
844#else /* #ifdef CONFIG_PREEMPT_RCU */ 870#else /* #ifdef CONFIG_PREEMPT_RCU */
845 871
846static struct rcu_state *const rcu_state_p = &rcu_sched_state; 872static struct rcu_state *const rcu_state_p = &rcu_sched_state;
@@ -949,6 +975,14 @@ void exit_rcu(void)
949{ 975{
950} 976}
951 977
978/*
979 * Dump the guaranteed-empty blocked-tasks state. Trust but verify.
980 */
981static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
982{
983 WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks));
984}
985
952#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ 986#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
953 987
954#ifdef CONFIG_RCU_BOOST 988#ifdef CONFIG_RCU_BOOST
diff --git a/kernel/torture.c b/kernel/torture.c
index 3de1efbecd6a..1ac24a826589 100644
--- a/kernel/torture.c
+++ b/kernel/torture.c
@@ -20,6 +20,9 @@
20 * Author: Paul E. McKenney <paulmck@us.ibm.com> 20 * Author: Paul E. McKenney <paulmck@us.ibm.com>
21 * Based on kernel/rcu/torture.c. 21 * Based on kernel/rcu/torture.c.
22 */ 22 */
23
24#define pr_fmt(fmt) fmt
25
23#include <linux/types.h> 26#include <linux/types.h>
24#include <linux/kernel.h> 27#include <linux/kernel.h>
25#include <linux/init.h> 28#include <linux/init.h>
@@ -53,7 +56,7 @@ MODULE_LICENSE("GPL");
53MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>"); 56MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
54 57
55static char *torture_type; 58static char *torture_type;
56static bool verbose; 59static int verbose;
57 60
58/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 61/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */
59#define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 62#define FULLSTOP_DONTSTOP 0 /* Normal operation. */
@@ -98,7 +101,7 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
98 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 101 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
99 return false; 102 return false;
100 103
101 if (verbose) 104 if (verbose > 1)
102 pr_alert("%s" TORTURE_FLAG 105 pr_alert("%s" TORTURE_FLAG
103 "torture_onoff task: offlining %d\n", 106 "torture_onoff task: offlining %d\n",
104 torture_type, cpu); 107 torture_type, cpu);
@@ -111,7 +114,7 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
111 "torture_onoff task: offline %d failed: errno %d\n", 114 "torture_onoff task: offline %d failed: errno %d\n",
112 torture_type, cpu, ret); 115 torture_type, cpu, ret);
113 } else { 116 } else {
114 if (verbose) 117 if (verbose > 1)
115 pr_alert("%s" TORTURE_FLAG 118 pr_alert("%s" TORTURE_FLAG
116 "torture_onoff task: offlined %d\n", 119 "torture_onoff task: offlined %d\n",
117 torture_type, cpu); 120 torture_type, cpu);
@@ -147,7 +150,7 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
147 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 150 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
148 return false; 151 return false;
149 152
150 if (verbose) 153 if (verbose > 1)
151 pr_alert("%s" TORTURE_FLAG 154 pr_alert("%s" TORTURE_FLAG
152 "torture_onoff task: onlining %d\n", 155 "torture_onoff task: onlining %d\n",
153 torture_type, cpu); 156 torture_type, cpu);
@@ -160,7 +163,7 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
160 "torture_onoff task: online %d failed: errno %d\n", 163 "torture_onoff task: online %d failed: errno %d\n",
161 torture_type, cpu, ret); 164 torture_type, cpu, ret);
162 } else { 165 } else {
163 if (verbose) 166 if (verbose > 1)
164 pr_alert("%s" TORTURE_FLAG 167 pr_alert("%s" TORTURE_FLAG
165 "torture_onoff task: onlined %d\n", 168 "torture_onoff task: onlined %d\n",
166 torture_type, cpu); 169 torture_type, cpu);
@@ -647,7 +650,7 @@ static void torture_stutter_cleanup(void)
647 * The runnable parameter points to a flag that controls whether or not 650 * The runnable parameter points to a flag that controls whether or not
648 * the test is currently runnable. If there is no such flag, pass in NULL. 651 * the test is currently runnable. If there is no such flag, pass in NULL.
649 */ 652 */
650bool torture_init_begin(char *ttype, bool v) 653bool torture_init_begin(char *ttype, int v)
651{ 654{
652 mutex_lock(&fullstop_mutex); 655 mutex_lock(&fullstop_mutex);
653 if (torture_type != NULL) { 656 if (torture_type != NULL) {