aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/rcu/rcu_segcblist.h8
-rw-r--r--kernel/rcu/tree.c9
-rw-r--r--kernel/rcu/tree_plugin.h4
3 files changed, 7 insertions, 14 deletions
diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h
index 86bc1101b806..7d18d41f0116 100644
--- a/kernel/rcu/rcu_segcblist.h
+++ b/kernel/rcu/rcu_segcblist.h
@@ -22,12 +22,6 @@
22 22
23#include <linux/rcu_segcblist.h> 23#include <linux/rcu_segcblist.h>
24 24
25/* Is simple callback list empty? */
26static inline bool rcu_cblist_empty(struct rcu_cblist *rclp)
27{
28 return !rclp->head;
29}
30
31/* Return number of callbacks in simple callback list. */ 25/* Return number of callbacks in simple callback list. */
32static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) 26static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
33{ 27{
@@ -66,7 +60,7 @@ static inline struct rcu_head *rcu_cblist_head(struct rcu_cblist *rclp)
66 */ 60 */
67static inline struct rcu_head **rcu_cblist_tail(struct rcu_cblist *rclp) 61static inline struct rcu_head **rcu_cblist_tail(struct rcu_cblist *rclp)
68{ 62{
69 WARN_ON_ONCE(rcu_cblist_empty(rclp)); 63 WARN_ON_ONCE(!rclp->head);
70 return rclp->tail; 64 return rclp->tail;
71} 65}
72 66
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 91fff49d5869..35152414760d 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2647,9 +2647,9 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp, unsigned long flags)
2647 2647
2648 /* First adopt the ready-to-invoke callbacks, then the done ones. */ 2648 /* First adopt the ready-to-invoke callbacks, then the done ones. */
2649 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rsp->orphan_done); 2649 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rsp->orphan_done);
2650 WARN_ON_ONCE(!rcu_cblist_empty(&rsp->orphan_done)); 2650 WARN_ON_ONCE(rsp->orphan_done.head);
2651 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rsp->orphan_pend); 2651 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rsp->orphan_pend);
2652 WARN_ON_ONCE(!rcu_cblist_empty(&rsp->orphan_pend)); 2652 WARN_ON_ONCE(rsp->orphan_pend.head);
2653 WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != 2653 WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) !=
2654 !rcu_segcblist_n_cbs(&rdp->cblist)); 2654 !rcu_segcblist_n_cbs(&rdp->cblist));
2655} 2655}
@@ -2800,9 +2800,8 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
2800 2800
2801 local_irq_save(flags); 2801 local_irq_save(flags);
2802 count = -rcu_cblist_n_cbs(&rcl); 2802 count = -rcu_cblist_n_cbs(&rcl);
2803 trace_rcu_batch_end(rsp->name, count, !rcu_cblist_empty(&rcl), 2803 trace_rcu_batch_end(rsp->name, count, !!rcl.head, need_resched(),
2804 need_resched(), is_idle_task(current), 2804 is_idle_task(current), rcu_is_callbacks_kthread());
2805 rcu_is_callbacks_kthread());
2806 2805
2807 /* Update counts and requeue any remaining callbacks. */ 2806 /* Update counts and requeue any remaining callbacks. */
2808 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); 2807 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 7f1d677a2a25..6b4b1f8a272d 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1942,12 +1942,12 @@ static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
1942 return false; 1942 return false;
1943 1943
1944 /* First, enqueue the donelist, if any. This preserves CB ordering. */ 1944 /* First, enqueue the donelist, if any. This preserves CB ordering. */
1945 if (!rcu_cblist_empty(&rsp->orphan_done)) { 1945 if (rsp->orphan_done.head) {
1946 __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_done), 1946 __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_done),
1947 rcu_cblist_tail(&rsp->orphan_done), 1947 rcu_cblist_tail(&rsp->orphan_done),
1948 ql, qll, flags); 1948 ql, qll, flags);
1949 } 1949 }
1950 if (!rcu_cblist_empty(&rsp->orphan_pend)) { 1950 if (rsp->orphan_pend.head) {
1951 __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_pend), 1951 __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_pend),
1952 rcu_cblist_tail(&rsp->orphan_pend), 1952 rcu_cblist_tail(&rsp->orphan_pend),
1953 ql, qll, flags); 1953 ql, qll, flags);