aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcu/tree.h
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcu/tree.h')
-rw-r--r--kernel/rcu/tree.h19
1 files changed, 2 insertions, 17 deletions
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index a21d403a6010..9329c1ff695f 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -88,6 +88,7 @@ struct rcu_node {
88 /* This will either be equal to or one */ 88 /* This will either be equal to or one */
89 /* behind the root rcu_node's gpnum. */ 89 /* behind the root rcu_node's gpnum. */
90 unsigned long gp_seq; /* Track rsp->rcu_gp_seq. */ 90 unsigned long gp_seq; /* Track rsp->rcu_gp_seq. */
91 unsigned long gp_seq_needed; /* Track rsp->rcu_gp_seq_needed. */
91 unsigned long completedqs; /* All QSes done for this node. */ 92 unsigned long completedqs; /* All QSes done for this node. */
92 unsigned long qsmask; /* CPUs or groups that need to switch in */ 93 unsigned long qsmask; /* CPUs or groups that need to switch in */
93 /* order for current grace period to proceed.*/ 94 /* order for current grace period to proceed.*/
@@ -160,7 +161,6 @@ struct rcu_node {
160 struct swait_queue_head nocb_gp_wq[2]; 161 struct swait_queue_head nocb_gp_wq[2];
161 /* Place for rcu_nocb_kthread() to wait GP. */ 162 /* Place for rcu_nocb_kthread() to wait GP. */
162#endif /* #ifdef CONFIG_RCU_NOCB_CPU */ 163#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
163 u8 need_future_gp[4]; /* Counts of upcoming GP requests. */
164 raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp; 164 raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp;
165 165
166 spinlock_t exp_lock ____cacheline_internodealigned_in_smp; 166 spinlock_t exp_lock ____cacheline_internodealigned_in_smp;
@@ -170,22 +170,6 @@ struct rcu_node {
170 bool exp_need_flush; /* Need to flush workitem? */ 170 bool exp_need_flush; /* Need to flush workitem? */
171} ____cacheline_internodealigned_in_smp; 171} ____cacheline_internodealigned_in_smp;
172 172
173/* Accessors for ->need_future_gp[] array. */
174#define need_future_gp_mask() \
175 (ARRAY_SIZE(((struct rcu_node *)NULL)->need_future_gp) - 1)
176#define need_future_gp_element(rnp, c) \
177 ((rnp)->need_future_gp[(c >> RCU_SEQ_CTR_SHIFT) & need_future_gp_mask()])
178#define need_any_future_gp(rnp) \
179({ \
180 int __i; \
181 bool __nonzero = false; \
182 \
183 for (__i = 0; __i < ARRAY_SIZE((rnp)->need_future_gp); __i++) \
184 __nonzero = __nonzero || \
185 READ_ONCE((rnp)->need_future_gp[__i]); \
186 __nonzero; \
187})
188
189/* 173/*
190 * Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and 174 * Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and
191 * are indexed relative to this interval rather than the global CPU ID space. 175 * are indexed relative to this interval rather than the global CPU ID space.
@@ -213,6 +197,7 @@ struct rcu_data {
213 unsigned long gpnum; /* Highest gp number that this CPU */ 197 unsigned long gpnum; /* Highest gp number that this CPU */
214 /* is aware of having started. */ 198 /* is aware of having started. */
215 unsigned long gp_seq; /* Track rsp->rcu_gp_seq counter. */ 199 unsigned long gp_seq; /* Track rsp->rcu_gp_seq counter. */
200 unsigned long gp_seq_needed; /* Track rsp->rcu_gp_seq_needed ctr. */
216 unsigned long rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */ 201 unsigned long rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */
217 /* for rcu_all_qs() invocations. */ 202 /* for rcu_all_qs() invocations. */
218 union rcu_noqs cpu_no_qs; /* No QSes yet for this CPU. */ 203 union rcu_noqs cpu_no_qs; /* No QSes yet for this CPU. */