diff options
Diffstat (limited to 'kernel/rcu/srcutiny.c')
-rw-r--r-- | kernel/rcu/srcutiny.c | 120 |
1 files changed, 60 insertions, 60 deletions
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index b46e6683f8c9..32dfd6522548 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c | |||
@@ -37,30 +37,30 @@ int rcu_scheduler_active __read_mostly; | |||
37 | static LIST_HEAD(srcu_boot_list); | 37 | static LIST_HEAD(srcu_boot_list); |
38 | static bool srcu_init_done; | 38 | static bool srcu_init_done; |
39 | 39 | ||
40 | static int init_srcu_struct_fields(struct srcu_struct *sp) | 40 | static int init_srcu_struct_fields(struct srcu_struct *ssp) |
41 | { | 41 | { |
42 | sp->srcu_lock_nesting[0] = 0; | 42 | ssp->srcu_lock_nesting[0] = 0; |
43 | sp->srcu_lock_nesting[1] = 0; | 43 | ssp->srcu_lock_nesting[1] = 0; |
44 | init_swait_queue_head(&sp->srcu_wq); | 44 | init_swait_queue_head(&ssp->srcu_wq); |
45 | sp->srcu_cb_head = NULL; | 45 | ssp->srcu_cb_head = NULL; |
46 | sp->srcu_cb_tail = &sp->srcu_cb_head; | 46 | ssp->srcu_cb_tail = &ssp->srcu_cb_head; |
47 | sp->srcu_gp_running = false; | 47 | ssp->srcu_gp_running = false; |
48 | sp->srcu_gp_waiting = false; | 48 | ssp->srcu_gp_waiting = false; |
49 | sp->srcu_idx = 0; | 49 | ssp->srcu_idx = 0; |
50 | INIT_WORK(&sp->srcu_work, srcu_drive_gp); | 50 | INIT_WORK(&ssp->srcu_work, srcu_drive_gp); |
51 | INIT_LIST_HEAD(&sp->srcu_work.entry); | 51 | INIT_LIST_HEAD(&ssp->srcu_work.entry); |
52 | return 0; | 52 | return 0; |
53 | } | 53 | } |
54 | 54 | ||
55 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 55 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
56 | 56 | ||
57 | int __init_srcu_struct(struct srcu_struct *sp, const char *name, | 57 | int __init_srcu_struct(struct srcu_struct *ssp, const char *name, |
58 | struct lock_class_key *key) | 58 | struct lock_class_key *key) |
59 | { | 59 | { |
60 | /* Don't re-initialize a lock while it is held. */ | 60 | /* Don't re-initialize a lock while it is held. */ |
61 | debug_check_no_locks_freed((void *)sp, sizeof(*sp)); | 61 | debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); |
62 | lockdep_init_map(&sp->dep_map, name, key, 0); | 62 | lockdep_init_map(&ssp->dep_map, name, key, 0); |
63 | return init_srcu_struct_fields(sp); | 63 | return init_srcu_struct_fields(ssp); |
64 | } | 64 | } |
65 | EXPORT_SYMBOL_GPL(__init_srcu_struct); | 65 | EXPORT_SYMBOL_GPL(__init_srcu_struct); |
66 | 66 | ||
@@ -68,15 +68,15 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct); | |||
68 | 68 | ||
69 | /* | 69 | /* |
70 | * init_srcu_struct - initialize a sleep-RCU structure | 70 | * init_srcu_struct - initialize a sleep-RCU structure |
71 | * @sp: structure to initialize. | 71 | * @ssp: structure to initialize. |
72 | * | 72 | * |
73 | * Must invoke this on a given srcu_struct before passing that srcu_struct | 73 | * Must invoke this on a given srcu_struct before passing that srcu_struct |
74 | * to any other function. Each srcu_struct represents a separate domain | 74 | * to any other function. Each srcu_struct represents a separate domain |
75 | * of SRCU protection. | 75 | * of SRCU protection. |
76 | */ | 76 | */ |
77 | int init_srcu_struct(struct srcu_struct *sp) | 77 | int init_srcu_struct(struct srcu_struct *ssp) |
78 | { | 78 | { |
79 | return init_srcu_struct_fields(sp); | 79 | return init_srcu_struct_fields(ssp); |
80 | } | 80 | } |
81 | EXPORT_SYMBOL_GPL(init_srcu_struct); | 81 | EXPORT_SYMBOL_GPL(init_srcu_struct); |
82 | 82 | ||
@@ -84,22 +84,22 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); | |||
84 | 84 | ||
85 | /* | 85 | /* |
86 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | 86 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure |
87 | * @sp: structure to clean up. | 87 | * @ssp: structure to clean up. |
88 | * | 88 | * |
89 | * Must invoke this after you are finished using a given srcu_struct that | 89 | * Must invoke this after you are finished using a given srcu_struct that |
90 | * was initialized via init_srcu_struct(), else you leak memory. | 90 | * was initialized via init_srcu_struct(), else you leak memory. |
91 | */ | 91 | */ |
92 | void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced) | 92 | void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced) |
93 | { | 93 | { |
94 | WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); | 94 | WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); |
95 | if (quiesced) | 95 | if (quiesced) |
96 | WARN_ON(work_pending(&sp->srcu_work)); | 96 | WARN_ON(work_pending(&ssp->srcu_work)); |
97 | else | 97 | else |
98 | flush_work(&sp->srcu_work); | 98 | flush_work(&ssp->srcu_work); |
99 | WARN_ON(sp->srcu_gp_running); | 99 | WARN_ON(ssp->srcu_gp_running); |
100 | WARN_ON(sp->srcu_gp_waiting); | 100 | WARN_ON(ssp->srcu_gp_waiting); |
101 | WARN_ON(sp->srcu_cb_head); | 101 | WARN_ON(ssp->srcu_cb_head); |
102 | WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail); | 102 | WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); |
103 | } | 103 | } |
104 | EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); | 104 | EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); |
105 | 105 | ||
@@ -107,13 +107,13 @@ EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); | |||
107 | * Removes the count for the old reader from the appropriate element of | 107 | * Removes the count for the old reader from the appropriate element of |
108 | * the srcu_struct. | 108 | * the srcu_struct. |
109 | */ | 109 | */ |
110 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) | 110 | void __srcu_read_unlock(struct srcu_struct *ssp, int idx) |
111 | { | 111 | { |
112 | int newval = sp->srcu_lock_nesting[idx] - 1; | 112 | int newval = ssp->srcu_lock_nesting[idx] - 1; |
113 | 113 | ||
114 | WRITE_ONCE(sp->srcu_lock_nesting[idx], newval); | 114 | WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval); |
115 | if (!newval && READ_ONCE(sp->srcu_gp_waiting)) | 115 | if (!newval && READ_ONCE(ssp->srcu_gp_waiting)) |
116 | swake_up_one(&sp->srcu_wq); | 116 | swake_up_one(&ssp->srcu_wq); |
117 | } | 117 | } |
118 | EXPORT_SYMBOL_GPL(__srcu_read_unlock); | 118 | EXPORT_SYMBOL_GPL(__srcu_read_unlock); |
119 | 119 | ||
@@ -127,24 +127,24 @@ void srcu_drive_gp(struct work_struct *wp) | |||
127 | int idx; | 127 | int idx; |
128 | struct rcu_head *lh; | 128 | struct rcu_head *lh; |
129 | struct rcu_head *rhp; | 129 | struct rcu_head *rhp; |
130 | struct srcu_struct *sp; | 130 | struct srcu_struct *ssp; |
131 | 131 | ||
132 | sp = container_of(wp, struct srcu_struct, srcu_work); | 132 | ssp = container_of(wp, struct srcu_struct, srcu_work); |
133 | if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head)) | 133 | if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head)) |
134 | return; /* Already running or nothing to do. */ | 134 | return; /* Already running or nothing to do. */ |
135 | 135 | ||
136 | /* Remove recently arrived callbacks and wait for readers. */ | 136 | /* Remove recently arrived callbacks and wait for readers. */ |
137 | WRITE_ONCE(sp->srcu_gp_running, true); | 137 | WRITE_ONCE(ssp->srcu_gp_running, true); |
138 | local_irq_disable(); | 138 | local_irq_disable(); |
139 | lh = sp->srcu_cb_head; | 139 | lh = ssp->srcu_cb_head; |
140 | sp->srcu_cb_head = NULL; | 140 | ssp->srcu_cb_head = NULL; |
141 | sp->srcu_cb_tail = &sp->srcu_cb_head; | 141 | ssp->srcu_cb_tail = &ssp->srcu_cb_head; |
142 | local_irq_enable(); | 142 | local_irq_enable(); |
143 | idx = sp->srcu_idx; | 143 | idx = ssp->srcu_idx; |
144 | WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx); | 144 | WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx); |
145 | WRITE_ONCE(sp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ | 145 | WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ |
146 | swait_event_exclusive(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx])); | 146 | swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx])); |
147 | WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ | 147 | WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ |
148 | 148 | ||
149 | /* Invoke the callbacks we removed above. */ | 149 | /* Invoke the callbacks we removed above. */ |
150 | while (lh) { | 150 | while (lh) { |
@@ -161,9 +161,9 @@ void srcu_drive_gp(struct work_struct *wp) | |||
161 | * at interrupt level, but the ->srcu_gp_running checks will | 161 | * at interrupt level, but the ->srcu_gp_running checks will |
162 | * straighten that out. | 162 | * straighten that out. |
163 | */ | 163 | */ |
164 | WRITE_ONCE(sp->srcu_gp_running, false); | 164 | WRITE_ONCE(ssp->srcu_gp_running, false); |
165 | if (READ_ONCE(sp->srcu_cb_head)) | 165 | if (READ_ONCE(ssp->srcu_cb_head)) |
166 | schedule_work(&sp->srcu_work); | 166 | schedule_work(&ssp->srcu_work); |
167 | } | 167 | } |
168 | EXPORT_SYMBOL_GPL(srcu_drive_gp); | 168 | EXPORT_SYMBOL_GPL(srcu_drive_gp); |
169 | 169 | ||
@@ -171,7 +171,7 @@ EXPORT_SYMBOL_GPL(srcu_drive_gp); | |||
171 | * Enqueue an SRCU callback on the specified srcu_struct structure, | 171 | * Enqueue an SRCU callback on the specified srcu_struct structure, |
172 | * initiating grace-period processing if it is not already running. | 172 | * initiating grace-period processing if it is not already running. |
173 | */ | 173 | */ |
174 | void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, | 174 | void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, |
175 | rcu_callback_t func) | 175 | rcu_callback_t func) |
176 | { | 176 | { |
177 | unsigned long flags; | 177 | unsigned long flags; |
@@ -179,14 +179,14 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, | |||
179 | rhp->func = func; | 179 | rhp->func = func; |
180 | rhp->next = NULL; | 180 | rhp->next = NULL; |
181 | local_irq_save(flags); | 181 | local_irq_save(flags); |
182 | *sp->srcu_cb_tail = rhp; | 182 | *ssp->srcu_cb_tail = rhp; |
183 | sp->srcu_cb_tail = &rhp->next; | 183 | ssp->srcu_cb_tail = &rhp->next; |
184 | local_irq_restore(flags); | 184 | local_irq_restore(flags); |
185 | if (!READ_ONCE(sp->srcu_gp_running)) { | 185 | if (!READ_ONCE(ssp->srcu_gp_running)) { |
186 | if (likely(srcu_init_done)) | 186 | if (likely(srcu_init_done)) |
187 | schedule_work(&sp->srcu_work); | 187 | schedule_work(&ssp->srcu_work); |
188 | else if (list_empty(&sp->srcu_work.entry)) | 188 | else if (list_empty(&ssp->srcu_work.entry)) |
189 | list_add(&sp->srcu_work.entry, &srcu_boot_list); | 189 | list_add(&ssp->srcu_work.entry, &srcu_boot_list); |
190 | } | 190 | } |
191 | } | 191 | } |
192 | EXPORT_SYMBOL_GPL(call_srcu); | 192 | EXPORT_SYMBOL_GPL(call_srcu); |
@@ -194,13 +194,13 @@ EXPORT_SYMBOL_GPL(call_srcu); | |||
194 | /* | 194 | /* |
195 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | 195 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion |
196 | */ | 196 | */ |
197 | void synchronize_srcu(struct srcu_struct *sp) | 197 | void synchronize_srcu(struct srcu_struct *ssp) |
198 | { | 198 | { |
199 | struct rcu_synchronize rs; | 199 | struct rcu_synchronize rs; |
200 | 200 | ||
201 | init_rcu_head_on_stack(&rs.head); | 201 | init_rcu_head_on_stack(&rs.head); |
202 | init_completion(&rs.completion); | 202 | init_completion(&rs.completion); |
203 | call_srcu(sp, &rs.head, wakeme_after_rcu); | 203 | call_srcu(ssp, &rs.head, wakeme_after_rcu); |
204 | wait_for_completion(&rs.completion); | 204 | wait_for_completion(&rs.completion); |
205 | destroy_rcu_head_on_stack(&rs.head); | 205 | destroy_rcu_head_on_stack(&rs.head); |
206 | } | 206 | } |
@@ -219,13 +219,13 @@ void __init rcu_scheduler_starting(void) | |||
219 | */ | 219 | */ |
220 | void __init srcu_init(void) | 220 | void __init srcu_init(void) |
221 | { | 221 | { |
222 | struct srcu_struct *sp; | 222 | struct srcu_struct *ssp; |
223 | 223 | ||
224 | srcu_init_done = true; | 224 | srcu_init_done = true; |
225 | while (!list_empty(&srcu_boot_list)) { | 225 | while (!list_empty(&srcu_boot_list)) { |
226 | sp = list_first_entry(&srcu_boot_list, | 226 | ssp = list_first_entry(&srcu_boot_list, |
227 | struct srcu_struct, srcu_work.entry); | 227 | struct srcu_struct, srcu_work.entry); |
228 | list_del_init(&sp->srcu_work.entry); | 228 | list_del_init(&ssp->srcu_work.entry); |
229 | schedule_work(&sp->srcu_work); | 229 | schedule_work(&ssp->srcu_work); |
230 | } | 230 | } |
231 | } | 231 | } |