aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/locking/locktorture.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/locking/locktorture.c')
-rw-r--r--kernel/locking/locktorture.c392
1 files changed, 323 insertions, 69 deletions
diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c
index 0955b885d0dc..540d5dfe1112 100644
--- a/kernel/locking/locktorture.c
+++ b/kernel/locking/locktorture.c
@@ -27,6 +27,7 @@
27#include <linux/kthread.h> 27#include <linux/kthread.h>
28#include <linux/err.h> 28#include <linux/err.h>
29#include <linux/spinlock.h> 29#include <linux/spinlock.h>
30#include <linux/mutex.h>
30#include <linux/smp.h> 31#include <linux/smp.h>
31#include <linux/interrupt.h> 32#include <linux/interrupt.h>
32#include <linux/sched.h> 33#include <linux/sched.h>
@@ -51,6 +52,8 @@ MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
51 52
52torture_param(int, nwriters_stress, -1, 53torture_param(int, nwriters_stress, -1,
53 "Number of write-locking stress-test threads"); 54 "Number of write-locking stress-test threads");
55torture_param(int, nreaders_stress, -1,
56 "Number of read-locking stress-test threads");
54torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 57torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
55torture_param(int, onoff_interval, 0, 58torture_param(int, onoff_interval, 0,
56 "Time between CPU hotplugs (s), 0=disable"); 59 "Time between CPU hotplugs (s), 0=disable");
@@ -66,30 +69,28 @@ torture_param(bool, verbose, true,
66static char *torture_type = "spin_lock"; 69static char *torture_type = "spin_lock";
67module_param(torture_type, charp, 0444); 70module_param(torture_type, charp, 0444);
68MODULE_PARM_DESC(torture_type, 71MODULE_PARM_DESC(torture_type,
69 "Type of lock to torture (spin_lock, spin_lock_irq, ...)"); 72 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
70
71static atomic_t n_lock_torture_errors;
72 73
73static struct task_struct *stats_task; 74static struct task_struct *stats_task;
74static struct task_struct **writer_tasks; 75static struct task_struct **writer_tasks;
76static struct task_struct **reader_tasks;
75 77
76static int nrealwriters_stress;
77static bool lock_is_write_held; 78static bool lock_is_write_held;
79static bool lock_is_read_held;
78 80
79struct lock_writer_stress_stats { 81struct lock_stress_stats {
80 long n_write_lock_fail; 82 long n_lock_fail;
81 long n_write_lock_acquired; 83 long n_lock_acquired;
82}; 84};
83static struct lock_writer_stress_stats *lwsa;
84 85
85#if defined(MODULE) 86#if defined(MODULE)
86#define LOCKTORTURE_RUNNABLE_INIT 1 87#define LOCKTORTURE_RUNNABLE_INIT 1
87#else 88#else
88#define LOCKTORTURE_RUNNABLE_INIT 0 89#define LOCKTORTURE_RUNNABLE_INIT 0
89#endif 90#endif
90int locktorture_runnable = LOCKTORTURE_RUNNABLE_INIT; 91int torture_runnable = LOCKTORTURE_RUNNABLE_INIT;
91module_param(locktorture_runnable, int, 0444); 92module_param(torture_runnable, int, 0444);
92MODULE_PARM_DESC(locktorture_runnable, "Start locktorture at module init"); 93MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init");
93 94
94/* Forward reference. */ 95/* Forward reference. */
95static void lock_torture_cleanup(void); 96static void lock_torture_cleanup(void);
@@ -102,12 +103,25 @@ struct lock_torture_ops {
102 int (*writelock)(void); 103 int (*writelock)(void);
103 void (*write_delay)(struct torture_random_state *trsp); 104 void (*write_delay)(struct torture_random_state *trsp);
104 void (*writeunlock)(void); 105 void (*writeunlock)(void);
106 int (*readlock)(void);
107 void (*read_delay)(struct torture_random_state *trsp);
108 void (*readunlock)(void);
105 unsigned long flags; 109 unsigned long flags;
106 const char *name; 110 const char *name;
107}; 111};
108 112
109static struct lock_torture_ops *cur_ops; 113struct lock_torture_cxt {
110 114 int nrealwriters_stress;
115 int nrealreaders_stress;
116 bool debug_lock;
117 atomic_t n_lock_torture_errors;
118 struct lock_torture_ops *cur_ops;
119 struct lock_stress_stats *lwsa; /* writer statistics */
120 struct lock_stress_stats *lrsa; /* reader statistics */
121};
122static struct lock_torture_cxt cxt = { 0, 0, false,
123 ATOMIC_INIT(0),
124 NULL, NULL};
111/* 125/*
112 * Definitions for lock torture testing. 126 * Definitions for lock torture testing.
113 */ 127 */
@@ -123,10 +137,10 @@ static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
123 137
124 /* We want a long delay occasionally to force massive contention. */ 138 /* We want a long delay occasionally to force massive contention. */
125 if (!(torture_random(trsp) % 139 if (!(torture_random(trsp) %
126 (nrealwriters_stress * 2000 * longdelay_us))) 140 (cxt.nrealwriters_stress * 2000 * longdelay_us)))
127 mdelay(longdelay_us); 141 mdelay(longdelay_us);
128#ifdef CONFIG_PREEMPT 142#ifdef CONFIG_PREEMPT
129 if (!(torture_random(trsp) % (nrealwriters_stress * 20000))) 143 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
130 preempt_schedule(); /* Allow test to be preempted. */ 144 preempt_schedule(); /* Allow test to be preempted. */
131#endif 145#endif
132} 146}
@@ -140,6 +154,9 @@ static struct lock_torture_ops lock_busted_ops = {
140 .writelock = torture_lock_busted_write_lock, 154 .writelock = torture_lock_busted_write_lock,
141 .write_delay = torture_lock_busted_write_delay, 155 .write_delay = torture_lock_busted_write_delay,
142 .writeunlock = torture_lock_busted_write_unlock, 156 .writeunlock = torture_lock_busted_write_unlock,
157 .readlock = NULL,
158 .read_delay = NULL,
159 .readunlock = NULL,
143 .name = "lock_busted" 160 .name = "lock_busted"
144}; 161};
145 162
@@ -160,13 +177,13 @@ static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
160 * we want a long delay occasionally to force massive contention. 177 * we want a long delay occasionally to force massive contention.
161 */ 178 */
162 if (!(torture_random(trsp) % 179 if (!(torture_random(trsp) %
163 (nrealwriters_stress * 2000 * longdelay_us))) 180 (cxt.nrealwriters_stress * 2000 * longdelay_us)))
164 mdelay(longdelay_us); 181 mdelay(longdelay_us);
165 if (!(torture_random(trsp) % 182 if (!(torture_random(trsp) %
166 (nrealwriters_stress * 2 * shortdelay_us))) 183 (cxt.nrealwriters_stress * 2 * shortdelay_us)))
167 udelay(shortdelay_us); 184 udelay(shortdelay_us);
168#ifdef CONFIG_PREEMPT 185#ifdef CONFIG_PREEMPT
169 if (!(torture_random(trsp) % (nrealwriters_stress * 20000))) 186 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
170 preempt_schedule(); /* Allow test to be preempted. */ 187 preempt_schedule(); /* Allow test to be preempted. */
171#endif 188#endif
172} 189}
@@ -180,6 +197,9 @@ static struct lock_torture_ops spin_lock_ops = {
180 .writelock = torture_spin_lock_write_lock, 197 .writelock = torture_spin_lock_write_lock,
181 .write_delay = torture_spin_lock_write_delay, 198 .write_delay = torture_spin_lock_write_delay,
182 .writeunlock = torture_spin_lock_write_unlock, 199 .writeunlock = torture_spin_lock_write_unlock,
200 .readlock = NULL,
201 .read_delay = NULL,
202 .readunlock = NULL,
183 .name = "spin_lock" 203 .name = "spin_lock"
184}; 204};
185 205
@@ -189,30 +209,137 @@ __acquires(torture_spinlock_irq)
189 unsigned long flags; 209 unsigned long flags;
190 210
191 spin_lock_irqsave(&torture_spinlock, flags); 211 spin_lock_irqsave(&torture_spinlock, flags);
192 cur_ops->flags = flags; 212 cxt.cur_ops->flags = flags;
193 return 0; 213 return 0;
194} 214}
195 215
196static void torture_lock_spin_write_unlock_irq(void) 216static void torture_lock_spin_write_unlock_irq(void)
197__releases(torture_spinlock) 217__releases(torture_spinlock)
198{ 218{
199 spin_unlock_irqrestore(&torture_spinlock, cur_ops->flags); 219 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
200} 220}
201 221
202static struct lock_torture_ops spin_lock_irq_ops = { 222static struct lock_torture_ops spin_lock_irq_ops = {
203 .writelock = torture_spin_lock_write_lock_irq, 223 .writelock = torture_spin_lock_write_lock_irq,
204 .write_delay = torture_spin_lock_write_delay, 224 .write_delay = torture_spin_lock_write_delay,
205 .writeunlock = torture_lock_spin_write_unlock_irq, 225 .writeunlock = torture_lock_spin_write_unlock_irq,
226 .readlock = NULL,
227 .read_delay = NULL,
228 .readunlock = NULL,
206 .name = "spin_lock_irq" 229 .name = "spin_lock_irq"
207}; 230};
208 231
232static DEFINE_MUTEX(torture_mutex);
233
234static int torture_mutex_lock(void) __acquires(torture_mutex)
235{
236 mutex_lock(&torture_mutex);
237 return 0;
238}
239
240static void torture_mutex_delay(struct torture_random_state *trsp)
241{
242 const unsigned long longdelay_ms = 100;
243
244 /* We want a long delay occasionally to force massive contention. */
245 if (!(torture_random(trsp) %
246 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
247 mdelay(longdelay_ms * 5);
248 else
249 mdelay(longdelay_ms / 5);
250#ifdef CONFIG_PREEMPT
251 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
252 preempt_schedule(); /* Allow test to be preempted. */
253#endif
254}
255
256static void torture_mutex_unlock(void) __releases(torture_mutex)
257{
258 mutex_unlock(&torture_mutex);
259}
260
261static struct lock_torture_ops mutex_lock_ops = {
262 .writelock = torture_mutex_lock,
263 .write_delay = torture_mutex_delay,
264 .writeunlock = torture_mutex_unlock,
265 .readlock = NULL,
266 .read_delay = NULL,
267 .readunlock = NULL,
268 .name = "mutex_lock"
269};
270
271static DECLARE_RWSEM(torture_rwsem);
272static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
273{
274 down_write(&torture_rwsem);
275 return 0;
276}
277
278static void torture_rwsem_write_delay(struct torture_random_state *trsp)
279{
280 const unsigned long longdelay_ms = 100;
281
282 /* We want a long delay occasionally to force massive contention. */
283 if (!(torture_random(trsp) %
284 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
285 mdelay(longdelay_ms * 10);
286 else
287 mdelay(longdelay_ms / 10);
288#ifdef CONFIG_PREEMPT
289 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
290 preempt_schedule(); /* Allow test to be preempted. */
291#endif
292}
293
294static void torture_rwsem_up_write(void) __releases(torture_rwsem)
295{
296 up_write(&torture_rwsem);
297}
298
299static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
300{
301 down_read(&torture_rwsem);
302 return 0;
303}
304
305static void torture_rwsem_read_delay(struct torture_random_state *trsp)
306{
307 const unsigned long longdelay_ms = 100;
308
309 /* We want a long delay occasionally to force massive contention. */
310 if (!(torture_random(trsp) %
311 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
312 mdelay(longdelay_ms * 2);
313 else
314 mdelay(longdelay_ms / 2);
315#ifdef CONFIG_PREEMPT
316 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
317 preempt_schedule(); /* Allow test to be preempted. */
318#endif
319}
320
321static void torture_rwsem_up_read(void) __releases(torture_rwsem)
322{
323 up_read(&torture_rwsem);
324}
325
326static struct lock_torture_ops rwsem_lock_ops = {
327 .writelock = torture_rwsem_down_write,
328 .write_delay = torture_rwsem_write_delay,
329 .writeunlock = torture_rwsem_up_write,
330 .readlock = torture_rwsem_down_read,
331 .read_delay = torture_rwsem_read_delay,
332 .readunlock = torture_rwsem_up_read,
333 .name = "rwsem_lock"
334};
335
209/* 336/*
210 * Lock torture writer kthread. Repeatedly acquires and releases 337 * Lock torture writer kthread. Repeatedly acquires and releases
211 * the lock, checking for duplicate acquisitions. 338 * the lock, checking for duplicate acquisitions.
212 */ 339 */
213static int lock_torture_writer(void *arg) 340static int lock_torture_writer(void *arg)
214{ 341{
215 struct lock_writer_stress_stats *lwsp = arg; 342 struct lock_stress_stats *lwsp = arg;
216 static DEFINE_TORTURE_RANDOM(rand); 343 static DEFINE_TORTURE_RANDOM(rand);
217 344
218 VERBOSE_TOROUT_STRING("lock_torture_writer task started"); 345 VERBOSE_TOROUT_STRING("lock_torture_writer task started");
@@ -221,14 +348,14 @@ static int lock_torture_writer(void *arg)
221 do { 348 do {
222 if ((torture_random(&rand) & 0xfffff) == 0) 349 if ((torture_random(&rand) & 0xfffff) == 0)
223 schedule_timeout_uninterruptible(1); 350 schedule_timeout_uninterruptible(1);
224 cur_ops->writelock(); 351 cxt.cur_ops->writelock();
225 if (WARN_ON_ONCE(lock_is_write_held)) 352 if (WARN_ON_ONCE(lock_is_write_held))
226 lwsp->n_write_lock_fail++; 353 lwsp->n_lock_fail++;
227 lock_is_write_held = 1; 354 lock_is_write_held = 1;
228 lwsp->n_write_lock_acquired++; 355 lwsp->n_lock_acquired++;
229 cur_ops->write_delay(&rand); 356 cxt.cur_ops->write_delay(&rand);
230 lock_is_write_held = 0; 357 lock_is_write_held = 0;
231 cur_ops->writeunlock(); 358 cxt.cur_ops->writeunlock();
232 stutter_wait("lock_torture_writer"); 359 stutter_wait("lock_torture_writer");
233 } while (!torture_must_stop()); 360 } while (!torture_must_stop());
234 torture_kthread_stopping("lock_torture_writer"); 361 torture_kthread_stopping("lock_torture_writer");
@@ -236,32 +363,61 @@ static int lock_torture_writer(void *arg)
236} 363}
237 364
238/* 365/*
366 * Lock torture reader kthread. Repeatedly acquires and releases
367 * the reader lock.
368 */
369static int lock_torture_reader(void *arg)
370{
371 struct lock_stress_stats *lrsp = arg;
372 static DEFINE_TORTURE_RANDOM(rand);
373
374 VERBOSE_TOROUT_STRING("lock_torture_reader task started");
375 set_user_nice(current, MAX_NICE);
376
377 do {
378 if ((torture_random(&rand) & 0xfffff) == 0)
379 schedule_timeout_uninterruptible(1);
380 cxt.cur_ops->readlock();
381 lock_is_read_held = 1;
382 lrsp->n_lock_acquired++;
383 cxt.cur_ops->read_delay(&rand);
384 lock_is_read_held = 0;
385 cxt.cur_ops->readunlock();
386 stutter_wait("lock_torture_reader");
387 } while (!torture_must_stop());
388 torture_kthread_stopping("lock_torture_reader");
389 return 0;
390}
391
392/*
239 * Create an lock-torture-statistics message in the specified buffer. 393 * Create an lock-torture-statistics message in the specified buffer.
240 */ 394 */
241static void lock_torture_printk(char *page) 395static void __torture_print_stats(char *page,
396 struct lock_stress_stats *statp, bool write)
242{ 397{
243 bool fail = 0; 398 bool fail = 0;
244 int i; 399 int i, n_stress;
245 long max = 0; 400 long max = 0;
246 long min = lwsa[0].n_write_lock_acquired; 401 long min = statp[0].n_lock_acquired;
247 long long sum = 0; 402 long long sum = 0;
248 403
249 for (i = 0; i < nrealwriters_stress; i++) { 404 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
250 if (lwsa[i].n_write_lock_fail) 405 for (i = 0; i < n_stress; i++) {
406 if (statp[i].n_lock_fail)
251 fail = true; 407 fail = true;
252 sum += lwsa[i].n_write_lock_acquired; 408 sum += statp[i].n_lock_acquired;
253 if (max < lwsa[i].n_write_lock_fail) 409 if (max < statp[i].n_lock_fail)
254 max = lwsa[i].n_write_lock_fail; 410 max = statp[i].n_lock_fail;
255 if (min > lwsa[i].n_write_lock_fail) 411 if (min > statp[i].n_lock_fail)
256 min = lwsa[i].n_write_lock_fail; 412 min = statp[i].n_lock_fail;
257 } 413 }
258 page += sprintf(page, "%s%s ", torture_type, TORTURE_FLAG);
259 page += sprintf(page, 414 page += sprintf(page,
260 "Writes: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", 415 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n",
416 write ? "Writes" : "Reads ",
261 sum, max, min, max / 2 > min ? "???" : "", 417 sum, max, min, max / 2 > min ? "???" : "",
262 fail, fail ? "!!!" : ""); 418 fail, fail ? "!!!" : "");
263 if (fail) 419 if (fail)
264 atomic_inc(&n_lock_torture_errors); 420 atomic_inc(&cxt.n_lock_torture_errors);
265} 421}
266 422
267/* 423/*
@@ -274,18 +430,35 @@ static void lock_torture_printk(char *page)
274 */ 430 */
275static void lock_torture_stats_print(void) 431static void lock_torture_stats_print(void)
276{ 432{
277 int size = nrealwriters_stress * 200 + 8192; 433 int size = cxt.nrealwriters_stress * 200 + 8192;
278 char *buf; 434 char *buf;
279 435
436 if (cxt.cur_ops->readlock)
437 size += cxt.nrealreaders_stress * 200 + 8192;
438
280 buf = kmalloc(size, GFP_KERNEL); 439 buf = kmalloc(size, GFP_KERNEL);
281 if (!buf) { 440 if (!buf) {
282 pr_err("lock_torture_stats_print: Out of memory, need: %d", 441 pr_err("lock_torture_stats_print: Out of memory, need: %d",
283 size); 442 size);
284 return; 443 return;
285 } 444 }
286 lock_torture_printk(buf); 445
446 __torture_print_stats(buf, cxt.lwsa, true);
287 pr_alert("%s", buf); 447 pr_alert("%s", buf);
288 kfree(buf); 448 kfree(buf);
449
450 if (cxt.cur_ops->readlock) {
451 buf = kmalloc(size, GFP_KERNEL);
452 if (!buf) {
453 pr_err("lock_torture_stats_print: Out of memory, need: %d",
454 size);
455 return;
456 }
457
458 __torture_print_stats(buf, cxt.lrsa, false);
459 pr_alert("%s", buf);
460 kfree(buf);
461 }
289} 462}
290 463
291/* 464/*
@@ -312,9 +485,10 @@ lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
312 const char *tag) 485 const char *tag)
313{ 486{
314 pr_alert("%s" TORTURE_FLAG 487 pr_alert("%s" TORTURE_FLAG
315 "--- %s: nwriters_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n", 488 "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
316 torture_type, tag, nrealwriters_stress, stat_interval, verbose, 489 torture_type, tag, cxt.debug_lock ? " [debug]": "",
317 shuffle_interval, stutter, shutdown_secs, 490 cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
491 verbose, shuffle_interval, stutter, shutdown_secs,
318 onoff_interval, onoff_holdoff); 492 onoff_interval, onoff_holdoff);
319} 493}
320 494
@@ -322,46 +496,56 @@ static void lock_torture_cleanup(void)
322{ 496{
323 int i; 497 int i;
324 498
325 if (torture_cleanup()) 499 if (torture_cleanup_begin())
326 return; 500 return;
327 501
328 if (writer_tasks) { 502 if (writer_tasks) {
329 for (i = 0; i < nrealwriters_stress; i++) 503 for (i = 0; i < cxt.nrealwriters_stress; i++)
330 torture_stop_kthread(lock_torture_writer, 504 torture_stop_kthread(lock_torture_writer,
331 writer_tasks[i]); 505 writer_tasks[i]);
332 kfree(writer_tasks); 506 kfree(writer_tasks);
333 writer_tasks = NULL; 507 writer_tasks = NULL;
334 } 508 }
335 509
510 if (reader_tasks) {
511 for (i = 0; i < cxt.nrealreaders_stress; i++)
512 torture_stop_kthread(lock_torture_reader,
513 reader_tasks[i]);
514 kfree(reader_tasks);
515 reader_tasks = NULL;
516 }
517
336 torture_stop_kthread(lock_torture_stats, stats_task); 518 torture_stop_kthread(lock_torture_stats, stats_task);
337 lock_torture_stats_print(); /* -After- the stats thread is stopped! */ 519 lock_torture_stats_print(); /* -After- the stats thread is stopped! */
338 520
339 if (atomic_read(&n_lock_torture_errors)) 521 if (atomic_read(&cxt.n_lock_torture_errors))
340 lock_torture_print_module_parms(cur_ops, 522 lock_torture_print_module_parms(cxt.cur_ops,
341 "End of test: FAILURE"); 523 "End of test: FAILURE");
342 else if (torture_onoff_failures()) 524 else if (torture_onoff_failures())
343 lock_torture_print_module_parms(cur_ops, 525 lock_torture_print_module_parms(cxt.cur_ops,
344 "End of test: LOCK_HOTPLUG"); 526 "End of test: LOCK_HOTPLUG");
345 else 527 else
346 lock_torture_print_module_parms(cur_ops, 528 lock_torture_print_module_parms(cxt.cur_ops,
347 "End of test: SUCCESS"); 529 "End of test: SUCCESS");
530 torture_cleanup_end();
348} 531}
349 532
350static int __init lock_torture_init(void) 533static int __init lock_torture_init(void)
351{ 534{
352 int i; 535 int i, j;
353 int firsterr = 0; 536 int firsterr = 0;
354 static struct lock_torture_ops *torture_ops[] = { 537 static struct lock_torture_ops *torture_ops[] = {
355 &lock_busted_ops, &spin_lock_ops, &spin_lock_irq_ops, 538 &lock_busted_ops, &spin_lock_ops, &spin_lock_irq_ops,
539 &mutex_lock_ops, &rwsem_lock_ops,
356 }; 540 };
357 541
358 if (!torture_init_begin(torture_type, verbose, &locktorture_runnable)) 542 if (!torture_init_begin(torture_type, verbose, &torture_runnable))
359 return -EBUSY; 543 return -EBUSY;
360 544
361 /* Process args and tell the world that the torturer is on the job. */ 545 /* Process args and tell the world that the torturer is on the job. */
362 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 546 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
363 cur_ops = torture_ops[i]; 547 cxt.cur_ops = torture_ops[i];
364 if (strcmp(torture_type, cur_ops->name) == 0) 548 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
365 break; 549 break;
366 } 550 }
367 if (i == ARRAY_SIZE(torture_ops)) { 551 if (i == ARRAY_SIZE(torture_ops)) {
@@ -374,31 +558,68 @@ static int __init lock_torture_init(void)
374 torture_init_end(); 558 torture_init_end();
375 return -EINVAL; 559 return -EINVAL;
376 } 560 }
377 if (cur_ops->init) 561 if (cxt.cur_ops->init)
378 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 562 cxt.cur_ops->init(); /* no "goto unwind" prior to this point!!! */
379 563
380 if (nwriters_stress >= 0) 564 if (nwriters_stress >= 0)
381 nrealwriters_stress = nwriters_stress; 565 cxt.nrealwriters_stress = nwriters_stress;
382 else 566 else
383 nrealwriters_stress = 2 * num_online_cpus(); 567 cxt.nrealwriters_stress = 2 * num_online_cpus();
384 lock_torture_print_module_parms(cur_ops, "Start of test"); 568
569#ifdef CONFIG_DEBUG_MUTEXES
570 if (strncmp(torture_type, "mutex", 5) == 0)
571 cxt.debug_lock = true;
572#endif
573#ifdef CONFIG_DEBUG_SPINLOCK
574 if (strncmp(torture_type, "spin", 4) == 0)
575 cxt.debug_lock = true;
576#endif
385 577
386 /* Initialize the statistics so that each run gets its own numbers. */ 578 /* Initialize the statistics so that each run gets its own numbers. */
387 579
388 lock_is_write_held = 0; 580 lock_is_write_held = 0;
389 lwsa = kmalloc(sizeof(*lwsa) * nrealwriters_stress, GFP_KERNEL); 581 cxt.lwsa = kmalloc(sizeof(*cxt.lwsa) * cxt.nrealwriters_stress, GFP_KERNEL);
390 if (lwsa == NULL) { 582 if (cxt.lwsa == NULL) {
391 VERBOSE_TOROUT_STRING("lwsa: Out of memory"); 583 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
392 firsterr = -ENOMEM; 584 firsterr = -ENOMEM;
393 goto unwind; 585 goto unwind;
394 } 586 }
395 for (i = 0; i < nrealwriters_stress; i++) { 587 for (i = 0; i < cxt.nrealwriters_stress; i++) {
396 lwsa[i].n_write_lock_fail = 0; 588 cxt.lwsa[i].n_lock_fail = 0;
397 lwsa[i].n_write_lock_acquired = 0; 589 cxt.lwsa[i].n_lock_acquired = 0;
398 } 590 }
399 591
400 /* Start up the kthreads. */ 592 if (cxt.cur_ops->readlock) {
593 if (nreaders_stress >= 0)
594 cxt.nrealreaders_stress = nreaders_stress;
595 else {
596 /*
597 * By default distribute evenly the number of
598 * readers and writers. We still run the same number
599 * of threads as the writer-only locks default.
600 */
601 if (nwriters_stress < 0) /* user doesn't care */
602 cxt.nrealwriters_stress = num_online_cpus();
603 cxt.nrealreaders_stress = cxt.nrealwriters_stress;
604 }
605
606 lock_is_read_held = 0;
607 cxt.lrsa = kmalloc(sizeof(*cxt.lrsa) * cxt.nrealreaders_stress, GFP_KERNEL);
608 if (cxt.lrsa == NULL) {
609 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
610 firsterr = -ENOMEM;
611 kfree(cxt.lwsa);
612 goto unwind;
613 }
614
615 for (i = 0; i < cxt.nrealreaders_stress; i++) {
616 cxt.lrsa[i].n_lock_fail = 0;
617 cxt.lrsa[i].n_lock_acquired = 0;
618 }
619 }
620 lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
401 621
622 /* Prepare torture context. */
402 if (onoff_interval > 0) { 623 if (onoff_interval > 0) {
403 firsterr = torture_onoff_init(onoff_holdoff * HZ, 624 firsterr = torture_onoff_init(onoff_holdoff * HZ,
404 onoff_interval * HZ); 625 onoff_interval * HZ);
@@ -422,18 +643,51 @@ static int __init lock_torture_init(void)
422 goto unwind; 643 goto unwind;
423 } 644 }
424 645
425 writer_tasks = kzalloc(nrealwriters_stress * sizeof(writer_tasks[0]), 646 writer_tasks = kzalloc(cxt.nrealwriters_stress * sizeof(writer_tasks[0]),
426 GFP_KERNEL); 647 GFP_KERNEL);
427 if (writer_tasks == NULL) { 648 if (writer_tasks == NULL) {
428 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory"); 649 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
429 firsterr = -ENOMEM; 650 firsterr = -ENOMEM;
430 goto unwind; 651 goto unwind;
431 } 652 }
432 for (i = 0; i < nrealwriters_stress; i++) { 653
433 firsterr = torture_create_kthread(lock_torture_writer, &lwsa[i], 654 if (cxt.cur_ops->readlock) {
655 reader_tasks = kzalloc(cxt.nrealreaders_stress * sizeof(reader_tasks[0]),
656 GFP_KERNEL);
657 if (reader_tasks == NULL) {
658 VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
659 firsterr = -ENOMEM;
660 goto unwind;
661 }
662 }
663
664 /*
665 * Create the kthreads and start torturing (oh, those poor little locks).
666 *
667 * TODO: Note that we interleave writers with readers, giving writers a
668 * slight advantage, by creating its kthread first. This can be modified
669 * for very specific needs, or even let the user choose the policy, if
670 * ever wanted.
671 */
672 for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
673 j < cxt.nrealreaders_stress; i++, j++) {
674 if (i >= cxt.nrealwriters_stress)
675 goto create_reader;
676
677 /* Create writer. */
678 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
434 writer_tasks[i]); 679 writer_tasks[i]);
435 if (firsterr) 680 if (firsterr)
436 goto unwind; 681 goto unwind;
682
683 create_reader:
684 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
685 continue;
686 /* Create reader. */
687 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
688 reader_tasks[j]);
689 if (firsterr)
690 goto unwind;
437 } 691 }
438 if (stat_interval > 0) { 692 if (stat_interval > 0) {
439 firsterr = torture_create_kthread(lock_torture_stats, NULL, 693 firsterr = torture_create_kthread(lock_torture_stats, NULL,