aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcutorture.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcutorture.c')
-rw-r--r--kernel/rcutorture.c226
1 files changed, 63 insertions, 163 deletions
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index c898f14a5b7d..ddef61871878 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -65,6 +65,8 @@ static int irqreader = 1; /* RCU readers from irq (timers). */
65static int fqs_duration; /* Duration of bursts (us), 0 to disable. */ 65static int fqs_duration; /* Duration of bursts (us), 0 to disable. */
66static int fqs_holdoff; /* Hold time within burst (us). */ 66static int fqs_holdoff; /* Hold time within burst (us). */
67static int fqs_stutter = 3; /* Wait time between bursts (s). */ 67static int fqs_stutter = 3; /* Wait time between bursts (s). */
68static bool gp_exp; /* Use expedited GP wait primitives. */
69static bool gp_normal; /* Use normal GP wait primitives. */
68static int n_barrier_cbs; /* Number of callbacks to test RCU barriers. */ 70static int n_barrier_cbs; /* Number of callbacks to test RCU barriers. */
69static int object_debug; /* Test object-debug double call_rcu()?. */ 71static int object_debug; /* Test object-debug double call_rcu()?. */
70static int onoff_interval; /* Wait time between CPU hotplugs, 0=disable. */ 72static int onoff_interval; /* Wait time between CPU hotplugs, 0=disable. */
@@ -99,6 +101,10 @@ module_param(fqs_holdoff, int, 0444);
99MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)"); 101MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
100module_param(fqs_stutter, int, 0444); 102module_param(fqs_stutter, int, 0444);
101MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)"); 103MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
104module_param(gp_normal, bool, 0444);
105MODULE_PARM_DESC(gp_normal, "Use normal (non-expedited) GP wait primitives");
106module_param(gp_exp, bool, 0444);
107MODULE_PARM_DESC(gp_exp, "Use expedited GP wait primitives");
102module_param(n_barrier_cbs, int, 0444); 108module_param(n_barrier_cbs, int, 0444);
103MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing"); 109MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
104module_param(object_debug, int, 0444); 110module_param(object_debug, int, 0444);
@@ -363,6 +369,7 @@ struct rcu_torture_ops {
363 int (*completed)(void); 369 int (*completed)(void);
364 void (*deferred_free)(struct rcu_torture *p); 370 void (*deferred_free)(struct rcu_torture *p);
365 void (*sync)(void); 371 void (*sync)(void);
372 void (*exp_sync)(void);
366 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 373 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
367 void (*cb_barrier)(void); 374 void (*cb_barrier)(void);
368 void (*fqs)(void); 375 void (*fqs)(void);
@@ -446,81 +453,27 @@ static void rcu_torture_deferred_free(struct rcu_torture *p)
446 call_rcu(&p->rtort_rcu, rcu_torture_cb); 453 call_rcu(&p->rtort_rcu, rcu_torture_cb);
447} 454}
448 455
449static struct rcu_torture_ops rcu_ops = {
450 .init = NULL,
451 .readlock = rcu_torture_read_lock,
452 .read_delay = rcu_read_delay,
453 .readunlock = rcu_torture_read_unlock,
454 .completed = rcu_torture_completed,
455 .deferred_free = rcu_torture_deferred_free,
456 .sync = synchronize_rcu,
457 .call = call_rcu,
458 .cb_barrier = rcu_barrier,
459 .fqs = rcu_force_quiescent_state,
460 .stats = NULL,
461 .irq_capable = 1,
462 .can_boost = rcu_can_boost(),
463 .name = "rcu"
464};
465
466static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
467{
468 int i;
469 struct rcu_torture *rp;
470 struct rcu_torture *rp1;
471
472 cur_ops->sync();
473 list_add(&p->rtort_free, &rcu_torture_removed);
474 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
475 i = rp->rtort_pipe_count;
476 if (i > RCU_TORTURE_PIPE_LEN)
477 i = RCU_TORTURE_PIPE_LEN;
478 atomic_inc(&rcu_torture_wcount[i]);
479 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
480 rp->rtort_mbtest = 0;
481 list_del(&rp->rtort_free);
482 rcu_torture_free(rp);
483 }
484 }
485}
486
487static void rcu_sync_torture_init(void) 456static void rcu_sync_torture_init(void)
488{ 457{
489 INIT_LIST_HEAD(&rcu_torture_removed); 458 INIT_LIST_HEAD(&rcu_torture_removed);
490} 459}
491 460
492static struct rcu_torture_ops rcu_sync_ops = { 461static struct rcu_torture_ops rcu_ops = {
493 .init = rcu_sync_torture_init, 462 .init = rcu_sync_torture_init,
494 .readlock = rcu_torture_read_lock, 463 .readlock = rcu_torture_read_lock,
495 .read_delay = rcu_read_delay, 464 .read_delay = rcu_read_delay,
496 .readunlock = rcu_torture_read_unlock, 465 .readunlock = rcu_torture_read_unlock,
497 .completed = rcu_torture_completed, 466 .completed = rcu_torture_completed,
498 .deferred_free = rcu_sync_torture_deferred_free, 467 .deferred_free = rcu_torture_deferred_free,
499 .sync = synchronize_rcu, 468 .sync = synchronize_rcu,
500 .call = NULL, 469 .exp_sync = synchronize_rcu_expedited,
501 .cb_barrier = NULL, 470 .call = call_rcu,
502 .fqs = rcu_force_quiescent_state, 471 .cb_barrier = rcu_barrier,
503 .stats = NULL,
504 .irq_capable = 1,
505 .can_boost = rcu_can_boost(),
506 .name = "rcu_sync"
507};
508
509static struct rcu_torture_ops rcu_expedited_ops = {
510 .init = rcu_sync_torture_init,
511 .readlock = rcu_torture_read_lock,
512 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
513 .readunlock = rcu_torture_read_unlock,
514 .completed = rcu_no_completed,
515 .deferred_free = rcu_sync_torture_deferred_free,
516 .sync = synchronize_rcu_expedited,
517 .call = NULL,
518 .cb_barrier = NULL,
519 .fqs = rcu_force_quiescent_state, 472 .fqs = rcu_force_quiescent_state,
520 .stats = NULL, 473 .stats = NULL,
521 .irq_capable = 1, 474 .irq_capable = 1,
522 .can_boost = rcu_can_boost(), 475 .can_boost = rcu_can_boost(),
523 .name = "rcu_expedited" 476 .name = "rcu"
524}; 477};
525 478
526/* 479/*
@@ -549,13 +502,14 @@ static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
549} 502}
550 503
551static struct rcu_torture_ops rcu_bh_ops = { 504static struct rcu_torture_ops rcu_bh_ops = {
552 .init = NULL, 505 .init = rcu_sync_torture_init,
553 .readlock = rcu_bh_torture_read_lock, 506 .readlock = rcu_bh_torture_read_lock,
554 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 507 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
555 .readunlock = rcu_bh_torture_read_unlock, 508 .readunlock = rcu_bh_torture_read_unlock,
556 .completed = rcu_bh_torture_completed, 509 .completed = rcu_bh_torture_completed,
557 .deferred_free = rcu_bh_torture_deferred_free, 510 .deferred_free = rcu_bh_torture_deferred_free,
558 .sync = synchronize_rcu_bh, 511 .sync = synchronize_rcu_bh,
512 .exp_sync = synchronize_rcu_bh_expedited,
559 .call = call_rcu_bh, 513 .call = call_rcu_bh,
560 .cb_barrier = rcu_barrier_bh, 514 .cb_barrier = rcu_barrier_bh,
561 .fqs = rcu_bh_force_quiescent_state, 515 .fqs = rcu_bh_force_quiescent_state,
@@ -564,38 +518,6 @@ static struct rcu_torture_ops rcu_bh_ops = {
564 .name = "rcu_bh" 518 .name = "rcu_bh"
565}; 519};
566 520
567static struct rcu_torture_ops rcu_bh_sync_ops = {
568 .init = rcu_sync_torture_init,
569 .readlock = rcu_bh_torture_read_lock,
570 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
571 .readunlock = rcu_bh_torture_read_unlock,
572 .completed = rcu_bh_torture_completed,
573 .deferred_free = rcu_sync_torture_deferred_free,
574 .sync = synchronize_rcu_bh,
575 .call = NULL,
576 .cb_barrier = NULL,
577 .fqs = rcu_bh_force_quiescent_state,
578 .stats = NULL,
579 .irq_capable = 1,
580 .name = "rcu_bh_sync"
581};
582
583static struct rcu_torture_ops rcu_bh_expedited_ops = {
584 .init = rcu_sync_torture_init,
585 .readlock = rcu_bh_torture_read_lock,
586 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
587 .readunlock = rcu_bh_torture_read_unlock,
588 .completed = rcu_bh_torture_completed,
589 .deferred_free = rcu_sync_torture_deferred_free,
590 .sync = synchronize_rcu_bh_expedited,
591 .call = NULL,
592 .cb_barrier = NULL,
593 .fqs = rcu_bh_force_quiescent_state,
594 .stats = NULL,
595 .irq_capable = 1,
596 .name = "rcu_bh_expedited"
597};
598
599/* 521/*
600 * Definitions for srcu torture testing. 522 * Definitions for srcu torture testing.
601 */ 523 */
@@ -670,6 +592,11 @@ static int srcu_torture_stats(char *page)
670 return cnt; 592 return cnt;
671} 593}
672 594
595static void srcu_torture_synchronize_expedited(void)
596{
597 synchronize_srcu_expedited(&srcu_ctl);
598}
599
673static struct rcu_torture_ops srcu_ops = { 600static struct rcu_torture_ops srcu_ops = {
674 .init = rcu_sync_torture_init, 601 .init = rcu_sync_torture_init,
675 .readlock = srcu_torture_read_lock, 602 .readlock = srcu_torture_read_lock,
@@ -678,45 +605,13 @@ static struct rcu_torture_ops srcu_ops = {
678 .completed = srcu_torture_completed, 605 .completed = srcu_torture_completed,
679 .deferred_free = srcu_torture_deferred_free, 606 .deferred_free = srcu_torture_deferred_free,
680 .sync = srcu_torture_synchronize, 607 .sync = srcu_torture_synchronize,
608 .exp_sync = srcu_torture_synchronize_expedited,
681 .call = srcu_torture_call, 609 .call = srcu_torture_call,
682 .cb_barrier = srcu_torture_barrier, 610 .cb_barrier = srcu_torture_barrier,
683 .stats = srcu_torture_stats, 611 .stats = srcu_torture_stats,
684 .name = "srcu" 612 .name = "srcu"
685}; 613};
686 614
687static struct rcu_torture_ops srcu_sync_ops = {
688 .init = rcu_sync_torture_init,
689 .readlock = srcu_torture_read_lock,
690 .read_delay = srcu_read_delay,
691 .readunlock = srcu_torture_read_unlock,
692 .completed = srcu_torture_completed,
693 .deferred_free = rcu_sync_torture_deferred_free,
694 .sync = srcu_torture_synchronize,
695 .call = NULL,
696 .cb_barrier = NULL,
697 .stats = srcu_torture_stats,
698 .name = "srcu_sync"
699};
700
701static void srcu_torture_synchronize_expedited(void)
702{
703 synchronize_srcu_expedited(&srcu_ctl);
704}
705
706static struct rcu_torture_ops srcu_expedited_ops = {
707 .init = rcu_sync_torture_init,
708 .readlock = srcu_torture_read_lock,
709 .read_delay = srcu_read_delay,
710 .readunlock = srcu_torture_read_unlock,
711 .completed = srcu_torture_completed,
712 .deferred_free = rcu_sync_torture_deferred_free,
713 .sync = srcu_torture_synchronize_expedited,
714 .call = NULL,
715 .cb_barrier = NULL,
716 .stats = srcu_torture_stats,
717 .name = "srcu_expedited"
718};
719
720/* 615/*
721 * Definitions for sched torture testing. 616 * Definitions for sched torture testing.
722 */ 617 */
@@ -745,6 +640,8 @@ static struct rcu_torture_ops sched_ops = {
745 .completed = rcu_no_completed, 640 .completed = rcu_no_completed,
746 .deferred_free = rcu_sched_torture_deferred_free, 641 .deferred_free = rcu_sched_torture_deferred_free,
747 .sync = synchronize_sched, 642 .sync = synchronize_sched,
643 .exp_sync = synchronize_sched_expedited,
644 .call = call_rcu_sched,
748 .cb_barrier = rcu_barrier_sched, 645 .cb_barrier = rcu_barrier_sched,
749 .fqs = rcu_sched_force_quiescent_state, 646 .fqs = rcu_sched_force_quiescent_state,
750 .stats = NULL, 647 .stats = NULL,
@@ -752,35 +649,6 @@ static struct rcu_torture_ops sched_ops = {
752 .name = "sched" 649 .name = "sched"
753}; 650};
754 651
755static struct rcu_torture_ops sched_sync_ops = {
756 .init = rcu_sync_torture_init,
757 .readlock = sched_torture_read_lock,
758 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
759 .readunlock = sched_torture_read_unlock,
760 .completed = rcu_no_completed,
761 .deferred_free = rcu_sync_torture_deferred_free,
762 .sync = synchronize_sched,
763 .cb_barrier = NULL,
764 .fqs = rcu_sched_force_quiescent_state,
765 .stats = NULL,
766 .name = "sched_sync"
767};
768
769static struct rcu_torture_ops sched_expedited_ops = {
770 .init = rcu_sync_torture_init,
771 .readlock = sched_torture_read_lock,
772 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
773 .readunlock = sched_torture_read_unlock,
774 .completed = rcu_no_completed,
775 .deferred_free = rcu_sync_torture_deferred_free,
776 .sync = synchronize_sched_expedited,
777 .cb_barrier = NULL,
778 .fqs = rcu_sched_force_quiescent_state,
779 .stats = NULL,
780 .irq_capable = 1,
781 .name = "sched_expedited"
782};
783
784/* 652/*
785 * RCU torture priority-boost testing. Runs one real-time thread per 653 * RCU torture priority-boost testing. Runs one real-time thread per
786 * CPU for moderate bursts, repeatedly registering RCU callbacks and 654 * CPU for moderate bursts, repeatedly registering RCU callbacks and
@@ -930,9 +798,11 @@ rcu_torture_fqs(void *arg)
930static int 798static int
931rcu_torture_writer(void *arg) 799rcu_torture_writer(void *arg)
932{ 800{
801 bool exp;
933 int i; 802 int i;
934 long oldbatch = rcu_batches_completed(); 803 long oldbatch = rcu_batches_completed();
935 struct rcu_torture *rp; 804 struct rcu_torture *rp;
805 struct rcu_torture *rp1;
936 struct rcu_torture *old_rp; 806 struct rcu_torture *old_rp;
937 static DEFINE_RCU_RANDOM(rand); 807 static DEFINE_RCU_RANDOM(rand);
938 808
@@ -957,7 +827,31 @@ rcu_torture_writer(void *arg)
957 i = RCU_TORTURE_PIPE_LEN; 827 i = RCU_TORTURE_PIPE_LEN;
958 atomic_inc(&rcu_torture_wcount[i]); 828 atomic_inc(&rcu_torture_wcount[i]);
959 old_rp->rtort_pipe_count++; 829 old_rp->rtort_pipe_count++;
960 cur_ops->deferred_free(old_rp); 830 if (gp_normal == gp_exp)
831 exp = !!(rcu_random(&rand) & 0x80);
832 else
833 exp = gp_exp;
834 if (!exp) {
835 cur_ops->deferred_free(old_rp);
836 } else {
837 cur_ops->exp_sync();
838 list_add(&old_rp->rtort_free,
839 &rcu_torture_removed);
840 list_for_each_entry_safe(rp, rp1,
841 &rcu_torture_removed,
842 rtort_free) {
843 i = rp->rtort_pipe_count;
844 if (i > RCU_TORTURE_PIPE_LEN)
845 i = RCU_TORTURE_PIPE_LEN;
846 atomic_inc(&rcu_torture_wcount[i]);
847 if (++rp->rtort_pipe_count >=
848 RCU_TORTURE_PIPE_LEN) {
849 rp->rtort_mbtest = 0;
850 list_del(&rp->rtort_free);
851 rcu_torture_free(rp);
852 }
853 }
854 }
961 } 855 }
962 rcutorture_record_progress(++rcu_torture_current_version); 856 rcutorture_record_progress(++rcu_torture_current_version);
963 oldbatch = cur_ops->completed(); 857 oldbatch = cur_ops->completed();
@@ -986,10 +880,18 @@ rcu_torture_fakewriter(void *arg)
986 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10); 880 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
987 udelay(rcu_random(&rand) & 0x3ff); 881 udelay(rcu_random(&rand) & 0x3ff);
988 if (cur_ops->cb_barrier != NULL && 882 if (cur_ops->cb_barrier != NULL &&
989 rcu_random(&rand) % (nfakewriters * 8) == 0) 883 rcu_random(&rand) % (nfakewriters * 8) == 0) {
990 cur_ops->cb_barrier(); 884 cur_ops->cb_barrier();
991 else 885 } else if (gp_normal == gp_exp) {
886 if (rcu_random(&rand) & 0x80)
887 cur_ops->sync();
888 else
889 cur_ops->exp_sync();
890 } else if (gp_normal) {
992 cur_ops->sync(); 891 cur_ops->sync();
892 } else {
893 cur_ops->exp_sync();
894 }
993 rcu_stutter_wait("rcu_torture_fakewriter"); 895 rcu_stutter_wait("rcu_torture_fakewriter");
994 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 896 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
995 897
@@ -2000,11 +1902,9 @@ rcu_torture_init(void)
2000 int cpu; 1902 int cpu;
2001 int firsterr = 0; 1903 int firsterr = 0;
2002 int retval; 1904 int retval;
2003 static struct rcu_torture_ops *torture_ops[] = 1905 static struct rcu_torture_ops *torture_ops[] = {
2004 { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops, 1906 &rcu_ops, &rcu_bh_ops, &srcu_ops, &sched_ops,
2005 &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops, 1907 };
2006 &srcu_ops, &srcu_sync_ops, &srcu_expedited_ops,
2007 &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
2008 1908
2009 mutex_lock(&fullstop_mutex); 1909 mutex_lock(&fullstop_mutex);
2010 1910
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-15 17:08:12 -0500