aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcu
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcu')
-rw-r--r--kernel/rcu/tree.c22
-rw-r--r--kernel/rcu/tree_plugin.h8
2 files changed, 15 insertions, 15 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3e3f13e8b429..f1ba77363fbb 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -458,9 +458,9 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
458 } 458 }
459 rcu_prepare_for_idle(smp_processor_id()); 459 rcu_prepare_for_idle(smp_processor_id());
460 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 460 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
461 smp_mb__before_atomic_inc(); /* See above. */ 461 smp_mb__before_atomic(); /* See above. */
462 atomic_inc(&rdtp->dynticks); 462 atomic_inc(&rdtp->dynticks);
463 smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */ 463 smp_mb__after_atomic(); /* Force ordering with next sojourn. */
464 WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 464 WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
465 465
466 /* 466 /*
@@ -578,10 +578,10 @@ void rcu_irq_exit(void)
578static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval, 578static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
579 int user) 579 int user)
580{ 580{
581 smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ 581 smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */
582 atomic_inc(&rdtp->dynticks); 582 atomic_inc(&rdtp->dynticks);
583 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 583 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
584 smp_mb__after_atomic_inc(); /* See above. */ 584 smp_mb__after_atomic(); /* See above. */
585 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 585 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
586 rcu_cleanup_after_idle(smp_processor_id()); 586 rcu_cleanup_after_idle(smp_processor_id());
587 trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting); 587 trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
@@ -706,10 +706,10 @@ void rcu_nmi_enter(void)
706 (atomic_read(&rdtp->dynticks) & 0x1)) 706 (atomic_read(&rdtp->dynticks) & 0x1))
707 return; 707 return;
708 rdtp->dynticks_nmi_nesting++; 708 rdtp->dynticks_nmi_nesting++;
709 smp_mb__before_atomic_inc(); /* Force delay from prior write. */ 709 smp_mb__before_atomic(); /* Force delay from prior write. */
710 atomic_inc(&rdtp->dynticks); 710 atomic_inc(&rdtp->dynticks);
711 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 711 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
712 smp_mb__after_atomic_inc(); /* See above. */ 712 smp_mb__after_atomic(); /* See above. */
713 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 713 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
714} 714}
715 715
@@ -728,9 +728,9 @@ void rcu_nmi_exit(void)
728 --rdtp->dynticks_nmi_nesting != 0) 728 --rdtp->dynticks_nmi_nesting != 0)
729 return; 729 return;
730 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 730 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
731 smp_mb__before_atomic_inc(); /* See above. */ 731 smp_mb__before_atomic(); /* See above. */
732 atomic_inc(&rdtp->dynticks); 732 atomic_inc(&rdtp->dynticks);
733 smp_mb__after_atomic_inc(); /* Force delay to next write. */ 733 smp_mb__after_atomic(); /* Force delay to next write. */
734 WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 734 WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
735} 735}
736 736
@@ -2918,7 +2918,7 @@ void synchronize_sched_expedited(void)
2918 s = atomic_long_read(&rsp->expedited_done); 2918 s = atomic_long_read(&rsp->expedited_done);
2919 if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) { 2919 if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
2920 /* ensure test happens before caller kfree */ 2920 /* ensure test happens before caller kfree */
2921 smp_mb__before_atomic_inc(); /* ^^^ */ 2921 smp_mb__before_atomic(); /* ^^^ */
2922 atomic_long_inc(&rsp->expedited_workdone1); 2922 atomic_long_inc(&rsp->expedited_workdone1);
2923 return; 2923 return;
2924 } 2924 }
@@ -2936,7 +2936,7 @@ void synchronize_sched_expedited(void)
2936 s = atomic_long_read(&rsp->expedited_done); 2936 s = atomic_long_read(&rsp->expedited_done);
2937 if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) { 2937 if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
2938 /* ensure test happens before caller kfree */ 2938 /* ensure test happens before caller kfree */
2939 smp_mb__before_atomic_inc(); /* ^^^ */ 2939 smp_mb__before_atomic(); /* ^^^ */
2940 atomic_long_inc(&rsp->expedited_workdone2); 2940 atomic_long_inc(&rsp->expedited_workdone2);
2941 return; 2941 return;
2942 } 2942 }
@@ -2965,7 +2965,7 @@ void synchronize_sched_expedited(void)
2965 s = atomic_long_read(&rsp->expedited_done); 2965 s = atomic_long_read(&rsp->expedited_done);
2966 if (ULONG_CMP_GE((ulong)s, (ulong)snap)) { 2966 if (ULONG_CMP_GE((ulong)s, (ulong)snap)) {
2967 /* ensure test happens before caller kfree */ 2967 /* ensure test happens before caller kfree */
2968 smp_mb__before_atomic_inc(); /* ^^^ */ 2968 smp_mb__before_atomic(); /* ^^^ */
2969 atomic_long_inc(&rsp->expedited_done_lost); 2969 atomic_long_inc(&rsp->expedited_done_lost);
2970 break; 2970 break;
2971 } 2971 }
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 29977ae84e7e..cbc2c45265e2 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2462,9 +2462,9 @@ static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
2462 /* Record start of fully idle period. */ 2462 /* Record start of fully idle period. */
2463 j = jiffies; 2463 j = jiffies;
2464 ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j; 2464 ACCESS_ONCE(rdtp->dynticks_idle_jiffies) = j;
2465 smp_mb__before_atomic_inc(); 2465 smp_mb__before_atomic();
2466 atomic_inc(&rdtp->dynticks_idle); 2466 atomic_inc(&rdtp->dynticks_idle);
2467 smp_mb__after_atomic_inc(); 2467 smp_mb__after_atomic();
2468 WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1); 2468 WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
2469} 2469}
2470 2470
@@ -2529,9 +2529,9 @@ static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
2529 } 2529 }
2530 2530
2531 /* Record end of idle period. */ 2531 /* Record end of idle period. */
2532 smp_mb__before_atomic_inc(); 2532 smp_mb__before_atomic();
2533 atomic_inc(&rdtp->dynticks_idle); 2533 atomic_inc(&rdtp->dynticks_idle);
2534 smp_mb__after_atomic_inc(); 2534 smp_mb__after_atomic();
2535 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1)); 2535 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
2536 2536
2537 /* 2537 /*