From 0a3aee0da4402aa19b66e458038533c896fb80c6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 18 Dec 2010 16:28:55 +0100 Subject: x86: Use this_cpu_ops to optimize code Go through x86 code and replace __get_cpu_var and get_cpu_var instances that refer to a scalar and are not used for address determinations. Cc: Yinghai Lu Cc: Ingo Molnar Acked-by: Tejun Heo Acked-by: "H. Peter Anvin" Signed-off-by: Christoph Lameter Signed-off-by: Tejun Heo --- arch/x86/kernel/apic/io_apic.c | 4 ++-- arch/x86/kernel/apic/nmi.c | 24 ++++++++++++------------ arch/x86/kernel/apic/x2apic_uv_x.c | 8 ++++---- 3 files changed, 18 insertions(+), 18 deletions(-) (limited to 'arch/x86/kernel/apic') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 7cc0a721f628..8d50922687af 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2302,7 +2302,7 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void) unsigned int irr; struct irq_desc *desc; struct irq_cfg *cfg; - irq = __get_cpu_var(vector_irq)[vector]; + irq = __this_cpu_read(vector_irq[vector]); if (irq == -1) continue; @@ -2336,7 +2336,7 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void) apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR); goto unlock; } - __get_cpu_var(vector_irq)[vector] = -1; + __this_cpu_write(vector_irq[vector], -1); unlock: raw_spin_unlock(&desc->lock); } diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c index c90041ccb742..b387dce0b409 100644 --- a/arch/x86/kernel/apic/nmi.c +++ b/arch/x86/kernel/apic/nmi.c @@ -306,12 +306,12 @@ void acpi_nmi_disable(void) */ void cpu_nmi_set_wd_enabled(void) { - __get_cpu_var(wd_enabled) = 1; + __this_cpu_write(wd_enabled, 1); } void setup_apic_nmi_watchdog(void *unused) { - if (__get_cpu_var(wd_enabled)) + if (__this_cpu_read(wd_enabled)) return; /* cheap hack to support suspend/resume */ @@ -322,12 +322,12 @@ void setup_apic_nmi_watchdog(void *unused) switch (nmi_watchdog) { case NMI_LOCAL_APIC: if (lapic_watchdog_init(nmi_hz) < 0) { - __get_cpu_var(wd_enabled) = 0; + __this_cpu_write(wd_enabled, 0); return; } /* FALL THROUGH */ case NMI_IO_APIC: - __get_cpu_var(wd_enabled) = 1; + __this_cpu_write(wd_enabled, 1); atomic_inc(&nmi_active); } } @@ -337,13 +337,13 @@ void stop_apic_nmi_watchdog(void *unused) /* only support LOCAL and IO APICs for now */ if (!nmi_watchdog_active()) return; - if (__get_cpu_var(wd_enabled) == 0) + if (__this_cpu_read(wd_enabled) == 0) return; if (nmi_watchdog == NMI_LOCAL_APIC) lapic_watchdog_stop(); else __acpi_nmi_disable(NULL); - __get_cpu_var(wd_enabled) = 0; + __this_cpu_write(wd_enabled, 0); atomic_dec(&nmi_active); } @@ -403,8 +403,8 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) sum = get_timer_irqs(cpu); - if (__get_cpu_var(nmi_touch)) { - __get_cpu_var(nmi_touch) = 0; + if (__this_cpu_read(nmi_touch)) { + __this_cpu_write(nmi_touch, 0); touched = 1; } @@ -427,7 +427,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) touched = 1; /* if the none of the timers isn't firing, this cpu isn't doing much */ - if (!touched && __get_cpu_var(last_irq_sum) == sum) { + if (!touched && __this_cpu_read(last_irq_sum) == sum) { /* * Ayiee, looks like this CPU is stuck ... * wait a few IRQs (5 seconds) before doing the oops ... @@ -440,12 +440,12 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) die_nmi("BUG: NMI Watchdog detected LOCKUP", regs, panic_on_timeout); } else { - __get_cpu_var(last_irq_sum) = sum; + __this_cpu_write(last_irq_sum, sum); __this_cpu_write(alert_counter, 0); } /* see if the nmi watchdog went off */ - if (!__get_cpu_var(wd_enabled)) + if (!__this_cpu_read(wd_enabled)) return rc; switch (nmi_watchdog) { case NMI_LOCAL_APIC: @@ -467,7 +467,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) static void enable_ioapic_nmi_watchdog_single(void *unused) { - __get_cpu_var(wd_enabled) = 1; + __this_cpu_write(wd_enabled, 1); atomic_inc(&nmi_active); __acpi_nmi_enable(NULL); } diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index c1c52c341f40..26ec9a7c3518 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -118,8 +118,8 @@ static int __init uv_acpi_madt_oem_check(char *oem_id, char *oem_table_id) else if (!strcmp(oem_table_id, "UVX")) uv_system_type = UV_X2APIC; else if (!strcmp(oem_table_id, "UVH")) { - __get_cpu_var(x2apic_extra_bits) = - nodeid << (uvh_apicid.s.pnode_shift - 1); + __this_cpu_write(x2apic_extra_bits, + nodeid << (uvh_apicid.s.pnode_shift - 1)); uv_system_type = UV_NON_UNIQUE_APIC; uv_set_apicid_hibit(); return 1; @@ -284,7 +284,7 @@ static unsigned int x2apic_get_apic_id(unsigned long x) unsigned int id; WARN_ON(preemptible() && num_online_cpus() > 1); - id = x | __get_cpu_var(x2apic_extra_bits); + id = x | __this_cpu_read(x2apic_extra_bits); return id; } @@ -376,7 +376,7 @@ struct apic __refdata apic_x2apic_uv_x = { static __cpuinit void set_x2apic_extra_bits(int pnode) { - __get_cpu_var(x2apic_extra_bits) = (pnode << 6); + __this_cpu_write(x2apic_extra_bits, (pnode << 6)); } /* -- cgit v1.2.2 From 7b543a5334ff4ea2e3ad3b777fc23cdb8072a988 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 18 Dec 2010 16:30:05 +0100 Subject: x86: Replace uses of current_cpu_data with this_cpu ops Replace all uses of current_cpu_data with this_cpu operations on the per cpu structure cpu_info. The scala accesses are replaced with the matching this_cpu ops which results in smaller and more efficient code. In the long run, it might be a good idea to remove cpu_data() macro too and use per_cpu macro directly. tj: updated description Cc: Yinghai Lu Cc: Ingo Molnar Acked-by: H. Peter Anvin Acked-by: Tejun Heo Signed-off-by: Christoph Lameter Signed-off-by: Tejun Heo --- arch/x86/kernel/apic/apic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel/apic') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 3f838d537392..8accfe3b34d7 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -516,7 +516,7 @@ static void __cpuinit setup_APIC_timer(void) { struct clock_event_device *levt = &__get_cpu_var(lapic_events); - if (cpu_has(¤t_cpu_data, X86_FEATURE_ARAT)) { + if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_ARAT)) { lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP; /* Make LAPIC timer preferrable over percpu HPET */ lapic_clockevent.rating = 150; -- cgit v1.2.2 From c1955b5f3a95717ce1f5235f6e9968da068e3183 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 18 Dec 2010 16:30:48 +0100 Subject: x86: Use this_cpu_inc_return for nmi counter this_cpu_inc_return() saves us a memory access there. Reviewed-by: Pekka Enberg Reviewed-by: Mathieu Desnoyers Acked-by: H. Peter Anvin Acked-by: Tejun Heo Signed-off-by: Christoph Lameter Signed-off-by: Tejun Heo --- arch/x86/kernel/apic/nmi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'arch/x86/kernel/apic') diff --git a/arch/x86/kernel/apic/nmi.c b/arch/x86/kernel/apic/nmi.c index b387dce0b409..37769cc4fe55 100644 --- a/arch/x86/kernel/apic/nmi.c +++ b/arch/x86/kernel/apic/nmi.c @@ -432,8 +432,7 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) * Ayiee, looks like this CPU is stuck ... * wait a few IRQs (5 seconds) before doing the oops ... */ - __this_cpu_inc(alert_counter); - if (__this_cpu_read(alert_counter) == 5 * nmi_hz) + if (__this_cpu_inc_return(alert_counter) == 5 * nmi_hz) /* * die_nmi will return ONLY if NOTIFY_STOP happens.. */ -- cgit v1.2.2