aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-03-16 11:22:41 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2011-03-16 11:22:41 -0400
commit79d8a8f736151b12129984b1250fd708440e742c (patch)
treef67eebe2bafca8820955ee9f851985a41fb32e66 /include
parentbd2895eeade5f11f3e5906283c630bbdb4b57454 (diff)
parentb9ec40af0e18fb7d02106be148036c2ea490fdf9 (diff)
Merge branch 'for-2.6.39' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
* 'for-2.6.39' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: percpu, x86: Add arch-specific this_cpu_cmpxchg_double() support percpu: Generic support for this_cpu_cmpxchg_double() alpha: use L1_CACHE_BYTES for cacheline size in the linker script percpu: align percpu readmostly subsection to cacheline Fix up trivial conflict in arch/x86/kernel/vmlinux.lds.S due to the percpu alignment having changed ("x86: Reduce back the alignment of the per-CPU data section")
Diffstat (limited to 'include')
-rw-r--r--include/asm-generic/vmlinux.lds.h35
-rw-r--r--include/linux/percpu.h128
2 files changed, 150 insertions, 13 deletions
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 906c3ceca9a2..32c45e5fe0ab 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -15,7 +15,7 @@
15 * HEAD_TEXT_SECTION 15 * HEAD_TEXT_SECTION
16 * INIT_TEXT_SECTION(PAGE_SIZE) 16 * INIT_TEXT_SECTION(PAGE_SIZE)
17 * INIT_DATA_SECTION(...) 17 * INIT_DATA_SECTION(...)
18 * PERCPU(PAGE_SIZE) 18 * PERCPU(CACHELINE_SIZE, PAGE_SIZE)
19 * __init_end = .; 19 * __init_end = .;
20 * 20 *
21 * _stext = .; 21 * _stext = .;
@@ -689,13 +689,18 @@
689 689
690/** 690/**
691 * PERCPU_VADDR - define output section for percpu area 691 * PERCPU_VADDR - define output section for percpu area
692 * @cacheline: cacheline size
692 * @vaddr: explicit base address (optional) 693 * @vaddr: explicit base address (optional)
693 * @phdr: destination PHDR (optional) 694 * @phdr: destination PHDR (optional)
694 * 695 *
695 * Macro which expands to output section for percpu area. If @vaddr 696 * Macro which expands to output section for percpu area.
696 * is not blank, it specifies explicit base address and all percpu 697 *
697 * symbols will be offset from the given address. If blank, @vaddr 698 * @cacheline is used to align subsections to avoid false cacheline
698 * always equals @laddr + LOAD_OFFSET. 699 * sharing between subsections for different purposes.
700 *
701 * If @vaddr is not blank, it specifies explicit base address and all
702 * percpu symbols will be offset from the given address. If blank,
703 * @vaddr always equals @laddr + LOAD_OFFSET.
699 * 704 *
700 * @phdr defines the output PHDR to use if not blank. Be warned that 705 * @phdr defines the output PHDR to use if not blank. Be warned that
701 * output PHDR is sticky. If @phdr is specified, the next output 706 * output PHDR is sticky. If @phdr is specified, the next output
@@ -706,7 +711,7 @@
706 * If there is no need to put the percpu section at a predetermined 711 * If there is no need to put the percpu section at a predetermined
707 * address, use PERCPU(). 712 * address, use PERCPU().
708 */ 713 */
709#define PERCPU_VADDR(vaddr, phdr) \ 714#define PERCPU_VADDR(cacheline, vaddr, phdr) \
710 VMLINUX_SYMBOL(__per_cpu_load) = .; \ 715 VMLINUX_SYMBOL(__per_cpu_load) = .; \
711 .data..percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load) \ 716 .data..percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load) \
712 - LOAD_OFFSET) { \ 717 - LOAD_OFFSET) { \
@@ -714,7 +719,9 @@
714 *(.data..percpu..first) \ 719 *(.data..percpu..first) \
715 . = ALIGN(PAGE_SIZE); \ 720 . = ALIGN(PAGE_SIZE); \
716 *(.data..percpu..page_aligned) \ 721 *(.data..percpu..page_aligned) \
722 . = ALIGN(cacheline); \
717 *(.data..percpu..readmostly) \ 723 *(.data..percpu..readmostly) \
724 . = ALIGN(cacheline); \
718 *(.data..percpu) \ 725 *(.data..percpu) \
719 *(.data..percpu..shared_aligned) \ 726 *(.data..percpu..shared_aligned) \
720 VMLINUX_SYMBOL(__per_cpu_end) = .; \ 727 VMLINUX_SYMBOL(__per_cpu_end) = .; \
@@ -723,18 +730,18 @@
723 730
724/** 731/**
725 * PERCPU - define output section for percpu area, simple version 732 * PERCPU - define output section for percpu area, simple version
733 * @cacheline: cacheline size
726 * @align: required alignment 734 * @align: required alignment
727 * 735 *
728 * Align to @align and outputs output section for percpu area. This 736 * Align to @align and outputs output section for percpu area. This macro
729 * macro doesn't maniuplate @vaddr or @phdr and __per_cpu_load and 737 * doesn't manipulate @vaddr or @phdr and __per_cpu_load and
730 * __per_cpu_start will be identical. 738 * __per_cpu_start will be identical.
731 * 739 *
732 * This macro is equivalent to ALIGN(align); PERCPU_VADDR( , ) except 740 * This macro is equivalent to ALIGN(@align); PERCPU_VADDR(@cacheline,,)
733 * that __per_cpu_load is defined as a relative symbol against 741 * except that __per_cpu_load is defined as a relative symbol against
734 * .data..percpu which is required for relocatable x86_32 742 * .data..percpu which is required for relocatable x86_32 configuration.
735 * configuration.
736 */ 743 */
737#define PERCPU(align) \ 744#define PERCPU(cacheline, align) \
738 . = ALIGN(align); \ 745 . = ALIGN(align); \
739 .data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \ 746 .data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \
740 VMLINUX_SYMBOL(__per_cpu_load) = .; \ 747 VMLINUX_SYMBOL(__per_cpu_load) = .; \
@@ -742,7 +749,9 @@
742 *(.data..percpu..first) \ 749 *(.data..percpu..first) \
743 . = ALIGN(PAGE_SIZE); \ 750 . = ALIGN(PAGE_SIZE); \
744 *(.data..percpu..page_aligned) \ 751 *(.data..percpu..page_aligned) \
752 . = ALIGN(cacheline); \
745 *(.data..percpu..readmostly) \ 753 *(.data..percpu..readmostly) \
754 . = ALIGN(cacheline); \
746 *(.data..percpu) \ 755 *(.data..percpu) \
747 *(.data..percpu..shared_aligned) \ 756 *(.data..percpu..shared_aligned) \
748 VMLINUX_SYMBOL(__per_cpu_end) = .; \ 757 VMLINUX_SYMBOL(__per_cpu_end) = .; \
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 27c3c6fcfad3..3a5c4449fd36 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -255,6 +255,30 @@ extern void __bad_size_call_parameter(void);
255 pscr2_ret__; \ 255 pscr2_ret__; \
256}) 256})
257 257
258/*
259 * Special handling for cmpxchg_double. cmpxchg_double is passed two
260 * percpu variables. The first has to be aligned to a double word
261 * boundary and the second has to follow directly thereafter.
262 */
263#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
264({ \
265 bool pdcrb_ret__; \
266 __verify_pcpu_ptr(&pcp1); \
267 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
268 VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
269 VM_BUG_ON((unsigned long)(&pcp2) != \
270 (unsigned long)(&pcp1) + sizeof(pcp1)); \
271 switch(sizeof(pcp1)) { \
272 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
273 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
274 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
275 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
276 default: \
277 __bad_size_call_parameter(); break; \
278 } \
279 pdcrb_ret__; \
280})
281
258#define __pcpu_size_call(stem, variable, ...) \ 282#define __pcpu_size_call(stem, variable, ...) \
259do { \ 283do { \
260 __verify_pcpu_ptr(&(variable)); \ 284 __verify_pcpu_ptr(&(variable)); \
@@ -501,6 +525,45 @@ do { \
501#endif 525#endif
502 526
503/* 527/*
528 * cmpxchg_double replaces two adjacent scalars at once. The first
529 * two parameters are per cpu variables which have to be of the same
530 * size. A truth value is returned to indicate success or failure
531 * (since a double register result is difficult to handle). There is
532 * very limited hardware support for these operations, so only certain
533 * sizes may work.
534 */
535#define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
536({ \
537 int ret__; \
538 preempt_disable(); \
539 ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \
540 oval1, oval2, nval1, nval2); \
541 preempt_enable(); \
542 ret__; \
543})
544
545#ifndef this_cpu_cmpxchg_double
546# ifndef this_cpu_cmpxchg_double_1
547# define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
548 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
549# endif
550# ifndef this_cpu_cmpxchg_double_2
551# define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
552 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
553# endif
554# ifndef this_cpu_cmpxchg_double_4
555# define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
556 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
557# endif
558# ifndef this_cpu_cmpxchg_double_8
559# define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
560 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
561# endif
562# define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
563 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
564#endif
565
566/*
504 * Generic percpu operations that do not require preemption handling. 567 * Generic percpu operations that do not require preemption handling.
505 * Either we do not care about races or the caller has the 568 * Either we do not care about races or the caller has the
506 * responsibility of handling preemptions issues. Arch code can still 569 * responsibility of handling preemptions issues. Arch code can still
@@ -703,6 +766,39 @@ do { \
703 __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval) 766 __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval)
704#endif 767#endif
705 768
769#define __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
770({ \
771 int __ret = 0; \
772 if (__this_cpu_read(pcp1) == (oval1) && \
773 __this_cpu_read(pcp2) == (oval2)) { \
774 __this_cpu_write(pcp1, (nval1)); \
775 __this_cpu_write(pcp2, (nval2)); \
776 __ret = 1; \
777 } \
778 (__ret); \
779})
780
781#ifndef __this_cpu_cmpxchg_double
782# ifndef __this_cpu_cmpxchg_double_1
783# define __this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
784 __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
785# endif
786# ifndef __this_cpu_cmpxchg_double_2
787# define __this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
788 __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
789# endif
790# ifndef __this_cpu_cmpxchg_double_4
791# define __this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
792 __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
793# endif
794# ifndef __this_cpu_cmpxchg_double_8
795# define __this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
796 __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
797# endif
798# define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
799 __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
800#endif
801
706/* 802/*
707 * IRQ safe versions of the per cpu RMW operations. Note that these operations 803 * IRQ safe versions of the per cpu RMW operations. Note that these operations
708 * are *not* safe against modification of the same variable from another 804 * are *not* safe against modification of the same variable from another
@@ -823,4 +919,36 @@ do { \
823 __pcpu_size_call_return2(irqsafe_cpu_cmpxchg_, (pcp), oval, nval) 919 __pcpu_size_call_return2(irqsafe_cpu_cmpxchg_, (pcp), oval, nval)
824#endif 920#endif
825 921
922#define irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
923({ \
924 int ret__; \
925 unsigned long flags; \
926 local_irq_save(flags); \
927 ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \
928 oval1, oval2, nval1, nval2); \
929 local_irq_restore(flags); \
930 ret__; \
931})
932
933#ifndef irqsafe_cpu_cmpxchg_double
934# ifndef irqsafe_cpu_cmpxchg_double_1
935# define irqsafe_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
936 irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
937# endif
938# ifndef irqsafe_cpu_cmpxchg_double_2
939# define irqsafe_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
940 irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
941# endif
942# ifndef irqsafe_cpu_cmpxchg_double_4
943# define irqsafe_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
944 irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
945# endif
946# ifndef irqsafe_cpu_cmpxchg_double_8
947# define irqsafe_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
948 irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
949# endif
950# define irqsafe_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
951 __pcpu_double_call_return_int(irqsafe_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
952#endif
953
826#endif /* __LINUX_PERCPU_H */ 954#endif /* __LINUX_PERCPU_H */