diff options
author | Glenn Elliott <gelliott@cs.unc.edu> | 2012-03-04 19:47:13 -0500 |
---|---|---|
committer | Glenn Elliott <gelliott@cs.unc.edu> | 2012-03-04 19:47:13 -0500 |
commit | c71c03bda1e86c9d5198c5d83f712e695c4f2a1e (patch) | |
tree | ecb166cb3e2b7e2adb3b5e292245fefd23381ac8 /include/linux/percpu.h | |
parent | ea53c912f8a86a8567697115b6a0d8152beee5c8 (diff) | |
parent | 6a00f206debf8a5c8899055726ad127dbeeed098 (diff) |
Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp
Conflicts:
litmus/sched_cedf.c
Diffstat (limited to 'include/linux/percpu.h')
-rw-r--r-- | include/linux/percpu.h | 374 |
1 files changed, 349 insertions, 25 deletions
diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 49466b13c5c6..9ca008f0c542 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h | |||
@@ -39,10 +39,17 @@ | |||
39 | preempt_enable(); \ | 39 | preempt_enable(); \ |
40 | } while (0) | 40 | } while (0) |
41 | 41 | ||
42 | #ifdef CONFIG_SMP | 42 | #define get_cpu_ptr(var) ({ \ |
43 | preempt_disable(); \ | ||
44 | this_cpu_ptr(var); }) | ||
45 | |||
46 | #define put_cpu_ptr(var) do { \ | ||
47 | (void)(var); \ | ||
48 | preempt_enable(); \ | ||
49 | } while (0) | ||
43 | 50 | ||
44 | /* minimum unit size, also is the maximum supported allocation size */ | 51 | /* minimum unit size, also is the maximum supported allocation size */ |
45 | #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10) | 52 | #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) |
46 | 53 | ||
47 | /* | 54 | /* |
48 | * Percpu allocator can serve percpu allocations before slab is | 55 | * Percpu allocator can serve percpu allocations before slab is |
@@ -137,37 +144,20 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size, | |||
137 | * dynamically allocated. Non-atomic access to the current CPU's | 144 | * dynamically allocated. Non-atomic access to the current CPU's |
138 | * version should probably be combined with get_cpu()/put_cpu(). | 145 | * version should probably be combined with get_cpu()/put_cpu(). |
139 | */ | 146 | */ |
147 | #ifdef CONFIG_SMP | ||
140 | #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) | 148 | #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) |
149 | #else | ||
150 | #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) | ||
151 | #endif | ||
141 | 152 | ||
142 | extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); | 153 | extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); |
143 | extern bool is_kernel_percpu_address(unsigned long addr); | 154 | extern bool is_kernel_percpu_address(unsigned long addr); |
144 | 155 | ||
145 | #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA | 156 | #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) |
146 | extern void __init setup_per_cpu_areas(void); | 157 | extern void __init setup_per_cpu_areas(void); |
147 | #endif | 158 | #endif |
148 | extern void __init percpu_init_late(void); | 159 | extern void __init percpu_init_late(void); |
149 | 160 | ||
150 | #else /* CONFIG_SMP */ | ||
151 | |||
152 | #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) | ||
153 | |||
154 | /* can't distinguish from other static vars, always false */ | ||
155 | static inline bool is_kernel_percpu_address(unsigned long addr) | ||
156 | { | ||
157 | return false; | ||
158 | } | ||
159 | |||
160 | static inline void __init setup_per_cpu_areas(void) { } | ||
161 | |||
162 | static inline void __init percpu_init_late(void) { } | ||
163 | |||
164 | static inline void *pcpu_lpage_remapped(void *kaddr) | ||
165 | { | ||
166 | return NULL; | ||
167 | } | ||
168 | |||
169 | #endif /* CONFIG_SMP */ | ||
170 | |||
171 | extern void __percpu *__alloc_percpu(size_t size, size_t align); | 161 | extern void __percpu *__alloc_percpu(size_t size, size_t align); |
172 | extern void free_percpu(void __percpu *__pdata); | 162 | extern void free_percpu(void __percpu *__pdata); |
173 | extern phys_addr_t per_cpu_ptr_to_phys(void *addr); | 163 | extern phys_addr_t per_cpu_ptr_to_phys(void *addr); |
@@ -250,6 +240,48 @@ extern void __bad_size_call_parameter(void); | |||
250 | pscr_ret__; \ | 240 | pscr_ret__; \ |
251 | }) | 241 | }) |
252 | 242 | ||
243 | #define __pcpu_size_call_return2(stem, variable, ...) \ | ||
244 | ({ \ | ||
245 | typeof(variable) pscr2_ret__; \ | ||
246 | __verify_pcpu_ptr(&(variable)); \ | ||
247 | switch(sizeof(variable)) { \ | ||
248 | case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ | ||
249 | case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ | ||
250 | case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ | ||
251 | case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ | ||
252 | default: \ | ||
253 | __bad_size_call_parameter(); break; \ | ||
254 | } \ | ||
255 | pscr2_ret__; \ | ||
256 | }) | ||
257 | |||
258 | /* | ||
259 | * Special handling for cmpxchg_double. cmpxchg_double is passed two | ||
260 | * percpu variables. The first has to be aligned to a double word | ||
261 | * boundary and the second has to follow directly thereafter. | ||
262 | * We enforce this on all architectures even if they don't support | ||
263 | * a double cmpxchg instruction, since it's a cheap requirement, and it | ||
264 | * avoids breaking the requirement for architectures with the instruction. | ||
265 | */ | ||
266 | #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \ | ||
267 | ({ \ | ||
268 | bool pdcrb_ret__; \ | ||
269 | __verify_pcpu_ptr(&pcp1); \ | ||
270 | BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ | ||
271 | VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \ | ||
272 | VM_BUG_ON((unsigned long)(&pcp2) != \ | ||
273 | (unsigned long)(&pcp1) + sizeof(pcp1)); \ | ||
274 | switch(sizeof(pcp1)) { \ | ||
275 | case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ | ||
276 | case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ | ||
277 | case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ | ||
278 | case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ | ||
279 | default: \ | ||
280 | __bad_size_call_parameter(); break; \ | ||
281 | } \ | ||
282 | pdcrb_ret__; \ | ||
283 | }) | ||
284 | |||
253 | #define __pcpu_size_call(stem, variable, ...) \ | 285 | #define __pcpu_size_call(stem, variable, ...) \ |
254 | do { \ | 286 | do { \ |
255 | __verify_pcpu_ptr(&(variable)); \ | 287 | __verify_pcpu_ptr(&(variable)); \ |
@@ -412,6 +444,128 @@ do { \ | |||
412 | # define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) | 444 | # define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) |
413 | #endif | 445 | #endif |
414 | 446 | ||
447 | #define _this_cpu_generic_add_return(pcp, val) \ | ||
448 | ({ \ | ||
449 | typeof(pcp) ret__; \ | ||
450 | preempt_disable(); \ | ||
451 | __this_cpu_add(pcp, val); \ | ||
452 | ret__ = __this_cpu_read(pcp); \ | ||
453 | preempt_enable(); \ | ||
454 | ret__; \ | ||
455 | }) | ||
456 | |||
457 | #ifndef this_cpu_add_return | ||
458 | # ifndef this_cpu_add_return_1 | ||
459 | # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val) | ||
460 | # endif | ||
461 | # ifndef this_cpu_add_return_2 | ||
462 | # define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val) | ||
463 | # endif | ||
464 | # ifndef this_cpu_add_return_4 | ||
465 | # define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val) | ||
466 | # endif | ||
467 | # ifndef this_cpu_add_return_8 | ||
468 | # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val) | ||
469 | # endif | ||
470 | # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) | ||
471 | #endif | ||
472 | |||
473 | #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(val)) | ||
474 | #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) | ||
475 | #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) | ||
476 | |||
477 | #define _this_cpu_generic_xchg(pcp, nval) \ | ||
478 | ({ typeof(pcp) ret__; \ | ||
479 | preempt_disable(); \ | ||
480 | ret__ = __this_cpu_read(pcp); \ | ||
481 | __this_cpu_write(pcp, nval); \ | ||
482 | preempt_enable(); \ | ||
483 | ret__; \ | ||
484 | }) | ||
485 | |||
486 | #ifndef this_cpu_xchg | ||
487 | # ifndef this_cpu_xchg_1 | ||
488 | # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval) | ||
489 | # endif | ||
490 | # ifndef this_cpu_xchg_2 | ||
491 | # define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval) | ||
492 | # endif | ||
493 | # ifndef this_cpu_xchg_4 | ||
494 | # define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval) | ||
495 | # endif | ||
496 | # ifndef this_cpu_xchg_8 | ||
497 | # define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval) | ||
498 | # endif | ||
499 | # define this_cpu_xchg(pcp, nval) \ | ||
500 | __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval) | ||
501 | #endif | ||
502 | |||
503 | #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \ | ||
504 | ({ typeof(pcp) ret__; \ | ||
505 | preempt_disable(); \ | ||
506 | ret__ = __this_cpu_read(pcp); \ | ||
507 | if (ret__ == (oval)) \ | ||
508 | __this_cpu_write(pcp, nval); \ | ||
509 | preempt_enable(); \ | ||
510 | ret__; \ | ||
511 | }) | ||
512 | |||
513 | #ifndef this_cpu_cmpxchg | ||
514 | # ifndef this_cpu_cmpxchg_1 | ||
515 | # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
516 | # endif | ||
517 | # ifndef this_cpu_cmpxchg_2 | ||
518 | # define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
519 | # endif | ||
520 | # ifndef this_cpu_cmpxchg_4 | ||
521 | # define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
522 | # endif | ||
523 | # ifndef this_cpu_cmpxchg_8 | ||
524 | # define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
525 | # endif | ||
526 | # define this_cpu_cmpxchg(pcp, oval, nval) \ | ||
527 | __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) | ||
528 | #endif | ||
529 | |||
530 | /* | ||
531 | * cmpxchg_double replaces two adjacent scalars at once. The first | ||
532 | * two parameters are per cpu variables which have to be of the same | ||
533 | * size. A truth value is returned to indicate success or failure | ||
534 | * (since a double register result is difficult to handle). There is | ||
535 | * very limited hardware support for these operations, so only certain | ||
536 | * sizes may work. | ||
537 | */ | ||
538 | #define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
539 | ({ \ | ||
540 | int ret__; \ | ||
541 | preempt_disable(); \ | ||
542 | ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \ | ||
543 | oval1, oval2, nval1, nval2); \ | ||
544 | preempt_enable(); \ | ||
545 | ret__; \ | ||
546 | }) | ||
547 | |||
548 | #ifndef this_cpu_cmpxchg_double | ||
549 | # ifndef this_cpu_cmpxchg_double_1 | ||
550 | # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
551 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
552 | # endif | ||
553 | # ifndef this_cpu_cmpxchg_double_2 | ||
554 | # define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
555 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
556 | # endif | ||
557 | # ifndef this_cpu_cmpxchg_double_4 | ||
558 | # define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
559 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
560 | # endif | ||
561 | # ifndef this_cpu_cmpxchg_double_8 | ||
562 | # define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
563 | _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
564 | # endif | ||
565 | # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
566 | __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) | ||
567 | #endif | ||
568 | |||
415 | /* | 569 | /* |
416 | * Generic percpu operations that do not require preemption handling. | 570 | * Generic percpu operations that do not require preemption handling. |
417 | * Either we do not care about races or the caller has the | 571 | * Either we do not care about races or the caller has the |
@@ -539,11 +693,120 @@ do { \ | |||
539 | # define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val)) | 693 | # define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val)) |
540 | #endif | 694 | #endif |
541 | 695 | ||
696 | #define __this_cpu_generic_add_return(pcp, val) \ | ||
697 | ({ \ | ||
698 | __this_cpu_add(pcp, val); \ | ||
699 | __this_cpu_read(pcp); \ | ||
700 | }) | ||
701 | |||
702 | #ifndef __this_cpu_add_return | ||
703 | # ifndef __this_cpu_add_return_1 | ||
704 | # define __this_cpu_add_return_1(pcp, val) __this_cpu_generic_add_return(pcp, val) | ||
705 | # endif | ||
706 | # ifndef __this_cpu_add_return_2 | ||
707 | # define __this_cpu_add_return_2(pcp, val) __this_cpu_generic_add_return(pcp, val) | ||
708 | # endif | ||
709 | # ifndef __this_cpu_add_return_4 | ||
710 | # define __this_cpu_add_return_4(pcp, val) __this_cpu_generic_add_return(pcp, val) | ||
711 | # endif | ||
712 | # ifndef __this_cpu_add_return_8 | ||
713 | # define __this_cpu_add_return_8(pcp, val) __this_cpu_generic_add_return(pcp, val) | ||
714 | # endif | ||
715 | # define __this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) | ||
716 | #endif | ||
717 | |||
718 | #define __this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(val)) | ||
719 | #define __this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) | ||
720 | #define __this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) | ||
721 | |||
722 | #define __this_cpu_generic_xchg(pcp, nval) \ | ||
723 | ({ typeof(pcp) ret__; \ | ||
724 | ret__ = __this_cpu_read(pcp); \ | ||
725 | __this_cpu_write(pcp, nval); \ | ||
726 | ret__; \ | ||
727 | }) | ||
728 | |||
729 | #ifndef __this_cpu_xchg | ||
730 | # ifndef __this_cpu_xchg_1 | ||
731 | # define __this_cpu_xchg_1(pcp, nval) __this_cpu_generic_xchg(pcp, nval) | ||
732 | # endif | ||
733 | # ifndef __this_cpu_xchg_2 | ||
734 | # define __this_cpu_xchg_2(pcp, nval) __this_cpu_generic_xchg(pcp, nval) | ||
735 | # endif | ||
736 | # ifndef __this_cpu_xchg_4 | ||
737 | # define __this_cpu_xchg_4(pcp, nval) __this_cpu_generic_xchg(pcp, nval) | ||
738 | # endif | ||
739 | # ifndef __this_cpu_xchg_8 | ||
740 | # define __this_cpu_xchg_8(pcp, nval) __this_cpu_generic_xchg(pcp, nval) | ||
741 | # endif | ||
742 | # define __this_cpu_xchg(pcp, nval) \ | ||
743 | __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval) | ||
744 | #endif | ||
745 | |||
746 | #define __this_cpu_generic_cmpxchg(pcp, oval, nval) \ | ||
747 | ({ \ | ||
748 | typeof(pcp) ret__; \ | ||
749 | ret__ = __this_cpu_read(pcp); \ | ||
750 | if (ret__ == (oval)) \ | ||
751 | __this_cpu_write(pcp, nval); \ | ||
752 | ret__; \ | ||
753 | }) | ||
754 | |||
755 | #ifndef __this_cpu_cmpxchg | ||
756 | # ifndef __this_cpu_cmpxchg_1 | ||
757 | # define __this_cpu_cmpxchg_1(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
758 | # endif | ||
759 | # ifndef __this_cpu_cmpxchg_2 | ||
760 | # define __this_cpu_cmpxchg_2(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
761 | # endif | ||
762 | # ifndef __this_cpu_cmpxchg_4 | ||
763 | # define __this_cpu_cmpxchg_4(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
764 | # endif | ||
765 | # ifndef __this_cpu_cmpxchg_8 | ||
766 | # define __this_cpu_cmpxchg_8(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval) | ||
767 | # endif | ||
768 | # define __this_cpu_cmpxchg(pcp, oval, nval) \ | ||
769 | __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval) | ||
770 | #endif | ||
771 | |||
772 | #define __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
773 | ({ \ | ||
774 | int __ret = 0; \ | ||
775 | if (__this_cpu_read(pcp1) == (oval1) && \ | ||
776 | __this_cpu_read(pcp2) == (oval2)) { \ | ||
777 | __this_cpu_write(pcp1, (nval1)); \ | ||
778 | __this_cpu_write(pcp2, (nval2)); \ | ||
779 | __ret = 1; \ | ||
780 | } \ | ||
781 | (__ret); \ | ||
782 | }) | ||
783 | |||
784 | #ifndef __this_cpu_cmpxchg_double | ||
785 | # ifndef __this_cpu_cmpxchg_double_1 | ||
786 | # define __this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
787 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
788 | # endif | ||
789 | # ifndef __this_cpu_cmpxchg_double_2 | ||
790 | # define __this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
791 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
792 | # endif | ||
793 | # ifndef __this_cpu_cmpxchg_double_4 | ||
794 | # define __this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
795 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
796 | # endif | ||
797 | # ifndef __this_cpu_cmpxchg_double_8 | ||
798 | # define __this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
799 | __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
800 | # endif | ||
801 | # define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
802 | __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) | ||
803 | #endif | ||
804 | |||
542 | /* | 805 | /* |
543 | * IRQ safe versions of the per cpu RMW operations. Note that these operations | 806 | * IRQ safe versions of the per cpu RMW operations. Note that these operations |
544 | * are *not* safe against modification of the same variable from another | 807 | * are *not* safe against modification of the same variable from another |
545 | * processors (which one gets when using regular atomic operations) | 808 | * processors (which one gets when using regular atomic operations) |
546 | . They are guaranteed to be atomic vs. local interrupts and | 809 | * They are guaranteed to be atomic vs. local interrupts and |
547 | * preemption only. | 810 | * preemption only. |
548 | */ | 811 | */ |
549 | #define irqsafe_cpu_generic_to_op(pcp, val, op) \ | 812 | #define irqsafe_cpu_generic_to_op(pcp, val, op) \ |
@@ -630,4 +893,65 @@ do { \ | |||
630 | # define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val)) | 893 | # define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val)) |
631 | #endif | 894 | #endif |
632 | 895 | ||
896 | #define irqsafe_cpu_generic_cmpxchg(pcp, oval, nval) \ | ||
897 | ({ \ | ||
898 | typeof(pcp) ret__; \ | ||
899 | unsigned long flags; \ | ||
900 | local_irq_save(flags); \ | ||
901 | ret__ = __this_cpu_read(pcp); \ | ||
902 | if (ret__ == (oval)) \ | ||
903 | __this_cpu_write(pcp, nval); \ | ||
904 | local_irq_restore(flags); \ | ||
905 | ret__; \ | ||
906 | }) | ||
907 | |||
908 | #ifndef irqsafe_cpu_cmpxchg | ||
909 | # ifndef irqsafe_cpu_cmpxchg_1 | ||
910 | # define irqsafe_cpu_cmpxchg_1(pcp, oval, nval) irqsafe_cpu_generic_cmpxchg(pcp, oval, nval) | ||
911 | # endif | ||
912 | # ifndef irqsafe_cpu_cmpxchg_2 | ||
913 | # define irqsafe_cpu_cmpxchg_2(pcp, oval, nval) irqsafe_cpu_generic_cmpxchg(pcp, oval, nval) | ||
914 | # endif | ||
915 | # ifndef irqsafe_cpu_cmpxchg_4 | ||
916 | # define irqsafe_cpu_cmpxchg_4(pcp, oval, nval) irqsafe_cpu_generic_cmpxchg(pcp, oval, nval) | ||
917 | # endif | ||
918 | # ifndef irqsafe_cpu_cmpxchg_8 | ||
919 | # define irqsafe_cpu_cmpxchg_8(pcp, oval, nval) irqsafe_cpu_generic_cmpxchg(pcp, oval, nval) | ||
920 | # endif | ||
921 | # define irqsafe_cpu_cmpxchg(pcp, oval, nval) \ | ||
922 | __pcpu_size_call_return2(irqsafe_cpu_cmpxchg_, (pcp), oval, nval) | ||
923 | #endif | ||
924 | |||
925 | #define irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
926 | ({ \ | ||
927 | int ret__; \ | ||
928 | unsigned long flags; \ | ||
929 | local_irq_save(flags); \ | ||
930 | ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \ | ||
931 | oval1, oval2, nval1, nval2); \ | ||
932 | local_irq_restore(flags); \ | ||
933 | ret__; \ | ||
934 | }) | ||
935 | |||
936 | #ifndef irqsafe_cpu_cmpxchg_double | ||
937 | # ifndef irqsafe_cpu_cmpxchg_double_1 | ||
938 | # define irqsafe_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
939 | irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
940 | # endif | ||
941 | # ifndef irqsafe_cpu_cmpxchg_double_2 | ||
942 | # define irqsafe_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
943 | irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
944 | # endif | ||
945 | # ifndef irqsafe_cpu_cmpxchg_double_4 | ||
946 | # define irqsafe_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
947 | irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
948 | # endif | ||
949 | # ifndef irqsafe_cpu_cmpxchg_double_8 | ||
950 | # define irqsafe_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
951 | irqsafe_generic_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) | ||
952 | # endif | ||
953 | # define irqsafe_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ | ||
954 | __pcpu_double_call_return_bool(irqsafe_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) | ||
955 | #endif | ||
956 | |||
633 | #endif /* __LINUX_PERCPU_H */ | 957 | #endif /* __LINUX_PERCPU_H */ |