diff options
| -rw-r--r-- | arch/x86/include/asm/percpu.h | 98 | ||||
| -rw-r--r-- | arch/x86/include/asm/preempt.h | 16 | ||||
| -rw-r--r-- | include/asm-generic/percpu.h | 13 | ||||
| -rw-r--r-- | include/linux/percpu.h | 331 |
4 files changed, 260 insertions, 198 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 94220d14d5cc..851bcdc5db04 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h | |||
| @@ -52,7 +52,7 @@ | |||
| 52 | * Compared to the generic __my_cpu_offset version, the following | 52 | * Compared to the generic __my_cpu_offset version, the following |
| 53 | * saves one instruction and avoids clobbering a temp register. | 53 | * saves one instruction and avoids clobbering a temp register. |
| 54 | */ | 54 | */ |
| 55 | #define __this_cpu_ptr(ptr) \ | 55 | #define raw_cpu_ptr(ptr) \ |
| 56 | ({ \ | 56 | ({ \ |
| 57 | unsigned long tcp_ptr__; \ | 57 | unsigned long tcp_ptr__; \ |
| 58 | __verify_pcpu_ptr(ptr); \ | 58 | __verify_pcpu_ptr(ptr); \ |
| @@ -362,25 +362,25 @@ do { \ | |||
| 362 | */ | 362 | */ |
| 363 | #define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var))) | 363 | #define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var))) |
| 364 | 364 | ||
| 365 | #define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 365 | #define raw_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| 366 | #define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 366 | #define raw_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| 367 | #define __this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 367 | #define raw_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| 368 | 368 | ||
| 369 | #define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) | 369 | #define raw_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
| 370 | #define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) | 370 | #define raw_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
| 371 | #define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) | 371 | #define raw_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
| 372 | #define __this_cpu_add_1(pcp, val) percpu_add_op((pcp), val) | 372 | #define raw_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
| 373 | #define __this_cpu_add_2(pcp, val) percpu_add_op((pcp), val) | 373 | #define raw_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
| 374 | #define __this_cpu_add_4(pcp, val) percpu_add_op((pcp), val) | 374 | #define raw_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
| 375 | #define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) | 375 | #define raw_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
| 376 | #define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) | 376 | #define raw_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
| 377 | #define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) | 377 | #define raw_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
| 378 | #define __this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) | 378 | #define raw_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) |
| 379 | #define __this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) | 379 | #define raw_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) |
| 380 | #define __this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) | 380 | #define raw_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) |
| 381 | #define __this_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val) | 381 | #define raw_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val) |
| 382 | #define __this_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val) | 382 | #define raw_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val) |
| 383 | #define __this_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val) | 383 | #define raw_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val) |
| 384 | 384 | ||
| 385 | #define this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 385 | #define this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| 386 | #define this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 386 | #define this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| @@ -401,16 +401,16 @@ do { \ | |||
| 401 | #define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval) | 401 | #define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval) |
| 402 | #define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval) | 402 | #define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval) |
| 403 | 403 | ||
| 404 | #define __this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) | 404 | #define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) |
| 405 | #define __this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) | 405 | #define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) |
| 406 | #define __this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) | 406 | #define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) |
| 407 | #define __this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 407 | #define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 408 | #define __this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 408 | #define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 409 | #define __this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 409 | #define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 410 | 410 | ||
| 411 | #define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) | 411 | #define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) |
| 412 | #define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) | 412 | #define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) |
| 413 | #define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) | 413 | #define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) |
| 414 | #define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 414 | #define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 415 | #define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 415 | #define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 416 | #define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 416 | #define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| @@ -427,7 +427,7 @@ do { \ | |||
| 427 | __ret; \ | 427 | __ret; \ |
| 428 | }) | 428 | }) |
| 429 | 429 | ||
| 430 | #define __this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double | 430 | #define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double |
| 431 | #define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double | 431 | #define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double |
| 432 | #endif /* CONFIG_X86_CMPXCHG64 */ | 432 | #endif /* CONFIG_X86_CMPXCHG64 */ |
| 433 | 433 | ||
| @@ -436,22 +436,22 @@ do { \ | |||
| 436 | * 32 bit must fall back to generic operations. | 436 | * 32 bit must fall back to generic operations. |
| 437 | */ | 437 | */ |
| 438 | #ifdef CONFIG_X86_64 | 438 | #ifdef CONFIG_X86_64 |
| 439 | #define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 439 | #define raw_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| 440 | #define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) | 440 | #define raw_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
| 441 | #define __this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) | 441 | #define raw_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
| 442 | #define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) | 442 | #define raw_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
| 443 | #define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) | 443 | #define raw_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
| 444 | #define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) | 444 | #define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) |
| 445 | #define __this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) | 445 | #define raw_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) |
| 446 | #define __this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 446 | #define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 447 | 447 | ||
| 448 | #define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) | 448 | #define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) |
| 449 | #define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) | 449 | #define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
| 450 | #define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) | 450 | #define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
| 451 | #define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) | 451 | #define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
| 452 | #define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) | 452 | #define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
| 453 | #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) | 453 | #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) |
| 454 | #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) | 454 | #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) |
| 455 | #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) | 455 | #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
| 456 | 456 | ||
| 457 | /* | 457 | /* |
| @@ -474,7 +474,7 @@ do { \ | |||
| 474 | __ret; \ | 474 | __ret; \ |
| 475 | }) | 475 | }) |
| 476 | 476 | ||
| 477 | #define __this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double | 477 | #define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double |
| 478 | #define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double | 478 | #define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double |
| 479 | 479 | ||
| 480 | #endif | 480 | #endif |
| @@ -495,9 +495,9 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, | |||
| 495 | unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG; | 495 | unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG; |
| 496 | 496 | ||
| 497 | #ifdef CONFIG_X86_64 | 497 | #ifdef CONFIG_X86_64 |
| 498 | return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0; | 498 | return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0; |
| 499 | #else | 499 | #else |
| 500 | return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0; | 500 | return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0; |
| 501 | #endif | 501 | #endif |
| 502 | } | 502 | } |
| 503 | 503 | ||
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index c8b051933b1b..7024c12f7bfe 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h | |||
| @@ -19,12 +19,12 @@ DECLARE_PER_CPU(int, __preempt_count); | |||
| 19 | */ | 19 | */ |
| 20 | static __always_inline int preempt_count(void) | 20 | static __always_inline int preempt_count(void) |
| 21 | { | 21 | { |
| 22 | return __this_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; | 22 | return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; |
| 23 | } | 23 | } |
| 24 | 24 | ||
| 25 | static __always_inline void preempt_count_set(int pc) | 25 | static __always_inline void preempt_count_set(int pc) |
| 26 | { | 26 | { |
| 27 | __this_cpu_write_4(__preempt_count, pc); | 27 | raw_cpu_write_4(__preempt_count, pc); |
| 28 | } | 28 | } |
| 29 | 29 | ||
