aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-mips/bitops.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/asm-mips/bitops.h')
-rw-r--r--include/asm-mips/bitops.h209
1 files changed, 163 insertions, 46 deletions
diff --git a/include/asm-mips/bitops.h b/include/asm-mips/bitops.h
index eb8d79dba11c..5496f9064a6a 100644
--- a/include/asm-mips/bitops.h
+++ b/include/asm-mips/bitops.h
@@ -12,20 +12,21 @@
12#include <linux/config.h> 12#include <linux/config.h>
13#include <linux/compiler.h> 13#include <linux/compiler.h>
14#include <linux/types.h> 14#include <linux/types.h>
15#include <asm/bug.h>
15#include <asm/byteorder.h> /* sigh ... */ 16#include <asm/byteorder.h> /* sigh ... */
16#include <asm/cpu-features.h> 17#include <asm/cpu-features.h>
17 18
18#if (_MIPS_SZLONG == 32) 19#if (_MIPS_SZLONG == 32)
19#define SZLONG_LOG 5 20#define SZLONG_LOG 5
20#define SZLONG_MASK 31UL 21#define SZLONG_MASK 31UL
21#define __LL "ll " 22#define __LL "ll "
22#define __SC "sc " 23#define __SC "sc "
23#define cpu_to_lelongp(x) cpu_to_le32p((__u32 *) (x)) 24#define cpu_to_lelongp(x) cpu_to_le32p((__u32 *) (x))
24#elif (_MIPS_SZLONG == 64) 25#elif (_MIPS_SZLONG == 64)
25#define SZLONG_LOG 6 26#define SZLONG_LOG 6
26#define SZLONG_MASK 63UL 27#define SZLONG_MASK 63UL
27#define __LL "lld " 28#define __LL "lld "
28#define __SC "scd " 29#define __SC "scd "
29#define cpu_to_lelongp(x) cpu_to_le64p((__u64 *) (x)) 30#define cpu_to_lelongp(x) cpu_to_le64p((__u64 *) (x))
30#endif 31#endif
31 32
@@ -72,18 +73,22 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
72 73
73 if (cpu_has_llsc && R10000_LLSC_WAR) { 74 if (cpu_has_llsc && R10000_LLSC_WAR) {
74 __asm__ __volatile__( 75 __asm__ __volatile__(
76 " .set mips3 \n"
75 "1: " __LL "%0, %1 # set_bit \n" 77 "1: " __LL "%0, %1 # set_bit \n"
76 " or %0, %2 \n" 78 " or %0, %2 \n"
77 " "__SC "%0, %1 \n" 79 " " __SC "%0, %1 \n"
78 " beqzl %0, 1b \n" 80 " beqzl %0, 1b \n"
81 " .set mips0 \n"
79 : "=&r" (temp), "=m" (*m) 82 : "=&r" (temp), "=m" (*m)
80 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); 83 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
81 } else if (cpu_has_llsc) { 84 } else if (cpu_has_llsc) {
82 __asm__ __volatile__( 85 __asm__ __volatile__(
86 " .set mips3 \n"
83 "1: " __LL "%0, %1 # set_bit \n" 87 "1: " __LL "%0, %1 # set_bit \n"
84 " or %0, %2 \n" 88 " or %0, %2 \n"
85 " "__SC "%0, %1 \n" 89 " " __SC "%0, %1 \n"
86 " beqz %0, 1b \n" 90 " beqz %0, 1b \n"
91 " .set mips0 \n"
87 : "=&r" (temp), "=m" (*m) 92 : "=&r" (temp), "=m" (*m)
88 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); 93 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
89 } else { 94 } else {
@@ -132,18 +137,22 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
132 137
133 if (cpu_has_llsc && R10000_LLSC_WAR) { 138 if (cpu_has_llsc && R10000_LLSC_WAR) {
134 __asm__ __volatile__( 139 __asm__ __volatile__(
140 " .set mips3 \n"
135 "1: " __LL "%0, %1 # clear_bit \n" 141 "1: " __LL "%0, %1 # clear_bit \n"
136 " and %0, %2 \n" 142 " and %0, %2 \n"
137 " " __SC "%0, %1 \n" 143 " " __SC "%0, %1 \n"
138 " beqzl %0, 1b \n" 144 " beqzl %0, 1b \n"
145 " .set mips0 \n"
139 : "=&r" (temp), "=m" (*m) 146 : "=&r" (temp), "=m" (*m)
140 : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m)); 147 : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m));
141 } else if (cpu_has_llsc) { 148 } else if (cpu_has_llsc) {
142 __asm__ __volatile__( 149 __asm__ __volatile__(
150 " .set mips3 \n"
143 "1: " __LL "%0, %1 # clear_bit \n" 151 "1: " __LL "%0, %1 # clear_bit \n"
144 " and %0, %2 \n" 152 " and %0, %2 \n"
145 " " __SC "%0, %1 \n" 153 " " __SC "%0, %1 \n"
146 " beqz %0, 1b \n" 154 " beqz %0, 1b \n"
155 " .set mips0 \n"
147 : "=&r" (temp), "=m" (*m) 156 : "=&r" (temp), "=m" (*m)
148 : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m)); 157 : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m));
149 } else { 158 } else {
@@ -191,10 +200,12 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
191 unsigned long temp; 200 unsigned long temp;
192 201
193 __asm__ __volatile__( 202 __asm__ __volatile__(
203 " .set mips3 \n"
194 "1: " __LL "%0, %1 # change_bit \n" 204 "1: " __LL "%0, %1 # change_bit \n"
195 " xor %0, %2 \n" 205 " xor %0, %2 \n"
196 " "__SC "%0, %1 \n" 206 " " __SC "%0, %1 \n"
197 " beqzl %0, 1b \n" 207 " beqzl %0, 1b \n"
208 " .set mips0 \n"
198 : "=&r" (temp), "=m" (*m) 209 : "=&r" (temp), "=m" (*m)
199 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); 210 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
200 } else if (cpu_has_llsc) { 211 } else if (cpu_has_llsc) {
@@ -202,10 +213,12 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
202 unsigned long temp; 213 unsigned long temp;
203 214
204 __asm__ __volatile__( 215 __asm__ __volatile__(
216 " .set mips3 \n"
205 "1: " __LL "%0, %1 # change_bit \n" 217 "1: " __LL "%0, %1 # change_bit \n"
206 " xor %0, %2 \n" 218 " xor %0, %2 \n"
207 " "__SC "%0, %1 \n" 219 " " __SC "%0, %1 \n"
208 " beqz %0, 1b \n" 220 " beqz %0, 1b \n"
221 " .set mips0 \n"
209 : "=&r" (temp), "=m" (*m) 222 : "=&r" (temp), "=m" (*m)
210 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); 223 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
211 } else { 224 } else {
@@ -253,14 +266,16 @@ static inline int test_and_set_bit(unsigned long nr,
253 unsigned long temp, res; 266 unsigned long temp, res;
254 267
255 __asm__ __volatile__( 268 __asm__ __volatile__(
269 " .set mips3 \n"
256 "1: " __LL "%0, %1 # test_and_set_bit \n" 270 "1: " __LL "%0, %1 # test_and_set_bit \n"
257 " or %2, %0, %3 \n" 271 " or %2, %0, %3 \n"
258 " " __SC "%2, %1 \n" 272 " " __SC "%2, %1 \n"
259 " beqzl %2, 1b \n" 273 " beqzl %2, 1b \n"
260 " and %2, %0, %3 \n" 274 " and %2, %0, %3 \n"
261#ifdef CONFIG_SMP 275#ifdef CONFIG_SMP
262 "sync \n" 276 " sync \n"
263#endif 277#endif
278 " .set mips0 \n"
264 : "=&r" (temp), "=m" (*m), "=&r" (res) 279 : "=&r" (temp), "=m" (*m), "=&r" (res)
265 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) 280 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
266 : "memory"); 281 : "memory");
@@ -271,16 +286,18 @@ static inline int test_and_set_bit(unsigned long nr,
271 unsigned long temp, res; 286 unsigned long temp, res;
272 287
273 __asm__ __volatile__( 288 __asm__ __volatile__(
274 " .set noreorder # test_and_set_bit \n" 289 " .set push \n"
275 "1: " __LL "%0, %1 \n" 290 " .set noreorder \n"
291 " .set mips3 \n"
292 "1: " __LL "%0, %1 # test_and_set_bit \n"
276 " or %2, %0, %3 \n" 293 " or %2, %0, %3 \n"
277 " " __SC "%2, %1 \n" 294 " " __SC "%2, %1 \n"
278 " beqz %2, 1b \n" 295 " beqz %2, 1b \n"
279 " and %2, %0, %3 \n" 296 " and %2, %0, %3 \n"
280#ifdef CONFIG_SMP 297#ifdef CONFIG_SMP
281 "sync \n" 298 " sync \n"
282#endif 299#endif
283 ".set\treorder" 300 " .set pop \n"
284 : "=&r" (temp), "=m" (*m), "=&r" (res) 301 : "=&r" (temp), "=m" (*m), "=&r" (res)
285 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) 302 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
286 : "memory"); 303 : "memory");
@@ -343,15 +360,17 @@ static inline int test_and_clear_bit(unsigned long nr,
343 unsigned long temp, res; 360 unsigned long temp, res;
344 361
345 __asm__ __volatile__( 362 __asm__ __volatile__(
363 " .set mips3 \n"
346 "1: " __LL "%0, %1 # test_and_clear_bit \n" 364 "1: " __LL "%0, %1 # test_and_clear_bit \n"
347 " or %2, %0, %3 \n" 365 " or %2, %0, %3 \n"
348 " xor %2, %3 \n" 366 " xor %2, %3 \n"
349 __SC "%2, %1 \n" 367 " " __SC "%2, %1 \n"
350 " beqzl %2, 1b \n" 368 " beqzl %2, 1b \n"
351 " and %2, %0, %3 \n" 369 " and %2, %0, %3 \n"
352#ifdef CONFIG_SMP 370#ifdef CONFIG_SMP
353 " sync \n" 371 " sync \n"
354#endif 372#endif
373 " .set mips0 \n"
355 : "=&r" (temp), "=m" (*m), "=&r" (res) 374 : "=&r" (temp), "=m" (*m), "=&r" (res)
356 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) 375 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
357 : "memory"); 376 : "memory");
@@ -362,17 +381,19 @@ static inline int test_and_clear_bit(unsigned long nr,
362 unsigned long temp, res; 381 unsigned long temp, res;
363 382
364 __asm__ __volatile__( 383 __asm__ __volatile__(
365 " .set noreorder # test_and_clear_bit \n" 384 " .set push \n"
366 "1: " __LL "%0, %1 \n" 385 " .set noreorder \n"
386 " .set mips3 \n"
387 "1: " __LL "%0, %1 # test_and_clear_bit \n"
367 " or %2, %0, %3 \n" 388 " or %2, %0, %3 \n"
368 " xor %2, %3 \n" 389 " xor %2, %3 \n"
369 __SC "%2, %1 \n" 390 " " __SC "%2, %1 \n"
370 " beqz %2, 1b \n" 391 " beqz %2, 1b \n"
371 " and %2, %0, %3 \n" 392 " and %2, %0, %3 \n"
372#ifdef CONFIG_SMP 393#ifdef CONFIG_SMP
373 " sync \n" 394 " sync \n"
374#endif 395#endif
375 " .set reorder \n" 396 " .set pop \n"
376 : "=&r" (temp), "=m" (*m), "=&r" (res) 397 : "=&r" (temp), "=m" (*m), "=&r" (res)
377 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) 398 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
378 : "memory"); 399 : "memory");
@@ -435,14 +456,16 @@ static inline int test_and_change_bit(unsigned long nr,
435 unsigned long temp, res; 456 unsigned long temp, res;
436 457
437 __asm__ __volatile__( 458 __asm__ __volatile__(
438 "1: " __LL " %0, %1 # test_and_change_bit \n" 459 " .set mips3 \n"
460 "1: " __LL "%0, %1 # test_and_change_bit \n"
439 " xor %2, %0, %3 \n" 461 " xor %2, %0, %3 \n"
440 " "__SC "%2, %1 \n" 462 " " __SC "%2, %1 \n"
441 " beqzl %2, 1b \n" 463 " beqzl %2, 1b \n"
442 " and %2, %0, %3 \n" 464 " and %2, %0, %3 \n"
443#ifdef CONFIG_SMP 465#ifdef CONFIG_SMP
444 " sync \n" 466 " sync \n"
445#endif 467#endif
468 " .set mips0 \n"
446 : "=&r" (temp), "=m" (*m), "=&r" (res) 469 : "=&r" (temp), "=m" (*m), "=&r" (res)
447 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) 470 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
448 : "memory"); 471 : "memory");
@@ -453,16 +476,18 @@ static inline int test_and_change_bit(unsigned long nr,
453 unsigned long temp, res; 476 unsigned long temp, res;
454 477
455 __asm__ __volatile__( 478 __asm__ __volatile__(
456 " .set noreorder # test_and_change_bit \n" 479 " .set push \n"
457 "1: " __LL " %0, %1 \n" 480 " .set noreorder \n"
481 " .set mips3 \n"
482 "1: " __LL "%0, %1 # test_and_change_bit \n"
458 " xor %2, %0, %3 \n" 483 " xor %2, %0, %3 \n"
459 " "__SC "\t%2, %1 \n" 484 " " __SC "\t%2, %1 \n"
460 " beqz %2, 1b \n" 485 " beqz %2, 1b \n"
461 " and %2, %0, %3 \n" 486 " and %2, %0, %3 \n"
462#ifdef CONFIG_SMP 487#ifdef CONFIG_SMP
463 " sync \n" 488 " sync \n"
464#endif 489#endif
465 " .set reorder \n" 490 " .set pop \n"
466 : "=&r" (temp), "=m" (*m), "=&r" (res) 491 : "=&r" (temp), "=m" (*m), "=&r" (res)
467 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) 492 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
468 : "memory"); 493 : "memory");
@@ -523,22 +548,60 @@ static inline int test_bit(unsigned long nr, const volatile unsigned long *addr)
523} 548}
524 549
525/* 550/*
526 * ffz - find first zero in word. 551 * Return the bit position (0..63) of the most significant 1 bit in a word
552 * Returns -1 if no 1 bit exists
553 */
554static inline int __ilog2(unsigned long x)
555{
556 int lz;
557
558 if (sizeof(x) == 4) {
559 __asm__ (
560 " .set push \n"
561 " .set mips32 \n"
562 " clz %0, %1 \n"
563 " .set pop \n"
564 : "=r" (lz)
565 : "r" (x));
566
567 return 31 - lz;
568 }
569
570 BUG_ON(sizeof(x) != 8);
571
572 __asm__ (
573 " .set push \n"
574 " .set mips64 \n"
575 " dclz %0, %1 \n"
576 " .set pop \n"
577 : "=r" (lz)
578 : "r" (x));
579
580 return 63 - lz;
581}
582
583/*
584 * __ffs - find first bit in word.
527 * @word: The word to search 585 * @word: The word to search
528 * 586 *
529 * Undefined if no zero exists, so code should check against ~0UL first. 587 * Returns 0..SZLONG-1
588 * Undefined if no bit exists, so code should check against 0 first.
530 */ 589 */
531static inline unsigned long ffz(unsigned long word) 590static inline unsigned long __ffs(unsigned long word)
532{ 591{
592#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64)
593 return __ilog2(word & -word);
594#else
533 int b = 0, s; 595 int b = 0, s;
534 596
535 word = ~word;
536#ifdef CONFIG_32BIT 597#ifdef CONFIG_32BIT
537 s = 16; if (word << 16 != 0) s = 0; b += s; word >>= s; 598 s = 16; if (word << 16 != 0) s = 0; b += s; word >>= s;
538 s = 8; if (word << 24 != 0) s = 0; b += s; word >>= s; 599 s = 8; if (word << 24 != 0) s = 0; b += s; word >>= s;
539 s = 4; if (word << 28 != 0) s = 0; b += s; word >>= s; 600 s = 4; if (word << 28 != 0) s = 0; b += s; word >>= s;
540 s = 2; if (word << 30 != 0) s = 0; b += s; word >>= s; 601 s = 2; if (word << 30 != 0) s = 0; b += s; word >>= s;
541 s = 1; if (word << 31 != 0) s = 0; b += s; 602 s = 1; if (word << 31 != 0) s = 0; b += s;
603
604 return b;
542#endif 605#endif
543#ifdef CONFIG_64BIT 606#ifdef CONFIG_64BIT
544 s = 32; if (word << 32 != 0) s = 0; b += s; word >>= s; 607 s = 32; if (word << 32 != 0) s = 0; b += s; word >>= s;
@@ -547,27 +610,92 @@ static inline unsigned long ffz(unsigned long word)
547 s = 4; if (word << 60 != 0) s = 0; b += s; word >>= s; 610 s = 4; if (word << 60 != 0) s = 0; b += s; word >>= s;
548 s = 2; if (word << 62 != 0) s = 0; b += s; word >>= s; 611 s = 2; if (word << 62 != 0) s = 0; b += s; word >>= s;
549 s = 1; if (word << 63 != 0) s = 0; b += s; 612 s = 1; if (word << 63 != 0) s = 0; b += s;
550#endif
551 613
552 return b; 614 return b;
615#endif
616#endif
553} 617}
554 618
555/* 619/*
556 * __ffs - find first bit in word. 620 * ffs - find first bit set.
557 * @word: The word to search 621 * @word: The word to search
558 * 622 *
559 * Undefined if no bit exists, so code should check against 0 first. 623 * Returns 1..SZLONG
624 * Returns 0 if no bit exists
560 */ 625 */
561static inline unsigned long __ffs(unsigned long word) 626
627static inline unsigned long ffs(unsigned long word)
562{ 628{
563 return ffz(~word); 629 if (!word)
630 return 0;
631
632 return __ffs(word) + 1;
564} 633}
565 634
566/* 635/*
567 * fls: find last bit set. 636 * ffz - find first zero in word.
637 * @word: The word to search
638 *
639 * Undefined if no zero exists, so code should check against ~0UL first.
640 */
641static inline unsigned long ffz(unsigned long word)
642{
643 return __ffs (~word);
644}
645
646/*
647 * flz - find last zero in word.
648 * @word: The word to search
649 *
650 * Returns 0..SZLONG-1
651 * Undefined if no zero exists, so code should check against ~0UL first.
652 */
653static inline unsigned long flz(unsigned long word)
654{
655#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64)
656 return __ilog2(~word);
657#else
658#ifdef CONFIG_32BIT
659 int r = 31, s;
660 word = ~word;
661 s = 16; if ((word & 0xffff0000)) s = 0; r -= s; word <<= s;
662 s = 8; if ((word & 0xff000000)) s = 0; r -= s; word <<= s;
663 s = 4; if ((word & 0xf0000000)) s = 0; r -= s; word <<= s;
664 s = 2; if ((word & 0xc0000000)) s = 0; r -= s; word <<= s;
665 s = 1; if ((word & 0x80000000)) s = 0; r -= s;
666
667 return r;
668#endif
669#ifdef CONFIG_64BIT
670 int r = 63, s;
671 word = ~word;
672 s = 32; if ((word & 0xffffffff00000000UL)) s = 0; r -= s; word <<= s;
673 s = 16; if ((word & 0xffff000000000000UL)) s = 0; r -= s; word <<= s;
674 s = 8; if ((word & 0xff00000000000000UL)) s = 0; r -= s; word <<= s;
675 s = 4; if ((word & 0xf000000000000000UL)) s = 0; r -= s; word <<= s;
676 s = 2; if ((word & 0xc000000000000000UL)) s = 0; r -= s; word <<= s;
677 s = 1; if ((word & 0x8000000000000000UL)) s = 0; r -= s;
678
679 return r;
680#endif
681#endif
682}
683
684/*
685 * fls - find last bit set.
686 * @word: The word to search
687 *
688 * Returns 1..SZLONG
689 * Returns 0 if no bit exists
568 */ 690 */
691static inline unsigned long fls(unsigned long word)
692{
693 if (word == 0)
694 return 0;
695
696 return flz(~word) + 1;
697}
569 698
570#define fls(x) generic_fls(x)
571 699
572/* 700/*
573 * find_next_zero_bit - find the first zero bit in a memory region 701 * find_next_zero_bit - find the first zero bit in a memory region
@@ -704,17 +832,6 @@ static inline int sched_find_first_bit(const unsigned long *b)
704} 832}
705 833
706/* 834/*
707 * ffs - find first bit set
708 * @x: the word to search
709 *
710 * This is defined the same way as
711 * the libc and compiler builtin ffs routines, therefore
712 * differs in spirit from the above ffz (man ffs).
713 */
714
715#define ffs(x) generic_ffs(x)
716
717/*
718 * hweightN - returns the hamming weight of a N-bit word 835 * hweightN - returns the hamming weight of a N-bit word
719 * @x: the word to weigh 836 * @x: the word to weigh
720 * 837 *
generated by cgit v1.2.2 (git 2.25.0) at 2025-11-02 16:20:42 -0500