1 files changed, 197 insertions, 155 deletions

diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index b235c96167d3..da39c608a28c 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -17,6 +17,7 @@
  */
 
 #include <stdarg.h>
+#include <linux/clk-provider.h>
 #include <linux/module.h>       /* for KSYM_SYMBOL_LEN */
 #include <linux/types.h>
 #include <linux/string.h>
@@ -32,6 +33,7 @@
 
 #include <asm/page.h>           /* for PAGE_SIZE */
 #include <asm/sections.h>       /* for dereference_function_descriptor() */
+#include <asm/byteorder.h>      /* cpu_to_le16 */
 
 #include <linux/string_helpers.h>
 #include "kstrtox.h"
@@ -121,142 +123,145 @@ int skip_atoi(const char **s)
         return i;
 }
 
-/* Decimal conversion is by far the most typical, and is used
- * for /proc and /sys data. This directly impacts e.g. top performance
- * with many processes running. We optimize it for speed
- * using ideas described at <http://www.cs.uiowa.edu/~jones/bcd/divide.html>
- * (with permission from the author, Douglas W. Jones).
+/*
+ * Decimal conversion is by far the most typical, and is used for
+ * /proc and /sys data. This directly impacts e.g. top performance
+ * with many processes running. We optimize it for speed by emitting
+ * two characters at a time, using a 200 byte lookup table. This
+ * roughly halves the number of multiplications compared to computing
+ * the digits one at a time. Implementation strongly inspired by the
+ * previous version, which in turn used ideas described at
+ * <http://www.cs.uiowa.edu/~jones/bcd/divide.html> (with permission
+ * from the author, Douglas W. Jones).
+ *
+ * It turns out there is precisely one 26 bit fixed-point
+ * approximation a of 64/100 for which x/100 == (x * (u64)a) >> 32
+ * holds for all x in [0, 10^8-1], namely a = 0x28f5c29. The actual
+ * range happens to be somewhat larger (x <= 1073741898), but that's
+ * irrelevant for our purpose.
+ *
+ * For dividing a number in the range [10^4, 10^6-1] by 100, we still
+ * need a 32x32->64 bit multiply, so we simply use the same constant.
+ *
+ * For dividing a number in the range [100, 10^4-1] by 100, there are
+ * several options. The simplest is (x * 0x147b) >> 19, which is valid
+ * for all x <= 43698.
  */
 
-#if BITS_PER_LONG != 32 || BITS_PER_LONG_LONG != 64
-/* Formats correctly any integer in [0, 999999999] */
+static const u16 decpair[100] = {
+#define _(x) (__force u16) cpu_to_le16(((x % 10) | ((x / 10) << 8)) + 0x3030)
+        _( 0), _( 1), _( 2), _( 3), _( 4), _( 5), _( 6), _( 7), _( 8), _( 9),
+        _(10), _(11), _(12), _(13), _(14), _(15), _(16), _(17), _(18), _(19),
+        _(20), _(21), _(22), _(23), _(24), _(25), _(26), _(27), _(28), _(29),
+        _(30), _(31), _(32), _(33), _(34), _(35), _(36), _(37), _(38), _(39),
+        _(40), _(41), _(42), _(43), _(44), _(45), _(46), _(47), _(48), _(49),
+        _(50), _(51), _(52), _(53), _(54), _(55), _(56), _(57), _(58), _(59),
+        _(60), _(61), _(62), _(63), _(64), _(65), _(66), _(67), _(68), _(69),
+        _(70), _(71), _(72), _(73), _(74), _(75), _(76), _(77), _(78), _(79),
+        _(80), _(81), _(82), _(83), _(84), _(85), _(86), _(87), _(88), _(89),
+        _(90), _(91), _(92), _(93), _(94), _(95), _(96), _(97), _(98), _(99),
+#undef _
+};
+
+/*
+ * This will print a single '0' even if r == 0, since we would
+ * immediately jump to out_r where two 0s would be written but only
+ * one of them accounted for in buf. This is needed by ip4_string
+ * below. All other callers pass a non-zero value of r.
+*/
 static noinline_for_stack
-char *put_dec_full9(char *buf, unsigned q)
+char *put_dec_trunc8(char *buf, unsigned r)
 {
-        unsigned r;
+        unsigned q;
 
-        /*
-         * Possible ways to approx. divide by 10
-         * (x * 0x1999999a) >> 32 x < 1073741829 (multiply must be 64-bit)
-         * (x * 0xcccd) >> 19     x <      81920 (x < 262149 when 64-bit mul)
-         * (x * 0x6667) >> 18     x <      43699
-         * (x * 0x3334) >> 17     x <      16389
-         * (x * 0x199a) >> 16     x <      16389
-         * (x * 0x0ccd) >> 15     x <      16389
-         * (x * 0x0667) >> 14     x <       2739
-         * (x * 0x0334) >> 13     x <       1029
-         * (x * 0x019a) >> 12     x <       1029
-         * (x * 0x00cd) >> 11     x <       1029 shorter code than * 0x67 (on i386)
-         * (x * 0x0067) >> 10     x <        179
-         * (x * 0x0034) >>  9     x <         69 same
-         * (x * 0x001a) >>  8     x <         69 same
-         * (x * 0x000d) >>  7     x <         69 same, shortest code (on i386)
-         * (x * 0x0007) >>  6     x <         19
-         * See <http://www.cs.uiowa.edu/~jones/bcd/divide.html>
-         */
-        r      = (q * (uint64_t)0x1999999a) >> 32;
-        *buf++ = (q - 10 * r) + '0'; /* 1 */
-        q      = (r * (uint64_t)0x1999999a) >> 32;
-        *buf++ = (r - 10 * q) + '0'; /* 2 */
-        r      = (q * (uint64_t)0x1999999a) >> 32;
-        *buf++ = (q - 10 * r) + '0'; /* 3 */
-        q      = (r * (uint64_t)0x1999999a) >> 32;
-        *buf++ = (r - 10 * q) + '0'; /* 4 */
-        r      = (q * (uint64_t)0x1999999a) >> 32;
-        *buf++ = (q - 10 * r) + '0'; /* 5 */
-        /* Now value is under 10000, can avoid 64-bit multiply */
-        q      = (r * 0x199a) >> 16;
-        *buf++ = (r - 10 * q)  + '0'; /* 6 */
-        r      = (q * 0xcd) >> 11;
-        *buf++ = (q - 10 * r)  + '0'; /* 7 */
-        q      = (r * 0xcd) >> 11;
-        *buf++ = (r - 10 * q) + '0'; /* 8 */
-        *buf++ = q + '0'; /* 9 */
+        /* 1 <= r < 10^8 */
+        if (r < 100)
+                goto out_r;
+
+        /* 100 <= r < 10^8 */
+        q = (r * (u64)0x28f5c29) >> 32;
+        *((u16 *)buf) = decpair[r - 100*q];
+        buf += 2;
+
+        /* 1 <= q < 10^6 */
+        if (q < 100)
+                goto out_q;
+
+        /*  100 <= q < 10^6 */
+        r = (q * (u64)0x28f5c29) >> 32;
+        *((u16 *)buf) = decpair[q - 100*r];
+        buf += 2;
+
+        /* 1 <= r < 10^4 */
+        if (r < 100)
+                goto out_r;
+
+        /* 100 <= r < 10^4 */
+        q = (r * 0x147b) >> 19;
+        *((u16 *)buf) = decpair[r - 100*q];
+        buf += 2;
+out_q:
+        /* 1 <= q < 100 */
+        r = q;
+out_r:
+        /* 1 <= r < 100 */
+        *((u16 *)buf) = decpair[r];
+        buf += r < 10 ? 1 : 2;
         return buf;
 }
-#endif
 
-/* Similar to above but do not pad with zeros.
- * Code can be easily arranged to print 9 digits too, but our callers
- * always call put_dec_full9() instead when the number has 9 decimal digits.
- */
+#if BITS_PER_LONG == 64 && BITS_PER_LONG_LONG == 64
 static noinline_for_stack
-char *put_dec_trunc8(char *buf, unsigned r)
+char *put_dec_full8(char *buf, unsigned r)
 {
         unsigned q;
 
-        /* Copy of previous function's body with added early returns */
-        while (r >= 10000) {
-                q = r + '0';
-                r  = (r * (uint64_t)0x1999999a) >> 32;
-                *buf++ = q - 10*r;
-        }
-
-        q      = (r * 0x199a) >> 16;    /* r <= 9999 */
-        *buf++ = (r - 10 * q)  + '0';
-        if (q == 0)
-                return buf;
-        r      = (q * 0xcd) >> 11;      /* q <= 999 */
-        *buf++ = (q - 10 * r)  + '0';
-        if (r == 0)
-                return buf;
-        q      = (r * 0xcd) >> 11;      /* r <= 99 */
-        *buf++ = (r - 10 * q) + '0';
-        if (q == 0)
-                return buf;
-        *buf++ = q + '0';                /* q <= 9 */
-        return buf;
-}
+        /* 0 <= r < 10^8 */
+        q = (r * (u64)0x28f5c29) >> 32;
+        *((u16 *)buf) = decpair[r - 100*q];
+        buf += 2;
 
-/* There are two algorithms to print larger numbers.
- * One is generic: divide by 1000000000 and repeatedly print
- * groups of (up to) 9 digits. It's conceptually simple,
- * but requires a (unsigned long long) / 1000000000 division.
- *
- * Second algorithm splits 64-bit unsigned long long into 16-bit chunks,
- * manipulates them cleverly and generates groups of 4 decimal digits.
- * It so happens that it does NOT require long long division.
- *
- * If long is > 32 bits, division of 64-bit values is relatively easy,
- * and we will use the first algorithm.
- * If long long is > 64 bits (strange architecture with VERY large long long),
- * second algorithm can't be used, and we again use the first one.
- *
- * Else (if long is 32 bits and long long is 64 bits) we use second one.
- */
+        /* 0 <= q < 10^6 */
+        r = (q * (u64)0x28f5c29) >> 32;
+        *((u16 *)buf) = decpair[q - 100*r];
+        buf += 2;
 
-#if BITS_PER_LONG != 32 || BITS_PER_LONG_LONG != 64
+        /* 0 <= r < 10^4 */
+        q = (r * 0x147b) >> 19;
+        *((u16 *)buf) = decpair[r - 100*q];
+        buf += 2;
 
-/* First algorithm: generic */
+        /* 0 <= q < 100 */
+        *((u16 *)buf) = decpair[q];
+        buf += 2;
+        return buf;
+}
 
-static
+static noinline_for_stack
 char *put_dec(char *buf, unsigned long long n)
 {
-        if (n >= 100*1000*1000) {
-                while (n >= 1000*1000*1000)
-                        buf = put_dec_full9(buf, do_div(n, 1000*1000*1000));
-                if (n >= 100*1000*1000)
-                        return put_dec_full9(buf, n);
-        }
+        if (n >= 100*1000*1000)
+                buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
+        /* 1 <= n <= 1.6e11 */
+        if (n >= 100*1000*1000)
+                buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
+        /* 1 <= n < 1e8 */
         return put_dec_trunc8(buf, n);
 }
 
-#else
-
-/* Second algorithm: valid only for 64-bit long longs */
+#elif BITS_PER_LONG == 32 && BITS_PER_LONG_LONG == 64
 
-/* See comment in put_dec_full9 for choice of constants */
-static noinline_for_stack
-void put_dec_full4(char *buf, unsigned q)
+static void
+put_dec_full4(char *buf, unsigned r)
 {
-        unsigned r;
-        r      = (q * 0xccd) >> 15;
-        buf[0] = (q - 10 * r) + '0';
-        q      = (r * 0xcd) >> 11;
-        buf[1] = (r - 10 * q)  + '0';
-        r      = (q * 0xcd) >> 11;
-        buf[2] = (q - 10 * r)  + '0';
-        buf[3] = r + '0';
+        unsigned q;
+
+        /* 0 <= r < 10^4 */
+        q = (r * 0x147b) >> 19;
+        *((u16 *)buf) = decpair[r - 100*q];
+        buf += 2;
+        /* 0 <= q < 100 */
+        *((u16 *)buf) = decpair[q];
 }
 
 /*
@@ -264,9 +269,9 @@ void put_dec_full4(char *buf, unsigned q)
  * The approximation x/10000 == (x * 0x346DC5D7) >> 43
  * holds for all x < 1,128,869,999.  The largest value this
  * helper will ever be asked to convert is 1,125,520,955.
- * (d1 in the put_dec code, assuming n is all-ones).
+ * (second call in the put_dec code, assuming n is all-ones).
  */
-static
+static noinline_for_stack
 unsigned put_dec_helper4(char *buf, unsigned x)
 {
         uint32_t q = (x * (uint64_t)0x346DC5D7) >> 43;
@@ -293,6 +298,8 @@ char *put_dec(char *buf, unsigned long long n)
         d2  = (h      ) & 0xffff;
         d3  = (h >> 16); /* implicit "& 0xffff" */
 
+        /* n = 2^48 d3 + 2^32 d2 + 2^16 d1 + d0
+             = 281_4749_7671_0656 d3 + 42_9496_7296 d2 + 6_5536 d1 + d0 */
         q   = 656 * d3 + 7296 * d2 + 5536 * d1 + ((uint32_t)n & 0xffff);
         q = put_dec_helper4(buf, q);
 
@@ -322,7 +329,8 @@ char *put_dec(char *buf, unsigned long long n)
  */
 int num_to_str(char *buf, int size, unsigned long long num)
 {
-        char tmp[sizeof(num) * 3];
+        /* put_dec requires 2-byte alignment of the buffer. */
+        char tmp[sizeof(num) * 3] __aligned(2);
         int idx, len;
 
         /* put_dec() may work incorrectly for num = 0 (generate "", not "0") */
@@ -340,11 +348,11 @@ int num_to_str(char *buf, int size, unsigned long long num)
         return len;
 }
 
-#define ZEROPAD 1               /* pad with zero */
-#define SIGN    2               /* unsigned/signed long */
+#define SIGN    1               /* unsigned/signed, must be 1 */
+#define LEFT    2               /* left justified */
 #define PLUS    4               /* show plus */
 #define SPACE   8               /* space if plus */
-#define LEFT    16              /* left justified */
+#define ZEROPAD 16              /* pad with zero, must be 16 == '0' - ' ' */
 #define SMALL   32              /* use lowercase in hex (must be 32 == 0x20) */
 #define SPECIAL 64              /* prefix hex with "0x", octal with "0" */
 
@@ -383,10 +391,8 @@ static noinline_for_stack
 char *number(char *buf, char *end, unsigned long long num,
              struct printf_spec spec)
 {
-        /* we are called with base 8, 10 or 16, only, thus don't need "G..."  */
-        static const char digits[16] = "0123456789ABCDEF"; /* "GHIJKLMNOPQRSTUVWXYZ"; */
-
-        char tmp[66];
+        /* put_dec requires 2-byte alignment of the buffer. */
+        char tmp[3 * sizeof(num)] __aligned(2);
         char sign;
         char locase;
         int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10);
@@ -422,12 +428,7 @@ char *number(char *buf, char *end, unsigned long long num,
         /* generate full string in tmp[], in reverse order */
         i = 0;
         if (num < spec.base)
-                tmp[i++] = digits[num] | locase;
-        /* Generic code, for any base:
-        else do {
-                tmp[i++] = (digits[do_div(num,base)] | locase);
-        } while (num != 0);
-        */
+                tmp[i++] = hex_asc_upper[num] | locase;
         else if (spec.base != 10) { /* 8 or 16 */
                 int mask = spec.base - 1;
                 int shift = 3;
@@ -435,7 +436,7 @@ char *number(char *buf, char *end, unsigned long long num,
                 if (spec.base == 16)
                         shift = 4;
                 do {
-                        tmp[i++] = (digits[((unsigned char)num) & mask] | locase);
+                        tmp[i++] = (hex_asc_upper[((unsigned char)num) & mask] | locase);
                         num >>= shift;
                 } while (num);
         } else { /* base 10 */
@@ -447,7 +448,7 @@ char *number(char *buf, char *end, unsigned long long num,
                 spec.precision = i;
         /* leading space padding */
         spec.field_width -= spec.precision;
-        if (!(spec.flags & (ZEROPAD+LEFT))) {
+        if (!(spec.flags & (ZEROPAD | LEFT))) {
                 while (--spec.field_width >= 0) {
                         if (buf < end)
                                 *buf = ' ';
@@ -475,7 +476,8 @@ char *number(char *buf, char *end, unsigned long long num,
         }
         /* zero or space padding */
         if (!(spec.flags & LEFT)) {
-                char c = (spec.flags & ZEROPAD) ? '0' : ' ';
+                char c = ' ' + (spec.flags & ZEROPAD);
+                BUILD_BUG_ON(' ' + ZEROPAD != '0');
                 while (--spec.field_width >= 0) {
                         if (buf < end)
                                 *buf = c;
@@ -783,11 +785,19 @@ char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
         if (spec.field_width > 0)
                 len = min_t(int, spec.field_width, 64);
 
-        for (i = 0; i < len && buf < end - 1; i++) {
-                buf = hex_byte_pack(buf, addr[i]);
+        for (i = 0; i < len; ++i) {
+                if (buf < end)
+                        *buf = hex_asc_hi(addr[i]);
+                ++buf;
+                if (buf < end)
+                        *buf = hex_asc_lo(addr[i]);
+                ++buf;
 
-                if (buf < end && separator && i != len - 1)
-                        *buf++ = separator;
+                if (separator && i != len - 1) {
+                        if (buf < end)
+                                *buf = separator;
+                        ++buf;
+                }
         }
 
         return buf;
@@ -942,7 +952,7 @@ char *ip4_string(char *p, const u8 *addr, const char *fmt)
                 break;
         }
         for (i = 0; i < 4; i++) {
-                char temp[3];   /* hold each IP quad in reverse order */
+                char temp[4] __aligned(2);      /* hold each IP quad in reverse order */
                 int digits = put_dec_trunc8(temp, addr[index]) - temp;
                 if (leading_zeros) {
                         if (digits < 3)
@@ -1233,8 +1243,12 @@ char *escaped_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
 
         len = spec.field_width < 0 ? 1 : spec.field_width;
 
-        /* Ignore the error. We print as many characters as we can */
-        string_escape_mem(addr, len, &buf, end - buf, flags, NULL);
+        /*
+         * string_escape_mem() writes as many characters as it can to
+         * the given buffer, and returns the total size of the output
+         * had the buffer been big enough.
+         */
+        buf += string_escape_mem(addr, len, buf, buf < end ? end - buf : 0, flags, NULL);
 
         return buf;
 }
@@ -1322,6 +1336,30 @@ char *address_val(char *buf, char *end, const void *addr,
         return number(buf, end, num, spec);
 }
 
+static noinline_for_stack
+char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec,
+            const char *fmt)
+{
+        if (!IS_ENABLED(CONFIG_HAVE_CLK) || !clk)
+                return string(buf, end, NULL, spec);
+
+        switch (fmt[1]) {
+        case 'r':
+                return number(buf, end, clk_get_rate(clk), spec);
+
+        case 'n':
+        default:
+#ifdef CONFIG_COMMON_CLK
+                return string(buf, end, __clk_get_name(clk), spec);
+#else
+                spec.base = 16;
+                spec.field_width = sizeof(unsigned long) * 2 + 2;
+                spec.flags |= SPECIAL | SMALL | ZEROPAD;
+                return number(buf, end, (unsigned long)clk, spec);
+#endif
+        }
+}
+
 int kptr_restrict __read_mostly;
 
 /*
@@ -1404,6 +1442,11 @@ int kptr_restrict __read_mostly;
  *           (default assumed to be phys_addr_t, passed by reference)
  * - 'd[234]' For a dentry name (optionally 2-4 last components)
  * - 'D[234]' Same as 'd' but for a struct file
+ * - 'C' For a clock, it prints the name (Common Clock Framework) or address
+ *       (legacy clock framework) of the clock
+ * - 'Cn' For a clock, it prints the name (Common Clock Framework) or address
+ *        (legacy clock framework) of the clock
+ * - 'Cr' For a clock, it prints the current rate of the clock
  *
  * Note: The difference between 'S' and 'F' is that on ia64 and ppc64
  * function pointers are really function descriptors, which contain a
@@ -1548,6 +1591,8 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
                 return address_val(buf, end, ptr, spec, fmt);
         case 'd':
                 return dentry_name(buf, end, ptr, spec, fmt);
+        case 'C':
+                return clock(buf, end, ptr, spec, fmt);
         case 'D':
                 return dentry_name(buf, end,
                                    ((const struct file *)ptr)->f_path.dentry,
@@ -1738,29 +1783,21 @@ qualifier:
         if (spec->qualifier == 'L')
                 spec->type = FORMAT_TYPE_LONG_LONG;
         else if (spec->qualifier == 'l') {
-                if (spec->flags & SIGN)
-                        spec->type = FORMAT_TYPE_LONG;
-                else
-                        spec->type = FORMAT_TYPE_ULONG;
+                BUILD_BUG_ON(FORMAT_TYPE_ULONG + SIGN != FORMAT_TYPE_LONG);
+                spec->type = FORMAT_TYPE_ULONG + (spec->flags & SIGN);
         } else if (_tolower(spec->qualifier) == 'z') {
                 spec->type = FORMAT_TYPE_SIZE_T;
         } else if (spec->qualifier == 't') {
                 spec->type = FORMAT_TYPE_PTRDIFF;
         } else if (spec->qualifier == 'H') {
-                if (spec->flags & SIGN)
-                        spec->type = FORMAT_TYPE_BYTE;
-                else
-                        spec->type = FORMAT_TYPE_UBYTE;
+                BUILD_BUG_ON(FORMAT_TYPE_UBYTE + SIGN != FORMAT_TYPE_BYTE);
+                spec->type = FORMAT_TYPE_UBYTE + (spec->flags & SIGN);
         } else if (spec->qualifier == 'h') {
-                if (spec->flags & SIGN)
-                        spec->type = FORMAT_TYPE_SHORT;
-                else
-                        spec->type = FORMAT_TYPE_USHORT;
+                BUILD_BUG_ON(FORMAT_TYPE_USHORT + SIGN != FORMAT_TYPE_SHORT);
+                spec->type = FORMAT_TYPE_USHORT + (spec->flags & SIGN);
         } else {
-                if (spec->flags & SIGN)
-                        spec->type = FORMAT_TYPE_INT;
-                else
-                        spec->type = FORMAT_TYPE_UINT;
+                BUILD_BUG_ON(FORMAT_TYPE_UINT + SIGN != FORMAT_TYPE_INT);
+                spec->type = FORMAT_TYPE_UINT + (spec->flags & SIGN);
         }
 
         return ++fmt - start;
@@ -1800,6 +1837,11 @@ qualifier:
  * %*pE[achnops] print an escaped buffer
  * %*ph[CDN] a variable-length hex string with a separator (supports up to 64
  *           bytes of the input)
+ * %pC output the name (Common Clock Framework) or address (legacy clock
+ *     framework) of a clock
+ * %pCn output the name (Common Clock Framework) or address (legacy clock
+ *      framework) of a clock
+ * %pCr output the current rate of a clock
  * %n is ignored
  *
  * ** Please update Documentation/printk-formats.txt when making changes **