Merge tag 'perf-core-for-mingo-5.1-20190311' of git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux into perf/urgent

Pull perf/core improvements and fixes from Arnaldo:

kernel:

  Stephane Eranian :

  - Restore mmap record type correctly when handling PERF_RECORD_MMAP2
    events, as the same template is used for all the threads interested
    in mmap events, some may want just PERF_RECORD_MMAP, while some
    may want the extra info in MMAP2 records.

perf probe:

  Adrian Hunter:

  - Fix getting the kernel map, because since changes related to x86 PTI
    entry trampolines handling, there are more than one kernel map.

perf script:

  Andi Kleen:

  - Support insn output for normal samples, i.e.:

    perf script -F ip,sym,insn --xed

    Will fetch the sample IP from the thread address space and feed it
    to Intel's XED disassembler, producing lines such as:

      ffffffffa4068804 native_write_msr            wrmsr
      ffffffffa415b95e __hrtimer_next_event_base   movq  0x18(%rax), %rdx

    That match 'perf annotate's output.

  - Make the --cpu filter apply to  PERF_RECORD_COMM/FORK/... events, in
    addition to PERF_RECORD_SAMPLE.

perf report:

  - Add a new --samples option to save a small random number of samples
    per hist entry, using a reservoir technique to select a representative
    number of samples.

    Then allow browsing the samples using 'perf script' as part of the hist
    entry context menu. This automatically adds the right filters, so only
    the thread or CPU of the sample is displayed. Then we use less' search
    functionality to directly jump to the time stamp of the selected sample.

    It uses different menus for assembler and source display.  Assembler
    needs xed installed and source needs debuginfo.

  - Fix the UI browser scripts pop up menu when there are many scripts
    available.

perf report:

  Andi Kleen:

  - Add 'time' sort option. E.g.:

    % perf report --sort time,overhead,symbol --time-quantum 1ms --stdio
    ...
         0.67%  277061.87300  [.] _dl_start
         0.50%  277061.87300  [.] f1
         0.50%  277061.87300  [.] f2
         0.33%  277061.87300  [.] main
         0.29%  277061.87300  [.] _dl_lookup_symbol_x
         0.29%  277061.87300  [.] dl_main
         0.29%  277061.87300  [.] do_lookup_x
         0.17%  277061.87300  [.] _dl_debug_initialize
         0.17%  277061.87300  [.] _dl_init_paths
         0.08%  277061.87300  [.] check_match
         0.04%  277061.87300  [.] _dl_count_modids
         1.33%  277061.87400  [.] f1
         1.33%  277061.87400  [.] f2
         1.33%  277061.87400  [.] main
         1.17%  277061.87500  [.] main
         1.08%  277061.87500  [.] f1
         1.08%  277061.87500  [.] f2
         1.00%  277061.87600  [.] main
         0.83%  277061.87600  [.] f1
         0.83%  277061.87600  [.] f2
         1.00%  277061.87700  [.] main

tools headers:

  Arnaldo Carvalho de Melo:

  - Update x86's syscall_64.tbl, no change in tools/perf behaviour.

  -  Sync copies asm-generic/unistd.h and linux/in with the kernel sources.

perf data:

  Jiri Olsa:

  - Prep work to support having perf.data stored as a directory, with one
    file per CPU, that ultimately will allow having one ring buffer reading
    thread per CPU.

Vendor events:

  Martin Liška:

  - perf PMU events for AMD Family 17h.

perf script python:

  Tony Jones:

  - Add python3 support for the remaining Intel PT related scripts, with
    these we should have a clean build of perf with python3 while still
    supporting the build with python2.

libbpf:

  Arnaldo Carvalho de Melo:

  - Fix the build on uCLibc, adding the missing stdarg.h since we use
    va_list in one typedef.

Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>

3 files changed, 89 insertions, 40 deletions

diff --git a/tools/perf/scripts/python/export-to-postgresql.py b/tools/perf/scripts/python/export-to-postgresql.py
index 390a351d15ea..c3eae1d77d36 100644
--- a/tools/perf/scripts/python/export-to-postgresql.py
+++ b/tools/perf/scripts/python/export-to-postgresql.py
@@ -10,6 +10,8 @@
 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 # more details.
 
+from __future__ import print_function
+
 import os
 import sys
 import struct
@@ -199,6 +201,18 @@ import datetime
 
 from PySide.QtSql import *
 
+if sys.version_info < (3, 0):
+        def toserverstr(str):
+                return str
+        def toclientstr(str):
+                return str
+else:
+        # Assume UTF-8 server_encoding and client_encoding
+        def toserverstr(str):
+                return bytes(str, "UTF_8")
+        def toclientstr(str):
+                return bytes(str, "UTF_8")
+
 # Need to access PostgreSQL C library directly to use COPY FROM STDIN
 from ctypes import *
 libpq = CDLL("libpq.so.5")
@@ -234,12 +248,17 @@ perf_db_export_mode = True
 perf_db_export_calls = False
 perf_db_export_callchains = False
 
+def printerr(*args, **kw_args):
+        print(*args, file=sys.stderr, **kw_args)
+
+def printdate(*args, **kw_args):
+        print(datetime.datetime.today(), *args, sep=' ', **kw_args)
 
 def usage():
-        print >> sys.stderr, "Usage is: export-to-postgresql.py <database name> [<columns>] [<calls>] [<callchains>]"
-        print >> sys.stderr, "where:    columns         'all' or 'branches'"
-        print >> sys.stderr, "          calls           'calls' => create calls and call_paths table"
-        print >> sys.stderr, "          callchains      'callchains' => create call_paths table"
+        printerr("Usage is: export-to-postgresql.py <database name> [<columns>] [<calls>] [<callchains>]")
+        printerr("where:        columns         'all' or 'branches'")
+        printerr("              calls           'calls' => create calls and call_paths table")
+        printerr("              callchains      'callchains' => create call_paths table")
         raise Exception("Too few arguments")
 
 if (len(sys.argv) < 2):
@@ -273,7 +292,7 @@ def do_query(q, s):
                 return
         raise Exception("Query failed: " + q.lastError().text())
 
-print datetime.datetime.today(), "Creating database..."
+printdate("Creating database...")
 
 db = QSqlDatabase.addDatabase('QPSQL')
 query = QSqlQuery(db)
@@ -506,12 +525,12 @@ do_query(query, 'CREATE VIEW samples_view AS '
         ' FROM samples')
 
 
-file_header = struct.pack("!11sii", "PGCOPY\n\377\r\n\0", 0, 0)
-file_trailer = "\377\377"
+file_header = struct.pack("!11sii", b"PGCOPY\n\377\r\n\0", 0, 0)
+file_trailer = b"\377\377"
 
 def open_output_file(file_name):
         path_name = output_dir_name + "/" + file_name
-        file = open(path_name, "w+")
+        file = open(path_name, "wb+")
         file.write(file_header)
         return file
 
@@ -526,13 +545,13 @@ def copy_output_file_direct(file, table_name):
 
 # Use COPY FROM STDIN because security may prevent postgres from accessing the files directly
 def copy_output_file(file, table_name):
-        conn = PQconnectdb("dbname = " + dbname)
+        conn = PQconnectdb(toclientstr("dbname = " + dbname))
         if (PQstatus(conn)):
                 raise Exception("COPY FROM STDIN PQconnectdb failed")
         file.write(file_trailer)
         file.seek(0)
         sql = "COPY " + table_name + " FROM STDIN (FORMAT 'binary')"
-        res = PQexec(conn, sql)
+        res = PQexec(conn, toclientstr(sql))
         if (PQresultStatus(res) != 4):
                 raise Exception("COPY FROM STDIN PQexec failed")
         data = file.read(65536)
@@ -566,7 +585,7 @@ if perf_db_export_calls:
         call_file               = open_output_file("call_table.bin")
 
 def trace_begin():
-        print datetime.datetime.today(), "Writing to intermediate files..."
+        printdate("Writing to intermediate files...")
         # id == 0 means unknown.  It is easier to create records for them than replace the zeroes with NULLs
         evsel_table(0, "unknown")
         machine_table(0, 0, "unknown")
@@ -582,7 +601,7 @@ def trace_begin():
 unhandled_count = 0
 
 def trace_end():
-        print datetime.datetime.today(), "Copying to database..."
+        printdate("Copying to database...")
         copy_output_file(evsel_file,            "selected_events")
         copy_output_file(machine_file,          "machines")
         copy_output_file(thread_file,           "threads")
@@ -597,7 +616,7 @@ def trace_end():
         if perf_db_export_calls:
                 copy_output_file(call_file,             "calls")
 
-        print datetime.datetime.today(), "Removing intermediate files..."
+        printdate("Removing intermediate files...")
         remove_output_file(evsel_file)
         remove_output_file(machine_file)
         remove_output_file(thread_file)
@@ -612,7 +631,7 @@ def trace_end():
         if perf_db_export_calls:
                 remove_output_file(call_file)
         os.rmdir(output_dir_name)
-        print datetime.datetime.today(), "Adding primary keys"
+        printdate("Adding primary keys")
         do_query(query, 'ALTER TABLE selected_events ADD PRIMARY KEY (id)')
         do_query(query, 'ALTER TABLE machines        ADD PRIMARY KEY (id)')
         do_query(query, 'ALTER TABLE threads         ADD PRIMARY KEY (id)')
@@ -627,7 +646,7 @@ def trace_end():
         if perf_db_export_calls:
                 do_query(query, 'ALTER TABLE calls           ADD PRIMARY KEY (id)')
 
-        print datetime.datetime.today(), "Adding foreign keys"
+        printdate("Adding foreign keys")
         do_query(query, 'ALTER TABLE threads '
                                         'ADD CONSTRAINT machinefk  FOREIGN KEY (machine_id)   REFERENCES machines   (id),'
                                         'ADD CONSTRAINT processfk  FOREIGN KEY (process_id)   REFERENCES threads    (id)')
@@ -663,8 +682,8 @@ def trace_end():
                 do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)')
 
         if (unhandled_count):
-                print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
-        print datetime.datetime.today(), "Done"
+                printdate("Warning: ", unhandled_count, " unhandled events")
+        printdate("Done")
 
 def trace_unhandled(event_name, context, event_fields_dict):
         global unhandled_count
@@ -674,12 +693,14 @@ def sched__sched_switch(*x):
         pass
 
 def evsel_table(evsel_id, evsel_name, *x):
+        evsel_name = toserverstr(evsel_name)
         n = len(evsel_name)
         fmt = "!hiqi" + str(n) + "s"
         value = struct.pack(fmt, 2, 8, evsel_id, n, evsel_name)
         evsel_file.write(value)
 
 def machine_table(machine_id, pid, root_dir, *x):
+        root_dir = toserverstr(root_dir)
         n = len(root_dir)
         fmt = "!hiqiii" + str(n) + "s"
         value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, root_dir)
@@ -690,6 +711,7 @@ def thread_table(thread_id, machine_id, process_id, pid, tid, *x):
         thread_file.write(value)
 
 def comm_table(comm_id, comm_str, *x):
+        comm_str = toserverstr(comm_str)
         n = len(comm_str)
         fmt = "!hiqi" + str(n) + "s"
         value = struct.pack(fmt, 2, 8, comm_id, n, comm_str)
@@ -701,6 +723,9 @@ def comm_thread_table(comm_thread_id, comm_id, thread_id, *x):
         comm_thread_file.write(value)
 
 def dso_table(dso_id, machine_id, short_name, long_name, build_id, *x):
+        short_name = toserverstr(short_name)
+        long_name = toserverstr(long_name)
+        build_id = toserverstr(build_id)
         n1 = len(short_name)
         n2 = len(long_name)
         n3 = len(build_id)
@@ -709,12 +734,14 @@ def dso_table(dso_id, machine_id, short_name, long_name, build_id, *x):
         dso_file.write(value)
 
 def symbol_table(symbol_id, dso_id, sym_start, sym_end, binding, symbol_name, *x):
+        symbol_name = toserverstr(symbol_name)
         n = len(symbol_name)
         fmt = "!hiqiqiqiqiii" + str(n) + "s"
         value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, sym_start, 8, sym_end, 4, binding, n, symbol_name)
         symbol_file.write(value)
 
 def branch_type_table(branch_type, name, *x):
+        name = toserverstr(name)
         n = len(name)
         fmt = "!hiii" + str(n) + "s"
         value = struct.pack(fmt, 2, 4, branch_type, n, name)
diff --git a/tools/perf/scripts/python/export-to-sqlite.py b/tools/perf/scripts/python/export-to-sqlite.py
index eb63e6c7107f..3b71902a5a21 100644
--- a/tools/perf/scripts/python/export-to-sqlite.py
+++ b/tools/perf/scripts/python/export-to-sqlite.py
@@ -10,6 +10,8 @@
 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 # more details.
 
+from __future__ import print_function
+
 import os
 import sys
 import struct
@@ -60,11 +62,17 @@ perf_db_export_mode = True
 perf_db_export_calls = False
 perf_db_export_callchains = False
 
+def printerr(*args, **keyword_args):
+        print(*args, file=sys.stderr, **keyword_args)
+
+def printdate(*args, **kw_args):
+        print(datetime.datetime.today(), *args, sep=' ', **kw_args)
+
 def usage():
-        print >> sys.stderr, "Usage is: export-to-sqlite.py <database name> [<columns>] [<calls>] [<callchains>]"
-        print >> sys.stderr, "where:    columns         'all' or 'branches'"
-        print >> sys.stderr, "          calls           'calls' => create calls and call_paths table"
-        print >> sys.stderr, "          callchains      'callchains' => create call_paths table"
+        printerr("Usage is: export-to-sqlite.py <database name> [<columns>] [<calls>] [<callchains>]");
+        printerr("where:        columns         'all' or 'branches'");
+        printerr("              calls           'calls' => create calls and call_paths table");
+        printerr("              callchains      'callchains' => create call_paths table");
         raise Exception("Too few arguments")
 
 if (len(sys.argv) < 2):
@@ -100,7 +108,7 @@ def do_query_(q):
                 return
         raise Exception("Query failed: " + q.lastError().text())
 
-print datetime.datetime.today(), "Creating database..."
+printdate("Creating database ...")
 
 db_exists = False
 try:
@@ -378,7 +386,7 @@ if perf_db_export_calls:
         call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
 
 def trace_begin():
-        print datetime.datetime.today(), "Writing records..."
+        printdate("Writing records...")
         do_query(query, 'BEGIN TRANSACTION')
         # id == 0 means unknown.  It is easier to create records for them than replace the zeroes with NULLs
         evsel_table(0, "unknown")
@@ -397,14 +405,14 @@ unhandled_count = 0
 def trace_end():
         do_query(query, 'END TRANSACTION')
 
-        print datetime.datetime.today(), "Adding indexes"
+        printdate("Adding indexes")
         if perf_db_export_calls:
                 do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
                 do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)')
 
         if (unhandled_count):
-                print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
-        print datetime.datetime.today(), "Done"
+                printdate("Warning: ", unhandled_count, " unhandled events")
+        printdate("Done")
 
 def trace_unhandled(event_name, context, event_fields_dict):
         global unhandled_count
diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py
index afec9479ca7f..e38518cdcbc3 100755
--- a/tools/perf/scripts/python/exported-sql-viewer.py
+++ b/tools/perf/scripts/python/exported-sql-viewer.py
@@ -88,11 +88,20 @@
 #                                                                              7fab593ea956 48 89 15 3b 13 22 00                            movq  %rdx, 0x22133b(%rip)
 # 8107675243232  2    ls       22011  22011  hardware interrupt     No         7fab593ea956 _dl_start+0x26 (ld-2.19.so) -> ffffffff86a012e0 page_fault ([kernel])
 
+from __future__ import print_function
+
 import sys
 import weakref
 import threading
 import string
-import cPickle
+try:
+        # Python2
+        import cPickle as pickle
+        # size of pickled integer big enough for record size
+        glb_nsz = 8
+except ImportError:
+        import pickle
+        glb_nsz = 16
 import re
 import os
 from PySide.QtCore import *
@@ -102,6 +111,15 @@ from decimal import *
 from ctypes import *
 from multiprocessing import Process, Array, Value, Event
 
+# xrange is range in Python3
+try:
+        xrange
+except NameError:
+        xrange = range
+
+def printerr(*args, **keyword_args):
+        print(*args, file=sys.stderr, **keyword_args)
+
 # Data formatting helpers
 
 def tohex(ip):
@@ -1004,10 +1022,6 @@ class ChildDataItemFinder():
 
 glb_chunk_sz = 10000
 
-# size of pickled integer big enough for record size
-
-glb_nsz = 8
-
 # Background process for SQL data fetcher
 
 class SQLFetcherProcess():
@@ -1066,7 +1080,7 @@ class SQLFetcherProcess():
                                 return True
                         if space >= glb_nsz:
                                 # Use 0 (or space < glb_nsz) to mean there is no more at the top of the buffer
-                                nd = cPickle.dumps(0, cPickle.HIGHEST_PROTOCOL)
+                                nd = pickle.dumps(0, pickle.HIGHEST_PROTOCOL)
                                 self.buffer[self.local_head : self.local_head + len(nd)] = nd
                         self.local_head = 0
                 if self.local_tail - self.local_head > sz:
@@ -1084,9 +1098,9 @@ class SQLFetcherProcess():
                         self.wait_event.wait()
 
         def AddToBuffer(self, obj):
-                d = cPickle.dumps(obj, cPickle.HIGHEST_PROTOCOL)
+                d = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
                 n = len(d)
-                nd = cPickle.dumps(n, cPickle.HIGHEST_PROTOCOL)
+                nd = pickle.dumps(n, pickle.HIGHEST_PROTOCOL)
                 sz = n + glb_nsz
                 self.WaitForSpace(sz)
                 pos = self.local_head
@@ -1198,12 +1212,12 @@ class SQLFetcher(QObject):
                 pos = self.local_tail
                 if len(self.buffer) - pos < glb_nsz:
                         pos = 0
-                n = cPickle.loads(self.buffer[pos : pos + glb_nsz])
+                n = pickle.loads(self.buffer[pos : pos + glb_nsz])
                 if n == 0:
                         pos = 0
-                        n = cPickle.loads(self.buffer[0 : glb_nsz])
+                        n = pickle.loads(self.buffer[0 : glb_nsz])
                 pos += glb_nsz
-                obj = cPickle.loads(self.buffer[pos : pos + n])
+                obj = pickle.loads(self.buffer[pos : pos + n])
                 self.local_tail = pos + n
                 return obj
 
@@ -2973,7 +2987,7 @@ class DBRef():
 
 def Main():
         if (len(sys.argv) < 2):
-                print >> sys.stderr, "Usage is: exported-sql-viewer.py {<database name> | --help-only}"
+                printerr("Usage is: exported-sql-viewer.py {<database name> | --help-only}");
                 raise Exception("Too few arguments")
 
         dbname = sys.argv[1]
@@ -2986,8 +3000,8 @@ def Main():
 
         is_sqlite3 = False
         try:
-                f = open(dbname)
-                if f.read(15) == "SQLite format 3":
+                f = open(dbname, "rb")
+                if f.read(15) == b'SQLite format 3':
                         is_sqlite3 = True
                 f.close()
         except: