Update to pm-graph 5.4

bootgraph:
 - dmesg log format has changed, update parser in two places
 - fix prints in preparation for upgrade to python3

sleepgraph:
 - fix prints in preparation for upgrade to python3
 - add new trace events and kprobes to cover freeze more completely
 - add new -ftop callgraph trace over suspend_devices_and_enter
 - add -wifi option to check if a wifi connection is active
 - add -skipkprobe option to suppress unwanted kprobes in dev mode
 - add kernel params and sysinfo to the log output
 - don't crash if /dev/mem is throwing IO errors, ignore FPDT and DMI
 - fix kprobe length calculation when calls are recursive
 - add several new kernel issue definitions for USB, ACPI, ATA, etc
 - enable turbostat output to be read from stdout instead of from file
 - add BIOS call data to the timeline from acpi_ps_execute_method kprobe

Signed-off-by: Todd Brandt <todd.e.brandt@linux.intel.com>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>

2 files changed, 347 insertions, 152 deletions

diff --git a/tools/power/pm-graph/bootgraph.py b/tools/power/pm-graph/bootgraph.py
index 6dae57041537..d7f4bd152bf1 100755
--- a/tools/power/pm-graph/bootgraph.py
+++ b/tools/power/pm-graph/bootgraph.py
@@ -333,9 +333,9 @@ def parseKernelLog():
                         if(not sysvals.stamp['kernel']):
                                 sysvals.stamp['kernel'] = sysvals.kernelVersion(msg)
                         continue
-                m = re.match('.* setting system clock to (?P<t>.*) UTC.*', msg)
+                m = re.match('.* setting system clock to (?P<d>[0-9\-]*)[ A-Z](?P<t>[0-9:]*) UTC.*', msg)
                 if(m):
-                        bt = datetime.strptime(m.group('t'), '%Y-%m-%d %H:%M:%S')
+                        bt = datetime.strptime(m.group('d')+' '+m.group('t'), '%Y-%m-%d %H:%M:%S')
                         bt = bt - timedelta(seconds=int(ktime))
                         data.boottime = bt.strftime('%Y-%m-%d_%H:%M:%S')
                         sysvals.stamp['time'] = bt.strftime('%B %d %Y, %I:%M:%S %p')
@@ -356,7 +356,7 @@ def parseKernelLog():
                                 data.newAction(phase, f, pid, start, ktime, int(r), int(t))
                                 del devtemp[f]
                         continue
-                if(re.match('^Freeing unused kernel memory.*', msg)):
+                if(re.match('^Freeing unused kernel .*', msg)):
                         data.tUserMode = ktime
                         data.dmesg['kernel']['end'] = ktime
                         data.dmesg['user']['start'] = ktime
@@ -1016,7 +1016,7 @@ if __name__ == '__main__':
                         updateKernelParams()
                 elif cmd == 'flistall':
                         for f in sysvals.getBootFtraceFilterFunctions():
-                                print f
+                                print(f)
                 elif cmd == 'checkbl':
                         sysvals.getBootLoader()
                         pprint('Boot Loader: %s\n%s' % (sysvals.bootloader, sysvals.blexec))
diff --git a/tools/power/pm-graph/sleepgraph.py b/tools/power/pm-graph/sleepgraph.py
index 41d28d63e7c9..ccd0f3917c51 100755
--- a/tools/power/pm-graph/sleepgraph.py
+++ b/tools/power/pm-graph/sleepgraph.py
@@ -17,9 +17,9 @@
 #
 # Links:
 #        Home Page
-#          https://01.org/suspendresume
+#          https://01.org/pm-graph
 #        Source repo
-#          git@github.com:01org/pm-graph
+#          git@github.com:intel/pm-graph
 #
 # Description:
 #        This tool is designed to assist kernel and OS developers in optimizing
@@ -32,6 +32,7 @@
 #        viewed in firefox or chrome.
 #
 #        The following kernel build options are required:
+#                CONFIG_DEVMEM=y
 #                CONFIG_PM_DEBUG=y
 #                CONFIG_PM_SLEEP_DEBUG=y
 #                CONFIG_FTRACE=y
@@ -75,7 +76,7 @@ def pprint(msg):
 #        store system values and test parameters
 class SystemValues:
         title = 'SleepGraph'
-        version = '5.3'
+        version = '5.4'
         ansi = False
         rs = 0
         display = ''
@@ -83,8 +84,9 @@ class SystemValues:
         sync = False
         verbose = False
         testlog = True
-        dmesglog = False
+        dmesglog = True
         ftracelog = False
+        tstat = False
         mindevlen = 0.0
         mincglen = 0.0
         cgphase = ''
@@ -108,6 +110,8 @@ class SystemValues:
         pmdpath = '/sys/power/pm_debug_messages'
         traceevents = [
                 'suspend_resume',
+                'wakeup_source_activate',
+                'wakeup_source_deactivate',
                 'device_pm_callback_end',
                 'device_pm_callback_start'
         ]
@@ -139,6 +143,8 @@ class SystemValues:
         x2delay = 0
         skiphtml = False
         usecallgraph = False
+        ftopfunc = 'suspend_devices_and_enter'
+        ftop = False
         usetraceevents = False
         usetracemarkers = True
         usekprobes = True
@@ -167,6 +173,13 @@ class SystemValues:
                 'acpi_hibernation_leave': {},
                 'acpi_pm_freeze': {},
                 'acpi_pm_thaw': {},
+                'acpi_s2idle_end': {},
+                'acpi_s2idle_sync': {},
+                'acpi_s2idle_begin': {},
+                'acpi_s2idle_prepare': {},
+                'acpi_s2idle_wake': {},
+                'acpi_s2idle_wakeup': {},
+                'acpi_s2idle_restore': {},
                 'hibernate_preallocate_memory': {},
                 'create_basic_memory_bitmaps': {},
                 'swsusp_write': {},
@@ -203,7 +216,11 @@ class SystemValues:
                 'rt_mutex_slowlock': {'ub': 1},
                 # ACPI
                 'acpi_resume_power_resources': {},
-                'acpi_ps_parse_aml': {},
+                'acpi_ps_execute_method': { 'args_x86_64': {
+                        'fullpath':'+0(+40(%di)):string',
+                }},
+                # mei_me
+                'mei_reset': {},
                 # filesystem
                 'ext4_sync_fs': {},
                 # 80211
@@ -252,6 +269,7 @@ class SystemValues:
         timeformat = '%.3f'
         cmdline = '%s %s' % \
                         (os.path.basename(sys.argv[0]), ' '.join(sys.argv[1:]))
+        kparams = ''
         sudouser = ''
         def __init__(self):
                 self.archargs = 'args_'+platform.machine()
@@ -330,6 +348,7 @@ class SystemValues:
                 args['date'] = n.strftime('%y%m%d')
                 args['time'] = n.strftime('%H%M%S')
                 args['hostname'] = args['host'] = self.hostname
+                args['mode'] = self.suspendmode
                 return value.format(**args)
         def setOutputFile(self):
                 if self.dmesgfile != '':
@@ -341,7 +360,7 @@ class SystemValues:
                         if(m):
                                 self.htmlfile = m.group('name')+'.html'
         def systemInfo(self, info):
-                p = c = m = b = ''
+                p = m = ''
                 if 'baseboard-manufacturer' in info:
                         m = info['baseboard-manufacturer']
                 elif 'system-manufacturer' in info:
@@ -352,12 +371,17 @@ class SystemValues:
                         p = info['baseboard-product-name']
                 if m[:5].lower() == 'intel' and 'baseboard-product-name' in info:
                         p = info['baseboard-product-name']
-                if 'processor-version' in info:
-                        c = info['processor-version']
-                if 'bios-version' in info:
-                        b = info['bios-version']
-                self.sysstamp = '# sysinfo | man:%s | plat:%s | cpu:%s | bios:%s | numcpu:%d | memsz:%d | memfr:%d' % \
-                        (m, p, c, b, self.cpucount, self.memtotal, self.memfree)
+                c = info['processor-version'] if 'processor-version' in info else ''
+                b = info['bios-version'] if 'bios-version' in info else ''
+                r = info['bios-release-date'] if 'bios-release-date' in info else ''
+                self.sysstamp = '# sysinfo | man:%s | plat:%s | cpu:%s | bios:%s | biosdate:%s | numcpu:%d | memsz:%d | memfr:%d' % \
+                        (m, p, c, b, r, self.cpucount, self.memtotal, self.memfree)
+                try:
+                        kcmd = open('/proc/cmdline', 'r').read().strip()
+                except:
+                        kcmd = ''
+                if kcmd:
+                        self.sysstamp += '\n# kparams | %s' % kcmd
         def printSystemInfo(self, fatal=False):
                 self.rootCheck(True)
                 out = dmidecode(self.mempath, fatal)
@@ -365,10 +389,10 @@ class SystemValues:
                         return
                 fmt = '%-24s: %s'
                 for name in sorted(out):
-                        print fmt % (name, out[name])
-                print fmt % ('cpucount', ('%d' % self.cpucount))
-                print fmt % ('memtotal', ('%d kB' % self.memtotal))
-                print fmt % ('memfree', ('%d kB' % self.memfree))
+                        print(fmt % (name, out[name]))
+                print(fmt % ('cpucount', ('%d' % self.cpucount)))
+                print(fmt % ('memtotal', ('%d kB' % self.memtotal)))
+                print(fmt % ('memfree', ('%d kB' % self.memfree)))
         def cpuInfo(self):
                 self.cpucount = 0
                 fp = open('/proc/cpuinfo', 'r')
@@ -388,7 +412,7 @@ class SystemValues:
         def initTestOutput(self, name):
                 self.prefix = self.hostname
                 v = open('/proc/version', 'r').read().strip()
-                kver = string.split(v)[2]
+                kver = v.split()[2]
                 fmt = name+'-%m%d%y-%H%M%S'
                 testtime = datetime.now().strftime(fmt)
                 self.teststamp = \
@@ -403,7 +427,7 @@ class SystemValues:
                 self.htmlfile = \
                         self.testdir+'/'+self.prefix+'_'+self.suspendmode+'.html'
                 if not os.path.isdir(self.testdir):
-                        os.mkdir(self.testdir)
+                        os.makedirs(self.testdir)
         def getValueList(self, value):
                 out = []
                 for i in value.split(','):
@@ -414,6 +438,12 @@ class SystemValues:
                 self.devicefilter = self.getValueList(value)
         def setCallgraphFilter(self, value):
                 self.cgfilter = self.getValueList(value)
+        def skipKprobes(self, value):
+                for k in self.getValueList(value):
+                        if k in self.tracefuncs:
+                                del self.tracefuncs[k]
+                        if k in self.dev_tracefuncs:
+                                del self.dev_tracefuncs[k]
         def setCallgraphBlacklist(self, file):
                 self.cgblacklist = self.listFromFile(file)
         def rtcWakeAlarmOn(self):
@@ -483,9 +513,9 @@ class SystemValues:
                         if 'func' in self.tracefuncs[i]:
                                 i = self.tracefuncs[i]['func']
                         if i in master:
-                                print i
+                                print(i)
                         else:
-                                print self.colorText(i)
+                                print(self.colorText(i))
         def setFtraceFilterFunctions(self, list):
                 master = self.listFromFile(self.tpath+'available_filter_functions')
                 flist = ''
@@ -728,7 +758,10 @@ class SystemValues:
                                         cf.append(self.tracefuncs[fn]['func'])
                                 else:
                                         cf.append(fn)
-                        self.setFtraceFilterFunctions(cf)
+                        if self.ftop:
+                                self.setFtraceFilterFunctions([self.ftopfunc])
+                        else:
+                                self.setFtraceFilterFunctions(cf)
                 # initialize the kprobe trace
                 elif self.usekprobes:
                         for name in self.tracefuncs:
@@ -788,6 +821,14 @@ class SystemValues:
                         if 'bat' in test:
                                 (a1, c1), (a2, c2) = test['bat']
                                 fp.write('# battery %s %d %s %d\n' % (a1, c1, a2, c2))
+                        if 'wifi' in test:
+                                wstr = []
+                                for wifi in test['wifi']:
+                                        tmp = []
+                                        for key in sorted(wifi):
+                                                tmp.append('%s:%s' % (key, wifi[key]))
+                                        wstr.append('|'.join(tmp))
+                                fp.write('# wifi %s\n' % (','.join(wstr)))
                         if test['error'] or len(testdata) > 1:
                                 fp.write('# enter_sleep_error %s\n' % test['error'])
                 return fp
@@ -852,26 +893,22 @@ class SystemValues:
                         return ''
                 return base64.b64encode(out.encode('zlib'))
         def haveTurbostat(self):
+                if not self.tstat:
+                        return False
                 cmd = self.getExec('turbostat')
                 if not cmd:
                         return False
                 fp = Popen([cmd, '-v'], stdout=PIPE, stderr=PIPE).stderr
                 out = fp.read().strip()
                 fp.close()
-                return re.match('turbostat version [0-8.]* .*', out)
+                return re.match('turbostat version [0-9\.]* .*', out)
         def turbostat(self):
                 cmd = self.getExec('turbostat')
                 if not cmd:
                         return 'missing turbostat executable'
-                outfile = '/tmp/pm-graph-turbostat.txt'
-                res = call('%s -o %s -q -S echo freeze > %s' % \
-                        (cmd, outfile, self.powerfile), shell=True)
-                if res != 0:
-                        return 'turbosat returned %d' % res
-                if not os.path.exists(outfile):
-                        return 'turbostat output missing'
-                fp = open(outfile, 'r')
                 text = []
+                fullcmd = '%s -q -S echo freeze > %s' % (cmd, self.powerfile)
+                fp = Popen(['sh', '-c', fullcmd], stdout=PIPE, stderr=PIPE).stderr
                 for line in fp:
                         if re.match('[0-9.]* sec', line):
                                 continue
@@ -888,6 +925,60 @@ class SystemValues:
                                         values.append(line[idx])
                         out.append('%s=%s' % (key, ','.join(values)))
                 return '|'.join(out)
+        def checkWifi(self):
+                out = dict()
+                iwcmd, ifcmd = self.getExec('iwconfig'), self.getExec('ifconfig')
+                if not iwcmd or not ifcmd:
+                        return out
+                fp = Popen(iwcmd, stdout=PIPE, stderr=PIPE).stdout
+                for line in fp:
+                        m = re.match('(?P<dev>\S*) .* ESSID:(?P<ess>\S*)', line)
+                        if not m:
+                                continue
+                        out['device'] = m.group('dev')
+                        if '"' in m.group('ess'):
+                                out['essid'] = m.group('ess').strip('"')
+                                break
+                fp.close()
+                if 'device' in out:
+                        fp = Popen([ifcmd, out['device']], stdout=PIPE, stderr=PIPE).stdout
+                        for line in fp:
+                                m = re.match('.* inet (?P<ip>[0-9\.]*)', line)
+                                if m:
+                                        out['ip'] = m.group('ip')
+                                        break
+                        fp.close()
+                return out
+        def errorSummary(self, errinfo, msg):
+                found = False
+                for entry in errinfo:
+                        if re.match(entry['match'], msg):
+                                entry['count'] += 1
+                                if self.hostname not in entry['urls']:
+                                        entry['urls'][self.hostname] = [self.htmlfile]
+                                elif self.htmlfile not in entry['urls'][self.hostname]:
+                                        entry['urls'][self.hostname].append(self.htmlfile)
+                                found = True
+                                break
+                if found:
+                        return
+                arr = msg.split()
+                for j in range(len(arr)):
+                        if re.match('^[0-9,\-\.]*$', arr[j]):
+                                arr[j] = '[0-9,\-\.]*'
+                        else:
+                                arr[j] = arr[j]\
+                                        .replace('\\', '\\\\').replace(']', '\]').replace('[', '\[')\
+                                        .replace('.', '\.').replace('+', '\+').replace('*', '\*')\
+                                        .replace('(', '\(').replace(')', '\)')
+                mstr = ' '.join(arr)
+                entry = {
+                        'line': msg,
+                        'match': mstr,
+                        'count': 1,
+                        'urls': {self.hostname: [self.htmlfile]}
+                }
+                errinfo.append(entry)
 
 sysvals = SystemValues()
 switchvalues = ['enable', 'disable', 'on', 'off', 'true', 'false', '1', '0']
@@ -982,7 +1073,14 @@ class Data:
                 'ERROR'   : '.*ERROR.*',
                 'WARNING' : '.*WARNING.*',
                 'IRQ'     : '.*genirq: .*',
-                'TASKFAIL': '.*Freezing of tasks failed.*',
+                'TASKFAIL': '.*Freezing of tasks *.*',
+                'ACPI'    : '.*ACPI *(?P<b>[A-Za-z]*) *Error[: ].*',
+                'DEVFAIL' : '.* failed to (?P<b>[a-z]*) async: .*',
+                'DISKFULL': '.*No space left on device.*',
+                'USBERR'  : '.*usb .*device .*, error [0-9-]*',
+                'ATAERR'  : ' *ata[0-9\.]*: .*failed.*',
+                'MEIERR'  : ' *mei.*: .*failed.*',
+                'TPMERR'  : '(?i) *tpm *tpm[0-9]*: .*error.*',
         }
         def __init__(self, num):
                 idchar = 'abcdefghij'
@@ -1000,6 +1098,7 @@ class Data:
                 self.outfile = ''
                 self.kerror = False
                 self.battery = 0
+                self.wifi = 0
                 self.turbostat = 0
                 self.mcelog = 0
                 self.enterfail = ''
@@ -1036,35 +1135,21 @@ class Data:
                 if len(plist) < 1:
                         return ''
                 return plist[-1]
-        def errorSummary(self, errinfo, msg):
-                found = False
-                for entry in errinfo:
-                        if re.match(entry['match'], msg):
-                                entry['count'] += 1
-                                if sysvals.hostname not in entry['urls']:
-                                        entry['urls'][sysvals.hostname] = sysvals.htmlfile
-                                found = True
-                                break
-                if found:
-                        return
-                arr = msg.split()
-                for j in range(len(arr)):
-                        if re.match('^[0-9\-\.]*$', arr[j]):
-                                arr[j] = '[0-9\-\.]*'
-                        else:
-                                arr[j] = arr[j]\
-                                        .replace(']', '\]').replace('[', '\[').replace('.', '\.')\
-                                        .replace('+', '\+').replace('*', '\*').replace('(', '\(')\
-                                        .replace(')', '\)')
-                mstr = ' '.join(arr)
-                entry = {
-                        'line': msg,
-                        'match': mstr,
-                        'count': 1,
-                        'urls': {sysvals.hostname: sysvals.htmlfile}
-                }
-                errinfo.append(entry)
-        def extractErrorInfo(self, issues=0):
+        def turbostatInfo(self):
+                tp = TestProps()
+                out = {'syslpi':'N/A','pkgpc10':'N/A'}
+                for line in self.dmesgtext:
+                        m = re.match(tp.tstatfmt, line)
+                        if not m:
+                                continue
+                        for i in m.group('t').split('|'):
+                                if 'SYS%LPI' in i:
+                                        out['syslpi'] = i.split('=')[-1]+'%'
+                                elif 'pc10' in i:
+                                        out['pkgpc10'] = i.split('=')[-1]+'%'
+                        break
+                return out
+        def extractErrorInfo(self):
                 lf = self.dmesgtext
                 if len(self.dmesgtext) < 1 and sysvals.dmesgfile:
                         lf = sysvals.openlog(sysvals.dmesgfile, 'r')
@@ -1082,19 +1167,19 @@ class Data:
                         msg = m.group('msg')
                         for err in self.errlist:
                                 if re.match(self.errlist[err], msg):
-                                        list.append((err, dir, t, i, i))
+                                        list.append((msg, err, dir, t, i, i))
                                         self.kerror = True
-                                        if not isinstance(issues, int):
-                                                self.errorSummary(issues, msg)
                                         break
-                for e in list:
-                        type, dir, t, idx1, idx2 = e
+                msglist = []
+                for msg, type, dir, t, idx1, idx2 in list:
+                        msglist.append(msg)
                         sysvals.vprint('kernel %s found in %s at %f' % (type, dir, t))
                         self.errorinfo[dir].append((type, t, idx1, idx2))
                 if self.kerror:
                         sysvals.dmesglog = True
                 if len(self.dmesgtext) < 1 and sysvals.dmesgfile:
                         lf.close()
+                return msglist
         def setStart(self, time):
                 self.start = time
         def setEnd(self, time):
@@ -2147,7 +2232,7 @@ class FTraceCallGraph:
                         if(data.dmesg[p]['start'] <= self.start and
                                 self.start <= data.dmesg[p]['end']):
                                 list = data.dmesg[p]['list']
-                                for devname in list:
+                                for devname in sorted(list, key=lambda k:list[k]['start']):
                                         dev = list[devname]
                                         if(pid == dev['pid'] and
                                                 self.start <= dev['start'] and
@@ -2452,6 +2537,7 @@ class TestProps:
                                 '(?P<H>[0-9]{2})(?P<M>[0-9]{2})(?P<S>[0-9]{2})'+\
                                 ' (?P<host>.*) (?P<mode>.*) (?P<kernel>.*)$'
         batteryfmt = '^# battery (?P<a1>\w*) (?P<c1>\d*) (?P<a2>\w*) (?P<c2>\d*)'
+        wififmt    = '^# wifi (?P<w>.*)'
         tstatfmt   = '^# turbostat (?P<t>\S*)'
         mcelogfmt  = '^# mcelog (?P<m>\S*)'
         testerrfmt = '^# enter_sleep_error (?P<e>.*)'
@@ -2479,6 +2565,7 @@ class TestProps:
                 self.mcelog = []
                 self.turbostat = []
                 self.battery = []
+                self.wifi = []
                 self.fwdata = []
                 self.ftrace_line_fmt = self.ftrace_line_fmt_nop
                 self.cgformat = False
@@ -2505,6 +2592,9 @@ class TestProps:
                 elif re.match(self.sysinfofmt, line):
                         self.sysinfo = line
                         return True
+                elif re.match(self.kparamsfmt, line):
+                        self.kparams = line
+                        return True
                 elif re.match(self.cmdlinefmt, line):
                         self.cmdline = line
                         return True
@@ -2517,6 +2607,9 @@ class TestProps:
                 elif re.match(self.batteryfmt, line):
                         self.battery.append(line)
                         return True
+                elif re.match(self.wififmt, line):
+                        self.wifi.append(line)
+                        return True
                 elif re.match(self.testerrfmt, line):
                         self.testerror.append(line)
                         return True
@@ -2586,6 +2679,11 @@ class TestProps:
                         m = re.match(self.batteryfmt, self.battery[data.testnumber])
                         if m:
                                 data.battery = m.groups()
+                # wifi data
+                if len(self.wifi) > data.testnumber:
+                        m = re.match(self.wififmt, self.wifi[data.testnumber])
+                        if m:
+                                data.wifi = m.group('w')
                 # sleep mode enter errors
                 if len(self.testerror) > data.testnumber:
                         m = re.match(self.testerrfmt, self.testerror[data.testnumber])
@@ -2655,9 +2753,9 @@ class ProcessMonitor:
 #        Quickly determine if the ftrace log has all of the trace events,
 #        markers, and/or kprobes required for primary parsing.
 def doesTraceLogHaveTraceEvents():
-        kpcheck = ['_cal: (', '_cpu_down()']
+        kpcheck = ['_cal: (', '_ret: (']
         techeck = ['suspend_resume', 'device_pm_callback']
-        tmcheck = ['tracing_mark_write']
+        tmcheck = ['SUSPEND START', 'RESUME COMPLETE']
         sysvals.usekprobes = False
         fp = sysvals.openlog(sysvals.ftracefile, 'r')
         for line in fp:
@@ -3042,7 +3140,7 @@ def parseTraceLog(live=False):
                                 tp.ktemp[key].append({
                                         'pid': pid,
                                         'begin': t.time,
-                                        'end': t.time,
+                                        'end': -1,
                                         'name': displayname,
                                         'cdata': kprobedata,
                                         'proc': m_proc,
@@ -3053,12 +3151,11 @@ def parseTraceLog(live=False):
                         elif(t.freturn):
                                 if(key not in tp.ktemp) or len(tp.ktemp[key]) < 1:
                                         continue
-                                e = tp.ktemp[key][-1]
-                                if e['begin'] < 0.0 or t.time - e['begin'] < 0.000001:
-                                        tp.ktemp[key].pop()
-                                else:
-                                        e['end'] = t.time
-                                        e['rdata'] = kprobedata
+                                e = next((x for x in reversed(tp.ktemp[key]) if x['end'] < 0), 0)
+                                if not e:
+                                        continue
+                                e['end'] = t.time
+                                e['rdata'] = kprobedata
                                 # end of kernel resume
                                 if(phase != 'suspend_prepare' and kprobename in krescalls):
                                         if phase in data.dmesg:
@@ -3080,8 +3177,10 @@ def parseTraceLog(live=False):
                         if(res == -1):
                                 testrun.ftemp[key][-1].addLine(t)
         tf.close()
+        if len(testdata) < 1:
+                sysvals.vprint('WARNING: ftrace start marker is missing')
         if data and not data.devicegroups:
-                sysvals.vprint('WARNING: end marker is missing')
+                sysvals.vprint('WARNING: ftrace end marker is missing')
                 data.handleEndMarker(t.time)
 
         if sysvals.suspendmode == 'command':
@@ -3130,9 +3229,11 @@ def parseTraceLog(live=False):
                                 name, pid = key
                                 if name not in sysvals.tracefuncs:
                                         continue
+                                if pid not in data.devpids:
+                                        data.devpids.append(pid)
                                 for e in tp.ktemp[key]:
                                         kb, ke = e['begin'], e['end']
-                                        if kb == ke or tlb > kb or tle <= kb:
+                                        if ke - kb < 0.000001 or tlb > kb or tle <= kb:
                                                 continue
                                         color = sysvals.kprobeColor(name)
                                         data.newActionGlobal(e['name'], kb, ke, pid, color)
@@ -3144,7 +3245,7 @@ def parseTraceLog(live=False):
                                                 continue
                                         for e in tp.ktemp[key]:
                                                 kb, ke = e['begin'], e['end']
-                                                if kb == ke or tlb > kb or tle <= kb:
+                                                if ke - kb < 0.000001 or tlb > kb or tle <= kb:
                                                         continue
                                                 data.addDeviceFunctionCall(e['name'], name, e['proc'], pid, kb,
                                                         ke, e['cdata'], e['rdata'])
@@ -3168,7 +3269,7 @@ def parseTraceLog(live=False):
                                         if not devname:
                                                 sortkey = '%f%f%d' % (cg.start, cg.end, pid)
                                                 sortlist[sortkey] = cg
-                                        elif len(cg.list) > 1000000:
+                                        elif len(cg.list) > 1000000 and cg.name != sysvals.ftopfunc:
                                                 sysvals.vprint('WARNING: the callgraph for %s is massive (%d lines)' %\
                                                         (devname, len(cg.list)))
                         # create blocks for orphan cg data
@@ -3275,7 +3376,7 @@ def loadKernelLog():
         if data:
                 testruns.append(data)
         if len(testruns) < 1:
-                pprint('ERROR: dmesg log has no suspend/resume data: %s' \
+                doError('dmesg log has no suspend/resume data: %s' \
                         % sysvals.dmesgfile)
 
         # fix lines with same timestamp/function with the call and return swapped
@@ -3614,6 +3715,8 @@ def addCallgraphs(sv, hf, data):
                                 name += ' '+p
                         if('ftrace' in dev):
                                 cg = dev['ftrace']
+                                if cg.name == sv.ftopfunc:
+                                        name = 'top level suspend/resume call'
                                 num = callgraphHTML(sv, hf, num, cg,
                                         name, color, dev['id'])
                         if('ftraces' in dev):
@@ -3653,9 +3756,10 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
 
         # extract the test data into list
         list = dict()
-        tAvg, tMin, tMax, tMed = [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [[], []]
+        tAvg, tMin, tMax, tMed = [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [dict(), dict()]
         iMin, iMed, iMax = [0, 0], [0, 0], [0, 0]
         num = 0
+        useturbo = False
         lastmode = ''
         cnt = dict()
         for data in sorted(testruns, key=lambda v:(v['mode'], v['host'], v['kernel'], v['time'])):
@@ -3666,20 +3770,25 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
                         for i in range(2):
                                 s = sorted(tMed[i])
                                 list[lastmode]['med'][i] = s[int(len(s)/2)]
-                                iMed[i] = tMed[i].index(list[lastmode]['med'][i])
+                                iMed[i] = tMed[i][list[lastmode]['med'][i]]
                         list[lastmode]['avg'] = [tAvg[0] / num, tAvg[1] / num]
                         list[lastmode]['min'] = tMin
                         list[lastmode]['max'] = tMax
                         list[lastmode]['idx'] = (iMin, iMed, iMax)
-                        tAvg, tMin, tMax, tMed = [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [[], []]
+                        tAvg, tMin, tMax, tMed = [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [dict(), dict()]
                         iMin, iMed, iMax = [0, 0], [0, 0], [0, 0]
                         num = 0
+                pkgpc10 = syslpi = ''
+                if 'pkgpc10' in data and 'syslpi' in data:
+                        pkgpc10 = data['pkgpc10']
+                        syslpi = data['syslpi']
+                        useturbo = True
                 res = data['result']
                 tVal = [float(data['suspend']), float(data['resume'])]
                 list[mode]['data'].append([data['host'], data['kernel'],
                         data['time'], tVal[0], tVal[1], data['url'], res,
                         data['issues'], data['sus_worst'], data['sus_worsttime'],
-                        data['res_worst'], data['res_worsttime']])
+                        data['res_worst'], data['res_worsttime'], pkgpc10, syslpi])
                 idx = len(list[mode]['data']) - 1
                 if res.startswith('fail in'):
                         res = 'fail'
@@ -3689,7 +3798,7 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
                         cnt[res] += 1
                 if res == 'pass':
                         for i in range(2):
-                                tMed[i].append(tVal[i])
+                                tMed[i][tVal[i]] = idx
                                 tAvg[i] += tVal[i]
                                 if tMin[i] == 0 or tVal[i] < tMin[i]:
                                         iMin[i] = idx
@@ -3703,7 +3812,7 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
                 for i in range(2):
                         s = sorted(tMed[i])
                         list[lastmode]['med'][i] = s[int(len(s)/2)]
-                        iMed[i] = tMed[i].index(list[lastmode]['med'][i])
+                        iMed[i] = tMed[i][list[lastmode]['med'][i]]
                 list[lastmode]['avg'] = [tAvg[0] / num, tAvg[1] / num]
                 list[lastmode]['min'] = tMin
                 list[lastmode]['max'] = tMax
@@ -3719,6 +3828,7 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
         td = '\t<td>{0}</td>\n'
         tdh = '\t<td{1}>{0}</td>\n'
         tdlink = '\t<td><a href="{0}">html</a></td>\n'
+        colspan = '14' if useturbo else '12'
 
         # table header
         html += '<table>\n<tr>\n' + th.format('#') +\
@@ -3726,12 +3836,13 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
                 th.format('Test Time') + th.format('Result') + th.format('Issues') +\
                 th.format('Suspend') + th.format('Resume') +\
                 th.format('Worst Suspend Device') + th.format('SD Time') +\
-                th.format('Worst Resume Device') + th.format('RD Time') +\
-                th.format('Detail') + '</tr>\n'
-
+                th.format('Worst Resume Device') + th.format('RD Time')
+        if useturbo:
+                html += th.format('PkgPC10') + th.format('SysLPI')
+        html += th.format('Detail')+'</tr>\n'
         # export list into html
         head = '<tr class="head"><td>{0}</td><td>{1}</td>'+\
-                '<td colspan=12 class="sus">Suspend Avg={2} '+\
+                '<td colspan='+colspan+' class="sus">Suspend Avg={2} '+\
                 '<span class=minval><a href="#s{10}min">Min={3}</a></span> '+\
                 '<span class=medval><a href="#s{10}med">Med={4}</a></span> '+\
                 '<span class=maxval><a href="#s{10}max">Max={5}</a></span> '+\
@@ -3740,7 +3851,8 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
                 '<span class=medval><a href="#r{10}med">Med={8}</a></span> '+\
                 '<span class=maxval><a href="#r{10}max">Max={9}</a></span></td>'+\
                 '</tr>\n'
-        headnone = '<tr class="head"><td>{0}</td><td>{1}</td><td colspan=12></td></tr>\n'
+        headnone = '<tr class="head"><td>{0}</td><td>{1}</td><td colspan='+\
+                colspan+'></td></tr>\n'
         for mode in list:
                 # header line for each suspend mode
                 num = 0
@@ -3787,6 +3899,9 @@ def createHTMLSummarySimple(testruns, htmlfile, title):
                         html += td.format('%.3f ms' % d[9])     if d[9] else td.format('')              # sus_worst time
                         html += td.format(d[10])                                                                        # res_worst
                         html += td.format('%.3f ms' % d[11]) if d[11] else td.format('')        # res_worst time
+                        if useturbo:
+                                html += td.format(d[12])                                                                # pkg_pc10
+                                html += td.format(d[13])                                                                # syslpi
                         html += tdlink.format(d[5]) if d[5] else td.format('')          # url
                         html += '</tr>\n'
                         num += 1
@@ -3861,8 +3976,10 @@ def createHTMLDeviceSummary(testruns, htmlfile, title):
         hf.close()
         return devall
 
-def createHTMLIssuesSummary(issues, htmlfile, title):
+def createHTMLIssuesSummary(testruns, issues, htmlfile, title, extra=''):
+        multihost = len([e for e in issues if len(e['urls']) > 1]) > 0
         html = summaryCSS('Issues Summary - SleepGraph', False)
+        total = len(testruns)
 
         # generate the html
         th = '\t<th>{0}</th>\n'
@@ -3870,27 +3987,36 @@ def createHTMLIssuesSummary(issues, htmlfile, title):
         tdlink = '<a href="{1}">{0}</a>'
         subtitle = '%d issues' % len(issues) if len(issues) > 0 else 'no issues'
         html += '<div class="stamp">%s (%s)</div><table>\n' % (title, subtitle)
-        html += '<tr>\n' + th.format('Count') + th.format('Issue') +\
-                th.format('Hosts') + th.format('First Instance') + '</tr>\n'
+        html += '<tr>\n' + th.format('Issue') + th.format('Count')
+        if multihost:
+                html += th.format('Hosts')
+        html += th.format('Tests') + th.format('Fail Rate') +\
+                th.format('First Instance') + '</tr>\n'
 
         num = 0
         for e in sorted(issues, key=lambda v:v['count'], reverse=True):
+                testtotal = 0
                 links = []
                 for host in sorted(e['urls']):
-                        links.append(tdlink.format(host, e['urls'][host]))
+                        links.append(tdlink.format(host, e['urls'][host][0]))
+                        testtotal += len(e['urls'][host])
+                rate = '%d/%d (%.2f%%)' % (testtotal, total, 100*float(testtotal)/float(total))
                 # row classes - alternate row color
                 rcls = ['alt'] if num % 2 == 1 else []
                 html += '<tr class="'+(' '.join(rcls))+'">\n' if len(rcls) > 0 else '<tr>\n'
-                html += td.format('center', e['count'])         # count
                 html += td.format('left', e['line'])            # issue
-                html += td.format('center', len(e['urls']))     # hosts
+                html += td.format('center', e['count'])         # count
+                if multihost:
+                        html += td.format('center', len(e['urls']))     # hosts
+                html += td.format('center', testtotal)          # test count
+                html += td.format('center', rate)                       # test rate
                 html += td.format('center nowrap', '<br>'.join(links))  # links
                 html += '</tr>\n'
                 num += 1
 
         # flush the data to file
         hf = open(htmlfile, 'w')
-        hf.write(html+'</table>\n</body>\n</html>\n')
+        hf.write(html+'</table>\n'+extra+'</body>\n</html>\n')
         hf.close()
         return issues
 
@@ -4195,7 +4321,7 @@ def createHTML(testruns, testfail):
                         for word in phase.split('_'):
                                 id += word[0]
                         order = '%.2f' % ((p['order'] * pdelta) + pmargin)
-                        name = string.replace(phase, '_', ' &nbsp;')
+                        name = phase.replace('_', ' &nbsp;')
                         devtl.html += devtl.html_legend.format(order, p['color'], name, id)
                 devtl.html += '</div>\n'
 
@@ -4784,6 +4910,7 @@ def setRuntimeSuspend(before=True):
 def executeSuspend():
         pm = ProcessMonitor()
         tp = sysvals.tpath
+        wifi = sysvals.checkWifi()
         testdata = []
         battery = True if getBattery() else False
         # run these commands to prepare the system for suspend
@@ -4857,6 +4984,8 @@ def executeSuspend():
                                 else:
                                         tdata['error'] = turbo
                         else:
+                                if sysvals.haveTurbostat():
+                                        sysvals.vprint('WARNING: ignoring turbostat in mode "%s"' % mode)
                                 pf = open(sysvals.powerfile, 'w')
                                 pf.write(mode)
                                 # execution will pause here
@@ -4883,6 +5012,8 @@ def executeSuspend():
                 bat2 = getBattery() if battery else False
                 if battery and bat1 and bat2:
                         tdata['bat'] = (bat1, bat2)
+                if 'device' in wifi and 'ip' in wifi:
+                        tdata['wifi'] = (wifi, sysvals.checkWifi())
                 testdata.append(tdata)
         # stop ftrace
         if(sysvals.usecallgraph or sysvals.usetraceevents):
@@ -4989,7 +5120,7 @@ def deviceInfo(output=''):
                         ms2nice(power['runtime_active_time']), \
                         ms2nice(power['runtime_suspended_time']))
         for i in sorted(lines):
-                print lines[i]
+                print(lines[i])
         return res
 
 # Function: devProps
@@ -5122,12 +5253,12 @@ def getModes():
         modes = []
         if(os.path.exists(sysvals.powerfile)):
                 fp = open(sysvals.powerfile, 'r')
-                modes = string.split(fp.read())
+                modes = fp.read().split()
                 fp.close()
         if(os.path.exists(sysvals.mempowerfile)):
                 deep = False
                 fp = open(sysvals.mempowerfile, 'r')
-                for m in string.split(fp.read()):
+                for m in fp.read().split():
                         memmode = m.strip('[]')
                         if memmode == 'deep':
                                 deep = True
@@ -5138,7 +5269,7 @@ def getModes():
                         modes.remove('mem')
         if('disk' in modes and os.path.exists(sysvals.diskpowerfile)):
                 fp = open(sysvals.diskpowerfile, 'r')
-                for m in string.split(fp.read()):
+                for m in fp.read().split():
                         modes.append('disk-%s' % m.strip('[]'))
                 fp.close()
         return modes
@@ -5201,14 +5332,15 @@ def dmidecode(mempath, fatal=False):
                                 continue
 
         # read in the memory for scanning
-        fp = open(mempath, 'rb')
         try:
+                fp = open(mempath, 'rb')
                 fp.seek(memaddr)
                 buf = fp.read(memsize)
         except:
                 if(fatal):
                         doError('DMI table is unreachable, sorry')
                 else:
+                        pprint('WARNING: /dev/mem is not readable, ignoring DMI data')
                         return out
         fp.close()
 
@@ -5231,14 +5363,15 @@ def dmidecode(mempath, fatal=False):
                         return out
 
         # read in the SM or DMI table
-        fp = open(mempath, 'rb')
         try:
+                fp = open(mempath, 'rb')
                 fp.seek(base)
                 buf = fp.read(length)
         except:
                 if(fatal):
                         doError('DMI table is unreachable, sorry')
                 else:
+                        pprint('WARNING: /dev/mem is not readable, ignoring DMI data')
                         return out
         fp.close()
 
@@ -5382,7 +5515,11 @@ def getFPDT(output):
         i = 0
         fwData = [0, 0]
         records = buf[36:]
-        fp = open(sysvals.mempath, 'rb')
+        try:
+                fp = open(sysvals.mempath, 'rb')
+        except:
+                pprint('WARNING: /dev/mem is not readable, ignoring the FPDT data')
+                return False
         while(i < len(records)):
                 header = struct.unpack('HBB', records[i:i+4])
                 if(header[0] not in rectype):
@@ -5499,13 +5636,14 @@ def statusCheck(probecheck=False):
         pprint('    is ftrace supported: %s' % res)
 
         # check if kprobes are available
-        res = sysvals.colorText('NO')
-        sysvals.usekprobes = sysvals.verifyKprobes()
-        if(sysvals.usekprobes):
-                res = 'YES'
-        else:
-                sysvals.usedevsrc = False
-        pprint('    are kprobes supported: %s' % res)
+        if sysvals.usekprobes:
+                res = sysvals.colorText('NO')
+                sysvals.usekprobes = sysvals.verifyKprobes()
+                if(sysvals.usekprobes):
+                        res = 'YES'
+                else:
+                        sysvals.usedevsrc = False
+                pprint('    are kprobes supported: %s' % res)
 
         # what data source are we using
         res = 'DMESG'
@@ -5593,6 +5731,8 @@ def getArgFloat(name, args, min, max, main=True):
 
 def processData(live=False):
         pprint('PROCESSING DATA')
+        sysvals.vprint('usetraceevents=%s, usetracemarkers=%s, usekprobes=%s' % \
+                (sysvals.usetraceevents, sysvals.usetracemarkers, sysvals.usekprobes))
         error = ''
         if(sysvals.usetraceevents):
                 testruns, error = parseTraceLog(live)
@@ -5605,6 +5745,11 @@ def processData(live=False):
                         parseKernelLog(data)
                 if(sysvals.ftracefile and (sysvals.usecallgraph or sysvals.usetraceevents)):
                         appendIncompleteTraceLog(testruns)
+        sysvals.vprint('System Info:')
+        for key in sorted(sysvals.stamp):
+                sysvals.vprint('    %-8s : %s' % (key.upper(), sysvals.stamp[key]))
+        if sysvals.kparams:
+                sysvals.vprint('Kparams:\n    %s' % sysvals.kparams)
         sysvals.vprint('Command:\n    %s' % sysvals.cmdline)
         for data in testruns:
                 if data.mcelog:
@@ -5612,12 +5757,24 @@ def processData(live=False):
                         for line in data.mcelog.split('\n'):
                                 sysvals.vprint('    %s' % line)
                 if data.turbostat:
-                        sysvals.vprint('Turbostat:\n    %s' % data.turbostat.replace('|', ' '))
+                        idx, s = 0, 'Turbostat:\n    '
+                        for val in data.turbostat.split('|'):
+                                idx += len(val) + 1
+                                if idx >= 80:
+                                        idx = 0
+                                        s += '\n    '
+                                s += val + ' '
+                        sysvals.vprint(s)
                 if data.battery:
                         a1, c1, a2, c2 = data.battery
                         s = 'Battery:\n    Before - AC: %s, Charge: %d\n     After - AC: %s, Charge: %d' % \
                                 (a1, int(c1), a2, int(c2))
                         sysvals.vprint(s)
+                if data.wifi:
+                        w = data.wifi.replace('|', ' ').split(',')
+                        s = 'Wifi:\n    Before %s\n     After %s' % \
+                                (w[0], w[1])
+                        sysvals.vprint(s)
                 data.printDetails()
         if sysvals.cgdump:
                 for data in testruns:
@@ -5641,13 +5798,13 @@ def processData(live=False):
 # Function: rerunTest
 # Description:
 #        generate an output from an existing set of ftrace/dmesg logs
-def rerunTest():
+def rerunTest(htmlfile=''):
         if sysvals.ftracefile:
                 doesTraceLogHaveTraceEvents()
         if not sysvals.dmesgfile and not sysvals.usetraceevents:
                 doError('recreating this html output requires a dmesg file')
-        if sysvals.outdir:
-                sysvals.htmlfile = sysvals.outdir
+        if htmlfile:
+                sysvals.htmlfile = htmlfile
         else:
                 sysvals.setOutputFile()
         if os.path.exists(sysvals.htmlfile):
@@ -5673,7 +5830,7 @@ def runTest(n=0):
         if sysvals.skiphtml:
                 sysvals.sudoUserchown(sysvals.testdir)
                 return
-        if len(testdata) > 0 and not testdata[0]['error']:
+        if not testdata[0]['error']:
                 testruns, stamp = processData(True)
                 for data in testruns:
                         del data
@@ -5709,15 +5866,13 @@ def find_in_html(html, start, end, firstonly=True):
                 return ''
         return out
 
-def data_from_html(file, outpath, issues):
-        if '<html>' not in file:
-                html = open(file, 'r').read()
-                sysvals.htmlfile = os.path.relpath(file, outpath)
-        else:
-                html = file
+def data_from_html(file, outpath, issues, fulldetail=False):
+        html = open(file, 'r').read()
+        sysvals.htmlfile = os.path.relpath(file, outpath)
         # extract general info
         suspend = find_in_html(html, 'Kernel Suspend', 'ms')
         resume = find_in_html(html, 'Kernel Resume', 'ms')
+        sysinfo = find_in_html(html, '<div class="stamp sysinfo">', '</div>')
         line = find_in_html(html, '<div class="stamp">', '</div>')
         stmp = line.split()
         if not suspend or not resume or len(stmp) != 8:
@@ -5739,13 +5894,18 @@ def data_from_html(file, outpath, issues):
                 result = 'pass'
         # extract error info
         ilist = []
+        extra = dict()
         log = find_in_html(html, '<div id="dmesglog" style="display:none;">',
                 '</div>').strip()
         if log:
                 d = Data(0)
                 d.end = 999999999
                 d.dmesgtext = log.split('\n')
-                d.extractErrorInfo(issues)
+                msglist = d.extractErrorInfo()
+                for msg in msglist:
+                        sysvals.errorSummary(issues, msg)
+                if stmp[2] == 'freeze':
+                        extra = d.turbostatInfo()
                 elist = dict()
                 for dir in d.errorinfo:
                         for err in d.errorinfo[dir]:
@@ -5761,14 +5921,15 @@ def data_from_html(file, outpath, issues):
                 if len(match) > 0:
                         match[0]['count'] += 1
                         if sysvals.hostname not in match[0]['urls']:
-                                match[0]['urls'][sysvals.hostname] = sysvals.htmlfile
+                                match[0]['urls'][sysvals.hostname] = [sysvals.htmlfile]
+                        elif sysvals.htmlfile not in match[0]['urls'][sysvals.hostname]:
+                                match[0]['urls'][sysvals.hostname].append(sysvals.htmlfile)
                 else:
                         issues.append({
                                 'match': issue, 'count': 1, 'line': issue,
-                                'urls': {sysvals.hostname: sysvals.htmlfile},
+                                'urls': {sysvals.hostname: [sysvals.htmlfile]},
                         })
                 ilist.append(issue)
-
         # extract device info
         devices = dict()
         for line in html.split('\n'):
@@ -5804,6 +5965,7 @@ def data_from_html(file, outpath, issues):
                 'mode': stmp[2],
                 'host': stmp[0],
                 'kernel': stmp[1],
+                'sysinfo': sysinfo,
                 'time': tstr,
                 'result': result,
                 'issues': ' '.join(ilist),
@@ -5816,8 +5978,28 @@ def data_from_html(file, outpath, issues):
                 'res_worsttime': worst['resume']['time'],
                 'url': sysvals.htmlfile,
         }
+        for key in extra:
+                data[key] = extra[key]
+        if fulldetail:
+                data['funclist'] = find_in_html(html, '<div title="', '" class="traceevent"', False)
         return data
 
+def genHtml(subdir):
+        for dirname, dirnames, filenames in os.walk(subdir):
+                sysvals.dmesgfile = sysvals.ftracefile = sysvals.htmlfile = ''
+                for filename in filenames:
+                        if(re.match('.*_dmesg.txt', filename)):
+                                sysvals.dmesgfile = os.path.join(dirname, filename)
+                        elif(re.match('.*_ftrace.txt', filename)):
+                                sysvals.ftracefile = os.path.join(dirname, filename)
+                sysvals.setOutputFile()
+                if sysvals.ftracefile and sysvals.htmlfile and \
+                        not os.path.exists(sysvals.htmlfile):
+                        pprint('FTRACE: %s' % sysvals.ftracefile)
+                        if sysvals.dmesgfile:
+                                pprint('DMESG : %s' % sysvals.dmesgfile)
+                        rerunTest()
+
 # Function: runSummary
 # Description:
 #        create a summary of tests in a sub-directory
@@ -5826,20 +6008,7 @@ def runSummary(subdir, local=True, genhtml=False):
         outpath = os.path.abspath('.') if local else inpath
         pprint('Generating a summary of folder:\n   %s' % inpath)
         if genhtml:
-                for dirname, dirnames, filenames in os.walk(subdir):
-                        sysvals.dmesgfile = sysvals.ftracefile = sysvals.htmlfile = ''
-                        for filename in filenames:
-                                if(re.match('.*_dmesg.txt', filename)):
-                                        sysvals.dmesgfile = os.path.join(dirname, filename)
-                                elif(re.match('.*_ftrace.txt', filename)):
-                                        sysvals.ftracefile = os.path.join(dirname, filename)
-                        sysvals.setOutputFile()
-                        if sysvals.ftracefile and sysvals.htmlfile and \
-                                not os.path.exists(sysvals.htmlfile):
-                                pprint('FTRACE: %s' % sysvals.ftracefile)
-                                if sysvals.dmesgfile:
-                                        pprint('DMESG : %s' % sysvals.dmesgfile)
-                                rerunTest()
+                genHtml(subdir)
         issues = []
         testruns = []
         desc = {'host':[],'mode':[],'kernel':[]}
@@ -5863,7 +6032,7 @@ def runSummary(subdir, local=True, genhtml=False):
         pprint('   summary.html         - tabular list of test data found')
         createHTMLDeviceSummary(testruns, os.path.join(outpath, 'summary-devices.html'), title)
         pprint('   summary-devices.html - kernel device list sorted by total execution time')
-        createHTMLIssuesSummary(issues, os.path.join(outpath, 'summary-issues.html'), title)
+        createHTMLIssuesSummary(testruns, issues, os.path.join(outpath, 'summary-issues.html'), title)
         pprint('   summary-issues.html  - kernel issues found sorted by frequency')
 
 # Function: checkArgBool
@@ -6109,6 +6278,7 @@ def printHelp():
         '                default: suspend-{date}-{time}\n'\
         '   -rtcwake t   Wakeup t seconds after suspend, set t to "off" to disable (default: 15)\n'\
         '   -addlogs     Add the dmesg and ftrace logs to the html output\n'\
+        '   -turbostat   Use turbostat to execute the command in freeze mode (default: disabled)\n'\
         '   -srgap       Add a visible gap in the timeline between sus/res (default: disabled)\n'\
         '   -skiphtml    Run the test and capture the trace logs, but skip the timeline (default: disabled)\n'\
         '   -result fn   Export a results table to a text file for parsing.\n'\
@@ -6130,6 +6300,7 @@ def printHelp():
         '                be created in a new subdirectory with a summary page.\n'\
         '  [debug]\n'\
         '   -f           Use ftrace to create device callgraphs (default: disabled)\n'\
+        '   -ftop        Use ftrace on the top level call: "%s" (default: disabled)\n'\
         '   -maxdepth N  limit the callgraph data to N call levels (default: 0=all)\n'\
         '   -expandcg    pre-expand the callgraph data in the html output (default: disabled)\n'\
         '   -fadd file   Add functions to be graphed in the timeline from a list in a text file\n'\
@@ -6149,6 +6320,7 @@ def printHelp():
         '   -status      Test to see if the system is enabled to run this tool\n'\
         '   -fpdt        Print out the contents of the ACPI Firmware Performance Data Table\n'\
         '   -battery     Print out battery info (if available)\n'\
+        '   -wifi        Print out wifi connection info (if wireless-tools and device exists)\n'\
         '   -x<mode>     Test xset by toggling the given mode (on/off/standby/suspend)\n'\
         '   -sysinfo     Print out system info extracted from BIOS\n'\
         '   -devinfo     Print out the pm settings of all devices which support runtime suspend\n'\
@@ -6158,7 +6330,7 @@ def printHelp():
         '  [redo]\n'\
         '   -ftrace ftracefile  Create HTML output using ftrace input (used with -dmesg)\n'\
         '   -dmesg dmesgfile    Create HTML output using dmesg (used with -ftrace)\n'\
-        '' % (sysvals.title, sysvals.version, sysvals.suspendmode))
+        '' % (sysvals.title, sysvals.version, sysvals.suspendmode, sysvals.ftopfunc))
         return True
 
 # ----------------- MAIN --------------------
@@ -6168,7 +6340,7 @@ if __name__ == '__main__':
         cmd = ''
         simplecmds = ['-sysinfo', '-modes', '-fpdt', '-flist', '-flistall',
                 '-devinfo', '-status', '-battery', '-xon', '-xoff', '-xstandby',
-                '-xsuspend', '-xinit', '-xreset', '-xstat']
+                '-xsuspend', '-xinit', '-xreset', '-xstat', '-wifi']
         if '-f' in sys.argv:
                 sysvals.cgskip = sysvals.configFile('cgskip.txt')
         # loop through the command line arguments
@@ -6200,6 +6372,10 @@ if __name__ == '__main__':
                         sysvals.postdelay = getArgInt('-postdelay', args, 0, 60000)
                 elif(arg == '-f'):
                         sysvals.usecallgraph = True
+                elif(arg == '-ftop'):
+                        sysvals.usecallgraph = True
+                        sysvals.ftop = True
+                        sysvals.usekprobes = False
                 elif(arg == '-skiphtml'):
                         sysvals.skiphtml = True
                 elif(arg == '-cgdump'):
@@ -6210,10 +6386,16 @@ if __name__ == '__main__':
                         genhtml = True
                 elif(arg == '-addlogs'):
                         sysvals.dmesglog = sysvals.ftracelog = True
+                elif(arg == '-nologs'):
+                        sysvals.dmesglog = sysvals.ftracelog = False
                 elif(arg == '-addlogdmesg'):
                         sysvals.dmesglog = True
                 elif(arg == '-addlogftrace'):
                         sysvals.ftracelog = True
+                elif(arg == '-turbostat'):
+                        sysvals.tstat = True
+                        if not sysvals.haveTurbostat():
+                                doError('Turbostat command not found')
                 elif(arg == '-verbose'):
                         sysvals.verbose = True
                 elif(arg == '-proc'):
@@ -6283,6 +6465,12 @@ if __name__ == '__main__':
                         except:
                                 doError('No callgraph functions supplied', True)
                         sysvals.setCallgraphFilter(val)
+                elif(arg == '-skipkprobe'):
+                        try:
+                                val = args.next()
+                        except:
+                                doError('No kprobe functions supplied', True)
+                        sysvals.skipKprobes(val)
                 elif(arg == '-cgskip'):
                         try:
                                 val = args.next()
@@ -6421,7 +6609,7 @@ if __name__ == '__main__':
                 elif(cmd == 'devinfo'):
                         deviceInfo()
                 elif(cmd == 'modes'):
-                        print getModes()
+                        pprint(getModes())
                 elif(cmd == 'flist'):
                         sysvals.getFtraceFilterFunctions(True)
                 elif(cmd == 'flistall'):
@@ -6433,11 +6621,18 @@ if __name__ == '__main__':
                         ret = displayControl(cmd[1:])
                 elif(cmd == 'xstat'):
                         pprint('Display Status: %s' % displayControl('stat').upper())
+                elif(cmd == 'wifi'):
+                        out = sysvals.checkWifi()
+                        if 'device' not in out:
+                                pprint('WIFI interface not found')
+                        else:
+                                for key in sorted(out):
+                                        pprint('%6s: %s' % (key.upper(), out[key]))
                 sys.exit(ret)
 
         # if instructed, re-analyze existing data files
         if(sysvals.notestrun):
-                stamp = rerunTest()
+                stamp = rerunTest(sysvals.outdir)
                 sysvals.outputResult(stamp)
                 sys.exit(0)
 
@@ -6474,7 +6669,7 @@ if __name__ == '__main__':
                         s = 'suspend-x%d' % sysvals.multitest['count']
                         sysvals.outdir = datetime.now().strftime(s+'-%y%m%d-%H%M%S')
                 if not os.path.isdir(sysvals.outdir):
-                        os.mkdir(sysvals.outdir)
+                        os.makedirs(sysvals.outdir)
                 for i in range(sysvals.multitest['count']):
                         if(i != 0):
                                 pprint('Waiting %d seconds...' % (sysvals.multitest['delay']))