aboutsummaryrefslogtreecommitdiffstats
path: root/tools/perf/bench
diff options
context:
space:
mode:
authorIngo Molnar <mingo@kernel.org>2012-12-06 07:51:59 -0500
committerArnaldo Carvalho de Melo <acme@redhat.com>2013-01-30 08:35:36 -0500
commit1c13f3c9042f9d222959af7c9da6db93ea9f7e4c (patch)
tree43fb380a1aa8acca2042f7ecf9bedc273bf5bf8b /tools/perf/bench
parent7e010562e01aff929126f671ff9e730e22dbdb1b (diff)
perf: Add 'perf bench numa mem' NUMA performance measurement suite
Add a suite of NUMA performance benchmarks. The goal was simulate the behavior and access patterns of real NUMA workloads, via a wide range of parameters, so this tool goes well beyond simple bzero() measurements that most NUMA micro-benchmarks use: - It processes the data and creates a chain of data dependencies, like a real workload would. Neither the compiler, nor the kernel (via KSM and other optimizations) nor the CPU can eliminate parts of the workload. - It randomizes the initial state and also randomizes the target addresses of the processing - it's not a simple forward scan of addresses. - It provides flexible options to set process, thread and memory relationship information: -G sets "global" memory shared between all test processes, -P sets "process" memory shared by all threads of a process and -T sets "thread" private memory. - There's a NUMA convergence monitoring and convergence latency measurement option via -c and -m. - Micro-sleeps and synchronization can be injected to provoke lock contention and scheduling, via the -u and -S options. This simulates IO and contention. - The -x option instructs the workload to 'perturb' itself artificially every N seconds, by moving to the first and last CPU of the system periodically. This way the stability of convergence equilibrium and the number of steps taken for the scheduler to reach equilibrium again can be measured. - The amount of work can be specified via the -l loop count, and/or via a -s seconds-timeout value. - CPU and node memory binding options, to test hard binding scenarios. THP can be turned on and off via madvise() calls. - Live reporting of convergence progress in an 'at glance' output format. Printing of convergence and deconvergence events. The 'perf bench numa mem -a' option will start an array of about 30 individual tests that will each output such measurements: # Running 5x5-bw-thread, "perf bench numa mem -p 5 -t 5 -P 512 -s 20 -zZ0q --thp 1" 5x5-bw-thread, 20.276, secs, runtime-max/thread 5x5-bw-thread, 20.004, secs, runtime-min/thread 5x5-bw-thread, 20.155, secs, runtime-avg/thread 5x5-bw-thread, 0.671, %, spread-runtime/thread 5x5-bw-thread, 21.153, GB, data/thread 5x5-bw-thread, 528.818, GB, data-total 5x5-bw-thread, 0.959, nsecs, runtime/byte/thread 5x5-bw-thread, 1.043, GB/sec, thread-speed 5x5-bw-thread, 26.081, GB/sec, total-speed See the help text and the code for more details. Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Hugh Dickins <hughd@google.com> Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'tools/perf/bench')
-rw-r--r--tools/perf/bench/bench.h1
-rw-r--r--tools/perf/bench/numa.c1731
2 files changed, 1732 insertions, 0 deletions
diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h
index 8f89998eeaf4..a5223e6a7b43 100644
--- a/tools/perf/bench/bench.h
+++ b/tools/perf/bench/bench.h
@@ -1,6 +1,7 @@
1#ifndef BENCH_H 1#ifndef BENCH_H
2#define BENCH_H 2#define BENCH_H
3 3
4extern int bench_numa(int argc, const char **argv, const char *prefix);
4extern int bench_sched_messaging(int argc, const char **argv, const char *prefix); 5extern int bench_sched_messaging(int argc, const char **argv, const char *prefix);
5extern int bench_sched_pipe(int argc, const char **argv, const char *prefix); 6extern int bench_sched_pipe(int argc, const char **argv, const char *prefix);
6extern int bench_mem_memcpy(int argc, const char **argv, 7extern int bench_mem_memcpy(int argc, const char **argv,
diff --git a/tools/perf/bench/numa.c b/tools/perf/bench/numa.c
new file mode 100644
index 000000000000..30d1c3225b46
--- /dev/null
+++ b/tools/perf/bench/numa.c
@@ -0,0 +1,1731 @@
1/*
2 * numa.c
3 *
4 * numa: Simulate NUMA-sensitive workload and measure their NUMA performance
5 */
6
7#include "../perf.h"
8#include "../builtin.h"
9#include "../util/util.h"
10#include "../util/parse-options.h"
11
12#include "bench.h"
13
14#include <errno.h>
15#include <sched.h>
16#include <stdio.h>
17#include <assert.h>
18#include <malloc.h>
19#include <signal.h>
20#include <stdlib.h>
21#include <string.h>
22#include <unistd.h>
23#include <pthread.h>
24#include <sys/mman.h>
25#include <sys/time.h>
26#include <sys/wait.h>
27#include <sys/prctl.h>
28#include <sys/types.h>
29
30#include <numa.h>
31#include <numaif.h>
32
33/*
34 * Regular printout to the terminal, supressed if -q is specified:
35 */
36#define tprintf(x...) do { if (g && g->p.show_details >= 0) printf(x); } while (0)
37
38/*
39 * Debug printf:
40 */
41#define dprintf(x...) do { if (g && g->p.show_details >= 1) printf(x); } while (0)
42
43struct thread_data {
44 int curr_cpu;
45 cpu_set_t bind_cpumask;
46 int bind_node;
47 u8 *process_data;
48 int process_nr;
49 int thread_nr;
50 int task_nr;
51 unsigned int loops_done;
52 u64 val;
53 u64 runtime_ns;
54 pthread_mutex_t *process_lock;
55};
56
57/* Parameters set by options: */
58
59struct params {
60 /* Startup synchronization: */
61 bool serialize_startup;
62
63 /* Task hierarchy: */
64 int nr_proc;
65 int nr_threads;
66
67 /* Working set sizes: */
68 const char *mb_global_str;
69 const char *mb_proc_str;
70 const char *mb_proc_locked_str;
71 const char *mb_thread_str;
72
73 double mb_global;
74 double mb_proc;
75 double mb_proc_locked;
76 double mb_thread;
77
78 /* Access patterns to the working set: */
79 bool data_reads;
80 bool data_writes;
81 bool data_backwards;
82 bool data_zero_memset;
83 bool data_rand_walk;
84 u32 nr_loops;
85 u32 nr_secs;
86 u32 sleep_usecs;
87
88 /* Working set initialization: */
89 bool init_zero;
90 bool init_random;
91 bool init_cpu0;
92
93 /* Misc options: */
94 int show_details;
95 int run_all;
96 int thp;
97
98 long bytes_global;
99 long bytes_process;
100 long bytes_process_locked;
101 long bytes_thread;
102
103 int nr_tasks;
104 bool show_quiet;
105
106 bool show_convergence;
107 bool measure_convergence;
108
109 int perturb_secs;
110 int nr_cpus;
111 int nr_nodes;
112
113 /* Affinity options -C and -N: */
114 char *cpu_list_str;
115 char *node_list_str;
116};
117
118
119/* Global, read-writable area, accessible to all processes and threads: */
120
121struct global_info {
122 u8 *data;
123
124 pthread_mutex_t startup_mutex;
125 int nr_tasks_started;
126
127 pthread_mutex_t startup_done_mutex;
128
129 pthread_mutex_t start_work_mutex;
130 int nr_tasks_working;
131
132 pthread_mutex_t stop_work_mutex;
133 u64 bytes_done;
134
135 struct thread_data *threads;
136
137 /* Convergence latency measurement: */
138 bool all_converged;
139 bool stop_work;
140
141 int print_once;
142
143 struct params p;
144};
145
146static struct global_info *g = NULL;
147
148static int parse_cpus_opt(const struct option *opt, const char *arg, int unset);
149static int parse_nodes_opt(const struct option *opt, const char *arg, int unset);
150
151struct params p0;
152
153static const struct option options[] = {
154 OPT_INTEGER('p', "nr_proc" , &p0.nr_proc, "number of processes"),
155 OPT_INTEGER('t', "nr_threads" , &p0.nr_threads, "number of threads per process"),
156
157 OPT_STRING('G', "mb_global" , &p0.mb_global_str, "MB", "global memory (MBs)"),
158 OPT_STRING('P', "mb_proc" , &p0.mb_proc_str, "MB", "process memory (MBs)"),
159 OPT_STRING('L', "mb_proc_locked", &p0.mb_proc_locked_str,"MB", "process serialized/locked memory access (MBs), <= process_memory"),
160 OPT_STRING('T', "mb_thread" , &p0.mb_thread_str, "MB", "thread memory (MBs)"),
161
162 OPT_UINTEGER('l', "nr_loops" , &p0.nr_loops, "max number of loops to run"),
163 OPT_UINTEGER('s', "nr_secs" , &p0.nr_secs, "max number of seconds to run"),
164 OPT_UINTEGER('u', "usleep" , &p0.sleep_usecs, "usecs to sleep per loop iteration"),
165
166 OPT_BOOLEAN('R', "data_reads" , &p0.data_reads, "access the data via writes (can be mixed with -W)"),
167 OPT_BOOLEAN('W', "data_writes" , &p0.data_writes, "access the data via writes (can be mixed with -R)"),
168 OPT_BOOLEAN('B', "data_backwards", &p0.data_backwards, "access the data backwards as well"),
169 OPT_BOOLEAN('Z', "data_zero_memset", &p0.data_zero_memset,"access the data via glibc bzero only"),
170 OPT_BOOLEAN('r', "data_rand_walk", &p0.data_rand_walk, "access the data with random (32bit LFSR) walk"),
171
172
173 OPT_BOOLEAN('z', "init_zero" , &p0.init_zero, "bzero the initial allocations"),
174 OPT_BOOLEAN('I', "init_random" , &p0.init_random, "randomize the contents of the initial allocations"),
175 OPT_BOOLEAN('0', "init_cpu0" , &p0.init_cpu0, "do the initial allocations on CPU#0"),
176 OPT_INTEGER('x', "perturb_secs", &p0.perturb_secs, "perturb thread 0/0 every X secs, to test convergence stability"),
177
178 OPT_INCR ('d', "show_details" , &p0.show_details, "Show details"),
179 OPT_INCR ('a', "all" , &p0.run_all, "Run all tests in the suite"),
180 OPT_INTEGER('H', "thp" , &p0.thp, "MADV_NOHUGEPAGE < 0 < MADV_HUGEPAGE"),
181 OPT_BOOLEAN('c', "show_convergence", &p0.show_convergence, "show convergence details"),
182 OPT_BOOLEAN('m', "measure_convergence", &p0.measure_convergence, "measure convergence latency"),
183 OPT_BOOLEAN('q', "quiet" , &p0.show_quiet, "bzero the initial allocations"),
184 OPT_BOOLEAN('S', "serialize-startup", &p0.serialize_startup,"serialize thread startup"),
185
186 /* Special option string parsing callbacks: */
187 OPT_CALLBACK('C', "cpus", NULL, "cpu[,cpu2,...cpuN]",
188 "bind the first N tasks to these specific cpus (the rest is unbound)",
189 parse_cpus_opt),
190 OPT_CALLBACK('M', "memnodes", NULL, "node[,node2,...nodeN]",
191 "bind the first N tasks to these specific memory nodes (the rest is unbound)",
192 parse_nodes_opt),
193 OPT_END()
194};
195
196static const char * const bench_numa_usage[] = {
197 "perf bench numa <options>",
198 NULL
199};
200
201static const char * const numa_usage[] = {
202 "perf bench numa mem [<options>]",
203 NULL
204};
205
206static cpu_set_t bind_to_cpu(int target_cpu)
207{
208 cpu_set_t orig_mask, mask;
209 int ret;
210
211 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask);
212 BUG_ON(ret);
213
214 CPU_ZERO(&mask);
215
216 if (target_cpu == -1) {
217 int cpu;
218
219 for (cpu = 0; cpu < g->p.nr_cpus; cpu++)
220 CPU_SET(cpu, &mask);
221 } else {
222 BUG_ON(target_cpu < 0 || target_cpu >= g->p.nr_cpus);
223 CPU_SET(target_cpu, &mask);
224 }
225
226 ret = sched_setaffinity(0, sizeof(mask), &mask);
227 BUG_ON(ret);
228
229 return orig_mask;
230}
231
232static cpu_set_t bind_to_node(int target_node)
233{
234 int cpus_per_node = g->p.nr_cpus/g->p.nr_nodes;
235 cpu_set_t orig_mask, mask;
236 int cpu;
237 int ret;
238
239 BUG_ON(cpus_per_node*g->p.nr_nodes != g->p.nr_cpus);
240 BUG_ON(!cpus_per_node);
241
242 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask);
243 BUG_ON(ret);
244
245 CPU_ZERO(&mask);
246
247 if (target_node == -1) {
248 for (cpu = 0; cpu < g->p.nr_cpus; cpu++)
249 CPU_SET(cpu, &mask);
250 } else {
251 int cpu_start = (target_node + 0) * cpus_per_node;
252 int cpu_stop = (target_node + 1) * cpus_per_node;
253
254 BUG_ON(cpu_stop > g->p.nr_cpus);
255
256 for (cpu = cpu_start; cpu < cpu_stop; cpu++)
257 CPU_SET(cpu, &mask);
258 }
259
260 ret = sched_setaffinity(0, sizeof(mask), &mask);
261 BUG_ON(ret);
262
263 return orig_mask;
264}
265
266static void bind_to_cpumask(cpu_set_t mask)
267{
268 int ret;
269
270 ret = sched_setaffinity(0, sizeof(mask), &mask);
271 BUG_ON(ret);
272}
273
274static void mempol_restore(void)
275{
276 int ret;
277
278 ret = set_mempolicy(MPOL_DEFAULT, NULL, g->p.nr_nodes-1);
279
280 BUG_ON(ret);
281}
282
283static void bind_to_memnode(int node)
284{
285 unsigned long nodemask;
286 int ret;
287
288 if (node == -1)
289 return;
290
291 BUG_ON(g->p.nr_nodes > (int)sizeof(nodemask));
292 nodemask = 1L << node;
293
294 ret = set_mempolicy(MPOL_BIND, &nodemask, sizeof(nodemask)*8);
295 dprintf("binding to node %d, mask: %016lx => %d\n", node, nodemask, ret);
296
297 BUG_ON(ret);
298}
299
300#define HPSIZE (2*1024*1024)
301
302#define set_taskname(fmt...) \
303do { \
304 char name[20]; \
305 \
306 snprintf(name, 20, fmt); \
307 prctl(PR_SET_NAME, name); \
308} while (0)
309
310static u8 *alloc_data(ssize_t bytes0, int map_flags,
311 int init_zero, int init_cpu0, int thp, int init_random)
312{
313 cpu_set_t orig_mask;
314 ssize_t bytes;
315 u8 *buf;
316 int ret;
317
318 if (!bytes0)
319 return NULL;
320
321 /* Allocate and initialize all memory on CPU#0: */
322 if (init_cpu0) {
323 orig_mask = bind_to_node(0);
324 bind_to_memnode(0);
325 }
326
327 bytes = bytes0 + HPSIZE;
328
329 buf = (void *)mmap(0, bytes, PROT_READ|PROT_WRITE, MAP_ANON|map_flags, -1, 0);
330 BUG_ON(buf == (void *)-1);
331
332 if (map_flags == MAP_PRIVATE) {
333 if (thp > 0) {
334 ret = madvise(buf, bytes, MADV_HUGEPAGE);
335 if (ret && !g->print_once) {
336 g->print_once = 1;
337 printf("WARNING: Could not enable THP - do: 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'\n");
338 }
339 }
340 if (thp < 0) {
341 ret = madvise(buf, bytes, MADV_NOHUGEPAGE);
342 if (ret && !g->print_once) {
343 g->print_once = 1;
344 printf("WARNING: Could not disable THP: run a CONFIG_TRANSPARENT_HUGEPAGE kernel?\n");
345 }
346 }
347 }
348
349 if (init_zero) {
350 bzero(buf, bytes);
351 } else {
352 /* Initialize random contents, different in each word: */
353 if (init_random) {
354 u64 *wbuf = (void *)buf;
355 long off = rand();
356 long i;
357
358 for (i = 0; i < bytes/8; i++)
359 wbuf[i] = i + off;
360 }
361 }
362
363 /* Align to 2MB boundary: */
364 buf = (void *)(((unsigned long)buf + HPSIZE-1) & ~(HPSIZE-1));
365
366 /* Restore affinity: */
367 if (init_cpu0) {
368 bind_to_cpumask(orig_mask);
369 mempol_restore();
370 }
371
372 return buf;
373}
374
375static void free_data(void *data, ssize_t bytes)
376{
377 int ret;
378
379 if (!data)
380 return;
381
382 ret = munmap(data, bytes);
383 BUG_ON(ret);
384}
385
386/*
387 * Create a shared memory buffer that can be shared between processes, zeroed:
388 */
389static void * zalloc_shared_data(ssize_t bytes)
390{
391 return alloc_data(bytes, MAP_SHARED, 1, g->p.init_cpu0, g->p.thp, g->p.init_random);
392}
393
394/*
395 * Create a shared memory buffer that can be shared between processes:
396 */
397static void * setup_shared_data(ssize_t bytes)
398{
399 return alloc_data(bytes, MAP_SHARED, 0, g->p.init_cpu0, g->p.thp, g->p.init_random);
400}
401
402/*
403 * Allocate process-local memory - this will either be shared between
404 * threads of this process, or only be accessed by this thread:
405 */
406static void * setup_private_data(ssize_t bytes)
407{
408 return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0, g->p.thp, g->p.init_random);
409}
410
411/*
412 * Return a process-shared (global) mutex:
413 */
414static void init_global_mutex(pthread_mutex_t *mutex)
415{
416 pthread_mutexattr_t attr;
417
418 pthread_mutexattr_init(&attr);
419 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
420 pthread_mutex_init(mutex, &attr);
421}
422
423static int parse_cpu_list(const char *arg)
424{
425 p0.cpu_list_str = strdup(arg);
426
427 dprintf("got CPU list: {%s}\n", p0.cpu_list_str);
428
429 return 0;
430}
431
432static void parse_setup_cpu_list(void)
433{
434 struct thread_data *td;
435 char *str0, *str;
436 int t;
437
438 if (!g->p.cpu_list_str)
439 return;
440
441 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);
442
443 str0 = str = strdup(g->p.cpu_list_str);
444 t = 0;
445
446 BUG_ON(!str);
447
448 tprintf("# binding tasks to CPUs:\n");
449 tprintf("# ");
450
451 while (true) {
452 int bind_cpu, bind_cpu_0, bind_cpu_1;
453 char *tok, *tok_end, *tok_step, *tok_len, *tok_mul;
454 int bind_len;
455 int step;
456 int mul;
457
458 tok = strsep(&str, ",");
459 if (!tok)
460 break;
461
462 tok_end = strstr(tok, "-");
463
464 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end);
465 if (!tok_end) {
466 /* Single CPU specified: */
467 bind_cpu_0 = bind_cpu_1 = atol(tok);
468 } else {
469 /* CPU range specified (for example: "5-11"): */
470 bind_cpu_0 = atol(tok);
471 bind_cpu_1 = atol(tok_end + 1);
472 }
473
474 step = 1;
475 tok_step = strstr(tok, "#");
476 if (tok_step) {
477 step = atol(tok_step + 1);
478 BUG_ON(step <= 0 || step >= g->p.nr_cpus);
479 }
480
481 /*
482 * Mask length.
483 * Eg: "--cpus 8_4-16#4" means: '--cpus 8_4,12_4,16_4',
484 * where the _4 means the next 4 CPUs are allowed.
485 */
486 bind_len = 1;
487 tok_len = strstr(tok, "_");
488 if (tok_len) {
489 bind_len = atol(tok_len + 1);
490 BUG_ON(bind_len <= 0 || bind_len > g->p.nr_cpus);
491 }
492
493 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */
494 mul = 1;
495 tok_mul = strstr(tok, "x");
496 if (tok_mul) {
497 mul = atol(tok_mul + 1);
498 BUG_ON(mul <= 0);
499 }
500
501 dprintf("CPUs: %d_%d-%d#%dx%d\n", bind_cpu_0, bind_len, bind_cpu_1, step, mul);
502
503 BUG_ON(bind_cpu_0 < 0 || bind_cpu_0 >= g->p.nr_cpus);
504 BUG_ON(bind_cpu_1 < 0 || bind_cpu_1 >= g->p.nr_cpus);
505 BUG_ON(bind_cpu_0 > bind_cpu_1);
506
507 for (bind_cpu = bind_cpu_0; bind_cpu <= bind_cpu_1; bind_cpu += step) {
508 int i;
509
510 for (i = 0; i < mul; i++) {
511 int cpu;
512
513 if (t >= g->p.nr_tasks) {
514 printf("\n# NOTE: ignoring bind CPUs starting at CPU#%d\n #", bind_cpu);
515 goto out;
516 }
517 td = g->threads + t;
518
519 if (t)
520 tprintf(",");
521 if (bind_len > 1) {
522 tprintf("%2d/%d", bind_cpu, bind_len);
523 } else {
524 tprintf("%2d", bind_cpu);
525 }
526
527 CPU_ZERO(&td->bind_cpumask);
528 for (cpu = bind_cpu; cpu < bind_cpu+bind_len; cpu++) {
529 BUG_ON(cpu < 0 || cpu >= g->p.nr_cpus);
530 CPU_SET(cpu, &td->bind_cpumask);
531 }
532 t++;
533 }
534 }
535 }
536out:
537
538 tprintf("\n");
539
540 if (t < g->p.nr_tasks)
541 printf("# NOTE: %d tasks bound, %d tasks unbound\n", t, g->p.nr_tasks - t);
542
543 free(str0);
544}
545
546static int parse_cpus_opt(const struct option *opt __maybe_unused,
547 const char *arg, int unset __maybe_unused)
548{
549 if (!arg)
550 return -1;
551
552 return parse_cpu_list(arg);
553}
554
555static int parse_node_list(const char *arg)
556{
557 p0.node_list_str = strdup(arg);
558
559 dprintf("got NODE list: {%s}\n", p0.node_list_str);
560
561 return 0;
562}
563
564static void parse_setup_node_list(void)
565{
566 struct thread_data *td;
567 char *str0, *str;
568 int t;
569
570 if (!g->p.node_list_str)
571 return;
572
573 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);
574
575 str0 = str = strdup(g->p.node_list_str);
576 t = 0;
577
578 BUG_ON(!str);
579
580 tprintf("# binding tasks to NODEs:\n");
581 tprintf("# ");
582
583 while (true) {
584 int bind_node, bind_node_0, bind_node_1;
585 char *tok, *tok_end, *tok_step, *tok_mul;
586 int step;
587 int mul;
588
589 tok = strsep(&str, ",");
590 if (!tok)
591 break;
592
593 tok_end = strstr(tok, "-");
594
595 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end);
596 if (!tok_end) {
597 /* Single NODE specified: */
598 bind_node_0 = bind_node_1 = atol(tok);
599 } else {
600 /* NODE range specified (for example: "5-11"): */
601 bind_node_0 = atol(tok);
602 bind_node_1 = atol(tok_end + 1);
603 }
604
605 step = 1;
606 tok_step = strstr(tok, "#");
607 if (tok_step) {
608 step = atol(tok_step + 1);
609 BUG_ON(step <= 0 || step >= g->p.nr_nodes);
610 }
611
612 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */
613 mul = 1;
614 tok_mul = strstr(tok, "x");
615 if (tok_mul) {
616 mul = atol(tok_mul + 1);
617 BUG_ON(mul <= 0);
618 }
619
620 dprintf("NODEs: %d-%d #%d\n", bind_node_0, bind_node_1, step);
621
622 BUG_ON(bind_node_0 < 0 || bind_node_0 >= g->p.nr_nodes);
623 BUG_ON(bind_node_1 < 0 || bind_node_1 >= g->p.nr_nodes);
624 BUG_ON(bind_node_0 > bind_node_1);
625
626 for (bind_node = bind_node_0; bind_node <= bind_node_1; bind_node += step) {
627 int i;
628
629 for (i = 0; i < mul; i++) {
630 if (t >= g->p.nr_tasks) {
631 printf("\n# NOTE: ignoring bind NODEs starting at NODE#%d\n", bind_node);
632 goto out;
633 }
634 td = g->threads + t;
635
636 if (!t)
637 tprintf(" %2d", bind_node);
638 else
639 tprintf(",%2d", bind_node);
640
641 td->bind_node = bind_node;
642 t++;
643 }
644 }
645 }
646out:
647
648 tprintf("\n");
649
650 if (t < g->p.nr_tasks)
651 printf("# NOTE: %d tasks mem-bound, %d tasks unbound\n", t, g->p.nr_tasks - t);
652
653 free(str0);
654}
655
656static int parse_nodes_opt(const struct option *opt __maybe_unused,
657 const char *arg, int unset __maybe_unused)
658{
659 if (!arg)
660 return -1;
661
662 return parse_node_list(arg);
663
664 return 0;
665}
666
667#define BIT(x) (1ul << x)
668
669static inline uint32_t lfsr_32(uint32_t lfsr)
670{
671 const uint32_t taps = BIT(1) | BIT(5) | BIT(6) | BIT(31);
672 return (lfsr>>1) ^ ((0x0u - (lfsr & 0x1u)) & taps);
673}
674
675/*
676 * Make sure there's real data dependency to RAM (when read
677 * accesses are enabled), so the compiler, the CPU and the
678 * kernel (KSM, zero page, etc.) cannot optimize away RAM
679 * accesses:
680 */
681static inline u64 access_data(u64 *data __attribute__((unused)), u64 val)
682{
683 if (g->p.data_reads)
684 val += *data;
685 if (g->p.data_writes)
686 *data = val + 1;
687 return val;
688}
689
690/*
691 * The worker process does two types of work, a forwards going
692 * loop and a backwards going loop.
693 *
694 * We do this so that on multiprocessor systems we do not create
695 * a 'train' of processing, with highly synchronized processes,
696 * skewing the whole benchmark.
697 */
698static u64 do_work(u8 *__data, long bytes, int nr, int nr_max, int loop, u64 val)
699{
700 long words = bytes/sizeof(u64);
701 u64 *data = (void *)__data;
702 long chunk_0, chunk_1;
703 u64 *d0, *d, *d1;
704 long off;
705 long i;
706
707 BUG_ON(!data && words);
708 BUG_ON(data && !words);
709
710 if (!data)
711 return val;
712
713 /* Very simple memset() work variant: */
714 if (g->p.data_zero_memset && !g->p.data_rand_walk) {
715 bzero(data, bytes);
716 return val;
717 }
718
719 /* Spread out by PID/TID nr and by loop nr: */
720 chunk_0 = words/nr_max;
721 chunk_1 = words/g->p.nr_loops;
722 off = nr*chunk_0 + loop*chunk_1;
723
724 while (off >= words)
725 off -= words;
726
727 if (g->p.data_rand_walk) {
728 u32 lfsr = nr + loop + val;
729 int j;
730
731 for (i = 0; i < words/1024; i++) {
732 long start, end;
733
734 lfsr = lfsr_32(lfsr);
735
736 start = lfsr % words;
737 end = min(start + 1024, words-1);
738
739 if (g->p.data_zero_memset) {
740 bzero(data + start, (end-start) * sizeof(u64));
741 } else {
742 for (j = start; j < end; j++)
743 val = access_data(data + j, val);
744 }
745 }
746 } else if (!g->p.data_backwards || (nr + loop) & 1) {
747
748 d0 = data + off;
749 d = data + off + 1;
750 d1 = data + words;
751
752 /* Process data forwards: */
753 for (;;) {
754 if (unlikely(d >= d1))
755 d = data;
756 if (unlikely(d == d0))
757 break;
758
759 val = access_data(d, val);
760
761 d++;
762 }
763 } else {
764 /* Process data backwards: */
765
766 d0 = data + off;
767 d = data + off - 1;
768 d1 = data + words;
769
770 /* Process data forwards: */
771 for (;;) {
772 if (unlikely(d < data))
773 d = data + words-1;
774 if (unlikely(d == d0))
775 break;
776
777 val = access_data(d, val);
778
779 d--;
780 }
781 }
782
783 return val;
784}
785
786static void update_curr_cpu(int task_nr, unsigned long bytes_worked)
787{
788 unsigned int cpu;
789
790 cpu = sched_getcpu();
791
792 g->threads[task_nr].curr_cpu = cpu;
793 prctl(0, bytes_worked);
794}
795
796#define MAX_NR_NODES 64
797
798/*
799 * Count the number of nodes a process's threads
800 * are spread out on.
801 *
802 * A count of 1 means that the process is compressed
803 * to a single node. A count of g->p.nr_nodes means it's
804 * spread out on the whole system.
805 */
806static int count_process_nodes(int process_nr)
807{
808 char node_present[MAX_NR_NODES] = { 0, };
809 int nodes;
810 int n, t;
811
812 for (t = 0; t < g->p.nr_threads; t++) {
813 struct thread_data *td;
814 int task_nr;
815 int node;
816
817 task_nr = process_nr*g->p.nr_threads + t;
818 td = g->threads + task_nr;
819
820 node = numa_node_of_cpu(td->curr_cpu);
821 node_present[node] = 1;
822 }
823
824 nodes = 0;
825
826 for (n = 0; n < MAX_NR_NODES; n++)
827 nodes += node_present[n];
828
829 return nodes;
830}
831
832/*
833 * Count the number of distinct process-threads a node contains.
834 *
835 * A count of 1 means that the node contains only a single
836 * process. If all nodes on the system contain at most one
837 * process then we are well-converged.
838 */
839static int count_node_processes(int node)
840{
841 int processes = 0;
842 int t, p;
843
844 for (p = 0; p < g->p.nr_proc; p++) {
845 for (t = 0; t < g->p.nr_threads; t++) {
846 struct thread_data *td;
847 int task_nr;
848 int n;
849
850 task_nr = p*g->p.nr_threads + t;
851 td = g->threads + task_nr;
852
853 n = numa_node_of_cpu(td->curr_cpu);
854 if (n == node) {
855 processes++;
856 break;
857 }
858 }
859 }
860
861 return processes;
862}
863
864static void calc_convergence_compression(int *strong)
865{
866 unsigned int nodes_min, nodes_max;
867 int p;
868
869 nodes_min = -1;
870 nodes_max = 0;
871
872 for (p = 0; p < g->p.nr_proc; p++) {
873 unsigned int nodes = count_process_nodes(p);
874
875 nodes_min = min(nodes, nodes_min);
876 nodes_max = max(nodes, nodes_max);
877 }
878
879 /* Strong convergence: all threads compress on a single node: */
880 if (nodes_min == 1 && nodes_max == 1) {
881 *strong = 1;
882 } else {
883 *strong = 0;
884 tprintf(" {%d-%d}", nodes_min, nodes_max);
885 }
886}
887
888static void calc_convergence(double runtime_ns_max, double *convergence)
889{
890 unsigned int loops_done_min, loops_done_max;
891 int process_groups;
892 int nodes[MAX_NR_NODES];
893 int distance;
894 int nr_min;
895 int nr_max;
896 int strong;
897 int sum;
898 int nr;
899 int node;
900 int cpu;
901 int t;
902
903 if (!g->p.show_convergence && !g->p.measure_convergence)
904 return;
905
906 for (node = 0; node < g->p.nr_nodes; node++)
907 nodes[node] = 0;
908
909 loops_done_min = -1;
910 loops_done_max = 0;
911
912 for (t = 0; t < g->p.nr_tasks; t++) {
913 struct thread_data *td = g->threads + t;
914 unsigned int loops_done;
915
916 cpu = td->curr_cpu;
917
918 /* Not all threads have written it yet: */
919 if (cpu < 0)
920 continue;
921
922 node = numa_node_of_cpu(cpu);
923
924 nodes[node]++;
925
926 loops_done = td->loops_done;
927 loops_done_min = min(loops_done, loops_done_min);
928 loops_done_max = max(loops_done, loops_done_max);
929 }
930
931 nr_max = 0;
932 nr_min = g->p.nr_tasks;
933 sum = 0;
934
935 for (node = 0; node < g->p.nr_nodes; node++) {
936 nr = nodes[node];
937 nr_min = min(nr, nr_min);
938 nr_max = max(nr, nr_max);
939 sum += nr;
940 }
941 BUG_ON(nr_min > nr_max);
942
943 BUG_ON(sum > g->p.nr_tasks);
944
945 if (0 && (sum < g->p.nr_tasks))
946 return;
947
948 /*
949 * Count the number of distinct process groups present
950 * on nodes - when we are converged this will decrease
951 * to g->p.nr_proc:
952 */
953 process_groups = 0;
954
955 for (node = 0; node < g->p.nr_nodes; node++) {
956 int processes = count_node_processes(node);
957
958 nr = nodes[node];
959 tprintf(" %2d/%-2d", nr, processes);
960
961 process_groups += processes;
962 }
963
964 distance = nr_max - nr_min;
965
966 tprintf(" [%2d/%-2d]", distance, process_groups);
967
968 tprintf(" l:%3d-%-3d (%3d)",
969 loops_done_min, loops_done_max, loops_done_max-loops_done_min);
970
971 if (loops_done_min && loops_done_max) {
972 double skew = 1.0 - (double)loops_done_min/loops_done_max;
973
974 tprintf(" [%4.1f%%]", skew * 100.0);
975 }
976
977 calc_convergence_compression(&strong);
978
979 if (strong && process_groups == g->p.nr_proc) {
980 if (!*convergence) {
981 *convergence = runtime_ns_max;
982 tprintf(" (%6.1fs converged)\n", *convergence/1e9);
983 if (g->p.measure_convergence) {
984 g->all_converged = true;
985 g->stop_work = true;
986 }
987 }
988 } else {
989 if (*convergence) {
990 tprintf(" (%6.1fs de-converged)", runtime_ns_max/1e9);
991 *convergence = 0;
992 }
993 tprintf("\n");
994 }
995}
996
997static void show_summary(double runtime_ns_max, int l, double *convergence)
998{
999 tprintf("\r # %5.1f%% [%.1f mins]",
1000 (double)(l+1)/g->p.nr_loops*100.0, runtime_ns_max/1e9 / 60.0);
1001
1002 calc_convergence(runtime_ns_max, convergence);
1003
1004 if (g->p.show_details >= 0)
1005 fflush(stdout);
1006}
1007
1008static void *worker_thread(void *__tdata)
1009{
1010 struct thread_data *td = __tdata;
1011 struct timeval start0, start, stop, diff;
1012 int process_nr = td->process_nr;
1013 int thread_nr = td->thread_nr;
1014 unsigned long last_perturbance;
1015 int task_nr = td->task_nr;
1016 int details = g->p.show_details;
1017 int first_task, last_task;
1018 double convergence = 0;
1019 u64 val = td->val;
1020 double runtime_ns_max;
1021 u8 *global_data;
1022 u8 *process_data;
1023 u8 *thread_data;
1024 u64 bytes_done;
1025 long work_done;
1026 u32 l;
1027
1028 bind_to_cpumask(td->bind_cpumask);
1029 bind_to_memnode(td->bind_node);
1030
1031 set_taskname("thread %d/%d", process_nr, thread_nr);
1032
1033 global_data = g->data;
1034 process_data = td->process_data;
1035 thread_data = setup_private_data(g->p.bytes_thread);
1036
1037 bytes_done = 0;
1038
1039 last_task = 0;
1040 if (process_nr == g->p.nr_proc-1 && thread_nr == g->p.nr_threads-1)
1041 last_task = 1;
1042
1043 first_task = 0;
1044 if (process_nr == 0 && thread_nr == 0)
1045 first_task = 1;
1046
1047 if (details >= 2) {
1048 printf("# thread %2d / %2d global mem: %p, process mem: %p, thread mem: %p\n",
1049 process_nr, thread_nr, global_data, process_data, thread_data);
1050 }
1051
1052 if (g->p.serialize_startup) {
1053 pthread_mutex_lock(&g->startup_mutex);
1054 g->nr_tasks_started++;
1055 pthread_mutex_unlock(&g->startup_mutex);
1056
1057 /* Here we will wait for the main process to start us all at once: */
1058 pthread_mutex_lock(&g->start_work_mutex);
1059 g->nr_tasks_working++;
1060
1061 /* Last one wake the main process: */
1062 if (g->nr_tasks_working == g->p.nr_tasks)
1063 pthread_mutex_unlock(&g->startup_done_mutex);
1064
1065 pthread_mutex_unlock(&g->start_work_mutex);
1066 }
1067
1068 gettimeofday(&start0, NULL);
1069
1070 start = stop = start0;
1071 last_perturbance = start.tv_sec;
1072
1073 for (l = 0; l < g->p.nr_loops; l++) {
1074 start = stop;
1075
1076 if (g->stop_work)
1077 break;
1078
1079 val += do_work(global_data, g->p.bytes_global, process_nr, g->p.nr_proc, l, val);
1080 val += do_work(process_data, g->p.bytes_process, thread_nr, g->p.nr_threads, l, val);
1081 val += do_work(thread_data, g->p.bytes_thread, 0, 1, l, val);
1082
1083 if (g->p.sleep_usecs) {
1084 pthread_mutex_lock(td->process_lock);
1085 usleep(g->p.sleep_usecs);
1086 pthread_mutex_unlock(td->process_lock);
1087 }
1088 /*
1089 * Amount of work to be done under a process-global lock:
1090 */
1091 if (g->p.bytes_process_locked) {
1092 pthread_mutex_lock(td->process_lock);
1093 val += do_work(process_data, g->p.bytes_process_locked, thread_nr, g->p.nr_threads, l, val);
1094 pthread_mutex_unlock(td->process_lock);
1095 }
1096
1097 work_done = g->p.bytes_global + g->p.bytes_process +
1098 g->p.bytes_process_locked + g->p.bytes_thread;
1099
1100 update_curr_cpu(task_nr, work_done);
1101 bytes_done += work_done;
1102
1103 if (details < 0 && !g->p.perturb_secs && !g->p.measure_convergence && !g->p.nr_secs)
1104 continue;
1105
1106 td->loops_done = l;
1107
1108 gettimeofday(&stop, NULL);
1109
1110 /* Check whether our max runtime timed out: */
1111 if (g->p.nr_secs) {
1112 timersub(&stop, &start0, &diff);
1113 if (diff.tv_sec >= g->p.nr_secs) {
1114 g->stop_work = true;
1115 break;
1116 }
1117 }
1118
1119 /* Update the summary at most once per second: */
1120 if (start.tv_sec == stop.tv_sec)
1121 continue;
1122
1123 /*
1124 * Perturb the first task's equilibrium every g->p.perturb_secs seconds,
1125 * by migrating to CPU#0:
1126 */
1127 if (first_task && g->p.perturb_secs && (int)(stop.tv_sec - last_perturbance) >= g->p.perturb_secs) {
1128 cpu_set_t orig_mask;
1129 int target_cpu;
1130 int this_cpu;
1131
1132 last_perturbance = stop.tv_sec;
1133
1134 /*
1135 * Depending on where we are running, move into
1136 * the other half of the system, to create some
1137 * real disturbance:
1138 */
1139 this_cpu = g->threads[task_nr].curr_cpu;
1140 if (this_cpu < g->p.nr_cpus/2)
1141 target_cpu = g->p.nr_cpus-1;
1142 else
1143 target_cpu = 0;
1144
1145 orig_mask = bind_to_cpu(target_cpu);
1146
1147 /* Here we are running on the target CPU already */
1148 if (details >= 1)
1149 printf(" (injecting perturbalance, moved to CPU#%d)\n", target_cpu);
1150
1151 bind_to_cpumask(orig_mask);
1152 }
1153
1154 if (details >= 3) {
1155 timersub(&stop, &start, &diff);
1156 runtime_ns_max = diff.tv_sec * 1000000000;
1157 runtime_ns_max += diff.tv_usec * 1000;
1158
1159 if (details >= 0) {
1160 printf(" #%2d / %2d: %14.2lf nsecs/op [val: %016lx]\n",
1161 process_nr, thread_nr, runtime_ns_max / bytes_done, val);
1162 }
1163 fflush(stdout);
1164 }
1165 if (!last_task)
1166 continue;
1167
1168 timersub(&stop, &start0, &diff);
1169 runtime_ns_max = diff.tv_sec * 1000000000ULL;
1170 runtime_ns_max += diff.tv_usec * 1000ULL;
1171
1172 show_summary(runtime_ns_max, l, &convergence);
1173 }
1174
1175 gettimeofday(&stop, NULL);
1176 timersub(&stop, &start0, &diff);
1177 td->runtime_ns = diff.tv_sec * 1000000000ULL;
1178 td->runtime_ns += diff.tv_usec * 1000ULL;
1179
1180 free_data(thread_data, g->p.bytes_thread);
1181
1182 pthread_mutex_lock(&g->stop_work_mutex);
1183 g->bytes_done += bytes_done;
1184 pthread_mutex_unlock(&g->stop_work_mutex);
1185
1186 return NULL;
1187}
1188
1189/*
1190 * A worker process starts a couple of threads:
1191 */
1192static void worker_process(int process_nr)
1193{
1194 pthread_mutex_t process_lock;
1195 struct thread_data *td;
1196 pthread_t *pthreads;
1197 u8 *process_data;
1198 int task_nr;
1199 int ret;
1200 int t;
1201
1202 pthread_mutex_init(&process_lock, NULL);
1203 set_taskname("process %d", process_nr);
1204
1205 /*
1206 * Pick up the memory policy and the CPU binding of our first thread,
1207 * so that we initialize memory accordingly:
1208 */
1209 task_nr = process_nr*g->p.nr_threads;
1210 td = g->threads + task_nr;
1211
1212 bind_to_memnode(td->bind_node);
1213 bind_to_cpumask(td->bind_cpumask);
1214
1215 pthreads = zalloc(g->p.nr_threads * sizeof(pthread_t));
1216 process_data = setup_private_data(g->p.bytes_process);
1217
1218 if (g->p.show_details >= 3) {
1219 printf(" # process %2d global mem: %p, process mem: %p\n",
1220 process_nr, g->data, process_data);
1221 }
1222
1223 for (t = 0; t < g->p.nr_threads; t++) {
1224 task_nr = process_nr*g->p.nr_threads + t;
1225 td = g->threads + task_nr;
1226
1227 td->process_data = process_data;
1228 td->process_nr = process_nr;
1229 td->thread_nr = t;
1230 td->task_nr = task_nr;
1231 td->val = rand();
1232 td->curr_cpu = -1;
1233 td->process_lock = &process_lock;
1234
1235 ret = pthread_create(pthreads + t, NULL, worker_thread, td);
1236 BUG_ON(ret);
1237 }
1238
1239 for (t = 0; t < g->p.nr_threads; t++) {
1240 ret = pthread_join(pthreads[t], NULL);
1241 BUG_ON(ret);
1242 }
1243
1244 free_data(process_data, g->p.bytes_process);
1245 free(pthreads);
1246}
1247
1248static void print_summary(void)
1249{
1250 if (g->p.show_details < 0)
1251 return;
1252
1253 printf("\n ###\n");
1254 printf(" # %d %s will execute (on %d nodes, %d CPUs):\n",
1255 g->p.nr_tasks, g->p.nr_tasks == 1 ? "task" : "tasks", g->p.nr_nodes, g->p.nr_cpus);
1256 printf(" # %5dx %5ldMB global shared mem operations\n",
1257 g->p.nr_loops, g->p.bytes_global/1024/1024);
1258 printf(" # %5dx %5ldMB process shared mem operations\n",
1259 g->p.nr_loops, g->p.bytes_process/1024/1024);
1260 printf(" # %5dx %5ldMB thread local mem operations\n",
1261 g->p.nr_loops, g->p.bytes_thread/1024/1024);
1262
1263 printf(" ###\n");
1264
1265 printf("\n ###\n"); fflush(stdout);
1266}
1267
1268static void init_thread_data(void)
1269{
1270 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;
1271 int t;
1272
1273 g->threads = zalloc_shared_data(size);
1274
1275 for (t = 0; t < g->p.nr_tasks; t++) {
1276 struct thread_data *td = g->threads + t;
1277 int cpu;
1278
1279 /* Allow all nodes by default: */
1280 td->bind_node = -1;
1281
1282 /* Allow all CPUs by default: */
1283 CPU_ZERO(&td->bind_cpumask);
1284 for (cpu = 0; cpu < g->p.nr_cpus; cpu++)
1285 CPU_SET(cpu, &td->bind_cpumask);
1286 }
1287}
1288
1289static void deinit_thread_data(void)
1290{
1291 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;
1292
1293 free_data(g->threads, size);
1294}
1295
1296static int init(void)
1297{
1298 g = (void *)alloc_data(sizeof(*g), MAP_SHARED, 1, 0, 0 /* THP */, 0);
1299
1300 /* Copy over options: */
1301 g->p = p0;
1302
1303 g->p.nr_cpus = numa_num_configured_cpus();
1304
1305 g->p.nr_nodes = numa_max_node() + 1;
1306
1307 /* char array in count_process_nodes(): */
1308 BUG_ON(g->p.nr_nodes > MAX_NR_NODES || g->p.nr_nodes < 0);
1309
1310 if (g->p.show_quiet && !g->p.show_details)
1311 g->p.show_details = -1;
1312
1313 /* Some memory should be specified: */
1314 if (!g->p.mb_global_str && !g->p.mb_proc_str && !g->p.mb_thread_str)
1315 return -1;
1316
1317 if (g->p.mb_global_str) {
1318 g->p.mb_global = atof(g->p.mb_global_str);
1319 BUG_ON(g->p.mb_global < 0);
1320 }
1321
1322 if (g->p.mb_proc_str) {
1323 g->p.mb_proc = atof(g->p.mb_proc_str);
1324 BUG_ON(g->p.mb_proc < 0);
1325 }
1326
1327 if (g->p.mb_proc_locked_str) {
1328 g->p.mb_proc_locked = atof(g->p.mb_proc_locked_str);
1329 BUG_ON(g->p.mb_proc_locked < 0);
1330 BUG_ON(g->p.mb_proc_locked > g->p.mb_proc);
1331 }
1332
1333 if (g->p.mb_thread_str) {
1334 g->p.mb_thread = atof(g->p.mb_thread_str);
1335 BUG_ON(g->p.mb_thread < 0);
1336 }
1337
1338 BUG_ON(g->p.nr_threads <= 0);
1339 BUG_ON(g->p.nr_proc <= 0);
1340
1341 g->p.nr_tasks = g->p.nr_proc*g->p.nr_threads;
1342
1343 g->p.bytes_global = g->p.mb_global *1024L*1024L;
1344 g->p.bytes_process = g->p.mb_proc *1024L*1024L;
1345 g->p.bytes_process_locked = g->p.mb_proc_locked *1024L*1024L;
1346 g->p.bytes_thread = g->p.mb_thread *1024L*1024L;
1347
1348 g->data = setup_shared_data(g->p.bytes_global);
1349
1350 /* Startup serialization: */
1351 init_global_mutex(&g->start_work_mutex);
1352 init_global_mutex(&g->startup_mutex);
1353 init_global_mutex(&g->startup_done_mutex);
1354 init_global_mutex(&g->stop_work_mutex);
1355
1356 init_thread_data();
1357
1358 tprintf("#\n");
1359 parse_setup_cpu_list();
1360 parse_setup_node_list();
1361 tprintf("#\n");
1362
1363 print_summary();
1364
1365 return 0;
1366}
1367
1368static void deinit(void)
1369{
1370 free_data(g->data, g->p.bytes_global);
1371 g->data = NULL;
1372
1373 deinit_thread_data();
1374
1375 free_data(g, sizeof(*g));
1376 g = NULL;
1377}
1378
1379/*
1380 * Print a short or long result, depending on the verbosity setting:
1381 */
1382static void print_res(const char *name, double val,
1383 const char *txt_unit, const char *txt_short, const char *txt_long)
1384{
1385 if (!name)
1386 name = "main,";
1387
1388 if (g->p.show_quiet)
1389 printf(" %-30s %15.3f, %-15s %s\n", name, val, txt_unit, txt_short);
1390 else
1391 printf(" %14.3f %s\n", val, txt_long);
1392}
1393
1394static int __bench_numa(const char *name)
1395{
1396 struct timeval start, stop, diff;
1397 u64 runtime_ns_min, runtime_ns_sum;
1398 pid_t *pids, pid, wpid;
1399 double delta_runtime;
1400 double runtime_avg;
1401 double runtime_sec_max;
1402 double runtime_sec_min;
1403 int wait_stat;
1404 double bytes;
1405 int i, t;
1406
1407 if (init())
1408 return -1;
1409
1410 pids = zalloc(g->p.nr_proc * sizeof(*pids));
1411 pid = -1;
1412
1413 /* All threads try to acquire it, this way we can wait for them to start up: */
1414 pthread_mutex_lock(&g->start_work_mutex);
1415
1416 if (g->p.serialize_startup) {
1417 tprintf(" #\n");
1418 tprintf(" # Startup synchronization: ..."); fflush(stdout);
1419 }
1420
1421 gettimeofday(&start, NULL);
1422
1423 for (i = 0; i < g->p.nr_proc; i++) {
1424 pid = fork();
1425 dprintf(" # process %2d: PID %d\n", i, pid);
1426
1427 BUG_ON(pid < 0);
1428 if (!pid) {
1429 /* Child process: */
1430 worker_process(i);
1431
1432 exit(0);
1433 }
1434 pids[i] = pid;
1435
1436 }
1437 /* Wait for all the threads to start up: */
1438 while (g->nr_tasks_started != g->p.nr_tasks)
1439 usleep(1000);
1440
1441 BUG_ON(g->nr_tasks_started != g->p.nr_tasks);
1442
1443 if (g->p.serialize_startup) {
1444 double startup_sec;
1445
1446 pthread_mutex_lock(&g->startup_done_mutex);
1447
1448 /* This will start all threads: */
1449 pthread_mutex_unlock(&g->start_work_mutex);
1450
1451 /* This mutex is locked - the last started thread will wake us: */
1452 pthread_mutex_lock(&g->startup_done_mutex);
1453
1454 gettimeofday(&stop, NULL);
1455
1456 timersub(&stop, &start, &diff);
1457
1458 startup_sec = diff.tv_sec * 1000000000.0;
1459 startup_sec += diff.tv_usec * 1000.0;
1460 startup_sec /= 1e9;
1461
1462 tprintf(" threads initialized in %.6f seconds.\n", startup_sec);
1463 tprintf(" #\n");
1464
1465 start = stop;
1466 pthread_mutex_unlock(&g->startup_done_mutex);
1467 } else {
1468 gettimeofday(&start, NULL);
1469 }
1470
1471 /* Parent process: */
1472
1473
1474 for (i = 0; i < g->p.nr_proc; i++) {
1475 wpid = waitpid(pids[i], &wait_stat, 0);
1476 BUG_ON(wpid < 0);
1477 BUG_ON(!WIFEXITED(wait_stat));
1478
1479 }
1480
1481 runtime_ns_sum = 0;
1482 runtime_ns_min = -1LL;
1483
1484 for (t = 0; t < g->p.nr_tasks; t++) {
1485 u64 thread_runtime_ns = g->threads[t].runtime_ns;
1486
1487 runtime_ns_sum += thread_runtime_ns;
1488 runtime_ns_min = min(thread_runtime_ns, runtime_ns_min);
1489 }
1490
1491 gettimeofday(&stop, NULL);
1492 timersub(&stop, &start, &diff);
1493
1494 BUG_ON(bench_format != BENCH_FORMAT_DEFAULT);
1495
1496 tprintf("\n ###\n");
1497 tprintf("\n");
1498
1499 runtime_sec_max = diff.tv_sec * 1000000000.0;
1500 runtime_sec_max += diff.tv_usec * 1000.0;
1501 runtime_sec_max /= 1e9;
1502
1503 runtime_sec_min = runtime_ns_min/1e9;
1504
1505 bytes = g->bytes_done;
1506 runtime_avg = (double)runtime_ns_sum / g->p.nr_tasks / 1e9;
1507
1508 if (g->p.measure_convergence) {
1509 print_res(name, runtime_sec_max,
1510 "secs,", "NUMA-convergence-latency", "secs latency to NUMA-converge");
1511 }
1512
1513 print_res(name, runtime_sec_max,
1514 "secs,", "runtime-max/thread", "secs slowest (max) thread-runtime");
1515
1516 print_res(name, runtime_sec_min,
1517 "secs,", "runtime-min/thread", "secs fastest (min) thread-runtime");
1518
1519 print_res(name, runtime_avg,
1520 "secs,", "runtime-avg/thread", "secs average thread-runtime");
1521
1522 delta_runtime = (runtime_sec_max - runtime_sec_min)/2.0;
1523 print_res(name, delta_runtime / runtime_sec_max * 100.0,
1524 "%,", "spread-runtime/thread", "% difference between max/avg runtime");
1525
1526 print_res(name, bytes / g->p.nr_tasks / 1e9,
1527 "GB,", "data/thread", "GB data processed, per thread");
1528
1529 print_res(name, bytes / 1e9,
1530 "GB,", "data-total", "GB data processed, total");
1531
1532 print_res(name, runtime_sec_max * 1e9 / (bytes / g->p.nr_tasks),
1533 "nsecs,", "runtime/byte/thread","nsecs/byte/thread runtime");
1534
1535 print_res(name, bytes / g->p.nr_tasks / 1e9 / runtime_sec_max,
1536 "GB/sec,", "thread-speed", "GB/sec/thread speed");
1537
1538 print_res(name, bytes / runtime_sec_max / 1e9,
1539 "GB/sec,", "total-speed", "GB/sec total speed");
1540
1541 free(pids);
1542
1543 deinit();
1544
1545 return 0;
1546}
1547
1548#define MAX_ARGS 50
1549
1550static int command_size(const char **argv)
1551{
1552 int size = 0;
1553
1554 while (*argv) {
1555 size++;
1556 argv++;
1557 }
1558
1559 BUG_ON(size >= MAX_ARGS);
1560
1561 return size;
1562}
1563
1564static void init_params(struct params *p, const char *name, int argc, const char **argv)
1565{
1566 int i;
1567
1568 printf("\n # Running %s \"perf bench numa", name);
1569
1570 for (i = 0; i < argc; i++)
1571 printf(" %s", argv[i]);
1572
1573 printf("\"\n");
1574
1575 memset(p, 0, sizeof(*p));
1576
1577 /* Initialize nonzero defaults: */
1578
1579 p->serialize_startup = 1;
1580 p->data_reads = true;
1581 p->data_writes = true;
1582 p->data_backwards = true;
1583 p->data_rand_walk = true;
1584 p->nr_loops = -1;
1585 p->init_random = true;
1586}
1587
1588static int run_bench_numa(const char *name, const char **argv)
1589{
1590 int argc = command_size(argv);
1591
1592 init_params(&p0, name, argc, argv);
1593 argc = parse_options(argc, argv, options, bench_numa_usage, 0);
1594 if (argc)
1595 goto err;
1596
1597 if (__bench_numa(name))
1598 goto err;
1599
1600 return 0;
1601
1602err:
1603 usage_with_options(numa_usage, options);
1604 return -1;
1605}
1606
1607#define OPT_BW_RAM "-s", "20", "-zZq", "--thp", " 1", "--no-data_rand_walk"
1608#define OPT_BW_RAM_NOTHP OPT_BW_RAM, "--thp", "-1"
1609
1610#define OPT_CONV "-s", "100", "-zZ0qcm", "--thp", " 1"
1611#define OPT_CONV_NOTHP OPT_CONV, "--thp", "-1"
1612
1613#define OPT_BW "-s", "20", "-zZ0q", "--thp", " 1"
1614#define OPT_BW_NOTHP OPT_BW, "--thp", "-1"
1615
1616/*
1617 * The built-in test-suite executed by "perf bench numa -a".
1618 *
1619 * (A minimum of 4 nodes and 16 GB of RAM is recommended.)
1620 */
1621static const char *tests[][MAX_ARGS] = {
1622 /* Basic single-stream NUMA bandwidth measurements: */
1623 { "RAM-bw-local,", "mem", "-p", "1", "-t", "1", "-P", "1024",
1624 "-C" , "0", "-M", "0", OPT_BW_RAM },
1625 { "RAM-bw-local-NOTHP,",
1626 "mem", "-p", "1", "-t", "1", "-P", "1024",
1627 "-C" , "0", "-M", "0", OPT_BW_RAM_NOTHP },
1628 { "RAM-bw-remote,", "mem", "-p", "1", "-t", "1", "-P", "1024",
1629 "-C" , "0", "-M", "1", OPT_BW_RAM },
1630
1631 /* 2-stream NUMA bandwidth measurements: */
1632 { "RAM-bw-local-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024",
1633 "-C", "0,2", "-M", "0x2", OPT_BW_RAM },
1634 { "RAM-bw-remote-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024",
1635 "-C", "0,2", "-M", "1x2", OPT_BW_RAM },
1636
1637 /* Cross-stream NUMA bandwidth measurement: */
1638 { "RAM-bw-cross,", "mem", "-p", "2", "-t", "1", "-P", "1024",
1639 "-C", "0,8", "-M", "1,0", OPT_BW_RAM },
1640
1641 /* Convergence latency measurements: */
1642 { " 1x3-convergence,", "mem", "-p", "1", "-t", "3", "-P", "512", OPT_CONV },
1643 { " 1x4-convergence,", "mem", "-p", "1", "-t", "4", "-P", "512", OPT_CONV },
1644 { " 1x6-convergence,", "mem", "-p", "1", "-t", "6", "-P", "1020", OPT_CONV },
1645 { " 2x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV },
1646 { " 3x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV },
1647 { " 4x4-convergence,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV },
1648 { " 4x4-convergence-NOTHP,",
1649 "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV_NOTHP },
1650 { " 4x6-convergence,", "mem", "-p", "4", "-t", "6", "-P", "1020", OPT_CONV },
1651 { " 4x8-convergence,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_CONV },
1652 { " 8x4-convergence,", "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV },
1653 { " 8x4-convergence-NOTHP,",
1654 "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV_NOTHP },
1655 { " 3x1-convergence,", "mem", "-p", "3", "-t", "1", "-P", "512", OPT_CONV },
1656 { " 4x1-convergence,", "mem", "-p", "4", "-t", "1", "-P", "512", OPT_CONV },
1657 { " 8x1-convergence,", "mem", "-p", "8", "-t", "1", "-P", "512", OPT_CONV },
1658 { "16x1-convergence,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_CONV },
1659 { "32x1-convergence,", "mem", "-p", "32", "-t", "1", "-P", "128", OPT_CONV },
1660
1661 /* Various NUMA process/thread layout bandwidth measurements: */
1662 { " 2x1-bw-process,", "mem", "-p", "2", "-t", "1", "-P", "1024", OPT_BW },
1663 { " 3x1-bw-process,", "mem", "-p", "3", "-t", "1", "-P", "1024", OPT_BW },
1664 { " 4x1-bw-process,", "mem", "-p", "4", "-t", "1", "-P", "1024", OPT_BW },
1665 { " 8x1-bw-process,", "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW },
1666 { " 8x1-bw-process-NOTHP,",
1667 "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW_NOTHP },
1668 { "16x1-bw-process,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_BW },
1669
1670 { " 4x1-bw-thread,", "mem", "-p", "1", "-t", "4", "-T", "256", OPT_BW },
1671 { " 8x1-bw-thread,", "mem", "-p", "1", "-t", "8", "-T", "256", OPT_BW },
1672 { "16x1-bw-thread,", "mem", "-p", "1", "-t", "16", "-T", "128", OPT_BW },
1673 { "32x1-bw-thread,", "mem", "-p", "1", "-t", "32", "-T", "64", OPT_BW },
1674
1675 { " 2x3-bw-thread,", "mem", "-p", "2", "-t", "3", "-P", "512", OPT_BW },
1676 { " 4x4-bw-thread,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_BW },
1677 { " 4x6-bw-thread,", "mem", "-p", "4", "-t", "6", "-P", "512", OPT_BW },
1678 { " 4x8-bw-thread,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW },
1679 { " 4x8-bw-thread-NOTHP,",
1680 "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW_NOTHP },
1681 { " 3x3-bw-thread,", "mem", "-p", "3", "-t", "3", "-P", "512", OPT_BW },
1682 { " 5x5-bw-thread,", "mem", "-p", "5", "-t", "5", "-P", "512", OPT_BW },
1683
1684 { "2x16-bw-thread,", "mem", "-p", "2", "-t", "16", "-P", "512", OPT_BW },
1685 { "1x32-bw-thread,", "mem", "-p", "1", "-t", "32", "-P", "2048", OPT_BW },
1686
1687 { "numa02-bw,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW },
1688 { "numa02-bw-NOTHP,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW_NOTHP },
1689 { "numa01-bw-thread,", "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW },
1690 { "numa01-bw-thread-NOTHP,",
1691 "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW_NOTHP },
1692};
1693
1694static int bench_all(void)
1695{
1696 int nr = ARRAY_SIZE(tests);
1697 int ret;
1698 int i;
1699
1700 ret = system("echo ' #'; echo ' # Running test on: '$(uname -a); echo ' #'");
1701 BUG_ON(ret < 0);
1702
1703 for (i = 0; i < nr; i++) {
1704 if (run_bench_numa(tests[i][0], tests[i] + 1))
1705 return -1;
1706 }
1707
1708 printf("\n");
1709
1710 return 0;
1711}
1712
1713int bench_numa(int argc, const char **argv, const char *prefix __maybe_unused)
1714{
1715 init_params(&p0, "main,", argc, argv);
1716 argc = parse_options(argc, argv, options, bench_numa_usage, 0);
1717 if (argc)
1718 goto err;
1719
1720 if (p0.run_all)
1721 return bench_all();
1722
1723 if (__bench_numa(NULL))
1724 goto err;
1725
1726 return 0;
1727
1728err:
1729 usage_with_options(numa_usage, options);
1730 return -1;
1731}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-21 16:00:09 -0500