diff options
author | Nicolas Benech <nbenech@nvidia.com> | 2018-09-25 14:37:16 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2018-09-27 17:04:17 -0400 |
commit | c6eae929fd74f11ab13d469a38bffd4e8ba50fb5 (patch) | |
tree | 12525d29aa8934cba58c703e67bf1aa8928ccacd | |
parent | 442cb2b1db40ae23bbcfda0624c0546da186694c (diff) |
gpu: nvgpu: posix: Multithreading for unit tests
Add a -j argument to enable running unit tests on several
threads. Also adds signal handling to prevent a fatal
error in one thread from killing the whole unit test
framework.
JIRA NVGPU-1043
Change-Id: I891a547640cd005a50ffa5c06367ed46c54de012
Signed-off-by: Nicolas Benech <nbenech@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1847740
Reviewed-by: svc-misra-checker <svc-misra-checker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Alex Waterman <alexw@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
-rw-r--r-- | userspace/include/unit/args.h | 1 | ||||
-rw-r--r-- | userspace/include/unit/unit.h | 4 | ||||
-rw-r--r-- | userspace/src/args.c | 13 | ||||
-rw-r--r-- | userspace/src/exec.c | 142 | ||||
-rw-r--r-- | userspace/src/results.c | 26 | ||||
-rw-r--r-- | userspace/src/unit_main.c | 8 |
6 files changed, 173 insertions, 21 deletions
diff --git a/userspace/include/unit/args.h b/userspace/include/unit/args.h index def84a29..708a1655 100644 --- a/userspace/include/unit/args.h +++ b/userspace/include/unit/args.h | |||
@@ -42,6 +42,7 @@ struct unit_fw_args { | |||
42 | bool help; | 42 | bool help; |
43 | int verbose_lvl; | 43 | int verbose_lvl; |
44 | bool no_color; | 44 | bool no_color; |
45 | int thread_count; | ||
45 | 46 | ||
46 | const char *unit_name; | 47 | const char *unit_name; |
47 | const char *unit_load_path; | 48 | const char *unit_load_path; |
diff --git a/userspace/include/unit/unit.h b/userspace/include/unit/unit.h index 2f93bab5..442a73f1 100644 --- a/userspace/include/unit/unit.h +++ b/userspace/include/unit/unit.h | |||
@@ -23,6 +23,8 @@ | |||
23 | #ifndef __UNIT_UNIT_H__ | 23 | #ifndef __UNIT_UNIT_H__ |
24 | #define __UNIT_UNIT_H__ | 24 | #define __UNIT_UNIT_H__ |
25 | 25 | ||
26 | #include <pthread.h> | ||
27 | |||
26 | struct gk20a; | 28 | struct gk20a; |
27 | 29 | ||
28 | struct unit_module; | 30 | struct unit_module; |
@@ -84,6 +86,8 @@ struct unit_module { | |||
84 | */ | 86 | */ |
85 | void *lib_handle; | 87 | void *lib_handle; |
86 | struct unit_fw *fw; | 88 | struct unit_fw *fw; |
89 | |||
90 | pthread_t thread; | ||
87 | }; | 91 | }; |
88 | 92 | ||
89 | /* | 93 | /* |
diff --git a/userspace/src/args.c b/userspace/src/args.c index d91c6f6e..cf17c983 100644 --- a/userspace/src/args.c +++ b/userspace/src/args.c | |||
@@ -36,11 +36,12 @@ static struct option core_opts[] = { | |||
36 | { "no-color", 0, NULL, 'C' }, | 36 | { "no-color", 0, NULL, 'C' }, |
37 | 37 | ||
38 | { "unit-load-path", 1, NULL, 'L' }, | 38 | { "unit-load-path", 1, NULL, 'L' }, |
39 | { "num-threads", 1, NULL, 'j' }, | ||
39 | 40 | ||
40 | { NULL, 0, NULL, 0 } | 41 | { NULL, 0, NULL, 0 } |
41 | }; | 42 | }; |
42 | 43 | ||
43 | static const char *core_opts_str = "hvqCL:"; | 44 | static const char *core_opts_str = "hvqCL:j:"; |
44 | 45 | ||
45 | void core_print_help(struct unit_fw *fw) | 46 | void core_print_help(struct unit_fw *fw) |
46 | { | 47 | { |
@@ -63,6 +64,8 @@ void core_print_help(struct unit_fw *fw) | |||
63 | " corrupt that file.\n", | 64 | " corrupt that file.\n", |
64 | " -L, --unit-load-path <PATH>\n", | 65 | " -L, --unit-load-path <PATH>\n", |
65 | " Path to where the unit test libraries reside.\n", | 66 | " Path to where the unit test libraries reside.\n", |
67 | " -j, --num-threads <COUNT>\n", | ||
68 | " Number of threads to use while running all tests.\n", | ||
66 | "\n", | 69 | "\n", |
67 | "Note: mandatory arguments to long arguments are mandatory for short\n", | 70 | "Note: mandatory arguments to long arguments are mandatory for short\n", |
68 | "arguments as well.\n", | 71 | "arguments as well.\n", |
@@ -79,6 +82,7 @@ NULL | |||
79 | static void set_arg_defaults(struct unit_fw_args *args) | 82 | static void set_arg_defaults(struct unit_fw_args *args) |
80 | { | 83 | { |
81 | args->unit_load_path = DEFAULT_ARG_UNIT_LOAD_PATH; | 84 | args->unit_load_path = DEFAULT_ARG_UNIT_LOAD_PATH; |
85 | args->thread_count = 1; | ||
82 | } | 86 | } |
83 | 87 | ||
84 | /* | 88 | /* |
@@ -121,6 +125,13 @@ int core_parse_args(struct unit_fw *fw, int argc, char **argv) | |||
121 | case 'L': | 125 | case 'L': |
122 | args->unit_load_path = optarg; | 126 | args->unit_load_path = optarg; |
123 | break; | 127 | break; |
128 | case 'j': | ||
129 | args->thread_count = strtol(optarg, NULL, 10); | ||
130 | if (args->thread_count == 0) { | ||
131 | core_err(fw, "Invalid number of threads\n"); | ||
132 | return -1; | ||
133 | } | ||
134 | break; | ||
124 | case '?': | 135 | case '?': |
125 | args->help = true; | 136 | args->help = true; |
126 | return -1; | 137 | return -1; |
diff --git a/userspace/src/exec.c b/userspace/src/exec.c index b9ba1336..8a99437b 100644 --- a/userspace/src/exec.c +++ b/userspace/src/exec.c | |||
@@ -21,6 +21,10 @@ | |||
21 | */ | 21 | */ |
22 | 22 | ||
23 | #include <stdlib.h> | 23 | #include <stdlib.h> |
24 | #include <string.h> | ||
25 | #include <pthread.h> | ||
26 | #include <semaphore.h> | ||
27 | #include <signal.h> | ||
24 | 28 | ||
25 | #include <unit/io.h> | 29 | #include <unit/io.h> |
26 | #include <unit/core.h> | 30 | #include <unit/core.h> |
@@ -31,19 +35,39 @@ | |||
31 | #include <nvgpu/posix/probe.h> | 35 | #include <nvgpu/posix/probe.h> |
32 | 36 | ||
33 | /* | 37 | /* |
38 | * Sempaphore to limit the number of threads | ||
39 | */ | ||
40 | sem_t unit_thread_semaphore; | ||
41 | |||
42 | /* | ||
43 | * C11 thread local storage, used to access test context when a signal is | ||
44 | * received (ex: SIGSEGV) in a thread. | ||
45 | */ | ||
46 | _Thread_local struct unit_module *thread_local_module; | ||
47 | _Thread_local struct unit_module_test *thread_local_test; | ||
48 | |||
49 | /* | ||
34 | * Execute a module and all its subtests. This function builds a gk20a for the | 50 | * Execute a module and all its subtests. This function builds a gk20a for the |
35 | * test to use by executing nvgpu_posix_probe() and nvgpu_posix_cleanup(); | 51 | * test to use by executing nvgpu_posix_probe() and nvgpu_posix_cleanup(); |
36 | */ | 52 | */ |
37 | static int core_exec_module(struct unit_fw *fw, | 53 | static void *core_exec_module(void *module_param) |
38 | struct unit_module *module) | ||
39 | { | 54 | { |
40 | unsigned int i; | 55 | unsigned int i; |
41 | struct gk20a *g = fw->nvgpu.nvgpu_posix_probe(); | 56 | struct unit_module *module = (struct unit_module *) module_param; |
57 | struct gk20a *g; | ||
58 | |||
59 | g = module->fw->nvgpu.nvgpu_posix_probe(); | ||
60 | |||
61 | if (!g) { | ||
62 | core_msg_color(module->fw, C_RED, | ||
63 | " nvgpu_posix_probe failed: Module %s\n", | ||
64 | module->name); | ||
65 | goto thread_exit; | ||
66 | } | ||
42 | 67 | ||
43 | if (!g) | 68 | core_vbs(module->fw, 1, "Execing module: %s\n", module->name); |
44 | return -1; | ||
45 | 69 | ||
46 | core_vbs(fw, 1, "Execing module: %s\n", module->name); | 70 | thread_local_module = module; |
47 | 71 | ||
48 | /* | 72 | /* |
49 | * Execute each test within the module. No reinit is done between tests. | 73 | * Execute each test within the module. No reinit is done between tests. |
@@ -53,21 +77,86 @@ static int core_exec_module(struct unit_fw *fw, | |||
53 | for (i = 0; i < module->nr_tests; i++) { | 77 | for (i = 0; i < module->nr_tests; i++) { |
54 | struct unit_module_test *t = module->tests + i; | 78 | struct unit_module_test *t = module->tests + i; |
55 | int test_status; | 79 | int test_status; |
80 | thread_local_test = t; | ||
56 | 81 | ||
57 | core_msg(fw, "Running %s.%s\n", module->name, t->name); | 82 | core_msg(module->fw, "Running %s.%s\n", module->name, |
83 | t->name); | ||
58 | test_status = t->fn(module, g, t->args); | 84 | test_status = t->fn(module, g, t->args); |
59 | 85 | ||
60 | if (test_status != UNIT_SUCCESS) | 86 | if (test_status != UNIT_SUCCESS) |
61 | core_msg_color(fw, C_RED, | 87 | core_msg_color(module->fw, C_RED, |
62 | " Unit error! Test %s.%s FAILED!\n", | 88 | " Unit error! Test %s.%s FAILED!\n", |
63 | module->name, t->name); | 89 | module->name, t->name); |
64 | 90 | ||
65 | core_add_test_record(fw, module, t, | 91 | core_add_test_record(module->fw, module, t, |
66 | test_status == UNIT_SUCCESS); | 92 | test_status == UNIT_SUCCESS); |
67 | } | 93 | } |
68 | 94 | ||
69 | fw->nvgpu.nvgpu_posix_cleanup(g); | 95 | module->fw->nvgpu.nvgpu_posix_cleanup(g); |
96 | |||
97 | core_vbs(module->fw, 1, "Module completed: %s\n", module->name); | ||
98 | thread_exit: | ||
99 | sem_post(&unit_thread_semaphore); | ||
100 | return NULL; | ||
101 | } | ||
102 | |||
103 | /* | ||
104 | * According to POSIX, "Signals which are generated by some action attributable | ||
105 | * to a particular thread, such as a hardware fault, shall be generated for the | ||
106 | * thread that caused the signal to be generated." | ||
107 | * This custom signal handler will be run from within the thread that caused the | ||
108 | * exception. Thanks to the context being saved in local thread storage, it is | ||
109 | * then trivial to report which test case failed, and then terminate the thread. | ||
110 | */ | ||
111 | static void thread_error_handler(int sig, siginfo_t *siginfo, void *context) | ||
112 | { | ||
113 | core_msg_color(thread_local_module->fw, C_RED, | ||
114 | " Signal %d in Test: %s.%s!\n", sig, | ||
115 | thread_local_module->name, thread_local_test->name); | ||
116 | core_add_test_record(thread_local_module->fw, thread_local_module, | ||
117 | thread_local_test, false); | ||
118 | sem_post(&unit_thread_semaphore); | ||
119 | pthread_exit(NULL); | ||
120 | } | ||
121 | |||
122 | /* | ||
123 | * Install a custom signal handler for several signals to be used when running | ||
124 | * in multithreaded environment. | ||
125 | */ | ||
126 | static int install_thread_error_handler(void) | ||
127 | { | ||
128 | struct sigaction action; | ||
129 | int err; | ||
130 | |||
131 | memset(&action, 0, sizeof(action)); | ||
132 | action.sa_sigaction = &thread_error_handler; | ||
133 | action.sa_flags = SA_SIGINFO; | ||
70 | 134 | ||
135 | /* SIGSEGV: Invalid memory reference */ | ||
136 | err = sigaction(SIGSEGV, &action, NULL); | ||
137 | if (err < 0) { | ||
138 | return err; | ||
139 | } | ||
140 | /* SIGILL: Illegal Instruction */ | ||
141 | err = sigaction(SIGILL, &action, NULL); | ||
142 | if (err < 0) { | ||
143 | return err; | ||
144 | } | ||
145 | /* SIGFPE: Floating-point exception */ | ||
146 | err = sigaction(SIGFPE, &action, NULL); | ||
147 | if (err < 0) { | ||
148 | return err; | ||
149 | } | ||
150 | /* SIGBUS: Bus error */ | ||
151 | err = sigaction(SIGBUS, &action, NULL); | ||
152 | if (err < 0) { | ||
153 | return err; | ||
154 | } | ||
155 | /* SIGSYS: Bad system call */ | ||
156 | err = sigaction(SIGSYS, &action, NULL); | ||
157 | if (err < 0) { | ||
158 | return err; | ||
159 | } | ||
71 | return 0; | 160 | return 0; |
72 | } | 161 | } |
73 | 162 | ||
@@ -76,14 +165,39 @@ static int core_exec_module(struct unit_fw *fw, | |||
76 | */ | 165 | */ |
77 | int core_exec(struct unit_fw *fw) | 166 | int core_exec(struct unit_fw *fw) |
78 | { | 167 | { |
79 | int ret; | ||
80 | struct unit_module **modules; | 168 | struct unit_module **modules; |
169 | int err = 0; | ||
170 | |||
171 | core_vbs(fw, 1, "Using %d threads\n", fw->args->thread_count); | ||
172 | sem_init(&unit_thread_semaphore, 0, fw->args->thread_count); | ||
173 | |||
174 | /* | ||
175 | * If running single threaded, keep the default SIGSEGV handler to make | ||
176 | * interactive debugging easier, otherwise install the custom one. | ||
177 | */ | ||
178 | if (fw->args->thread_count > 1) { | ||
179 | err = install_thread_error_handler(); | ||
180 | if (err != 0) { | ||
181 | core_msg_color(fw, C_RED, | ||
182 | " Failed to install signal handler!\n"); | ||
183 | return err; | ||
184 | } | ||
185 | } | ||
81 | 186 | ||
82 | for (modules = fw->modules; *modules != NULL; modules++) { | 187 | for (modules = fw->modules; *modules != NULL; modules++) { |
83 | ret = core_exec_module(fw, *modules); | 188 | if (fw->args->thread_count == 1) { |
189 | core_exec_module(*modules); | ||
190 | } else { | ||
191 | sem_wait(&unit_thread_semaphore); | ||
192 | pthread_create(&((*modules)->thread), NULL, | ||
193 | core_exec_module, (void *) *modules); | ||
194 | } | ||
195 | } | ||
84 | 196 | ||
85 | if (ret != 0) | 197 | if (fw->args->thread_count > 1) { |
86 | return ret; | 198 | for (modules = fw->modules; *modules != NULL; modules++) { |
199 | pthread_join((*modules)->thread, NULL); | ||
200 | } | ||
87 | } | 201 | } |
88 | 202 | ||
89 | return 0; | 203 | return 0; |
diff --git a/userspace/src/results.c b/userspace/src/results.c index ae077b82..4c30c4db 100644 --- a/userspace/src/results.c +++ b/userspace/src/results.c | |||
@@ -22,12 +22,18 @@ | |||
22 | 22 | ||
23 | #include <stdlib.h> | 23 | #include <stdlib.h> |
24 | #include <string.h> | 24 | #include <string.h> |
25 | #include <pthread.h> | ||
25 | 26 | ||
26 | #include <unit/io.h> | 27 | #include <unit/io.h> |
27 | #include <unit/core.h> | 28 | #include <unit/core.h> |
28 | #include <unit/unit.h> | 29 | #include <unit/unit.h> |
29 | #include <unit/results.h> | 30 | #include <unit/results.h> |
30 | 31 | ||
32 | /* | ||
33 | * Mutex to ensure core_add_test_record() is thread safe. | ||
34 | */ | ||
35 | pthread_mutex_t mutex_results = PTHREAD_MUTEX_INITIALIZER; | ||
36 | |||
31 | static int __init_results(struct unit_fw *fw) | 37 | static int __init_results(struct unit_fw *fw) |
32 | { | 38 | { |
33 | struct unit_results *results; | 39 | struct unit_results *results; |
@@ -72,16 +78,22 @@ int core_add_test_record(struct unit_fw *fw, | |||
72 | bool success) | 78 | bool success) |
73 | { | 79 | { |
74 | struct unit_test_record *tr; | 80 | struct unit_test_record *tr; |
81 | int err = 0; | ||
75 | 82 | ||
83 | pthread_mutex_lock(&mutex_results); | ||
76 | /* | 84 | /* |
77 | * Dones nothing if results are already inited. | 85 | * Does nothing if results are already inited. |
78 | */ | 86 | */ |
79 | if (__init_results(fw) != 0) | 87 | if (__init_results(fw) != 0) { |
80 | return -1; | 88 | err = -1; |
89 | goto done; | ||
90 | } | ||
81 | 91 | ||
82 | tr = malloc(sizeof(*tr)); | 92 | tr = malloc(sizeof(*tr)); |
83 | if (tr == NULL) | 93 | if (tr == NULL) { |
84 | return -1; | 94 | err = -1; |
95 | goto done; | ||
96 | } | ||
85 | 97 | ||
86 | tr->mod = mod; | 98 | tr->mod = mod; |
87 | tr->test = test; | 99 | tr->test = test; |
@@ -97,7 +109,9 @@ int core_add_test_record(struct unit_fw *fw, | |||
97 | if (success) | 109 | if (success) |
98 | fw->results->nr_passing += 1; | 110 | fw->results->nr_passing += 1; |
99 | 111 | ||
100 | return 0; | 112 | done: |
113 | pthread_mutex_unlock(&mutex_results); | ||
114 | return err; | ||
101 | } | 115 | } |
102 | 116 | ||
103 | void core_print_test_status(struct unit_fw *fw) | 117 | void core_print_test_status(struct unit_fw *fw) |
diff --git a/userspace/src/unit_main.c b/userspace/src/unit_main.c index 31c31d50..64344bf0 100644 --- a/userspace/src/unit_main.c +++ b/userspace/src/unit_main.c | |||
@@ -72,5 +72,13 @@ int main(int argc, char **argv) | |||
72 | 72 | ||
73 | core_print_test_status(fw); | 73 | core_print_test_status(fw); |
74 | 74 | ||
75 | if (fw->results->nr_tests == 0) { | ||
76 | /* No tests were run */ | ||
77 | return -1; | ||
78 | } else if ((fw->results->nr_tests - fw->results->nr_passing) != 0) { | ||
79 | /* Some tests failed */ | ||
80 | return -1; | ||
81 | } | ||
82 | |||
75 | return 0; | 83 | return 0; |
76 | } | 84 | } |