diff options
author | Yonghong Song <yhs@fb.com> | 2018-05-24 14:21:58 -0400 |
---|---|---|
committer | Alexei Starovoitov <ast@kernel.org> | 2018-05-24 21:18:20 -0400 |
commit | b04df400c30235fa347313c9e2a0695549bd2c8e (patch) | |
tree | de94bbbf01500ee35347b4351c2934e88ae710e5 | |
parent | f699cf7aa41f7b5a48316c0d21b76861b45e46e8 (diff) |
tools/bpftool: add perf subcommand
The new command "bpftool perf [show | list]" will traverse
all processes under /proc, and if any fd is associated
with a perf event, it will print out related perf event
information. Documentation is also added.
Below is an example to show the results using bcc commands.
Running the following 4 bcc commands:
kprobe: trace.py '__x64_sys_nanosleep'
kretprobe: trace.py 'r::__x64_sys_nanosleep'
tracepoint: trace.py 't:syscalls:sys_enter_nanosleep'
uprobe: trace.py 'p:/home/yhs/a.out:main'
The bpftool command line and result:
$ bpftool perf
pid 21711 fd 5: prog_id 5 kprobe func __x64_sys_write offset 0
pid 21765 fd 5: prog_id 7 kretprobe func __x64_sys_nanosleep offset 0
pid 21767 fd 5: prog_id 8 tracepoint sys_enter_nanosleep
pid 21800 fd 5: prog_id 9 uprobe filename /home/yhs/a.out offset 1159
$ bpftool -j perf
[{"pid":21711,"fd":5,"prog_id":5,"fd_type":"kprobe","func":"__x64_sys_write","offset":0}, \
{"pid":21765,"fd":5,"prog_id":7,"fd_type":"kretprobe","func":"__x64_sys_nanosleep","offset":0}, \
{"pid":21767,"fd":5,"prog_id":8,"fd_type":"tracepoint","tracepoint":"sys_enter_nanosleep"}, \
{"pid":21800,"fd":5,"prog_id":9,"fd_type":"uprobe","filename":"/home/yhs/a.out","offset":1159}]
$ bpftool prog
5: kprobe name probe___x64_sys tag e495a0c82f2c7a8d gpl
loaded_at 2018-05-15T04:46:37-0700 uid 0
xlated 200B not jited memlock 4096B map_ids 4
7: kprobe name probe___x64_sys tag f2fdee479a503abf gpl
loaded_at 2018-05-15T04:48:32-0700 uid 0
xlated 200B not jited memlock 4096B map_ids 7
8: tracepoint name tracepoint__sys tag 5390badef2395fcf gpl
loaded_at 2018-05-15T04:48:48-0700 uid 0
xlated 200B not jited memlock 4096B map_ids 8
9: kprobe name probe_main_1 tag 0a87bdc2e2953b6d gpl
loaded_at 2018-05-15T04:49:52-0700 uid 0
xlated 200B not jited memlock 4096B map_ids 9
$ ps ax | grep "python ./trace.py"
21711 pts/0 T 0:03 python ./trace.py __x64_sys_write
21765 pts/0 S+ 0:00 python ./trace.py r::__x64_sys_nanosleep
21767 pts/2 S+ 0:00 python ./trace.py t:syscalls:sys_enter_nanosleep
21800 pts/3 S+ 0:00 python ./trace.py p:/home/yhs/a.out:main
22374 pts/1 S+ 0:00 grep --color=auto python ./trace.py
Reviewed-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
-rw-r--r-- | tools/bpf/bpftool/Documentation/bpftool-perf.rst | 81 | ||||
-rw-r--r-- | tools/bpf/bpftool/Documentation/bpftool.rst | 5 | ||||
-rw-r--r-- | tools/bpf/bpftool/bash-completion/bpftool | 9 | ||||
-rw-r--r-- | tools/bpf/bpftool/main.c | 3 | ||||
-rw-r--r-- | tools/bpf/bpftool/main.h | 1 | ||||
-rw-r--r-- | tools/bpf/bpftool/perf.c | 246 |
6 files changed, 343 insertions, 2 deletions
diff --git a/tools/bpf/bpftool/Documentation/bpftool-perf.rst b/tools/bpf/bpftool/Documentation/bpftool-perf.rst new file mode 100644 index 000000000000..e3eb0eab7641 --- /dev/null +++ b/tools/bpf/bpftool/Documentation/bpftool-perf.rst | |||
@@ -0,0 +1,81 @@ | |||
1 | ================ | ||
2 | bpftool-perf | ||
3 | ================ | ||
4 | ------------------------------------------------------------------------------- | ||
5 | tool for inspection of perf related bpf prog attachments | ||
6 | ------------------------------------------------------------------------------- | ||
7 | |||
8 | :Manual section: 8 | ||
9 | |||
10 | SYNOPSIS | ||
11 | ======== | ||
12 | |||
13 | **bpftool** [*OPTIONS*] **perf** *COMMAND* | ||
14 | |||
15 | *OPTIONS* := { [{ **-j** | **--json** }] [{ **-p** | **--pretty** }] } | ||
16 | |||
17 | *COMMANDS* := | ||
18 | { **show** | **list** | **help** } | ||
19 | |||
20 | PERF COMMANDS | ||
21 | ============= | ||
22 | |||
23 | | **bpftool** **perf { show | list }** | ||
24 | | **bpftool** **perf help** | ||
25 | |||
26 | DESCRIPTION | ||
27 | =========== | ||
28 | **bpftool perf { show | list }** | ||
29 | List all raw_tracepoint, tracepoint, kprobe attachment in the system. | ||
30 | |||
31 | Output will start with process id and file descriptor in that process, | ||
32 | followed by bpf program id, attachment information, and attachment point. | ||
33 | The attachment point for raw_tracepoint/tracepoint is the trace probe name. | ||
34 | The attachment point for k[ret]probe is either symbol name and offset, | ||
35 | or a kernel virtual address. | ||
36 | The attachment point for u[ret]probe is the file name and the file offset. | ||
37 | |||
38 | **bpftool perf help** | ||
39 | Print short help message. | ||
40 | |||
41 | OPTIONS | ||
42 | ======= | ||
43 | -h, --help | ||
44 | Print short generic help message (similar to **bpftool help**). | ||
45 | |||
46 | -v, --version | ||
47 | Print version number (similar to **bpftool version**). | ||
48 | |||
49 | -j, --json | ||
50 | Generate JSON output. For commands that cannot produce JSON, this | ||
51 | option has no effect. | ||
52 | |||
53 | -p, --pretty | ||
54 | Generate human-readable JSON output. Implies **-j**. | ||
55 | |||
56 | EXAMPLES | ||
57 | ======== | ||
58 | |||
59 | | **# bpftool perf** | ||
60 | |||
61 | :: | ||
62 | |||
63 | pid 21711 fd 5: prog_id 5 kprobe func __x64_sys_write offset 0 | ||
64 | pid 21765 fd 5: prog_id 7 kretprobe func __x64_sys_nanosleep offset 0 | ||
65 | pid 21767 fd 5: prog_id 8 tracepoint sys_enter_nanosleep | ||
66 | pid 21800 fd 5: prog_id 9 uprobe filename /home/yhs/a.out offset 1159 | ||
67 | |||
68 | | | ||
69 | | **# bpftool -j perf** | ||
70 | |||
71 | :: | ||
72 | |||
73 | [{"pid":21711,"fd":5,"prog_id":5,"fd_type":"kprobe","func":"__x64_sys_write","offset":0}, \ | ||
74 | {"pid":21765,"fd":5,"prog_id":7,"fd_type":"kretprobe","func":"__x64_sys_nanosleep","offset":0}, \ | ||
75 | {"pid":21767,"fd":5,"prog_id":8,"fd_type":"tracepoint","tracepoint":"sys_enter_nanosleep"}, \ | ||
76 | {"pid":21800,"fd":5,"prog_id":9,"fd_type":"uprobe","filename":"/home/yhs/a.out","offset":1159}] | ||
77 | |||
78 | |||
79 | SEE ALSO | ||
80 | ======== | ||
81 | **bpftool**\ (8), **bpftool-prog**\ (8), **bpftool-map**\ (8) | ||
diff --git a/tools/bpf/bpftool/Documentation/bpftool.rst b/tools/bpf/bpftool/Documentation/bpftool.rst index 564cb0d9692b..b6f5d560460d 100644 --- a/tools/bpf/bpftool/Documentation/bpftool.rst +++ b/tools/bpf/bpftool/Documentation/bpftool.rst | |||
@@ -16,7 +16,7 @@ SYNOPSIS | |||
16 | 16 | ||
17 | **bpftool** **version** | 17 | **bpftool** **version** |
18 | 18 | ||
19 | *OBJECT* := { **map** | **program** | **cgroup** } | 19 | *OBJECT* := { **map** | **program** | **cgroup** | **perf** } |
20 | 20 | ||
21 | *OPTIONS* := { { **-V** | **--version** } | { **-h** | **--help** } | 21 | *OPTIONS* := { { **-V** | **--version** } | { **-h** | **--help** } |
22 | | { **-j** | **--json** } [{ **-p** | **--pretty** }] } | 22 | | { **-j** | **--json** } [{ **-p** | **--pretty** }] } |
@@ -30,6 +30,8 @@ SYNOPSIS | |||
30 | 30 | ||
31 | *CGROUP-COMMANDS* := { **show** | **list** | **attach** | **detach** | **help** } | 31 | *CGROUP-COMMANDS* := { **show** | **list** | **attach** | **detach** | **help** } |
32 | 32 | ||
33 | *PERF-COMMANDS* := { **show** | **list** | **help** } | ||
34 | |||
33 | DESCRIPTION | 35 | DESCRIPTION |
34 | =========== | 36 | =========== |
35 | *bpftool* allows for inspection and simple modification of BPF objects | 37 | *bpftool* allows for inspection and simple modification of BPF objects |
@@ -56,3 +58,4 @@ OPTIONS | |||
56 | SEE ALSO | 58 | SEE ALSO |
57 | ======== | 59 | ======== |
58 | **bpftool-map**\ (8), **bpftool-prog**\ (8), **bpftool-cgroup**\ (8) | 60 | **bpftool-map**\ (8), **bpftool-prog**\ (8), **bpftool-cgroup**\ (8) |
61 | **bpftool-perf**\ (8) | ||
diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index b301c9b315f1..7bc198d60de2 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool | |||
@@ -448,6 +448,15 @@ _bpftool() | |||
448 | ;; | 448 | ;; |
449 | esac | 449 | esac |
450 | ;; | 450 | ;; |
451 | perf) | ||
452 | case $command in | ||
453 | *) | ||
454 | [[ $prev == $object ]] && \ | ||
455 | COMPREPLY=( $( compgen -W 'help \ | ||
456 | show list' -- "$cur" ) ) | ||
457 | ;; | ||
458 | esac | ||
459 | ;; | ||
451 | esac | 460 | esac |
452 | } && | 461 | } && |
453 | complete -F _bpftool bpftool | 462 | complete -F _bpftool bpftool |
diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 1ec852d21d44..eea7f14355f3 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c | |||
@@ -87,7 +87,7 @@ static int do_help(int argc, char **argv) | |||
87 | " %s batch file FILE\n" | 87 | " %s batch file FILE\n" |
88 | " %s version\n" | 88 | " %s version\n" |
89 | "\n" | 89 | "\n" |
90 | " OBJECT := { prog | map | cgroup }\n" | 90 | " OBJECT := { prog | map | cgroup | perf }\n" |
91 | " " HELP_SPEC_OPTIONS "\n" | 91 | " " HELP_SPEC_OPTIONS "\n" |
92 | "", | 92 | "", |
93 | bin_name, bin_name, bin_name); | 93 | bin_name, bin_name, bin_name); |
@@ -216,6 +216,7 @@ static const struct cmd cmds[] = { | |||
216 | { "prog", do_prog }, | 216 | { "prog", do_prog }, |
217 | { "map", do_map }, | 217 | { "map", do_map }, |
218 | { "cgroup", do_cgroup }, | 218 | { "cgroup", do_cgroup }, |
219 | { "perf", do_perf }, | ||
219 | { "version", do_version }, | 220 | { "version", do_version }, |
220 | { 0 } | 221 | { 0 } |
221 | }; | 222 | }; |
diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 6173cd997e7a..63fdb310b9a4 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h | |||
@@ -119,6 +119,7 @@ int do_prog(int argc, char **arg); | |||
119 | int do_map(int argc, char **arg); | 119 | int do_map(int argc, char **arg); |
120 | int do_event_pipe(int argc, char **argv); | 120 | int do_event_pipe(int argc, char **argv); |
121 | int do_cgroup(int argc, char **arg); | 121 | int do_cgroup(int argc, char **arg); |
122 | int do_perf(int argc, char **arg); | ||
122 | 123 | ||
123 | int prog_parse_fd(int *argc, char ***argv); | 124 | int prog_parse_fd(int *argc, char ***argv); |
124 | int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len); | 125 | int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len); |
diff --git a/tools/bpf/bpftool/perf.c b/tools/bpf/bpftool/perf.c new file mode 100644 index 000000000000..ac6b1a12c9b7 --- /dev/null +++ b/tools/bpf/bpftool/perf.c | |||
@@ -0,0 +1,246 @@ | |||
1 | // SPDX-License-Identifier: GPL-2.0+ | ||
2 | // Copyright (C) 2018 Facebook | ||
3 | // Author: Yonghong Song <yhs@fb.com> | ||
4 | |||
5 | #define _GNU_SOURCE | ||
6 | #include <ctype.h> | ||
7 | #include <errno.h> | ||
8 | #include <fcntl.h> | ||
9 | #include <stdlib.h> | ||
10 | #include <string.h> | ||
11 | #include <sys/stat.h> | ||
12 | #include <sys/types.h> | ||
13 | #include <unistd.h> | ||
14 | #include <ftw.h> | ||
15 | |||
16 | #include <bpf.h> | ||
17 | |||
18 | #include "main.h" | ||
19 | |||
20 | /* 0: undecided, 1: supported, 2: not supported */ | ||
21 | static int perf_query_supported; | ||
22 | static bool has_perf_query_support(void) | ||
23 | { | ||
24 | __u64 probe_offset, probe_addr; | ||
25 | __u32 len, prog_id, fd_type; | ||
26 | char buf[256]; | ||
27 | int fd; | ||
28 | |||
29 | if (perf_query_supported) | ||
30 | goto out; | ||
31 | |||
32 | fd = open(bin_name, O_RDONLY); | ||
33 | if (fd < 0) { | ||
34 | p_err("perf_query_support: %s", strerror(errno)); | ||
35 | goto out; | ||
36 | } | ||
37 | |||
38 | /* the following query will fail as no bpf attachment, | ||
39 | * the expected errno is ENOTSUPP | ||
40 | */ | ||
41 | errno = 0; | ||
42 | len = sizeof(buf); | ||
43 | bpf_task_fd_query(getpid(), fd, 0, buf, &len, &prog_id, | ||
44 | &fd_type, &probe_offset, &probe_addr); | ||
45 | |||
46 | if (errno == 524 /* ENOTSUPP */) { | ||
47 | perf_query_supported = 1; | ||
48 | goto close_fd; | ||
49 | } | ||
50 | |||
51 | perf_query_supported = 2; | ||
52 | p_err("perf_query_support: %s", strerror(errno)); | ||
53 | fprintf(stderr, | ||
54 | "HINT: non root or kernel doesn't support TASK_FD_QUERY\n"); | ||
55 | |||
56 | close_fd: | ||
57 | close(fd); | ||
58 | out: | ||
59 | return perf_query_supported == 1; | ||
60 | } | ||
61 | |||
62 | static void print_perf_json(int pid, int fd, __u32 prog_id, __u32 fd_type, | ||
63 | char *buf, __u64 probe_offset, __u64 probe_addr) | ||
64 | { | ||
65 | jsonw_start_object(json_wtr); | ||
66 | jsonw_int_field(json_wtr, "pid", pid); | ||
67 | jsonw_int_field(json_wtr, "fd", fd); | ||
68 | jsonw_uint_field(json_wtr, "prog_id", prog_id); | ||
69 | switch (fd_type) { | ||
70 | case BPF_FD_TYPE_RAW_TRACEPOINT: | ||
71 | jsonw_string_field(json_wtr, "fd_type", "raw_tracepoint"); | ||
72 | jsonw_string_field(json_wtr, "tracepoint", buf); | ||
73 | break; | ||
74 | case BPF_FD_TYPE_TRACEPOINT: | ||
75 | jsonw_string_field(json_wtr, "fd_type", "tracepoint"); | ||
76 | jsonw_string_field(json_wtr, "tracepoint", buf); | ||
77 | break; | ||
78 | case BPF_FD_TYPE_KPROBE: | ||
79 | jsonw_string_field(json_wtr, "fd_type", "kprobe"); | ||
80 | if (buf[0] != '\0') { | ||
81 | jsonw_string_field(json_wtr, "func", buf); | ||
82 | jsonw_lluint_field(json_wtr, "offset", probe_offset); | ||
83 | } else { | ||
84 | jsonw_lluint_field(json_wtr, "addr", probe_addr); | ||
85 | } | ||
86 | break; | ||
87 | case BPF_FD_TYPE_KRETPROBE: | ||
88 | jsonw_string_field(json_wtr, "fd_type", "kretprobe"); | ||
89 | if (buf[0] != '\0') { | ||
90 | jsonw_string_field(json_wtr, "func", buf); | ||
91 | jsonw_lluint_field(json_wtr, "offset", probe_offset); | ||
92 | } else { | ||
93 | jsonw_lluint_field(json_wtr, "addr", probe_addr); | ||
94 | } | ||
95 | break; | ||
96 | case BPF_FD_TYPE_UPROBE: | ||
97 | jsonw_string_field(json_wtr, "fd_type", "uprobe"); | ||
98 | jsonw_string_field(json_wtr, "filename", buf); | ||
99 | jsonw_lluint_field(json_wtr, "offset", probe_offset); | ||
100 | break; | ||
101 | case BPF_FD_TYPE_URETPROBE: | ||
102 | jsonw_string_field(json_wtr, "fd_type", "uretprobe"); | ||
103 | jsonw_string_field(json_wtr, "filename", buf); | ||
104 | jsonw_lluint_field(json_wtr, "offset", probe_offset); | ||
105 | break; | ||
106 | } | ||
107 | jsonw_end_object(json_wtr); | ||
108 | } | ||
109 | |||
110 | static void print_perf_plain(int pid, int fd, __u32 prog_id, __u32 fd_type, | ||
111 | char *buf, __u64 probe_offset, __u64 probe_addr) | ||
112 | { | ||
113 | printf("pid %d fd %d: prog_id %u ", pid, fd, prog_id); | ||
114 | switch (fd_type) { | ||
115 | case BPF_FD_TYPE_RAW_TRACEPOINT: | ||
116 | printf("raw_tracepoint %s\n", buf); | ||
117 | break; | ||
118 | case BPF_FD_TYPE_TRACEPOINT: | ||
119 | printf("tracepoint %s\n", buf); | ||
120 | break; | ||
121 | case BPF_FD_TYPE_KPROBE: | ||
122 | if (buf[0] != '\0') | ||
123 | printf("kprobe func %s offset %llu\n", buf, | ||
124 | probe_offset); | ||
125 | else | ||
126 | printf("kprobe addr %llu\n", probe_addr); | ||
127 | break; | ||
128 | case BPF_FD_TYPE_KRETPROBE: | ||
129 | if (buf[0] != '\0') | ||
130 | printf("kretprobe func %s offset %llu\n", buf, | ||
131 | probe_offset); | ||
132 | else | ||
133 | printf("kretprobe addr %llu\n", probe_addr); | ||
134 | break; | ||
135 | case BPF_FD_TYPE_UPROBE: | ||
136 | printf("uprobe filename %s offset %llu\n", buf, probe_offset); | ||
137 | break; | ||
138 | case BPF_FD_TYPE_URETPROBE: | ||
139 | printf("uretprobe filename %s offset %llu\n", buf, | ||
140 | probe_offset); | ||
141 | break; | ||
142 | } | ||
143 | } | ||
144 | |||
145 | static int show_proc(const char *fpath, const struct stat *sb, | ||
146 | int tflag, struct FTW *ftwbuf) | ||
147 | { | ||
148 | __u64 probe_offset, probe_addr; | ||
149 | __u32 len, prog_id, fd_type; | ||
150 | int err, pid = 0, fd = 0; | ||
151 | const char *pch; | ||
152 | char buf[4096]; | ||
153 | |||
154 | /* prefix always /proc */ | ||
155 | pch = fpath + 5; | ||
156 | if (*pch == '\0') | ||
157 | return 0; | ||
158 | |||
159 | /* pid should be all numbers */ | ||
160 | pch++; | ||
161 | while (isdigit(*pch)) { | ||
162 | pid = pid * 10 + *pch - '0'; | ||
163 | pch++; | ||
164 | } | ||
165 | if (*pch == '\0') | ||
166 | return 0; | ||
167 | if (*pch != '/') | ||
168 | return FTW_SKIP_SUBTREE; | ||
169 | |||
170 | /* check /proc/<pid>/fd directory */ | ||
171 | pch++; | ||
172 | if (strncmp(pch, "fd", 2)) | ||
173 | return FTW_SKIP_SUBTREE; | ||
174 | pch += 2; | ||
175 | if (*pch == '\0') | ||
176 | return 0; | ||
177 | if (*pch != '/') | ||
178 | return FTW_SKIP_SUBTREE; | ||
179 | |||
180 | /* check /proc/<pid>/fd/<fd_num> */ | ||
181 | pch++; | ||
182 | while (isdigit(*pch)) { | ||
183 | fd = fd * 10 + *pch - '0'; | ||
184 | pch++; | ||
185 | } | ||
186 | if (*pch != '\0') | ||
187 | return FTW_SKIP_SUBTREE; | ||
188 | |||
189 | /* query (pid, fd) for potential perf events */ | ||
190 | len = sizeof(buf); | ||
191 | err = bpf_task_fd_query(pid, fd, 0, buf, &len, &prog_id, &fd_type, | ||
192 | &probe_offset, &probe_addr); | ||
193 | if (err < 0) | ||
194 | return 0; | ||
195 | |||
196 | if (json_output) | ||
197 | print_perf_json(pid, fd, prog_id, fd_type, buf, probe_offset, | ||
198 | probe_addr); | ||
199 | else | ||
200 | print_perf_plain(pid, fd, prog_id, fd_type, buf, probe_offset, | ||
201 | probe_addr); | ||
202 | |||
203 | return 0; | ||
204 | } | ||
205 | |||
206 | static int do_show(int argc, char **argv) | ||
207 | { | ||
208 | int flags = FTW_ACTIONRETVAL | FTW_PHYS; | ||
209 | int err = 0, nopenfd = 16; | ||
210 | |||
211 | if (!has_perf_query_support()) | ||
212 | return -1; | ||
213 | |||
214 | if (json_output) | ||
215 | jsonw_start_array(json_wtr); | ||
216 | if (nftw("/proc", show_proc, nopenfd, flags) == -1) { | ||
217 | p_err("%s", strerror(errno)); | ||
218 | err = -1; | ||
219 | } | ||
220 | if (json_output) | ||
221 | jsonw_end_array(json_wtr); | ||
222 | |||
223 | return err; | ||
224 | } | ||
225 | |||
226 | static int do_help(int argc, char **argv) | ||
227 | { | ||
228 | fprintf(stderr, | ||
229 | "Usage: %s %s { show | list | help }\n" | ||
230 | "", | ||
231 | bin_name, argv[-2]); | ||
232 | |||
233 | return 0; | ||
234 | } | ||
235 | |||
236 | static const struct cmd cmds[] = { | ||
237 | { "show", do_show }, | ||
238 | { "list", do_show }, | ||
239 | { "help", do_help }, | ||
240 | { 0 } | ||
241 | }; | ||
242 | |||
243 | int do_perf(int argc, char **argv) | ||
244 | { | ||
245 | return cmd_select(cmds, argc, argv, do_help); | ||
246 | } | ||