diff options
author | Thomas Fleury <tfleury@nvidia.com> | 2018-04-16 19:52:08 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2018-05-09 16:25:18 -0400 |
commit | 703d00d730d230f9ac9970e7d2d22a7d8f0cd2d1 (patch) | |
tree | 2dec0fb4b51b79d0d6f4f1b19e9f3cbd71ba1d85 /drivers | |
parent | f9e55fbaf66c024125a19e1a773a1a4f0e9648f4 (diff) |
gpu: nvgpu: nvlink endpoint ops to common code
Move nvlink endpoint operations to common code. These operations
are invoked when handling nvlink core driver requests.
Jira VQRM-3523
Change-Id: I93024bf88a8caa3765b33c1264dde452c1a85ee3
Signed-off-by: Thomas Fleury <tfleury@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1698686
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/gpu/nvgpu/Makefile | 1 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/nvlink.c | 536 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/nvlink.c | 505 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/hal_gv100.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/nvlink_gv100.c | 3 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/linux/nvlink.h | 31 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/nvlink.h | 11 |
7 files changed, 586 insertions, 503 deletions
diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 6d0fcad0..fd5e8bb2 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile | |||
@@ -65,6 +65,7 @@ nvgpu-y := \ | |||
65 | common/mm/mm.o \ | 65 | common/mm/mm.o \ |
66 | common/bus.o \ | 66 | common/bus.o \ |
67 | common/enabled.o \ | 67 | common/enabled.o \ |
68 | common/nvlink.o \ | ||
68 | common/pramin.o \ | 69 | common/pramin.o \ |
69 | common/semaphore.o \ | 70 | common/semaphore.o \ |
70 | common/as.o \ | 71 | common/as.o \ |
diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c index 5a2f4ded..faa27239 100644 --- a/drivers/gpu/nvgpu/common/linux/nvlink.c +++ b/drivers/gpu/nvgpu/common/linux/nvlink.c | |||
@@ -20,524 +20,58 @@ | |||
20 | #include "module.h" | 20 | #include "module.h" |
21 | 21 | ||
22 | #ifdef CONFIG_TEGRA_NVLINK | 22 | #ifdef CONFIG_TEGRA_NVLINK |
23 | #include <linux/platform/tegra/tegra-nvlink.h> | 23 | int nvgpu_nvlink_read_dt_props(struct gk20a *g) |
24 | #endif | ||
25 | |||
26 | #ifdef CONFIG_TEGRA_NVLINK | ||
27 | |||
28 | /* | ||
29 | * WAR: use this function to find detault link, as only one is supported | ||
30 | * on the library for now | ||
31 | * Returns NVLINK_MAX_LINKS_SW on failure | ||
32 | */ | ||
33 | static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) | ||
34 | { | ||
35 | u32 link_id; | ||
36 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
37 | |||
38 | if (!g) | ||
39 | return NVLINK_MAX_LINKS_SW; | ||
40 | |||
41 | /* Lets find the detected link */ | ||
42 | if (g->nvlink.initialized_links) | ||
43 | link_id = ffs(g->nvlink.initialized_links) - 1; | ||
44 | else | ||
45 | return NVLINK_MAX_LINKS_SW; | ||
46 | |||
47 | if (g->nvlink.links[link_id].remote_info.is_connected) | ||
48 | return link_id; | ||
49 | |||
50 | return NVLINK_MAX_LINKS_SW; | ||
51 | } | ||
52 | |||
53 | static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) | ||
54 | { | ||
55 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
56 | int err; | ||
57 | |||
58 | /* For now master topology is the only one supported */ | ||
59 | if (!ndev->is_master) { | ||
60 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | ||
61 | "dGPU is not master of Nvlink link"); | ||
62 | return -EINVAL; | ||
63 | } | ||
64 | |||
65 | err = g->ops.nvlink.early_init(g); | ||
66 | return err; | ||
67 | } | ||
68 | |||
69 | static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev) | ||
70 | { | ||
71 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
72 | int err; | ||
73 | u32 link_id; | ||
74 | |||
75 | /* | ||
76 | * First check the topology and setup connectivity | ||
77 | * HACK: we are only enabling one link for now!!! | ||
78 | */ | ||
79 | link_id = ffs(g->nvlink.discovered_links) - 1; | ||
80 | g->nvlink.links[link_id].remote_info.is_connected = true; | ||
81 | g->nvlink.links[link_id].remote_info.device_type = | ||
82 | nvgpu_nvlink_endp_tegra; | ||
83 | err = g->ops.nvlink.link_early_init(g, BIT(link_id)); | ||
84 | |||
85 | if (err == 0) { | ||
86 | g->nvlink.links[link_id].priv = (void *) &(ndev->link); | ||
87 | ndev->link.priv = (void *) g; | ||
88 | } | ||
89 | return err; | ||
90 | } | ||
91 | |||
92 | static int nvgpu_nvlink_interface_init(struct nvlink_device *ndev) | ||
93 | { | ||
94 | int err; | ||
95 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
96 | |||
97 | err = g->ops.nvlink.interface_init(g); | ||
98 | return err; | ||
99 | } | ||
100 | |||
101 | static int nvgpu_nvlink_shutdown(struct nvlink_device *ndev) | ||
102 | { | ||
103 | int err; | ||
104 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
105 | |||
106 | err = g->ops.nvlink.shutdown(g); | ||
107 | return 0; | ||
108 | } | ||
109 | |||
110 | static int nvgpu_nvlink_reg_init(struct nvlink_device *ndev) | ||
111 | { | ||
112 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
113 | int err; | ||
114 | |||
115 | err = g->ops.nvlink.reg_init(g); | ||
116 | |||
117 | return err; | ||
118 | } | ||
119 | |||
120 | static u32 nvgpu_nvlink_get_link_mode(struct nvlink_device *ndev) | ||
121 | { | ||
122 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
123 | u32 link_id; | ||
124 | u32 mode; | ||
125 | |||
126 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
127 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
128 | return -EINVAL; | ||
129 | |||
130 | mode = g->ops.nvlink.link_get_mode(g, link_id); | ||
131 | |||
132 | switch (mode) { | ||
133 | case nvgpu_nvlink_link_off: | ||
134 | return NVLINK_LINK_OFF; | ||
135 | case nvgpu_nvlink_link_hs: | ||
136 | return NVLINK_LINK_HS; | ||
137 | case nvgpu_nvlink_link_safe: | ||
138 | return NVLINK_LINK_SAFE; | ||
139 | case nvgpu_nvlink_link_fault: | ||
140 | return NVLINK_LINK_FAULT; | ||
141 | case nvgpu_nvlink_link_rcvy_ac: | ||
142 | return NVLINK_LINK_RCVY_AC; | ||
143 | case nvgpu_nvlink_link_rcvy_sw: | ||
144 | return NVLINK_LINK_RCVY_SW; | ||
145 | case nvgpu_nvlink_link_rcvy_rx: | ||
146 | return NVLINK_LINK_RCVY_RX; | ||
147 | case nvgpu_nvlink_link_detect: | ||
148 | return NVLINK_LINK_DETECT; | ||
149 | case nvgpu_nvlink_link_reset: | ||
150 | return NVLINK_LINK_RESET; | ||
151 | case nvgpu_nvlink_link_enable_pm: | ||
152 | return NVLINK_LINK_ENABLE_PM; | ||
153 | case nvgpu_nvlink_link_disable_pm: | ||
154 | return NVLINK_LINK_DISABLE_PM; | ||
155 | case nvgpu_nvlink_link_disable_err_detect: | ||
156 | return NVLINK_LINK_DISABLE_ERR_DETECT; | ||
157 | case nvgpu_nvlink_link_lane_disable: | ||
158 | return NVLINK_LINK_LANE_DISABLE; | ||
159 | case nvgpu_nvlink_link_lane_shutdown: | ||
160 | return NVLINK_LINK_LANE_SHUTDOWN; | ||
161 | default: | ||
162 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | ||
163 | "unsupported mode %u", mode); | ||
164 | } | ||
165 | |||
166 | return NVLINK_LINK_OFF; | ||
167 | } | ||
168 | |||
169 | static u32 nvgpu_nvlink_get_link_state(struct nvlink_device *ndev) | ||
170 | { | ||
171 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
172 | u32 link_id; | ||
173 | |||
174 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
175 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
176 | return -EINVAL; | ||
177 | |||
178 | return g->ops.nvlink.link_get_state(g, link_id); | ||
179 | } | ||
180 | |||
181 | static int nvgpu_nvlink_set_link_mode(struct nvlink_device *ndev, u32 mode) | ||
182 | { | ||
183 | |||
184 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
185 | u32 link_id; | ||
186 | u32 mode_sw; | ||
187 | |||
188 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
189 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
190 | return -EINVAL; | ||
191 | |||
192 | switch (mode) { | ||
193 | case NVLINK_LINK_OFF: | ||
194 | mode_sw = nvgpu_nvlink_link_off; | ||
195 | break; | ||
196 | case NVLINK_LINK_HS: | ||
197 | mode_sw = nvgpu_nvlink_link_hs; | ||
198 | break; | ||
199 | case NVLINK_LINK_SAFE: | ||
200 | mode_sw = nvgpu_nvlink_link_safe; | ||
201 | break; | ||
202 | case NVLINK_LINK_FAULT: | ||
203 | mode_sw = nvgpu_nvlink_link_fault; | ||
204 | break; | ||
205 | case NVLINK_LINK_RCVY_AC: | ||
206 | mode_sw = nvgpu_nvlink_link_rcvy_ac; | ||
207 | break; | ||
208 | case NVLINK_LINK_RCVY_SW: | ||
209 | mode_sw = nvgpu_nvlink_link_rcvy_sw; | ||
210 | break; | ||
211 | case NVLINK_LINK_RCVY_RX: | ||
212 | mode_sw = nvgpu_nvlink_link_rcvy_rx; | ||
213 | break; | ||
214 | case NVLINK_LINK_DETECT: | ||
215 | mode_sw = nvgpu_nvlink_link_detect; | ||
216 | break; | ||
217 | case NVLINK_LINK_RESET: | ||
218 | mode_sw = nvgpu_nvlink_link_reset; | ||
219 | break; | ||
220 | case NVLINK_LINK_ENABLE_PM: | ||
221 | mode_sw = nvgpu_nvlink_link_enable_pm; | ||
222 | break; | ||
223 | case NVLINK_LINK_DISABLE_PM: | ||
224 | mode_sw = nvgpu_nvlink_link_disable_pm; | ||
225 | break; | ||
226 | case NVLINK_LINK_LANE_DISABLE: | ||
227 | mode_sw = nvgpu_nvlink_link_lane_disable; | ||
228 | break; | ||
229 | case NVLINK_LINK_LANE_SHUTDOWN: | ||
230 | mode_sw = nvgpu_nvlink_link_lane_shutdown; | ||
231 | break; | ||
232 | default: | ||
233 | mode_sw = nvgpu_nvlink_link_off; | ||
234 | } | ||
235 | |||
236 | return g->ops.nvlink.link_set_mode(g, link_id, mode_sw); | ||
237 | } | ||
238 | |||
239 | static void nvgpu_nvlink_get_tx_sublink_state(struct nvlink_device *ndev, | ||
240 | u32 *state) | ||
241 | { | ||
242 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
243 | u32 link_id; | ||
244 | |||
245 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
246 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
247 | return; | ||
248 | if (state) | ||
249 | *state = g->ops.nvlink.get_tx_sublink_state(g, link_id); | ||
250 | } | ||
251 | |||
252 | static void nvgpu_nvlink_get_rx_sublink_state(struct nvlink_device *ndev, | ||
253 | u32 *state) | ||
254 | { | ||
255 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
256 | u32 link_id; | ||
257 | |||
258 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
259 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
260 | return; | ||
261 | if (state) | ||
262 | *state = g->ops.nvlink.get_rx_sublink_state(g, link_id); | ||
263 | } | ||
264 | |||
265 | static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev, | ||
266 | bool is_rx_sublink) | ||
267 | { | ||
268 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
269 | u32 link_id; | ||
270 | u32 mode; | ||
271 | |||
272 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
273 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
274 | return -EINVAL; | ||
275 | |||
276 | mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); | ||
277 | |||
278 | switch (mode) { | ||
279 | case nvgpu_nvlink_sublink_tx_hs: | ||
280 | return NVLINK_TX_HS; | ||
281 | case nvgpu_nvlink_sublink_tx_off: | ||
282 | return NVLINK_TX_OFF; | ||
283 | case nvgpu_nvlink_sublink_tx_single_lane: | ||
284 | return NVLINK_TX_SINGLE_LANE; | ||
285 | case nvgpu_nvlink_sublink_tx_safe: | ||
286 | return NVLINK_TX_SAFE; | ||
287 | case nvgpu_nvlink_sublink_tx_enable_pm: | ||
288 | return NVLINK_TX_ENABLE_PM; | ||
289 | case nvgpu_nvlink_sublink_tx_disable_pm: | ||
290 | return NVLINK_TX_DISABLE_PM; | ||
291 | case nvgpu_nvlink_sublink_tx_common: | ||
292 | return NVLINK_TX_COMMON; | ||
293 | case nvgpu_nvlink_sublink_tx_common_disable: | ||
294 | return NVLINK_TX_COMMON_DISABLE; | ||
295 | case nvgpu_nvlink_sublink_tx_data_ready: | ||
296 | return NVLINK_TX_DATA_READY; | ||
297 | case nvgpu_nvlink_sublink_tx_prbs_en: | ||
298 | return NVLINK_TX_PRBS_EN; | ||
299 | case nvgpu_nvlink_sublink_rx_hs: | ||
300 | return NVLINK_RX_HS; | ||
301 | case nvgpu_nvlink_sublink_rx_enable_pm: | ||
302 | return NVLINK_RX_ENABLE_PM; | ||
303 | case nvgpu_nvlink_sublink_rx_disable_pm: | ||
304 | return NVLINK_RX_DISABLE_PM; | ||
305 | case nvgpu_nvlink_sublink_rx_single_lane: | ||
306 | return NVLINK_RX_SINGLE_LANE; | ||
307 | case nvgpu_nvlink_sublink_rx_safe: | ||
308 | return NVLINK_RX_SAFE; | ||
309 | case nvgpu_nvlink_sublink_rx_off: | ||
310 | return NVLINK_RX_OFF; | ||
311 | case nvgpu_nvlink_sublink_rx_rxcal: | ||
312 | return NVLINK_RX_RXCAL; | ||
313 | default: | ||
314 | nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode); | ||
315 | break; | ||
316 | } | ||
317 | |||
318 | if (is_rx_sublink) | ||
319 | return NVLINK_RX_OFF; | ||
320 | return NVLINK_TX_OFF; | ||
321 | } | ||
322 | |||
323 | static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev, | ||
324 | bool is_rx_sublink, u32 mode) | ||
325 | { | ||
326 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
327 | u32 link_id; | ||
328 | u32 mode_sw; | ||
329 | |||
330 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
331 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
332 | return -EINVAL; | ||
333 | |||
334 | if (!is_rx_sublink) { | ||
335 | switch (mode) { | ||
336 | case NVLINK_TX_HS: | ||
337 | mode_sw = nvgpu_nvlink_sublink_tx_hs; | ||
338 | break; | ||
339 | case NVLINK_TX_ENABLE_PM: | ||
340 | mode_sw = nvgpu_nvlink_sublink_tx_enable_pm; | ||
341 | break; | ||
342 | case NVLINK_TX_DISABLE_PM: | ||
343 | mode_sw = nvgpu_nvlink_sublink_tx_disable_pm; | ||
344 | break; | ||
345 | case NVLINK_TX_SINGLE_LANE: | ||
346 | mode_sw = nvgpu_nvlink_sublink_tx_single_lane; | ||
347 | break; | ||
348 | case NVLINK_TX_SAFE: | ||
349 | mode_sw = nvgpu_nvlink_sublink_tx_safe; | ||
350 | break; | ||
351 | case NVLINK_TX_OFF: | ||
352 | mode_sw = nvgpu_nvlink_sublink_tx_off; | ||
353 | break; | ||
354 | case NVLINK_TX_COMMON: | ||
355 | mode_sw = nvgpu_nvlink_sublink_tx_common; | ||
356 | break; | ||
357 | case NVLINK_TX_COMMON_DISABLE: | ||
358 | mode_sw = nvgpu_nvlink_sublink_tx_common_disable; | ||
359 | break; | ||
360 | case NVLINK_TX_DATA_READY: | ||
361 | mode_sw = nvgpu_nvlink_sublink_tx_data_ready; | ||
362 | break; | ||
363 | case NVLINK_TX_PRBS_EN: | ||
364 | mode_sw = nvgpu_nvlink_sublink_tx_prbs_en; | ||
365 | break; | ||
366 | default: | ||
367 | return -EINVAL; | ||
368 | } | ||
369 | } else { | ||
370 | switch (mode) { | ||
371 | case NVLINK_RX_HS: | ||
372 | mode_sw = nvgpu_nvlink_sublink_rx_hs; | ||
373 | break; | ||
374 | case NVLINK_RX_ENABLE_PM: | ||
375 | mode_sw = nvgpu_nvlink_sublink_rx_enable_pm; | ||
376 | break; | ||
377 | case NVLINK_RX_DISABLE_PM: | ||
378 | mode_sw = nvgpu_nvlink_sublink_rx_disable_pm; | ||
379 | break; | ||
380 | case NVLINK_RX_SINGLE_LANE: | ||
381 | mode_sw = nvgpu_nvlink_sublink_rx_single_lane; | ||
382 | break; | ||
383 | case NVLINK_RX_SAFE: | ||
384 | mode_sw = nvgpu_nvlink_sublink_rx_safe; | ||
385 | break; | ||
386 | case NVLINK_RX_OFF: | ||
387 | mode_sw = nvgpu_nvlink_sublink_rx_off; | ||
388 | break; | ||
389 | case NVLINK_RX_RXCAL: | ||
390 | mode_sw = nvgpu_nvlink_sublink_rx_rxcal; | ||
391 | break; | ||
392 | default: | ||
393 | return -EINVAL; | ||
394 | } | ||
395 | } | ||
396 | |||
397 | return g->ops.nvlink.set_sublink_mode(g, link_id, is_rx_sublink, | ||
398 | mode_sw); | ||
399 | } | ||
400 | #endif | ||
401 | |||
402 | int nvgpu_nvlink_enumerate(struct gk20a *g) | ||
403 | { | ||
404 | int err = -ENODEV; | ||
405 | #ifdef CONFIG_TEGRA_NVLINK | ||
406 | struct nvlink_device *ndev; | ||
407 | |||
408 | ndev = (struct nvlink_device *) g->nvlink.priv; | ||
409 | |||
410 | if (ndev) | ||
411 | err = nvlink_enumerate(ndev); | ||
412 | #endif | ||
413 | return err; | ||
414 | } | ||
415 | |||
416 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) | ||
417 | { | 24 | { |
418 | int err = -ENODEV; | 25 | struct device_node *np; |
419 | #ifdef CONFIG_TEGRA_NVLINK | 26 | struct nvlink_device *ndev = g->nvlink.priv; |
420 | struct nvlink_device *ndev; | 27 | u32 local_dev_id; |
421 | 28 | u32 local_link_id; | |
422 | ndev = (struct nvlink_device *) g->nvlink.priv; | 29 | u32 remote_dev_id; |
423 | 30 | u32 remote_link_id; | |
424 | if (!ndev) | 31 | u32 physical_link; |
425 | return -ENODEV; | 32 | bool is_master; |
426 | |||
427 | /* Check if the link is connected */ | ||
428 | if (!g->nvlink.links[link_id].remote_info.is_connected) | ||
429 | return -ENODEV; | ||
430 | |||
431 | if (from_off) | ||
432 | return nvlink_transition_intranode_conn_off_to_safe(ndev); | ||
433 | return nvlink_train_intranode_conn_safe_to_hs(ndev); | ||
434 | |||
435 | #endif | ||
436 | return err; | ||
437 | } | ||
438 | |||
439 | int nvgpu_nvlink_probe(struct gk20a *g) | ||
440 | { | ||
441 | #ifdef CONFIG_TEGRA_NVLINK | ||
442 | int err = 0; | ||
443 | struct device_node *np = nvgpu_get_node(g); | ||
444 | struct device_node *nvlink_np = NULL, *endp_np = NULL; | ||
445 | struct nvlink_device *ndev; | ||
446 | u32 phys_link_id; | ||
447 | 33 | ||
448 | /* Parse DT */ | 34 | /* Parse DT */ |
449 | if (np) { | 35 | np = nvgpu_get_node(g); |
450 | nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); | 36 | if (!np) |
451 | if (nvlink_np) { | 37 | goto fail; |
452 | endp_np = of_get_child_by_name(nvlink_np, "endpoint"); | ||
453 | } | ||
454 | } | ||
455 | 38 | ||
456 | if (!endp_np) { | 39 | np = of_get_child_by_name(np, "nvidia,nvlink"); |
457 | nvgpu_info(g, "no nvlink DT detected"); | 40 | if (!np) |
458 | return -ENODEV; | 41 | goto fail; |
459 | } | ||
460 | 42 | ||
461 | /* Allocating structures */ | 43 | np = of_get_child_by_name(np, "endpoint"); |
462 | ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device)); | 44 | if (!np) |
463 | if (!ndev) { | 45 | goto fail; |
464 | nvgpu_err(g, "OOM while allocating nvlink device struct"); | ||
465 | return -ENOMEM; | ||
466 | } | ||
467 | |||
468 | ndev->priv = (void *) g; | ||
469 | g->nvlink.priv = (void *) ndev; | ||
470 | 46 | ||
471 | /* Parse DT structure to detect endpoint topology */ | 47 | /* Parse DT structure to detect endpoint topology */ |
472 | of_property_read_u32(endp_np, "local_dev_id", &ndev->device_id); | 48 | of_property_read_u32(np, "local_dev_id", &local_dev_id); |
473 | of_property_read_u32(endp_np, "local_link_id", &ndev->link.link_id); | 49 | of_property_read_u32(np, "local_link_id", &local_link_id); |
474 | ndev->is_master = of_property_read_bool(endp_np, "is_master"); | 50 | of_property_read_u32(np, "remote_dev_id", &remote_dev_id); |
475 | of_property_read_u32(endp_np, "remote_dev_id", | 51 | of_property_read_u32(np, "remote_link_id", &remote_link_id); |
476 | &ndev->link.remote_dev_info.device_id); | 52 | of_property_read_u32(np, "physical_link", &physical_link); |
477 | of_property_read_u32(endp_np, "remote_link_id", | 53 | is_master = of_property_read_bool(np, "is_master"); |
478 | &ndev->link.remote_dev_info.link_id); | ||
479 | of_property_read_u32(endp_np, "physical_link", | ||
480 | &phys_link_id); | ||
481 | |||
482 | g->nvlink.connected_links = BIT(phys_link_id); | ||
483 | 54 | ||
484 | /* Check that we are in dGPU mode */ | 55 | /* Check that we are in dGPU mode */ |
485 | if (ndev->device_id != NVLINK_ENDPT_GV100) { | 56 | if (local_dev_id != NVLINK_ENDPT_GV100) { |
486 | nvgpu_err(g, "Local nvlink device is not dGPU"); | 57 | nvgpu_err(g, "Local nvlink device is not dGPU"); |
487 | err = -EINVAL; | 58 | return -EINVAL; |
488 | goto free_ndev; | ||
489 | } | ||
490 | |||
491 | /* Fill in device struct */ | ||
492 | ndev->dev_ops.dev_early_init = nvgpu_nvlink_early_init; | ||
493 | ndev->dev_ops.dev_interface_init = nvgpu_nvlink_interface_init; | ||
494 | ndev->dev_ops.dev_reg_init = nvgpu_nvlink_reg_init; | ||
495 | ndev->dev_ops.dev_shutdown = nvgpu_nvlink_shutdown; | ||
496 | |||
497 | /* Fill in the link struct */ | ||
498 | ndev->link.device_id = ndev->device_id; | ||
499 | ndev->link.mode = NVLINK_LINK_OFF; | ||
500 | ndev->link.link_ops.get_link_mode = nvgpu_nvlink_get_link_mode; | ||
501 | ndev->link.link_ops.set_link_mode = nvgpu_nvlink_set_link_mode; | ||
502 | ndev->link.link_ops.get_sublink_mode = nvgpu_nvlink_get_sublink_mode; | ||
503 | ndev->link.link_ops.set_sublink_mode = nvgpu_nvlink_set_sublink_mode; | ||
504 | ndev->link.link_ops.get_link_state = nvgpu_nvlink_get_link_state; | ||
505 | ndev->link.link_ops.get_tx_sublink_state = | ||
506 | nvgpu_nvlink_get_tx_sublink_state; | ||
507 | ndev->link.link_ops.get_rx_sublink_state = | ||
508 | nvgpu_nvlink_get_rx_sublink_state; | ||
509 | ndev->link.link_ops.link_early_init = | ||
510 | nvgpu_nvlink_link_early_init; | ||
511 | |||
512 | /* Register device with core driver*/ | ||
513 | err = nvlink_register_device(ndev); | ||
514 | if (err) { | ||
515 | nvgpu_err(g, "failed on nvlink device registration"); | ||
516 | goto free_ndev; | ||
517 | } | 59 | } |
518 | 60 | ||
519 | /* Register link with core driver */ | 61 | ndev->is_master = is_master; |
520 | err = nvlink_register_link(&ndev->link); | 62 | ndev->device_id = local_dev_id; |
521 | if (err) { | 63 | ndev->link.link_id = local_link_id; |
522 | nvgpu_err(g, "failed on nvlink link registration"); | 64 | ndev->link.remote_dev_info.device_id = remote_dev_id; |
523 | goto unregister_ndev; | 65 | ndev->link.remote_dev_info.link_id = remote_link_id; |
524 | } | ||
525 | 66 | ||
526 | /* Enable NVLINK support */ | 67 | g->nvlink.connected_links = BIT(physical_link); |
527 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); | ||
528 | return 0; | 68 | return 0; |
529 | 69 | ||
530 | unregister_ndev: | 70 | fail: |
531 | nvlink_unregister_device(ndev); | 71 | nvgpu_info(g, "nvlink endpoint not found or invaling in DT"); |
532 | |||
533 | free_ndev: | ||
534 | nvgpu_kfree(g, ndev); | ||
535 | g->nvlink.priv = NULL; | ||
536 | return err; | ||
537 | #else | ||
538 | return -ENODEV; | 72 | return -ENODEV; |
539 | #endif | ||
540 | } | 73 | } |
74 | #endif /* CONFIG_TEGRA_NVLINK */ | ||
541 | 75 | ||
542 | void nvgpu_mss_nvlink_init_credits(struct gk20a *g) | 76 | void nvgpu_mss_nvlink_init_credits(struct gk20a *g) |
543 | { | 77 | { |
diff --git a/drivers/gpu/nvgpu/common/nvlink.c b/drivers/gpu/nvgpu/common/nvlink.c new file mode 100644 index 00000000..25c1b10b --- /dev/null +++ b/drivers/gpu/nvgpu/common/nvlink.c | |||
@@ -0,0 +1,505 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #include <gk20a/gk20a.h> | ||
18 | #include <nvgpu/nvlink.h> | ||
19 | #include <nvgpu/enabled.h> | ||
20 | |||
21 | #ifdef CONFIG_TEGRA_NVLINK | ||
22 | /* | ||
23 | * WAR: use this function to find detault link, as only one is supported | ||
24 | * on the library for now | ||
25 | * Returns NVLINK_MAX_LINKS_SW on failure | ||
26 | */ | ||
27 | static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) | ||
28 | { | ||
29 | u32 link_id; | ||
30 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
31 | |||
32 | if (!g) | ||
33 | return NVLINK_MAX_LINKS_SW; | ||
34 | |||
35 | /* Lets find the detected link */ | ||
36 | if (g->nvlink.initialized_links) | ||
37 | link_id = ffs(g->nvlink.initialized_links) - 1; | ||
38 | else | ||
39 | return NVLINK_MAX_LINKS_SW; | ||
40 | |||
41 | if (g->nvlink.links[link_id].remote_info.is_connected) | ||
42 | return link_id; | ||
43 | |||
44 | return NVLINK_MAX_LINKS_SW; | ||
45 | } | ||
46 | |||
47 | static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) | ||
48 | { | ||
49 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
50 | int err; | ||
51 | |||
52 | /* For now master topology is the only one supported */ | ||
53 | if (!ndev->is_master) { | ||
54 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | ||
55 | "dGPU is not master of Nvlink link"); | ||
56 | return -EINVAL; | ||
57 | } | ||
58 | |||
59 | err = g->ops.nvlink.early_init(g); | ||
60 | return err; | ||
61 | } | ||
62 | |||
63 | static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev) | ||
64 | { | ||
65 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
66 | int err; | ||
67 | u32 link_id; | ||
68 | |||
69 | /* | ||
70 | * First check the topology and setup connectivity | ||
71 | * HACK: we are only enabling one link for now!!! | ||
72 | */ | ||
73 | link_id = ffs(g->nvlink.discovered_links) - 1; | ||
74 | g->nvlink.links[link_id].remote_info.is_connected = true; | ||
75 | g->nvlink.links[link_id].remote_info.device_type = | ||
76 | nvgpu_nvlink_endp_tegra; | ||
77 | err = g->ops.nvlink.link_early_init(g, BIT(link_id)); | ||
78 | |||
79 | if (err == 0) { | ||
80 | g->nvlink.links[link_id].priv = (void *) &(ndev->link); | ||
81 | ndev->link.priv = (void *) g; | ||
82 | } | ||
83 | return err; | ||
84 | } | ||
85 | |||
86 | static int nvgpu_nvlink_interface_init(struct nvlink_device *ndev) | ||
87 | { | ||
88 | int err; | ||
89 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
90 | |||
91 | err = g->ops.nvlink.interface_init(g); | ||
92 | return err; | ||
93 | } | ||
94 | |||
95 | static int nvgpu_nvlink_shutdown(struct nvlink_device *ndev) | ||
96 | { | ||
97 | int err; | ||
98 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
99 | |||
100 | err = g->ops.nvlink.shutdown(g); | ||
101 | return 0; | ||
102 | } | ||
103 | |||
104 | static int nvgpu_nvlink_reg_init(struct nvlink_device *ndev) | ||
105 | { | ||
106 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
107 | int err; | ||
108 | |||
109 | err = g->ops.nvlink.reg_init(g); | ||
110 | |||
111 | return err; | ||
112 | } | ||
113 | |||
114 | static u32 nvgpu_nvlink_get_link_mode(struct nvlink_device *ndev) | ||
115 | { | ||
116 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
117 | u32 link_id; | ||
118 | u32 mode; | ||
119 | |||
120 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
121 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
122 | return -EINVAL; | ||
123 | |||
124 | mode = g->ops.nvlink.link_get_mode(g, link_id); | ||
125 | |||
126 | switch (mode) { | ||
127 | case nvgpu_nvlink_link_off: | ||
128 | return NVLINK_LINK_OFF; | ||
129 | case nvgpu_nvlink_link_hs: | ||
130 | return NVLINK_LINK_HS; | ||
131 | case nvgpu_nvlink_link_safe: | ||
132 | return NVLINK_LINK_SAFE; | ||
133 | case nvgpu_nvlink_link_fault: | ||
134 | return NVLINK_LINK_FAULT; | ||
135 | case nvgpu_nvlink_link_rcvy_ac: | ||
136 | return NVLINK_LINK_RCVY_AC; | ||
137 | case nvgpu_nvlink_link_rcvy_sw: | ||
138 | return NVLINK_LINK_RCVY_SW; | ||
139 | case nvgpu_nvlink_link_rcvy_rx: | ||
140 | return NVLINK_LINK_RCVY_RX; | ||
141 | case nvgpu_nvlink_link_detect: | ||
142 | return NVLINK_LINK_DETECT; | ||
143 | case nvgpu_nvlink_link_reset: | ||
144 | return NVLINK_LINK_RESET; | ||
145 | case nvgpu_nvlink_link_enable_pm: | ||
146 | return NVLINK_LINK_ENABLE_PM; | ||
147 | case nvgpu_nvlink_link_disable_pm: | ||
148 | return NVLINK_LINK_DISABLE_PM; | ||
149 | case nvgpu_nvlink_link_disable_err_detect: | ||
150 | return NVLINK_LINK_DISABLE_ERR_DETECT; | ||
151 | case nvgpu_nvlink_link_lane_disable: | ||
152 | return NVLINK_LINK_LANE_DISABLE; | ||
153 | case nvgpu_nvlink_link_lane_shutdown: | ||
154 | return NVLINK_LINK_LANE_SHUTDOWN; | ||
155 | default: | ||
156 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | ||
157 | "unsupported mode %u", mode); | ||
158 | } | ||
159 | |||
160 | return NVLINK_LINK_OFF; | ||
161 | } | ||
162 | |||
163 | static u32 nvgpu_nvlink_get_link_state(struct nvlink_device *ndev) | ||
164 | { | ||
165 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
166 | u32 link_id; | ||
167 | |||
168 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
169 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
170 | return -EINVAL; | ||
171 | |||
172 | return g->ops.nvlink.link_get_state(g, link_id); | ||
173 | } | ||
174 | |||
175 | static int nvgpu_nvlink_set_link_mode(struct nvlink_device *ndev, u32 mode) | ||
176 | { | ||
177 | |||
178 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
179 | u32 link_id; | ||
180 | u32 mode_sw; | ||
181 | |||
182 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
183 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
184 | return -EINVAL; | ||
185 | |||
186 | switch (mode) { | ||
187 | case NVLINK_LINK_OFF: | ||
188 | mode_sw = nvgpu_nvlink_link_off; | ||
189 | break; | ||
190 | case NVLINK_LINK_HS: | ||
191 | mode_sw = nvgpu_nvlink_link_hs; | ||
192 | break; | ||
193 | case NVLINK_LINK_SAFE: | ||
194 | mode_sw = nvgpu_nvlink_link_safe; | ||
195 | break; | ||
196 | case NVLINK_LINK_FAULT: | ||
197 | mode_sw = nvgpu_nvlink_link_fault; | ||
198 | break; | ||
199 | case NVLINK_LINK_RCVY_AC: | ||
200 | mode_sw = nvgpu_nvlink_link_rcvy_ac; | ||
201 | break; | ||
202 | case NVLINK_LINK_RCVY_SW: | ||
203 | mode_sw = nvgpu_nvlink_link_rcvy_sw; | ||
204 | break; | ||
205 | case NVLINK_LINK_RCVY_RX: | ||
206 | mode_sw = nvgpu_nvlink_link_rcvy_rx; | ||
207 | break; | ||
208 | case NVLINK_LINK_DETECT: | ||
209 | mode_sw = nvgpu_nvlink_link_detect; | ||
210 | break; | ||
211 | case NVLINK_LINK_RESET: | ||
212 | mode_sw = nvgpu_nvlink_link_reset; | ||
213 | break; | ||
214 | case NVLINK_LINK_ENABLE_PM: | ||
215 | mode_sw = nvgpu_nvlink_link_enable_pm; | ||
216 | break; | ||
217 | case NVLINK_LINK_DISABLE_PM: | ||
218 | mode_sw = nvgpu_nvlink_link_disable_pm; | ||
219 | break; | ||
220 | case NVLINK_LINK_LANE_DISABLE: | ||
221 | mode_sw = nvgpu_nvlink_link_lane_disable; | ||
222 | break; | ||
223 | case NVLINK_LINK_LANE_SHUTDOWN: | ||
224 | mode_sw = nvgpu_nvlink_link_lane_shutdown; | ||
225 | break; | ||
226 | default: | ||
227 | mode_sw = nvgpu_nvlink_link_off; | ||
228 | } | ||
229 | |||
230 | return g->ops.nvlink.link_set_mode(g, link_id, mode_sw); | ||
231 | } | ||
232 | |||
233 | static void nvgpu_nvlink_get_tx_sublink_state(struct nvlink_device *ndev, u32 *state) | ||
234 | { | ||
235 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
236 | u32 link_id; | ||
237 | |||
238 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
239 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
240 | return; | ||
241 | if (state) | ||
242 | *state = g->ops.nvlink.get_tx_sublink_state(g, link_id); | ||
243 | } | ||
244 | |||
245 | static void nvgpu_nvlink_get_rx_sublink_state(struct nvlink_device *ndev, u32 *state) | ||
246 | { | ||
247 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
248 | u32 link_id; | ||
249 | |||
250 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
251 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
252 | return; | ||
253 | if (state) | ||
254 | *state = g->ops.nvlink.get_rx_sublink_state(g, link_id); | ||
255 | } | ||
256 | |||
257 | static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev, | ||
258 | bool is_rx_sublink) | ||
259 | { | ||
260 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
261 | u32 link_id; | ||
262 | u32 mode; | ||
263 | |||
264 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
265 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
266 | return -EINVAL; | ||
267 | |||
268 | mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); | ||
269 | |||
270 | switch (mode) { | ||
271 | case nvgpu_nvlink_sublink_tx_hs: | ||
272 | return NVLINK_TX_HS; | ||
273 | case nvgpu_nvlink_sublink_tx_off: | ||
274 | return NVLINK_TX_OFF; | ||
275 | case nvgpu_nvlink_sublink_tx_single_lane: | ||
276 | return NVLINK_TX_SINGLE_LANE; | ||
277 | case nvgpu_nvlink_sublink_tx_safe: | ||
278 | return NVLINK_TX_SAFE; | ||
279 | case nvgpu_nvlink_sublink_tx_enable_pm: | ||
280 | return NVLINK_TX_ENABLE_PM; | ||
281 | case nvgpu_nvlink_sublink_tx_disable_pm: | ||
282 | return NVLINK_TX_DISABLE_PM; | ||
283 | case nvgpu_nvlink_sublink_tx_common: | ||
284 | return NVLINK_TX_COMMON; | ||
285 | case nvgpu_nvlink_sublink_tx_common_disable: | ||
286 | return NVLINK_TX_COMMON_DISABLE; | ||
287 | case nvgpu_nvlink_sublink_tx_data_ready: | ||
288 | return NVLINK_TX_DATA_READY; | ||
289 | case nvgpu_nvlink_sublink_tx_prbs_en: | ||
290 | return NVLINK_TX_PRBS_EN; | ||
291 | case nvgpu_nvlink_sublink_rx_hs: | ||
292 | return NVLINK_RX_HS; | ||
293 | case nvgpu_nvlink_sublink_rx_enable_pm: | ||
294 | return NVLINK_RX_ENABLE_PM; | ||
295 | case nvgpu_nvlink_sublink_rx_disable_pm: | ||
296 | return NVLINK_RX_DISABLE_PM; | ||
297 | case nvgpu_nvlink_sublink_rx_single_lane: | ||
298 | return NVLINK_RX_SINGLE_LANE; | ||
299 | case nvgpu_nvlink_sublink_rx_safe: | ||
300 | return NVLINK_RX_SAFE; | ||
301 | case nvgpu_nvlink_sublink_rx_off: | ||
302 | return NVLINK_RX_OFF; | ||
303 | case nvgpu_nvlink_sublink_rx_rxcal: | ||
304 | return NVLINK_RX_RXCAL; | ||
305 | default: | ||
306 | nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode); | ||
307 | break; | ||
308 | } | ||
309 | |||
310 | if (is_rx_sublink) | ||
311 | return NVLINK_RX_OFF; | ||
312 | return NVLINK_TX_OFF; | ||
313 | } | ||
314 | |||
315 | static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev, | ||
316 | bool is_rx_sublink, u32 mode) | ||
317 | { | ||
318 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
319 | u32 link_id; | ||
320 | u32 mode_sw; | ||
321 | |||
322 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
323 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
324 | return -EINVAL; | ||
325 | |||
326 | if (!is_rx_sublink) { | ||
327 | switch (mode) { | ||
328 | case NVLINK_TX_HS: | ||
329 | mode_sw = nvgpu_nvlink_sublink_tx_hs; | ||
330 | break; | ||
331 | case NVLINK_TX_ENABLE_PM: | ||
332 | mode_sw = nvgpu_nvlink_sublink_tx_enable_pm; | ||
333 | break; | ||
334 | case NVLINK_TX_DISABLE_PM: | ||
335 | mode_sw = nvgpu_nvlink_sublink_tx_disable_pm; | ||
336 | break; | ||
337 | case NVLINK_TX_SINGLE_LANE: | ||
338 | mode_sw = nvgpu_nvlink_sublink_tx_single_lane; | ||
339 | break; | ||
340 | case NVLINK_TX_SAFE: | ||
341 | mode_sw = nvgpu_nvlink_sublink_tx_safe; | ||
342 | break; | ||
343 | case NVLINK_TX_OFF: | ||
344 | mode_sw = nvgpu_nvlink_sublink_tx_off; | ||
345 | break; | ||
346 | case NVLINK_TX_COMMON: | ||
347 | mode_sw = nvgpu_nvlink_sublink_tx_common; | ||
348 | break; | ||
349 | case NVLINK_TX_COMMON_DISABLE: | ||
350 | mode_sw = nvgpu_nvlink_sublink_tx_common_disable; | ||
351 | break; | ||
352 | case NVLINK_TX_DATA_READY: | ||
353 | mode_sw = nvgpu_nvlink_sublink_tx_data_ready; | ||
354 | break; | ||
355 | case NVLINK_TX_PRBS_EN: | ||
356 | mode_sw = nvgpu_nvlink_sublink_tx_prbs_en; | ||
357 | break; | ||
358 | default: | ||
359 | return -EINVAL; | ||
360 | } | ||
361 | } else { | ||
362 | switch (mode) { | ||
363 | case NVLINK_RX_HS: | ||
364 | mode_sw = nvgpu_nvlink_sublink_rx_hs; | ||
365 | break; | ||
366 | case NVLINK_RX_ENABLE_PM: | ||
367 | mode_sw = nvgpu_nvlink_sublink_rx_enable_pm; | ||
368 | break; | ||
369 | case NVLINK_RX_DISABLE_PM: | ||
370 | mode_sw = nvgpu_nvlink_sublink_rx_disable_pm; | ||
371 | break; | ||
372 | case NVLINK_RX_SINGLE_LANE: | ||
373 | mode_sw = nvgpu_nvlink_sublink_rx_single_lane; | ||
374 | break; | ||
375 | case NVLINK_RX_SAFE: | ||
376 | mode_sw = nvgpu_nvlink_sublink_rx_safe; | ||
377 | break; | ||
378 | case NVLINK_RX_OFF: | ||
379 | mode_sw = nvgpu_nvlink_sublink_rx_off; | ||
380 | break; | ||
381 | case NVLINK_RX_RXCAL: | ||
382 | mode_sw = nvgpu_nvlink_sublink_rx_rxcal; | ||
383 | break; | ||
384 | default: | ||
385 | return -EINVAL; | ||
386 | } | ||
387 | } | ||
388 | |||
389 | return g->ops.nvlink.set_sublink_mode(g, link_id, is_rx_sublink, | ||
390 | mode_sw); | ||
391 | } | ||
392 | |||
393 | static int nvgpu_nvlink_init_ops(struct gk20a *g) | ||
394 | { | ||
395 | struct nvlink_device *ndev = g->nvlink.priv; | ||
396 | |||
397 | if (!ndev) | ||
398 | return -EINVAL; | ||
399 | |||
400 | /* Fill in device struct */ | ||
401 | ndev->dev_ops.dev_early_init = nvgpu_nvlink_early_init; | ||
402 | ndev->dev_ops.dev_interface_init = nvgpu_nvlink_interface_init; | ||
403 | ndev->dev_ops.dev_reg_init = nvgpu_nvlink_reg_init; | ||
404 | ndev->dev_ops.dev_shutdown = nvgpu_nvlink_shutdown; | ||
405 | |||
406 | /* Fill in the link struct */ | ||
407 | ndev->link.device_id = ndev->device_id; | ||
408 | ndev->link.mode = NVLINK_LINK_OFF; | ||
409 | ndev->link.link_ops.get_link_mode = nvgpu_nvlink_get_link_mode; | ||
410 | ndev->link.link_ops.set_link_mode = nvgpu_nvlink_set_link_mode; | ||
411 | ndev->link.link_ops.get_sublink_mode = nvgpu_nvlink_get_sublink_mode; | ||
412 | ndev->link.link_ops.set_sublink_mode = nvgpu_nvlink_set_sublink_mode; | ||
413 | ndev->link.link_ops.get_link_state = nvgpu_nvlink_get_link_state; | ||
414 | ndev->link.link_ops.get_tx_sublink_state = | ||
415 | nvgpu_nvlink_get_tx_sublink_state; | ||
416 | ndev->link.link_ops.get_rx_sublink_state = | ||
417 | nvgpu_nvlink_get_rx_sublink_state; | ||
418 | ndev->link.link_ops.link_early_init = | ||
419 | nvgpu_nvlink_link_early_init; | ||
420 | |||
421 | return 0; | ||
422 | } | ||
423 | |||
424 | int nvgpu_nvlink_enumerate(struct gk20a *g) | ||
425 | { | ||
426 | struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; | ||
427 | |||
428 | if (!ndev) | ||
429 | return -ENODEV; | ||
430 | |||
431 | return nvlink_enumerate(ndev); | ||
432 | } | ||
433 | |||
434 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) | ||
435 | { | ||
436 | struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; | ||
437 | |||
438 | if (!ndev) | ||
439 | return -ENODEV; | ||
440 | |||
441 | /* Check if the link is connected */ | ||
442 | if (!g->nvlink.links[link_id].remote_info.is_connected) | ||
443 | return -ENODEV; | ||
444 | |||
445 | if (from_off) | ||
446 | return nvlink_transition_intranode_conn_off_to_safe(ndev); | ||
447 | |||
448 | return nvlink_train_intranode_conn_safe_to_hs(ndev); | ||
449 | } | ||
450 | |||
451 | #endif | ||
452 | |||
453 | int nvgpu_nvlink_probe(struct gk20a *g) | ||
454 | { | ||
455 | #ifdef CONFIG_TEGRA_NVLINK | ||
456 | int err; | ||
457 | struct nvlink_device *ndev; | ||
458 | |||
459 | /* Allocating structures */ | ||
460 | ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device)); | ||
461 | if (!ndev) { | ||
462 | nvgpu_err(g, "OOM while allocating nvlink device struct"); | ||
463 | return -ENOMEM; | ||
464 | } | ||
465 | |||
466 | ndev->priv = (void *) g; | ||
467 | g->nvlink.priv = (void *) ndev; | ||
468 | |||
469 | err = nvgpu_nvlink_read_dt_props(g); | ||
470 | if (err) | ||
471 | goto free_ndev; | ||
472 | |||
473 | err = nvgpu_nvlink_init_ops(g); | ||
474 | if (err) | ||
475 | goto free_ndev; | ||
476 | |||
477 | /* Register device with core driver*/ | ||
478 | err = nvlink_register_device(ndev); | ||
479 | if (err) { | ||
480 | nvgpu_err(g, "failed on nvlink device registration"); | ||
481 | goto free_ndev; | ||
482 | } | ||
483 | |||
484 | /* Register link with core driver */ | ||
485 | err = nvlink_register_link(&ndev->link); | ||
486 | if (err) { | ||
487 | nvgpu_err(g, "failed on nvlink link registration"); | ||
488 | goto unregister_ndev; | ||
489 | } | ||
490 | |||
491 | /* Enable NVLINK support */ | ||
492 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); | ||
493 | return 0; | ||
494 | |||
495 | unregister_ndev: | ||
496 | nvlink_unregister_device(ndev); | ||
497 | |||
498 | free_ndev: | ||
499 | nvgpu_kfree(g, ndev); | ||
500 | g->nvlink.priv = NULL; | ||
501 | return err; | ||
502 | #else | ||
503 | return -ENODEV; | ||
504 | #endif | ||
505 | } | ||
diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index 99cd8731..dc404b7a 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c | |||
@@ -816,6 +816,7 @@ static const struct gpu_ops gv100_ops = { | |||
816 | .isr = gp10b_priv_ring_isr, | 816 | .isr = gp10b_priv_ring_isr, |
817 | .decode_error_code = gp10b_priv_ring_decode_error_code, | 817 | .decode_error_code = gp10b_priv_ring_decode_error_code, |
818 | }, | 818 | }, |
819 | #if defined(CONFIG_TEGRA_NVLINK) | ||
819 | .nvlink = { | 820 | .nvlink = { |
820 | .discover_ioctrl = gv100_nvlink_discover_ioctrl, | 821 | .discover_ioctrl = gv100_nvlink_discover_ioctrl, |
821 | .discover_link = gv100_nvlink_discover_link, | 822 | .discover_link = gv100_nvlink_discover_link, |
@@ -835,6 +836,7 @@ static const struct gpu_ops gv100_ops = { | |||
835 | .shutdown = gv100_nvlink_shutdown, | 836 | .shutdown = gv100_nvlink_shutdown, |
836 | .early_init = gv100_nvlink_early_init, | 837 | .early_init = gv100_nvlink_early_init, |
837 | }, | 838 | }, |
839 | #endif | ||
838 | .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, | 840 | .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, |
839 | .get_litter_value = gv100_get_litter_value, | 841 | .get_litter_value = gv100_get_litter_value, |
840 | }; | 842 | }; |
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c index 776d1d66..9d99242f 100644 --- a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c | |||
@@ -20,6 +20,8 @@ | |||
20 | * DEALINGS IN THE SOFTWARE. | 20 | * DEALINGS IN THE SOFTWARE. |
21 | */ | 21 | */ |
22 | 22 | ||
23 | #ifdef CONFIG_TEGRA_NVLINK | ||
24 | |||
23 | #include <nvgpu/nvgpu_common.h> | 25 | #include <nvgpu/nvgpu_common.h> |
24 | #include <nvgpu/bios.h> | 26 | #include <nvgpu/bios.h> |
25 | #include <nvgpu/firmware.h> | 27 | #include <nvgpu/firmware.h> |
@@ -2677,3 +2679,4 @@ nvlink_init_exit: | |||
2677 | return err; | 2679 | return err; |
2678 | } | 2680 | } |
2679 | 2681 | ||
2682 | #endif /* CONFIG_TEGRA_NVLINK */ | ||
diff --git a/drivers/gpu/nvgpu/include/nvgpu/linux/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/linux/nvlink.h new file mode 100644 index 00000000..550a897a --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/linux/nvlink.h | |||
@@ -0,0 +1,31 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef __NVGPU_LINUX_NVLINK_H__ | ||
24 | #define __NVGPU_LINUX_NVLINK_H__ | ||
25 | |||
26 | #ifdef CONFIG_TEGRA_NVLINK | ||
27 | #include <linux/mutex.h> | ||
28 | #include <linux/platform/tegra/tegra-nvlink.h> | ||
29 | #endif | ||
30 | |||
31 | #endif | ||
diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h index dfa3be3f..00f0eda9 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h | |||
@@ -25,6 +25,12 @@ | |||
25 | 25 | ||
26 | #include <nvgpu/types.h> | 26 | #include <nvgpu/types.h> |
27 | 27 | ||
28 | #ifdef __KERNEL__ | ||
29 | #include <nvgpu/linux/nvlink.h> | ||
30 | #else | ||
31 | #include <nvgpu_rmos/include/nvlink.h> | ||
32 | #endif | ||
33 | |||
28 | struct gk20a; | 34 | struct gk20a; |
29 | 35 | ||
30 | struct nvgpu_nvlink_ioctrl_list { | 36 | struct nvgpu_nvlink_ioctrl_list { |
@@ -207,11 +213,12 @@ struct nvgpu_nvlink_dev { | |||
207 | void *priv; | 213 | void *priv; |
208 | }; | 214 | }; |
209 | 215 | ||
210 | |||
211 | int nvgpu_nvlink_enumerate(struct gk20a *g); | 216 | int nvgpu_nvlink_enumerate(struct gk20a *g); |
212 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); | 217 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); |
218 | int nvgpu_nvlink_read_dt_props(struct gk20a *g); | ||
219 | |||
213 | int nvgpu_nvlink_probe(struct gk20a *g); | 220 | int nvgpu_nvlink_probe(struct gk20a *g); |
214 | 221 | ||
215 | void nvgpu_mss_nvlink_init_credits(struct gk20a *g); | 222 | void nvgpu_mss_nvlink_init_credits(struct gk20a *g); |
216 | 223 | ||
217 | #endif | 224 | #endif /* __NVGPU_NVLINK_H__ */ |