diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/nvlink.c | 536 |
1 files changed, 35 insertions, 501 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c index 5a2f4ded..faa27239 100644 --- a/drivers/gpu/nvgpu/common/linux/nvlink.c +++ b/drivers/gpu/nvgpu/common/linux/nvlink.c | |||
@@ -20,524 +20,58 @@ | |||
20 | #include "module.h" | 20 | #include "module.h" |
21 | 21 | ||
22 | #ifdef CONFIG_TEGRA_NVLINK | 22 | #ifdef CONFIG_TEGRA_NVLINK |
23 | #include <linux/platform/tegra/tegra-nvlink.h> | 23 | int nvgpu_nvlink_read_dt_props(struct gk20a *g) |
24 | #endif | ||
25 | |||
26 | #ifdef CONFIG_TEGRA_NVLINK | ||
27 | |||
28 | /* | ||
29 | * WAR: use this function to find detault link, as only one is supported | ||
30 | * on the library for now | ||
31 | * Returns NVLINK_MAX_LINKS_SW on failure | ||
32 | */ | ||
33 | static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) | ||
34 | { | ||
35 | u32 link_id; | ||
36 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
37 | |||
38 | if (!g) | ||
39 | return NVLINK_MAX_LINKS_SW; | ||
40 | |||
41 | /* Lets find the detected link */ | ||
42 | if (g->nvlink.initialized_links) | ||
43 | link_id = ffs(g->nvlink.initialized_links) - 1; | ||
44 | else | ||
45 | return NVLINK_MAX_LINKS_SW; | ||
46 | |||
47 | if (g->nvlink.links[link_id].remote_info.is_connected) | ||
48 | return link_id; | ||
49 | |||
50 | return NVLINK_MAX_LINKS_SW; | ||
51 | } | ||
52 | |||
53 | static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) | ||
54 | { | ||
55 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
56 | int err; | ||
57 | |||
58 | /* For now master topology is the only one supported */ | ||
59 | if (!ndev->is_master) { | ||
60 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | ||
61 | "dGPU is not master of Nvlink link"); | ||
62 | return -EINVAL; | ||
63 | } | ||
64 | |||
65 | err = g->ops.nvlink.early_init(g); | ||
66 | return err; | ||
67 | } | ||
68 | |||
69 | static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev) | ||
70 | { | ||
71 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
72 | int err; | ||
73 | u32 link_id; | ||
74 | |||
75 | /* | ||
76 | * First check the topology and setup connectivity | ||
77 | * HACK: we are only enabling one link for now!!! | ||
78 | */ | ||
79 | link_id = ffs(g->nvlink.discovered_links) - 1; | ||
80 | g->nvlink.links[link_id].remote_info.is_connected = true; | ||
81 | g->nvlink.links[link_id].remote_info.device_type = | ||
82 | nvgpu_nvlink_endp_tegra; | ||
83 | err = g->ops.nvlink.link_early_init(g, BIT(link_id)); | ||
84 | |||
85 | if (err == 0) { | ||
86 | g->nvlink.links[link_id].priv = (void *) &(ndev->link); | ||
87 | ndev->link.priv = (void *) g; | ||
88 | } | ||
89 | return err; | ||
90 | } | ||
91 | |||
92 | static int nvgpu_nvlink_interface_init(struct nvlink_device *ndev) | ||
93 | { | ||
94 | int err; | ||
95 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
96 | |||
97 | err = g->ops.nvlink.interface_init(g); | ||
98 | return err; | ||
99 | } | ||
100 | |||
101 | static int nvgpu_nvlink_shutdown(struct nvlink_device *ndev) | ||
102 | { | ||
103 | int err; | ||
104 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
105 | |||
106 | err = g->ops.nvlink.shutdown(g); | ||
107 | return 0; | ||
108 | } | ||
109 | |||
110 | static int nvgpu_nvlink_reg_init(struct nvlink_device *ndev) | ||
111 | { | ||
112 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
113 | int err; | ||
114 | |||
115 | err = g->ops.nvlink.reg_init(g); | ||
116 | |||
117 | return err; | ||
118 | } | ||
119 | |||
120 | static u32 nvgpu_nvlink_get_link_mode(struct nvlink_device *ndev) | ||
121 | { | ||
122 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
123 | u32 link_id; | ||
124 | u32 mode; | ||
125 | |||
126 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
127 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
128 | return -EINVAL; | ||
129 | |||
130 | mode = g->ops.nvlink.link_get_mode(g, link_id); | ||
131 | |||
132 | switch (mode) { | ||
133 | case nvgpu_nvlink_link_off: | ||
134 | return NVLINK_LINK_OFF; | ||
135 | case nvgpu_nvlink_link_hs: | ||
136 | return NVLINK_LINK_HS; | ||
137 | case nvgpu_nvlink_link_safe: | ||
138 | return NVLINK_LINK_SAFE; | ||
139 | case nvgpu_nvlink_link_fault: | ||
140 | return NVLINK_LINK_FAULT; | ||
141 | case nvgpu_nvlink_link_rcvy_ac: | ||
142 | return NVLINK_LINK_RCVY_AC; | ||
143 | case nvgpu_nvlink_link_rcvy_sw: | ||
144 | return NVLINK_LINK_RCVY_SW; | ||
145 | case nvgpu_nvlink_link_rcvy_rx: | ||
146 | return NVLINK_LINK_RCVY_RX; | ||
147 | case nvgpu_nvlink_link_detect: | ||
148 | return NVLINK_LINK_DETECT; | ||
149 | case nvgpu_nvlink_link_reset: | ||
150 | return NVLINK_LINK_RESET; | ||
151 | case nvgpu_nvlink_link_enable_pm: | ||
152 | return NVLINK_LINK_ENABLE_PM; | ||
153 | case nvgpu_nvlink_link_disable_pm: | ||
154 | return NVLINK_LINK_DISABLE_PM; | ||
155 | case nvgpu_nvlink_link_disable_err_detect: | ||
156 | return NVLINK_LINK_DISABLE_ERR_DETECT; | ||
157 | case nvgpu_nvlink_link_lane_disable: | ||
158 | return NVLINK_LINK_LANE_DISABLE; | ||
159 | case nvgpu_nvlink_link_lane_shutdown: | ||
160 | return NVLINK_LINK_LANE_SHUTDOWN; | ||
161 | default: | ||
162 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | ||
163 | "unsupported mode %u", mode); | ||
164 | } | ||
165 | |||
166 | return NVLINK_LINK_OFF; | ||
167 | } | ||
168 | |||
169 | static u32 nvgpu_nvlink_get_link_state(struct nvlink_device *ndev) | ||
170 | { | ||
171 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
172 | u32 link_id; | ||
173 | |||
174 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
175 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
176 | return -EINVAL; | ||
177 | |||
178 | return g->ops.nvlink.link_get_state(g, link_id); | ||
179 | } | ||
180 | |||
181 | static int nvgpu_nvlink_set_link_mode(struct nvlink_device *ndev, u32 mode) | ||
182 | { | ||
183 | |||
184 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
185 | u32 link_id; | ||
186 | u32 mode_sw; | ||
187 | |||
188 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
189 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
190 | return -EINVAL; | ||
191 | |||
192 | switch (mode) { | ||
193 | case NVLINK_LINK_OFF: | ||
194 | mode_sw = nvgpu_nvlink_link_off; | ||
195 | break; | ||
196 | case NVLINK_LINK_HS: | ||
197 | mode_sw = nvgpu_nvlink_link_hs; | ||
198 | break; | ||
199 | case NVLINK_LINK_SAFE: | ||
200 | mode_sw = nvgpu_nvlink_link_safe; | ||
201 | break; | ||
202 | case NVLINK_LINK_FAULT: | ||
203 | mode_sw = nvgpu_nvlink_link_fault; | ||
204 | break; | ||
205 | case NVLINK_LINK_RCVY_AC: | ||
206 | mode_sw = nvgpu_nvlink_link_rcvy_ac; | ||
207 | break; | ||
208 | case NVLINK_LINK_RCVY_SW: | ||
209 | mode_sw = nvgpu_nvlink_link_rcvy_sw; | ||
210 | break; | ||
211 | case NVLINK_LINK_RCVY_RX: | ||
212 | mode_sw = nvgpu_nvlink_link_rcvy_rx; | ||
213 | break; | ||
214 | case NVLINK_LINK_DETECT: | ||
215 | mode_sw = nvgpu_nvlink_link_detect; | ||
216 | break; | ||
217 | case NVLINK_LINK_RESET: | ||
218 | mode_sw = nvgpu_nvlink_link_reset; | ||
219 | break; | ||
220 | case NVLINK_LINK_ENABLE_PM: | ||
221 | mode_sw = nvgpu_nvlink_link_enable_pm; | ||
222 | break; | ||
223 | case NVLINK_LINK_DISABLE_PM: | ||
224 | mode_sw = nvgpu_nvlink_link_disable_pm; | ||
225 | break; | ||
226 | case NVLINK_LINK_LANE_DISABLE: | ||
227 | mode_sw = nvgpu_nvlink_link_lane_disable; | ||
228 | break; | ||
229 | case NVLINK_LINK_LANE_SHUTDOWN: | ||
230 | mode_sw = nvgpu_nvlink_link_lane_shutdown; | ||
231 | break; | ||
232 | default: | ||
233 | mode_sw = nvgpu_nvlink_link_off; | ||
234 | } | ||
235 | |||
236 | return g->ops.nvlink.link_set_mode(g, link_id, mode_sw); | ||
237 | } | ||
238 | |||
239 | static void nvgpu_nvlink_get_tx_sublink_state(struct nvlink_device *ndev, | ||
240 | u32 *state) | ||
241 | { | ||
242 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
243 | u32 link_id; | ||
244 | |||
245 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
246 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
247 | return; | ||
248 | if (state) | ||
249 | *state = g->ops.nvlink.get_tx_sublink_state(g, link_id); | ||
250 | } | ||
251 | |||
252 | static void nvgpu_nvlink_get_rx_sublink_state(struct nvlink_device *ndev, | ||
253 | u32 *state) | ||
254 | { | ||
255 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
256 | u32 link_id; | ||
257 | |||
258 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
259 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
260 | return; | ||
261 | if (state) | ||
262 | *state = g->ops.nvlink.get_rx_sublink_state(g, link_id); | ||
263 | } | ||
264 | |||
265 | static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev, | ||
266 | bool is_rx_sublink) | ||
267 | { | ||
268 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
269 | u32 link_id; | ||
270 | u32 mode; | ||
271 | |||
272 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
273 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
274 | return -EINVAL; | ||
275 | |||
276 | mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); | ||
277 | |||
278 | switch (mode) { | ||
279 | case nvgpu_nvlink_sublink_tx_hs: | ||
280 | return NVLINK_TX_HS; | ||
281 | case nvgpu_nvlink_sublink_tx_off: | ||
282 | return NVLINK_TX_OFF; | ||
283 | case nvgpu_nvlink_sublink_tx_single_lane: | ||
284 | return NVLINK_TX_SINGLE_LANE; | ||
285 | case nvgpu_nvlink_sublink_tx_safe: | ||
286 | return NVLINK_TX_SAFE; | ||
287 | case nvgpu_nvlink_sublink_tx_enable_pm: | ||
288 | return NVLINK_TX_ENABLE_PM; | ||
289 | case nvgpu_nvlink_sublink_tx_disable_pm: | ||
290 | return NVLINK_TX_DISABLE_PM; | ||
291 | case nvgpu_nvlink_sublink_tx_common: | ||
292 | return NVLINK_TX_COMMON; | ||
293 | case nvgpu_nvlink_sublink_tx_common_disable: | ||
294 | return NVLINK_TX_COMMON_DISABLE; | ||
295 | case nvgpu_nvlink_sublink_tx_data_ready: | ||
296 | return NVLINK_TX_DATA_READY; | ||
297 | case nvgpu_nvlink_sublink_tx_prbs_en: | ||
298 | return NVLINK_TX_PRBS_EN; | ||
299 | case nvgpu_nvlink_sublink_rx_hs: | ||
300 | return NVLINK_RX_HS; | ||
301 | case nvgpu_nvlink_sublink_rx_enable_pm: | ||
302 | return NVLINK_RX_ENABLE_PM; | ||
303 | case nvgpu_nvlink_sublink_rx_disable_pm: | ||
304 | return NVLINK_RX_DISABLE_PM; | ||
305 | case nvgpu_nvlink_sublink_rx_single_lane: | ||
306 | return NVLINK_RX_SINGLE_LANE; | ||
307 | case nvgpu_nvlink_sublink_rx_safe: | ||
308 | return NVLINK_RX_SAFE; | ||
309 | case nvgpu_nvlink_sublink_rx_off: | ||
310 | return NVLINK_RX_OFF; | ||
311 | case nvgpu_nvlink_sublink_rx_rxcal: | ||
312 | return NVLINK_RX_RXCAL; | ||
313 | default: | ||
314 | nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode); | ||
315 | break; | ||
316 | } | ||
317 | |||
318 | if (is_rx_sublink) | ||
319 | return NVLINK_RX_OFF; | ||
320 | return NVLINK_TX_OFF; | ||
321 | } | ||
322 | |||
323 | static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev, | ||
324 | bool is_rx_sublink, u32 mode) | ||
325 | { | ||
326 | struct gk20a *g = (struct gk20a *) ndev->priv; | ||
327 | u32 link_id; | ||
328 | u32 mode_sw; | ||
329 | |||
330 | link_id = __nvgpu_nvlink_get_link(ndev); | ||
331 | if (link_id == NVLINK_MAX_LINKS_SW) | ||
332 | return -EINVAL; | ||
333 | |||
334 | if (!is_rx_sublink) { | ||
335 | switch (mode) { | ||
336 | case NVLINK_TX_HS: | ||
337 | mode_sw = nvgpu_nvlink_sublink_tx_hs; | ||
338 | break; | ||
339 | case NVLINK_TX_ENABLE_PM: | ||
340 | mode_sw = nvgpu_nvlink_sublink_tx_enable_pm; | ||
341 | break; | ||
342 | case NVLINK_TX_DISABLE_PM: | ||
343 | mode_sw = nvgpu_nvlink_sublink_tx_disable_pm; | ||
344 | break; | ||
345 | case NVLINK_TX_SINGLE_LANE: | ||
346 | mode_sw = nvgpu_nvlink_sublink_tx_single_lane; | ||
347 | break; | ||
348 | case NVLINK_TX_SAFE: | ||
349 | mode_sw = nvgpu_nvlink_sublink_tx_safe; | ||
350 | break; | ||
351 | case NVLINK_TX_OFF: | ||
352 | mode_sw = nvgpu_nvlink_sublink_tx_off; | ||
353 | break; | ||
354 | case NVLINK_TX_COMMON: | ||
355 | mode_sw = nvgpu_nvlink_sublink_tx_common; | ||
356 | break; | ||
357 | case NVLINK_TX_COMMON_DISABLE: | ||
358 | mode_sw = nvgpu_nvlink_sublink_tx_common_disable; | ||
359 | break; | ||
360 | case NVLINK_TX_DATA_READY: | ||
361 | mode_sw = nvgpu_nvlink_sublink_tx_data_ready; | ||
362 | break; | ||
363 | case NVLINK_TX_PRBS_EN: | ||
364 | mode_sw = nvgpu_nvlink_sublink_tx_prbs_en; | ||
365 | break; | ||
366 | default: | ||
367 | return -EINVAL; | ||
368 | } | ||
369 | } else { | ||
370 | switch (mode) { | ||
371 | case NVLINK_RX_HS: | ||
372 | mode_sw = nvgpu_nvlink_sublink_rx_hs; | ||
373 | break; | ||
374 | case NVLINK_RX_ENABLE_PM: | ||
375 | mode_sw = nvgpu_nvlink_sublink_rx_enable_pm; | ||
376 | break; | ||
377 | case NVLINK_RX_DISABLE_PM: | ||
378 | mode_sw = nvgpu_nvlink_sublink_rx_disable_pm; | ||
379 | break; | ||
380 | case NVLINK_RX_SINGLE_LANE: | ||
381 | mode_sw = nvgpu_nvlink_sublink_rx_single_lane; | ||
382 | break; | ||
383 | case NVLINK_RX_SAFE: | ||
384 | mode_sw = nvgpu_nvlink_sublink_rx_safe; | ||
385 | break; | ||
386 | case NVLINK_RX_OFF: | ||
387 | mode_sw = nvgpu_nvlink_sublink_rx_off; | ||
388 | break; | ||
389 | case NVLINK_RX_RXCAL: | ||
390 | mode_sw = nvgpu_nvlink_sublink_rx_rxcal; | ||
391 | break; | ||
392 | default: | ||
393 | return -EINVAL; | ||
394 | } | ||
395 | } | ||
396 | |||
397 | return g->ops.nvlink.set_sublink_mode(g, link_id, is_rx_sublink, | ||
398 | mode_sw); | ||
399 | } | ||
400 | #endif | ||
401 | |||
402 | int nvgpu_nvlink_enumerate(struct gk20a *g) | ||
403 | { | ||
404 | int err = -ENODEV; | ||
405 | #ifdef CONFIG_TEGRA_NVLINK | ||
406 | struct nvlink_device *ndev; | ||
407 | |||
408 | ndev = (struct nvlink_device *) g->nvlink.priv; | ||
409 | |||
410 | if (ndev) | ||
411 | err = nvlink_enumerate(ndev); | ||
412 | #endif | ||
413 | return err; | ||
414 | } | ||
415 | |||
416 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) | ||
417 | { | 24 | { |
418 | int err = -ENODEV; | 25 | struct device_node *np; |
419 | #ifdef CONFIG_TEGRA_NVLINK | 26 | struct nvlink_device *ndev = g->nvlink.priv; |
420 | struct nvlink_device *ndev; | 27 | u32 local_dev_id; |
421 | 28 | u32 local_link_id; | |
422 | ndev = (struct nvlink_device *) g->nvlink.priv; | 29 | u32 remote_dev_id; |
423 | 30 | u32 remote_link_id; | |
424 | if (!ndev) | 31 | u32 physical_link; |
425 | return -ENODEV; | 32 | bool is_master; |
426 | |||
427 | /* Check if the link is connected */ | ||
428 | if (!g->nvlink.links[link_id].remote_info.is_connected) | ||
429 | return -ENODEV; | ||
430 | |||
431 | if (from_off) | ||
432 | return nvlink_transition_intranode_conn_off_to_safe(ndev); | ||
433 | return nvlink_train_intranode_conn_safe_to_hs(ndev); | ||
434 | |||
435 | #endif | ||
436 | return err; | ||
437 | } | ||
438 | |||
439 | int nvgpu_nvlink_probe(struct gk20a *g) | ||
440 | { | ||
441 | #ifdef CONFIG_TEGRA_NVLINK | ||
442 | int err = 0; | ||
443 | struct device_node *np = nvgpu_get_node(g); | ||
444 | struct device_node *nvlink_np = NULL, *endp_np = NULL; | ||
445 | struct nvlink_device *ndev; | ||
446 | u32 phys_link_id; | ||
447 | 33 | ||
448 | /* Parse DT */ | 34 | /* Parse DT */ |
449 | if (np) { | 35 | np = nvgpu_get_node(g); |
450 | nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); | 36 | if (!np) |
451 | if (nvlink_np) { | 37 | goto fail; |
452 | endp_np = of_get_child_by_name(nvlink_np, "endpoint"); | ||
453 | } | ||
454 | } | ||
455 | 38 | ||
456 | if (!endp_np) { | 39 | np = of_get_child_by_name(np, "nvidia,nvlink"); |
457 | nvgpu_info(g, "no nvlink DT detected"); | 40 | if (!np) |
458 | return -ENODEV; | 41 | goto fail; |
459 | } | ||
460 | 42 | ||
461 | /* Allocating structures */ | 43 | np = of_get_child_by_name(np, "endpoint"); |
462 | ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device)); | 44 | if (!np) |
463 | if (!ndev) { | 45 | goto fail; |
464 | nvgpu_err(g, "OOM while allocating nvlink device struct"); | ||
465 | return -ENOMEM; | ||
466 | } | ||
467 | |||
468 | ndev->priv = (void *) g; | ||
469 | g->nvlink.priv = (void *) ndev; | ||
470 | 46 | ||
471 | /* Parse DT structure to detect endpoint topology */ | 47 | /* Parse DT structure to detect endpoint topology */ |
472 | of_property_read_u32(endp_np, "local_dev_id", &ndev->device_id); | 48 | of_property_read_u32(np, "local_dev_id", &local_dev_id); |
473 | of_property_read_u32(endp_np, "local_link_id", &ndev->link.link_id); | 49 | of_property_read_u32(np, "local_link_id", &local_link_id); |
474 | ndev->is_master = of_property_read_bool(endp_np, "is_master"); | 50 | of_property_read_u32(np, "remote_dev_id", &remote_dev_id); |
475 | of_property_read_u32(endp_np, "remote_dev_id", | 51 | of_property_read_u32(np, "remote_link_id", &remote_link_id); |
476 | &ndev->link.remote_dev_info.device_id); | 52 | of_property_read_u32(np, "physical_link", &physical_link); |
477 | of_property_read_u32(endp_np, "remote_link_id", | 53 | is_master = of_property_read_bool(np, "is_master"); |
478 | &ndev->link.remote_dev_info.link_id); | ||
479 | of_property_read_u32(endp_np, "physical_link", | ||
480 | &phys_link_id); | ||
481 | |||
482 | g->nvlink.connected_links = BIT(phys_link_id); | ||
483 | 54 | ||
484 | /* Check that we are in dGPU mode */ | 55 | /* Check that we are in dGPU mode */ |
485 | if (ndev->device_id != NVLINK_ENDPT_GV100) { | 56 | if (local_dev_id != NVLINK_ENDPT_GV100) { |
486 | nvgpu_err(g, "Local nvlink device is not dGPU"); | 57 | nvgpu_err(g, "Local nvlink device is not dGPU"); |
487 | err = -EINVAL; | 58 | return -EINVAL; |
488 | goto free_ndev; | ||
489 | } | ||
490 | |||
491 | /* Fill in device struct */ | ||
492 | ndev->dev_ops.dev_early_init = nvgpu_nvlink_early_init; | ||
493 | ndev->dev_ops.dev_interface_init = nvgpu_nvlink_interface_init; | ||
494 | ndev->dev_ops.dev_reg_init = nvgpu_nvlink_reg_init; | ||
495 | ndev->dev_ops.dev_shutdown = nvgpu_nvlink_shutdown; | ||
496 | |||
497 | /* Fill in the link struct */ | ||
498 | ndev->link.device_id = ndev->device_id; | ||
499 | ndev->link.mode = NVLINK_LINK_OFF; | ||
500 | ndev->link.link_ops.get_link_mode = nvgpu_nvlink_get_link_mode; | ||
501 | ndev->link.link_ops.set_link_mode = nvgpu_nvlink_set_link_mode; | ||
502 | ndev->link.link_ops.get_sublink_mode = nvgpu_nvlink_get_sublink_mode; | ||
503 | ndev->link.link_ops.set_sublink_mode = nvgpu_nvlink_set_sublink_mode; | ||
504 | ndev->link.link_ops.get_link_state = nvgpu_nvlink_get_link_state; | ||
505 | ndev->link.link_ops.get_tx_sublink_state = | ||
506 | nvgpu_nvlink_get_tx_sublink_state; | ||
507 | ndev->link.link_ops.get_rx_sublink_state = | ||
508 | nvgpu_nvlink_get_rx_sublink_state; | ||
509 | ndev->link.link_ops.link_early_init = | ||
510 | nvgpu_nvlink_link_early_init; | ||
511 | |||
512 | /* Register device with core driver*/ | ||
513 | err = nvlink_register_device(ndev); | ||
514 | if (err) { | ||
515 | nvgpu_err(g, "failed on nvlink device registration"); | ||
516 | goto free_ndev; | ||
517 | } | 59 | } |
518 | 60 | ||
519 | /* Register link with core driver */ | 61 | ndev->is_master = is_master; |
520 | err = nvlink_register_link(&ndev->link); | 62 | ndev->device_id = local_dev_id; |
521 | if (err) { | 63 | ndev->link.link_id = local_link_id; |
522 | nvgpu_err(g, "failed on nvlink link registration"); | 64 | ndev->link.remote_dev_info.device_id = remote_dev_id; |
523 | goto unregister_ndev; | 65 | ndev->link.remote_dev_info.link_id = remote_link_id; |
524 | } | ||
525 | 66 | ||
526 | /* Enable NVLINK support */ | 67 | g->nvlink.connected_links = BIT(physical_link); |
527 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); | ||
528 | return 0; | 68 | return 0; |
529 | 69 | ||
530 | unregister_ndev: | 70 | fail: |
531 | nvlink_unregister_device(ndev); | 71 | nvgpu_info(g, "nvlink endpoint not found or invaling in DT"); |
532 | |||
533 | free_ndev: | ||
534 | nvgpu_kfree(g, ndev); | ||
535 | g->nvlink.priv = NULL; | ||
536 | return err; | ||
537 | #else | ||
538 | return -ENODEV; | 72 | return -ENODEV; |
539 | #endif | ||
540 | } | 73 | } |
74 | #endif /* CONFIG_TEGRA_NVLINK */ | ||
541 | 75 | ||
542 | void nvgpu_mss_nvlink_init_credits(struct gk20a *g) | 76 | void nvgpu_mss_nvlink_init_credits(struct gk20a *g) |
543 | { | 77 | { |