From ac60151ea0a4a1f3882fde3c486af870029b7977 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Mon, 8 Apr 2024 13:33:28 -0400 Subject: Rework LCE<->PCE and GRCE->LCE configuration printing API Rather than up to dozens of individual files exposing part of each copy engine's configuration, have one file which exposes a unified view of the full topology. Example new output on RTX 2080 Ti: $ cat /proc/gpu0/copy_topology GRCE0 -> LCE04 GRCE1 -> LCE03 LCE02 -> PCE02 LCE03 -> PCE03 LCE04 -> PCE01 Old output: $ tail -n 1 /proc/gpu0/lce_for_pce* ==> /proc/gpu0/lce_for_pce0 <== 0xf ==> /proc/gpu0/lce_for_pce1 <== 0x4 ==> /proc/gpu0/lce_for_pce2 <== 0x2 ==> /proc/gpu0/lce_for_pce3 <== 0x3 $ tail -n 1 /proc/gpu1/shared_lce_for_grce* ==> /proc/gpu0/shared_lce_for_grce0 <== 0x4 ==> /proc/gpu0/shared_lce_for_grce1 <== 0x3 Specifically: - Add `copy_topology` API - Remove `shared_lce_for_grce#` and `lce_for_pce#` APIs - Move logic from `nvdebug_entry.c` to `copy_topology_procfs.c` - Do not print PCE or Shared LCE configuration if flagged absent - Refer to LCE0 and LCE1 as GRCE0 and GRCE1 - Print by LCE ID, which is move helpful when attempting to trace how a given copy runlist maps to a physical copy engine. - Document two errata with CE registers Tested working on Pascal Integrated, Pascal, Volta Integrated Volta, Turing, and Ampere Integrated on Linux 4.9 through 5.10. --- nvdebug_entry.c | 106 ++++++++------------------------------------------------ 1 file changed, 14 insertions(+), 92 deletions(-) (limited to 'nvdebug_entry.c') diff --git a/nvdebug_entry.c b/nvdebug_entry.c index 78860e6..ed82e58 100644 --- a/nvdebug_entry.c +++ b/nvdebug_entry.c @@ -1,4 +1,4 @@ -/* Copyright 2021 Joshua Bakita +/* Copyright 2024 Joshua Bakita * SPDX-License-Identifier: MIT */ @@ -27,6 +27,7 @@ extern struct file_operations disable_channel_file_ops; extern struct file_operations enable_channel_file_ops; extern struct file_operations switch_to_tsg_file_ops; extern struct file_operations device_info_file_ops; +extern struct file_operations copy_topology_file_ops; extern struct file_operations nvdebug_read_reg32_file_ops; extern struct file_operations nvdebug_read_reg_range_file_ops; @@ -204,7 +205,7 @@ int create_tpc_mask_files(int device_id, struct proc_dir_entry *dir) { int __init nvdebug_init(void) { struct proc_dir_entry *dir, *preempt_entry, *disable_channel_entry, *enable_channel_entry, *switch_to_tsg_entry, *device_info_entry, - *num_gpcs_entry, *lce_for_pce_entry, *grce_for_pce_entry; + *num_gpcs_entry; int rl_create_err, tpc_masks_create_err; // Check that an NVIDIA GPU is present and initialize g_nvdebug_state int res = probe_and_cache_device(); @@ -260,97 +261,18 @@ int __init nvdebug_init(void) { num_gpcs_entry = proc_create_data( "gpc_mask", 0444, dir, compat_ops(&nvdebug_read_reg32_file_ops), (void*)NV_FUSE_GPC); - // In both nouveau and nvgpu, the PCE_MAP register is available on Pascal+ - if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_PASCAL){ - // Used for reading a subset of a register on pascal - union reg_range pascal_reg; - // Create a pce mask for iteration - u32 ce_pce_map = nvdebug_readl(&g_nvdebug_state[device_id], NV_CE_PCE_MAP); - char file_name[21]; - int pce_id = 0; - int pce_num = 0; - int i; - for (pce_id = 0; pce_id < MAP_SIZE; pce_id++) { - // If pce is enabled, create files and iterate pce_id; otherwise, do nothing - if ((1 << pce_id) & ce_pce_map) { - snprintf(file_name, 20, "lce_for_pce%d", pce_num); - // Depending on GPU architecture, fetch data for the LCE of particular PCE - switch (g_nvdebug_state[res].chip_id & 0xff0) { - case NV_CHIP_ID_PASCAL: - // On Pascal, two PCE configurations are packed per-byte. - // Work around this by leveraging that we only run on 64-bit - // platforms (can assume that a void* is 64-bits), and that - // GPU register offsets are only 32-bits. Use the other 32 - // bits to store which bits to print. - pascal_reg.offset = NV_LCE_FOR_PCE_GP100(0); - pascal_reg.start_bit = pce_id * 4; - pascal_reg.stop_bit = pce_id * 4 + 4; - lce_for_pce_entry = proc_create_data( - file_name, 0444, dir, compat_ops(&nvdebug_read_reg_range_file_ops), - (void*)pascal_reg.raw); - break; - case NV_CHIP_ID_VOLTA: - case NV_CHIP_ID_VOLTA_INTEGRATED: - case NV_CHIP_ID_TURING: - lce_for_pce_entry = proc_create_data( - file_name, 0444, dir, compat_ops(&nvdebug_read_reg32_file_ops), - (void*)(uintptr_t)NV_LCE_FOR_PCE_GV100(pce_id)); - break; - case NV_CHIP_ID_AMPERE: - case NV_CHIP_ID_HOPPER: - case NV_CHIP_ID_ADA: - lce_for_pce_entry = proc_create_data( - file_name, 0444, dir, compat_ops(&nvdebug_read_reg32_file_ops), - (void*)(uintptr_t)NV_LCE_FOR_PCE_GA100(pce_id)); - break; - } - if (!lce_for_pce_entry) - return -ENOMEM; - pce_num++; - } - } - // We assume 2 GRCEs (reminder: GRCE0 and 1 are just LCE0 and 1) - for (i = 0; i < 2; i++) { - union reg_range grce_reg = {0}; - snprintf(file_name, 21, "shared_lce_for_grce%d", i); - // The offset used here is only documented for Turing - // Actually, Pascal through Turing - // On Pascal, it's only 3 bits, every 8 bits - // On Volta-Turing, it start at same offset, but it's lower 4 bits, every 32 bits - // On Ampere+ it starts at 0x001041c0, but is the same layout as Volta-Turing - switch (g_nvdebug_state[res].chip_id & 0xff0) { - case NV_CHIP_ID_PASCAL: - grce_reg.offset = NV_GRCE_FOR_CE_GP100(0); - grce_reg.start_bit = i * 8; - grce_reg.stop_bit = grce_reg.start_bit + 3; - break; - case NV_CHIP_ID_VOLTA: - case NV_CHIP_ID_VOLTA_INTEGRATED: - case NV_CHIP_ID_TURING: - grce_reg.offset = NV_GRCE_FOR_CE_GP100(i); - grce_reg.start_bit = 0; - grce_reg.stop_bit = grce_reg.start_bit + 4; - break; - case NV_CHIP_ID_AMPERE: - case NV_CHIP_ID_HOPPER: - case NV_CHIP_ID_ADA: - grce_reg.offset = NV_GRCE_FOR_CE_GA100(i); - grce_reg.start_bit = 0; - grce_reg.stop_bit = grce_reg.start_bit + 4; - break; - } - grce_for_pce_entry = proc_create_data( - file_name, 0444, dir, compat_ops(&nvdebug_read_reg_range_file_ops), - (void*)grce_reg.raw); - if (!grce_for_pce_entry) - return -ENOMEM; - } - - // TODO: Redo to num_pces + // Create files exposing LCE and PCE configuration (Pascal+) + if (g_nvdebug_state[res].chip_id >= NV_CHIP_ID_PASCAL) { + // Create file `/proc/gpu#/copy_topology`, world readable + if (!proc_create_data( + "copy_topology", 0444, dir, compat_ops(©_topology_file_ops), + (void*)0)) + goto out_nomem; // Create file `/proc/gpu#/pce_map`, world readable - num_gpcs_entry = proc_create_data( - "pce_map", 0444, dir, compat_ops(&nvdebug_read_reg32_file_ops), - (void*)NV_CE_PCE_MAP); + if (!proc_create_data( + "pce_map", 0444, dir, compat_ops(&nvdebug_read_reg32_file_ops), + (void*)NV_CE_PCE_MAP)) + goto out_nomem; } // ProcFS entry creation only fails if out of memory if (rl_create_err || tpc_masks_create_err || !preempt_entry || -- cgit v1.2.2