From 89177fce34edb5ad0059a41548888d05588cc1c5 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Mon, 16 Jun 2025 19:29:07 -0400 Subject: Rewrite nvtaskset and implementation of partitioning for unmodified tasks Rather than requiring libsmctrl.so to be preloaded, we now wrap libcuda.so.1. All CUDA-using applications will load libcuda.so.1, ensuring that our wrapper will always be dynamically loaded, no matter if LD_PRELOAD is enabled, or if a program has been staticly linked. All that needs to be done is that the location of our "fake" libcuda.so.1 need to be put within the loader search path. This can be done by setting LD_LIBRARY_PATH, or by installing our wrapper into /lib/x86_64-linux-gnu. The mask can still be set via the LIBSMCTRL_MASK environment variable, but the easier-to-use nvtaskset tool is now the recommended way to view or change the supreme TPC mask for any CUDA-using application. This allows launching a program on the first two GPCs via a command as simple as: ./nvtaskset -g 0-1 ./a_program a_program_args (Note that use of the -g option requires the nvdebug kernel module to first be loaded.) These changes support the final version of the ECRTS'25 paper. Note that nvtaskset does not yet fully support multi-GPU systems. Bugfixes: - Fix crash that would occur if both libsmctrl.so and libsmctrl.a were built into an application. - Correctly use GPU ID when initializing a context in `libsmctrl_test_gpc_info`. - Include `nvtaskset` as a prerequisite for `libsmctrl_test_supreme_mask`. - Fix malfunction of `libsmctrl_test_gpc_info` if CUDA_VISIBLE_DEVICES is set. Other minor changes: - Adds make target to run all the tests. - Fixes typos in comments. - Enables -Wall build option. - Upgrades supreme mask from 64 to 128 bits. - Removes `detect_parker_soc()` from the global namespace. - Adjusts test messages to be more succinct. - Updates README with overview of how to partition unmodified applications, more details on the tests, and information on the new ECRTS'25 paper. --- Makefile | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 62ec245..87a5708 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,11 @@ CUDA ?= /usr/local/cuda # Note that CXX and CC are predefined as g++ and cc (respectively) by Make NVCC ?= $(CUDA)/bin/nvcc # Everything has to have -lcuda, as it's needed for libsmctrl -LDFLAGS := -lcuda -I$(CUDA)/include -L$(CUDA)/lib64 +LDFLAGS := -ldl -lcuda -I$(CUDA)/include -L$(CUDA)/lib64 +ARCH = $(shell $(CC) -dumpmachine) +CFLAGS := -Wall -Wno-parentheses -.PHONY: clean tests all +.PHONY: clean tests all install remove run_tests # ----- Main Library ----- libsmctrl.so: libsmctrl.c libsmctrl.h @@ -14,9 +16,14 @@ libsmctrl.so: libsmctrl.c libsmctrl.h # -fPIC is needed even if built as a static library, in case we are linked into # another shared library libsmctrl.a: libsmctrl.c libsmctrl.h - $(CC) $< -c -o libsmctrl.o -fPIC $(CFLAGS) $(LDFLAGS) + $(CC) $< -c -o libsmctrl.o -fPIC -DLIBSMCTRL_STATIC $(CFLAGS) $(LDFLAGS) ar rcs $@ libsmctrl.o +# ----- CUDA Wrapper ----- +libcuda.so.1: libsmctrl.c libsmctrl.h + $(CC) $< -shared -o $@ -fPIC -DLIBSMCTRL_WRAPPER $(CFLAGS) $(LDFLAGS) + patchelf libcuda.so.1 --add-needed libcuda.so + # ----- Utilities ----- # Use static linking with tests to avoid LD_LIBRARY_PATH issues nvtaskset: nvtaskset.c libsmctrl.so libsmctrl.a @@ -29,7 +36,7 @@ libsmctrl_test_gpc_info: libsmctrl_test_gpc_info.c libsmctrl.a testbench.h libsmctrl_test_mask_shared.o: libsmctrl_test_mask_shared.cu testbench.h $(NVCC) -ccbin $(CXX) $< -c -g -libsmctrl_test_supreme_mask: libsmctrl_test_supreme_mask.c libsmctrl.a libsmctrl_test_mask_shared.o +libsmctrl_test_supreme_mask: libsmctrl_test_supreme_mask.c libsmctrl.a libsmctrl_test_mask_shared.o libcuda.so.1 nvtaskset $(NVCC) -ccbin $(CXX) $@.c -o $@ libsmctrl_test_mask_shared.o -g -L. -l:libsmctrl.a $(LDFLAGS) libsmctrl_test_global_mask: libsmctrl_test_global_mask.c libsmctrl.a libsmctrl_test_mask_shared.o @@ -52,7 +59,7 @@ tests: libsmctrl_test_gpc_info libsmctrl_test_supreme_mask \ libsmctrl_test_stream_mask_override libsmctrl_test_next_mask \ libsmctrl_test_next_mask_override -all: libsmctrl.so nvtaskset tests +all: libsmctrl.so libcuda.so.1 nvtaskset tests clean: rm -f libsmctrl.so libsmctrl.o libsmctrl.a libsmctrl_test_gpc_info \ @@ -60,4 +67,32 @@ clean: libsmctrl_test_global_mask \ libsmctrl_test_stream_mask libsmctrl_test_stream_mask_override \ libsmctrl_test_next_mask libsmctrl_test_next_mask_override \ - nvtaskset + nvtaskset libcuda.so.1 + +install: libcuda.so.1 + @# Check that CUDA is installed first + test -f /lib/$(ARCH)/libcuda.so.*.* + @# Change libcuda.so link to bypass libcuda.so.1 + sudo ln -sf /lib/$(ARCH)/libcuda.so.*.* /lib/$(ARCH)/libcuda.so + @# Remove libcuda.so.1 symlink + sudo rm /lib/$(ARCH)/libcuda.so.1 + @# Install wrapper as libcuda.so.1 + sudo cp libcuda.so.1 /lib/$(ARCH)/libcuda.so.1 + +remove: + @# Test that our library in installed first + test ! -L /lib/$(ARCH)/libcuda.so.1 + @# Overwrite install with original symlinks + sudo ln -sf libcuda.so.1 /lib/$(ARCH)/libcuda.so + sudo ln -sf /lib/$(ARCH)/libcuda.so.*.* /lib/$(ARCH)/libcuda.so.1 + +run_tests: tests + ./libsmctrl_test_global_mask + ./libsmctrl_test_next_mask + ./libsmctrl_test_stream_mask + ./libsmctrl_test_next_mask_override + ./libsmctrl_test_stream_mask_override + @# Must set LD_LIBRARY_PATH in case make install has not been run + LD_LIBRARY_PATH=. ./libsmctrl_test_supreme_mask + ./libsmctrl_test_gpc_info + @ echo "All tests passed!" -- cgit v1.2.2