Rewrite nvtaskset and implementation of partitioning for unmodified tasks

Rather than requiring libsmctrl.so to be preloaded, we now wrap
libcuda.so.1. All CUDA-using applications will load libcuda.so.1,
ensuring that our wrapper will always be dynamically loaded, no
matter if LD_PRELOAD is enabled, or if a program has been staticly
linked. All that needs to be done is that the location of our
"fake" libcuda.so.1 need to be put within the loader search path.
This can be done by setting LD_LIBRARY_PATH, or by installing
our wrapper into /lib/x86_64-linux-gnu.

The mask can still be set via the LIBSMCTRL_MASK environment
variable, but the easier-to-use nvtaskset tool is now the
recommended way to view or change the supreme TPC mask for any
CUDA-using application. This allows launching a program on the
first two GPCs via a command as simple as:
    ./nvtaskset -g 0-1 ./a_program a_program_args
(Note that use of the -g option requires the nvdebug kernel module
to first be loaded.)

These changes support the final version of the ECRTS'25 paper.

Note that nvtaskset does not yet fully support multi-GPU systems.

Bugfixes:
- Fix crash that would occur if both libsmctrl.so and libsmctrl.a
  were built into an application.
- Correctly use GPU ID when initializing a context in
  `libsmctrl_test_gpc_info`.
- Include `nvtaskset` as a prerequisite for
  `libsmctrl_test_supreme_mask`.
- Fix malfunction of `libsmctrl_test_gpc_info` if
  CUDA_VISIBLE_DEVICES is set.

Other minor changes:
- Adds make target to run all the tests.
- Fixes typos in comments.
- Enables -Wall build option.
- Upgrades supreme mask from 64 to 128 bits.
- Removes `detect_parker_soc()` from the global namespace.
- Adjusts test messages to be more succinct.
- Updates README with overview of how to partition unmodified
  applications, more details on the tests, and information on the
  new ECRTS'25 paper.

1 files changed, 41 insertions, 6 deletions

diff --git a/Makefile b/Makefile
index 62ec245..87a5708 100644
--- a/Makefile
+++ b/Makefile
@@ -3,9 +3,11 @@ CUDA ?= /usr/local/cuda
 # Note that CXX and CC are predefined as g++ and cc (respectively) by Make
 NVCC ?= $(CUDA)/bin/nvcc
 # Everything has to have -lcuda, as it's needed for libsmctrl
-LDFLAGS := -lcuda -I$(CUDA)/include -L$(CUDA)/lib64
+LDFLAGS := -ldl -lcuda -I$(CUDA)/include -L$(CUDA)/lib64
+ARCH = $(shell $(CC) -dumpmachine)
+CFLAGS := -Wall -Wno-parentheses
 
-.PHONY: clean tests all
+.PHONY: clean tests all install remove run_tests
 
 # ----- Main Library -----
 libsmctrl.so: libsmctrl.c libsmctrl.h
@@ -14,9 +16,14 @@ libsmctrl.so: libsmctrl.c libsmctrl.h
 # -fPIC is needed even if built as a static library, in case we are linked into
 # another shared library
 libsmctrl.a: libsmctrl.c libsmctrl.h
-        $(CC) $< -c -o libsmctrl.o -fPIC $(CFLAGS) $(LDFLAGS)
+        $(CC) $< -c -o libsmctrl.o -fPIC -DLIBSMCTRL_STATIC $(CFLAGS) $(LDFLAGS)
         ar rcs $@ libsmctrl.o
 
+# ----- CUDA Wrapper -----
+libcuda.so.1: libsmctrl.c libsmctrl.h
+        $(CC) $< -shared -o $@ -fPIC -DLIBSMCTRL_WRAPPER $(CFLAGS) $(LDFLAGS)
+        patchelf libcuda.so.1 --add-needed libcuda.so
+
 # ----- Utilities -----
 # Use static linking with tests to avoid LD_LIBRARY_PATH issues
 nvtaskset: nvtaskset.c libsmctrl.so libsmctrl.a
@@ -29,7 +36,7 @@ libsmctrl_test_gpc_info: libsmctrl_test_gpc_info.c libsmctrl.a testbench.h
 libsmctrl_test_mask_shared.o: libsmctrl_test_mask_shared.cu testbench.h
         $(NVCC) -ccbin $(CXX) $< -c -g
 
-libsmctrl_test_supreme_mask: libsmctrl_test_supreme_mask.c libsmctrl.a libsmctrl_test_mask_shared.o
+libsmctrl_test_supreme_mask: libsmctrl_test_supreme_mask.c libsmctrl.a libsmctrl_test_mask_shared.o libcuda.so.1 nvtaskset
         $(NVCC) -ccbin $(CXX) $@.c -o $@ libsmctrl_test_mask_shared.o -g -L. -l:libsmctrl.a $(LDFLAGS)
 
 libsmctrl_test_global_mask: libsmctrl_test_global_mask.c libsmctrl.a libsmctrl_test_mask_shared.o
@@ -52,7 +59,7 @@ tests: libsmctrl_test_gpc_info libsmctrl_test_supreme_mask \
        libsmctrl_test_stream_mask_override libsmctrl_test_next_mask \
        libsmctrl_test_next_mask_override
 
-all: libsmctrl.so nvtaskset tests
+all: libsmctrl.so libcuda.so.1 nvtaskset tests
 
 clean:
         rm -f libsmctrl.so libsmctrl.o libsmctrl.a libsmctrl_test_gpc_info \
@@ -60,4 +67,32 @@ clean:
               libsmctrl_test_global_mask \
               libsmctrl_test_stream_mask libsmctrl_test_stream_mask_override \
               libsmctrl_test_next_mask libsmctrl_test_next_mask_override \
-              nvtaskset
+              nvtaskset libcuda.so.1
+
+install: libcuda.so.1
+        @# Check that CUDA is installed first
+        test -f /lib/$(ARCH)/libcuda.so.*.*
+        @# Change libcuda.so link to bypass libcuda.so.1
+        sudo ln -sf /lib/$(ARCH)/libcuda.so.*.* /lib/$(ARCH)/libcuda.so
+        @# Remove libcuda.so.1 symlink
+        sudo rm /lib/$(ARCH)/libcuda.so.1
+        @# Install wrapper as libcuda.so.1
+        sudo cp libcuda.so.1 /lib/$(ARCH)/libcuda.so.1
+
+remove:
+        @# Test that our library in installed first
+        test ! -L /lib/$(ARCH)/libcuda.so.1
+        @# Overwrite install with original symlinks
+        sudo ln -sf libcuda.so.1 /lib/$(ARCH)/libcuda.so
+        sudo ln -sf /lib/$(ARCH)/libcuda.so.*.* /lib/$(ARCH)/libcuda.so.1
+
+run_tests: tests
+        ./libsmctrl_test_global_mask
+        ./libsmctrl_test_next_mask
+        ./libsmctrl_test_stream_mask
+        ./libsmctrl_test_next_mask_override
+        ./libsmctrl_test_stream_mask_override
+        @# Must set LD_LIBRARY_PATH in case make install has not been run
+        LD_LIBRARY_PATH=. ./libsmctrl_test_supreme_mask
+        ./libsmctrl_test_gpc_info
+        @ echo "All tests passed!"