gpu: nvgpu: unit: Add unit testing FW

Full documentation for this is in the unit testing
confluence page.

JIRA NVGPU-525
Bug 2261555

Change-Id: I463e6267eb0eb12b7313f8b275266e8faabe5ccf
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1683915
GVS: Gerrit_Virtual_Submit
Reviewed-by: Konsta Holtta <kholtta@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

17 files changed, 1343 insertions, 17 deletions

diff --git a/userspace/Makefile b/userspace/Makefile
index 70e93bad..56709ec4 100644
--- a/userspace/Makefile
+++ b/userspace/Makefile
@@ -18,6 +18,10 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
+# TODO:
+# - Separate rule for nvgpu_unit shared library
+# - Proper header dependency checking.
+
 # Turn off suffix rules. They are deprecated.
 .SUFFIXES:
 
@@ -28,25 +32,35 @@ TWD=$(CURDIR)
 # Top level out dir.
 OUT=$(TWD)/build
 
+# Core unit test framework.
+CORE_SRC=$(TWD)/src
+CORE_OUT=$(OUT)/nvgpu_unit_core
+
 # Nvgpu driver code.
 NVGPU_SRC=$(TWD)/../drivers/gpu/nvgpu
 NVGPU_OUT=$(OUT)/libnvgpu
 
-INCLUDES=                       \
-        -I$(NVGPU_SRC)          \
-        -I$(NVGPU_SRC)/include  \
-        -I../include            \
-        -I../include/uapi
+# Unit tests themselves.
+UNIT_SRC=$(TWD)/units
+UNIT_OUT=$(OUT)/units
 
-CONFIGS=                        \
-        -D__NVGPU_POSIX__
+INCLUDES=                               \
+        -I$(NVGPU_SRC)                  \
+        -I$(NVGPU_SRC)/include          \
+        -I$(TWD)/../include             \
+        -I$(TWD)/../include/uapi        \
+        -I$(TWD)/include
+
+CONFIGS=-D__NVGPU_POSIX__
 
 # Compiler, c-flags, etc.
 
-# CC    = clang
-CC      = gcc
-CFLAGS  = -Wall -ggdb  -Werror -fPIC $(INCLUDES) $(CONFIGS)
-LIBS    = -lpthread -pthread
+# CC      = clang
+CC        = gcc
+CFLAGS    = -Wall -ggdb  -Werror -fPIC $(INCLUDES) $(CONFIGS)
+LIB_PATHS = -L$(OUT)
+LIBS      = -lpthread -pthread -lgcov -ldl
+
 
 # Source files. We expect $(OBJS) and $(HEADERS) to get filled in here.
 include Makefile.sources
@@ -55,19 +69,68 @@ include Makefile.sources
 # Linux kernel.
 include Makefile.configs
 
-all: $(OUT)/libnvgpu-drv.so
+all: $(OUT)/nvgpu_unit $(UNITS)
+
+# Convenience targets.
+.PHONY: libnvgpu core units
+libnvgpu: $(OUT)/libnvgpu-drv.so
+core: $(OUT)/nvgpu_unit
+units: $(UNITS)
+
+# Note the weird libnvgpu_unit.so file: this is a bit of a hack. It lets the
+# unit tests link back against the nvgpu_unit executable so that they can call
+# functions (like unit_info()) directly. This shared library isn't actually
+# used for anything beyond that.
+#
+# Also it really should have its own rule...
+$(OUT)/nvgpu_unit: $(OUT)/libnvgpu-drv.so $(CORE_OBJS)
+        $(CC) -shared -o $(OUT)/libnvgpu_unit.so        \
+                $(CORE_OBJS) $(LIB_PATHS) $(LIBS)
+        $(CC) --coverage                                \
+                -o $(OUT)/nvgpu_unit $(CORE_OBJS) $(LIB_PATHS) $(LIBS)
 
 $(OUT)/libnvgpu-drv.so: $(OBJS)
         $(CC) -shared -o $(OUT)/libnvgpu-drv.so $(OBJS) -lgcov
 
-# Default build target for all the object files we want to build in userspace.
+# Default build target for all the nvgpu driver object files we want to build in
+# userspace. These get bundled into libnvgpu-drv.so.
 $(NVGPU_OUT)/%.o : $(NVGPU_SRC)/%.c $(HEADERS)
         @if [ ! -d $(dir $@) ] ; then \
                 mkdir -p $(dir $@) ; \
         fi
-        $(CC) $(CFLAGS) $(configs) -c -o $@ $<
+        $(CC) --coverage $(CFLAGS) $(configs) -c -o $@ $<
+
+# Build target for unit test files. These are not part of the libnvgpu-drv.so.
+# These comprise the unit test framework.
+$(CORE_OUT)/%.o : $(CORE_SRC)/%.c $(CORE_HEADERS)
+        @if [ ! -d $(dir $@) ] ; then \
+                mkdir -p $(dir $@) ; \
+        fi
+        $(CC) --coverage $(CFLAGS) $(configs) -c -o $@ $<
+
+# Certain variables should be exported to the unit test module builds.
+export TWD INCLUDES CONFIGS UNIT_SRC UNIT_OUT
+export CC CFLAGS LIB_PATHS LIBS
 
-.PHONY: clean
+.PHONY: $(UNITS)
+$(UNITS): $(OUT)/libnvgpu-drv.so
+        @echo "Building unit module: $@"
+        @+$(MAKE) --no-print-directory -C $@
 
-clean:
+.PHONY: clean nvgpu_clean core_clean unit_clean
+
+clean: nvgpu_clean core_clean unit_clean
         rm -rf $(OUT)
+
+nvgpu_clean:
+        rm -rf $(OUT)/libnvgpu*
+
+core_clean:
+        rm -rf $(OUT)/nvgpu_unit*
+
+unit_clean:
+        @for d in $(UNITS); do                                  \
+                echo Cleaning $$d;                              \
+                $(MAKE) --no-print-directory -C $$d clean;      \
+        done
+        rm -rf $(OUT)/units
diff --git a/userspace/Makefile.sources b/userspace/Makefile.sources
index 3598c971..cb46573a 100644
--- a/userspace/Makefile.sources
+++ b/userspace/Makefile.sources
@@ -24,7 +24,6 @@ include ../drivers/gpu/nvgpu/Makefile.sources
 
 OBJS := $(srcs:%.c=$(NVGPU_OUT)/%.o)
 
-
 HEADERS :=      \
         $(NVGPU_SRC)/include/nvgpu/*.h          \
         $(NVGPU_SRC)/include/nvgpu/hw/*/*.h     \
@@ -32,3 +31,18 @@ HEADERS :=	\
         $(NVGPU_SRC)/gm20b/*.h                  \
         $(NVGPU_SRC)/gp10b/*.h                  \
         $(NVGPU_SRC)/gv11b/*.h
+
+CORE_OBJS :=    \
+        $(CORE_OUT)/unit_main.o                 \
+        $(CORE_OUT)/nvgpu.o                     \
+        $(CORE_OUT)/args.o                      \
+        $(CORE_OUT)/io.o                        \
+        $(CORE_OUT)/module.o                    \
+        $(CORE_OUT)/results.o                   \
+        $(CORE_OUT)/exec.o
+
+CORE_HEADERS := \
+        $(CORE_SRC)/../include/unit/*.h
+
+# Each directory under the UNIT_SRC directory should correspond to one module.
+UNITS :=
diff --git a/userspace/include/unit/args.h b/userspace/include/unit/args.h
new file mode 100644
index 00000000..def84a29
--- /dev/null
+++ b/userspace/include/unit/args.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UNIT_ARGS_H__
+#define __UNIT_ARGS_H__
+
+#include <stdbool.h>
+
+/*
+ * Allow defaults to be changed at compile time.
+ */
+#define __stringify(x)                  #x
+#define stringify(x)                    __stringify(x)
+
+#ifndef __DEFAULT_ARG_UNIT_LOAD_PATH
+#define __DEFAULT_ARG_UNIT_LOAD_PATH    build/units
+#endif
+#define DEFAULT_ARG_UNIT_LOAD_PATH      stringify(__DEFAULT_ARG_UNIT_LOAD_PATH)
+
+struct unit_fw;
+
+struct unit_fw_args {
+        bool             help;
+        int              verbose_lvl;
+        bool             no_color;
+
+        const char      *unit_name;
+        const char      *unit_load_path;
+};
+
+int core_parse_args(struct unit_fw *fw, int argc, char **argv);
+void core_print_help(struct unit_fw *fw);
+
+/*
+ * Convenience for getting the args struct pointer.
+ */
+#define args(fw)                ((fw)->args)
+
+#endif
diff --git a/userspace/include/unit/core.h b/userspace/include/unit/core.h
new file mode 100644
index 00000000..d9d119db
--- /dev/null
+++ b/userspace/include/unit/core.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UNIT_CORE_H__
+#define __UNIT_CORE_H__
+
+struct unit_fw_args;
+struct unit_modules;
+struct unit_results;
+
+struct gk20a;
+
+/*
+ * The core unit testing framework data structure. Keeps track of global state
+ * for the unit test app.
+ */
+struct unit_fw {
+        struct unit_fw_args      *args;
+
+        struct unit_module      **modules;
+
+        struct unit_results      *results;
+
+        /*
+         * nvgpu-drv interface. Currently the only two directly referenced
+         * functions are:
+         *
+         *   nvgpu_posix_probe()
+         *   nvgpu_posix_cleanup()
+         *
+         * There will get populated so that we can call them before/after each
+         * module.
+         */
+        void                     *nvgpu_so;
+        struct {
+                struct gk20a    *(*nvgpu_posix_probe)(void);
+                void             (*nvgpu_posix_cleanup)(struct gk20a *g);
+        } nvgpu;
+};
+
+int core_load_nvgpu(struct unit_fw *fw);
+int core_exec(struct unit_fw *fw);
+
+#endif
diff --git a/userspace/include/unit/io.h b/userspace/include/unit/io.h
new file mode 100644
index 00000000..1fd0fa43
--- /dev/null
+++ b/userspace/include/unit/io.h
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UNIT_IO_H__
+#define __UNIT_IO_H__
+
+struct unit_fw;
+struct unit_module;
+
+/*
+ * necessary for args(fw) macro. IO will always, in general, depend on args
+ * since the args will specify where IO should be directed to.
+ */
+#include <unit/args.h>
+
+#define core_msg(fw, msg, ...)                                          \
+        core_vbs(fw, 0, msg, ##__VA_ARGS__)
+#define core_msg_color(fw, color, msg, ...)                             \
+        core_vbs_color(fw, color, 0, msg, ##__VA_ARGS__)
+
+#define core_vbs_color(fw, color, lvl, msg, ...)                        \
+        do {                                                            \
+                if ((lvl) > args(fw)->verbose_lvl)                      \
+                        continue;                                       \
+                                                                        \
+                /* Print if verbosity level is high enough. */          \
+                __core_print_stdout(fw, color, msg, ##__VA_ARGS__);     \
+        } while (0)
+#define core_vbs(fw, lvl, msg, ...)                                     \
+        core_vbs_color(fw, NULL, lvl, msg, ##__VA_ARGS__)
+
+#define core_err(fw, msg, ...)                                          \
+        __core_print_stderr(fw, "(%s:%d) " msg,                         \
+                            __func__, __LINE__, ##__VA_ARGS__)
+
+/*
+ * Output macro for unit tests to use.
+ */
+#define unit_info(unit, msg, ...)                                       \
+        __unit_info_color(unit, NULL, msg, ##__VA_ARGS__)
+#define unit_err(unit, msg, ...)                                        \
+        __unit_info_color(unit, C_RED, msg, ##__VA_ARGS__)
+
+/*
+ * Don't go overboard with these!!!
+ */
+#define C_RED     "\x1B[31m"
+#define C_GREEN   "\x1B[32m"
+#define C_YELLOW  "\x1B[33m"
+#define C_BLUE    "\x1B[34m"
+#define C_MAGENTA "\x1B[35m"
+#define C_CYAN    "\x1B[36m"
+#define C_WHITE   "\x1B[37m"
+#define C_RESET   "\x1B[0m"
+
+/*
+ * Printing functions. Do not use these directly. Instead use the provided
+ * macros.
+ */
+__attribute__((format (printf, 3, 4)))
+void __core_print_stdout(struct unit_fw *fw, const char *color,
+                         const char *fmt, ...);
+__attribute__((format (printf, 2, 3)))
+void __core_print_stderr(struct unit_fw *fw, const char *fmt, ...);
+__attribute__((format (printf, 3, 4)))
+void __unit_info_color(struct unit_module *unit, const char *color,
+                       const char *fmt, ...);
+
+#endif
diff --git a/userspace/include/unit/module.h b/userspace/include/unit/module.h
new file mode 100644
index 00000000..79737d61
--- /dev/null
+++ b/userspace/include/unit/module.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UNIT_MODULE_H__
+#define __UNIT_MODULE_H__
+
+struct unit_fw;
+struct unit_module;
+
+struct unit_module **core_load_modules(struct unit_fw *fw);
+
+#endif
diff --git a/userspace/include/unit/results.h b/userspace/include/unit/results.h
new file mode 100644
index 00000000..fb88e59d
--- /dev/null
+++ b/userspace/include/unit/results.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UNIT_RESULTS_H__
+#define __UNIT_RESULTS_H__
+
+/*
+ * Keep track of the the results of a set of unit tests. This is effectively
+ * just a single linked list of records for each test.
+ */
+
+struct unit_test_record {
+        /*
+         * Let's us determine the name of the test.
+         */
+        struct unit_module *mod;
+        struct unit_module_test *test;
+
+        /*
+         * True for pass, false for fail.
+         */
+        bool status;
+
+        struct unit_test_record *next;
+};
+
+struct unit_test_list {
+        struct unit_test_record *head;
+        struct unit_test_record *last;
+};
+
+struct unit_results {
+        struct unit_test_list passing;
+        struct unit_test_list failing;
+
+        int nr_tests;
+        int nr_passing;
+};
+
+#define for_record_in_test_list(__test_list, __test)    \
+        for ((__test) = (__test_list)->head;            \
+             (__test) != NULL;                          \
+             (__test) = (__test)->next)
+
+int core_add_test_record(struct unit_fw *fw,
+                         struct unit_module *mod,
+                         struct unit_module_test *test,
+                         bool success);
+void core_print_test_status(struct unit_fw *fw);
+
+#endif
diff --git a/userspace/include/unit/unit.h b/userspace/include/unit/unit.h
new file mode 100644
index 00000000..9438d4d9
--- /dev/null
+++ b/userspace/include/unit/unit.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __UNIT_UNIT_H__
+#define __UNIT_UNIT_H__
+
+struct gk20a;
+
+struct unit_module;
+typedef int (*module_test_fn)(struct unit_module *m,
+                              struct gk20a *g, void *args);
+
+#define UNIT_SUCCESS            0
+#define UNIT_FAIL               -1
+
+struct unit_module_test {
+        /*
+         * Name of the test.
+         */
+        const char *name;
+
+        /*
+         * Function to call to execute the test.
+         */
+        module_test_fn fn;
+
+        /*
+         * A void pointer to arbitrary arguments. Lets the same unit test
+         * function perform multiple tests. This gets passed into the
+         * module_test_fn as @args.
+         */
+        void *args;
+};
+
+/*
+ * Interface to the unit test framework module loader. Each unit test module
+ * will have exactly one of these.
+ */
+struct unit_module {
+        /*
+         * Name of the module.
+         */
+        const char                      *name;
+
+        /*
+         * NULL terminated list of tests within the module.
+         */
+        struct unit_module_test         *tests;
+        unsigned long                    nr_tests;
+
+        /*
+         * For the core FW to use. Not for modules!!!
+         */
+        void                            *lib_handle;
+        struct unit_fw                  *fw;
+};
+
+#define UNIT_MODULE(__name, __tests)                                    \
+        struct unit_module __unit_module__ = {                          \
+                .name = #__name,                                        \
+                .tests = __tests,                                       \
+                .nr_tests = (sizeof(__tests) /                          \
+                             sizeof(struct unit_module_test)),          \
+                .lib_handle = NULL,                                     \
+        }
+
+#define UNIT_TEST(__name, __fn, __args)                                 \
+        {                                                               \
+                .name = #__name,                                        \
+                .fn = __fn,                                             \
+                .args = __args,                                         \
+        }
+
+#define unit_return_fail(m, msg, ...)                                   \
+        do {                                                            \
+                unit_err(m, "%s():%d " msg,                             \
+                         __func__, __LINE__, ##__VA_ARGS__);            \
+                return UNIT_FAIL;                                       \
+        } while (0)
+
+#endif
diff --git a/userspace/src/args.c b/userspace/src/args.c
new file mode 100644
index 00000000..d91c6f6e
--- /dev/null
+++ b/userspace/src/args.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <getopt.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unit/core.h>
+#include <unit/args.h>
+#include <unit/io.h>
+
+static struct option core_opts[] = {
+        { "help",               0, NULL, 'h' },
+        { "verbose",            0, NULL, 'v' },
+        { "quiet",              0, NULL, 'q' },
+        { "no-color",           0, NULL, 'C' },
+
+        { "unit-load-path",     1, NULL, 'L' },
+
+        { NULL,                 0, NULL,  0  }
+};
+
+static const char *core_opts_str = "hvqCL:";
+
+void core_print_help(struct unit_fw *fw)
+{
+        const char **line, *help_msg[] = {
+"NvGpu Unit Testing FW. Basic usage\n",
+"\n",
+"  $ nvgpu_unit [options] <unit>\n",
+"\n",
+"Basic usage consists of one or more options and a particular unit test to\n",
+"execute.\n",
+"\n",
+"Available options are as follows:\n",
+"\n",
+"  -h, --help             Print this help message and exit.\n",
+"  -v, --verbose          Increment the verbosity level. Can be specified\n",
+"                         multiple times.\n",
+"  -q, --quiet            Set the verbose level back to 0.\n",
+"  -C, --no-color         Disable color printing; for example, if writing\n",
+"                         output to a file the color escape sequences will\n",
+"                         corrupt that file.\n",
+"  -L, --unit-load-path <PATH>\n",
+"                         Path to where the unit test libraries reside.\n",
+"\n",
+"Note: mandatory arguments to long arguments are mandatory for short\n",
+"arguments as well.\n",
+NULL
+        };
+
+        line = help_msg;
+        while (*line != NULL) {
+                core_msg(fw, "%s", *line);
+                line++;
+        }
+}
+
+static void set_arg_defaults(struct unit_fw_args *args)
+{
+        args->unit_load_path = DEFAULT_ARG_UNIT_LOAD_PATH;
+}
+
+/*
+ * Parse command line arguments.
+ */
+int core_parse_args(struct unit_fw *fw, int argc, char **argv)
+{
+        int c, opt_index;
+        struct unit_fw_args *args;
+
+        args = malloc(sizeof(*args));
+        if (!args)
+                return -1;
+
+        memset(args, 0, sizeof(*args));
+        set_arg_defaults(args);
+
+        fw->args = args;
+
+        while (1) {
+                c = getopt_long(argc, argv,
+                                core_opts_str, core_opts, &opt_index);
+
+                if (c == -1)
+                        break;
+
+                switch (c) {
+                case 'h':
+                        args->help = true;
+                        break;
+                case 'v':
+                        args->verbose_lvl += 1;
+                        break;
+                case 'q':
+                        args->verbose_lvl = 0;
+                        break;
+                case 'C':
+                        args->no_color = true;
+                        break;
+                case 'L':
+                        args->unit_load_path = optarg;
+                        break;
+                case '?':
+                        args->help = true;
+                        return -1;
+                default:
+                        core_err(fw, "bug?!\n");
+                        return -1;
+                }
+        }
+
+        return 0;
+}
diff --git a/userspace/src/exec.c b/userspace/src/exec.c
new file mode 100644
index 00000000..b9ba1336
--- /dev/null
+++ b/userspace/src/exec.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+
+#include <unit/io.h>
+#include <unit/core.h>
+#include <unit/unit.h>
+#include <unit/module.h>
+#include <unit/results.h>
+
+#include <nvgpu/posix/probe.h>
+
+/*
+ * Execute a module and all its subtests. This function builds a gk20a for the
+ * test to use by executing nvgpu_posix_probe() and nvgpu_posix_cleanup();
+ */
+static int core_exec_module(struct unit_fw *fw,
+                            struct unit_module *module)
+{
+        unsigned int i;
+        struct gk20a *g = fw->nvgpu.nvgpu_posix_probe();
+
+        if (!g)
+                return -1;
+
+        core_vbs(fw, 1, "Execing module: %s\n", module->name);
+
+        /*
+         * Execute each test within the module. No reinit is done between tests.
+         * Thats up to the module itself to handle. Any setup/teardown between
+         * unit tests must be handled within the module.
+         */
+        for (i = 0; i < module->nr_tests; i++) {
+                struct unit_module_test *t = module->tests + i;
+                int test_status;
+
+                core_msg(fw, "Running %s.%s\n", module->name, t->name);
+                test_status = t->fn(module, g, t->args);
+
+                if (test_status != UNIT_SUCCESS)
+                        core_msg_color(fw, C_RED,
+                                       "  Unit error! Test %s.%s FAILED!\n",
+                                       module->name, t->name);
+
+                core_add_test_record(fw, module, t,
+                                     test_status == UNIT_SUCCESS);
+        }
+
+        fw->nvgpu.nvgpu_posix_cleanup(g);
+
+        return 0;
+}
+
+/*
+ * Execute all modules loaded by the unit test framework.
+ */
+int core_exec(struct unit_fw *fw)
+{
+        int ret;
+        struct unit_module **modules;
+
+        for (modules = fw->modules; *modules != NULL; modules++) {
+                ret = core_exec_module(fw, *modules);
+
+                if (ret != 0)
+                        return ret;
+        }
+
+        return 0;
+}
diff --git a/userspace/src/io.c b/userspace/src/io.c
new file mode 100644
index 00000000..21ac54da
--- /dev/null
+++ b/userspace/src/io.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+
+#include <unit/io.h>
+#include <unit/args.h>
+#include <unit/core.h>
+#include <unit/unit.h>
+
+#define MAX_LOG_LINE_LENGTH             4096
+
+static void __core_print_file(struct unit_fw *fw, FILE *filp,
+                              const char *prefix, const char *msg,
+                              const char *color)
+{
+        if (color == NULL || args(fw)->no_color)
+                color = "";
+
+        fprintf(filp, "[%s%s%s] %s%s%s",
+                color, prefix, C_RESET,
+                color, msg, C_RESET);
+}
+
+__attribute__((format (printf, 3, 4)))
+void __core_print_stdout(struct unit_fw *fw, const char *color,
+                         const char *fmt, ...)
+{
+        va_list args;
+        char buf[MAX_LOG_LINE_LENGTH];
+
+        va_start(args, fmt);
+        vsnprintf(buf, MAX_LOG_LINE_LENGTH, fmt, args);
+        va_end(args);
+
+        buf[MAX_LOG_LINE_LENGTH - 1] = 0;
+
+        __core_print_file(fw, stdout, "C", buf, color);
+}
+
+__attribute__((format (printf, 2, 3)))
+void __core_print_stderr(struct unit_fw *fw, const char *fmt, ...)
+{
+        va_list args;
+        char buf[MAX_LOG_LINE_LENGTH];
+
+        va_start(args, fmt);
+        vsnprintf(buf, MAX_LOG_LINE_LENGTH, fmt, args);
+        va_end(args);
+
+        buf[MAX_LOG_LINE_LENGTH - 1] = 0;
+
+        __core_print_file(fw, stdout, "E", buf, C_RED);
+}
+
+__attribute__((format (printf, 3, 4)))
+void __unit_info_color(struct unit_module *unit, const char *color,
+                       const char *fmt, ...)
+{
+        va_list args;
+        char buf[MAX_LOG_LINE_LENGTH];
+        char *msg_start;
+        int written;
+
+        /*
+         * Default color for module prints is blue. Users can still turn this
+         * off with '-C'.
+         */
+        if (color == NULL)
+                color = C_BLUE;
+
+        /*
+         * First prepend the unit name to the print.
+         */
+        written = snprintf(buf, MAX_LOG_LINE_LENGTH, "  [%s] ", unit->name);
+
+        msg_start = buf + written;
+
+        va_start(args, fmt);
+        vsnprintf(msg_start, MAX_LOG_LINE_LENGTH - written, fmt, args);
+        va_end(args);
+
+        __core_print_file(unit->fw, stdout, "T", buf, color);
+}
diff --git a/userspace/src/module.c b/userspace/src/module.c
new file mode 100644
index 00000000..c111c6cb
--- /dev/null
+++ b/userspace/src/module.c
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <errno.h>
+#include <dlfcn.h>
+#include <dirent.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include <sys/types.h>
+
+#include <unit/core.h>
+#include <unit/io.h>
+#include <unit/args.h>
+#include <unit/unit.h>
+
+static int check_module(struct unit_fw *fw, struct unit_module *mod)
+{
+        unsigned int i;
+
+        /*
+         * Make sure this module has reasonable data.
+         */
+        if (mod->name == NULL) {
+                core_err(fw, "Unnamed module!");
+                return -1;
+        }
+
+        if (mod->tests == NULL || mod->nr_tests == 0) {
+                core_err(fw, "%s: Empty module!\n", mod->name);
+                return -1;
+        }
+
+        for (i = 0; i < mod->nr_tests; i++) {
+                struct unit_module_test *test = &mod->tests[i];
+
+                if (test->name == NULL) {
+                        core_err(fw, "%s: Unnamed test\n", mod->name);
+                        return -1;
+                }
+
+                if (test->fn == NULL) {
+                        core_err(fw, "%s: Test %s missing function \n",
+                                 mod->name, test->name);
+                        return -1;
+                }
+        }
+
+        return 0;
+}
+
+static struct unit_module *load_one_module(struct unit_fw *fw,
+                                           struct dirent *dent)
+{
+        void *lib_handle;
+        struct unit_module *mod;
+
+        core_vbs(fw, 1, "Loading: %s\n", dent->d_name);
+
+        lib_handle = dlopen(dent->d_name, RTLD_NOW);
+        if (lib_handle == NULL) {
+                core_err(fw, "Failed to load %s: %s\n",
+                         dent->d_name, dlerror());
+                return NULL;
+        }
+
+        mod = dlsym(lib_handle, "__unit_module__");
+        if (mod == NULL) {
+                core_vbs(fw, 1,
+                         "Failed to resolve __unit_module__ in %s: %s\n",
+                         dent->d_name, dlerror());
+                return NULL;
+        }
+
+        mod->lib_handle = lib_handle;
+        mod->fw = fw;
+
+        core_vbs(fw, 1, "  '%s' contains %lu tests\n", mod->name, mod->nr_tests);
+
+        return mod;
+}
+
+/*
+ * Load all the modules we can from the module load path. Return the list of
+ * loaded module as an array of pointers to modules. The returned list of
+ * modules is NULL terminated.
+ */
+struct unit_module **core_load_modules(struct unit_fw *fw)
+{
+        int nr_modules = 0, i;
+        DIR *load_dir;
+        struct dirent *ent;
+        const char *load_path = args(fw)->unit_load_path;
+        struct unit_module **modules, *mod;
+
+        core_vbs(fw, 1, "Loading modules from %s\n", load_path);
+
+        /*
+         * Open and count the number of files in the dir.
+         */
+        load_dir = opendir(load_path);
+        if (!load_dir) {
+                core_err(fw, "%s: Unable to open dir (%s)\n",
+                         load_path, strerror(errno));
+                return NULL;
+        }
+
+        while (readdir(load_dir) != NULL)
+                nr_modules += 1;
+
+        /* '.' and '..' should be skipped. */
+        nr_modules -= 2;
+
+        /*
+         * Now allocate necessary space for storing pointers to the modules and
+         * load the modules. +1 for the last NULL entry.
+         */
+        modules = malloc(sizeof(*modules) * (nr_modules + 1));
+        if (!modules) {
+                core_err(fw, "Out of mem! (huh?)\n");
+                goto err;
+        }
+
+        rewinddir(load_dir);
+        i = 0;
+        while ((ent = readdir(load_dir)) != NULL) {
+                if (strcmp(".", ent->d_name) == 0 ||
+                    strcmp("..", ent->d_name) == 0)
+                        continue;
+
+                mod = load_one_module(fw, ent);
+                if (mod == NULL)
+                        continue;
+
+                if (check_module(fw, mod) != 0)
+                        continue;
+
+                modules[i] = mod;
+                i++;
+        }
+
+        modules[i] = NULL;
+        return modules;
+
+err:
+        closedir(load_dir);
+        return NULL;
+}
diff --git a/userspace/src/nvgpu.c b/userspace/src/nvgpu.c
new file mode 100644
index 00000000..a9b7ced3
--- /dev/null
+++ b/userspace/src/nvgpu.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include <unit/io.h>
+#include <unit/core.h>
+
+/*
+ * Load libnvgpu-drv.so. This is done with dlopen() since this will make
+ * resolving addresses into symbols easier in the future.
+ *
+ * Also, this makes people think carefully about what functions to call in
+ * nvgpu-drv from the unit test FW. The interaction should really be limited
+ * and doing explicit name lookups is a good way to prevent too much coupling.
+ */
+int core_load_nvgpu(struct unit_fw *fw)
+{
+        const char *msg;
+
+        /*
+         * Specify a GLOBAL binding so that subsequently loaded unit tests see
+         * the nvgpu-drv library. They will of course need it (and will access
+         * it directly). I.e they will link against nvgpu-drv and this should
+         * satisfy that linkage.
+         */
+        fw->nvgpu_so = dlopen("libnvgpu-drv.so", RTLD_NOW | RTLD_GLOBAL);
+
+        if (fw->nvgpu_so == NULL) {
+                msg = dlerror();
+                core_err(fw, "Failed to load nvgpu-drv: %s\n", msg);
+                return -1;
+        }
+
+        /*
+         * We directly check the value of the returned symbol for these
+         * functions against NULL because if it is NULL then something is
+         * terribly wrong.
+         */
+
+        fw->nvgpu.nvgpu_posix_probe = dlsym(fw->nvgpu_so,
+                                            "nvgpu_posix_probe");
+        if (fw->nvgpu.nvgpu_posix_probe == NULL) {
+                msg = dlerror();
+                core_err(fw, "Failed to resolve nvgpu_posix_probe: %s\n", msg);
+                return -1;
+        }
+
+        fw->nvgpu.nvgpu_posix_cleanup = dlsym(fw->nvgpu_so,
+                                              "nvgpu_posix_cleanup");
+        if (fw->nvgpu.nvgpu_posix_cleanup == NULL) {
+                msg = dlerror();
+                core_err(fw, "Failed to resolve nvgpu_posix_cleanup: %s\n", msg);
+                return -1;
+        }
+
+        return 0;
+}
diff --git a/userspace/src/results.c b/userspace/src/results.c
new file mode 100644
index 00000000..ae077b82
--- /dev/null
+++ b/userspace/src/results.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <unit/io.h>
+#include <unit/core.h>
+#include <unit/unit.h>
+#include <unit/results.h>
+
+static int __init_results(struct unit_fw *fw)
+{
+        struct unit_results *results;
+
+        if (fw->results != NULL)
+                return 0;
+
+        results = malloc(sizeof(*results));
+        if (results == NULL)
+                return -1;
+
+        memset(results, 0, sizeof(*results));
+
+        fw->results = results;
+
+        return 0;
+}
+
+static void add_record(struct unit_test_list *list,
+                       struct unit_test_record *tr)
+{
+        /*
+         * First entry.
+         */
+        if (list->head == NULL) {
+                list->head = tr;
+                list->last = tr;
+                return;
+        }
+
+        /*
+         * Add to the end of the list and update the pointer to the last entry
+         * in the list. This gives us O(1) add time.
+         */
+        list->last->next = tr;
+        list->last = tr;
+}
+
+int core_add_test_record(struct unit_fw *fw,
+                         struct unit_module *mod,
+                         struct unit_module_test *test,
+                         bool success)
+{
+        struct unit_test_record *tr;
+
+        /*
+         * Dones nothing if results are already inited.
+         */
+        if (__init_results(fw) != 0)
+                return -1;
+
+        tr = malloc(sizeof(*tr));
+        if (tr == NULL)
+                return -1;
+
+        tr->mod = mod;
+        tr->test = test;
+        tr->status = success;
+        tr->next = NULL;
+
+        if (success)
+                add_record(&fw->results->passing, tr);
+        else
+                add_record(&fw->results->failing, tr);
+
+        fw->results->nr_tests += 1;
+        if (success)
+                fw->results->nr_passing += 1;
+
+        return 0;
+}
+
+void core_print_test_status(struct unit_fw *fw)
+{
+        struct unit_test_list *failing_tests = &fw->results->failing;
+        struct unit_test_record *rec;
+
+        /*
+         * Print stats for the tests.
+         */
+        core_msg(fw, "\n");
+        core_msg(fw, "Test results:\n");
+        core_msg(fw, "-------------\n");
+        core_msg(fw, "\n");
+        core_msg(fw, "  Passing: %d\n", fw->results->nr_passing);
+        core_msg(fw, "  Failing: %d\n",
+                 fw->results->nr_tests - fw->results->nr_passing);
+        core_msg(fw, "  Total:   %d\n", fw->results->nr_tests);
+        core_msg(fw, "\n");
+        core_msg(fw, "Failing tests:\n");
+        core_msg(fw, "\n");
+
+        for_record_in_test_list(failing_tests, rec) {
+                core_msg(fw, "  %s.%s\n",
+                         rec->mod->name,
+                         rec->test->name);
+        }
+}
diff --git a/userspace/src/unit_main.c b/userspace/src/unit_main.c
new file mode 100644
index 00000000..31c31d50
--- /dev/null
+++ b/userspace/src/unit_main.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * NvGpu unit testing framework!
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <unit/io.h>
+#include <unit/core.h>
+#include <unit/args.h>
+#include <unit/module.h>
+#include <unit/results.h>
+
+int main(int argc, char **argv)
+{
+        struct unit_fw *fw;
+        int ret;
+
+        fw = malloc(sizeof(*fw));
+        if (!fw)
+                return 1;
+
+        memset(fw, 0, sizeof(*fw));
+
+        ret = core_parse_args(fw, argc, argv);
+        if (ret) {
+                core_err(fw, "Enable to parse args.\n");
+                core_err(fw, "Exiting!\n");
+                return 1;
+        }
+
+        core_vbs(fw, 1, "Welcome to the nvgpu unit testing framework!\n");
+
+        if (args(fw)->help) {
+                core_print_help(fw);
+                return 1;
+        }
+
+        ret = core_load_nvgpu(fw);
+        if (ret != 0)
+                return ret;
+
+        fw->modules = core_load_modules(fw);
+        if (fw->modules == NULL)
+                return -1;
+
+        ret = core_exec(fw);
+        if (ret != 0)
+                return ret;
+
+        core_print_test_status(fw);
+
+        return 0;
+}
diff --git a/userspace/unit.sh b/userspace/unit.sh
new file mode 100755
index 00000000..078512f8
--- /dev/null
+++ b/userspace/unit.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#
+# Execute the unit test. Args to this script are passed on to the unit test
+# core. This just serves to set the LD_LIBRARY_PATH environment variable such
+# that unit tests are found and nvgpu-drv is found.
+#
+
+LD_LIBRARY_PATH="build:build/units"
+NVGPU_UNIT=build/nvgpu_unit
+
+export LD_LIBRARY_PATH
+
+echo "$ $NVGPU_UNIT $*"
+
+$NVGPU_UNIT $*
diff --git a/userspace/units/Makefile.units b/userspace/units/Makefile.units
new file mode 100644
index 00000000..64b9e07d
--- /dev/null
+++ b/userspace/units/Makefile.units
@@ -0,0 +1,45 @@
+# -*- mode: makefile -*-
+#
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+#
+# Inlcude this in unit test makefiles to get useful rules.
+#
+
+LOCAL_HEADERS = (shell ls *.h)
+
+.PHONY: all clean
+all: lib$(MODULE).so
+        @true
+
+#
+# Inherits from the top level makefile. As such the unit test make files cannot
+# be invoked by themselves.
+%.o : %.c # $(LOCAL_HEADERS)
+        $(CC) $(CFLAGS) -c -o $@ $<
+
+lib$(MODULE).so: $(OBJS)
+        $(CC) -shared -o $@ $(OBJS)
+        @mkdir -p $(UNIT_OUT)/
+        @cp $@ $(UNIT_OUT)/
+
+clean:
+        rm -rf *.o *.so