Clean up and split demand_paging_speed.c from paging_speed.c

This also removes the --no-seq flag from paging_speed.c, which
never worked.

Cleanups include:
- Copyright notices
- Better usage messages & code comments
- More similar structure in the CPU-side benchmarks
- Random disk buffer initialization in demand_paging_speed.c
- Output comments detailing data source
- Stripping of unused or deprecated codepaths

At this point, the mapping of benchmarks to RTSS'22 plots is:
- paging_speed: Fig. 4
- demand_paging_speed: Fig. 10
- directio_paging_speed: Fig. 10
- gpu_paging_speed: Fig. 10
- gpu_paging_overhead_speed: Fig. 11
- gpu_paging_evil_task: Figs. 12 and 13

6 files changed, 288 insertions, 91 deletions

diff --git a/demand_paging_speed.c b/demand_paging_speed.c
new file mode 100644
index 0000000..22c3942
--- /dev/null
+++ b/demand_paging_speed.c
@@ -0,0 +1,153 @@
+/**
+ * Copyright 2022 Joshua Bakita
+ * This program clocks how long it takes to page-fault in a 1GiB buffer, making
+ * efforts to exclude userspace overheads.
+ *
+ * More precisely, this program clocks:
+ *   mmap(big_buffer);
+ *   sequentially_walk(big_buffer);
+ * where `big_buffer` is randomly filled. It then clocks another:
+ *   sequentially_walk(big_buffer);
+ * subtracts that time from the first one, and outputs the result. /dev/nvme0n1
+ * is preinitialized with random non-zero bytes.
+ */
+#define _GNU_SOURCE
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#define GiB 1024l*1024l*1024l
+#define s2ns(s) ((s)*1000l*1000l*1000l)
+#define ns2us(ns) ((ns)/1000l)
+#define PAGED_FILE "/dev/nvme0n1"
+#define CLEAR_PAGECACHE_DENTRIES_INODES "3"
+int max(int x, int y) {return x > y ? x : y;}
+
+int seq_walk(char* mem, int len, char to_find) {
+        int num_42 = 0;
+        // Stride of 4096 bytes (one 4k page)
+        for (int i = 4096; i < len; i += 4096)
+                if (mem[i] == to_find)
+                        num_42++;
+        return num_42;
+}
+
+// Original function from copy_only.cu
+void fill_rand(char* buf, uint64_t buf_len) {
+        uint64_t i = 0;
+        for (; i < buf_len; i++)
+                buf[i] = max((rand() & 0xff), 1);
+}
+
+uint64_t count_zero(char* buf, uint64_t buf_len) {
+        uint64_t i = 0;
+        uint64_t num_zeros = 0;
+        for (; i < buf_len; i++)
+                num_zeros += (!buf[i]);
+        return num_zeros;
+}
+
+// Subtract first parameter from second parameter. Return as nanoseconds.
+long time_diff_ns(struct timespec start, struct timespec stop) {
+        return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
+}
+
+int main(int argc, char **argv) {
+        struct timespec start, stop, stop2;
+        int iters, res;
+
+        if (argc != 2 || argv[1][0] == '-') {
+                fprintf(stderr, "Usage: %s <number of iterations>\n", argv[0]);
+                return 1;
+        }
+        iters = atoi(argv[1]);
+
+        // If output is redirected, add comment with source details
+        if (!isatty(fileno(stdout)))
+                fprintf(stdout, "# Generated by '%s %s'\n", argv[0], argv[1]);
+
+        // Open control device for use to reset between iterations any caches
+        // that the demand paging system might use.
+        int clear_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
+        if (clear_fd == -1) {
+                perror("Unable to open /proc/sys/vm/drop_caches");
+                return 1;
+        }
+
+        // Fill the area that we'll read from with random data
+        // (Direct I/O just used here for convenience.)
+        char *mem_in, *mem_out; // In and out of SSD
+        int fd = open(PAGED_FILE, O_RDWR | O_DIRECT | O_SYNC);
+        if (fd == -1) {
+                perror("Unable to open " PAGED_FILE);
+                return 1;
+        }
+        // Aligned malloc(GiB) basicially
+        res = posix_memalign((void**)&mem_in, 4096, GiB);
+        fill_rand(mem_in, GiB);
+        res = write(fd, mem_in, GiB);
+        if (res == -1) {
+                perror("Unable to write inital 1GiB random buffer to " PAGED_FILE);
+                return 1;
+        }
+        if (res != GiB) {
+                fprintf(stderr, "Unable to write the buffer all at once!");
+                return 2;
+        }
+        free(mem_in);
+        close(fd);
+
+        // Output table header. One read sample per row.
+        printf("in (us)\n");
+        // Perform iterations of demand paging in
+        for (int i = 0; i < iters; i++) {
+                int fd = open(PAGED_FILE, O_RDWR);
+                if (fd == -1) {
+                        perror("Unable to open " PAGED_FILE);
+                        return 1;
+                }
+                // Clear page cache
+                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);
+
+                // Begin paging by mapping the file into (unbacked) virtual memory
+                clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+                mem_out = mmap(NULL, GiB, PROT_READ, MAP_PRIVATE, fd, 0);
+                if (mem_out == MAP_FAILED) {
+                        perror("Unable to mmap " PAGED_FILE);
+                        return 1;
+                }
+                // Page fault in all the data via a sequential walk
+                res = seq_walk(mem_out, GiB, 42); // Made up work to fool optimizer
+                assert(res > 0);
+                clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
+
+                // This benchmark attempts to capture data comparable to the
+                // GPU or Direct I/O paging benchmarks. As they don't have to
+                // perform a sequential walk, we emulate that here by measuring
+                // how long a sequential walk takes on its own, and subtract
+                // that from `stop - start`.
+                res = seq_walk(mem_out, GiB, 7); // Made up work to fool optimizer
+                assert(res > 0);
+                clock_gettime(CLOCK_MONOTONIC_RAW, &stop2);
+
+                // `stop2 - stop` is how long just a sequential walk takes
+                long seq_walk_time = time_diff_ns(stop, stop2);
+                // `stop - start` is how long the faulting sequential walk took
+                long demand_paging_raw_duration = time_diff_ns(start, stop);
+                // `(stop - start) - (stop2 - stop)` is emulated duration
+                long demand_paging_duration = demand_paging_raw_duration - seq_walk_time;
+                printf("%ld\n", ns2us(demand_paging_duration));
+
+                munmap(mem_out, GiB);
+                close(fd);
+        }
+        return 0;
+}
diff --git a/directio_paging_speed.c b/directio_paging_speed.c
index b0a01d3..9dd6598 100644
--- a/directio_paging_speed.c
+++ b/directio_paging_speed.c
@@ -1,19 +1,34 @@
+/**
+ * Copyright 2022 Joshua Bakita
+ * This program clocks how long it takes to read, and write, a 1GiB buffer via
+ * Linux direct I/O.
+ *
+ * More precisely, this program clocks:
+ *   write(random_buffer);
+ *   free(random_buffer);
+ * for writing data via direct I/O, and
+ *   malloc(big_buffer);
+ *   read(big_buffer);
+ * for reading in data via direct I/O. `random_buffer` is a preinitialized 1GiB
+ * buffer of random non-zero bytes.
+ */
 #define _GNU_SOURCE
 
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <fcntl.h>
-#include <stdio.h>
 #include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
-#include <stdlib.h>
 
 #define GiB 1024l*1024l*1024l
 #define s2ns(s) ((s)*1000l*1000l*1000l)
 #define ns2us(ns) ((ns)/1000l)
 #define PAGED_FILE "/dev/nvme0n1"
+#define CLEAR_PAGECACHE_DENTRIES_INODES "3"
 int max(int x, int y) {return x > y ? x : y;} 
 
 // Original function from copy_only.cu
@@ -31,16 +46,24 @@ uint64_t count_zero(char* buf, uint64_t buf_len) {
         return num_zeros;
 }
 
+// Subtract first parameter from second parameter. Return as nanoseconds.
 long time_diff_ns(struct timespec start, struct timespec stop) {
         return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
 }
 
 int main(int argc, char **argv) {
         struct timespec out_start, out_stop, in_start, in_stop;
-        int iters = 1;
-        int res;
-        if (argc > 1)
-                iters = atoi(argv[1]);
+        int iters, res;
+
+        if (argc != 2 || argv[1][0] == '-') {
+                fprintf(stderr, "Usage: %s <number of iterations>\n", argv[0]);
+                return 1;
+        }
+        iters = atoi(argv[1]);
+
+        // If output is redirected, add comment with source details
+        if (!isatty(fileno(stdout)))
+                fprintf(stdout, "# Generated by '%s %s'\n", argv[0], argv[1]);
 
         // Needed to allow page cache clearing between iterations
         // Note: Shouldn't be needed with O_DIRECT, but include it just in case
@@ -49,8 +72,8 @@ int main(int argc, char **argv) {
                 perror("Unable to open /proc/sys/vm/drop_caches");
                 return 1;
         }
-        char clear_cmd = '3';
 
+        // Print table header. One read, one write sample per following row
         printf("out (us)\tin (us)\n");
         for (int i = 0; i < iters; i++) {
                 char *mem_in, *mem_out;
@@ -60,7 +83,7 @@ int main(int argc, char **argv) {
                         return 1;
                 }
                 // Clear page cache
-                write(clear_fd, &clear_cmd, 1);
+                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);
                 // Allocate and fill a buffer with random data
                 // Aligned malloc(GiB) basicially
                 res = posix_memalign((void**)&mem_in, 4096, GiB);
@@ -81,7 +104,7 @@ int main(int argc, char **argv) {
                 }
                 
                 sleep(1); // Supposedly some other work would happen here
-                write(clear_fd, &clear_cmd, 1); // Just in case O_DIRECT misbehaves
+                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1); // Just in case O_DIRECT misbehaves
                 res = lseek(fd, 0, SEEK_SET); // Reposition offset
                 if (res == -1) {
                         perror("Unable to seek to offset 0 in " PAGED_FILE);
diff --git a/gpu_paging_evil_task.cu b/gpu_paging_evil_task.cu
index 7c1ab59..4b358a4 100644
--- a/gpu_paging_evil_task.cu
+++ b/gpu_paging_evil_task.cu
@@ -1,3 +1,9 @@
+/**
+ * Copyright 2022 Joshua Bakita
+ *
+ * Run many iterations of GPU paging on the same CUDA context, and print the
+ * average time per iteration.
+ */
 #include <stdio.h>
 #include <cuda.h>
 #include <curand_kernel.h> // curandState_t and curand
@@ -78,6 +84,7 @@ __global__ void count_zero(char* buf, uint64_t buf_len) {
         atomicAdd_block((unsigned long long int*)&gpu_res, (unsigned long long int)num_zero);
 }
 
+// Subtract first parameter from second parameter. Return as nanoseconds.
 long time_diff_ns(struct timespec start, struct timespec stop) {
         return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
 }
@@ -89,8 +96,8 @@ int main(int argc, char **argv) {
         cudaStream_t stream1;
         cudaError_t err;
         int iters;
-        if (argc != 2) {
-                fprintf(stderr, "Usage: %s <iterations>\n", argv[0]);
+        if (argc != 2 || argv[1][0] == '-') {
+                fprintf(stderr, "Usage: %s <number of iterations>\n", argv[0]);
                 return 1;
         }
         iters = atoi(argv[1]);
diff --git a/gpu_paging_overhead_speed.cu b/gpu_paging_overhead_speed.cu
index c7a0f3a..0cf364e 100644
--- a/gpu_paging_overhead_speed.cu
+++ b/gpu_paging_overhead_speed.cu
@@ -1,3 +1,6 @@
+/**
+ * Copyright 2022 Joshua Bakita
+ */
 #include <stdio.h>
 #include <cuda.h>
 #include <curand_kernel.h> // curandState_t and curand
@@ -78,6 +81,7 @@ __global__ void count_zero(char* buf, uint64_t buf_len) {
         atomicAdd_block((unsigned long long int*)&gpu_res, (unsigned long long int)num_zero);
 }
 
+// Subtract first parameter from second parameter. Return as nanoseconds.
 long time_diff_ns(struct timespec start, struct timespec stop) {
         return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
 }
diff --git a/gpu_paging_speed.cu b/gpu_paging_speed.cu
index 72cb82e..bca02d9 100644
--- a/gpu_paging_speed.cu
+++ b/gpu_paging_speed.cu
@@ -1,3 +1,6 @@
+/**
+ * Copyright 2022 Joshua Bakita
+ */
 #include <stdio.h>
 #include <cuda.h>
 #include <curand_kernel.h> // curandState_t and curand
@@ -78,6 +81,7 @@ __global__ void count_zero(char* buf, uint64_t buf_len) {
         atomicAdd_block((unsigned long long int*)&gpu_res, (unsigned long long int)num_zero);
 }
 
+// Subtract first parameter from second parameter. Return as nanoseconds.
 long time_diff_ns(struct timespec start, struct timespec stop) {
         return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
 }
diff --git a/paging_speed.c b/paging_speed.c
index 4ad56e2..7b0cd22 100644
--- a/paging_speed.c
+++ b/paging_speed.c
@@ -1,19 +1,37 @@
+/**
+ * Copyright 2022 Joshua Bakita
+ * This program clocks how long it takes to read in a 1Gib buffer via page
+ * faulting and via direct I/O, with efforts to match the overheads.
+ *
+ * More precisely, this program clocks:
+ *   mmap(big_buffer);
+ *   sequentially_walk(big_buffer);
+ * for demand paging, and:
+ *   malloc(big_buffer);
+ *   read(big_buffer);
+ *   walk(big_buffer);
+ * for direct I/O. `big_buffer` is the arbitrary bytes contained in the first
+ * 1GiB of /dev/nvme0n1, and should be filled with random data prior to
+ * executing this microbenchmark.
+ */
 #define _GNU_SOURCE
 
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
+#include <assert.h>
 #include <fcntl.h>
-#include <stdio.h>
 #include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
-#include <stdlib.h>
-#include <string.h> // strlen()
 
 #define GiB 1024l*1024l*1024l
 #define s2ns(s) ((s)*1000l*1000l*1000l)
-
+#define ns2us(ns) ((ns)/1000l)
+#define PAGED_FILE "/dev/nvme0n1"
+#define CLEAR_PAGECACHE_DENTRIES_INODES "3"
 
 int seq_walk(char* mem, int len, char to_find) {
         int num_42 = 0;
@@ -24,91 +42,82 @@ int seq_walk(char* mem, int len, char to_find) {
         return num_42;
 }
 
+// Subtract first parameter from second parameter. Return as nanoseconds.
 long time_diff_ns(struct timespec start, struct timespec stop) {
         return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
 }
-// TODO: take *num_42, return time
-
-//#define PAGED_FILE "/home/jbakita/1gib_random_f"
-#define PAGED_FILE "/dev/nvme0n1"
 
 int main(int argc, char **argv) {
-        int iters = 1;
-        int no_seq = 0;
-        if (argc > 1)
-                iters = atoi(argv[1]);
-        if (argc > 2) {
-                no_seq = strncmp(argv[2], "--no-seq", strlen(argv[2])) ? 1 : 0;
-                fprintf(stderr, "Skipping seq, but using no-seq emulation with demand paging\n");
+        struct timespec start, stop;
+        int iters, res;
+        char* mem_out;
+
+        if (argc != 2 || argv[1][0] == '-') {
+                fprintf(stderr, "Usage: %s <number of iterations>\n", argv[0]);
+                return 1;
         }
-        struct timespec start, stop, seq_stop;
+        iters = atoi(argv[1]);
+
+        // Add comment with source details
+        fprintf(stdout, "# Generated by '%s %s'. Row 1 is demand paging, row 2 is direct I/O.\n", argv[0], argv[1]);
+
+        // Open control device for use to reset between iterations any caches
+        // that the demand paging system might use.
         int clear_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
         if (clear_fd == -1) {
                 perror("Unable to open /proc/sys/vm/drop_caches");
                 return 1;
         }
 
-        char clear_cmd = '3';
+        // Perform iterations of demand paging in
         for (int i = 0; i < iters; i++) {
-                int fd = open(PAGED_FILE, O_RDWR);
-                if (fd == -1) {
-                        perror("Unable to open " PAGED_FILE);
-                        return 1;
-                }
-                // Clear page cache
-                write(clear_fd, &clear_cmd, 1);
-                // VIA MMAP
-                clock_gettime(CLOCK_MONOTONIC_RAW, &start);
-                char* mem = mmap(NULL, GiB, PROT_READ, MAP_PRIVATE, fd, 0);
-                if (mem == MAP_FAILED) {
-                        perror("Unable to mmap " PAGED_FILE);
-                        return 1;
-                }
-                // Fault on all the pages via a sequential walk
-                int num_42 = seq_walk(mem, GiB, 42);
-                clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
-                int num_52 = 0;
-                if (no_seq)
-                        num_52 = seq_walk(mem, GiB, 52);
-                clock_gettime(CLOCK_MONOTONIC_RAW, &seq_stop);
-                if (num_52)
-                        fprintf(stderr, "Something is seriously wrong! Found a 52 in a buffer that should be only 42s\n");
-                long duration = (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
-                // Emulate the time demand paging would take if we didn't have to walk
-                if (no_seq) {
-                        long seq_time = time_diff_ns(stop, seq_stop);
-                        duration -= seq_time;
-                }
-                if (iters == 1) {
-                        printf("Took %ldus via mmap\n", duration / 1000);
-                        printf("Read %d 42s of %ld expected\n", num_42, GiB/4096);
-                } else {
-                        printf("%ld, ", duration / 1000);
-                }
-                munmap(mem, GiB);
+                int fd = open(PAGED_FILE, O_RDWR);
+                if (fd == -1) {
+                        perror("Unable to open " PAGED_FILE);
+                        return 1;
+                }
+                // Clear page cache
+                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);
+
+                // Begin paging by mapping the file into (unbacked) virtual memory
+                clock_gettime(CLOCK_MONOTONIC_RAW, &start);
+                mem_out = mmap(NULL, GiB, PROT_READ, MAP_PRIVATE, fd, 0);
+                if (mem_out == MAP_FAILED) {
+                        perror("Unable to mmap " PAGED_FILE);
+                        return 1;
+                }
+                // Page fault in all the data via a sequential walk
+                res = seq_walk(mem_out, GiB, 42); // Made up work to fool optimizer
+                assert(res > 0);
+                clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
+
+                long duration = time_diff_ns(start, stop);
+                printf("%ld, ", ns2us(duration));
+
+                munmap(mem_out, GiB);
                 close(fd);
         }
-        if (iters > 1)
-                printf("\n");
+        printf("\n");
 
+        // Perform iterations of paging in via direct I/O
         for (int i = 0; i < iters; i++) {
-                char* mem;
                 int fd = open(PAGED_FILE, O_RDWR | O_DIRECT);
                 if (fd == -1) {
                         perror("Unable to open " PAGED_FILE);
                         return 1;
                 }
-                // Clear page cache
-                write(clear_fd, &clear_cmd, 1);
-                // VIA READ
+                // Clear page cache
+                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);
+
+                // Load in buffer via direct I/O
                 clock_gettime(CLOCK_MONOTONIC_RAW, &start);
                 // Aligned malloc(GiB) basicially
-                int res = posix_memalign((void**)&mem, 4096, GiB);
+                res = posix_memalign((void**)&mem_out, 4096, GiB);
                 if (res) {
                         fprintf(stderr, "posix_memalign() failure. Error %d.", res);
                         return 1;
                 }
-                res = read(fd, mem, GiB);
+                res = read(fd, mem_out, GiB);
                 if (res == -1) {
                         perror("Unable to read 1GiB from /dev/nvme0n1");
                         return 1;
@@ -117,21 +126,18 @@ int main(int argc, char **argv) {
                         fprintf(stderr, "Unable to read the buffer all at once!");
                         return 2;
                 }
-                int num_42 = 0;
-                if (!no_seq)
-                        num_42 = seq_walk(mem, GiB, 42); // Not strictly necessary, but to match mmap path overheads
+                // This sequential walk is not strictly necessary, but is
+                // included to match the overheads of the demand paging path
+                res = seq_walk(mem_out, GiB, 42); // Made up work to fool optimizer
+                assert(res);
                 clock_gettime(CLOCK_MONOTONIC_RAW, &stop);
-                if (iters == 1) {
-                        printf("Took %ldus via read\n", ((s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec)) / 1000);
-                        if (!no_seq)
-                                printf("Read %d 42s of %ld expected\n", num_42, GiB/4096);
-                } else {
-                        printf("%ld, ", ((s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec)) / 1000);
-                }
+
+                long duration = time_diff_ns(start, stop);
+                printf("%ld, ", ns2us(duration));
+
                 close(fd);
-                free(mem);
+                free(mem_out);
         }
-        if (iters > 1)
-                printf("\n");
+        printf("\n");
         return 0;
 }