/**
 * Copyright 2022 Joshua Bakita
 * This program clocks how long it takes to read in a 1Gib buffer via page
 * faulting and via direct I/O, with efforts to match the overheads.
 *
 * More precisely, this program clocks:
 *   mmap(big_buffer);
 *   sequentially_walk(big_buffer);
 * for demand paging, and:
 *   malloc(big_buffer);
 *   read(big_buffer);
 *   walk(big_buffer);
 * for direct I/O. `big_buffer` is the arbitrary bytes contained in the first
 * 1GiB of /dev/nvme0n1, and should be filled with random data prior to
 * executing this microbenchmark.
 */
#define _GNU_SOURCE

#include <assert.h>
#include <fcntl.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>

#define GiB 1024l*1024l*1024l
#define s2ns(s) ((s)*1000l*1000l*1000l)
#define ns2us(ns) ((ns)/1000l)
#define PAGED_FILE "/dev/nvme0n1"
#define CLEAR_PAGECACHE_DENTRIES_INODES "3"

int seq_walk(char* mem, int len, char to_find) {
	int num_42 = 0;
	// Stride of 4096 bytes (one 4k page)
	for (int i = 4096; i < len; i += 4096)
		if (mem[i] == to_find)
			num_42++;
	return num_42;
}

// Subtract first parameter from second parameter. Return as nanoseconds.
long time_diff_ns(struct timespec start, struct timespec stop) {
	return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
}

int main(int argc, char **argv) {
	struct timespec start, stop;
	int iters, res;
	char* mem_out;

	if (argc != 2 || argv[1][0] == '-') {
		fprintf(stderr, "Usage: %s <number of iterations>\n", argv[0]);
		return 1;
	}
	iters = atoi(argv[1]);

        // Add comment with source details
	fprintf(stdout, "# Generated by '%s %s'. Row 1 is demand paging, row 2 is direct I/O.\n", argv[0], argv[1]);

        // Open control device for use to reset between iterations any caches
        // that the demand paging system might use.
	int clear_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
	if (clear_fd == -1) {
		perror("Unable to open /proc/sys/vm/drop_caches");
		return 1;
	}

	// Perform iterations of demand paging in
	for (int i = 0; i < iters; i++) {
                int fd = open(PAGED_FILE, O_RDWR);
                if (fd == -1) {
                        perror("Unable to open " PAGED_FILE);
                        return 1;
                }
                // Clear page cache
                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);

                // Begin paging by mapping the file into (unbacked) virtual memory
                clock_gettime(CLOCK_MONOTONIC_RAW, &start);
                mem_out = mmap(NULL, GiB, PROT_READ, MAP_PRIVATE, fd, 0);
                if (mem_out == MAP_FAILED) {
                        perror("Unable to mmap " PAGED_FILE);
                        return 1;
                }
                // Page fault in all the data via a sequential walk
                res = seq_walk(mem_out, GiB, 42); // Made up work to fool optimizer
                assert(res > 0);
                clock_gettime(CLOCK_MONOTONIC_RAW, &stop);

		long duration = time_diff_ns(start, stop);
		printf("%ld, ", ns2us(duration));

		munmap(mem_out, GiB);
		close(fd);
	}
	printf("\n");

	// Perform iterations of paging in via direct I/O
	for (int i = 0; i < iters; i++) {
		int fd = open(PAGED_FILE, O_RDWR | O_DIRECT);
		if (fd == -1) {
			perror("Unable to open " PAGED_FILE);
			return 1;
		}
                // Clear page cache
                write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);

		// Load in buffer via direct I/O
		clock_gettime(CLOCK_MONOTONIC_RAW, &start);
		// Aligned malloc(GiB) basicially
		res = posix_memalign((void**)&mem_out, 4096, GiB);
		if (res) {
			fprintf(stderr, "posix_memalign() failure. Error %d.", res);
			return 1;
		}
		res = read(fd, mem_out, GiB);
		if (res == -1) {
			perror("Unable to read 1GiB from /dev/nvme0n1");
			return 1;
		}
		if (res < GiB) {
			fprintf(stderr, "Unable to read the buffer all at once!");
			return 2;
		}
		// This sequential walk is not strictly necessary, but is
		// included to match the overheads of the demand paging path
		res = seq_walk(mem_out, GiB, 42); // Made up work to fool optimizer
		assert(res);
		clock_gettime(CLOCK_MONOTONIC_RAW, &stop);

		long duration = time_diff_ns(start, stop);
		printf("%ld, ", ns2us(duration));

		close(fd);
		free(mem_out);
	}
	printf("\n");
	return 0;
}