1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
/**
* Copyright 2022 Joshua Bakita
* This program clocks how long it takes to read, and write, a 1GiB buffer via
* Linux direct I/O.
*
* More precisely, this program clocks:
* write(random_buffer);
* free(random_buffer);
* for writing data via direct I/O, and
* malloc(big_buffer);
* read(big_buffer);
* for reading in data via direct I/O. `random_buffer` is a preinitialized 1GiB
* buffer of random non-zero bytes.
*/
#define _GNU_SOURCE
#include <fcntl.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#define GiB 1024l*1024l*1024l
#define s2ns(s) ((s)*1000l*1000l*1000l)
#define ns2us(ns) ((ns)/1000l)
#define PAGED_FILE "/dev/nvme0n1"
#define CLEAR_PAGECACHE_DENTRIES_INODES "3"
int max(int x, int y) {return x > y ? x : y;}
// Original function from copy_only.cu
void fill_rand(char* buf, uint64_t buf_len) {
uint64_t i = 0;
for (; i < buf_len; i++)
buf[i] = max((rand() & 0xff), 1);
}
uint64_t count_zero(char* buf, uint64_t buf_len) {
uint64_t i = 0;
uint64_t num_zeros = 0;
for (; i < buf_len; i++)
num_zeros += (!buf[i]);
return num_zeros;
}
// Subtract first parameter from second parameter. Return as nanoseconds.
long time_diff_ns(struct timespec start, struct timespec stop) {
return (s2ns(stop.tv_sec) + stop.tv_nsec) - (s2ns(start.tv_sec) + start.tv_nsec);
}
int main(int argc, char **argv) {
struct timespec out_start, out_stop, in_start, in_stop;
int iters, res;
if (argc != 2 || argv[1][0] == '-') {
fprintf(stderr, "Usage: %s <number of iterations>\n", argv[0]);
return 1;
}
iters = atoi(argv[1]);
// If output is redirected, add comment with source details
if (!isatty(fileno(stdout)))
fprintf(stdout, "# Generated by '%s %s'\n", argv[0], argv[1]);
// Needed to allow page cache clearing between iterations
// Note: Shouldn't be needed with O_DIRECT, but include it just in case
int clear_fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
if (clear_fd == -1) {
perror("Unable to open /proc/sys/vm/drop_caches");
return 1;
}
// Print table header. One read, one write sample per following row
printf("out (us)\tin (us)\n");
for (int i = 0; i < iters; i++) {
char *mem_in, *mem_out;
int fd = open(PAGED_FILE, O_RDWR | O_DIRECT | O_SYNC);
if (fd == -1) {
perror("Unable to open " PAGED_FILE);
return 1;
}
// Clear page cache
write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1);
// Allocate and fill a buffer with random data
// Aligned malloc(GiB) basicially
res = posix_memalign((void**)&mem_in, 4096, GiB);
fill_rand(mem_in, GiB);
// Write and free buffer
clock_gettime(CLOCK_MONOTONIC_RAW, &out_start);
res = write(fd, mem_in, GiB);
free(mem_in);
clock_gettime(CLOCK_MONOTONIC_RAW, &out_stop);
if (res == -1) {
perror("Unable to write 1GiB to " PAGED_FILE);
return 1;
}
if (res != GiB) {
fprintf(stderr, "Unable to write the buffer all at once!");
return 2;
}
sleep(1); // Supposedly some other work would happen here
write(clear_fd, CLEAR_PAGECACHE_DENTRIES_INODES, 1); // Just in case O_DIRECT misbehaves
res = lseek(fd, 0, SEEK_SET); // Reposition offset
if (res == -1) {
perror("Unable to seek to offset 0 in " PAGED_FILE);
return 1;
}
if (res != 0) {
fprintf(stderr, "Unable to seek to offset 0 in " PAGED_FILE);
return 2;
}
// Allocate and read buffer
clock_gettime(CLOCK_MONOTONIC_RAW, &in_start);
// Aligned malloc(GiB) basicially
res = posix_memalign((void**)&mem_out, 4096, GiB);
if (res) {
fprintf(stderr, "posix_memalign() failure. Error %d.", res);
return 1;
}
res = read(fd, mem_out, GiB);
clock_gettime(CLOCK_MONOTONIC_RAW, &in_stop);
if (res == -1) {
perror("Unable to read 1GiB from " PAGED_FILE);
return 1;
}
if (res < GiB) {
fprintf(stderr, "Unable to read the buffer all at once!");
return 2;
}
// Check for valid contents
// TODO: Use CRC32 or something else a bit less dumb
res = count_zero(mem_out, GiB);
if (res > 0) {
fprintf(stderr, "Error: Found %d zeros in supposedly non-zero buffer after I/O!\n", res);
return 1;
}
// Print results as tab-seperated-values
printf("%ld\t%ld\n", ns2us(time_diff_ns(out_start, out_stop)),
ns2us(time_diff_ns(in_start, in_stop)));
close(fd);
free(mem_out);
}
return 0;
}
|
