[XFS] Concurrent Multi-File Data Streams

In media spaces, video is often stored in a frame-per-file format. When
dealing with uncompressed realtime HD video streams in this format, it is
crucial that files do not get fragmented and that multiple files a placed
contiguously on disk.

When multiple streams are being ingested and played out at the same time,
it is critical that the filesystem does not cross the streams and
interleave them together as this creates seek and readahead cache miss
latency and prevents both ingest and playout from meeting frame rate
targets.

This patch set creates a "stream of files" concept into the allocator to
place all the data from a single stream contiguously on disk so that RAID
array readahead can be used effectively. Each additional stream gets
placed in different allocation groups within the filesystem, thereby
ensuring that we don't cross any streams. When an AG fills up, we select a
new AG for the stream that is not in use.

The core of the functionality is the stream tracking - each inode that we
create in a directory needs to be associated with the directories' stream.
Hence every time we create a file, we look up the directories' stream
object and associate the new file with that object.

Once we have a stream object for a file, we use the AG that the stream
object point to for allocations. If we can't allocate in that AG (e.g. it
is full) we move the entire stream to another AG. Other inodes in the same
stream are moved to the new AG on their next allocation (i.e. lazy
update).

Stream objects are kept in a cache and hold a reference on the inode.
Hence the inode cannot be reclaimed while there is an outstanding stream
reference. This means that on unlink we need to remove the stream
association and we also need to flush all the associations on certain
events that want to reclaim all unreferenced inodes (e.g. filesystem
freeze).

SGI-PV: 964469
SGI-Modid: xfs-linux-melb:xfs-kern:29096a

Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: Barry Naujok <bnaujok@sgi.com>
Signed-off-by: Donald Douwsma <donaldd@sgi.com>
Signed-off-by: Christoph Hellwig <hch@infradead.org>
Signed-off-by: Tim Shimmin <tes@sgi.com>
Signed-off-by: Vlad Apostolov <vapo@sgi.com>

4 files changed, 15 insertions, 0 deletions

diff --git a/fs/xfs/linux-2.6/xfs_globals.c b/fs/xfs/linux-2.6/xfs_globals.c
index ed3a5e1b4b67..bb72c3d4141f 100644
--- a/fs/xfs/linux-2.6/xfs_globals.c
+++ b/fs/xfs/linux-2.6/xfs_globals.c
@@ -46,6 +46,7 @@ xfs_param_t xfs_params = {
         .inherit_nosym  = {     0,              0,              1       },
         .rotorstep      = {     1,              1,              255     },
         .inherit_nodfrg = {     0,              1,              1       },
+        .fstrm_timer    = {     1,              50,             3600*100},
 };
 
 /*
diff --git a/fs/xfs/linux-2.6/xfs_linux.h b/fs/xfs/linux-2.6/xfs_linux.h
index af24a457d3a3..330c4ba9d404 100644
--- a/fs/xfs/linux-2.6/xfs_linux.h
+++ b/fs/xfs/linux-2.6/xfs_linux.h
@@ -123,6 +123,7 @@
 #define xfs_inherit_nosymlinks  xfs_params.inherit_nosym.val
 #define xfs_rotorstep           xfs_params.rotorstep.val
 #define xfs_inherit_nodefrag    xfs_params.inherit_nodfrg.val
+#define xfs_fstrm_centisecs     xfs_params.fstrm_timer.val
 
 #define current_cpu()           (raw_smp_processor_id())
 #define current_pid()           (current->pid)
diff --git a/fs/xfs/linux-2.6/xfs_sysctl.c b/fs/xfs/linux-2.6/xfs_sysctl.c
index cd6eaa44aa2b..bb997d75c05c 100644
--- a/fs/xfs/linux-2.6/xfs_sysctl.c
+++ b/fs/xfs/linux-2.6/xfs_sysctl.c
@@ -210,6 +210,17 @@ static ctl_table xfs_table[] = {
                 .extra1         = &xfs_params.inherit_nodfrg.min,
                 .extra2         = &xfs_params.inherit_nodfrg.max
         },
+        {
+                .ctl_name       = XFS_FILESTREAM_TIMER,
+                .procname       = "filestream_centisecs",
+                .data           = &xfs_params.fstrm_timer.val,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec_minmax,
+                .strategy       = &sysctl_intvec,
+                .extra1         = &xfs_params.fstrm_timer.min,
+                .extra2         = &xfs_params.fstrm_timer.max,
+        },
         /* please keep this the last entry */
 #ifdef CONFIG_PROC_FS
         {
diff --git a/fs/xfs/linux-2.6/xfs_sysctl.h b/fs/xfs/linux-2.6/xfs_sysctl.h
index a631fb8cc5ac..98b97e399d6f 100644
--- a/fs/xfs/linux-2.6/xfs_sysctl.h
+++ b/fs/xfs/linux-2.6/xfs_sysctl.h
@@ -47,6 +47,7 @@ typedef struct xfs_param {
         xfs_sysctl_val_t inherit_nosym; /* Inherit the "nosymlinks" flag. */
         xfs_sysctl_val_t rotorstep;     /* inode32 AG rotoring control knob */
         xfs_sysctl_val_t inherit_nodfrg;/* Inherit the "nodefrag" inode flag. */
+        xfs_sysctl_val_t fstrm_timer;   /* Filestream dir-AG assoc'n timeout. */
 } xfs_param_t;
 
 /*
@@ -86,6 +87,7 @@ enum {
         XFS_INHERIT_NOSYM = 19,
         XFS_ROTORSTEP = 20,
         XFS_INHERIT_NODFRG = 21,
+        XFS_FILESTREAM_TIMER = 22,
 };
 
 extern xfs_param_t      xfs_params;