Support for raw read/write overheads.

We need information about the time it takes to actually
read and write data when we generate task sets for
actual execution. This patch adds these overheads and
support for them.

14 files changed, 253 insertions, 1 deletions

diff --git a/ecrts14/ecrts14.py b/ecrts14/ecrts14.py
index 9f55ea8..bdf9f5e 100755
--- a/ecrts14/ecrts14.py
+++ b/ecrts14/ecrts14.py
@@ -24,7 +24,7 @@ from pprint import pprint
 import traceback
 
 from schedcat.model.tasks import SporadicTask, TaskSystem
-from schedcat.overheads.model import Overheads, CacheDelay, ConsumerOverheads, ProducerOverheads
+from schedcat.overheads.model import Overheads, CacheDelay, ConsumerOverheads, ProducerOverheads, RawOverheads
 
 import schedcat.model.resources as resources
 import schedcat.generator.tasks as tasks
@@ -245,6 +245,23 @@ def get_ovh_dir():
     ovh_dir = os.path.join(parent, 'overheads')
     return ovh_dir
 
+def get_read_overheads(dp, _system):
+    # always assume no polluters
+    polluters = 'False'
+    read_file = '%s/read/dro_hot_host=%s_lvl=mem_polluters=%s_walk=%s_hpages=%s_upages=%s_type=%s.csv' % (dp.host, dp.host, polluters, dp.walk, str(dp.huge_pages), str(dp.uncached), dp.ovh_type)
+    read_file = os.path.join(get_ovh_dir(), read_file)
+    # reuse the CPMD class
+    ro = RawOverheads.from_file(read_file)
+    return ro
+
+def get_write_overheads(dp, _system):
+    # always assume no polluters
+    polluters = 'False'
+    write_file = '%s/write/dwo_cold_host=%s_lvl=mem_polluters=%s_walk=%s_hpages=%s_upages=%s_type=%s.csv' % (dp.host, dp.host, polluters, dp.walk, str(dp.huge_pages), str(dp.uncached), dp.ovh_type)
+    write_file = os.path.join(get_ovh_dir(), write_file)
+    wo = RawOverheads.from_file(write_file)
+    return wo
+
 def get_consumer_overheads(dp, _system):
     co_file = '%s/consumer/dco_host=%s_lvl=mem_polluters=%s_walk=%s_hpages=%s_upages=%s_type=%s.csv' % (dp.host, dp.host, str(dp.polluters), dp.walk, str(dp.huge_pages), str(dp.uncached), dp.ovh_type)
     co_file = os.path.join(get_ovh_dir(), co_file)
@@ -276,6 +293,8 @@ def get_overheads(dp, system = None):
     ovh.cache_affinity_loss.set_max_wss(max_wss)
     ovh.consumer = get_consumer_overheads(dp, system)
     ovh.producer = get_producer_overheads(dp)
+    ovh.read = get_read_overheads(dp, system)
+    ovh.write = get_write_overheads(dp, system)
     return ovh
 
 def process_dp(_dp):
diff --git a/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=max.csv b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=max.csv
new file mode 100644
index 0000000..8f6bb5f
--- /dev/null
+++ b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=max.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,1.496484,1.605251,1.605251,1.425223
+8,2.764182,3.082982,3.112986,2.726676
+16,5.404594,6.109705,5.963432,5.295827
+32,11.079231,11.578059,11.874355,10.700422
+64,20.808251,20.864510,20.823254,21.082044
+128,41.428973,41.387717,41.357712,41.380216
+256,82.250352,81.946554,82.175340,81.867792
+512,162.895452,163.120488,163.139241,163.000469
+1024,326.090952,326.451008,326.132208,326.315987
+2048,658.051571,657.972808,655.771214,656.405063
+3072,995.225504,994.617909,994.801688,999.381153
+4096,1344.270042,1344.491327,1342.784810,1363.052977
+8192,2945.789030,2984.885138,2944.322550,3134.586029
+12288,5172.493202,5166.522269,5052.181903,5128.266292
+16384,7473.162682,7465.072668,7407.204876,7055.939991
diff --git a/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=mean.csv b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=mean.csv
new file mode 100644
index 0000000..f753f01
--- /dev/null
+++ b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=mean.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,1.329539,1.365437,1.361324,1.329453
+8,2.623422,2.738913,2.733933,2.647975
+16,5.259937,5.443709,5.429345,5.290949
+32,10.445262,10.538843,11.026184,10.234948
+64,20.049110,20.188133,20.113750,20.198319
+128,40.522704,40.453958,40.402399,40.414077
+256,80.911254,80.857134,80.829766,80.828637
+512,161.796456,161.716880,161.702072,161.682193
+1024,324.034341,323.812891,324.030671,323.893428
+2048,644.784059,645.060955,649.029355,649.424119
+3072,983.714642,983.682240,986.656193,988.471229
+4096,1330.413409,1330.490865,1330.429751,1337.030575
+8192,2832.483927,2832.665365,2834.980499,2838.822303
+12288,4944.013842,4943.471211,4931.581177,4740.238789
+16384,7320.783576,7320.925097,7286.839523,6885.715781
diff --git a/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=median.csv b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=median.csv
new file mode 100644
index 0000000..dc1407b
--- /dev/null
+++ b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=median.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,1.361463,1.350211,1.350211,1.350211
+8,2.632911,2.670417,2.722925,2.659165
+16,5.273324,5.292077,5.292077,5.292077
+32,10.486639,10.340366,11.341772,10.209095
+64,19.938115,20.193155,20.151899,20.170652
+128,40.738865,40.581341,40.581341,40.558837
+256,81.023910,80.948898,80.877637,80.885138
+512,161.924051,161.875293,161.875293,161.811533
+1024,323.851852,323.848101,324.020628,323.893108
+2048,643.754337,644.200656,648.750117,649.128926
+3072,983.503047,983.493671,986.516643,987.900609
+4096,1329.880919,1329.847164,1329.834037,1334.608533
+8192,2830.242850,2830.207220,2830.792311,2843.643694
+12288,4950.128458,4951.099859,4941.670886,4712.350680
+16384,7327.756212,7327.041725,7288.555087,6884.159400
diff --git a/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=max.csv b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=max.csv
new file mode 100644
index 0000000..bcb6b96
--- /dev/null
+++ b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=max.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,1.556493,1.428973,1.428973,1.428973
+8,2.827942,2.869198,2.869198,2.730427
+16,5.539616,6.090952,6.068448,5.502110
+32,11.296765,11.214252,11.157993,11.049226
+64,22.503516,22.676043,22.428504,21.280825
+128,48.296296,48.840131,47.692452,43.889358
+256,106.411627,107.716831,102.949836,90.182841
+512,243.000469,245.164557,229.524613,202.051571
+1024,579.064229,582.796062,547.225504,535.816221
+2048,2182.833568,2185.473980,2191.606188,2172.988279
+3072,5326.788561,5353.481481,5324.823254,5248.521331
+4096,8164.429442,8140.868261,8138.816690,8055.752461
+8192,17917.442100,17901.022035,17920.127520,17761.218940
+12288,27582.949836,27589.633380,27509.858415,27347.454290
+16384,36948.091889,36974.330989,37027.971871,36769.800281
diff --git a/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=mean.csv b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=mean.csv
new file mode 100644
index 0000000..1c01e83
--- /dev/null
+++ b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=mean.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,1.301124,1.321736,1.323728,1.325868
+8,2.592648,2.665823,2.663852,2.660390
+16,5.178405,5.374015,5.362400,5.284798
+32,10.384490,10.362681,10.359040,10.348364
+64,20.623417,20.643917,20.618383,20.416869
+128,41.965868,42.074264,41.973729,41.178157
+256,86.732047,87.045687,86.553105,83.787203
+512,185.548887,185.469367,185.298756,178.074153
+1024,441.833512,442.755766,442.733299,436.729801
+2048,1870.433699,1866.428497,1846.958964,1919.773324
+3072,4716.482489,4696.972502,4716.464835,4841.703287
+4096,7398.841764,7392.313182,7408.211399,7456.805805
+8192,16881.990744,16884.659619,16889.357192,16740.921317
+12288,25915.902040,25899.189369,25924.135738,25704.645515
+16384,34941.498311,34941.853653,34974.096600,34683.863463
diff --git a/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=median.csv b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=median.csv
new file mode 100644
index 0000000..735000c
--- /dev/null
+++ b/ecrts14/overheads/ludwig/read/dro_hot_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=median.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,1.305204,1.299578,1.305204,1.346460
+8,2.632911,2.666667,2.666667,2.666667
+16,5.273324,5.292077,5.292077,5.288326
+32,10.332865,10.329114,10.347867,10.351617
+64,20.474449,20.474449,20.474449,20.391936
+128,41.382091,41.466479,41.436474,40.945148
+256,84.827004,84.973277,84.823254,83.401782
+512,180.759494,180.350680,181.331458,176.697609
+1024,436.770745,437.014534,439.900609,433.588373
+2048,1871.576184,1867.934365,1857.168308,1915.722457
+3072,4770.557900,4759.253633,4771.939991,4841.715893
+4096,7436.785748,7425.742147,7433.888420,7468.035631
+8192,16892.825129,16884.343179,16893.719644,16739.814346
+12288,25939.859353,25925.477731,25949.820910,25730.887951
+16384,34912.397562,34916.249414,34954.743554,34658.897328
diff --git a/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=max.csv b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=max.csv
new file mode 100644
index 0000000..95984cd
--- /dev/null
+++ b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=max.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,2.640413,2.636662,2.632911,2.632911
+8,5.149555,5.127051,5.157056,5.134552
+16,10.077825,10.074074,10.070323,10.092827
+32,20.571964,20.583216,20.526957,20.613221
+64,41.290202,41.421472,41.511486,41.293952
+128,84.725738,85.033286,85.025785,84.763244
+256,171.469292,171.413033,171.863104,171.645570
+512,354.287857,354.261603,354.595406,354.497890
+1024,735.471167,737.946554,739.180497,735.977496
+2048,1539.525551,1531.960619,1532.695734,1523.908111
+3072,2362.025316,2365.775902,2380.016878,2357.644632
+4096,3269.367089,3224.577590,3289.984060,3260.290670
+8192,6451.514299,6450.910455,6454.098453,6461.697140
+12288,8768.506329,8752.322550,8695.369902,8713.492733
+16384,10710.556024,10712.881388,10701.652133,10702.938584
diff --git a/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=mean.csv b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=mean.csv
new file mode 100644
index 0000000..ec87e50
--- /dev/null
+++ b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=mean.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,2.216750,2.214398,2.215836,2.211508
+8,4.483534,4.475152,4.476123,4.471763
+16,9.065764,9.057559,9.067796,9.062023
+32,18.327605,18.340298,18.332671,18.354392
+64,37.167393,37.167905,37.202521,37.136972
+128,75.376831,75.455987,75.391196,75.424597
+256,153.000048,152.854846,153.092226,153.043831
+512,310.460805,310.863753,310.850198,310.650921
+1024,631.166070,632.391552,631.946338,630.905431
+2048,1280.020718,1282.301684,1280.359659,1279.650137
+3072,1933.063224,1934.510189,1932.437136,1931.613963
+4096,2588.714297,2587.270341,2592.637676,2586.894382
+8192,5164.514362,5170.230562,5165.874272,5166.086110
+12288,7612.610216,7609.020434,7610.426490,7607.250761
+16384,10118.067882,10118.808164,10129.793417,10128.194430
diff --git a/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=median.csv b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=median.csv
new file mode 100644
index 0000000..d923a1c
--- /dev/null
+++ b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=False_walk=seq_hpages=False_upages=False_type=median.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,2.201594,2.205345,2.205345,2.197843
+8,4.493202,4.478200,4.474449,4.474449
+16,9.061416,9.053915,9.065166,9.061416
+32,18.317862,18.329114,18.310361,18.347867
+64,37.145804,37.136428,37.175809,37.115799
+128,75.353024,75.491796,75.349273,75.398031
+256,152.766995,152.596343,152.793249,152.690108
+512,309.918425,309.839662,310.160338,309.828411
+1024,629.652133,630.059072,630.289733,628.928270
+2048,1274.884201,1277.858415,1275.653071,1274.677918
+3072,1922.535396,1924.601969,1925.262072,1921.578997
+4096,2575.084857,2579.514299,2585.586498,2572.571964
+8192,5213.217065,5216.858884,5205.854665,5187.617440
+12288,7667.690577,7665.134552,7669.151430,7665.350211
+16384,10168.061885,10170.117206,10176.703235,10172.361932
diff --git a/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=max.csv b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=max.csv
new file mode 100644
index 0000000..0edc790
--- /dev/null
+++ b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=max.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,16.686357,16.888889,17.106423,16.948898
+8,31.448664,31.647445,31.767464,31.594937
+16,61.547117,62.000938,61.697140,61.899672
+32,120.937647,121.421472,121.080169,121.132677
+64,238.754805,239.088608,238.214721,237.982185
+128,464.652602,464.506329,465.897797,466.531646
+256,910.649789,911.047351,911.444913,913.507736
+512,1801.856540,1805.580872,1801.428973,1803.574308
+1024,3533.280825,3514.786685,3514.760431,3506.377872
+2048,6618.186592,6634.325363,6623.017346,6635.810595
+3072,9776.990155,9777.777778,9785.065166,9776.581341
+4096,13086.773558,13074.257853,13058.546648,13059.878106
+8192,26873.095171,26877.918425,26889.406470,26797.970933
+12288,41187.398031,41135.493671,41165.164557,41195.240506
+16384,55380.365682,55337.597750,55401.875293,55292.080638
diff --git a/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=mean.csv b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=mean.csv
new file mode 100644
index 0000000..df4d967
--- /dev/null
+++ b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=mean.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,12.946511,13.151410,13.137659,13.133257
+8,26.512099,26.671148,26.656004,26.667291
+16,53.603526,53.720409,53.682344,53.708226
+32,104.945119,105.195719,104.947769,105.056345
+64,212.230385,212.343369,212.253766,212.131969
+128,424.559671,424.733511,424.841250,424.607822
+256,833.394550,834.436350,836.000579,835.413044
+512,1689.457373,1689.088664,1689.878457,1688.922302
+1024,3246.377307,3250.898075,3255.745414,3254.031230
+2048,6221.741714,6223.563219,6220.932131,6223.255715
+3072,9149.937779,9146.135054,9151.912668,9148.621343
+4096,12126.347939,12127.131682,12126.603392,12117.177726
+8192,25279.034014,25277.283993,25277.905983,25263.617558
+12288,38599.828992,38595.603303,38619.806809,38570.023693
+16384,51919.226036,51930.338658,51902.725500,51876.951615
diff --git a/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=median.csv b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=median.csv
new file mode 100644
index 0000000..8583f79
--- /dev/null
+++ b/ecrts14/overheads/ludwig/write/dwo_cold_host=ludwig_lvl=mem_polluters=True_walk=seq_hpages=False_upages=False_type=median.csv
@@ -0,0 +1,16 @@
+WSS,L1,L2,L3,MEM
+4,12.932021,13.142053,13.134552,13.134552
+8,26.501641,26.655415,26.617909,26.617909
+16,53.607126,53.785279,53.695265,53.734646
+32,105.380216,105.548992,105.338959,105.316456
+64,212.962025,213.048289,212.937647,212.652602
+128,424.776371,424.917018,425.218940,424.969526
+256,835.521800,836.433193,837.974684,837.243319
+512,1688.789498,1689.918425,1689.942804,1688.751992
+1024,3258.659165,3261.970933,3263.264885,3262.672293
+2048,6216.596343,6218.231599,6218.580403,6216.045007
+3072,9144.386310,9141.046414,9140.373183,9140.725738
+4096,12143.163619,12147.810595,12151.219878,12139.956868
+8192,25292.558837,25296.635724,25283.165495,25276.153774
+12288,38645.340835,38631.349273,38666.072199,38599.909986
+16384,51876.472574,51895.126113,51859.122363,51827.332396
diff --git a/schedcat/overheads/model.py b/schedcat/overheads/model.py
index fdcb793..ea82862 100644
--- a/schedcat/overheads/model.py
+++ b/schedcat/overheads/model.py
@@ -118,6 +118,47 @@ class CacheDelay(object):
                 o.__dict__[name] = o.mem_hierarchy[idx]
         return o
 
+class RawOverheads(object):
+    MEM, L1, L2, L3 = 0, 1, 2, 3
+    SCHEDCAT_MAPPING = list(enumerate(["MEM", "L1", "L2", "L3"]))
+
+    def __init__(self, l1=0, l2=0, l3=0, mem=0):
+        self.mem_hierarchy  = [const(mem), const(l1), const(l2), const(l3)]
+        for (i, name) in RawOverheads.SCHEDCAT_MAPPING:
+            self.__dict__[name] = self.mem_hierarchy[i]
+
+    def cost(self, shared_mem_level, working_set_size):
+        wss = min(working_set_size, self.cache_size)
+        return self.mem_hierarchy[shared_mem_level](wss)
+
+    def __call__(self, wss):
+        # presume local: L1
+        return self.mem_hierarchy[1](wss)
+
+    @staticmethod
+    def get_idx_for_name(key):
+        for (i, name) in RawOverheads.SCHEDCAT_MAPPING:
+            if name == key:
+                return i
+        assert False # bad key
+
+    @staticmethod
+    def from_file(fname, non_decreasing=True):
+        data = load_column_csv(fname, convert=float)
+        if not 'WSS' in data.by_name:
+            raise IOError, 'WSS column is missing'
+
+        o = RawOverheads()
+
+        for idx, name in RawOverheads.SCHEDCAT_MAPPING:
+            if name in data.by_name:
+                points = zip(data.by_name['WSS'], data.by_name[name])
+                if non_decreasing:
+                    o.mem_hierarchy[idx] = monotonic_pwlin(points)
+                else:
+                    o.mem_hierarchy[idx] = piece_wise_linear(points)
+                o.__dict__[name] = o.mem_hierarchy[idx]
+        return o
 
 class ConsumerOverheads(object):
     """Consumption cost overheads