Use num_online_gpus()

Note that num_online_gpus() merely reports the
staticly configured maximum number of available
GPUs.  Will make dynamic in the future.

6 files changed, 30 insertions, 17 deletions

diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
index 6f354c8b00ac..8c2a5524512e 100644
--- a/include/litmus/nvidia_info.h
+++ b/include/litmus/nvidia_info.h
@@ -8,6 +8,10 @@
 
 #define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM
 
+/* TODO: Make this a function that checks the PCIe bus or maybe proc settings */
+#define num_online_gpus() (NV_DEVICE_NUM)
+
+
 /* Functions used for decoding NVIDIA blobs. */
 
 int init_nvidia_info(void);
@@ -21,8 +25,6 @@ void dump_nvidia_info(const struct tasklet_struct *t);
 u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
 u32 get_work_nv_device_num(const struct work_struct *t);
 
-
-
 /* Functions for figuring out the priority of GPU-using tasks */
 
 struct task_struct* get_nv_max_device_owner(u32 target_device_id);
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
index cfa218504d75..78004381a6cc 100644
--- a/include/litmus/sched_plugin.h
+++ b/include/litmus/sched_plugin.h
@@ -103,7 +103,7 @@ typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod,
 #endif
 
 #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
-typedef int (*default_cpu_for_gpu_t)(int gpu, int num_gpus);
+typedef int (*default_cpu_for_gpu_t)(int gpu);
 #endif
 
 
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
index 0b39dcc84115..7883296a7a18 100644
--- a/litmus/nvidia_info.c
+++ b/litmus/nvidia_info.c
@@ -444,13 +444,13 @@ static int init_nv_device_reg(void)
         mb();
 
 
-        for(i = 0; i < NV_DEVICE_NUM; ++i) {
+        for(i = 0; i < num_online_gpus(); ++i) {
                 raw_spin_lock_init(&NV_DEVICE_REG[i].lock);
                 INIT_BINHEAP_HANDLE(&NV_DEVICE_REG[i].owners, gpu_owner_max_priority_order);
 
 #ifdef CONFIG_LITMUS_SOFTIRQD
                 {
-                        int default_cpu = litmus->map_gpu_to_cpu(i, NV_DEVICE_NUM);
+                        int default_cpu = litmus->map_gpu_to_cpu(i);
 
                         NV_DEVICE_REG[i].callback.func = nvidia_klmirqd_cb;
                         NV_DEVICE_REG[i].callback.arg = (void*)(long long)(i);
@@ -478,7 +478,7 @@ static int shutdown_nv_device_reg(void)
                 int i;
                 nv_device_registry_t *reg;
 
-                for (i = 0; i < NV_DEVICE_NUM; ++i) {
+                for (i = 0; i < num_online_gpus(); ++i) {
 
                         TRACE("Shutting down GPU %d.\n", i);
 
@@ -829,7 +829,7 @@ int reg_nv_device(int reg_device_id, int reg_action, struct task_struct *t)
 {
         int ret;
 
-        if((reg_device_id < NV_DEVICE_NUM) && (reg_device_id >= 0))
+        if((reg_device_id < num_online_gpus()) && (reg_device_id >= 0))
         {
                 if(reg_action)
                         ret = __reg_nv_device(reg_device_id, t);
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 35ea1544ce69..46de8041cf59 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -157,6 +157,13 @@ static unsigned int cluster_size;
 
 static int clusters_allocated = 0;
 
+
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
+static int num_gpu_clusters;
+static unsigned int gpu_cluster_size;
+#endif
+
+
 #ifdef CONFIG_LITMUS_DGL_SUPPORT
 static raw_spinlock_t* cedf_get_dgl_spinlock(struct task_struct *t)
 {
@@ -1653,18 +1660,11 @@ static void cleanup_cedf(void)
 }
 
 #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
-static int cedf_map_gpu_to_cpu(int gpu, int num_gpus)
+static int cedf_map_gpu_to_cpu(int gpu)
 {
-        /* TODO: Some sort of smart clustering on the PCIe bus topology */
-        int num_gpu_clusters = num_clusters;
-        unsigned int gpu_cluster_size = num_gpus / num_gpu_clusters;
         int cpu_cluster = gpu / gpu_cluster_size;
         int default_cpu = cedf[cpu_cluster].cpus[0]->cpu;  // first CPU in given cluster
 
-        if(num_gpus % num_gpu_clusters != 0) {
-                TRACE("GPU clusters are of non-uniform size!\n");
-        }
-
         TRACE("CPU %d is default for GPU %d interrupt threads.\n", default_cpu, gpu);
 
         return default_cpu;
@@ -1717,6 +1717,17 @@ static long cedf_activate_plugin(void)
         printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n",
                         num_clusters, cluster_size);
 
+        
+#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
+        num_gpu_clusters = min(num_clusters, num_online_gpus());
+        gpu_cluster_size = num_online_gpus() / num_gpu_clusters;
+        
+        if (((num_online_gpus() % gpu_cluster_size) != 0) ||
+                (num_gpu_clusters != num_clusters)) {
+                printk(KERN_WARNING "C-EDF: GPUs not uniformly distributed among CPU clusters.\n");
+        }
+#endif
+        
         /* initialize clusters */
         cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC);
         for (i = 0; i < num_clusters; i++) {
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index 1b5d8d73dc16..4ac573a6f0f7 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -1836,7 +1836,7 @@ UNSUPPORTED_AFF_OBS:
 
 
 #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
-static int gsnedf_map_gpu_to_cpu(int gpu, int num_gpus)
+static int gsnedf_map_gpu_to_cpu(int gpu)
 {
         return 0;  // CPU_0 is default in all cases.
 }
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c
index ea89f5fedcab..76ff892122aa 100644
--- a/litmus/sched_plugin.c
+++ b/litmus/sched_plugin.c
@@ -223,7 +223,7 @@ static long litmus_dummy_allocate_aff_obs(struct affinity_observer **aff_obs,
 #endif
 
 #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
-static int litmus_dummy_map_gpu_to_cpu(int gpu, int num_gpus)
+static int litmus_dummy_map_gpu_to_cpu(int gpu)
 {
         return 0;
 }