From 2ccc2c4cc981a68e703082e6e32f5483ad87b61c Mon Sep 17 00:00:00 2001 From: Glenn Elliott Date: Wed, 12 Dec 2012 16:38:55 -0500 Subject: Use num_online_gpus() Note that num_online_gpus() merely reports the staticly configured maximum number of available GPUs. Will make dynamic in the future. --- include/litmus/nvidia_info.h | 6 ++++-- include/litmus/sched_plugin.h | 2 +- litmus/nvidia_info.c | 8 ++++---- litmus/sched_cedf.c | 27 +++++++++++++++++++-------- litmus/sched_gsn_edf.c | 2 +- litmus/sched_plugin.c | 2 +- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h index 6f354c8b00ac..8c2a5524512e 100644 --- a/include/litmus/nvidia_info.h +++ b/include/litmus/nvidia_info.h @@ -8,6 +8,10 @@ #define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM +/* TODO: Make this a function that checks the PCIe bus or maybe proc settings */ +#define num_online_gpus() (NV_DEVICE_NUM) + + /* Functions used for decoding NVIDIA blobs. */ int init_nvidia_info(void); @@ -21,8 +25,6 @@ void dump_nvidia_info(const struct tasklet_struct *t); u32 get_tasklet_nv_device_num(const struct tasklet_struct *t); u32 get_work_nv_device_num(const struct work_struct *t); - - /* Functions for figuring out the priority of GPU-using tasks */ struct task_struct* get_nv_max_device_owner(u32 target_device_id); diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h index cfa218504d75..78004381a6cc 100644 --- a/include/litmus/sched_plugin.h +++ b/include/litmus/sched_plugin.h @@ -103,7 +103,7 @@ typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod, #endif #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) -typedef int (*default_cpu_for_gpu_t)(int gpu, int num_gpus); +typedef int (*default_cpu_for_gpu_t)(int gpu); #endif diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c index 0b39dcc84115..7883296a7a18 100644 --- a/litmus/nvidia_info.c +++ b/litmus/nvidia_info.c @@ -444,13 +444,13 @@ static int init_nv_device_reg(void) mb(); - for(i = 0; i < NV_DEVICE_NUM; ++i) { + for(i = 0; i < num_online_gpus(); ++i) { raw_spin_lock_init(&NV_DEVICE_REG[i].lock); INIT_BINHEAP_HANDLE(&NV_DEVICE_REG[i].owners, gpu_owner_max_priority_order); #ifdef CONFIG_LITMUS_SOFTIRQD { - int default_cpu = litmus->map_gpu_to_cpu(i, NV_DEVICE_NUM); + int default_cpu = litmus->map_gpu_to_cpu(i); NV_DEVICE_REG[i].callback.func = nvidia_klmirqd_cb; NV_DEVICE_REG[i].callback.arg = (void*)(long long)(i); @@ -478,7 +478,7 @@ static int shutdown_nv_device_reg(void) int i; nv_device_registry_t *reg; - for (i = 0; i < NV_DEVICE_NUM; ++i) { + for (i = 0; i < num_online_gpus(); ++i) { TRACE("Shutting down GPU %d.\n", i); @@ -829,7 +829,7 @@ int reg_nv_device(int reg_device_id, int reg_action, struct task_struct *t) { int ret; - if((reg_device_id < NV_DEVICE_NUM) && (reg_device_id >= 0)) + if((reg_device_id < num_online_gpus()) && (reg_device_id >= 0)) { if(reg_action) ret = __reg_nv_device(reg_device_id, t); diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c index 35ea1544ce69..46de8041cf59 100644 --- a/litmus/sched_cedf.c +++ b/litmus/sched_cedf.c @@ -157,6 +157,13 @@ static unsigned int cluster_size; static int clusters_allocated = 0; + +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) +static int num_gpu_clusters; +static unsigned int gpu_cluster_size; +#endif + + #ifdef CONFIG_LITMUS_DGL_SUPPORT static raw_spinlock_t* cedf_get_dgl_spinlock(struct task_struct *t) { @@ -1653,18 +1660,11 @@ static void cleanup_cedf(void) } #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) -static int cedf_map_gpu_to_cpu(int gpu, int num_gpus) +static int cedf_map_gpu_to_cpu(int gpu) { - /* TODO: Some sort of smart clustering on the PCIe bus topology */ - int num_gpu_clusters = num_clusters; - unsigned int gpu_cluster_size = num_gpus / num_gpu_clusters; int cpu_cluster = gpu / gpu_cluster_size; int default_cpu = cedf[cpu_cluster].cpus[0]->cpu; // first CPU in given cluster - if(num_gpus % num_gpu_clusters != 0) { - TRACE("GPU clusters are of non-uniform size!\n"); - } - TRACE("CPU %d is default for GPU %d interrupt threads.\n", default_cpu, gpu); return default_cpu; @@ -1717,6 +1717,17 @@ static long cedf_activate_plugin(void) printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n", num_clusters, cluster_size); + +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) + num_gpu_clusters = min(num_clusters, num_online_gpus()); + gpu_cluster_size = num_online_gpus() / num_gpu_clusters; + + if (((num_online_gpus() % gpu_cluster_size) != 0) || + (num_gpu_clusters != num_clusters)) { + printk(KERN_WARNING "C-EDF: GPUs not uniformly distributed among CPU clusters.\n"); + } +#endif + /* initialize clusters */ cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC); for (i = 0; i < num_clusters; i++) { diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c index 1b5d8d73dc16..4ac573a6f0f7 100644 --- a/litmus/sched_gsn_edf.c +++ b/litmus/sched_gsn_edf.c @@ -1836,7 +1836,7 @@ UNSUPPORTED_AFF_OBS: #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) -static int gsnedf_map_gpu_to_cpu(int gpu, int num_gpus) +static int gsnedf_map_gpu_to_cpu(int gpu) { return 0; // CPU_0 is default in all cases. } diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c index ea89f5fedcab..76ff892122aa 100644 --- a/litmus/sched_plugin.c +++ b/litmus/sched_plugin.c @@ -223,7 +223,7 @@ static long litmus_dummy_allocate_aff_obs(struct affinity_observer **aff_obs, #endif #if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) -static int litmus_dummy_map_gpu_to_cpu(int gpu, int num_gpus) +static int litmus_dummy_map_gpu_to_cpu(int gpu) { return 0; } -- cgit v1.2.2