aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2013-01-15 14:07:05 -0500
committerGlenn Elliott <gelliott@cs.unc.edu>2013-01-15 14:07:05 -0500
commit92801864bece1ccab9cc8b3ad73c1edb667a27fb (patch)
treedb6adbc1bef673bd14c23a0c1e9be8b0712424d1
parent2f60844462d35e9a8f0103baf6aa34d9734b4ed1 (diff)
Remove tasklet_owner requirements.
Diffstat
-rw-r--r--include/litmus/rt_param.h1
-rw-r--r--include/litmus/sched_trace.h10
-rw-r--r--kernel/softirq.c202
-rw-r--r--litmus/Kconfig13
-rw-r--r--litmus/litmus_softirq.c5
-rw-r--r--litmus/sched_task_trace.c4
6 files changed, 84 insertions, 151 deletions
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 39685a351cb1..dd1ef076a4b2 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -255,7 +255,6 @@ typedef struct avg_est{
255struct klmirqd_info 255struct klmirqd_info
256{ 256{
257 struct task_struct* klmirqd; 257 struct task_struct* klmirqd;
258 struct task_struct* current_owner;
259 unsigned int terminating:1; 258 unsigned int terminating:1;
260 259
261 raw_spinlock_t lock; 260 raw_spinlock_t lock;
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
index 7af12f49c600..5fbd7f37b26d 100644
--- a/include/litmus/sched_trace.h
+++ b/include/litmus/sched_trace.h
@@ -84,7 +84,8 @@ struct st_sys_release_data {
84 84
85struct st_tasklet_release_data { 85struct st_tasklet_release_data {
86 u64 when; 86 u64 when;
87 u64 __unused; 87 u32 device;
88 u32 __unused;
88} __attribute__((packed)); 89} __attribute__((packed));
89 90
90struct st_tasklet_begin_data { 91struct st_tasklet_begin_data {
@@ -256,7 +257,8 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
256 257
257 258
258feather_callback void do_sched_trace_tasklet_release(unsigned long id, 259feather_callback void do_sched_trace_tasklet_release(unsigned long id,
259 struct task_struct* owner); 260 struct task_struct* owner,
261 u32 device);
260feather_callback void do_sched_trace_tasklet_begin(unsigned long id, 262feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
261 struct task_struct* owner); 263 struct task_struct* owner);
262feather_callback void do_sched_trace_tasklet_end(unsigned long id, 264feather_callback void do_sched_trace_tasklet_end(unsigned long id,
@@ -397,8 +399,8 @@ feather_callback void do_sched_trace_migration(unsigned long id,
397 trace_litmus_sys_release(when); \ 399 trace_litmus_sys_release(when); \
398 } while (0) 400 } while (0)
399 401
400#define sched_trace_tasklet_release(t) \ 402#define sched_trace_tasklet_release(t, d) \
401 SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t) 403 SCHED_TRACE2(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t, d)
402 404
403#define sched_trace_tasklet_begin(t) \ 405#define sched_trace_tasklet_begin(t) \
404 SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t) 406 SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index ea438a8635d0..b56b35452d0d 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -410,7 +410,7 @@ void open_softirq(int nr, void (*action)(struct softirq_action *))
410static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); 410static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
411static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); 411static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
412 412
413#ifdef CONFIG_LITMUS_NVIDIA 413#ifdef CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS
414static int __do_nv_now(struct tasklet_struct* tasklet) 414static int __do_nv_now(struct tasklet_struct* tasklet)
415{ 415{
416 int success = 1; 416 int success = 1;
@@ -437,65 +437,57 @@ static int __do_nv_now(struct tasklet_struct* tasklet)
437} 437}
438#endif 438#endif
439 439
440#ifdef CONFIG_LITMUS_SOFTIRQD
441typedef int (*klmirqd_tasklet_sched_t)(
442 struct tasklet_struct *t,
443 struct task_struct* klmirqd_th);
444
445/* returns true on success */
446static int __klmirqd_nv_tasklet_schedule(struct tasklet_struct *t,
447 klmirqd_tasklet_sched_t klmirqd_func)
448{
449 u32 nvidia_device = get_tasklet_nv_device_num(t);
450 struct task_struct* klmirqd_th = get_nv_klmirqd_thread(nvidia_device);
451
452 BUG_ON(!klmirqd_func);
453
454 TRACE("%s: Handling NVIDIA tasklet for device %u (klmirqd: %s/%d) at %llu\n",
455 __FUNCTION__, nvidia_device,
456 (klmirqd_th) ? klmirqd_th->comm : "nil",
457 (klmirqd_th) ? klmirqd_th->pid : 0,
458 litmus_clock());
459
460 sched_trace_tasklet_release(NULL, nvidia_device);
461
462 if (klmirqd_th && likely(klmirqd_func(t, klmirqd_th)))
463 return 1;
464 else
465 return 0;
466}
467#endif
468
440 469
441void __tasklet_schedule(struct tasklet_struct *t) 470void __tasklet_schedule(struct tasklet_struct *t)
442{ 471{
443#ifdef CONFIG_LITMUS_NVIDIA 472#ifdef CONFIG_LITMUS_NVIDIA
444 if(is_nvidia_func(t->func)) 473 if(is_nvidia_func(t->func))
445 { 474 {
446#if 1 475#if defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS)
447 // do nvidia tasklets right away and return 476 /* do nvidia tasklets right away and return */
448 if(__do_nv_now(t)) 477 if(__do_nv_now(t))
449 return; 478 return;
450#else 479#elif defined(CONFIG_LITMUS_SOFTIRQD)
451 u32 nvidia_device = get_tasklet_nv_device_num(t); 480 if(__klmirqd_nv_tasklet_schedule(t, _litmus_tasklet_schedule))
452 // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n", 481 return;
453 // __FUNCTION__, nvidia_device,litmus_clock());
454
455 unsigned long flags;
456 struct task_struct* device_owner;
457
458 lock_nv_registry(nvidia_device, &flags);
459
460 device_owner = get_nv_max_device_owner(nvidia_device);
461
462 if(device_owner==NULL)
463 {
464 t->owner = NULL;
465 }
466 else 482 else
467 { 483 goto default_linux_handling;
468 if(is_realtime(device_owner)) 484#elif defined(CONFIG_LITMUS_PAI_SOFTIRQD)
469 { 485 /* broken at the moment! */
470 TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
471 __FUNCTION__, nvidia_device,litmus_clock());
472 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
473 __FUNCTION__,device_owner->pid,nvidia_device);
474
475 t->owner = device_owner;
476 sched_trace_tasklet_release(t->owner);
477
478 if(likely(_litmus_tasklet_schedule(t,nvidia_device)))
479 {
480 unlock_nv_registry(nvidia_device, &flags);
481 return;
482 }
483 else
484 {
485 t->owner = NULL; /* fall through to normal scheduling */
486 }
487 }
488 else
489 {
490 t->owner = NULL;
491 }
492 }
493 unlock_nv_registry(nvidia_device, &flags);
494#endif 486#endif
495 } 487 }
496 488
489default_linux_handling:
497#endif 490#endif
498
499 ___tasklet_schedule(t); 491 ___tasklet_schedule(t);
500} 492}
501EXPORT_SYMBOL(__tasklet_schedule); 493EXPORT_SYMBOL(__tasklet_schedule);
@@ -519,58 +511,23 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
519{ 511{
520#ifdef CONFIG_LITMUS_NVIDIA 512#ifdef CONFIG_LITMUS_NVIDIA
521 if(is_nvidia_func(t->func)) 513 if(is_nvidia_func(t->func))
522 { 514 {
523#if 1 515#if defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS)
524 // do nvidia tasklets right away and return 516 /* do nvidia tasklets right away and return */
525 if(__do_nv_now(t)) 517 if(__do_nv_now(t))
526 return; 518 return;
527#else 519#elif defined(CONFIG_LITMUS_SOFTIRQD)
528 u32 nvidia_device = get_tasklet_nv_device_num(t); 520 if(__klmirqd_nv_tasklet_schedule(t, _litmus_tasklet_hi_schedule))
529 // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n", 521 return;
530 // __FUNCTION__, nvidia_device,litmus_clock());
531
532 unsigned long flags;
533 struct task_struct* device_owner;
534
535 lock_nv_registry(nvidia_device, &flags);
536
537 device_owner = get_nv_max_device_owner(nvidia_device);
538
539 if(device_owner==NULL)
540 {
541 t->owner = NULL;
542 }
543 else 522 else
544 { 523 goto default_linux_handling;
545 if( is_realtime(device_owner)) 524#elif defined(CONFIG_LITMUS_PAI_SOFTIRQD)
546 { 525 /* broken at the moment! */
547 TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
548 __FUNCTION__, nvidia_device,litmus_clock());
549 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
550 __FUNCTION__,device_owner->pid,nvidia_device);
551
552 t->owner = device_owner;
553 sched_trace_tasklet_release(t->owner);
554 if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
555 {
556 unlock_nv_registry(nvidia_device, &flags);
557 return;
558 }
559 else
560 {
561 t->owner = NULL; /* fall through to normal scheduling */
562 }
563 }
564 else
565 {
566 t->owner = NULL;
567 }
568 }
569 unlock_nv_registry(nvidia_device, &flags);
570#endif 526#endif
571 } 527 }
572#endif
573 528
529default_linux_handling:
530#endif
574 ___tasklet_hi_schedule(t); 531 ___tasklet_hi_schedule(t);
575} 532}
576EXPORT_SYMBOL(__tasklet_hi_schedule); 533EXPORT_SYMBOL(__tasklet_hi_schedule);
@@ -591,60 +548,25 @@ EXPORT_SYMBOL(___tasklet_hi_schedule);
591void __tasklet_hi_schedule_first(struct tasklet_struct *t) 548void __tasklet_hi_schedule_first(struct tasklet_struct *t)
592{ 549{
593 BUG_ON(!irqs_disabled()); 550 BUG_ON(!irqs_disabled());
594#ifdef CONFIG_LITMUS_NVIDIA 551#ifdef CONFIG_LITMUS_NVIDIA
595 if(is_nvidia_func(t->func)) 552 if(is_nvidia_func(t->func))
596 { 553 {
597#if 1 554#if defined(CONFIG_LITMUS_NVIDIA_NONSPLIT_INTERRUPTS)
598 // do nvidia tasklets right away and return 555 /* do nvidia tasklets right away and return */
599 if(__do_nv_now(t)) 556 if(__do_nv_now(t))
600 return; 557 return;
601#else 558#elif defined(CONFIG_LITMUS_SOFTIRQD)
602 u32 nvidia_device = get_tasklet_nv_device_num(t); 559 if(__klmirqd_nv_tasklet_schedule(t, _litmus_tasklet_hi_schedule_first))
603 // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n", 560 return;
604 // __FUNCTION__, nvidia_device,litmus_clock());
605 unsigned long flags;
606 struct task_struct* device_owner;
607
608 lock_nv_registry(nvidia_device, &flags);
609
610 device_owner = get_nv_max_device_owner(nvidia_device);
611
612 if(device_owner==NULL)
613 {
614 t->owner = NULL;
615 }
616 else 561 else
617 { 562 goto default_linux_handling;
618 if(is_realtime(device_owner)) 563#elif defined(CONFIG_LITMUS_PAI_SOFTIRQD)
619 { 564 /* broken at the moment! */
620 TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
621 __FUNCTION__, nvidia_device,litmus_clock());
622
623 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
624 __FUNCTION__,device_owner->pid,nvidia_device);
625
626 t->owner = device_owner;
627 sched_trace_tasklet_release(t->owner);
628 if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))
629 {
630 unlock_nv_registry(nvidia_device, &flags);
631 return;
632 }
633 else
634 {
635 t->owner = NULL; /* fall through to normal scheduling */
636 }
637 }
638 else
639 {
640 t->owner = NULL;
641 }
642 }
643 unlock_nv_registry(nvidia_device, &flags);
644#endif 565#endif
645 } 566 }
646#endif
647 567
568default_linux_handling:
569#endif
648 ___tasklet_hi_schedule_first(t); 570 ___tasklet_hi_schedule_first(t);
649} 571}
650EXPORT_SYMBOL(__tasklet_hi_schedule_first); 572EXPORT_SYMBOL(__tasklet_hi_schedule_first);
diff --git a/litmus/Kconfig b/litmus/Kconfig
index 594c54342bdc..fa470b27ace4 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -431,6 +431,19 @@ config LITMUS_NVIDIA
431 431
432 If unsure, say No. 432 If unsure, say No.
433 433
434config LITMUS_NVIDIA_NONSPLIT_INTERRUPTS
435 bool "Execute NVIDIA interrupts with top-halves."
436 depends on LITMUS_NVIDIA
437 default n
438 help
439 Tasklets orginating from the NVIDIA driver are executed
440 immediatly in interrupt-space. This implements non-split
441 interrupt handling for GPUs. Feature intended mainly for
442 debugging, as it allows one to avoid having to rely
443 upon PAI or klmirqd interrupt handling.
444
445 If unsure, say No.
446
434config LITMUS_AFFINITY_AWARE_GPU_ASSINGMENT 447config LITMUS_AFFINITY_AWARE_GPU_ASSINGMENT
435 bool "Enable affinity-aware heuristics to improve GPU assignment." 448 bool "Enable affinity-aware heuristics to improve GPU assignment."
436 depends on LITMUS_NVIDIA && LITMUS_AFFINITY_LOCKING 449 depends on LITMUS_NVIDIA && LITMUS_AFFINITY_LOCKING
diff --git a/litmus/litmus_softirq.c b/litmus/litmus_softirq.c
index 464a78d780ad..666ea7ed1cb1 100644
--- a/litmus/litmus_softirq.c
+++ b/litmus/litmus_softirq.c
@@ -410,16 +410,11 @@ int proc_read_klmirqd_stats(char *page, char **start,
410 len += 410 len +=
411 snprintf(page + len - 1, PAGE_SIZE, /* -1 to strip off \0 */ 411 snprintf(page + len - 1, PAGE_SIZE, /* -1 to strip off \0 */
412 "klmirqd_thread: %s/%d\n" 412 "klmirqd_thread: %s/%d\n"
413 "\tcurrent_owner: %s/%d\n"
414 "\tpending: %x\n" 413 "\tpending: %x\n"
415 "\tnum hi: %d\n" 414 "\tnum hi: %d\n"
416 "\tnum low: %d\n" 415 "\tnum low: %d\n"
417 "\tnum work: %d\n\n", 416 "\tnum work: %d\n\n",
418 info->klmirqd->comm, info->klmirqd->pid, 417 info->klmirqd->comm, info->klmirqd->pid,
419 (info->current_owner != NULL) ?
420 info->current_owner->comm : "(null)",
421 (info->current_owner != NULL) ?
422 info->current_owner->pid : 0,
423 info->pending, 418 info->pending,
424 atomic_read(&info->num_hi_pending), 419 atomic_read(&info->num_hi_pending),
425 atomic_read(&info->num_low_pending), 420 atomic_read(&info->num_low_pending),
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
index f7f575346b54..b14b0100e09c 100644
--- a/litmus/sched_task_trace.c
+++ b/litmus/sched_task_trace.c
@@ -295,13 +295,15 @@ feather_callback void do_sched_trace_migration(unsigned long id,
295 295
296 296
297feather_callback void do_sched_trace_tasklet_release(unsigned long id, 297feather_callback void do_sched_trace_tasklet_release(unsigned long id,
298 unsigned long _owner) 298 unsigned long _owner,
299 unsigned long _device)
299{ 300{
300 struct task_struct *t = (struct task_struct*) _owner; 301 struct task_struct *t = (struct task_struct*) _owner;
301 struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t); 302 struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t);
302 303
303 if (rec) { 304 if (rec) {
304 rec->data.tasklet_release.when = now(); 305 rec->data.tasklet_release.when = now();
306 rec->data.tasklet_release.device = _device;
305 put_record(rec); 307 put_record(rec);
306 } 308 }
307} 309}
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-19 21:23:25 -0400