/* * litmus/sched_mc2.c * * Implementation of the Mixed-Criticality on MultiCore scheduler * * Thus plugin implements a scheduling algorithm proposed in * "Mixed-Criticality Real-Time Scheduling for Multicore System" paper. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef CONFIG_PGMRT_SUPPORT #include #endif //#define TRACE(fmt, args...) do {} while (false) //#define TRACE_TASK(fmt, args...) do {} while (false) #define BUDGET_ENFORCEMENT_AT_C 0 extern int num_sync_released; extern void do_partition(enum crit_level lv, int cpu); /* _global_env - reservation container for level-C tasks*/ struct gmp_reservation_environment _global_env_modes[NR_MODES]; struct gmp_reservation_environment *_global_env; raw_spinlock_t global_lock; /* cpu_entry - keep track of a running task on a cpu * This state is used to decide the lowest priority cpu */ struct cpu_entry { struct task_struct *scheduled; lt_t deadline; int cpu; enum crit_level lv; /* if will_schedule is true, this cpu is already selected and call mc2_schedule() soon. */ bool will_schedule; }; /* cpu_priority - a global state for choosing the lowest priority CPU */ struct cpu_priority { raw_spinlock_t lock; struct cpu_entry cpu_entries[NR_CPUS]; }; struct cpu_priority _lowest_prio_cpu; /* mc2_task_state - a task state structure */ struct mc2_task_state { struct task_client res_info[NR_MODES]; /* if cpu == -1, this task is a global task (level C) */ int cpu; bool has_departed; struct mc2_task mc2_param; }; /* mc2_cpu_state - maintain the scheduled state and ghost jobs * timer : timer for partitioned tasks (level A and B) * g_timer : timer for global tasks (level C) */ struct mc2_cpu_state { raw_spinlock_t lock; struct sup_reservation_environment sup_env_modes[NR_MODES]; struct sup_reservation_environment *sup_env; struct hrtimer timer; int cpu; struct task_struct* scheduled; //struct crit_entry crit_entries[NUM_CRIT_LEVELS]; bool spin_flag; //not used on cpu 0 }; static int resched_cpu[NR_CPUS]; static DEFINE_PER_CPU(struct mc2_cpu_state, mc2_cpu_state); //level_a_priorities unused //static int level_a_priorities[NR_CPUS]; #define cpu_state_for(cpu_id) (&per_cpu(mc2_cpu_state, cpu_id)) #define local_cpu_state() (this_cpu_ptr(&mc2_cpu_state)) unsigned int mode; //currently executing mode, from 0 to NR_MODES-1 unsigned int requested_mode; //The pending mode /* Prevent multiple requests from entering and prevent request from entering while old * is being enacted */ raw_spinlock_t mode_lock; unsigned int mode_sizes[NR_MODES]; unsigned int res_reported; bool cpu_0_spin_flag; bool seen_once; bool cpu_0_task_exist; bool mode_changed; #define in_mode(t, modenum) (tsk_mc2_data(t)->mode_mask & (1 << modenum)) #define pending mode != requested_mode #define ready !res_reported /* * To be called from level A task's with period equal to * A and B hyperperiod */ asmlinkage long sys_enact_mode(void) { struct mc2_cpu_state *state = local_cpu_state(); struct reservation *res; struct list_head *pos; unsigned long flags; //lt_t now = litmus_clock(); TRACE_TASK(current, "ENACTING SYSCALL\n"); if (state->cpu == 0){ //preempt_disable(); mode_changed = false; local_irq_save(flags); if (pending){ //MCR has entered raw_spin_lock(&state->lock); raw_spin_lock(&global_lock); raw_spin_lock(&mode_lock); if (!seen_once){ TRACE_TASK(current, "REQUEST\n"); sched_trace_request_mode(current); //TS_MODE_CHANGE_START; //clean up jobs that are already done //after this jobs report themselves list_for_each(pos, &_global_env->active_reservations){ res = list_entry(pos, struct reservation, list); if (tsk_rt(res->tsk)->completed && res->mode == mode && !res->reported){ res->reported = 1; TRACE_CUR("R%d RES_REPORTED_ACTIVE = %d mode %d\n", res->id, res_reported, res->mode); res_reported--; } } list_for_each(pos, &_global_env->depleted_reservations){ res = list_entry(pos, struct reservation, list); if (tsk_rt(res->tsk)->completed && res->mode == mode && !res->reported){ res->reported = 1; TRACE_CUR("R%d RES_REPORTED_DEPLETED = %d mode %d\n",res->id, res_reported, res->mode); res_reported--; } } list_for_each(pos, &_global_env->inactive_reservations){ res = list_entry(pos, struct reservation, list); if (tsk_rt(res->tsk)->completed && res->mode == mode && !res->reported){ res->reported = 1; //TRACE_CUR("R%d RES_REPORTED_INACTIVE = %d mode %d\n", res->id, res_reported, res->mode); res_reported--; } } seen_once = true; } if( ready ){ //C is throttled lt_t new_mode_basetime = get_release(current); //TRACE("Timer canceled\n"); hrtimer_cancel(&state->timer);//stop listening to old mode timers mode = requested_mode; TRACE("Mode has been changed.\n"); mode_changed = true; _global_env = &_global_env_modes[mode]; //set res->reported for new global tasks list_for_each(pos, &_global_env->active_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); res->reported = 0; } list_for_each(pos, &_global_env->depleted_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); res->reported = 0; } list_for_each(pos, &_global_env->inactive_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); res->reported = 0; } gmp_update_time(_global_env, litmus_clock()); //raw_spin_lock(&state->lock); state->sup_env = &state->sup_env_modes[mode]; list_for_each(pos, &state->sup_env->active_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); } list_for_each(pos, &state->sup_env->depleted_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); } list_for_each(pos, &state->sup_env->inactive_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); } sup_update_time(state->sup_env, litmus_clock()); //raw_spin_unlock(&state->lock); //t=litmus_clock(); sched_trace_enact_mode(current); //TS_MODE_CHANGE_END; TRACE("ENACT\n"); } raw_spin_unlock(&mode_lock); raw_spin_unlock(&global_lock); raw_spin_unlock(&state->lock); } local_irq_restore(flags); cpu_0_spin_flag = !cpu_0_spin_flag; } else if (cpu_0_task_exist) { //spin, wait for CPU 0 to stabilize mode decision //before scheduling next hyperperiod //TRACE("CPU%d start spinning. %d\n",state->cpu, mode_changed); if (state->spin_flag) { //TRACE_CUR("state->spin_flag %d\n",state->spin_flag); while(cpu_0_spin_flag) udelay(1); //TRACE_CUR("state->spin_flag %d cpu_0_spin_flag %d\n",state->spin_flag, cpu_0_spin_flag); } else { //TRACE_CUR("state->spin_flag %d\n",state->spin_flag); while(!cpu_0_spin_flag) //TRACE_CUR("state->spin_flag %d cpu_0_spin_flag %d\n",state->spin_flag, cpu_0_spin_flag); udelay(1); } //TRACE("CPU%d flag check. %d\n",state->cpu, mode_changed); local_irq_save(flags); if (mode_changed) { lt_t new_mode_basetime = get_release(current); //TRACE("CPU%d mode changed\n",state->cpu); hrtimer_cancel(&state->timer); //stop listening to old mode timers //preempt_disable(); //local_irq_save(flags); raw_spin_lock(&state->lock); state->sup_env = &state->sup_env_modes[mode]; list_for_each(pos, &state->sup_env->active_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); } list_for_each(pos, &state->sup_env->depleted_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); } list_for_each(pos, &state->sup_env->inactive_reservations){ res = list_entry(pos, struct reservation, list); release_at(res->tsk, new_mode_basetime); } sup_update_time(state->sup_env, litmus_clock()); raw_spin_unlock(&state->lock); //local_irq_restore(flags); //preempt_enable(); } local_irq_restore(flags); state->spin_flag = !state->spin_flag; } else { //TRACE("CPU%d no cpu_0_task_exist.%d\n",state->cpu, mode_changed); return 0; } TRACE("CPU%d enact syscall ends m_c? %d new_mode %d\n",state->cpu, mode_changed, mode); //if mode didn't change this has no effect on what's being scheduled //raw_spin_lock(&state->lock); //state->sup_env = &state->sup_env_modes[mode]; //raw_spin_unlock(&state->lock); //sup_update_time(state->sup_env, litmus_clock()); return 0; } /* * Called from non-real time program * Protect by exclusive lock to prevent from occuring while mode change is enacted */ asmlinkage long sys_request_mode(int new_mode){ preempt_disable(); raw_spin_lock(&mode_lock); if (pending){ raw_spin_unlock(&mode_lock); preempt_enable(); return -EAGAIN; } if (mode == new_mode){ raw_spin_unlock(&mode_lock); preempt_enable(); return 0; } requested_mode = new_mode; TRACE("MCR received\n"); res_reported = mode_sizes[mode]; TRACE_CUR("RES_REPORTED = %d\n",res_reported); seen_once = false; raw_spin_unlock(&mode_lock); preempt_enable(); return 0; } /* get_mc2_state - get the task's state */ static struct mc2_task_state* get_mc2_state(struct task_struct *tsk) { struct mc2_task_state* tinfo; tinfo = (struct mc2_task_state*)tsk_rt(tsk)->plugin_state; if (tinfo) return tinfo; else return NULL; } /* get_task_crit_level - return the criticaility level of a task */ static enum crit_level get_task_crit_level(struct task_struct *tsk) { struct mc2_task *mp; if (!tsk || !is_realtime(tsk)) return NUM_CRIT_LEVELS; mp = tsk_rt(tsk)->mc2_data; if (!mp) return NUM_CRIT_LEVELS; else return mp->crit; } static int is_init_finished(struct task_struct *tsk) { struct mc2_task *mp; if (!tsk || !is_realtime(tsk)) return 0; mp = tsk_rt(tsk)->mc2_data; if (!mp) return 0; else return mp->init_finished; } /* task_depart - remove a task from its reservation * If the job has remaining budget, convert it to a ghost job * and update crit_entries[] * * @job_complete indicate whether job completes or not */ static void task_departs(struct task_struct *tsk, int job_complete) { struct mc2_task_state* tinfo = get_mc2_state(tsk); //struct mc2_cpu_state* state = local_cpu_state(); struct reservation* res = NULL; struct reservation_client *client = NULL; int i; BUG_ON(!is_realtime(tsk)); for(i = 0; i < NR_MODES; i++){ if (! in_mode(tsk, i) && i != 0) continue; res = tinfo->res_info[i].client.reservation; client = &tinfo->res_info[i].client; BUG_ON(!res); BUG_ON(!client); if (job_complete) res->cur_budget = 0; res->ops->client_departs(res, client, job_complete); } /* 9/18/2015 fix start - no ghost job handling, empty remaining budget */ /* if (job_complete) { //res->cur_budget = 0; } */ /* fix end */ tinfo->has_departed = true; TRACE_TASK(tsk, "CLIENT DEPART with budget %llu at %llu\n", res->cur_budget, litmus_clock()); } /* task_arrive - put a task into its reservation * If the job was a ghost job, remove it from crit_entries[] */ static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk) { struct mc2_task_state* tinfo = get_mc2_state(tsk); struct reservation* res; struct reservation_client *client; enum crit_level lv = get_task_crit_level(tsk); int i; switch(lv) { case CRIT_LEVEL_A: case CRIT_LEVEL_B: TS_RELEASE_START; break; case CRIT_LEVEL_C: TS_RELEASE_C_START; break; default: break; } tinfo->has_departed = false; TRACE_TASK(tsk, "CLIENT ARRIVES at %llu\n", litmus_clock()); for(i = 0; i < NR_MODES; i++){ if (! in_mode(tsk, i) && i != 0) continue; res = tinfo->res_info[i].client.reservation; client = &tinfo->res_info[i].client; res->ops->client_arrives(res, client); } switch(lv) { case CRIT_LEVEL_A: case CRIT_LEVEL_B: TS_RELEASE_END; break; case CRIT_LEVEL_C: TS_RELEASE_C_END; break; default: break; } } /* get_lowest_prio_cpu - return the lowest priority cpu * This will be used for scheduling level-C tasks. * If all CPUs are running tasks which has * higher priority than level C, return NO_CPU. */ static int get_lowest_prio_cpu(lt_t priority) { struct cpu_entry *ce; int cpu, ret = NO_CPU; lt_t latest_deadline = 0; if (priority == LITMUS_NO_PRIORITY) return ret; ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu]; if (!ce->will_schedule && !ce->scheduled) { TRACE("CPU %d (local) is the lowest!\n", ce->cpu); return ce->cpu; } else { TRACE("Local CPU will_schedule=%d, scheduled=(%s/%d)\n", ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0); } for_each_online_cpu(cpu) { ce = &_lowest_prio_cpu.cpu_entries[cpu]; /* If a CPU will call schedule() in the near future, we don't return that CPU. */ TRACE("CPU %d will_schedule=%d, scheduled=(%s/%d:%d)\n", cpu, ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0, ce->scheduled ? (ce->scheduled)->rt_param.job_params.job_no : 0); if (!ce->will_schedule) { if (!ce->scheduled) { /* Idle cpu, return this. */ TRACE("CPU %d is the lowest!\n", ce->cpu); return ce->cpu; } else if (ce->lv == CRIT_LEVEL_C && ce->deadline > latest_deadline) { latest_deadline = ce->deadline; ret = ce->cpu; } } } if (priority >= latest_deadline) ret = NO_CPU; TRACE("CPU %d is the lowest!\n", ret); return ret; } /* NOTE: drops state->lock */ /* mc2_update_timer_and_unlock - set a timer and g_timer and unlock * Whenever res_env.current_time is updated, * we check next_scheduler_update and set * a timer. * If there exist a global event which is * not armed on any CPU and g_timer is not * active, set a g_timer for that event. */ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) { int local, cpus; lt_t update, now; //enum crit_level lv = get_task_crit_level(state->scheduled); struct next_timer_event *event, *next; int reschedule[NR_CPUS]; for (cpus = 0; cpussup_env->next_scheduler_update; now = state->sup_env->env.current_time; /* Be sure we're actually running on the right core, * as pres_update_timer() is also called from pres_task_resume(), * which might be called on any CPU when a thread resumes. */ local = local_cpu_state() == state; raw_spin_lock(&global_lock); list_for_each_entry_safe(event, next, &_global_env->next_events, list) { /* If the event time is already passed, we call schedule() on the lowest priority cpu */ if (event->next_update >= update) { break; } if (event->next_update < litmus_clock()) { if (event->timer_armed_on == NO_CPU) { struct reservation *res = gmp_find_by_id(_global_env, event->id); int cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY); //TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu); list_del(&event->list); kfree(event); if (cpu != NO_CPU) { //raw_spin_lock(&_lowest_prio_cpu.lock); _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; //raw_spin_unlock(&_lowest_prio_cpu.lock); if (cpu == local_cpu_state()->cpu) litmus_reschedule_local(); else reschedule[cpu] = 1; } } } else if (event->next_update < update && (event->timer_armed_on == NO_CPU || event->timer_armed_on == state->cpu)) { event->timer_armed_on = state->cpu; update = event->next_update; break; } } /* Must drop state lock before calling into hrtimer_start(), which * may raise a softirq, which in turn may wake ksoftirqd. */ raw_spin_unlock(&global_lock); raw_spin_unlock(&state->lock); if (update <= now || reschedule[state->cpu]) { reschedule[state->cpu] = 0; litmus_reschedule(state->cpu); /* raw_spin_lock(&state->lock); preempt_if_preemptable(state->scheduled, state->cpu); raw_spin_unlock(&state->lock); */ } else if (likely(local && update != SUP_NO_SCHEDULER_UPDATE)) { /* Reprogram only if not already set correctly. */ if (!hrtimer_active(&state->timer) || ktime_to_ns(hrtimer_get_expires(&state->timer)) != update) { TRACE("canceling timer...at %llu\n", ktime_to_ns(hrtimer_get_expires(&state->timer))); hrtimer_cancel(&state->timer); TRACE("setting scheduler timer for %llu\n", update); /* We cannot use hrtimer_start() here because the * wakeup flag must be set to zero. */ __hrtimer_start_range_ns(&state->timer, ns_to_ktime(update), 0 /* timer coalescing slack */, HRTIMER_MODE_ABS_PINNED, 0 /* wakeup */); if (update < litmus_clock()) { /* uh oh, timer expired while trying to set it */ TRACE("timer expired during setting " "update:%llu now:%llu actual:%llu\n", update, now, litmus_clock()); /* The timer HW may not have been reprogrammed * correctly; force rescheduling now. */ litmus_reschedule(state->cpu); } } } else if (unlikely(!local && update != SUP_NO_SCHEDULER_UPDATE)) { /* Poke remote core only if timer needs to be set earlier than * it is currently set. */ TRACE("mc2_update_timer for remote CPU %d (update=%llu, " "active:%d, set:%llu)\n", state->cpu, update, hrtimer_active(&state->timer), ktime_to_ns(hrtimer_get_expires(&state->timer))); if (!hrtimer_active(&state->timer) || ktime_to_ns(hrtimer_get_expires(&state->timer)) > update) { TRACE("poking CPU %d so that it can update its " "scheduling timer (active:%d, set:%llu)\n", state->cpu, hrtimer_active(&state->timer), ktime_to_ns(hrtimer_get_expires(&state->timer))); //litmus_reschedule(state->cpu); /* raw_spin_lock(&state->lock); preempt_if_preemptable(state->scheduled, state->cpu); raw_spin_unlock(&state->lock); reschedule[state->cpu] = 0; */ } } /* for (cpus = 0; cpuscpu]; enum crit_level lv = get_task_crit_level(state->scheduled); if (!state->scheduled) { /* cpu is idle. */ ce->scheduled = NULL; ce->deadline = ULLONG_MAX; ce->lv = NUM_CRIT_LEVELS; } else if (lv == CRIT_LEVEL_C) { ce->scheduled = state->scheduled; ce->deadline = get_deadline(state->scheduled); ce->lv = lv; } else if (lv < CRIT_LEVEL_C) { /* If cpu is running level A or B tasks, it is not eligible to run level-C tasks */ ce->scheduled = state->scheduled; ce->deadline = 0; ce->lv = lv; } }; /* on_scheduling_timer - timer event for partitioned tasks */ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer) { unsigned long flags; enum hrtimer_restart restart = HRTIMER_NORESTART; struct mc2_cpu_state *state; lt_t update, now; int global_schedule_now; //lt_t remain_budget; // no ghost jobs int reschedule[NR_CPUS]; int cpus; for (cpus = 0; cpuscpu != raw_smp_processor_id()); TS_ISR_START; TRACE("Timer fired at %llu\n", litmus_clock()); raw_spin_lock_irqsave(&state->lock, flags); now = litmus_clock(); sup_update_time(state->sup_env, now); /* 9/20/2015 fix - no ghost job remain_budget = mc2_update_ghost_state(state); */ update = state->sup_env->next_scheduler_update; now = state->sup_env->env.current_time; if (update <= now) { litmus_reschedule_local(); } else if (update != SUP_NO_SCHEDULER_UPDATE) { hrtimer_set_expires(timer, ns_to_ktime(update)); restart = HRTIMER_RESTART; } raw_spin_lock(&global_lock); global_schedule_now = gmp_update_time(_global_env, now); BUG_ON(global_schedule_now < 0 || global_schedule_now > 4); /* Find the lowest cpu, and call reschedule */ while (global_schedule_now--) { int cpu = get_lowest_prio_cpu(0); if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { //raw_spin_lock(&_lowest_prio_cpu.lock); _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; //raw_spin_unlock(&_lowest_prio_cpu.lock); TRACE("LOWEST CPU = P%d\n", cpu); if (cpu == state->cpu && update > now) litmus_reschedule_local(); else reschedule[cpu] = 1; } } raw_spin_unlock(&global_lock); raw_spin_unlock_irqrestore(&state->lock, flags); TS_ISR_END; for (cpus = 0; cpuslock); preempt_if_preemptable(remote_state->scheduled, remote_state->cpu); raw_spin_unlock(&remote_state->lock); */ } } return restart; } /* mc2_complete_job - syscall backend for job completions */ static long mc2_complete_job(void) { ktime_t next_release; lt_t next_release_ns; long err; enum crit_level lv; raw_spin_lock(&mode_lock); tsk_rt(current)->completed = 1; raw_spin_unlock(&mode_lock); lv = get_task_crit_level(current); /* If this the first job instance, we need to reset replenish time to the next release time */ if (tsk_rt(current)->sporadic_release) { struct mc2_cpu_state *state; struct mc2_task_state *tinfo; struct reservation *res = NULL; unsigned long flags; local_irq_save(flags); tinfo = get_mc2_state(current); if (lv < CRIT_LEVEL_C) { int i; state = cpu_state_for(tinfo->cpu); raw_spin_lock(&state->lock); for (i = 0; isup_env_modes[i].env.time_zero = tsk_rt(current)->sporadic_release_time; } } res = sup_find_by_id(state->sup_env, tinfo->mc2_param.res_id); } else if (lv == CRIT_LEVEL_C) { int i; state = local_cpu_state(); raw_spin_lock(&state->lock); raw_spin_lock(&global_lock); for (i = 0; i < NR_MODES; i++) { if (in_mode(current,i) || i == 0) { _global_env_modes[i].env.time_zero = tsk_rt(current)->sporadic_release_time; } } res = gmp_find_by_id(_global_env, tinfo->mc2_param.res_id); } else BUG(); /* set next_replenish to synchronous release time */ BUG_ON(!res); res->next_replenishment = tsk_rt(current)->sporadic_release_time; /* if (get_task_crit_level(current) == CRIT_LEVEL_A) { struct table_driven_reservation *tdres; tdres = container_of(res, struct table_driven_reservation, res); tdres->next_interval = 0; tdres->major_cycle_start = tsk_rt(current)->sporadic_release_time; res->next_replenishment += tdres->intervals[0].start; } */ res->cur_budget = 0; res->env->change_state(res->env, res, RESERVATION_DEPLETED); // TRACE_CUR("CHANGE NEXT_REP = %llu NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env->next_scheduler_update); if (lv == CRIT_LEVEL_C) raw_spin_unlock(&global_lock); raw_spin_unlock(&state->lock); local_irq_restore(flags); } sched_trace_task_completion(current, 0); /* update the next release time and deadline */ prepare_for_next_period(current); sched_trace_task_release(current); next_release = ns_to_ktime(get_release(current)); preempt_disable(); TRACE_CUR("next_release=%llu\n", get_release(current)); /* * Changed logic for mode switch case * In case of mode switch, do not want to release * new job even if release time has passed */ raw_spin_lock(&mode_lock); if (lv == CRIT_LEVEL_C && pending){ struct reservation *res = NULL; res = gmp_find_by_id(_global_env, tsk_mc2_data(current)->res_id); if (res && !res->reported){ res_reported--; TRACE_CUR("RES_REPORTED = %d\n",res_reported); res->reported = 1; //Current task doesn't exist in new mode if ( !in_mode(current, requested_mode) ){ raw_spin_unlock(&mode_lock); litmus_reschedule_local(); } //Otherwise schedule normally else raw_spin_unlock(&mode_lock); } else raw_spin_unlock(&mode_lock); } else raw_spin_unlock(&mode_lock); if (get_release(current) > litmus_clock()) { /* sleep until next_release */ set_current_state(TASK_INTERRUPTIBLE); preempt_enable_no_resched(); TRACE_CUR("Sleep until %llu\n", next_release); err = schedule_hrtimeout(&next_release, HRTIMER_MODE_ABS); } else { /* release the next job immediately */ err = 0; TRACE_CUR("TARDY: release=%llu now=%llu\n", get_release(current), litmus_clock()); preempt_enable(); } TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock()); raw_spin_lock(&mode_lock); tsk_rt(current)->completed = 0; raw_spin_unlock(&mode_lock); return err; } /* mc2_dispatch - Select the next task to schedule. */ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, struct mc2_cpu_state* state) { struct reservation *res, *next; struct task_struct *tsk = NULL; //struct crit_entry *ce; enum crit_level lv; lt_t time_slice; list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { if (res->state == RESERVATION_ACTIVE) TRACE_TASK(tsk, "ACT_LIST R%d mode = %d budget = %llu\n", res->id, res->mode, res->cur_budget); } list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { if (res->state == RESERVATION_ACTIVE) { tsk = res->ops->dispatch_client(res, &time_slice); if (likely(tsk)) { lv = get_task_crit_level(tsk); if (lv == NUM_CRIT_LEVELS) { sup_scheduler_update_after(sup_env, res->cur_budget); return tsk; } else { if (!is_init_finished(tsk)) { //ce = &state->crit_entries[lv]; sup_scheduler_update_after(sup_env, res->cur_budget); res->blocked_by_ghost = 0; res->is_ghost = NO_CPU; return tsk; } else if (res->mode == mode) { sup_scheduler_update_after(sup_env, res->cur_budget); res->blocked_by_ghost = 0; res->is_ghost = NO_CPU; return tsk; } } } } } return NULL; } struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state) { struct reservation *res, *next; struct task_struct *tsk = NULL; //struct crit_entry *ce; enum crit_level lv; lt_t time_slice; raw_spin_lock(&mode_lock); list_for_each_entry_safe(res, next, &_global_env->active_reservations, list) { BUG_ON(!res); if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) { tsk = res->ops->dispatch_client(res, &time_slice); if (pending && res->reported && !in_mode(tsk, requested_mode)){ continue; } if (likely(tsk)) { lv = get_task_crit_level(tsk); if (lv == NUM_CRIT_LEVELS) { #if BUDGET_ENFORCEMENT_AT_C gmp_add_event_after(_global_env, res->cur_budget, res->id, EVENT_DRAIN); #endif res->event_added = 1; res->blocked_by_ghost = 0; res->is_ghost = NO_CPU; res->scheduled_on = state->cpu; raw_spin_unlock(&mode_lock); return tsk; } else { if (!is_init_finished(tsk)) { //ce = &state->crit_entries[lv]; //if (likely(!ce->running)) { #if BUDGET_ENFORCEMENT_AT_C gmp_add_event_after(_global_env, res->cur_budget, res->id, EVENT_DRAIN); #endif res->event_added = 1; res->blocked_by_ghost = 0; res->is_ghost = NO_CPU; res->scheduled_on = state->cpu; raw_spin_unlock(&mode_lock); return tsk; } else if (res->mode == mode) { #if BUDGET_ENFORCEMENT_AT_C gmp_add_event_after(_global_env, res->cur_budget, res->id, EVENT_DRAIN); #endif res->event_added = 1; res->blocked_by_ghost = 0; res->is_ghost = NO_CPU; res->scheduled_on = state->cpu; raw_spin_unlock(&mode_lock); return tsk; } //} else { // res->blocked_by_ghost = 1; // TRACE_TASK(ce->running, " is GHOST\n"); // return NULL; //} } } } } raw_spin_unlock(&mode_lock); return NULL; } static inline void pre_schedule(struct task_struct *prev, int cpu) { TS_SCHED_A_START; TS_SCHED_C_START; if (!prev || !is_realtime(prev)) return; do_partition(CRIT_LEVEL_C, cpu); } static inline void post_schedule(struct task_struct *next, int cpu) { enum crit_level lev; if ((!next) || !is_realtime(next)) return; lev = get_task_crit_level(next); if (is_mode_poll_task(next)) { lev = MODE_POLL_TASK; } do_partition(lev, cpu); switch(lev) { case CRIT_LEVEL_A: case CRIT_LEVEL_B: case MODE_POLL_TASK: TS_SCHED_A_END(next); break; case CRIT_LEVEL_C: TS_SCHED_C_END(next); break; default: break; } } /* mc2_schedule - main scheduler function. pick the next task to run */ static struct task_struct* mc2_schedule(struct task_struct * prev) { int np, blocks, exists, preempt, to_schedule; /* next == NULL means "schedule background work". */ lt_t now = litmus_clock(); struct mc2_cpu_state *state = local_cpu_state(); raw_spin_lock(&state->lock); raw_spin_lock(&global_lock); preempt = resched_cpu[state->cpu]; resched_cpu[state->cpu] = 0; raw_spin_unlock(&global_lock); pre_schedule(prev, state->cpu); BUG_ON(state->scheduled && state->scheduled != prev); BUG_ON(state->scheduled && !is_realtime(prev)); //if (state->scheduled && state->scheduled != prev) // printk(KERN_ALERT "BUG1!!!!!!!! %s %s\n", state->scheduled ? (state->scheduled)->comm : "null", prev ? (prev)->comm : "null"); //if (state->scheduled && !is_realtime(prev)) // printk(KERN_ALERT "BUG2!!!!!!!! \n"); /* (0) Determine state */ exists = state->scheduled != NULL; blocks = exists && !is_current_running(); np = exists && is_np(state->scheduled); /* update time */ state->sup_env->will_schedule = true; sup_update_time(state->sup_env, now); /* 9/20/2015 fix */ //raw_spin_lock(&_global_env.lock); //to_schedule = gmp_update_time(&_global_env, now); //raw_spin_unlock(&_global_env.lock); /* 9/20/2015 fix mc2_update_ghost_state(state); */ /* remove task from reservation if it blocks */ /* if (is_realtime(prev) && !is_running(prev)) { if (get_task_crit_level(prev) == CRIT_LEVEL_C) raw_spin_lock(&_global_env.lock); task_departs(prev, is_completed(prev)); if (get_task_crit_level(prev) == CRIT_LEVEL_C) raw_spin_unlock(&_global_env.lock); }*/ if (is_realtime(current) && blocks) { if (get_task_crit_level(current) == CRIT_LEVEL_C) raw_spin_lock(&global_lock); task_departs(current, is_completed(current)); if (get_task_crit_level(current) == CRIT_LEVEL_C) raw_spin_unlock(&global_lock); } /* figure out what to schedule next */ if (!np) state->scheduled = mc2_dispatch(state->sup_env, state); if (!state->scheduled) { raw_spin_lock(&global_lock); if (is_realtime(prev)) gmp_update_time(_global_env, now); state->scheduled = mc2_global_dispatch(state); raw_spin_unlock(&global_lock); } /* if (!state->scheduled) { raw_spin_lock(&global_lock); //to_schedule = gmp_update_time(_global_env, now); state->scheduled = mc2_global_dispatch(state); _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; update_cpu_prio(state); raw_spin_unlock(&global_lock); } else { raw_spin_lock(&global_lock); _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; update_cpu_prio(state); raw_spin_unlock(&global_lock); } */ //raw_spin_lock(&_lowest_prio_cpu.lock); //_lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; //update_cpu_prio(state); //raw_spin_unlock(&_lowest_prio_cpu.lock); /* Notify LITMUS^RT core that we've arrived at a scheduling decision. */ sched_state_task_picked(); /* program scheduler timer */ state->sup_env->will_schedule = false; /* NOTE: drops state->lock */ mc2_update_timer_and_unlock(state); raw_spin_lock(&state->lock); if (prev != state->scheduled && is_realtime(prev)) { struct mc2_task_state* tinfo = get_mc2_state(prev); struct reservation* res = tinfo->res_info[mode].client.reservation; if (res) { TRACE_TASK(prev, "PREV JOB was scheduled_on = P%d\n", res->scheduled_on); res->scheduled_on = NO_CPU; } TRACE_TASK(prev, "descheduled at %llu.\n", litmus_clock()); /* if prev is preempted and a global task, find the lowest cpu and reschedule */ if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) { int cpu; raw_spin_lock(&global_lock); cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY); if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { //raw_spin_lock(&_lowest_prio_cpu.lock); _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; resched_cpu[cpu] = 1; //raw_spin_unlock(&_lowest_prio_cpu.lock); TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu); } raw_spin_unlock(&global_lock); } } /* if (to_schedule != 0) { raw_spin_lock(&global_lock); while (to_schedule--) { int cpu = get_lowest_prio_cpu(0); if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; resched_cpu[cpu] = 1; } } raw_spin_unlock(&global_lock); } */ post_schedule(state->scheduled, state->cpu); raw_spin_lock(&global_lock); _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; update_cpu_prio(state); raw_spin_unlock(&global_lock); raw_spin_unlock(&state->lock); if (state->scheduled) { TRACE_TASK(state->scheduled, "scheduled.\n"); } return state->scheduled; } static void resume_legacy_task_model_updates(struct task_struct *tsk) { lt_t now; if (is_sporadic(tsk)) { /* If this sporadic task was gone for a "long" time and woke up past * its deadline, then give it a new budget by triggering a job * release. This is purely cosmetic and has no effect on the * MC2 scheduler. */ now = litmus_clock(); if (is_tardy(tsk, now)) { release_at(tsk, now); } } } /* mc2_task_resume - Called when the state of tsk changes back to * TASK_RUNNING. We need to requeue the task. */ static void mc2_task_resume(struct task_struct *tsk) { unsigned long flags; struct mc2_task_state* tinfo = get_mc2_state(tsk); struct mc2_cpu_state *state; TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock()); local_irq_save(flags); if (tinfo->cpu != -1) state = cpu_state_for(tinfo->cpu); else state = local_cpu_state(); /* 9/20/2015 fix raw_spin_lock(&_global_env.lock); */ /* Requeue only if self-suspension was already processed. */ if (tinfo->has_departed) { /* We don't want to consider jobs before synchronous releases */ if (tsk_rt(tsk)->job_params.job_no == 2) { /* switch(get_task_crit_level(tsk)) { case CRIT_LEVEL_A: TS_RELEASE_LATENCY_A(get_release(tsk)); break; case CRIT_LEVEL_B: TS_RELEASE_LATENCY_B(get_release(tsk)); break; case CRIT_LEVEL_C: TS_RELEASE_LATENCY_C(get_release(tsk)); break; default: break; } */ // TRACE_CUR("INIT_FINISHED is SET\n"); tsk_mc2_data(tsk)->init_finished = 1; raw_spin_lock(&global_lock); num_sync_released--; raw_spin_unlock(&global_lock); } raw_spin_lock(&state->lock); /* Assumption: litmus_clock() is synchronized across cores, * since we might not actually be executing on tinfo->cpu * at the moment. */ if (tinfo->cpu != -1) { sup_update_time(state->sup_env, litmus_clock()); task_arrives(state, tsk); } else { raw_spin_lock(&global_lock); gmp_update_time(_global_env, litmus_clock()); task_arrives(state, tsk); raw_spin_unlock(&global_lock); } /* 9/20/2015 fix mc2_update_ghost_state(state); */ //task_arrives(state, tsk); /* NOTE: drops state->lock */ TRACE_TASK(tsk, "mc2_resume()\n"); mc2_update_timer_and_unlock(state); } else { TRACE_TASK(tsk, "resume event ignored, still scheduled\n"); //raw_spin_unlock(&_global_env.lock); } local_irq_restore(flags); //gmp_free_passed_event(); resume_legacy_task_model_updates(tsk); } /* mc2_admit_task - Setup mc2 task parameters */ static long mc2_admit_task(struct task_struct *tsk) { long err = 0; unsigned long flags; struct reservation *res; struct mc2_cpu_state *state; struct mc2_task_state *tinfo = kzalloc(sizeof(*tinfo), GFP_ATOMIC); struct mc2_task *mp = tsk_rt(tsk)->mc2_data; enum crit_level lv; int i; TRACE_TASK(tsk, "MC2 admitting task\n"); if (!tinfo) return -ENOMEM; if (!mp) { TRACE("mc2_admit_task: criticality level has not been set\n"); return -ESRCH; } lv = mp->crit; if (lv < CRIT_LEVEL_C) { state = cpu_state_for(task_cpu(tsk)); raw_spin_lock_irqsave(&state->lock, flags); tinfo->mc2_param.crit = mp->crit; tinfo->cpu = task_cpu(tsk); tinfo->has_departed = true; tinfo->mc2_param.res_id = mp->res_id; tinfo->mc2_param.mode_mask = mp->mode_mask; tinfo->mc2_param.init_finished = 0; // TRACE_TASK(tsk, "mode_mask = %x\n", mp->mode_mask); // TRACE_TASK(tsk, "Mode 0\n"); res = sup_find_by_id(&(state->sup_env_modes[0]), mp->res_id); /* found the appropriate reservation */ if (res) { // TRACE_TASK(tsk, "SUP FOUND RES ID in mode 0\n"); /* initial values */ err = err? err:mc2_task_client_init(&tinfo->res_info[0], &tinfo->mc2_param, tsk, res); } else { //failed to find an expected reservation err = -ESRCH; } for(i = 1; i < NR_MODES; i++){ if (!in_mode(tsk, i)){ //task not present in mode continue; } // TRACE_TASK(tsk, "Mode %d\n",i); res = sup_find_by_id(&(state->sup_env_modes[i]), mp->res_id); /* found the appropriate reservation */ if (res) { // TRACE_TASK(tsk, "SUP FOUND RES ID in mode %d\n", i); /* initial values */ err = err? err:mc2_task_client_init(&tinfo->res_info[i], &tinfo->mc2_param, tsk, res); } else{ //failed to find an expected reservation err = -ESRCH; } } if (!err){ /* disable LITMUS^RT's per-thread budget enforcement */ tsk_rt(tsk)->plugin_state = tinfo; tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; } if (is_mode_poll_task(tsk) && tinfo->cpu == 0) { cpu_0_task_exist = true; } raw_spin_unlock_irqrestore(&state->lock, flags); } else if (lv == CRIT_LEVEL_C) { // TRACE_TASK(tsk, "Task being admitted is Level C\n"); state = local_cpu_state(); raw_spin_lock_irqsave(&state->lock, flags); //state = local_cpu_state(); //raw_spin_lock(&state->lock); tinfo->mc2_param.crit = mp->crit; tinfo->cpu = -1; tinfo->has_departed = true; tinfo->mc2_param.res_id = mp->res_id; tinfo->mc2_param.mode_mask = mp->mode_mask; tinfo->mc2_param.init_finished = 0; // TRACE_TASK(tsk, "mode_mask = %x\n", mp->mode_mask); // TRACE_TASK(tsk, "Mode 0\n"); raw_spin_lock(&global_lock); res = gmp_find_by_id(&(_global_env_modes[0]), mp->res_id); /* found the appropriate reservation */ if (res) { // TRACE_TASK(tsk, "GMP FOUND RES ID in mode 0\n"); /* initial values */ err = err? err:mc2_task_client_init(&tinfo->res_info[0], &tinfo->mc2_param, tsk, res); } else { //failed to find an expected reservation err = -ESRCH; } for(i = 1; i < NR_MODES; i++){ if (!in_mode(tsk, i)) continue; res = gmp_find_by_id(&(_global_env_modes[i]), mp->res_id); /* found the appropriate reservation (or vCPU) */ if (res) { TRACE_TASK(tsk, "GMP FOUND RES ID in mode %d\n", i); /* initial values */ err = err? err:mc2_task_client_init(&tinfo->res_info[i], &tinfo->mc2_param, tsk, res); } } if (!err){ /* disable LITMUS^RT's per-thread budget enforcement */ tsk_rt(tsk)->plugin_state = tinfo; tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; raw_spin_lock(&mode_lock); for(i = 0; i < NR_MODES; i++){ if (in_mode(tsk, i)){ mode_sizes[i]++; } } raw_spin_unlock(&mode_lock); } raw_spin_unlock(&global_lock); raw_spin_unlock_irqrestore(&state->lock, flags); } if (err) kfree(tinfo); TRACE_TASK(tsk, "MC2 task admitted %d\n", err); return err; } /* mc2_task_new - A new real-time job is arrived. Release the next job * at the next reservation replenish time */ static void mc2_task_new(struct task_struct *tsk, int on_runqueue, int is_running) { unsigned long flags; struct mc2_task_state* tinfo = get_mc2_state(tsk); struct mc2_cpu_state *state; // = cpu_state_for(tinfo->cpu); struct reservation *res; enum crit_level lv = get_task_crit_level(tsk); lt_t release = 0; BUG_ON(lv < CRIT_LEVEL_A || lv > CRIT_LEVEL_C); TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n", litmus_clock(), on_runqueue, is_running); if (tinfo->cpu == -1) state = local_cpu_state(); else state = cpu_state_for(tinfo->cpu); /* acquire the lock protecting the state and disable interrupts */ //raw_spin_lock(&_global_env.lock); //raw_spin_lock(&state->lock); if (is_running) { state->scheduled = tsk; /* make sure this task should actually be running */ litmus_reschedule_local(); } local_irq_save(flags); raw_spin_lock(&state->lock); if (lv == CRIT_LEVEL_C) { raw_spin_lock(&global_lock); res = gmp_find_by_id(_global_env, tinfo->mc2_param.res_id); } else { res = sup_find_by_id(state->sup_env, tinfo->mc2_param.res_id); } //BUG_ON(!res); // the current mode doesn't have this task. // do not update timer and set the next release time. //res = res_find_by_id(state, tinfo->mc2_param.res_id); BUG_ON(!res); if (on_runqueue || is_running) { /* Assumption: litmus_clock() is synchronized across cores * [see comment in pres_task_resume()] */ if (lv == CRIT_LEVEL_C) { gmp_update_time(_global_env, litmus_clock()); //raw_spin_unlock(&_global_env.lock); } else sup_update_time(state->sup_env, litmus_clock()); //mc2_update_time(lv, state, litmus_clock()); /* 9/20/2015 fix mc2_update_ghost_state(state); */ task_arrives(state, tsk); if (lv == CRIT_LEVEL_C) raw_spin_unlock(&global_lock); /* NOTE: drops state->lock */ TRACE("mc2_new()\n"); mc2_update_timer_and_unlock(state); } else { if (lv == CRIT_LEVEL_C) raw_spin_unlock(&global_lock); raw_spin_unlock(&state->lock); //raw_spin_unlock(&_global_env.lock); } release = res->next_replenishment; local_irq_restore(flags); if (!release) { /*TRACE_TASK(tsk, "mc2_task_new() next_release = NULL\n"); release = res->next_replenishment; TRACE_TASK(tsk, "mc2_task_new() next_release SET! = %llu\n", release); release_at(tsk, release); */ BUG(); } else TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release); } /* mc2_reservation_destroy - reservation_destroy system call backend */ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu) { long ret = -EINVAL; struct mc2_cpu_state *state; struct reservation *res = NULL, *next; struct sup_reservation_environment *sup_env; int found = 0; //enum crit_level lv = get_task_crit_level(current); unsigned long flags; int i; if (cpu == -1) { struct next_timer_event *event, *e_next; local_irq_save(flags); raw_spin_lock(&global_lock); /* if the reservation is global reservation */ //state = local_cpu_state(); //delete reservation id in all modes for(i = 0; i < NR_MODES; i++) { //raw_spin_lock(&state->lock); list_for_each_entry_safe(res, next, &_global_env_modes[i].depleted_reservations, list) { if (res->id == reservation_id) { list_del(&res->list); kfree(res); found = 1; ret = 0; } } if (!found) { list_for_each_entry_safe(res, next, &_global_env_modes[i].inactive_reservations, list) { if (res->id == reservation_id) { list_del(&res->list); kfree(res); found = 1; ret = 0; } } } if (!found) { list_for_each_entry_safe(res, next, &_global_env_modes[i].active_reservations, list) { if (res->id == reservation_id) { list_del(&res->list); kfree(res); found = 1; ret = 0; } } } //raw_spin_unlock(&state->lock); list_for_each_entry_safe(event, e_next, &_global_env_modes[i].next_events, list) { if (event->id == reservation_id) { list_del(&event->list); TRACE("EVENT id %d deleted\n", event->id); kfree(event); } } } raw_spin_unlock(&global_lock); local_irq_restore(flags); } else { /* if the reservation is partitioned reservation */ state = cpu_state_for(cpu); for (i = 0; i < NR_MODES; i++){ local_irq_save(flags); raw_spin_lock(&state->lock); // res = sup_find_by_id(state->sup_env, reservation_id); sup_env = &(state->sup_env_modes[i]); list_for_each_entry_safe(res, next, &sup_env->depleted_reservations, list) { if (res->id == reservation_id) { /* if (lv == CRIT_LEVEL_A) { struct table_driven_reservation *tdres; tdres = container_of(res, struct table_driven_reservation, res); kfree(tdres->intervals); } */ list_del(&res->list); kfree(res); found = 1; ret = 0; TRACE_CUR("FOUND id %d mode %d\n",res->id, res->mode); } } if (!found) { list_for_each_entry_safe(res, next, &sup_env->inactive_reservations, list) { if (res->id == reservation_id) { /* if (lv == CRIT_LEVEL_A) { struct table_driven_reservation *tdres; tdres = container_of(res, struct table_driven_reservation, res); kfree(tdres->intervals); } */ list_del(&res->list); kfree(res); found = 1; ret = 0; TRACE_CUR("FOUND id %d mode %d\n",res->id, res->mode); } } } if (!found) { list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { if (res->id == reservation_id) { /* if (lv == CRIT_LEVEL_A) { struct table_driven_reservation *tdres; tdres = container_of(res, struct table_driven_reservation, res); kfree(tdres->intervals); } */ list_del(&res->list); kfree(res); found = 1; ret = 0; TRACE_CUR("FOUND id %d mode %d\n",res->id, res->mode); } } } raw_spin_unlock(&state->lock); local_irq_restore(flags); } } TRACE("Rerservation destroyed ret = %d\n", ret); return ret; } /* mc2_task_exit - Task became a normal task (not real-time task) */ static void mc2_task_exit(struct task_struct *tsk) { unsigned long flags; struct mc2_task_state* tinfo = get_mc2_state(tsk); struct mc2_cpu_state *state; enum crit_level lv = tinfo->mc2_param.crit; //struct crit_entry* ce; int cpu; int i; local_irq_save(flags); if (tinfo->cpu != -1) state = cpu_state_for(tinfo->cpu); else state = local_cpu_state(); raw_spin_lock(&state->lock); if (state->scheduled == tsk) state->scheduled = NULL; //ce = &state->crit_entries[lv]; //if (ce->running == tsk) // ce->running = NULL; /* remove from queues */ if (is_running(tsk)) { /* Assumption: litmus_clock() is synchronized across cores * [see comment in pres_task_resume()] */ /* update both global and partitioned */ if (lv < CRIT_LEVEL_C) { sup_update_time(state->sup_env, litmus_clock()); /* raw_spin_lock(&global_lock); gmp_update_time(_global_env, litmus_clock()); raw_spin_unlock(&global_lock); */ } else if (lv == CRIT_LEVEL_C) { raw_spin_lock(&global_lock); gmp_update_time(_global_env, litmus_clock()); //raw_spin_unlock(&_global_env.lock); } /* 9/20/2015 fix mc2_update_ghost_state(state); */ task_departs(tsk, 0); if (lv == CRIT_LEVEL_C) raw_spin_unlock(&global_lock); /* NOTE: drops state->lock */ TRACE("mc2_exit()\n"); mc2_update_timer_and_unlock(state); } else { raw_spin_unlock(&state->lock); } if (lv == CRIT_LEVEL_C) { raw_spin_lock(&mode_lock); for(i = 0; i < NR_MODES; i++){ if ( !(tsk_mc2_data(tsk)->mode_mask & (1<lock); if (state->scheduled == tsk) state->scheduled = NULL; //ce = &state->crit_entries[lv]; //if (ce->running == tsk) // ce->running = NULL; raw_spin_unlock(&state->lock); } } local_irq_restore(flags); if (is_mode_poll_task(tsk) && (tinfo->cpu == 0)) { cpu_0_spin_flag = !cpu_0_spin_flag; // release other cpu before exit. cpu_0_task_exist = false; } kfree(tsk_rt(tsk)->plugin_state); tsk_rt(tsk)->plugin_state = NULL; kfree(tsk_rt(tsk)->mc2_data); tsk_rt(tsk)->mc2_data = NULL; } /* create_polling_reservation - create a new polling reservation */ static long create_polling_reservation( int res_type, struct reservation_config *config) { struct mc2_cpu_state *state = NULL; //struct reservation* res = NULL; struct polling_reservation *pres; unsigned long flags; int use_edf = config->priority == LITMUS_NO_PRIORITY; int periodic = res_type == PERIODIC_POLLING; long err = -EINVAL; bool resExist = false; /* sanity checks */ if (config->polling_params.budget > config->polling_params.period) { printk(KERN_ERR "invalid polling reservation (%u): " "budget > period\n", config->id); return -EINVAL; } if (config->polling_params.budget > config->polling_params.relative_deadline && config->polling_params.relative_deadline) { printk(KERN_ERR "invalid polling reservation (%u): " "budget > deadline\n", config->id); return -EINVAL; } if (config->polling_params.offset > config->polling_params.period) { printk(KERN_ERR "invalid polling reservation (%u): " "offset > period\n", config->id); return -EINVAL; } //Added sanity check for mode if (config->mode < 0 || config->mode >= NR_MODES){ printk(KERN_ERR "invalid polling reservation (%u): " "Mode outside range\n", config->id); return -EINVAL; } /* Allocate before we grab a spin lock. * Todo: would be nice to use a core-local allocation. */ pres = kzalloc(sizeof(*pres), GFP_KERNEL); if (!pres) return -ENOMEM; TRACE("CREATE_POLLING_RESERVATION id %d mode %d\n", config->id, config->mode); if (config->cpu != -1) { int i, is_exist = 0; //raw_spin_lock_irqsave(&_global_env.lock, flags); state = cpu_state_for(config->cpu); raw_spin_lock_irqsave(&state->lock, flags); /* check if it is the first creation of reservartion */ for (i = 0; i < NR_MODES; i++) { if( sup_find_by_id(&(state->sup_env_modes[i]), config->id) ) is_exist = 1; } if (!is_exist && config->mode != 0) { /* create mode 0 reservation first */ struct polling_reservation *pres_0 = kzalloc(sizeof(*pres_0), GFP_ATOMIC); TRACE_CUR("The first mode_num = %d\n",config->mode); if (!pres_0) { raw_spin_unlock_irqrestore(&state->lock, flags); kfree(pres); return -ENOMEM; } polling_reservation_init(pres_0, use_edf, periodic, config->polling_params.budget, config->polling_params.period, config->polling_params.relative_deadline, config->polling_params.offset); pres_0->res.id = config->id; pres_0->res.blocked_by_ghost = 0; pres_0->res.is_ghost = NO_CPU; pres_0->res.mode = config->mode; if (!use_edf) pres_0->res.priority = config->priority; sup_add_new_reservation(&(state->sup_env_modes[0]), &pres_0->res); TRACE_CUR("SUP reservation created R%d for mode 0 priority : %llu\n", config->id, pres_0->res.priority); pres_0->res.reported = 0; pres_0->res.tsk = current; } //force reservation id unique inside of res_config->mode if( sup_find_by_id(&(state->sup_env_modes[config->mode]), config->id) ){ resExist = true; } if (!resExist) { polling_reservation_init(pres, use_edf, periodic, config->polling_params.budget, config->polling_params.period, config->polling_params.relative_deadline, config->polling_params.offset); pres->res.id = config->id; pres->res.blocked_by_ghost = 0; pres->res.is_ghost = NO_CPU; pres->res.mode = config->mode; /*if (config->priority == LITMUS_MAX_PRIORITY) { level_a_priorities[config->cpu]++; pres->res.priority = level_a_priorities[config->cpu]; }*/ if (!use_edf) pres->res.priority = config->priority; sup_add_new_reservation(&(state->sup_env_modes[config->mode]), &pres->res); err = config->id; TRACE_CUR("SUP reservation created R%d for mode %d priority : %llu\n", config->id, config->mode, pres->res.priority); } else { err = -EEXIST; } raw_spin_unlock_irqrestore(&state->lock, flags); //raw_spin_unlock_irqrestore(&_global_env.lock, flags); } else { int i, is_exist = 0; raw_spin_lock_irqsave(&global_lock, flags); /* check if it is the first creation of reservartion */ for (i = 0; i < NR_MODES; i++) { if(gmp_find_by_id(&(_global_env_modes[i]), config->id)) is_exist = 1; } if (!is_exist && config->mode != 0) { /* create mode 0 reservation first */ struct polling_reservation *pres_0 = kzalloc(sizeof(*pres_0), GFP_ATOMIC); TRACE_CUR("The first mode_num = %d\n",config->mode); if (!pres_0) { raw_spin_unlock_irqrestore(&global_lock, flags); kfree(pres); return -ENOMEM; } polling_reservation_init(pres_0, use_edf, periodic, config->polling_params.budget, config->polling_params.period, config->polling_params.relative_deadline, config->polling_params.offset); pres_0->res.id = config->id; pres_0->res.blocked_by_ghost = 0; pres_0->res.scheduled_on = NO_CPU; pres_0->res.is_ghost = NO_CPU; pres_0->res.mode = config->mode; if (!use_edf) pres_0->res.priority = config->priority; gmp_add_new_reservation(&(_global_env_modes[0]), &pres_0->res); TRACE_CUR("GMP reservation created R%d for mode 0 priority : %llu\n", config->id, pres_0->res.priority); pres_0->res.reported = 0; pres_0->res.tsk = current; } //force id's unique within desired mode if (gmp_find_by_id(&(_global_env_modes[config->mode]), config->id)){ resExist = true; } if (!resExist) { polling_reservation_init(pres, use_edf, periodic, config->polling_params.budget, config->polling_params.period, config->polling_params.relative_deadline, config->polling_params.offset); pres->res.id = config->id; pres->res.blocked_by_ghost = 0; pres->res.scheduled_on = NO_CPU; pres->res.is_ghost = NO_CPU; pres->res.mode = config->mode; if (!use_edf) pres->res.priority = config->priority; gmp_add_new_reservation(&(_global_env_modes[config->mode]), &pres->res); TRACE_CUR("GMP reservation created R%d for mode %d priority : %llu\n", config->id, config->mode, pres->res.priority); err = config->id; } else { err = -EEXIST; } raw_spin_unlock_irqrestore(&global_lock, flags); } pres->res.reported = 0; pres->res.tsk = current; if (err < 0) kfree(pres); return err; } #define MAX_INTERVALS 1024 /* create_table_driven_reservation - create a table_driven reservation */ static long create_table_driven_reservation( struct reservation_config *config) { struct mc2_cpu_state *state; //struct reservation* res = NULL; struct table_driven_reservation *td_res = NULL; struct lt_interval *slots = NULL; size_t slots_size; unsigned int i, num_slots; unsigned long flags; long err = -EINVAL; bool resExist = false; if (!config->table_driven_params.num_intervals) { printk(KERN_ERR "invalid table-driven reservation (%u): " "no intervals\n", config->id); return -EINVAL; } if (config->table_driven_params.num_intervals > MAX_INTERVALS) { printk(KERN_ERR "invalid table-driven reservation (%u): " "too many intervals (max: %d)\n", config->id, MAX_INTERVALS); return -EINVAL; } if (config->mode >= NR_MODES || config->mode < 0){ printk(KERN_ERR "invalid table-driven reservation (%u): " "mode outside of range\n", config->id); return -EINVAL; } num_slots = config->table_driven_params.num_intervals; slots_size = sizeof(slots[0]) * num_slots; slots = kzalloc(slots_size, GFP_KERNEL); if (!slots) return -ENOMEM; td_res = kzalloc(sizeof(*td_res), GFP_KERNEL); if (!td_res) err = -ENOMEM; else err = copy_from_user(slots, config->table_driven_params.intervals, slots_size); if (!err) { /* sanity checks */ for (i = 0; !err && i < num_slots; i++) if (slots[i].end <= slots[i].start) { printk(KERN_ERR "invalid table-driven reservation (%u): " "invalid interval %u => [%llu, %llu]\n", config->id, i, slots[i].start, slots[i].end); err = -EINVAL; } for (i = 0; !err && i + 1 < num_slots; i++) if (slots[i + 1].start <= slots[i].end) { printk(KERN_ERR "invalid table-driven reservation (%u): " "overlapping intervals %u, %u\n", config->id, i, i + 1); err = -EINVAL; } if (slots[num_slots - 1].end > config->table_driven_params.major_cycle_length) { printk(KERN_ERR "invalid table-driven reservation (%u): last " "interval ends past major cycle %llu > %llu\n", config->id, slots[num_slots - 1].end, config->table_driven_params.major_cycle_length); err = -EINVAL; } } if (!err) { state = cpu_state_for(config->cpu); raw_spin_lock_irqsave(&state->lock, flags); //force unique id's across all modes for(i = 0; i < NR_MODES; i++){ if (sup_find_by_id(&(state->sup_env_modes[i]), config->id)){ resExist = true; break; } } if (!resExist) { table_driven_reservation_init(td_res, config->table_driven_params.major_cycle_length, slots, num_slots); td_res->res.id = config->id; td_res->res.priority = config->priority; td_res->res.blocked_by_ghost = 0; td_res->res.mode = config->mode; sup_add_new_reservation(&(state->sup_env_modes[config->mode]), &td_res->res); err = config->id; } else { err = -EEXIST; } raw_spin_unlock_irqrestore(&state->lock, flags); } td_res->res.reported = 0; td_res->res.tsk = current; if (err < 0) { kfree(slots); kfree(td_res); } return err; } /* mc2_reservation_create - reservation_create system call backend */ static long mc2_reservation_create(int res_type, void* __user _config) { long ret = -EINVAL; struct reservation_config config; TRACE("Attempt to create reservation (%d)\n", res_type); if (copy_from_user(&config, _config, sizeof(config))) return -EFAULT; TRACE("Attempt to create reservation id %d mode %d\n", config.id, config.mode); if (config.cpu != -1) { if (config.cpu < 0 || !cpu_online(config.cpu)) { printk(KERN_ERR "invalid polling reservation (%u): " "CPU %d offline\n", config.id, config.cpu); return -EINVAL; } } switch (res_type) { case PERIODIC_POLLING: case SPORADIC_POLLING: ret = create_polling_reservation(res_type, &config); break; case TABLE_DRIVEN: ret = create_table_driven_reservation(&config); break; default: return -EINVAL; }; return ret; } static struct domain_proc_info mc2_domain_proc_info; static long mc2_get_domain_proc_info(struct domain_proc_info **ret) { *ret = &mc2_domain_proc_info; return 0; } static void mc2_setup_domain_proc(void) { int i, cpu; int num_rt_cpus = num_online_cpus(); struct cd_mapping *cpu_map, *domain_map; memset(&mc2_domain_proc_info, sizeof(mc2_domain_proc_info), 0); init_domain_proc_info(&mc2_domain_proc_info, num_rt_cpus, num_rt_cpus); mc2_domain_proc_info.num_cpus = num_rt_cpus; mc2_domain_proc_info.num_domains = num_rt_cpus; i = 0; for_each_online_cpu(cpu) { cpu_map = &mc2_domain_proc_info.cpu_to_domains[i]; domain_map = &mc2_domain_proc_info.domain_to_cpus[i]; cpu_map->id = cpu; domain_map->id = i; cpumask_set_cpu(i, cpu_map->mask); cpumask_set_cpu(cpu, domain_map->mask); ++i; } } static long mc2_activate_plugin(void) { int cpu;//, lv; struct mc2_cpu_state *state; struct cpu_entry *ce; int i; for(i = 0; i < NR_MODES; i++){ gmp_init(&(_global_env_modes[i])); } _global_env = &_global_env_modes[0]; //raw_spin_lock_init(&_lowest_prio_cpu.lock); raw_spin_lock_init(&mode_lock); raw_spin_lock_init(&global_lock); seen_once = false; for_each_online_cpu(cpu) { TRACE("Initializing CPU%d...\n", cpu); resched_cpu[cpu] = 0; //level_a_priorities[cpu] = 0; state = cpu_state_for(cpu); ce = &_lowest_prio_cpu.cpu_entries[cpu]; ce->cpu = cpu; ce->scheduled = NULL; ce->deadline = ULLONG_MAX; ce->lv = NUM_CRIT_LEVELS; ce->will_schedule = false; raw_spin_lock_init(&state->lock); state->cpu = cpu; state->scheduled = NULL; //for (lv = 0; lv < NUM_CRIT_LEVELS; lv++) { // struct crit_entry *cr_entry = &state->crit_entries[lv]; // cr_entry->level = lv; // cr_entry->running = NULL; //} for(i = 0; i < NR_MODES; i++){ sup_init(&(state->sup_env_modes[i])); } state->sup_env = &(state->sup_env_modes[0]); hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); state->timer.function = on_scheduling_timer; state->spin_flag = false; } mc2_setup_domain_proc(); mode = 0; requested_mode = 0; for(i = 0; i < NR_MODES; i++){ mode_sizes[i] = 0; } res_reported = 0; cpu_0_spin_flag = false; cpu_0_task_exist = false; return 0; } static void mc2_finish_switch(struct task_struct *prev) { int cpus; enum crit_level lv = get_task_crit_level(prev); struct mc2_cpu_state *state = local_cpu_state(); state->scheduled = is_realtime(current) ? current : NULL; if (lv == CRIT_LEVEL_C) { for (cpus = 0; cpuslock); hrtimer_cancel(&state->timer); ce = &_lowest_prio_cpu.cpu_entries[cpu]; ce->cpu = cpu; ce->scheduled = NULL; ce->deadline = ULLONG_MAX; ce->lv = NUM_CRIT_LEVELS; ce->will_schedule = false; for(i = 0; i < NR_MODES; i++){ /* Delete all reservations --- assumes struct reservation * is prefix of containing struct. */ state->sup_env = &(state->sup_env_modes[i]); while (!list_empty(&state->sup_env->active_reservations)) { res = list_first_entry( &state->sup_env->active_reservations, struct reservation, list); list_del(&res->list); kfree(res); } while (!list_empty(&state->sup_env->inactive_reservations)) { res = list_first_entry( &state->sup_env->inactive_reservations, struct reservation, list); list_del(&res->list); kfree(res); } while (!list_empty(&state->sup_env->depleted_reservations)) { res = list_first_entry( &state->sup_env->depleted_reservations, struct reservation, list); list_del(&res->list); kfree(res); } } raw_spin_unlock(&state->lock); } raw_spin_lock(&global_lock); for(i = 0; i < NR_MODES; i++){ _global_env = &_global_env_modes[i]; while (!list_empty(&_global_env->active_reservations)) { res = list_first_entry( &_global_env->active_reservations, struct reservation, list); list_del(&res->list); kfree(res); } while (!list_empty(&_global_env->inactive_reservations)) { res = list_first_entry( &_global_env->inactive_reservations, struct reservation, list); list_del(&res->list); kfree(res); } while (!list_empty(&_global_env->depleted_reservations)) { res = list_first_entry( &_global_env->depleted_reservations, struct reservation, list); list_del(&res->list); kfree(res); } while (!list_empty(&_global_env->next_events)) { event = list_first_entry( &_global_env->next_events, struct next_timer_event, list); list_del(&event->list); kfree(event); } } raw_spin_unlock(&global_lock); destroy_domain_proc_info(&mc2_domain_proc_info); return 0; } static struct sched_plugin mc2_plugin = { .plugin_name = "MC2", .schedule = mc2_schedule, .finish_switch = mc2_finish_switch, .task_wake_up = mc2_task_resume, .admit_task = mc2_admit_task, .task_new = mc2_task_new, .task_exit = mc2_task_exit, .complete_job = mc2_complete_job, .get_domain_proc_info = mc2_get_domain_proc_info, .activate_plugin = mc2_activate_plugin, .deactivate_plugin = mc2_deactivate_plugin, .reservation_create = mc2_reservation_create, .reservation_destroy = mc2_reservation_destroy, }; static int __init init_mc2(void) { return register_sched_plugin(&mc2_plugin); } module_init(init_mc2);