|
@@ -119,8 +119,6 @@
|
|
|
*/
|
|
|
#define RUNTIME_INF ((u64)~0ULL)
|
|
|
|
|
|
-static void double_rq_lock(struct rq *rq1, struct rq *rq2);
|
|
|
-
|
|
|
static inline int rt_policy(int policy)
|
|
|
{
|
|
|
if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
|
|
@@ -378,13 +376,6 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
|
|
|
|
|
|
#else
|
|
|
|
|
|
-#ifdef CONFIG_SMP
|
|
|
-static int root_task_group_empty(void)
|
|
|
-{
|
|
|
- return 1;
|
|
|
-}
|
|
|
-#endif
|
|
|
-
|
|
|
static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
|
|
|
static inline struct task_group *task_group(struct task_struct *p)
|
|
|
{
|
|
@@ -514,14 +505,6 @@ struct root_domain {
|
|
|
#ifdef CONFIG_SMP
|
|
|
struct cpupri cpupri;
|
|
|
#endif
|
|
|
-#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
|
|
|
- /*
|
|
|
- * Preferred wake up cpu nominated by sched_mc balance that will be
|
|
|
- * used when most cpus are idle in the system indicating overall very
|
|
|
- * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
|
|
|
- */
|
|
|
- unsigned int sched_mc_preferred_wakeup_cpu;
|
|
|
-#endif
|
|
|
};
|
|
|
|
|
|
/*
|
|
@@ -646,9 +629,10 @@ struct rq {
|
|
|
|
|
|
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
|
|
|
|
|
|
-static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
|
|
|
+static inline
|
|
|
+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
|
|
|
{
|
|
|
- rq->curr->sched_class->check_preempt_curr(rq, p, sync);
|
|
|
+ rq->curr->sched_class->check_preempt_curr(rq, p, flags);
|
|
|
}
|
|
|
|
|
|
static inline int cpu_of(struct rq *rq)
|
|
@@ -1509,8 +1493,65 @@ static int tg_nop(struct task_group *tg, void *data)
|
|
|
#endif
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
-static unsigned long source_load(int cpu, int type);
|
|
|
-static unsigned long target_load(int cpu, int type);
|
|
|
+/* Used instead of source_load when we know the type == 0 */
|
|
|
+static unsigned long weighted_cpuload(const int cpu)
|
|
|
+{
|
|
|
+ return cpu_rq(cpu)->load.weight;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Return a low guess at the load of a migration-source cpu weighted
|
|
|
+ * according to the scheduling class and "nice" value.
|
|
|
+ *
|
|
|
+ * We want to under-estimate the load of migration sources, to
|
|
|
+ * balance conservatively.
|
|
|
+ */
|
|
|
+static unsigned long source_load(int cpu, int type)
|
|
|
+{
|
|
|
+ struct rq *rq = cpu_rq(cpu);
|
|
|
+ unsigned long total = weighted_cpuload(cpu);
|
|
|
+
|
|
|
+ if (type == 0 || !sched_feat(LB_BIAS))
|
|
|
+ return total;
|
|
|
+
|
|
|
+ return min(rq->cpu_load[type-1], total);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Return a high guess at the load of a migration-target cpu weighted
|
|
|
+ * according to the scheduling class and "nice" value.
|
|
|
+ */
|
|
|
+static unsigned long target_load(int cpu, int type)
|
|
|
+{
|
|
|
+ struct rq *rq = cpu_rq(cpu);
|
|
|
+ unsigned long total = weighted_cpuload(cpu);
|
|
|
+
|
|
|
+ if (type == 0 || !sched_feat(LB_BIAS))
|
|
|
+ return total;
|
|
|
+
|
|
|
+ return max(rq->cpu_load[type-1], total);
|
|
|
+}
|
|
|
+
|
|
|
+static struct sched_group *group_of(int cpu)
|
|
|
+{
|
|
|
+ struct sched_domain *sd = rcu_dereference(cpu_rq(cpu)->sd);
|
|
|
+
|
|
|
+ if (!sd)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ return sd->groups;
|
|
|
+}
|
|
|
+
|
|
|
+static unsigned long power_of(int cpu)
|
|
|
+{
|
|
|
+ struct sched_group *group = group_of(cpu);
|
|
|
+
|
|
|
+ if (!group)
|
|
|
+ return SCHED_LOAD_SCALE;
|
|
|
+
|
|
|
+ return group->cpu_power;
|
|
|
+}
|
|
|
+
|
|
|
static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
|
|
|
|
|
|
static unsigned long cpu_avg_load_per_task(int cpu)
|
|
@@ -1695,6 +1736,8 @@ static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
|
|
+static void double_rq_lock(struct rq *rq1, struct rq *rq2);
|
|
|
+
|
|
|
/*
|
|
|
* fair double_lock_balance: Safely acquires both rq->locks in a fair
|
|
|
* way at the expense of forcing extra atomic operations in all
|
|
@@ -1959,13 +2002,6 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
|
|
|
}
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
-
|
|
|
-/* Used instead of source_load when we know the type == 0 */
|
|
|
-static unsigned long weighted_cpuload(const int cpu)
|
|
|
-{
|
|
|
- return cpu_rq(cpu)->load.weight;
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* Is this task likely cache-hot:
|
|
|
*/
|
|
@@ -2239,185 +2275,6 @@ void kick_process(struct task_struct *p)
|
|
|
preempt_enable();
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(kick_process);
|
|
|
-
|
|
|
-/*
|
|
|
- * Return a low guess at the load of a migration-source cpu weighted
|
|
|
- * according to the scheduling class and "nice" value.
|
|
|
- *
|
|
|
- * We want to under-estimate the load of migration sources, to
|
|
|
- * balance conservatively.
|
|
|
- */
|
|
|
-static unsigned long source_load(int cpu, int type)
|
|
|
-{
|
|
|
- struct rq *rq = cpu_rq(cpu);
|
|
|
- unsigned long total = weighted_cpuload(cpu);
|
|
|
-
|
|
|
- if (type == 0 || !sched_feat(LB_BIAS))
|
|
|
- return total;
|
|
|
-
|
|
|
- return min(rq->cpu_load[type-1], total);
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * Return a high guess at the load of a migration-target cpu weighted
|
|
|
- * according to the scheduling class and "nice" value.
|
|
|
- */
|
|
|
-static unsigned long target_load(int cpu, int type)
|
|
|
-{
|
|
|
- struct rq *rq = cpu_rq(cpu);
|
|
|
- unsigned long total = weighted_cpuload(cpu);
|
|
|
-
|
|
|
- if (type == 0 || !sched_feat(LB_BIAS))
|
|
|
- return total;
|
|
|
-
|
|
|
- return max(rq->cpu_load[type-1], total);
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * find_idlest_group finds and returns the least busy CPU group within the
|
|
|
- * domain.
|
|
|
- */
|
|
|
-static struct sched_group *
|
|
|
-find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
|
|
|
-{
|
|
|
- struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
|
|
|
- unsigned long min_load = ULONG_MAX, this_load = 0;
|
|
|
- int load_idx = sd->forkexec_idx;
|
|
|
- int imbalance = 100 + (sd->imbalance_pct-100)/2;
|
|
|
-
|
|
|
- do {
|
|
|
- unsigned long load, avg_load;
|
|
|
- int local_group;
|
|
|
- int i;
|
|
|
-
|
|
|
- /* Skip over this group if it has no CPUs allowed */
|
|
|
- if (!cpumask_intersects(sched_group_cpus(group),
|
|
|
- &p->cpus_allowed))
|
|
|
- continue;
|
|
|
-
|
|
|
- local_group = cpumask_test_cpu(this_cpu,
|
|
|
- sched_group_cpus(group));
|
|
|
-
|
|
|
- /* Tally up the load of all CPUs in the group */
|
|
|
- avg_load = 0;
|
|
|
-
|
|
|
- for_each_cpu(i, sched_group_cpus(group)) {
|
|
|
- /* Bias balancing toward cpus of our domain */
|
|
|
- if (local_group)
|
|
|
- load = source_load(i, load_idx);
|
|
|
- else
|
|
|
- load = target_load(i, load_idx);
|
|
|
-
|
|
|
- avg_load += load;
|
|
|
- }
|
|
|
-
|
|
|
- /* Adjust by relative CPU power of the group */
|
|
|
- avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
|
|
|
-
|
|
|
- if (local_group) {
|
|
|
- this_load = avg_load;
|
|
|
- this = group;
|
|
|
- } else if (avg_load < min_load) {
|
|
|
- min_load = avg_load;
|
|
|
- idlest = group;
|
|
|
- }
|
|
|
- } while (group = group->next, group != sd->groups);
|
|
|
-
|
|
|
- if (!idlest || 100*this_load < imbalance*min_load)
|
|
|
- return NULL;
|
|
|
- return idlest;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * find_idlest_cpu - find the idlest cpu among the cpus in group.
|
|
|
- */
|
|
|
-static int
|
|
|
-find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
|
|
|
-{
|
|
|
- unsigned long load, min_load = ULONG_MAX;
|
|
|
- int idlest = -1;
|
|
|
- int i;
|
|
|
-
|
|
|
- /* Traverse only the allowed CPUs */
|
|
|
- for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
|
|
|
- load = weighted_cpuload(i);
|
|
|
-
|
|
|
- if (load < min_load || (load == min_load && i == this_cpu)) {
|
|
|
- min_load = load;
|
|
|
- idlest = i;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return idlest;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * sched_balance_self: balance the current task (running on cpu) in domains
|
|
|
- * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
|
|
|
- * SD_BALANCE_EXEC.
|
|
|
- *
|
|
|
- * Balance, ie. select the least loaded group.
|
|
|
- *
|
|
|
- * Returns the target CPU number, or the same CPU if no balancing is needed.
|
|
|
- *
|
|
|
- * preempt must be disabled.
|
|
|
- */
|
|
|
-static int sched_balance_self(int cpu, int flag)
|
|
|
-{
|
|
|
- struct task_struct *t = current;
|
|
|
- struct sched_domain *tmp, *sd = NULL;
|
|
|
-
|
|
|
- for_each_domain(cpu, tmp) {
|
|
|
- /*
|
|
|
- * If power savings logic is enabled for a domain, stop there.
|
|
|
- */
|
|
|
- if (tmp->flags & SD_POWERSAVINGS_BALANCE)
|
|
|
- break;
|
|
|
- if (tmp->flags & flag)
|
|
|
- sd = tmp;
|
|
|
- }
|
|
|
-
|
|
|
- if (sd)
|
|
|
- update_shares(sd);
|
|
|
-
|
|
|
- while (sd) {
|
|
|
- struct sched_group *group;
|
|
|
- int new_cpu, weight;
|
|
|
-
|
|
|
- if (!(sd->flags & flag)) {
|
|
|
- sd = sd->child;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- group = find_idlest_group(sd, t, cpu);
|
|
|
- if (!group) {
|
|
|
- sd = sd->child;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- new_cpu = find_idlest_cpu(group, t, cpu);
|
|
|
- if (new_cpu == -1 || new_cpu == cpu) {
|
|
|
- /* Now try balancing at a lower domain level of cpu */
|
|
|
- sd = sd->child;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- /* Now try balancing at a lower domain level of new_cpu */
|
|
|
- cpu = new_cpu;
|
|
|
- weight = cpumask_weight(sched_domain_span(sd));
|
|
|
- sd = NULL;
|
|
|
- for_each_domain(cpu, tmp) {
|
|
|
- if (weight <= cpumask_weight(sched_domain_span(tmp)))
|
|
|
- break;
|
|
|
- if (tmp->flags & flag)
|
|
|
- sd = tmp;
|
|
|
- }
|
|
|
- /* while loop will break here if sd == NULL */
|
|
|
- }
|
|
|
-
|
|
|
- return cpu;
|
|
|
-}
|
|
|
-
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
|
/**
|
|
@@ -2455,37 +2312,22 @@ void task_oncpu_function_call(struct task_struct *p,
|
|
|
*
|
|
|
* returns failure only if the task is already active.
|
|
|
*/
|
|
|
-static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
|
|
|
+static int try_to_wake_up(struct task_struct *p, unsigned int state,
|
|
|
+ int wake_flags)
|
|
|
{
|
|
|
int cpu, orig_cpu, this_cpu, success = 0;
|
|
|
unsigned long flags;
|
|
|
- long old_state;
|
|
|
struct rq *rq;
|
|
|
|
|
|
if (!sched_feat(SYNC_WAKEUPS))
|
|
|
- sync = 0;
|
|
|
-
|
|
|
-#ifdef CONFIG_SMP
|
|
|
- if (sched_feat(LB_WAKEUP_UPDATE) && !root_task_group_empty()) {
|
|
|
- struct sched_domain *sd;
|
|
|
+ wake_flags &= ~WF_SYNC;
|
|
|
|
|
|
- this_cpu = raw_smp_processor_id();
|
|
|
- cpu = task_cpu(p);
|
|
|
-
|
|
|
- for_each_domain(this_cpu, sd) {
|
|
|
- if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
|
|
|
- update_shares(sd);
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-#endif
|
|
|
+ this_cpu = get_cpu();
|
|
|
|
|
|
smp_wmb();
|
|
|
rq = task_rq_lock(p, &flags);
|
|
|
update_rq_clock(rq);
|
|
|
- old_state = p->state;
|
|
|
- if (!(old_state & state))
|
|
|
+ if (!(p->state & state))
|
|
|
goto out;
|
|
|
|
|
|
if (p->se.on_rq)
|
|
@@ -2493,27 +2335,29 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
|
|
|
|
|
|
cpu = task_cpu(p);
|
|
|
orig_cpu = cpu;
|
|
|
- this_cpu = smp_processor_id();
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
if (unlikely(task_running(rq, p)))
|
|
|
goto out_activate;
|
|
|
|
|
|
- cpu = p->sched_class->select_task_rq(p, sync);
|
|
|
- if (cpu != orig_cpu) {
|
|
|
+ /*
|
|
|
+ * In order to handle concurrent wakeups and release the rq->lock
|
|
|
+ * we put the task in TASK_WAKING state.
|
|
|
+ *
|
|
|
+ * First fix up the nr_uninterruptible count:
|
|
|
+ */
|
|
|
+ if (task_contributes_to_load(p))
|
|
|
+ rq->nr_uninterruptible--;
|
|
|
+ p->state = TASK_WAKING;
|
|
|
+ task_rq_unlock(rq, &flags);
|
|
|
+
|
|
|
+ cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
|
|
|
+ if (cpu != orig_cpu)
|
|
|
set_task_cpu(p, cpu);
|
|
|
- task_rq_unlock(rq, &flags);
|
|
|
- /* might preempt at this point */
|
|
|
- rq = task_rq_lock(p, &flags);
|
|
|
- old_state = p->state;
|
|
|
- if (!(old_state & state))
|
|
|
- goto out;
|
|
|
- if (p->se.on_rq)
|
|
|
- goto out_running;
|
|
|
|
|
|
- this_cpu = smp_processor_id();
|
|
|
- cpu = task_cpu(p);
|
|
|
- }
|
|
|
+ rq = task_rq_lock(p, &flags);
|
|
|
+ WARN_ON(p->state != TASK_WAKING);
|
|
|
+ cpu = task_cpu(p);
|
|
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
schedstat_inc(rq, ttwu_count);
|
|
@@ -2533,7 +2377,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
|
|
|
out_activate:
|
|
|
#endif /* CONFIG_SMP */
|
|
|
schedstat_inc(p, se.nr_wakeups);
|
|
|
- if (sync)
|
|
|
+ if (wake_flags & WF_SYNC)
|
|
|
schedstat_inc(p, se.nr_wakeups_sync);
|
|
|
if (orig_cpu != cpu)
|
|
|
schedstat_inc(p, se.nr_wakeups_migrate);
|
|
@@ -2562,7 +2406,7 @@ out_activate:
|
|
|
|
|
|
out_running:
|
|
|
trace_sched_wakeup(rq, p, success);
|
|
|
- check_preempt_curr(rq, p, sync);
|
|
|
+ check_preempt_curr(rq, p, wake_flags);
|
|
|
|
|
|
p->state = TASK_RUNNING;
|
|
|
#ifdef CONFIG_SMP
|
|
@@ -2571,6 +2415,7 @@ out_running:
|
|
|
#endif
|
|
|
out:
|
|
|
task_rq_unlock(rq, &flags);
|
|
|
+ put_cpu();
|
|
|
|
|
|
return success;
|
|
|
}
|
|
@@ -2613,6 +2458,7 @@ static void __sched_fork(struct task_struct *p)
|
|
|
p->se.avg_overlap = 0;
|
|
|
p->se.start_runtime = 0;
|
|
|
p->se.avg_wakeup = sysctl_sched_wakeup_granularity;
|
|
|
+ p->se.avg_running = 0;
|
|
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
p->se.wait_start = 0;
|
|
@@ -2674,11 +2520,6 @@ void sched_fork(struct task_struct *p, int clone_flags)
|
|
|
|
|
|
__sched_fork(p);
|
|
|
|
|
|
-#ifdef CONFIG_SMP
|
|
|
- cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
|
|
|
-#endif
|
|
|
- set_task_cpu(p, cpu);
|
|
|
-
|
|
|
/*
|
|
|
* Make sure we do not leak PI boosting priority to the child.
|
|
|
*/
|
|
@@ -2709,6 +2550,11 @@ void sched_fork(struct task_struct *p, int clone_flags)
|
|
|
if (!rt_prio(p->prio))
|
|
|
p->sched_class = &fair_sched_class;
|
|
|
|
|
|
+#ifdef CONFIG_SMP
|
|
|
+ cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0);
|
|
|
+#endif
|
|
|
+ set_task_cpu(p, cpu);
|
|
|
+
|
|
|
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
|
|
|
if (likely(sched_info_on()))
|
|
|
memset(&p->sched_info, 0, sizeof(p->sched_info));
|
|
@@ -2754,7 +2600,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
|
|
|
inc_nr_running(rq);
|
|
|
}
|
|
|
trace_sched_wakeup_new(rq, p, 1);
|
|
|
- check_preempt_curr(rq, p, 0);
|
|
|
+ check_preempt_curr(rq, p, WF_FORK);
|
|
|
#ifdef CONFIG_SMP
|
|
|
if (p->sched_class->task_wake_up)
|
|
|
p->sched_class->task_wake_up(rq, p);
|
|
@@ -3263,7 +3109,7 @@ out:
|
|
|
void sched_exec(void)
|
|
|
{
|
|
|
int new_cpu, this_cpu = get_cpu();
|
|
|
- new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
|
|
|
+ new_cpu = current->sched_class->select_task_rq(current, SD_BALANCE_EXEC, 0);
|
|
|
put_cpu();
|
|
|
if (new_cpu != this_cpu)
|
|
|
sched_migrate_task(current, new_cpu);
|
|
@@ -3683,11 +3529,6 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
|
|
|
*imbalance = sds->min_load_per_task;
|
|
|
sds->busiest = sds->group_min;
|
|
|
|
|
|
- if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
|
|
|
- cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
|
|
|
- group_first_cpu(sds->group_leader);
|
|
|
- }
|
|
|
-
|
|
|
return 1;
|
|
|
|
|
|
}
|
|
@@ -3711,7 +3552,18 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
|
|
|
}
|
|
|
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
|
|
|
|
|
|
-unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
+
|
|
|
+unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
|
|
|
+{
|
|
|
+ return SCHED_LOAD_SCALE;
|
|
|
+}
|
|
|
+
|
|
|
+unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
|
|
|
+{
|
|
|
+ return default_scale_freq_power(sd, cpu);
|
|
|
+}
|
|
|
+
|
|
|
+unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
{
|
|
|
unsigned long weight = cpumask_weight(sched_domain_span(sd));
|
|
|
unsigned long smt_gain = sd->smt_gain;
|
|
@@ -3721,6 +3573,11 @@ unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
return smt_gain;
|
|
|
}
|
|
|
|
|
|
+unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
|
|
|
+{
|
|
|
+ return default_scale_smt_power(sd, cpu);
|
|
|
+}
|
|
|
+
|
|
|
unsigned long scale_rt_power(int cpu)
|
|
|
{
|
|
|
struct rq *rq = cpu_rq(cpu);
|
|
@@ -3745,10 +3602,19 @@ static void update_cpu_power(struct sched_domain *sd, int cpu)
|
|
|
unsigned long power = SCHED_LOAD_SCALE;
|
|
|
struct sched_group *sdg = sd->groups;
|
|
|
|
|
|
- /* here we could scale based on cpufreq */
|
|
|
+ if (sched_feat(ARCH_POWER))
|
|
|
+ power *= arch_scale_freq_power(sd, cpu);
|
|
|
+ else
|
|
|
+ power *= default_scale_freq_power(sd, cpu);
|
|
|
+
|
|
|
+ power >>= SCHED_LOAD_SHIFT;
|
|
|
|
|
|
if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
|
|
|
- power *= arch_scale_smt_power(sd, cpu);
|
|
|
+ if (sched_feat(ARCH_POWER))
|
|
|
+ power *= arch_scale_smt_power(sd, cpu);
|
|
|
+ else
|
|
|
+ power *= default_scale_smt_power(sd, cpu);
|
|
|
+
|
|
|
power >>= SCHED_LOAD_SHIFT;
|
|
|
}
|
|
|
|
|
@@ -4161,26 +4027,6 @@ ret:
|
|
|
return NULL;
|
|
|
}
|
|
|
|
|
|
-static struct sched_group *group_of(int cpu)
|
|
|
-{
|
|
|
- struct sched_domain *sd = rcu_dereference(cpu_rq(cpu)->sd);
|
|
|
-
|
|
|
- if (!sd)
|
|
|
- return NULL;
|
|
|
-
|
|
|
- return sd->groups;
|
|
|
-}
|
|
|
-
|
|
|
-static unsigned long power_of(int cpu)
|
|
|
-{
|
|
|
- struct sched_group *group = group_of(cpu);
|
|
|
-
|
|
|
- if (!group)
|
|
|
- return SCHED_LOAD_SCALE;
|
|
|
-
|
|
|
- return group->cpu_power;
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* find_busiest_queue - find the busiest runqueue among the cpus in group.
|
|
|
*/
|
|
@@ -5465,14 +5311,13 @@ static inline void schedule_debug(struct task_struct *prev)
|
|
|
#endif
|
|
|
}
|
|
|
|
|
|
-static void put_prev_task(struct rq *rq, struct task_struct *prev)
|
|
|
+static void put_prev_task(struct rq *rq, struct task_struct *p)
|
|
|
{
|
|
|
- if (prev->state == TASK_RUNNING) {
|
|
|
- u64 runtime = prev->se.sum_exec_runtime;
|
|
|
+ u64 runtime = p->se.sum_exec_runtime - p->se.prev_sum_exec_runtime;
|
|
|
|
|
|
- runtime -= prev->se.prev_sum_exec_runtime;
|
|
|
- runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
|
|
|
+ update_avg(&p->se.avg_running, runtime);
|
|
|
|
|
|
+ if (p->state == TASK_RUNNING) {
|
|
|
/*
|
|
|
* In order to avoid avg_overlap growing stale when we are
|
|
|
* indeed overlapping and hence not getting put to sleep, grow
|
|
@@ -5482,9 +5327,12 @@ static void put_prev_task(struct rq *rq, struct task_struct *prev)
|
|
|
* correlates to the amount of cache footprint a task can
|
|
|
* build up.
|
|
|
*/
|
|
|
- update_avg(&prev->se.avg_overlap, runtime);
|
|
|
+ runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
|
|
|
+ update_avg(&p->se.avg_overlap, runtime);
|
|
|
+ } else {
|
|
|
+ update_avg(&p->se.avg_running, 0);
|
|
|
}
|
|
|
- prev->sched_class->put_prev_task(rq, prev);
|
|
|
+ p->sched_class->put_prev_task(rq, p);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -5716,10 +5564,10 @@ asmlinkage void __sched preempt_schedule_irq(void)
|
|
|
|
|
|
#endif /* CONFIG_PREEMPT */
|
|
|
|
|
|
-int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
|
|
|
+int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
|
|
|
void *key)
|
|
|
{
|
|
|
- return try_to_wake_up(curr->private, mode, sync);
|
|
|
+ return try_to_wake_up(curr->private, mode, wake_flags);
|
|
|
}
|
|
|
EXPORT_SYMBOL(default_wake_function);
|
|
|
|
|
@@ -5733,14 +5581,14 @@ EXPORT_SYMBOL(default_wake_function);
|
|
|
* zero in this (rare) case, and we handle it by continuing to scan the queue.
|
|
|
*/
|
|
|
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
|
|
|
- int nr_exclusive, int sync, void *key)
|
|
|
+ int nr_exclusive, int wake_flags, void *key)
|
|
|
{
|
|
|
wait_queue_t *curr, *next;
|
|
|
|
|
|
list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
|
|
|
unsigned flags = curr->flags;
|
|
|
|
|
|
- if (curr->func(curr, mode, sync, key) &&
|
|
|
+ if (curr->func(curr, mode, wake_flags, key) &&
|
|
|
(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
|
|
|
break;
|
|
|
}
|
|
@@ -5801,16 +5649,16 @@ void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
|
|
|
int nr_exclusive, void *key)
|
|
|
{
|
|
|
unsigned long flags;
|
|
|
- int sync = 1;
|
|
|
+ int wake_flags = WF_SYNC;
|
|
|
|
|
|
if (unlikely(!q))
|
|
|
return;
|
|
|
|
|
|
if (unlikely(!nr_exclusive))
|
|
|
- sync = 0;
|
|
|
+ wake_flags = 0;
|
|
|
|
|
|
spin_lock_irqsave(&q->lock, flags);
|
|
|
- __wake_up_common(q, mode, nr_exclusive, sync, key);
|
|
|
+ __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
|
|
|
spin_unlock_irqrestore(&q->lock, flags);
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(__wake_up_sync_key);
|
|
@@ -8000,9 +7848,7 @@ static int sd_degenerate(struct sched_domain *sd)
|
|
|
}
|
|
|
|
|
|
/* Following flags don't use groups */
|
|
|
- if (sd->flags & (SD_WAKE_IDLE |
|
|
|
- SD_WAKE_AFFINE |
|
|
|
- SD_WAKE_BALANCE))
|
|
|
+ if (sd->flags & (SD_WAKE_AFFINE))
|
|
|
return 0;
|
|
|
|
|
|
return 1;
|
|
@@ -8019,10 +7865,6 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
|
|
|
if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
|
|
|
return 0;
|
|
|
|
|
|
- /* Does parent contain flags not in child? */
|
|
|
- /* WAKE_BALANCE is a subset of WAKE_AFFINE */
|
|
|
- if (cflags & SD_WAKE_AFFINE)
|
|
|
- pflags &= ~SD_WAKE_BALANCE;
|
|
|
/* Flags needing groups don't count if only 1 group in parent */
|
|
|
if (parent->groups == parent->groups->next) {
|
|
|
pflags &= ~(SD_LOAD_BALANCE |
|
|
@@ -8708,10 +8550,10 @@ static void set_domain_attribute(struct sched_domain *sd,
|
|
|
request = attr->relax_domain_level;
|
|
|
if (request < sd->level) {
|
|
|
/* turn off idle balance on this domain */
|
|
|
- sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
|
|
|
+ sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
|
|
|
} else {
|
|
|
/* turn on idle balance on this domain */
|
|
|
- sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
|
|
|
+ sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
|
|
|
}
|
|
|
}
|
|
|
|