summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched/fair.c118
1 files changed, 60 insertions, 58 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 55b1f117419a..233d05171bf5 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2918,7 +2918,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
return;
/*
- * This is possible from callers such as pull_task(), in which we
+ * This is possible from callers such as move_task(), in which we
* unconditionally check_prempt_curr() after an enqueue (which may have
* lead to a throttle). This both saves work and prevents false
* next-buddy nomination below.
@@ -3084,17 +3084,37 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp
static unsigned long __read_mostly max_load_balance_interval = HZ/10;
+#define LBF_ALL_PINNED 0x01
+#define LBF_NEED_BREAK 0x02 /* clears into HAD_BREAK */
+#define LBF_HAD_BREAK 0x04
+#define LBF_HAD_BREAKS 0x0C /* count HAD_BREAKs overflows into ABORT */
+#define LBF_ABORT 0x10
+
+struct lb_env {
+ struct sched_domain *sd;
+
+ int src_cpu;
+ struct rq *src_rq;
+ struct cfs_rq *src_cfs_rq;
+
+ int dst_cpu;
+ struct rq *dst_rq;
+
+ enum cpu_idle_type idle;
+ unsigned long max_load_move;
+ unsigned int flags;
+};
+
/*
- * pull_task - move a task from a remote runqueue to the local runqueue.
+ * move_task - move a task from one runqueue to another runqueue.
* Both runqueues must be locked.
*/
-static void pull_task(struct rq *src_rq, struct task_struct *p,
- struct rq *this_rq, int this_cpu)
+static void move_task(struct task_struct *p, struct lb_env *env)
{
- deactivate_task(src_rq, p, 0);
- set_task_cpu(p, this_cpu);
- activate_task(this_rq, p, 0);
- check_preempt_curr(this_rq, p, 0);
+ deactivate_task(env->src_rq, p, 0);
+ set_task_cpu(p, env->dst_cpu);
+ activate_task(env->dst_rq, p, 0);
+ check_preempt_curr(env->dst_rq, p, 0);
}
/*
@@ -3129,26 +3149,6 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
return delta < (s64)sysctl_sched_migration_cost;
}
-#define LBF_ALL_PINNED 0x01
-#define LBF_NEED_BREAK 0x02 /* clears into HAD_BREAK */
-#define LBF_HAD_BREAK 0x04
-#define LBF_HAD_BREAKS 0x0C /* count HAD_BREAKs overflows into ABORT */
-#define LBF_ABORT 0x10
-
-struct lb_env {
- struct sched_domain *sd;
-
- int this_cpu;
- struct rq *this_rq;
-
- struct rq *busiest_rq;
- struct cfs_rq *busiest_cfs_rq;
-
- enum cpu_idle_type idle;
- unsigned long max_load_move;
- unsigned int flags;
-};
-
/*
* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
*/
@@ -3162,13 +3162,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
* 2) cannot be migrated to this CPU due to cpus_allowed, or
* 3) are cache-hot on their current CPU.
*/
- if (!cpumask_test_cpu(env->this_cpu, tsk_cpus_allowed(p))) {
+ if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) {
schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
return 0;
}
env->flags &= ~LBF_ALL_PINNED;
- if (task_running(env->busiest_rq, p)) {
+ if (task_running(env->src_rq, p)) {
schedstat_inc(p, se.statistics.nr_failed_migrations_running);
return 0;
}
@@ -3179,7 +3179,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
* 2) too many balance attempts have failed.
*/
- tsk_cache_hot = task_hot(p, env->busiest_rq->clock_task, env->sd);
+ tsk_cache_hot = task_hot(p, env->src_rq->clock_task, env->sd);
if (!tsk_cache_hot ||
env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
#ifdef CONFIG_SCHEDSTATS
@@ -3210,20 +3210,20 @@ static int move_one_task(struct lb_env *env)
struct task_struct *p, *n;
struct cfs_rq *cfs_rq;
- for_each_leaf_cfs_rq(env->busiest_rq, cfs_rq) {
+ for_each_leaf_cfs_rq(env->src_rq, cfs_rq) {
list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
if (throttled_lb_pair(task_group(p),
- env->busiest_rq->cpu, env->this_cpu))
+ env->src_cpu, env->dst_cpu))
break;
if (!can_migrate_task(p, env))
continue;
- pull_task(env->busiest_rq, p, env->this_rq, env->this_cpu);
+ move_task(p, env);
/*
- * Right now, this is only the second place pull_task()
- * is called, so we can safely collect pull_task()
- * stats here rather than inside pull_task().
+ * Right now, this is only the second place move_task()
+ * is called, so we can safely collect move_task()
+ * stats here rather than inside move_task().
*/
schedstat_inc(env->sd, lb_gained[env->idle]);
return 1;
@@ -3242,7 +3242,7 @@ static unsigned long balance_tasks(struct lb_env *env)
if (env->max_load_move == 0)
goto out;
- list_for_each_entry_safe(p, n, &env->busiest_cfs_rq->tasks, se.group_node) {
+ list_for_each_entry_safe(p, n, &env->src_cfs_rq->tasks, se.group_node) {
if (loops++ > sysctl_sched_nr_migrate) {
env->flags |= LBF_NEED_BREAK;
break;
@@ -3252,7 +3252,7 @@ static unsigned long balance_tasks(struct lb_env *env)
!can_migrate_task(p, env))
continue;
- pull_task(env->busiest_rq, p, env->this_rq, env->this_cpu);
+ move_task(p, env);
pulled++;
rem_load_move -= p->se.load.weight;
@@ -3277,9 +3277,9 @@ static unsigned long balance_tasks(struct lb_env *env)
}
out:
/*
- * Right now, this is one of only two places pull_task() is called,
- * so we can safely collect pull_task() stats here rather than
- * inside pull_task().
+ * Right now, this is one of only two places move_task() is called,
+ * so we can safely collect move_task() stats here rather than
+ * inside move_task().
*/
schedstat_add(env->sd, lb_gained[env->idle], pulled);
@@ -3372,11 +3372,11 @@ static unsigned long load_balance_fair(struct lb_env *env)
long rem_load_move = env->max_load_move;
rcu_read_lock();
- update_h_load(cpu_of(env->busiest_rq));
+ update_h_load(cpu_of(env->src_rq));
- for_each_leaf_cfs_rq(env->busiest_rq, env->busiest_cfs_rq) {
- unsigned long busiest_h_load = env->busiest_cfs_rq->h_load;
- unsigned long busiest_weight = env->busiest_cfs_rq->load.weight;
+ for_each_leaf_cfs_rq(env->src_rq, env->src_cfs_rq) {
+ unsigned long busiest_h_load = env->src_cfs_rq->h_load;
+ unsigned long busiest_weight = env->src_cfs_rq->load.weight;
u64 rem_load, moved_load;
if (env->flags & (LBF_NEED_BREAK|LBF_ABORT))
@@ -3385,12 +3385,12 @@ static unsigned long load_balance_fair(struct lb_env *env)
/*
* empty group or part of a throttled hierarchy
*/
- if (!env->busiest_cfs_rq->task_weight)
+ if (!env->src_cfs_rq->task_weight)
continue;
- if (throttled_lb_pair(env->busiest_cfs_rq->tg,
- cpu_of(env->busiest_rq),
- env->this_cpu))
+ if (throttled_lb_pair(env->src_cfs_rq->tg,
+ cpu_of(env->src_rq),
+ env->dst_cpu))
continue;
rem_load = (u64)rem_load_move * busiest_weight;
@@ -3420,7 +3420,7 @@ static inline void update_shares(int cpu)
static unsigned long load_balance_fair(struct lb_env *env)
{
- env->busiest_cfs_rq = &env->busiest_rq->cfs;
+ env->src_cfs_rq = &env->src_rq->cfs;
return balance_tasks(env);
}
#endif
@@ -3451,7 +3451,7 @@ static int move_tasks(struct lb_env *env)
* kernels will stop after the first task is pulled to minimize
* the critical section.
*/
- if (env->idle == CPU_NEWLY_IDLE && env->this_rq->nr_running) {
+ if (env->idle == CPU_NEWLY_IDLE && env->dst_rq->nr_running) {
env->flags |= LBF_ABORT;
break;
}
@@ -4461,8 +4461,8 @@ static int load_balance(int this_cpu, struct rq *this_rq,
struct lb_env env = {
.sd = sd,
- .this_cpu = this_cpu,
- .this_rq = this_rq,
+ .dst_cpu = this_cpu,
+ .dst_rq = this_rq,
.idle = idle,
};
@@ -4502,7 +4502,8 @@ redo:
*/
env.flags |= LBF_ALL_PINNED;
env.max_load_move = imbalance;
- env.busiest_rq = busiest;
+ env.src_cpu = busiest->cpu;
+ env.src_rq = busiest;
local_irq_save(flags);
double_rq_lock(this_rq, busiest);
@@ -4722,9 +4723,10 @@ static int active_load_balance_cpu_stop(void *data)
if (likely(sd)) {
struct lb_env env = {
.sd = sd,
- .this_cpu = target_cpu,
- .this_rq = target_rq,
- .busiest_rq = busiest_rq,
+ .dst_cpu = target_cpu,
+ .dst_rq = target_rq,
+ .src_cpu = busiest_rq->cpu,
+ .src_rq = busiest_rq,
.idle = CPU_IDLE,
};
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-23 21:45:07 +0100