sched: cpufreq: Allow remote cpufreq callbacks

With Android UI and benchmarks the latency of cpufreq response to
certain scheduling events can become very critical. Currently, callbacks
into cpufreq governors are only made from the scheduler if the target
CPU of the event is the same as the current CPU. This means there are
certain situations where a target CPU may not run the cpufreq governor
for some time.

One testcase to show this behavior is where a task starts running on
CPU0, then a new task is also spawned on CPU0 by a task on CPU1. If the
system is configured such that the new tasks should receive maximum
demand initially, this should result in CPU0 increasing frequency
immediately. But because of the above mentioned limitation though, this
does not occur.

This patch updates the scheduler core to call the cpufreq callbacks for
remote CPUs as well.

The schedutil, ondemand and conservative governors are updated to
process cpufreq utilization update hooks called for remote CPUs where
the remote CPU is managed by the cpufreq policy of the local CPU.

The intel_pstate driver is updated to always reject remote callbacks.

This is tested with couple of usecases (Android: hackbench, recentfling,
galleryfling, vellamo, Ubuntu: hackbench) on ARM hikey board (64 bit
octa-core, single policy). Only galleryfling showed minor improvements,
while others didn't had much deviation.

The reason being that this patch only targets a corner case, where
following are required to be true to improve performance and that
doesn't happen too often with these tests:

- Task is migrated to another CPU.
- The task has high demand, and should take the target CPU to higher
  OPPs.
- And the target CPU doesn't call into the cpufreq governor until the
  next tick.

Based on initial work from Steve Muckle.

Signed-off-by: Viresh Kumar <[email protected]>
Acked-by: Saravana Kannan <[email protected]>
Acked-by: Peter Zijlstra (Intel) <[email protected]>
Signed-off-by: Rafael J. Wysocki <[email protected]>

diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 47e24b5384b37978cde5ce1f0e2eeb440e6ea04b..ce5f3ec7ce710f56a806c001ade6386550ac14a4 100644 (file)
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -275,6 +275,9 @@ static void dbs_update_util_handler(struct update_util_data *data, u64 time,
        struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
        u64 delta_ns, lst;
 
+       if (!cpufreq_can_do_remote_dvfs(policy_dbs->policy))
+               return;
+
        /*
         * The work may not be allowed to be queued up right now.
         * Possible reasons:

diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 6cd50352563894bea24866d2d36d6b815d0abc20..d299b86a5a007e7548f689405c0c4a4be35567b2 100644 (file)
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1747,6 +1747,10 @@ static void intel_pstate_update_util_pid(struct update_util_data *data,
        struct cpudata *cpu = container_of(data, struct cpudata, update_util);
        u64 delta_ns = time - cpu->sample.time;
 
+       /* Don't allow remote callbacks */
+       if (smp_processor_id() != cpu->cpu)
+               return;
+
        if ((s64)delta_ns < pid_params.sample_rate_ns)
                return;
 
@@ -1764,6 +1768,10 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time,
        struct cpudata *cpu = container_of(data, struct cpudata, update_util);
        u64 delta_ns;
 
+       /* Don't allow remote callbacks */
+       if (smp_processor_id() != cpu->cpu)
+               return;
+
        if (flags & SCHED_CPUFREQ_IOWAIT) {
                cpu->iowait_boost = int_tofp(1);
        } else if (cpu->iowait_boost) {

diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index f10a9b3761cdbef777c38a131cc1cb1e4aec0c1e..c4035964e6b30703d73251c42be6d6a5e2c5b095 100644 (file)
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -562,6 +562,15 @@ struct governor_attr {
                         size_t count);
 };
 
+static inline bool cpufreq_can_do_remote_dvfs(struct cpufreq_policy *policy)
+{
+       /* Allow remote callbacks only on the CPUs sharing cpufreq policy */
+       if (cpumask_test_cpu(smp_processor_id(), policy->cpus))
+               return true;
+
+       return false;
+}
+
 /*********************************************************************
  *                     FREQUENCY TABLE HELPERS                       *
  *********************************************************************/

diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index ddd385f2a985216933f1c07e1e22d5c6ee02ea9c..7dbc76801f8627addf94dba3c250d21fe377f2d9 100644 (file)
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -52,6 +52,7 @@ struct sugov_policy {
 struct sugov_cpu {
        struct update_util_data update_util;
        struct sugov_policy *sg_policy;
+       unsigned int cpu;
 
        bool iowait_boost_pending;
        unsigned int iowait_boost;
@@ -77,6 +78,21 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
 {
        s64 delta_ns;
 
+       /*
+        * Since cpufreq_update_util() is called with rq->lock held for
+        * the @target_cpu, our per-cpu data is fully serialized.
+        *
+        * However, drivers cannot in general deal with cross-cpu
+        * requests, so while get_next_freq() will work, our
+        * sugov_update_commit() call may not.
+        *
+        * Hence stop here for remote requests if they aren't supported
+        * by the hardware, as calculating the frequency is pointless if
+        * we cannot in fact act on it.
+        */
+       if (!cpufreq_can_do_remote_dvfs(sg_policy->policy))
+               return false;
+
        if (sg_policy->work_in_progress)
                return false;
 
@@ -155,12 +171,12 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy,
        return cpufreq_driver_resolve_freq(policy, freq);
 }
 
-static void sugov_get_util(unsigned long *util, unsigned long *max)
+static void sugov_get_util(unsigned long *util, unsigned long *max, int cpu)
 {
-       struct rq *rq = this_rq();
+       struct rq *rq = cpu_rq(cpu);
        unsigned long cfs_max;
 
-       cfs_max = arch_scale_cpu_capacity(NULL, smp_processor_id());
+       cfs_max = arch_scale_cpu_capacity(NULL, cpu);
 
        *util = min(rq->cfs.avg.util_avg, cfs_max);
        *max = cfs_max;
@@ -254,7 +270,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
        if (flags & SCHED_CPUFREQ_RT_DL) {
                next_f = policy->cpuinfo.max_freq;
        } else {
-               sugov_get_util(&util, &max);
+               sugov_get_util(&util, &max, sg_cpu->cpu);
                sugov_iowait_boost(sg_cpu, &util, &max);
                next_f = get_next_freq(sg_policy, util, max);
                /*
@@ -316,7 +332,7 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
        unsigned long util, max;
        unsigned int next_f;
 
-       sugov_get_util(&util, &max);
+       sugov_get_util(&util, &max, sg_cpu->cpu);
 
        raw_spin_lock(&sg_policy->update_lock);
 
@@ -697,6 +713,11 @@ struct cpufreq_governor *cpufreq_default_governor(void)
 
 static int __init sugov_register(void)
 {
+       int cpu;
+
+       for_each_possible_cpu(cpu)
+               per_cpu(sugov_cpu, cpu).cpu = cpu;
+
        return cpufreq_register_governor(&schedutil_gov);
 }
 fs_initcall(sugov_register);

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 755bd3f1a1a93a8f1daf713dd36578637ce515c5..5c3bf4bd03275535918643a6c6b42266f07e58db 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1136,7 +1136,7 @@ static void update_curr_dl(struct rq *rq)
        }
 
        /* kick cpufreq (see the comment in kernel/sched/sched.h). */
-       cpufreq_update_this_cpu(rq, SCHED_CPUFREQ_DL);
+       cpufreq_update_util(rq, SCHED_CPUFREQ_DL);
 
        schedstat_set(curr->se.statistics.exec_max,
                      max(curr->se.statistics.exec_max, delta_exec));

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c95880e216f6900ed06e92e93ddc355d20e59092..d378d02fdfcb1eb14dd5036ee61b1bc37e300727 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3278,7 +3278,9 @@ static inline void set_tg_cfs_propagate(struct cfs_rq *cfs_rq) {}
 
 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq)
 {
-       if (&this_rq()->cfs == cfs_rq) {
+       struct rq *rq = rq_of(cfs_rq);
+
+       if (&rq->cfs == cfs_rq) {
                /*
                 * There are a few boundary cases this might miss but it should
                 * get called often enough that that should (hopefully) not be
@@ -3295,7 +3297,7 @@ static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq)
                 *
                 * See cpu_util().
                 */
-               cpufreq_update_util(rq_of(cfs_rq), 0);
+               cpufreq_update_util(rq, 0);
        }
 }
 
@@ -4875,7 +4877,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
         * passed.
         */
        if (p->in_iowait)
-               cpufreq_update_this_cpu(rq, SCHED_CPUFREQ_IOWAIT);
+               cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
 
        for_each_sched_entity(se) {
                if (se->on_rq)

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 45caf937ef90ead71864d4fbb12e7281e4b8ff9b..0af5ca9e3e3f1d3613e69fabba32d909d0a15f3b 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -970,7 +970,7 @@ static void update_curr_rt(struct rq *rq)
                return;
 
        /* Kick cpufreq (see the comment in kernel/sched/sched.h). */
-       cpufreq_update_this_cpu(rq, SCHED_CPUFREQ_RT);
+       cpufreq_update_util(rq, SCHED_CPUFREQ_RT);
 
        schedstat_set(curr->se.statistics.exec_max,
                      max(curr->se.statistics.exec_max, delta_exec));

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index eeef1a3086d1e74af034782cd780bbf7e44333b2..aa9d5b87b4f8af31981d18e7b9c24f86b7dc7851 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2070,19 +2070,13 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
 {
        struct update_util_data *data;
 
-       data = rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data));
+       data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data,
+                                                 cpu_of(rq)));
        if (data)
                data->func(data, rq_clock(rq), flags);
 }
-
-static inline void cpufreq_update_this_cpu(struct rq *rq, unsigned int flags)
-{
-       if (cpu_of(rq) == smp_processor_id())
-               cpufreq_update_util(rq, flags);
-}
 #else
 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
-static inline void cpufreq_update_this_cpu(struct rq *rq, unsigned int flags) {}
 #endif /* CONFIG_CPU_FREQ */
 
 #ifdef arch_scale_freq_capacity