From b7898fda5bc7e786e76ce24fbd2ec993b08ec518 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 30 Mar 2016 03:47:49 +0200 Subject: cpufreq: Support for fast frequency switching Modify the ACPI cpufreq driver to provide a method for switching CPU frequencies from interrupt context and update the cpufreq core to support that method if available. Introduce a new cpufreq driver callback, ->fast_switch, to be invoked for frequency switching from interrupt context by (future) governors supporting that feature via (new) helper function cpufreq_driver_fast_switch(). Add two new policy flags, fast_switch_possible, to be set by the cpufreq driver if fast frequency switching can be used for the given policy and fast_switch_enabled, to be set by the governor if it is going to use fast frequency switching for the given policy. Also add a helper for setting the latter. Since fast frequency switching is inherently incompatible with cpufreq transition notifiers, make it possible to set the fast_switch_enabled only if there are no transition notifiers already registered and make the registration of new transition notifiers fail if fast_switch_enabled is set for at least one policy. Implement the ->fast_switch callback in the ACPI cpufreq driver and make it set fast_switch_possible during policy initialization as appropriate. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 130 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 5 deletions(-) (limited to 'drivers/cpufreq/cpufreq.c') diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index b87596b591b3..a5b7d77d4816 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -77,6 +77,7 @@ static inline bool has_target(void) static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event); static unsigned int __cpufreq_get(struct cpufreq_policy *policy); static int cpufreq_start_governor(struct cpufreq_policy *policy); +static int cpufreq_exit_governor(struct cpufreq_policy *policy); /** * Two notifier lists: the "policy" list is involved in the @@ -429,6 +430,68 @@ void cpufreq_freq_transition_end(struct cpufreq_policy *policy, } EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); +/* + * Fast frequency switching status count. Positive means "enabled", negative + * means "disabled" and 0 means "not decided yet". + */ +static int cpufreq_fast_switch_count; +static DEFINE_MUTEX(cpufreq_fast_switch_lock); + +static void cpufreq_list_transition_notifiers(void) +{ + struct notifier_block *nb; + + pr_info("Registered transition notifiers:\n"); + + mutex_lock(&cpufreq_transition_notifier_list.mutex); + + for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) + pr_info("%pF\n", nb->notifier_call); + + mutex_unlock(&cpufreq_transition_notifier_list.mutex); +} + +/** + * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. + * @policy: cpufreq policy to enable fast frequency switching for. + * + * Try to enable fast frequency switching for @policy. + * + * The attempt will fail if there is at least one transition notifier registered + * at this point, as fast frequency switching is quite fundamentally at odds + * with transition notifiers. Thus if successful, it will make registration of + * transition notifiers fail going forward. + */ +void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) +{ + lockdep_assert_held(&policy->rwsem); + + if (!policy->fast_switch_possible) + return; + + mutex_lock(&cpufreq_fast_switch_lock); + if (cpufreq_fast_switch_count >= 0) { + cpufreq_fast_switch_count++; + policy->fast_switch_enabled = true; + } else { + pr_warn("CPU%u: Fast frequency switching not enabled\n", + policy->cpu); + cpufreq_list_transition_notifiers(); + } + mutex_unlock(&cpufreq_fast_switch_lock); +} +EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); + +static void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) +{ + mutex_lock(&cpufreq_fast_switch_lock); + if (policy->fast_switch_enabled) { + policy->fast_switch_enabled = false; + if (!WARN_ON(cpufreq_fast_switch_count <= 0)) + cpufreq_fast_switch_count--; + } + mutex_unlock(&cpufreq_fast_switch_lock); +} /********************************************************************* * SYSFS INTERFACE * @@ -1319,7 +1382,7 @@ static void cpufreq_offline(unsigned int cpu) /* If cpu is last user of policy, free policy */ if (has_target()) { - ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); + ret = cpufreq_exit_governor(policy); if (ret) pr_err("%s: Failed to exit governor\n", __func__); } @@ -1447,8 +1510,12 @@ static unsigned int __cpufreq_get(struct cpufreq_policy *policy) ret_freq = cpufreq_driver->get(policy->cpu); - /* Updating inactive policies is invalid, so avoid doing that. */ - if (unlikely(policy_is_inactive(policy))) + /* + * Updating inactive policies is invalid, so avoid doing that. Also + * if fast frequency switching is used with the given policy, the check + * against policy->cur is pointless, so skip it in that case too. + */ + if (unlikely(policy_is_inactive(policy)) || policy->fast_switch_enabled) return ret_freq; if (ret_freq && policy->cur && @@ -1672,8 +1739,18 @@ int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) switch (list) { case CPUFREQ_TRANSITION_NOTIFIER: + mutex_lock(&cpufreq_fast_switch_lock); + + if (cpufreq_fast_switch_count > 0) { + mutex_unlock(&cpufreq_fast_switch_lock); + return -EBUSY; + } ret = srcu_notifier_chain_register( &cpufreq_transition_notifier_list, nb); + if (!ret) + cpufreq_fast_switch_count--; + + mutex_unlock(&cpufreq_fast_switch_lock); break; case CPUFREQ_POLICY_NOTIFIER: ret = blocking_notifier_chain_register( @@ -1706,8 +1783,14 @@ int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) switch (list) { case CPUFREQ_TRANSITION_NOTIFIER: + mutex_lock(&cpufreq_fast_switch_lock); + ret = srcu_notifier_chain_unregister( &cpufreq_transition_notifier_list, nb); + if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) + cpufreq_fast_switch_count++; + + mutex_unlock(&cpufreq_fast_switch_lock); break; case CPUFREQ_POLICY_NOTIFIER: ret = blocking_notifier_chain_unregister( @@ -1726,6 +1809,37 @@ EXPORT_SYMBOL(cpufreq_unregister_notifier); * GOVERNORS * *********************************************************************/ +/** + * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. + * @policy: cpufreq policy to switch the frequency for. + * @target_freq: New frequency to set (may be approximate). + * + * Carry out a fast frequency switch without sleeping. + * + * The driver's ->fast_switch() callback invoked by this function must be + * suitable for being called from within RCU-sched read-side critical sections + * and it is expected to select the minimum available frequency greater than or + * equal to @target_freq (CPUFREQ_RELATION_L). + * + * This function must not be called if policy->fast_switch_enabled is unset. + * + * Governors calling this function must guarantee that it will never be invoked + * twice in parallel for the same policy and that it will never be called in + * parallel with either ->target() or ->target_index() for the same policy. + * + * If CPUFREQ_ENTRY_INVALID is returned by the driver's ->fast_switch() + * callback to indicate an error condition, the hardware configuration must be + * preserved. + */ +unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + clamp_val(target_freq, policy->min, policy->max); + + return cpufreq_driver->fast_switch(policy, target_freq); +} +EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); + /* Must set freqs->new to intermediate frequency */ static int __target_intermediate(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs, int index) @@ -1946,6 +2060,12 @@ static int cpufreq_start_governor(struct cpufreq_policy *policy) return ret ? ret : cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); } +static int cpufreq_exit_governor(struct cpufreq_policy *policy) +{ + cpufreq_disable_fast_switch(policy); + return cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); +} + int cpufreq_register_governor(struct cpufreq_governor *governor) { int err; @@ -2101,7 +2221,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, return ret; } - ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); + ret = cpufreq_exit_governor(policy); if (ret) { pr_err("%s: Failed to Exit Governor: %s (%d)\n", __func__, old_gov->name, ret); @@ -2118,7 +2238,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, pr_debug("cpufreq: governor change\n"); return 0; } - cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); + cpufreq_exit_governor(policy); } /* new governor failed, so re-start old one */ -- cgit v1.2.3 From 6c9d9c81924b4b63c7a487e90fddb3b2d0f7d458 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 7 Apr 2016 23:38:46 +0200 Subject: cpufreq: Call cpufreq_disable_fast_switch() in sugov_exit() Due to differences in the cpufreq core's handling of runtime CPU offline and nonboot CPUs disabling during system suspend-to-RAM, fast frequency switching gets disabled after a suspend-to-RAM and resume cycle on all of the nonboot CPUs. To prevent that from happening, move the invocation of cpufreq_disable_fast_switch() from cpufreq_exit_governor() to sugov_exit(), as the schedutil governor is the only user of fast frequency switching today anyway. That simply prevents cpufreq_disable_fast_switch() from being called without invoking the ->governor callback for the CPUFREQ_GOV_POLICY_EXIT event (which happens during system suspend now). Fixes: b7898fda5bc7 (cpufreq: Support for fast frequency switching) Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 19 +++++++++++-------- include/linux/cpufreq.h | 1 + kernel/sched/cpufreq_schedutil.c | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) (limited to 'drivers/cpufreq/cpufreq.c') diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index a5b7d77d4816..77d77a4e3b74 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -77,7 +77,11 @@ static inline bool has_target(void) static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event); static unsigned int __cpufreq_get(struct cpufreq_policy *policy); static int cpufreq_start_governor(struct cpufreq_policy *policy); -static int cpufreq_exit_governor(struct cpufreq_policy *policy); + +static inline int cpufreq_exit_governor(struct cpufreq_policy *policy) +{ + return cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); +} /** * Two notifier lists: the "policy" list is involved in the @@ -482,7 +486,11 @@ void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); -static void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) +/** + * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. + * @policy: cpufreq policy to disable fast frequency switching for. + */ +void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) { mutex_lock(&cpufreq_fast_switch_lock); if (policy->fast_switch_enabled) { @@ -492,6 +500,7 @@ static void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) } mutex_unlock(&cpufreq_fast_switch_lock); } +EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); /********************************************************************* * SYSFS INTERFACE * @@ -2060,12 +2069,6 @@ static int cpufreq_start_governor(struct cpufreq_policy *policy) return ret ? ret : cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); } -static int cpufreq_exit_governor(struct cpufreq_policy *policy) -{ - cpufreq_disable_fast_switch(policy); - return cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); -} - int cpufreq_register_governor(struct cpufreq_governor *governor) { int err; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 55e69ebb035c..4e81e08db752 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -168,6 +168,7 @@ int cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); void cpufreq_enable_fast_switch(struct cpufreq_policy *policy); +void cpufreq_disable_fast_switch(struct cpufreq_policy *policy); #else static inline unsigned int cpufreq_get(unsigned int cpu) { diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index d27ae064b476..154ae3a51e86 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -398,6 +398,8 @@ static int sugov_exit(struct cpufreq_policy *policy) struct sugov_tunables *tunables = sg_policy->tunables; unsigned int count; + cpufreq_disable_fast_switch(policy); + mutex_lock(&global_tunables_lock); count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); -- cgit v1.2.3 From cd73e9b01f635d25dbd17ed090f9351becf00280 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 7 Apr 2016 03:30:40 +0200 Subject: cpufreq: Simplify switch () in cpufreq_cpu_callback() Merge two switch entries that do the same thing in cpufreq_cpu_callback(). No functional changes. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/cpufreq/cpufreq.c') diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 77d77a4e3b74..859281592680 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2309,16 +2309,13 @@ static int cpufreq_cpu_callback(struct notifier_block *nfb, switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: + case CPU_DOWN_FAILED: cpufreq_online(cpu); break; case CPU_DOWN_PREPARE: cpufreq_offline(cpu); break; - - case CPU_DOWN_FAILED: - cpufreq_online(cpu); - break; } return NOTIFY_OK; } -- cgit v1.2.3 From a794d6138cda391b6a9ba28c4cf27651415669c6 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 7 Apr 2016 03:31:57 +0200 Subject: cpufreq: Rearrange cpufreq_add_dev() Reorganize the code in cpufreq_add_dev() to avoid using the ret variable and reduce the indentation level in it. No functional changes. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) (limited to 'drivers/cpufreq/cpufreq.c') diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 859281592680..2f1ae568f74b 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1320,26 +1320,24 @@ out_free_policy: */ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) { + struct cpufreq_policy *policy; unsigned cpu = dev->id; - int ret; dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); - if (cpu_online(cpu)) { - ret = cpufreq_online(cpu); - } else { - /* - * A hotplug notifier will follow and we will handle it as CPU - * online then. For now, just create the sysfs link, unless - * there is no policy or the link is already present. - */ - struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); + if (cpu_online(cpu)) + return cpufreq_online(cpu); - ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus) - ? add_cpu_dev_symlink(policy, cpu) : 0; - } + /* + * A hotplug notifier will follow and we will handle it as CPU online + * then. For now, just create the sysfs link, unless there is no policy + * or the link is already present. + */ + policy = per_cpu(cpufreq_cpu_data, cpu); + if (!policy || cpumask_test_and_set_cpu(cpu, policy->real_cpus)) + return 0; - return ret; + return add_cpu_dev_symlink(policy, cpu); } static void cpufreq_offline(unsigned int cpu) -- cgit v1.2.3