cpufreq_ondemand.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. /*
  2. * drivers/cpufreq/cpufreq_ondemand.c
  3. *
  4. * Copyright (C) 2001 Russell King
  5. * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
  6. * Jun Nakajima <jun.nakajima@intel.com>
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License version 2 as
  10. * published by the Free Software Foundation.
  11. */
  12. #include <linux/kernel.h>
  13. #include <linux/module.h>
  14. #include <linux/smp.h>
  15. #include <linux/init.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/ctype.h>
  18. #include <linux/cpufreq.h>
  19. #include <linux/sysctl.h>
  20. #include <linux/types.h>
  21. #include <linux/fs.h>
  22. #include <linux/sysfs.h>
  23. #include <linux/cpu.h>
  24. #include <linux/sched.h>
  25. #include <linux/kmod.h>
  26. #include <linux/workqueue.h>
  27. #include <linux/jiffies.h>
  28. #include <linux/kernel_stat.h>
  29. #include <linux/percpu.h>
  30. #include <linux/mutex.h>
  31. /*
  32. * dbs is used in this file as a shortform for demandbased switching
  33. * It helps to keep variable names smaller, simpler
  34. */
  35. #define DEF_FREQUENCY_UP_THRESHOLD (80)
  36. #define MIN_FREQUENCY_UP_THRESHOLD (11)
  37. #define MAX_FREQUENCY_UP_THRESHOLD (100)
  38. /*
  39. * The polling frequency of this governor depends on the capability of
  40. * the processor. Default polling frequency is 1000 times the transition
  41. * latency of the processor. The governor will work on any processor with
  42. * transition latency <= 10mS, using appropriate sampling
  43. * rate.
  44. * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
  45. * this governor will not work.
  46. * All times here are in uS.
  47. */
  48. static unsigned int def_sampling_rate;
  49. #define MIN_SAMPLING_RATE_RATIO (2)
  50. /* for correct statistics, we need at least 10 ticks between each measure */
  51. #define MIN_STAT_SAMPLING_RATE (MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
  52. #define MIN_SAMPLING_RATE (def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
  53. #define MAX_SAMPLING_RATE (500 * def_sampling_rate)
  54. #define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER (1000)
  55. #define TRANSITION_LATENCY_LIMIT (10 * 1000)
  56. static void do_dbs_timer(void *data);
  57. struct cpu_dbs_info_s {
  58. cputime64_t prev_cpu_idle;
  59. cputime64_t prev_cpu_wall;
  60. struct cpufreq_policy *cur_policy;
  61. struct work_struct work;
  62. unsigned int enable;
  63. };
  64. static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
  65. static unsigned int dbs_enable; /* number of CPUs using this policy */
  66. /*
  67. * DEADLOCK ALERT! There is a ordering requirement between cpu_hotplug
  68. * lock and dbs_mutex. cpu_hotplug lock should always be held before
  69. * dbs_mutex. If any function that can potentially take cpu_hotplug lock
  70. * (like __cpufreq_driver_target()) is being called with dbs_mutex taken, then
  71. * cpu_hotplug lock should be taken before that. Note that cpu_hotplug lock
  72. * is recursive for the same process. -Venki
  73. */
  74. static DEFINE_MUTEX (dbs_mutex);
  75. static DECLARE_WORK (dbs_work, do_dbs_timer, NULL);
  76. static struct workqueue_struct *kondemand_wq;
  77. struct dbs_tuners {
  78. unsigned int sampling_rate;
  79. unsigned int up_threshold;
  80. unsigned int ignore_nice;
  81. };
  82. static struct dbs_tuners dbs_tuners_ins = {
  83. .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
  84. .ignore_nice = 0,
  85. };
  86. static inline cputime64_t get_cpu_idle_time(unsigned int cpu)
  87. {
  88. cputime64_t retval;
  89. retval = cputime64_add(kstat_cpu(cpu).cpustat.idle,
  90. kstat_cpu(cpu).cpustat.iowait);
  91. if (dbs_tuners_ins.ignore_nice)
  92. retval = cputime64_add(retval, kstat_cpu(cpu).cpustat.nice);
  93. return retval;
  94. }
  95. /************************** sysfs interface ************************/
  96. static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
  97. {
  98. return sprintf (buf, "%u\n", MAX_SAMPLING_RATE);
  99. }
  100. static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
  101. {
  102. return sprintf (buf, "%u\n", MIN_SAMPLING_RATE);
  103. }
  104. #define define_one_ro(_name) \
  105. static struct freq_attr _name = \
  106. __ATTR(_name, 0444, show_##_name, NULL)
  107. define_one_ro(sampling_rate_max);
  108. define_one_ro(sampling_rate_min);
  109. /* cpufreq_ondemand Governor Tunables */
  110. #define show_one(file_name, object) \
  111. static ssize_t show_##file_name \
  112. (struct cpufreq_policy *unused, char *buf) \
  113. { \
  114. return sprintf(buf, "%u\n", dbs_tuners_ins.object); \
  115. }
  116. show_one(sampling_rate, sampling_rate);
  117. show_one(up_threshold, up_threshold);
  118. show_one(ignore_nice_load, ignore_nice);
  119. static ssize_t store_sampling_rate(struct cpufreq_policy *unused,
  120. const char *buf, size_t count)
  121. {
  122. unsigned int input;
  123. int ret;
  124. ret = sscanf (buf, "%u", &input);
  125. mutex_lock(&dbs_mutex);
  126. if (ret != 1 || input > MAX_SAMPLING_RATE || input < MIN_SAMPLING_RATE) {
  127. mutex_unlock(&dbs_mutex);
  128. return -EINVAL;
  129. }
  130. dbs_tuners_ins.sampling_rate = input;
  131. mutex_unlock(&dbs_mutex);
  132. return count;
  133. }
  134. static ssize_t store_up_threshold(struct cpufreq_policy *unused,
  135. const char *buf, size_t count)
  136. {
  137. unsigned int input;
  138. int ret;
  139. ret = sscanf (buf, "%u", &input);
  140. mutex_lock(&dbs_mutex);
  141. if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
  142. input < MIN_FREQUENCY_UP_THRESHOLD) {
  143. mutex_unlock(&dbs_mutex);
  144. return -EINVAL;
  145. }
  146. dbs_tuners_ins.up_threshold = input;
  147. mutex_unlock(&dbs_mutex);
  148. return count;
  149. }
  150. static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
  151. const char *buf, size_t count)
  152. {
  153. unsigned int input;
  154. int ret;
  155. unsigned int j;
  156. ret = sscanf (buf, "%u", &input);
  157. if ( ret != 1 )
  158. return -EINVAL;
  159. if ( input > 1 )
  160. input = 1;
  161. mutex_lock(&dbs_mutex);
  162. if ( input == dbs_tuners_ins.ignore_nice ) { /* nothing to do */
  163. mutex_unlock(&dbs_mutex);
  164. return count;
  165. }
  166. dbs_tuners_ins.ignore_nice = input;
  167. /* we need to re-evaluate prev_cpu_idle */
  168. for_each_online_cpu(j) {
  169. struct cpu_dbs_info_s *dbs_info;
  170. dbs_info = &per_cpu(cpu_dbs_info, j);
  171. dbs_info->prev_cpu_idle = get_cpu_idle_time(j);
  172. dbs_info->prev_cpu_wall = get_jiffies_64();
  173. }
  174. mutex_unlock(&dbs_mutex);
  175. return count;
  176. }
  177. #define define_one_rw(_name) \
  178. static struct freq_attr _name = \
  179. __ATTR(_name, 0644, show_##_name, store_##_name)
  180. define_one_rw(sampling_rate);
  181. define_one_rw(up_threshold);
  182. define_one_rw(ignore_nice_load);
  183. static struct attribute * dbs_attributes[] = {
  184. &sampling_rate_max.attr,
  185. &sampling_rate_min.attr,
  186. &sampling_rate.attr,
  187. &up_threshold.attr,
  188. &ignore_nice_load.attr,
  189. NULL
  190. };
  191. static struct attribute_group dbs_attr_group = {
  192. .attrs = dbs_attributes,
  193. .name = "ondemand",
  194. };
  195. /************************** sysfs end ************************/
  196. static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
  197. {
  198. unsigned int idle_ticks, total_ticks;
  199. unsigned int load;
  200. cputime64_t cur_jiffies;
  201. struct cpufreq_policy *policy;
  202. unsigned int j;
  203. if (!this_dbs_info->enable)
  204. return;
  205. policy = this_dbs_info->cur_policy;
  206. cur_jiffies = jiffies64_to_cputime64(get_jiffies_64());
  207. total_ticks = (unsigned int) cputime64_sub(cur_jiffies,
  208. this_dbs_info->prev_cpu_wall);
  209. this_dbs_info->prev_cpu_wall = cur_jiffies;
  210. /*
  211. * Every sampling_rate, we check, if current idle time is less
  212. * than 20% (default), then we try to increase frequency
  213. * Every sampling_rate, we look for a the lowest
  214. * frequency which can sustain the load while keeping idle time over
  215. * 30%. If such a frequency exist, we try to decrease to this frequency.
  216. *
  217. * Any frequency increase takes it to the maximum frequency.
  218. * Frequency reduction happens at minimum steps of
  219. * 5% (default) of current frequency
  220. */
  221. /* Get Idle Time */
  222. idle_ticks = UINT_MAX;
  223. for_each_cpu_mask(j, policy->cpus) {
  224. cputime64_t total_idle_ticks;
  225. unsigned int tmp_idle_ticks;
  226. struct cpu_dbs_info_s *j_dbs_info;
  227. j_dbs_info = &per_cpu(cpu_dbs_info, j);
  228. total_idle_ticks = get_cpu_idle_time(j);
  229. tmp_idle_ticks = (unsigned int) cputime64_sub(total_idle_ticks,
  230. j_dbs_info->prev_cpu_idle);
  231. j_dbs_info->prev_cpu_idle = total_idle_ticks;
  232. if (tmp_idle_ticks < idle_ticks)
  233. idle_ticks = tmp_idle_ticks;
  234. }
  235. load = (100 * (total_ticks - idle_ticks)) / total_ticks;
  236. /* Check for frequency increase */
  237. if (load > dbs_tuners_ins.up_threshold) {
  238. /* if we are already at full speed then break out early */
  239. if (policy->cur == policy->max)
  240. return;
  241. __cpufreq_driver_target(policy, policy->max,
  242. CPUFREQ_RELATION_H);
  243. return;
  244. }
  245. /* Check for frequency decrease */
  246. /* if we cannot reduce the frequency anymore, break out early */
  247. if (policy->cur == policy->min)
  248. return;
  249. /*
  250. * The optimal frequency is the frequency that is the lowest that
  251. * can support the current CPU usage without triggering the up
  252. * policy. To be safe, we focus 10 points under the threshold.
  253. */
  254. if (load < (dbs_tuners_ins.up_threshold - 10)) {
  255. unsigned int freq_next;
  256. freq_next = (policy->cur * load) /
  257. (dbs_tuners_ins.up_threshold - 10);
  258. __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L);
  259. }
  260. }
  261. static void do_dbs_timer(void *data)
  262. {
  263. unsigned int cpu = smp_processor_id();
  264. struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, cpu);
  265. dbs_check_cpu(dbs_info);
  266. queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work,
  267. usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
  268. }
  269. static inline void dbs_timer_init(unsigned int cpu)
  270. {
  271. struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, cpu);
  272. INIT_WORK(&dbs_info->work, do_dbs_timer, 0);
  273. queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work,
  274. usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
  275. return;
  276. }
  277. static inline void dbs_timer_exit(unsigned int cpu)
  278. {
  279. struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, cpu);
  280. cancel_rearming_delayed_workqueue(kondemand_wq, &dbs_info->work);
  281. }
  282. static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
  283. unsigned int event)
  284. {
  285. unsigned int cpu = policy->cpu;
  286. struct cpu_dbs_info_s *this_dbs_info;
  287. unsigned int j;
  288. this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
  289. switch (event) {
  290. case CPUFREQ_GOV_START:
  291. if ((!cpu_online(cpu)) ||
  292. (!policy->cur))
  293. return -EINVAL;
  294. if (policy->cpuinfo.transition_latency >
  295. (TRANSITION_LATENCY_LIMIT * 1000)) {
  296. printk(KERN_WARNING "ondemand governor failed to load "
  297. "due to too long transition latency\n");
  298. return -EINVAL;
  299. }
  300. if (this_dbs_info->enable) /* Already enabled */
  301. break;
  302. mutex_lock(&dbs_mutex);
  303. dbs_enable++;
  304. if (dbs_enable == 1) {
  305. kondemand_wq = create_workqueue("kondemand");
  306. if (!kondemand_wq) {
  307. printk(KERN_ERR "Creation of kondemand failed\n");
  308. dbs_enable--;
  309. mutex_unlock(&dbs_mutex);
  310. return -ENOSPC;
  311. }
  312. }
  313. for_each_cpu_mask(j, policy->cpus) {
  314. struct cpu_dbs_info_s *j_dbs_info;
  315. j_dbs_info = &per_cpu(cpu_dbs_info, j);
  316. j_dbs_info->cur_policy = policy;
  317. j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j);
  318. j_dbs_info->prev_cpu_wall = get_jiffies_64();
  319. }
  320. this_dbs_info->enable = 1;
  321. sysfs_create_group(&policy->kobj, &dbs_attr_group);
  322. /*
  323. * Start the timerschedule work, when this governor
  324. * is used for first time
  325. */
  326. if (dbs_enable == 1) {
  327. unsigned int latency;
  328. /* policy latency is in nS. Convert it to uS first */
  329. latency = policy->cpuinfo.transition_latency / 1000;
  330. if (latency == 0)
  331. latency = 1;
  332. def_sampling_rate = latency *
  333. DEF_SAMPLING_RATE_LATENCY_MULTIPLIER;
  334. if (def_sampling_rate < MIN_STAT_SAMPLING_RATE)
  335. def_sampling_rate = MIN_STAT_SAMPLING_RATE;
  336. dbs_tuners_ins.sampling_rate = def_sampling_rate;
  337. }
  338. dbs_timer_init(policy->cpu);
  339. mutex_unlock(&dbs_mutex);
  340. break;
  341. case CPUFREQ_GOV_STOP:
  342. mutex_lock(&dbs_mutex);
  343. dbs_timer_exit(policy->cpu);
  344. this_dbs_info->enable = 0;
  345. sysfs_remove_group(&policy->kobj, &dbs_attr_group);
  346. dbs_enable--;
  347. if (dbs_enable == 0)
  348. destroy_workqueue(kondemand_wq);
  349. mutex_unlock(&dbs_mutex);
  350. break;
  351. case CPUFREQ_GOV_LIMITS:
  352. lock_cpu_hotplug();
  353. mutex_lock(&dbs_mutex);
  354. if (policy->max < this_dbs_info->cur_policy->cur)
  355. __cpufreq_driver_target(
  356. this_dbs_info->cur_policy,
  357. policy->max, CPUFREQ_RELATION_H);
  358. else if (policy->min > this_dbs_info->cur_policy->cur)
  359. __cpufreq_driver_target(
  360. this_dbs_info->cur_policy,
  361. policy->min, CPUFREQ_RELATION_L);
  362. mutex_unlock(&dbs_mutex);
  363. unlock_cpu_hotplug();
  364. break;
  365. }
  366. return 0;
  367. }
  368. static struct cpufreq_governor cpufreq_gov_dbs = {
  369. .name = "ondemand",
  370. .governor = cpufreq_governor_dbs,
  371. .owner = THIS_MODULE,
  372. };
  373. static int __init cpufreq_gov_dbs_init(void)
  374. {
  375. return cpufreq_register_governor(&cpufreq_gov_dbs);
  376. }
  377. static void __exit cpufreq_gov_dbs_exit(void)
  378. {
  379. cpufreq_unregister_governor(&cpufreq_gov_dbs);
  380. }
  381. MODULE_AUTHOR ("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
  382. MODULE_DESCRIPTION ("'cpufreq_ondemand' - A dynamic cpufreq governor for "
  383. "Low Latency Frequency Transition capable processors");
  384. MODULE_LICENSE ("GPL");
  385. module_init(cpufreq_gov_dbs_init);
  386. module_exit(cpufreq_gov_dbs_exit);