cpuidle.c 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. /*
  2. * cpuidle.c - core cpuidle infrastructure
  3. *
  4. * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
  5. * Shaohua Li <shaohua.li@intel.com>
  6. * Adam Belay <abelay@novell.com>
  7. *
  8. * This code is licenced under the GPL.
  9. */
  10. #include <linux/kernel.h>
  11. #include <linux/mutex.h>
  12. #include <linux/sched.h>
  13. #include <linux/notifier.h>
  14. #include <linux/pm_qos_params.h>
  15. #include <linux/cpu.h>
  16. #include <linux/cpuidle.h>
  17. #include <linux/ktime.h>
  18. #include "cpuidle.h"
  19. DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
  20. DEFINE_MUTEX(cpuidle_lock);
  21. LIST_HEAD(cpuidle_detected_devices);
  22. static void (*pm_idle_old)(void);
  23. static int enabled_devices;
  24. #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT)
  25. static void cpuidle_kick_cpus(void)
  26. {
  27. cpu_idle_wait();
  28. }
  29. #elif defined(CONFIG_SMP)
  30. # error "Arch needs cpu_idle_wait() equivalent here"
  31. #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */
  32. static void cpuidle_kick_cpus(void) {}
  33. #endif
  34. /**
  35. * cpuidle_idle_call - the main idle loop
  36. *
  37. * NOTE: no locks or semaphores should be used here
  38. */
  39. static void cpuidle_idle_call(void)
  40. {
  41. struct cpuidle_device *dev = __get_cpu_var(cpuidle_devices);
  42. struct cpuidle_state *target_state;
  43. int next_state;
  44. /* check if the device is ready */
  45. if (!dev || !dev->enabled) {
  46. if (pm_idle_old)
  47. pm_idle_old();
  48. else
  49. local_irq_enable();
  50. return;
  51. }
  52. /* ask the governor for the next state */
  53. next_state = cpuidle_curr_governor->select(dev);
  54. if (need_resched())
  55. return;
  56. target_state = &dev->states[next_state];
  57. /* enter the state and update stats */
  58. dev->last_residency = target_state->enter(dev, target_state);
  59. dev->last_state = target_state;
  60. target_state->time += dev->last_residency;
  61. target_state->usage++;
  62. /* give the governor an opportunity to reflect on the outcome */
  63. if (cpuidle_curr_governor->reflect)
  64. cpuidle_curr_governor->reflect(dev);
  65. }
  66. /**
  67. * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
  68. */
  69. void cpuidle_install_idle_handler(void)
  70. {
  71. if (enabled_devices && (pm_idle != cpuidle_idle_call)) {
  72. /* Make sure all changes finished before we switch to new idle */
  73. smp_wmb();
  74. pm_idle = cpuidle_idle_call;
  75. }
  76. }
  77. /**
  78. * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
  79. */
  80. void cpuidle_uninstall_idle_handler(void)
  81. {
  82. if (enabled_devices && (pm_idle != pm_idle_old)) {
  83. pm_idle = pm_idle_old;
  84. cpuidle_kick_cpus();
  85. }
  86. }
  87. /**
  88. * cpuidle_pause_and_lock - temporarily disables CPUIDLE
  89. */
  90. void cpuidle_pause_and_lock(void)
  91. {
  92. mutex_lock(&cpuidle_lock);
  93. cpuidle_uninstall_idle_handler();
  94. }
  95. EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
  96. /**
  97. * cpuidle_resume_and_unlock - resumes CPUIDLE operation
  98. */
  99. void cpuidle_resume_and_unlock(void)
  100. {
  101. cpuidle_install_idle_handler();
  102. mutex_unlock(&cpuidle_lock);
  103. }
  104. EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
  105. /**
  106. * cpuidle_enable_device - enables idle PM for a CPU
  107. * @dev: the CPU
  108. *
  109. * This function must be called between cpuidle_pause_and_lock and
  110. * cpuidle_resume_and_unlock when used externally.
  111. */
  112. int cpuidle_enable_device(struct cpuidle_device *dev)
  113. {
  114. int ret, i;
  115. if (dev->enabled)
  116. return 0;
  117. if (!cpuidle_curr_driver || !cpuidle_curr_governor)
  118. return -EIO;
  119. if (!dev->state_count)
  120. return -EINVAL;
  121. if ((ret = cpuidle_add_state_sysfs(dev)))
  122. return ret;
  123. if (cpuidle_curr_governor->enable &&
  124. (ret = cpuidle_curr_governor->enable(dev)))
  125. goto fail_sysfs;
  126. for (i = 0; i < dev->state_count; i++) {
  127. dev->states[i].usage = 0;
  128. dev->states[i].time = 0;
  129. }
  130. dev->last_residency = 0;
  131. dev->last_state = NULL;
  132. smp_wmb();
  133. dev->enabled = 1;
  134. enabled_devices++;
  135. return 0;
  136. fail_sysfs:
  137. cpuidle_remove_state_sysfs(dev);
  138. return ret;
  139. }
  140. EXPORT_SYMBOL_GPL(cpuidle_enable_device);
  141. /**
  142. * cpuidle_disable_device - disables idle PM for a CPU
  143. * @dev: the CPU
  144. *
  145. * This function must be called between cpuidle_pause_and_lock and
  146. * cpuidle_resume_and_unlock when used externally.
  147. */
  148. void cpuidle_disable_device(struct cpuidle_device *dev)
  149. {
  150. if (!dev->enabled)
  151. return;
  152. if (!cpuidle_curr_driver || !cpuidle_curr_governor)
  153. return;
  154. dev->enabled = 0;
  155. if (cpuidle_curr_governor->disable)
  156. cpuidle_curr_governor->disable(dev);
  157. cpuidle_remove_state_sysfs(dev);
  158. enabled_devices--;
  159. }
  160. EXPORT_SYMBOL_GPL(cpuidle_disable_device);
  161. #ifdef CONFIG_ARCH_HAS_CPU_RELAX
  162. static int poll_idle(struct cpuidle_device *dev, struct cpuidle_state *st)
  163. {
  164. ktime_t t1, t2;
  165. s64 diff;
  166. int ret;
  167. t1 = ktime_get();
  168. local_irq_enable();
  169. while (!need_resched())
  170. cpu_relax();
  171. t2 = ktime_get();
  172. diff = ktime_to_us(ktime_sub(t2, t1));
  173. if (diff > INT_MAX)
  174. diff = INT_MAX;
  175. ret = (int) diff;
  176. return ret;
  177. }
  178. static void poll_idle_init(struct cpuidle_device *dev)
  179. {
  180. struct cpuidle_state *state = &dev->states[0];
  181. cpuidle_set_statedata(state, NULL);
  182. snprintf(state->name, CPUIDLE_NAME_LEN, "C0 (poll idle)");
  183. state->exit_latency = 0;
  184. state->target_residency = 0;
  185. state->power_usage = -1;
  186. state->flags = CPUIDLE_FLAG_POLL | CPUIDLE_FLAG_TIME_VALID;
  187. state->enter = poll_idle;
  188. }
  189. #else
  190. static void poll_idle_init(struct cpuidle_device *dev) {}
  191. #endif /* CONFIG_ARCH_HAS_CPU_RELAX */
  192. /**
  193. * cpuidle_register_device - registers a CPU's idle PM feature
  194. * @dev: the cpu
  195. */
  196. int cpuidle_register_device(struct cpuidle_device *dev)
  197. {
  198. int ret;
  199. struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
  200. if (!sys_dev)
  201. return -EINVAL;
  202. if (!try_module_get(cpuidle_curr_driver->owner))
  203. return -EINVAL;
  204. init_completion(&dev->kobj_unregister);
  205. mutex_lock(&cpuidle_lock);
  206. poll_idle_init(dev);
  207. per_cpu(cpuidle_devices, dev->cpu) = dev;
  208. list_add(&dev->device_list, &cpuidle_detected_devices);
  209. if ((ret = cpuidle_add_sysfs(sys_dev))) {
  210. mutex_unlock(&cpuidle_lock);
  211. module_put(cpuidle_curr_driver->owner);
  212. return ret;
  213. }
  214. cpuidle_enable_device(dev);
  215. cpuidle_install_idle_handler();
  216. mutex_unlock(&cpuidle_lock);
  217. return 0;
  218. }
  219. EXPORT_SYMBOL_GPL(cpuidle_register_device);
  220. /**
  221. * cpuidle_unregister_device - unregisters a CPU's idle PM feature
  222. * @dev: the cpu
  223. */
  224. void cpuidle_unregister_device(struct cpuidle_device *dev)
  225. {
  226. struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
  227. cpuidle_pause_and_lock();
  228. cpuidle_disable_device(dev);
  229. cpuidle_remove_sysfs(sys_dev);
  230. list_del(&dev->device_list);
  231. wait_for_completion(&dev->kobj_unregister);
  232. per_cpu(cpuidle_devices, dev->cpu) = NULL;
  233. cpuidle_resume_and_unlock();
  234. module_put(cpuidle_curr_driver->owner);
  235. }
  236. EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
  237. #ifdef CONFIG_SMP
  238. static void smp_callback(void *v)
  239. {
  240. /* we already woke the CPU up, nothing more to do */
  241. }
  242. /*
  243. * This function gets called when a part of the kernel has a new latency
  244. * requirement. This means we need to get all processors out of their C-state,
  245. * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
  246. * wakes them all right up.
  247. */
  248. static int cpuidle_latency_notify(struct notifier_block *b,
  249. unsigned long l, void *v)
  250. {
  251. smp_call_function(smp_callback, NULL, 0, 1);
  252. return NOTIFY_OK;
  253. }
  254. static struct notifier_block cpuidle_latency_notifier = {
  255. .notifier_call = cpuidle_latency_notify,
  256. };
  257. static inline void latency_notifier_init(struct notifier_block *n)
  258. {
  259. pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
  260. }
  261. #else /* CONFIG_SMP */
  262. #define latency_notifier_init(x) do { } while (0)
  263. #endif /* CONFIG_SMP */
  264. /**
  265. * cpuidle_init - core initializer
  266. */
  267. static int __init cpuidle_init(void)
  268. {
  269. int ret;
  270. pm_idle_old = pm_idle;
  271. ret = cpuidle_add_class_sysfs(&cpu_sysdev_class);
  272. if (ret)
  273. return ret;
  274. latency_notifier_init(&cpuidle_latency_notifier);
  275. return 0;
  276. }
  277. core_initcall(cpuidle_init);