perf_event.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644
  1. #undef DEBUG
  2. /*
  3. * ARM performance counter support.
  4. *
  5. * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
  6. * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
  7. *
  8. * This code is based on the sparc64 perf event code, which is in turn based
  9. * on the x86 code. Callchain code is based on the ARM OProfile backtrace
  10. * code.
  11. */
  12. #define pr_fmt(fmt) "hw perfevents: " fmt
  13. #include <linux/kernel.h>
  14. #include <linux/platform_device.h>
  15. #include <linux/pm_runtime.h>
  16. #include <linux/uaccess.h>
  17. #include <asm/irq_regs.h>
  18. #include <asm/pmu.h>
  19. #include <asm/stacktrace.h>
  20. static int
  21. armpmu_map_cache_event(const unsigned (*cache_map)
  22. [PERF_COUNT_HW_CACHE_MAX]
  23. [PERF_COUNT_HW_CACHE_OP_MAX]
  24. [PERF_COUNT_HW_CACHE_RESULT_MAX],
  25. u64 config)
  26. {
  27. unsigned int cache_type, cache_op, cache_result, ret;
  28. cache_type = (config >> 0) & 0xff;
  29. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  30. return -EINVAL;
  31. cache_op = (config >> 8) & 0xff;
  32. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  33. return -EINVAL;
  34. cache_result = (config >> 16) & 0xff;
  35. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  36. return -EINVAL;
  37. ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
  38. if (ret == CACHE_OP_UNSUPPORTED)
  39. return -ENOENT;
  40. return ret;
  41. }
  42. static int
  43. armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
  44. {
  45. int mapping;
  46. if (config >= PERF_COUNT_HW_MAX)
  47. return -EINVAL;
  48. mapping = (*event_map)[config];
  49. return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
  50. }
  51. static int
  52. armpmu_map_raw_event(u32 raw_event_mask, u64 config)
  53. {
  54. return (int)(config & raw_event_mask);
  55. }
  56. int
  57. armpmu_map_event(struct perf_event *event,
  58. const unsigned (*event_map)[PERF_COUNT_HW_MAX],
  59. const unsigned (*cache_map)
  60. [PERF_COUNT_HW_CACHE_MAX]
  61. [PERF_COUNT_HW_CACHE_OP_MAX]
  62. [PERF_COUNT_HW_CACHE_RESULT_MAX],
  63. u32 raw_event_mask)
  64. {
  65. u64 config = event->attr.config;
  66. switch (event->attr.type) {
  67. case PERF_TYPE_HARDWARE:
  68. return armpmu_map_hw_event(event_map, config);
  69. case PERF_TYPE_HW_CACHE:
  70. return armpmu_map_cache_event(cache_map, config);
  71. case PERF_TYPE_RAW:
  72. return armpmu_map_raw_event(raw_event_mask, config);
  73. }
  74. return -ENOENT;
  75. }
  76. int armpmu_event_set_period(struct perf_event *event)
  77. {
  78. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  79. struct hw_perf_event *hwc = &event->hw;
  80. s64 left = local64_read(&hwc->period_left);
  81. s64 period = hwc->sample_period;
  82. int ret = 0;
  83. /* The period may have been changed by PERF_EVENT_IOC_PERIOD */
  84. if (unlikely(period != hwc->last_period))
  85. left = period - (hwc->last_period - left);
  86. if (unlikely(left <= -period)) {
  87. left = period;
  88. local64_set(&hwc->period_left, left);
  89. hwc->last_period = period;
  90. ret = 1;
  91. }
  92. if (unlikely(left <= 0)) {
  93. left += period;
  94. local64_set(&hwc->period_left, left);
  95. hwc->last_period = period;
  96. ret = 1;
  97. }
  98. if (left > (s64)armpmu->max_period)
  99. left = armpmu->max_period;
  100. local64_set(&hwc->prev_count, (u64)-left);
  101. armpmu->write_counter(event, (u64)(-left) & 0xffffffff);
  102. perf_event_update_userpage(event);
  103. return ret;
  104. }
  105. u64 armpmu_event_update(struct perf_event *event)
  106. {
  107. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  108. struct hw_perf_event *hwc = &event->hw;
  109. u64 delta, prev_raw_count, new_raw_count;
  110. again:
  111. prev_raw_count = local64_read(&hwc->prev_count);
  112. new_raw_count = armpmu->read_counter(event);
  113. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  114. new_raw_count) != prev_raw_count)
  115. goto again;
  116. delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
  117. local64_add(delta, &event->count);
  118. local64_sub(delta, &hwc->period_left);
  119. return new_raw_count;
  120. }
  121. static void
  122. armpmu_read(struct perf_event *event)
  123. {
  124. armpmu_event_update(event);
  125. }
  126. static void
  127. armpmu_stop(struct perf_event *event, int flags)
  128. {
  129. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  130. struct hw_perf_event *hwc = &event->hw;
  131. /*
  132. * ARM pmu always has to update the counter, so ignore
  133. * PERF_EF_UPDATE, see comments in armpmu_start().
  134. */
  135. if (!(hwc->state & PERF_HES_STOPPED)) {
  136. armpmu->disable(event);
  137. armpmu_event_update(event);
  138. hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  139. }
  140. }
  141. static void armpmu_start(struct perf_event *event, int flags)
  142. {
  143. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  144. struct hw_perf_event *hwc = &event->hw;
  145. /*
  146. * ARM pmu always has to reprogram the period, so ignore
  147. * PERF_EF_RELOAD, see the comment below.
  148. */
  149. if (flags & PERF_EF_RELOAD)
  150. WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
  151. hwc->state = 0;
  152. /*
  153. * Set the period again. Some counters can't be stopped, so when we
  154. * were stopped we simply disabled the IRQ source and the counter
  155. * may have been left counting. If we don't do this step then we may
  156. * get an interrupt too soon or *way* too late if the overflow has
  157. * happened since disabling.
  158. */
  159. armpmu_event_set_period(event);
  160. armpmu->enable(event);
  161. }
  162. static void
  163. armpmu_del(struct perf_event *event, int flags)
  164. {
  165. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  166. struct pmu_hw_events *hw_events = armpmu->get_hw_events();
  167. struct hw_perf_event *hwc = &event->hw;
  168. int idx = hwc->idx;
  169. armpmu_stop(event, PERF_EF_UPDATE);
  170. hw_events->events[idx] = NULL;
  171. clear_bit(idx, hw_events->used_mask);
  172. perf_event_update_userpage(event);
  173. }
  174. static int
  175. armpmu_add(struct perf_event *event, int flags)
  176. {
  177. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  178. struct pmu_hw_events *hw_events = armpmu->get_hw_events();
  179. struct hw_perf_event *hwc = &event->hw;
  180. int idx;
  181. int err = 0;
  182. perf_pmu_disable(event->pmu);
  183. /* If we don't have a space for the counter then finish early. */
  184. idx = armpmu->get_event_idx(hw_events, event);
  185. if (idx < 0) {
  186. err = idx;
  187. goto out;
  188. }
  189. /*
  190. * If there is an event in the counter we are going to use then make
  191. * sure it is disabled.
  192. */
  193. event->hw.idx = idx;
  194. armpmu->disable(event);
  195. hw_events->events[idx] = event;
  196. hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  197. if (flags & PERF_EF_START)
  198. armpmu_start(event, PERF_EF_RELOAD);
  199. /* Propagate our changes to the userspace mapping. */
  200. perf_event_update_userpage(event);
  201. out:
  202. perf_pmu_enable(event->pmu);
  203. return err;
  204. }
  205. static int
  206. validate_event(struct pmu_hw_events *hw_events,
  207. struct perf_event *event)
  208. {
  209. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  210. struct pmu *leader_pmu = event->group_leader->pmu;
  211. if (is_software_event(event))
  212. return 1;
  213. if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF)
  214. return 1;
  215. if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
  216. return 1;
  217. return armpmu->get_event_idx(hw_events, event) >= 0;
  218. }
  219. static int
  220. validate_group(struct perf_event *event)
  221. {
  222. struct perf_event *sibling, *leader = event->group_leader;
  223. struct pmu_hw_events fake_pmu;
  224. DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS);
  225. /*
  226. * Initialise the fake PMU. We only need to populate the
  227. * used_mask for the purposes of validation.
  228. */
  229. memset(fake_used_mask, 0, sizeof(fake_used_mask));
  230. fake_pmu.used_mask = fake_used_mask;
  231. if (!validate_event(&fake_pmu, leader))
  232. return -EINVAL;
  233. list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
  234. if (!validate_event(&fake_pmu, sibling))
  235. return -EINVAL;
  236. }
  237. if (!validate_event(&fake_pmu, event))
  238. return -EINVAL;
  239. return 0;
  240. }
  241. static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
  242. {
  243. struct arm_pmu *armpmu = (struct arm_pmu *) dev;
  244. struct platform_device *plat_device = armpmu->plat_device;
  245. struct arm_pmu_platdata *plat = dev_get_platdata(&plat_device->dev);
  246. if (plat && plat->handle_irq)
  247. return plat->handle_irq(irq, dev, armpmu->handle_irq);
  248. else
  249. return armpmu->handle_irq(irq, dev);
  250. }
  251. static void
  252. armpmu_release_hardware(struct arm_pmu *armpmu)
  253. {
  254. armpmu->free_irq(armpmu);
  255. pm_runtime_put_sync(&armpmu->plat_device->dev);
  256. }
  257. static int
  258. armpmu_reserve_hardware(struct arm_pmu *armpmu)
  259. {
  260. int err;
  261. struct platform_device *pmu_device = armpmu->plat_device;
  262. if (!pmu_device)
  263. return -ENODEV;
  264. pm_runtime_get_sync(&pmu_device->dev);
  265. err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);
  266. if (err) {
  267. armpmu_release_hardware(armpmu);
  268. return err;
  269. }
  270. return 0;
  271. }
  272. static void
  273. hw_perf_event_destroy(struct perf_event *event)
  274. {
  275. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  276. atomic_t *active_events = &armpmu->active_events;
  277. struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
  278. if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
  279. armpmu_release_hardware(armpmu);
  280. mutex_unlock(pmu_reserve_mutex);
  281. }
  282. }
  283. static int
  284. event_requires_mode_exclusion(struct perf_event_attr *attr)
  285. {
  286. return attr->exclude_idle || attr->exclude_user ||
  287. attr->exclude_kernel || attr->exclude_hv;
  288. }
  289. static int
  290. __hw_perf_event_init(struct perf_event *event)
  291. {
  292. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  293. struct hw_perf_event *hwc = &event->hw;
  294. int mapping;
  295. mapping = armpmu->map_event(event);
  296. if (mapping < 0) {
  297. pr_debug("event %x:%llx not supported\n", event->attr.type,
  298. event->attr.config);
  299. return mapping;
  300. }
  301. /*
  302. * We don't assign an index until we actually place the event onto
  303. * hardware. Use -1 to signify that we haven't decided where to put it
  304. * yet. For SMP systems, each core has it's own PMU so we can't do any
  305. * clever allocation or constraints checking at this point.
  306. */
  307. hwc->idx = -1;
  308. hwc->config_base = 0;
  309. hwc->config = 0;
  310. hwc->event_base = 0;
  311. /*
  312. * Check whether we need to exclude the counter from certain modes.
  313. */
  314. if ((!armpmu->set_event_filter ||
  315. armpmu->set_event_filter(hwc, &event->attr)) &&
  316. event_requires_mode_exclusion(&event->attr)) {
  317. pr_debug("ARM performance counters do not support "
  318. "mode exclusion\n");
  319. return -EOPNOTSUPP;
  320. }
  321. /*
  322. * Store the event encoding into the config_base field.
  323. */
  324. hwc->config_base |= (unsigned long)mapping;
  325. if (!hwc->sample_period) {
  326. /*
  327. * For non-sampling runs, limit the sample_period to half
  328. * of the counter width. That way, the new counter value
  329. * is far less likely to overtake the previous one unless
  330. * you have some serious IRQ latency issues.
  331. */
  332. hwc->sample_period = armpmu->max_period >> 1;
  333. hwc->last_period = hwc->sample_period;
  334. local64_set(&hwc->period_left, hwc->sample_period);
  335. }
  336. if (event->group_leader != event) {
  337. if (validate_group(event) != 0)
  338. return -EINVAL;
  339. }
  340. return 0;
  341. }
  342. static int armpmu_event_init(struct perf_event *event)
  343. {
  344. struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  345. int err = 0;
  346. atomic_t *active_events = &armpmu->active_events;
  347. /* does not support taken branch sampling */
  348. if (has_branch_stack(event))
  349. return -EOPNOTSUPP;
  350. if (armpmu->map_event(event) == -ENOENT)
  351. return -ENOENT;
  352. event->destroy = hw_perf_event_destroy;
  353. if (!atomic_inc_not_zero(active_events)) {
  354. mutex_lock(&armpmu->reserve_mutex);
  355. if (atomic_read(active_events) == 0)
  356. err = armpmu_reserve_hardware(armpmu);
  357. if (!err)
  358. atomic_inc(active_events);
  359. mutex_unlock(&armpmu->reserve_mutex);
  360. }
  361. if (err)
  362. return err;
  363. err = __hw_perf_event_init(event);
  364. if (err)
  365. hw_perf_event_destroy(event);
  366. return err;
  367. }
  368. static void armpmu_enable(struct pmu *pmu)
  369. {
  370. struct arm_pmu *armpmu = to_arm_pmu(pmu);
  371. struct pmu_hw_events *hw_events = armpmu->get_hw_events();
  372. int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
  373. if (enabled)
  374. armpmu->start(armpmu);
  375. }
  376. static void armpmu_disable(struct pmu *pmu)
  377. {
  378. struct arm_pmu *armpmu = to_arm_pmu(pmu);
  379. armpmu->stop(armpmu);
  380. }
  381. #ifdef CONFIG_PM_RUNTIME
  382. static int armpmu_runtime_resume(struct device *dev)
  383. {
  384. struct arm_pmu_platdata *plat = dev_get_platdata(dev);
  385. if (plat && plat->runtime_resume)
  386. return plat->runtime_resume(dev);
  387. return 0;
  388. }
  389. static int armpmu_runtime_suspend(struct device *dev)
  390. {
  391. struct arm_pmu_platdata *plat = dev_get_platdata(dev);
  392. if (plat && plat->runtime_suspend)
  393. return plat->runtime_suspend(dev);
  394. return 0;
  395. }
  396. #endif
  397. const struct dev_pm_ops armpmu_dev_pm_ops = {
  398. SET_RUNTIME_PM_OPS(armpmu_runtime_suspend, armpmu_runtime_resume, NULL)
  399. };
  400. static void armpmu_init(struct arm_pmu *armpmu)
  401. {
  402. atomic_set(&armpmu->active_events, 0);
  403. mutex_init(&armpmu->reserve_mutex);
  404. armpmu->pmu = (struct pmu) {
  405. .pmu_enable = armpmu_enable,
  406. .pmu_disable = armpmu_disable,
  407. .event_init = armpmu_event_init,
  408. .add = armpmu_add,
  409. .del = armpmu_del,
  410. .start = armpmu_start,
  411. .stop = armpmu_stop,
  412. .read = armpmu_read,
  413. };
  414. }
  415. int armpmu_register(struct arm_pmu *armpmu, int type)
  416. {
  417. armpmu_init(armpmu);
  418. pm_runtime_enable(&armpmu->plat_device->dev);
  419. pr_info("enabled with %s PMU driver, %d counters available\n",
  420. armpmu->name, armpmu->num_events);
  421. return perf_pmu_register(&armpmu->pmu, armpmu->name, type);
  422. }
  423. /*
  424. * Callchain handling code.
  425. */
  426. /*
  427. * The registers we're interested in are at the end of the variable
  428. * length saved register structure. The fp points at the end of this
  429. * structure so the address of this struct is:
  430. * (struct frame_tail *)(xxx->fp)-1
  431. *
  432. * This code has been adapted from the ARM OProfile support.
  433. */
  434. struct frame_tail {
  435. struct frame_tail __user *fp;
  436. unsigned long sp;
  437. unsigned long lr;
  438. } __attribute__((packed));
  439. /*
  440. * Get the return address for a single stackframe and return a pointer to the
  441. * next frame tail.
  442. */
  443. static struct frame_tail __user *
  444. user_backtrace(struct frame_tail __user *tail,
  445. struct perf_callchain_entry *entry)
  446. {
  447. struct frame_tail buftail;
  448. /* Also check accessibility of one struct frame_tail beyond */
  449. if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
  450. return NULL;
  451. if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail)))
  452. return NULL;
  453. perf_callchain_store(entry, buftail.lr);
  454. /*
  455. * Frame pointers should strictly progress back up the stack
  456. * (towards higher addresses).
  457. */
  458. if (tail + 1 >= buftail.fp)
  459. return NULL;
  460. return buftail.fp - 1;
  461. }
  462. void
  463. perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
  464. {
  465. struct frame_tail __user *tail;
  466. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  467. /* We don't support guest os callchain now */
  468. return;
  469. }
  470. perf_callchain_store(entry, regs->ARM_pc);
  471. tail = (struct frame_tail __user *)regs->ARM_fp - 1;
  472. while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
  473. tail && !((unsigned long)tail & 0x3))
  474. tail = user_backtrace(tail, entry);
  475. }
  476. /*
  477. * Gets called by walk_stackframe() for every stackframe. This will be called
  478. * whist unwinding the stackframe and is like a subroutine return so we use
  479. * the PC.
  480. */
  481. static int
  482. callchain_trace(struct stackframe *fr,
  483. void *data)
  484. {
  485. struct perf_callchain_entry *entry = data;
  486. perf_callchain_store(entry, fr->pc);
  487. return 0;
  488. }
  489. void
  490. perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
  491. {
  492. struct stackframe fr;
  493. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  494. /* We don't support guest os callchain now */
  495. return;
  496. }
  497. fr.fp = regs->ARM_fp;
  498. fr.sp = regs->ARM_sp;
  499. fr.lr = regs->ARM_lr;
  500. fr.pc = regs->ARM_pc;
  501. walk_stackframe(&fr, callchain_trace, entry);
  502. }
  503. unsigned long perf_instruction_pointer(struct pt_regs *regs)
  504. {
  505. if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
  506. return perf_guest_cbs->get_guest_ip();
  507. return instruction_pointer(regs);
  508. }
  509. unsigned long perf_misc_flags(struct pt_regs *regs)
  510. {
  511. int misc = 0;
  512. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  513. if (perf_guest_cbs->is_user_mode())
  514. misc |= PERF_RECORD_MISC_GUEST_USER;
  515. else
  516. misc |= PERF_RECORD_MISC_GUEST_KERNEL;
  517. } else {
  518. if (user_mode(regs))
  519. misc |= PERF_RECORD_MISC_USER;
  520. else
  521. misc |= PERF_RECORD_MISC_KERNEL;
  522. }
  523. return misc;
  524. }