perf_event_amd_iommu.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502
  1. /*
  2. * Copyright (C) 2013 Advanced Micro Devices, Inc.
  3. *
  4. * Author: Steven Kinney <Steven.Kinney@amd.com>
  5. * Author: Suravee Suthikulpanit <Suraveee.Suthikulpanit@amd.com>
  6. *
  7. * Perf: amd_iommu - AMD IOMMU Performance Counter PMU implementation
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License version 2 as
  11. * published by the Free Software Foundation.
  12. */
  13. #include <linux/perf_event.h>
  14. #include <linux/module.h>
  15. #include <linux/cpumask.h>
  16. #include <linux/slab.h>
  17. #include "perf_event.h"
  18. #include "perf_event_amd_iommu.h"
  19. #define COUNTER_SHIFT 16
  20. #define _GET_BANK(ev) ((u8)(ev->hw.extra_reg.reg >> 8))
  21. #define _GET_CNTR(ev) ((u8)(ev->hw.extra_reg.reg))
  22. /* iommu pmu config masks */
  23. #define _GET_CSOURCE(ev) ((ev->hw.config & 0xFFULL))
  24. #define _GET_DEVID(ev) ((ev->hw.config >> 8) & 0xFFFFULL)
  25. #define _GET_PASID(ev) ((ev->hw.config >> 24) & 0xFFFFULL)
  26. #define _GET_DOMID(ev) ((ev->hw.config >> 40) & 0xFFFFULL)
  27. #define _GET_DEVID_MASK(ev) ((ev->hw.extra_reg.config) & 0xFFFFULL)
  28. #define _GET_PASID_MASK(ev) ((ev->hw.extra_reg.config >> 16) & 0xFFFFULL)
  29. #define _GET_DOMID_MASK(ev) ((ev->hw.extra_reg.config >> 32) & 0xFFFFULL)
  30. static struct perf_amd_iommu __perf_iommu;
  31. struct perf_amd_iommu {
  32. struct pmu pmu;
  33. u8 max_banks;
  34. u8 max_counters;
  35. u64 cntr_assign_mask;
  36. raw_spinlock_t lock;
  37. const struct attribute_group *attr_groups[4];
  38. };
  39. #define format_group attr_groups[0]
  40. #define cpumask_group attr_groups[1]
  41. #define events_group attr_groups[2]
  42. #define null_group attr_groups[3]
  43. /*---------------------------------------------
  44. * sysfs format attributes
  45. *---------------------------------------------*/
  46. PMU_FORMAT_ATTR(csource, "config:0-7");
  47. PMU_FORMAT_ATTR(devid, "config:8-23");
  48. PMU_FORMAT_ATTR(pasid, "config:24-39");
  49. PMU_FORMAT_ATTR(domid, "config:40-55");
  50. PMU_FORMAT_ATTR(devid_mask, "config1:0-15");
  51. PMU_FORMAT_ATTR(pasid_mask, "config1:16-31");
  52. PMU_FORMAT_ATTR(domid_mask, "config1:32-47");
  53. static struct attribute *iommu_format_attrs[] = {
  54. &format_attr_csource.attr,
  55. &format_attr_devid.attr,
  56. &format_attr_pasid.attr,
  57. &format_attr_domid.attr,
  58. &format_attr_devid_mask.attr,
  59. &format_attr_pasid_mask.attr,
  60. &format_attr_domid_mask.attr,
  61. NULL,
  62. };
  63. static struct attribute_group amd_iommu_format_group = {
  64. .name = "format",
  65. .attrs = iommu_format_attrs,
  66. };
  67. /*---------------------------------------------
  68. * sysfs events attributes
  69. *---------------------------------------------*/
  70. struct amd_iommu_event_desc {
  71. struct kobj_attribute attr;
  72. const char *event;
  73. };
  74. static ssize_t _iommu_event_show(struct kobject *kobj,
  75. struct kobj_attribute *attr, char *buf)
  76. {
  77. struct amd_iommu_event_desc *event =
  78. container_of(attr, struct amd_iommu_event_desc, attr);
  79. return sprintf(buf, "%s\n", event->event);
  80. }
  81. #define AMD_IOMMU_EVENT_DESC(_name, _event) \
  82. { \
  83. .attr = __ATTR(_name, 0444, _iommu_event_show, NULL), \
  84. .event = _event, \
  85. }
  86. static struct amd_iommu_event_desc amd_iommu_v2_event_descs[] = {
  87. AMD_IOMMU_EVENT_DESC(mem_pass_untrans, "csource=0x01"),
  88. AMD_IOMMU_EVENT_DESC(mem_pass_pretrans, "csource=0x02"),
  89. AMD_IOMMU_EVENT_DESC(mem_pass_excl, "csource=0x03"),
  90. AMD_IOMMU_EVENT_DESC(mem_target_abort, "csource=0x04"),
  91. AMD_IOMMU_EVENT_DESC(mem_trans_total, "csource=0x05"),
  92. AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_hit, "csource=0x06"),
  93. AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_mis, "csource=0x07"),
  94. AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_hit, "csource=0x08"),
  95. AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_mis, "csource=0x09"),
  96. AMD_IOMMU_EVENT_DESC(mem_dte_hit, "csource=0x0a"),
  97. AMD_IOMMU_EVENT_DESC(mem_dte_mis, "csource=0x0b"),
  98. AMD_IOMMU_EVENT_DESC(page_tbl_read_tot, "csource=0x0c"),
  99. AMD_IOMMU_EVENT_DESC(page_tbl_read_nst, "csource=0x0d"),
  100. AMD_IOMMU_EVENT_DESC(page_tbl_read_gst, "csource=0x0e"),
  101. AMD_IOMMU_EVENT_DESC(int_dte_hit, "csource=0x0f"),
  102. AMD_IOMMU_EVENT_DESC(int_dte_mis, "csource=0x10"),
  103. AMD_IOMMU_EVENT_DESC(cmd_processed, "csource=0x11"),
  104. AMD_IOMMU_EVENT_DESC(cmd_processed_inv, "csource=0x12"),
  105. AMD_IOMMU_EVENT_DESC(tlb_inv, "csource=0x13"),
  106. { /* end: all zeroes */ },
  107. };
  108. /*---------------------------------------------
  109. * sysfs cpumask attributes
  110. *---------------------------------------------*/
  111. static cpumask_t iommu_cpumask;
  112. static ssize_t _iommu_cpumask_show(struct device *dev,
  113. struct device_attribute *attr,
  114. char *buf)
  115. {
  116. int n = cpulist_scnprintf(buf, PAGE_SIZE - 2, &iommu_cpumask);
  117. buf[n++] = '\n';
  118. buf[n] = '\0';
  119. return n;
  120. }
  121. static DEVICE_ATTR(cpumask, S_IRUGO, _iommu_cpumask_show, NULL);
  122. static struct attribute *iommu_cpumask_attrs[] = {
  123. &dev_attr_cpumask.attr,
  124. NULL,
  125. };
  126. static struct attribute_group amd_iommu_cpumask_group = {
  127. .attrs = iommu_cpumask_attrs,
  128. };
  129. /*---------------------------------------------*/
  130. static int get_next_avail_iommu_bnk_cntr(struct perf_amd_iommu *perf_iommu)
  131. {
  132. unsigned long flags;
  133. int shift, bank, cntr, retval;
  134. int max_banks = perf_iommu->max_banks;
  135. int max_cntrs = perf_iommu->max_counters;
  136. raw_spin_lock_irqsave(&perf_iommu->lock, flags);
  137. for (bank = 0, shift = 0; bank < max_banks; bank++) {
  138. for (cntr = 0; cntr < max_cntrs; cntr++) {
  139. shift = bank + (bank*3) + cntr;
  140. if (perf_iommu->cntr_assign_mask & (1ULL<<shift)) {
  141. continue;
  142. } else {
  143. perf_iommu->cntr_assign_mask |= (1ULL<<shift);
  144. retval = ((u16)((u16)bank<<8) | (u8)(cntr));
  145. goto out;
  146. }
  147. }
  148. }
  149. retval = -ENOSPC;
  150. out:
  151. raw_spin_unlock_irqrestore(&perf_iommu->lock, flags);
  152. return retval;
  153. }
  154. static int clear_avail_iommu_bnk_cntr(struct perf_amd_iommu *perf_iommu,
  155. u8 bank, u8 cntr)
  156. {
  157. unsigned long flags;
  158. int max_banks, max_cntrs;
  159. int shift = 0;
  160. max_banks = perf_iommu->max_banks;
  161. max_cntrs = perf_iommu->max_counters;
  162. if ((bank > max_banks) || (cntr > max_cntrs))
  163. return -EINVAL;
  164. shift = bank + cntr + (bank*3);
  165. raw_spin_lock_irqsave(&perf_iommu->lock, flags);
  166. perf_iommu->cntr_assign_mask &= ~(1ULL<<shift);
  167. raw_spin_unlock_irqrestore(&perf_iommu->lock, flags);
  168. return 0;
  169. }
  170. static int perf_iommu_event_init(struct perf_event *event)
  171. {
  172. struct hw_perf_event *hwc = &event->hw;
  173. struct perf_amd_iommu *perf_iommu;
  174. u64 config, config1;
  175. /* test the event attr type check for PMU enumeration */
  176. if (event->attr.type != event->pmu->type)
  177. return -ENOENT;
  178. /*
  179. * IOMMU counters are shared across all cores.
  180. * Therefore, it does not support per-process mode.
  181. * Also, it does not support event sampling mode.
  182. */
  183. if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
  184. return -EINVAL;
  185. /* IOMMU counters do not have usr/os/guest/host bits */
  186. if (event->attr.exclude_user || event->attr.exclude_kernel ||
  187. event->attr.exclude_host || event->attr.exclude_guest)
  188. return -EINVAL;
  189. if (event->cpu < 0)
  190. return -EINVAL;
  191. perf_iommu = &__perf_iommu;
  192. if (event->pmu != &perf_iommu->pmu)
  193. return -ENOENT;
  194. if (perf_iommu) {
  195. config = event->attr.config;
  196. config1 = event->attr.config1;
  197. } else {
  198. return -EINVAL;
  199. }
  200. /* integrate with iommu base devid (0000), assume one iommu */
  201. perf_iommu->max_banks =
  202. amd_iommu_pc_get_max_banks(IOMMU_BASE_DEVID);
  203. perf_iommu->max_counters =
  204. amd_iommu_pc_get_max_counters(IOMMU_BASE_DEVID);
  205. if ((perf_iommu->max_banks == 0) || (perf_iommu->max_counters == 0))
  206. return -EINVAL;
  207. /* update the hw_perf_event struct with the iommu config data */
  208. hwc->config = config;
  209. hwc->extra_reg.config = config1;
  210. return 0;
  211. }
  212. static void perf_iommu_enable_event(struct perf_event *ev)
  213. {
  214. u8 csource = _GET_CSOURCE(ev);
  215. u16 devid = _GET_DEVID(ev);
  216. u64 reg = 0ULL;
  217. reg = csource;
  218. amd_iommu_pc_get_set_reg_val(devid,
  219. _GET_BANK(ev), _GET_CNTR(ev) ,
  220. IOMMU_PC_COUNTER_SRC_REG, &reg, true);
  221. reg = 0ULL | devid | (_GET_DEVID_MASK(ev) << 32);
  222. if (reg)
  223. reg |= (1UL << 31);
  224. amd_iommu_pc_get_set_reg_val(devid,
  225. _GET_BANK(ev), _GET_CNTR(ev) ,
  226. IOMMU_PC_DEVID_MATCH_REG, &reg, true);
  227. reg = 0ULL | _GET_PASID(ev) | (_GET_PASID_MASK(ev) << 32);
  228. if (reg)
  229. reg |= (1UL << 31);
  230. amd_iommu_pc_get_set_reg_val(devid,
  231. _GET_BANK(ev), _GET_CNTR(ev) ,
  232. IOMMU_PC_PASID_MATCH_REG, &reg, true);
  233. reg = 0ULL | _GET_DOMID(ev) | (_GET_DOMID_MASK(ev) << 32);
  234. if (reg)
  235. reg |= (1UL << 31);
  236. amd_iommu_pc_get_set_reg_val(devid,
  237. _GET_BANK(ev), _GET_CNTR(ev) ,
  238. IOMMU_PC_DOMID_MATCH_REG, &reg, true);
  239. }
  240. static void perf_iommu_disable_event(struct perf_event *event)
  241. {
  242. u64 reg = 0ULL;
  243. amd_iommu_pc_get_set_reg_val(_GET_DEVID(event),
  244. _GET_BANK(event), _GET_CNTR(event),
  245. IOMMU_PC_COUNTER_SRC_REG, &reg, true);
  246. }
  247. static void perf_iommu_start(struct perf_event *event, int flags)
  248. {
  249. struct hw_perf_event *hwc = &event->hw;
  250. pr_debug("perf: amd_iommu:perf_iommu_start\n");
  251. if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
  252. return;
  253. WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
  254. hwc->state = 0;
  255. if (flags & PERF_EF_RELOAD) {
  256. u64 prev_raw_count = local64_read(&hwc->prev_count);
  257. amd_iommu_pc_get_set_reg_val(_GET_DEVID(event),
  258. _GET_BANK(event), _GET_CNTR(event),
  259. IOMMU_PC_COUNTER_REG, &prev_raw_count, true);
  260. }
  261. perf_iommu_enable_event(event);
  262. perf_event_update_userpage(event);
  263. }
  264. static void perf_iommu_read(struct perf_event *event)
  265. {
  266. u64 count = 0ULL;
  267. u64 prev_raw_count = 0ULL;
  268. u64 delta = 0ULL;
  269. struct hw_perf_event *hwc = &event->hw;
  270. pr_debug("perf: amd_iommu:perf_iommu_read\n");
  271. amd_iommu_pc_get_set_reg_val(_GET_DEVID(event),
  272. _GET_BANK(event), _GET_CNTR(event),
  273. IOMMU_PC_COUNTER_REG, &count, false);
  274. /* IOMMU pc counter register is only 48 bits */
  275. count &= 0xFFFFFFFFFFFFULL;
  276. prev_raw_count = local64_read(&hwc->prev_count);
  277. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  278. count) != prev_raw_count)
  279. return;
  280. /* Handling 48-bit counter overflowing */
  281. delta = (count << COUNTER_SHIFT) - (prev_raw_count << COUNTER_SHIFT);
  282. delta >>= COUNTER_SHIFT;
  283. local64_add(delta, &event->count);
  284. }
  285. static void perf_iommu_stop(struct perf_event *event, int flags)
  286. {
  287. struct hw_perf_event *hwc = &event->hw;
  288. u64 config;
  289. pr_debug("perf: amd_iommu:perf_iommu_stop\n");
  290. if (hwc->state & PERF_HES_UPTODATE)
  291. return;
  292. perf_iommu_disable_event(event);
  293. WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
  294. hwc->state |= PERF_HES_STOPPED;
  295. if (hwc->state & PERF_HES_UPTODATE)
  296. return;
  297. config = hwc->config;
  298. perf_iommu_read(event);
  299. hwc->state |= PERF_HES_UPTODATE;
  300. }
  301. static int perf_iommu_add(struct perf_event *event, int flags)
  302. {
  303. int retval;
  304. struct perf_amd_iommu *perf_iommu =
  305. container_of(event->pmu, struct perf_amd_iommu, pmu);
  306. pr_debug("perf: amd_iommu:perf_iommu_add\n");
  307. event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  308. /* request an iommu bank/counter */
  309. retval = get_next_avail_iommu_bnk_cntr(perf_iommu);
  310. if (retval != -ENOSPC)
  311. event->hw.extra_reg.reg = (u16)retval;
  312. else
  313. return retval;
  314. if (flags & PERF_EF_START)
  315. perf_iommu_start(event, PERF_EF_RELOAD);
  316. return 0;
  317. }
  318. static void perf_iommu_del(struct perf_event *event, int flags)
  319. {
  320. struct perf_amd_iommu *perf_iommu =
  321. container_of(event->pmu, struct perf_amd_iommu, pmu);
  322. pr_debug("perf: amd_iommu:perf_iommu_del\n");
  323. perf_iommu_stop(event, PERF_EF_UPDATE);
  324. /* clear the assigned iommu bank/counter */
  325. clear_avail_iommu_bnk_cntr(perf_iommu,
  326. _GET_BANK(event),
  327. _GET_CNTR(event));
  328. perf_event_update_userpage(event);
  329. }
  330. static __init int _init_events_attrs(struct perf_amd_iommu *perf_iommu)
  331. {
  332. struct attribute **attrs;
  333. struct attribute_group *attr_group;
  334. int i = 0, j;
  335. while (amd_iommu_v2_event_descs[i].attr.attr.name)
  336. i++;
  337. attr_group = kzalloc(sizeof(struct attribute *)
  338. * (i + 1) + sizeof(*attr_group), GFP_KERNEL);
  339. if (!attr_group)
  340. return -ENOMEM;
  341. attrs = (struct attribute **)(attr_group + 1);
  342. for (j = 0; j < i; j++)
  343. attrs[j] = &amd_iommu_v2_event_descs[j].attr.attr;
  344. attr_group->name = "events";
  345. attr_group->attrs = attrs;
  346. perf_iommu->events_group = attr_group;
  347. return 0;
  348. }
  349. static __init void amd_iommu_pc_exit(void)
  350. {
  351. if (__perf_iommu.events_group != NULL) {
  352. kfree(__perf_iommu.events_group);
  353. __perf_iommu.events_group = NULL;
  354. }
  355. }
  356. static __init int _init_perf_amd_iommu(
  357. struct perf_amd_iommu *perf_iommu, char *name)
  358. {
  359. int ret;
  360. raw_spin_lock_init(&perf_iommu->lock);
  361. /* Init format attributes */
  362. perf_iommu->format_group = &amd_iommu_format_group;
  363. /* Init cpumask attributes to only core 0 */
  364. cpumask_set_cpu(0, &iommu_cpumask);
  365. perf_iommu->cpumask_group = &amd_iommu_cpumask_group;
  366. /* Init events attributes */
  367. if (_init_events_attrs(perf_iommu) != 0)
  368. pr_err("perf: amd_iommu: Only support raw events.\n");
  369. /* Init null attributes */
  370. perf_iommu->null_group = NULL;
  371. perf_iommu->pmu.attr_groups = perf_iommu->attr_groups;
  372. ret = perf_pmu_register(&perf_iommu->pmu, name, -1);
  373. if (ret) {
  374. pr_err("perf: amd_iommu: Failed to initialized.\n");
  375. amd_iommu_pc_exit();
  376. } else {
  377. pr_info("perf: amd_iommu: Detected. (%d banks, %d counters/bank)\n",
  378. amd_iommu_pc_get_max_banks(IOMMU_BASE_DEVID),
  379. amd_iommu_pc_get_max_counters(IOMMU_BASE_DEVID));
  380. }
  381. return ret;
  382. }
  383. static struct perf_amd_iommu __perf_iommu = {
  384. .pmu = {
  385. .event_init = perf_iommu_event_init,
  386. .add = perf_iommu_add,
  387. .del = perf_iommu_del,
  388. .start = perf_iommu_start,
  389. .stop = perf_iommu_stop,
  390. .read = perf_iommu_read,
  391. },
  392. .max_banks = 0x00,
  393. .max_counters = 0x00,
  394. .cntr_assign_mask = 0ULL,
  395. .format_group = NULL,
  396. .cpumask_group = NULL,
  397. .events_group = NULL,
  398. .null_group = NULL,
  399. };
  400. static __init int amd_iommu_pc_init(void)
  401. {
  402. /* Make sure the IOMMU PC resource is available */
  403. if (!amd_iommu_pc_supported())
  404. return -ENODEV;
  405. _init_perf_amd_iommu(&__perf_iommu, "amd_iommu");
  406. return 0;
  407. }
  408. device_initcall(amd_iommu_pc_init);