perf_event_mipsxx.c 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684
  1. /*
  2. * Linux performance counter support for MIPS.
  3. *
  4. * Copyright (C) 2010 MIPS Technologies, Inc.
  5. * Copyright (C) 2011 Cavium Networks, Inc.
  6. * Author: Deng-Cheng Zhu
  7. *
  8. * This code is based on the implementation for ARM, which is in turn
  9. * based on the sparc64 perf event code and the x86 code. Performance
  10. * counter access is based on the MIPS Oprofile code. And the callchain
  11. * support references the code of MIPS stacktrace.c.
  12. *
  13. * This program is free software; you can redistribute it and/or modify
  14. * it under the terms of the GNU General Public License version 2 as
  15. * published by the Free Software Foundation.
  16. */
  17. #include <linux/cpumask.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/smp.h>
  20. #include <linux/kernel.h>
  21. #include <linux/perf_event.h>
  22. #include <linux/uaccess.h>
  23. #include <asm/irq.h>
  24. #include <asm/irq_regs.h>
  25. #include <asm/stacktrace.h>
  26. #include <asm/time.h> /* For perf_irq */
  27. #define MIPS_MAX_HWEVENTS 4
  28. #define MIPS_TCS_PER_COUNTER 2
  29. #define MIPS_CPUID_TO_COUNTER_MASK (MIPS_TCS_PER_COUNTER - 1)
  30. struct cpu_hw_events {
  31. /* Array of events on this cpu. */
  32. struct perf_event *events[MIPS_MAX_HWEVENTS];
  33. /*
  34. * Set the bit (indexed by the counter number) when the counter
  35. * is used for an event.
  36. */
  37. unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
  38. /*
  39. * Software copy of the control register for each performance counter.
  40. * MIPS CPUs vary in performance counters. They use this differently,
  41. * and even may not use it.
  42. */
  43. unsigned int saved_ctrl[MIPS_MAX_HWEVENTS];
  44. };
  45. DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  46. .saved_ctrl = {0},
  47. };
  48. /* The description of MIPS performance events. */
  49. struct mips_perf_event {
  50. unsigned int event_id;
  51. /*
  52. * MIPS performance counters are indexed starting from 0.
  53. * CNTR_EVEN indicates the indexes of the counters to be used are
  54. * even numbers.
  55. */
  56. unsigned int cntr_mask;
  57. #define CNTR_EVEN 0x55555555
  58. #define CNTR_ODD 0xaaaaaaaa
  59. #define CNTR_ALL 0xffffffff
  60. #ifdef CONFIG_MIPS_MT_SMP
  61. enum {
  62. T = 0,
  63. V = 1,
  64. P = 2,
  65. } range;
  66. #else
  67. #define T
  68. #define V
  69. #define P
  70. #endif
  71. };
  72. static struct mips_perf_event raw_event;
  73. static DEFINE_MUTEX(raw_event_mutex);
  74. #define C(x) PERF_COUNT_HW_CACHE_##x
  75. struct mips_pmu {
  76. u64 max_period;
  77. u64 valid_count;
  78. u64 overflow;
  79. const char *name;
  80. int irq;
  81. u64 (*read_counter)(unsigned int idx);
  82. void (*write_counter)(unsigned int idx, u64 val);
  83. const struct mips_perf_event *(*map_raw_event)(u64 config);
  84. const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
  85. const struct mips_perf_event (*cache_event_map)
  86. [PERF_COUNT_HW_CACHE_MAX]
  87. [PERF_COUNT_HW_CACHE_OP_MAX]
  88. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  89. unsigned int num_counters;
  90. };
  91. static struct mips_pmu mipspmu;
  92. #define M_CONFIG1_PC (1 << 4)
  93. #define M_PERFCTL_EXL (1 << 0)
  94. #define M_PERFCTL_KERNEL (1 << 1)
  95. #define M_PERFCTL_SUPERVISOR (1 << 2)
  96. #define M_PERFCTL_USER (1 << 3)
  97. #define M_PERFCTL_INTERRUPT_ENABLE (1 << 4)
  98. #define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5)
  99. #define M_PERFCTL_VPEID(vpe) ((vpe) << 16)
  100. #ifdef CONFIG_CPU_BMIPS5000
  101. #define M_PERFCTL_MT_EN(filter) 0
  102. #else /* !CONFIG_CPU_BMIPS5000 */
  103. #define M_PERFCTL_MT_EN(filter) ((filter) << 20)
  104. #endif /* CONFIG_CPU_BMIPS5000 */
  105. #define M_TC_EN_ALL M_PERFCTL_MT_EN(0)
  106. #define M_TC_EN_VPE M_PERFCTL_MT_EN(1)
  107. #define M_TC_EN_TC M_PERFCTL_MT_EN(2)
  108. #define M_PERFCTL_TCID(tcid) ((tcid) << 22)
  109. #define M_PERFCTL_WIDE (1 << 30)
  110. #define M_PERFCTL_MORE (1 << 31)
  111. #define M_PERFCTL_TC (1 << 30)
  112. #define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \
  113. M_PERFCTL_KERNEL | \
  114. M_PERFCTL_USER | \
  115. M_PERFCTL_SUPERVISOR | \
  116. M_PERFCTL_INTERRUPT_ENABLE)
  117. #ifdef CONFIG_MIPS_MT_SMP
  118. #define M_PERFCTL_CONFIG_MASK 0x3fff801f
  119. #else
  120. #define M_PERFCTL_CONFIG_MASK 0x1f
  121. #endif
  122. #define M_PERFCTL_EVENT_MASK 0xfe0
  123. #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
  124. static int cpu_has_mipsmt_pertccounters;
  125. static DEFINE_RWLOCK(pmuint_rwlock);
  126. #if defined(CONFIG_CPU_BMIPS5000)
  127. #define vpe_id() (cpu_has_mipsmt_pertccounters ? \
  128. 0 : (smp_processor_id() & MIPS_CPUID_TO_COUNTER_MASK))
  129. #else
  130. /*
  131. * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because
  132. * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs.
  133. */
  134. #define vpe_id() (cpu_has_mipsmt_pertccounters ? \
  135. 0 : smp_processor_id())
  136. #endif
  137. /* Copied from op_model_mipsxx.c */
  138. static unsigned int vpe_shift(void)
  139. {
  140. if (num_possible_cpus() > 1)
  141. return 1;
  142. return 0;
  143. }
  144. static unsigned int counters_total_to_per_cpu(unsigned int counters)
  145. {
  146. return counters >> vpe_shift();
  147. }
  148. #else /* !CONFIG_MIPS_PERF_SHARED_TC_COUNTERS */
  149. #define vpe_id() 0
  150. #endif /* CONFIG_MIPS_PERF_SHARED_TC_COUNTERS */
  151. static void resume_local_counters(void);
  152. static void pause_local_counters(void);
  153. static irqreturn_t mipsxx_pmu_handle_irq(int, void *);
  154. static int mipsxx_pmu_handle_shared_irq(void);
  155. static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx)
  156. {
  157. if (vpe_id() == 1)
  158. idx = (idx + 2) & 3;
  159. return idx;
  160. }
  161. static u64 mipsxx_pmu_read_counter(unsigned int idx)
  162. {
  163. idx = mipsxx_pmu_swizzle_perf_idx(idx);
  164. switch (idx) {
  165. case 0:
  166. /*
  167. * The counters are unsigned, we must cast to truncate
  168. * off the high bits.
  169. */
  170. return (u32)read_c0_perfcntr0();
  171. case 1:
  172. return (u32)read_c0_perfcntr1();
  173. case 2:
  174. return (u32)read_c0_perfcntr2();
  175. case 3:
  176. return (u32)read_c0_perfcntr3();
  177. default:
  178. WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
  179. return 0;
  180. }
  181. }
  182. static u64 mipsxx_pmu_read_counter_64(unsigned int idx)
  183. {
  184. idx = mipsxx_pmu_swizzle_perf_idx(idx);
  185. switch (idx) {
  186. case 0:
  187. return read_c0_perfcntr0_64();
  188. case 1:
  189. return read_c0_perfcntr1_64();
  190. case 2:
  191. return read_c0_perfcntr2_64();
  192. case 3:
  193. return read_c0_perfcntr3_64();
  194. default:
  195. WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
  196. return 0;
  197. }
  198. }
  199. static void mipsxx_pmu_write_counter(unsigned int idx, u64 val)
  200. {
  201. idx = mipsxx_pmu_swizzle_perf_idx(idx);
  202. switch (idx) {
  203. case 0:
  204. write_c0_perfcntr0(val);
  205. return;
  206. case 1:
  207. write_c0_perfcntr1(val);
  208. return;
  209. case 2:
  210. write_c0_perfcntr2(val);
  211. return;
  212. case 3:
  213. write_c0_perfcntr3(val);
  214. return;
  215. }
  216. }
  217. static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val)
  218. {
  219. idx = mipsxx_pmu_swizzle_perf_idx(idx);
  220. switch (idx) {
  221. case 0:
  222. write_c0_perfcntr0_64(val);
  223. return;
  224. case 1:
  225. write_c0_perfcntr1_64(val);
  226. return;
  227. case 2:
  228. write_c0_perfcntr2_64(val);
  229. return;
  230. case 3:
  231. write_c0_perfcntr3_64(val);
  232. return;
  233. }
  234. }
  235. static unsigned int mipsxx_pmu_read_control(unsigned int idx)
  236. {
  237. idx = mipsxx_pmu_swizzle_perf_idx(idx);
  238. switch (idx) {
  239. case 0:
  240. return read_c0_perfctrl0();
  241. case 1:
  242. return read_c0_perfctrl1();
  243. case 2:
  244. return read_c0_perfctrl2();
  245. case 3:
  246. return read_c0_perfctrl3();
  247. default:
  248. WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
  249. return 0;
  250. }
  251. }
  252. static void mipsxx_pmu_write_control(unsigned int idx, unsigned int val)
  253. {
  254. idx = mipsxx_pmu_swizzle_perf_idx(idx);
  255. switch (idx) {
  256. case 0:
  257. write_c0_perfctrl0(val);
  258. return;
  259. case 1:
  260. write_c0_perfctrl1(val);
  261. return;
  262. case 2:
  263. write_c0_perfctrl2(val);
  264. return;
  265. case 3:
  266. write_c0_perfctrl3(val);
  267. return;
  268. }
  269. }
  270. static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc,
  271. struct hw_perf_event *hwc)
  272. {
  273. int i;
  274. /*
  275. * We only need to care the counter mask. The range has been
  276. * checked definitely.
  277. */
  278. unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff;
  279. for (i = mipspmu.num_counters - 1; i >= 0; i--) {
  280. /*
  281. * Note that some MIPS perf events can be counted by both
  282. * even and odd counters, wheresas many other are only by
  283. * even _or_ odd counters. This introduces an issue that
  284. * when the former kind of event takes the counter the
  285. * latter kind of event wants to use, then the "counter
  286. * allocation" for the latter event will fail. In fact if
  287. * they can be dynamically swapped, they both feel happy.
  288. * But here we leave this issue alone for now.
  289. */
  290. if (test_bit(i, &cntr_mask) &&
  291. !test_and_set_bit(i, cpuc->used_mask))
  292. return i;
  293. }
  294. return -EAGAIN;
  295. }
  296. static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx)
  297. {
  298. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  299. WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
  300. cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |
  301. (evt->config_base & M_PERFCTL_CONFIG_MASK) |
  302. /* Make sure interrupt enabled. */
  303. M_PERFCTL_INTERRUPT_ENABLE;
  304. if (IS_ENABLED(CONFIG_CPU_BMIPS5000))
  305. /* enable the counter for the calling thread */
  306. cpuc->saved_ctrl[idx] |=
  307. (1 << (12 + vpe_id())) | M_PERFCTL_TC;
  308. /*
  309. * We do not actually let the counter run. Leave it until start().
  310. */
  311. }
  312. static void mipsxx_pmu_disable_event(int idx)
  313. {
  314. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  315. unsigned long flags;
  316. WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
  317. local_irq_save(flags);
  318. cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) &
  319. ~M_PERFCTL_COUNT_EVENT_WHENEVER;
  320. mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]);
  321. local_irq_restore(flags);
  322. }
  323. static int mipspmu_event_set_period(struct perf_event *event,
  324. struct hw_perf_event *hwc,
  325. int idx)
  326. {
  327. u64 left = local64_read(&hwc->period_left);
  328. u64 period = hwc->sample_period;
  329. int ret = 0;
  330. if (unlikely((left + period) & (1ULL << 63))) {
  331. /* left underflowed by more than period. */
  332. left = period;
  333. local64_set(&hwc->period_left, left);
  334. hwc->last_period = period;
  335. ret = 1;
  336. } else if (unlikely((left + period) <= period)) {
  337. /* left underflowed by less than period. */
  338. left += period;
  339. local64_set(&hwc->period_left, left);
  340. hwc->last_period = period;
  341. ret = 1;
  342. }
  343. if (left > mipspmu.max_period) {
  344. left = mipspmu.max_period;
  345. local64_set(&hwc->period_left, left);
  346. }
  347. local64_set(&hwc->prev_count, mipspmu.overflow - left);
  348. mipspmu.write_counter(idx, mipspmu.overflow - left);
  349. perf_event_update_userpage(event);
  350. return ret;
  351. }
  352. static void mipspmu_event_update(struct perf_event *event,
  353. struct hw_perf_event *hwc,
  354. int idx)
  355. {
  356. u64 prev_raw_count, new_raw_count;
  357. u64 delta;
  358. again:
  359. prev_raw_count = local64_read(&hwc->prev_count);
  360. new_raw_count = mipspmu.read_counter(idx);
  361. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  362. new_raw_count) != prev_raw_count)
  363. goto again;
  364. delta = new_raw_count - prev_raw_count;
  365. local64_add(delta, &event->count);
  366. local64_sub(delta, &hwc->period_left);
  367. }
  368. static void mipspmu_start(struct perf_event *event, int flags)
  369. {
  370. struct hw_perf_event *hwc = &event->hw;
  371. if (flags & PERF_EF_RELOAD)
  372. WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
  373. hwc->state = 0;
  374. /* Set the period for the event. */
  375. mipspmu_event_set_period(event, hwc, hwc->idx);
  376. /* Enable the event. */
  377. mipsxx_pmu_enable_event(hwc, hwc->idx);
  378. }
  379. static void mipspmu_stop(struct perf_event *event, int flags)
  380. {
  381. struct hw_perf_event *hwc = &event->hw;
  382. if (!(hwc->state & PERF_HES_STOPPED)) {
  383. /* We are working on a local event. */
  384. mipsxx_pmu_disable_event(hwc->idx);
  385. barrier();
  386. mipspmu_event_update(event, hwc, hwc->idx);
  387. hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  388. }
  389. }
  390. static int mipspmu_add(struct perf_event *event, int flags)
  391. {
  392. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  393. struct hw_perf_event *hwc = &event->hw;
  394. int idx;
  395. int err = 0;
  396. perf_pmu_disable(event->pmu);
  397. /* To look for a free counter for this event. */
  398. idx = mipsxx_pmu_alloc_counter(cpuc, hwc);
  399. if (idx < 0) {
  400. err = idx;
  401. goto out;
  402. }
  403. /*
  404. * If there is an event in the counter we are going to use then
  405. * make sure it is disabled.
  406. */
  407. event->hw.idx = idx;
  408. mipsxx_pmu_disable_event(idx);
  409. cpuc->events[idx] = event;
  410. hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  411. if (flags & PERF_EF_START)
  412. mipspmu_start(event, PERF_EF_RELOAD);
  413. /* Propagate our changes to the userspace mapping. */
  414. perf_event_update_userpage(event);
  415. out:
  416. perf_pmu_enable(event->pmu);
  417. return err;
  418. }
  419. static void mipspmu_del(struct perf_event *event, int flags)
  420. {
  421. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  422. struct hw_perf_event *hwc = &event->hw;
  423. int idx = hwc->idx;
  424. WARN_ON(idx < 0 || idx >= mipspmu.num_counters);
  425. mipspmu_stop(event, PERF_EF_UPDATE);
  426. cpuc->events[idx] = NULL;
  427. clear_bit(idx, cpuc->used_mask);
  428. perf_event_update_userpage(event);
  429. }
  430. static void mipspmu_read(struct perf_event *event)
  431. {
  432. struct hw_perf_event *hwc = &event->hw;
  433. /* Don't read disabled counters! */
  434. if (hwc->idx < 0)
  435. return;
  436. mipspmu_event_update(event, hwc, hwc->idx);
  437. }
  438. static void mipspmu_enable(struct pmu *pmu)
  439. {
  440. #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
  441. write_unlock(&pmuint_rwlock);
  442. #endif
  443. resume_local_counters();
  444. }
  445. /*
  446. * MIPS performance counters can be per-TC. The control registers can
  447. * not be directly accessed accross CPUs. Hence if we want to do global
  448. * control, we need cross CPU calls. on_each_cpu() can help us, but we
  449. * can not make sure this function is called with interrupts enabled. So
  450. * here we pause local counters and then grab a rwlock and leave the
  451. * counters on other CPUs alone. If any counter interrupt raises while
  452. * we own the write lock, simply pause local counters on that CPU and
  453. * spin in the handler. Also we know we won't be switched to another
  454. * CPU after pausing local counters and before grabbing the lock.
  455. */
  456. static void mipspmu_disable(struct pmu *pmu)
  457. {
  458. pause_local_counters();
  459. #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
  460. write_lock(&pmuint_rwlock);
  461. #endif
  462. }
  463. static atomic_t active_events = ATOMIC_INIT(0);
  464. static DEFINE_MUTEX(pmu_reserve_mutex);
  465. static int (*save_perf_irq)(void);
  466. static int mipspmu_get_irq(void)
  467. {
  468. int err;
  469. if (mipspmu.irq >= 0) {
  470. /* Request my own irq handler. */
  471. err = request_irq(mipspmu.irq, mipsxx_pmu_handle_irq,
  472. IRQF_PERCPU | IRQF_NOBALANCING,
  473. "mips_perf_pmu", NULL);
  474. if (err) {
  475. pr_warning("Unable to request IRQ%d for MIPS "
  476. "performance counters!\n", mipspmu.irq);
  477. }
  478. } else if (cp0_perfcount_irq < 0) {
  479. /*
  480. * We are sharing the irq number with the timer interrupt.
  481. */
  482. save_perf_irq = perf_irq;
  483. perf_irq = mipsxx_pmu_handle_shared_irq;
  484. err = 0;
  485. } else {
  486. pr_warning("The platform hasn't properly defined its "
  487. "interrupt controller.\n");
  488. err = -ENOENT;
  489. }
  490. return err;
  491. }
  492. static void mipspmu_free_irq(void)
  493. {
  494. if (mipspmu.irq >= 0)
  495. free_irq(mipspmu.irq, NULL);
  496. else if (cp0_perfcount_irq < 0)
  497. perf_irq = save_perf_irq;
  498. }
  499. /*
  500. * mipsxx/rm9000/loongson2 have different performance counters, they have
  501. * specific low-level init routines.
  502. */
  503. static void reset_counters(void *arg);
  504. static int __hw_perf_event_init(struct perf_event *event);
  505. static void hw_perf_event_destroy(struct perf_event *event)
  506. {
  507. if (atomic_dec_and_mutex_lock(&active_events,
  508. &pmu_reserve_mutex)) {
  509. /*
  510. * We must not call the destroy function with interrupts
  511. * disabled.
  512. */
  513. on_each_cpu(reset_counters,
  514. (void *)(long)mipspmu.num_counters, 1);
  515. mipspmu_free_irq();
  516. mutex_unlock(&pmu_reserve_mutex);
  517. }
  518. }
  519. static int mipspmu_event_init(struct perf_event *event)
  520. {
  521. int err = 0;
  522. /* does not support taken branch sampling */
  523. if (has_branch_stack(event))
  524. return -EOPNOTSUPP;
  525. switch (event->attr.type) {
  526. case PERF_TYPE_RAW:
  527. case PERF_TYPE_HARDWARE:
  528. case PERF_TYPE_HW_CACHE:
  529. break;
  530. default:
  531. return -ENOENT;
  532. }
  533. if (event->cpu >= nr_cpumask_bits ||
  534. (event->cpu >= 0 && !cpu_online(event->cpu)))
  535. return -ENODEV;
  536. if (!atomic_inc_not_zero(&active_events)) {
  537. mutex_lock(&pmu_reserve_mutex);
  538. if (atomic_read(&active_events) == 0)
  539. err = mipspmu_get_irq();
  540. if (!err)
  541. atomic_inc(&active_events);
  542. mutex_unlock(&pmu_reserve_mutex);
  543. }
  544. if (err)
  545. return err;
  546. return __hw_perf_event_init(event);
  547. }
  548. static struct pmu pmu = {
  549. .pmu_enable = mipspmu_enable,
  550. .pmu_disable = mipspmu_disable,
  551. .event_init = mipspmu_event_init,
  552. .add = mipspmu_add,
  553. .del = mipspmu_del,
  554. .start = mipspmu_start,
  555. .stop = mipspmu_stop,
  556. .read = mipspmu_read,
  557. };
  558. static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev)
  559. {
  560. /*
  561. * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
  562. * event_id.
  563. */
  564. #ifdef CONFIG_MIPS_MT_SMP
  565. return ((unsigned int)pev->range << 24) |
  566. (pev->cntr_mask & 0xffff00) |
  567. (pev->event_id & 0xff);
  568. #else
  569. return (pev->cntr_mask & 0xffff00) |
  570. (pev->event_id & 0xff);
  571. #endif
  572. }
  573. static const struct mips_perf_event *mipspmu_map_general_event(int idx)
  574. {
  575. if ((*mipspmu.general_event_map)[idx].cntr_mask == 0)
  576. return ERR_PTR(-EOPNOTSUPP);
  577. return &(*mipspmu.general_event_map)[idx];
  578. }
  579. static const struct mips_perf_event *mipspmu_map_cache_event(u64 config)
  580. {
  581. unsigned int cache_type, cache_op, cache_result;
  582. const struct mips_perf_event *pev;
  583. cache_type = (config >> 0) & 0xff;
  584. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  585. return ERR_PTR(-EINVAL);
  586. cache_op = (config >> 8) & 0xff;
  587. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  588. return ERR_PTR(-EINVAL);
  589. cache_result = (config >> 16) & 0xff;
  590. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  591. return ERR_PTR(-EINVAL);
  592. pev = &((*mipspmu.cache_event_map)
  593. [cache_type]
  594. [cache_op]
  595. [cache_result]);
  596. if (pev->cntr_mask == 0)
  597. return ERR_PTR(-EOPNOTSUPP);
  598. return pev;
  599. }
  600. static int validate_group(struct perf_event *event)
  601. {
  602. struct perf_event *sibling, *leader = event->group_leader;
  603. struct cpu_hw_events fake_cpuc;
  604. memset(&fake_cpuc, 0, sizeof(fake_cpuc));
  605. if (mipsxx_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0)
  606. return -EINVAL;
  607. list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
  608. if (mipsxx_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0)
  609. return -EINVAL;
  610. }
  611. if (mipsxx_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0)
  612. return -EINVAL;
  613. return 0;
  614. }
  615. /* This is needed by specific irq handlers in perf_event_*.c */
  616. static void handle_associated_event(struct cpu_hw_events *cpuc,
  617. int idx, struct perf_sample_data *data,
  618. struct pt_regs *regs)
  619. {
  620. struct perf_event *event = cpuc->events[idx];
  621. struct hw_perf_event *hwc = &event->hw;
  622. mipspmu_event_update(event, hwc, idx);
  623. data->period = event->hw.last_period;
  624. if (!mipspmu_event_set_period(event, hwc, idx))
  625. return;
  626. if (perf_event_overflow(event, data, regs))
  627. mipsxx_pmu_disable_event(idx);
  628. }
  629. static int __n_counters(void)
  630. {
  631. if (!(read_c0_config1() & M_CONFIG1_PC))
  632. return 0;
  633. if (!(read_c0_perfctrl0() & M_PERFCTL_MORE))
  634. return 1;
  635. if (!(read_c0_perfctrl1() & M_PERFCTL_MORE))
  636. return 2;
  637. if (!(read_c0_perfctrl2() & M_PERFCTL_MORE))
  638. return 3;
  639. return 4;
  640. }
  641. static int n_counters(void)
  642. {
  643. int counters;
  644. switch (current_cpu_type()) {
  645. case CPU_R10000:
  646. counters = 2;
  647. break;
  648. case CPU_R12000:
  649. case CPU_R14000:
  650. counters = 4;
  651. break;
  652. default:
  653. counters = __n_counters();
  654. }
  655. return counters;
  656. }
  657. static void reset_counters(void *arg)
  658. {
  659. int counters = (int)(long)arg;
  660. switch (counters) {
  661. case 4:
  662. mipsxx_pmu_write_control(3, 0);
  663. mipspmu.write_counter(3, 0);
  664. case 3:
  665. mipsxx_pmu_write_control(2, 0);
  666. mipspmu.write_counter(2, 0);
  667. case 2:
  668. mipsxx_pmu_write_control(1, 0);
  669. mipspmu.write_counter(1, 0);
  670. case 1:
  671. mipsxx_pmu_write_control(0, 0);
  672. mipspmu.write_counter(0, 0);
  673. }
  674. }
  675. /* 24K/34K/1004K cores can share the same event map. */
  676. static const struct mips_perf_event mipsxxcore_event_map
  677. [PERF_COUNT_HW_MAX] = {
  678. [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
  679. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
  680. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T },
  681. [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T },
  682. };
  683. /* 74K core has different branch event code. */
  684. static const struct mips_perf_event mipsxx74Kcore_event_map
  685. [PERF_COUNT_HW_MAX] = {
  686. [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
  687. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
  688. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T },
  689. [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T },
  690. };
  691. static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = {
  692. [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL },
  693. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL },
  694. [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x2b, CNTR_ALL },
  695. [PERF_COUNT_HW_CACHE_MISSES] = { 0x2e, CNTR_ALL },
  696. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x08, CNTR_ALL },
  697. [PERF_COUNT_HW_BRANCH_MISSES] = { 0x09, CNTR_ALL },
  698. [PERF_COUNT_HW_BUS_CYCLES] = { 0x25, CNTR_ALL },
  699. };
  700. static const struct mips_perf_event bmips5000_event_map
  701. [PERF_COUNT_HW_MAX] = {
  702. [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, T },
  703. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
  704. [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T },
  705. };
  706. static const struct mips_perf_event xlp_event_map[PERF_COUNT_HW_MAX] = {
  707. [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL },
  708. [PERF_COUNT_HW_INSTRUCTIONS] = { 0x18, CNTR_ALL }, /* PAPI_TOT_INS */
  709. [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */
  710. [PERF_COUNT_HW_CACHE_MISSES] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */
  711. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x1b, CNTR_ALL }, /* PAPI_BR_CN */
  712. [PERF_COUNT_HW_BRANCH_MISSES] = { 0x1c, CNTR_ALL }, /* PAPI_BR_MSP */
  713. [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID },
  714. };
  715. /* 24K/34K/1004K cores can share the same cache event map. */
  716. static const struct mips_perf_event mipsxxcore_cache_map
  717. [PERF_COUNT_HW_CACHE_MAX]
  718. [PERF_COUNT_HW_CACHE_OP_MAX]
  719. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  720. [C(L1D)] = {
  721. /*
  722. * Like some other architectures (e.g. ARM), the performance
  723. * counters don't differentiate between read and write
  724. * accesses/misses, so this isn't strictly correct, but it's the
  725. * best we can do. Writes and reads get combined.
  726. */
  727. [C(OP_READ)] = {
  728. [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T },
  729. [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T },
  730. },
  731. [C(OP_WRITE)] = {
  732. [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T },
  733. [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T },
  734. },
  735. },
  736. [C(L1I)] = {
  737. [C(OP_READ)] = {
  738. [C(RESULT_ACCESS)] = { 0x09, CNTR_EVEN, T },
  739. [C(RESULT_MISS)] = { 0x09, CNTR_ODD, T },
  740. },
  741. [C(OP_WRITE)] = {
  742. [C(RESULT_ACCESS)] = { 0x09, CNTR_EVEN, T },
  743. [C(RESULT_MISS)] = { 0x09, CNTR_ODD, T },
  744. },
  745. [C(OP_PREFETCH)] = {
  746. [C(RESULT_ACCESS)] = { 0x14, CNTR_EVEN, T },
  747. /*
  748. * Note that MIPS has only "hit" events countable for
  749. * the prefetch operation.
  750. */
  751. },
  752. },
  753. [C(LL)] = {
  754. [C(OP_READ)] = {
  755. [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P },
  756. [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P },
  757. },
  758. [C(OP_WRITE)] = {
  759. [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P },
  760. [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P },
  761. },
  762. },
  763. [C(DTLB)] = {
  764. [C(OP_READ)] = {
  765. [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T },
  766. [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T },
  767. },
  768. [C(OP_WRITE)] = {
  769. [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T },
  770. [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T },
  771. },
  772. },
  773. [C(ITLB)] = {
  774. [C(OP_READ)] = {
  775. [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T },
  776. [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T },
  777. },
  778. [C(OP_WRITE)] = {
  779. [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T },
  780. [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T },
  781. },
  782. },
  783. [C(BPU)] = {
  784. /* Using the same code for *HW_BRANCH* */
  785. [C(OP_READ)] = {
  786. [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T },
  787. [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T },
  788. },
  789. [C(OP_WRITE)] = {
  790. [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T },
  791. [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T },
  792. },
  793. },
  794. };
  795. /* 74K core has completely different cache event map. */
  796. static const struct mips_perf_event mipsxx74Kcore_cache_map
  797. [PERF_COUNT_HW_CACHE_MAX]
  798. [PERF_COUNT_HW_CACHE_OP_MAX]
  799. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  800. [C(L1D)] = {
  801. /*
  802. * Like some other architectures (e.g. ARM), the performance
  803. * counters don't differentiate between read and write
  804. * accesses/misses, so this isn't strictly correct, but it's the
  805. * best we can do. Writes and reads get combined.
  806. */
  807. [C(OP_READ)] = {
  808. [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T },
  809. [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T },
  810. },
  811. [C(OP_WRITE)] = {
  812. [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T },
  813. [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T },
  814. },
  815. },
  816. [C(L1I)] = {
  817. [C(OP_READ)] = {
  818. [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T },
  819. [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T },
  820. },
  821. [C(OP_WRITE)] = {
  822. [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T },
  823. [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T },
  824. },
  825. [C(OP_PREFETCH)] = {
  826. [C(RESULT_ACCESS)] = { 0x34, CNTR_EVEN, T },
  827. /*
  828. * Note that MIPS has only "hit" events countable for
  829. * the prefetch operation.
  830. */
  831. },
  832. },
  833. [C(LL)] = {
  834. [C(OP_READ)] = {
  835. [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P },
  836. [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P },
  837. },
  838. [C(OP_WRITE)] = {
  839. [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P },
  840. [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P },
  841. },
  842. },
  843. [C(ITLB)] = {
  844. [C(OP_READ)] = {
  845. [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T },
  846. [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T },
  847. },
  848. [C(OP_WRITE)] = {
  849. [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T },
  850. [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T },
  851. },
  852. },
  853. [C(BPU)] = {
  854. /* Using the same code for *HW_BRANCH* */
  855. [C(OP_READ)] = {
  856. [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T },
  857. [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T },
  858. },
  859. [C(OP_WRITE)] = {
  860. [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T },
  861. [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T },
  862. },
  863. },
  864. };
  865. /* BMIPS5000 */
  866. static const struct mips_perf_event bmips5000_cache_map
  867. [PERF_COUNT_HW_CACHE_MAX]
  868. [PERF_COUNT_HW_CACHE_OP_MAX]
  869. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  870. [C(L1D)] = {
  871. /*
  872. * Like some other architectures (e.g. ARM), the performance
  873. * counters don't differentiate between read and write
  874. * accesses/misses, so this isn't strictly correct, but it's the
  875. * best we can do. Writes and reads get combined.
  876. */
  877. [C(OP_READ)] = {
  878. [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T },
  879. [C(RESULT_MISS)] = { 12, CNTR_ODD, T },
  880. },
  881. [C(OP_WRITE)] = {
  882. [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T },
  883. [C(RESULT_MISS)] = { 12, CNTR_ODD, T },
  884. },
  885. },
  886. [C(L1I)] = {
  887. [C(OP_READ)] = {
  888. [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T },
  889. [C(RESULT_MISS)] = { 10, CNTR_ODD, T },
  890. },
  891. [C(OP_WRITE)] = {
  892. [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T },
  893. [C(RESULT_MISS)] = { 10, CNTR_ODD, T },
  894. },
  895. [C(OP_PREFETCH)] = {
  896. [C(RESULT_ACCESS)] = { 23, CNTR_EVEN, T },
  897. /*
  898. * Note that MIPS has only "hit" events countable for
  899. * the prefetch operation.
  900. */
  901. },
  902. },
  903. [C(LL)] = {
  904. [C(OP_READ)] = {
  905. [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P },
  906. [C(RESULT_MISS)] = { 28, CNTR_ODD, P },
  907. },
  908. [C(OP_WRITE)] = {
  909. [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P },
  910. [C(RESULT_MISS)] = { 28, CNTR_ODD, P },
  911. },
  912. },
  913. [C(BPU)] = {
  914. /* Using the same code for *HW_BRANCH* */
  915. [C(OP_READ)] = {
  916. [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T },
  917. },
  918. [C(OP_WRITE)] = {
  919. [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T },
  920. },
  921. },
  922. };
  923. static const struct mips_perf_event octeon_cache_map
  924. [PERF_COUNT_HW_CACHE_MAX]
  925. [PERF_COUNT_HW_CACHE_OP_MAX]
  926. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  927. [C(L1D)] = {
  928. [C(OP_READ)] = {
  929. [C(RESULT_ACCESS)] = { 0x2b, CNTR_ALL },
  930. [C(RESULT_MISS)] = { 0x2e, CNTR_ALL },
  931. },
  932. [C(OP_WRITE)] = {
  933. [C(RESULT_ACCESS)] = { 0x30, CNTR_ALL },
  934. },
  935. },
  936. [C(L1I)] = {
  937. [C(OP_READ)] = {
  938. [C(RESULT_ACCESS)] = { 0x18, CNTR_ALL },
  939. },
  940. [C(OP_PREFETCH)] = {
  941. [C(RESULT_ACCESS)] = { 0x19, CNTR_ALL },
  942. },
  943. },
  944. [C(DTLB)] = {
  945. /*
  946. * Only general DTLB misses are counted use the same event for
  947. * read and write.
  948. */
  949. [C(OP_READ)] = {
  950. [C(RESULT_MISS)] = { 0x35, CNTR_ALL },
  951. },
  952. [C(OP_WRITE)] = {
  953. [C(RESULT_MISS)] = { 0x35, CNTR_ALL },
  954. },
  955. },
  956. [C(ITLB)] = {
  957. [C(OP_READ)] = {
  958. [C(RESULT_MISS)] = { 0x37, CNTR_ALL },
  959. },
  960. },
  961. };
  962. static const struct mips_perf_event xlp_cache_map
  963. [PERF_COUNT_HW_CACHE_MAX]
  964. [PERF_COUNT_HW_CACHE_OP_MAX]
  965. [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  966. [C(L1D)] = {
  967. [C(OP_READ)] = {
  968. [C(RESULT_ACCESS)] = { 0x31, CNTR_ALL }, /* PAPI_L1_DCR */
  969. [C(RESULT_MISS)] = { 0x30, CNTR_ALL }, /* PAPI_L1_LDM */
  970. },
  971. [C(OP_WRITE)] = {
  972. [C(RESULT_ACCESS)] = { 0x2f, CNTR_ALL }, /* PAPI_L1_DCW */
  973. [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, /* PAPI_L1_STM */
  974. },
  975. [C(OP_PREFETCH)] = {
  976. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  977. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  978. },
  979. },
  980. [C(L1I)] = {
  981. [C(OP_READ)] = {
  982. [C(RESULT_ACCESS)] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */
  983. [C(RESULT_MISS)] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */
  984. },
  985. [C(OP_WRITE)] = {
  986. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  987. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  988. },
  989. [C(OP_PREFETCH)] = {
  990. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  991. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  992. },
  993. },
  994. [C(LL)] = {
  995. [C(OP_READ)] = {
  996. [C(RESULT_ACCESS)] = { 0x35, CNTR_ALL }, /* PAPI_L2_DCR */
  997. [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, /* PAPI_L2_LDM */
  998. },
  999. [C(OP_WRITE)] = {
  1000. [C(RESULT_ACCESS)] = { 0x34, CNTR_ALL }, /* PAPI_L2_DCA */
  1001. [C(RESULT_MISS)] = { 0x36, CNTR_ALL }, /* PAPI_L2_DCM */
  1002. },
  1003. [C(OP_PREFETCH)] = {
  1004. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1005. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1006. },
  1007. },
  1008. [C(DTLB)] = {
  1009. /*
  1010. * Only general DTLB misses are counted use the same event for
  1011. * read and write.
  1012. */
  1013. [C(OP_READ)] = {
  1014. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1015. [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */
  1016. },
  1017. [C(OP_WRITE)] = {
  1018. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1019. [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */
  1020. },
  1021. [C(OP_PREFETCH)] = {
  1022. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1023. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1024. },
  1025. },
  1026. [C(ITLB)] = {
  1027. [C(OP_READ)] = {
  1028. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1029. [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */
  1030. },
  1031. [C(OP_WRITE)] = {
  1032. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1033. [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */
  1034. },
  1035. [C(OP_PREFETCH)] = {
  1036. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1037. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1038. },
  1039. },
  1040. [C(BPU)] = {
  1041. [C(OP_READ)] = {
  1042. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1043. [C(RESULT_MISS)] = { 0x25, CNTR_ALL },
  1044. },
  1045. [C(OP_WRITE)] = {
  1046. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1047. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1048. },
  1049. [C(OP_PREFETCH)] = {
  1050. [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1051. [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID },
  1052. },
  1053. },
  1054. };
  1055. #ifdef CONFIG_MIPS_MT_SMP
  1056. static void check_and_calc_range(struct perf_event *event,
  1057. const struct mips_perf_event *pev)
  1058. {
  1059. struct hw_perf_event *hwc = &event->hw;
  1060. if (event->cpu >= 0) {
  1061. if (pev->range > V) {
  1062. /*
  1063. * The user selected an event that is processor
  1064. * wide, while expecting it to be VPE wide.
  1065. */
  1066. hwc->config_base |= M_TC_EN_ALL;
  1067. } else {
  1068. /*
  1069. * FIXME: cpu_data[event->cpu].vpe_id reports 0
  1070. * for both CPUs.
  1071. */
  1072. hwc->config_base |= M_PERFCTL_VPEID(event->cpu);
  1073. hwc->config_base |= M_TC_EN_VPE;
  1074. }
  1075. } else
  1076. hwc->config_base |= M_TC_EN_ALL;
  1077. }
  1078. #else
  1079. static void check_and_calc_range(struct perf_event *event,
  1080. const struct mips_perf_event *pev)
  1081. {
  1082. }
  1083. #endif
  1084. static int __hw_perf_event_init(struct perf_event *event)
  1085. {
  1086. struct perf_event_attr *attr = &event->attr;
  1087. struct hw_perf_event *hwc = &event->hw;
  1088. const struct mips_perf_event *pev;
  1089. int err;
  1090. /* Returning MIPS event descriptor for generic perf event. */
  1091. if (PERF_TYPE_HARDWARE == event->attr.type) {
  1092. if (event->attr.config >= PERF_COUNT_HW_MAX)
  1093. return -EINVAL;
  1094. pev = mipspmu_map_general_event(event->attr.config);
  1095. } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
  1096. pev = mipspmu_map_cache_event(event->attr.config);
  1097. } else if (PERF_TYPE_RAW == event->attr.type) {
  1098. /* We are working on the global raw event. */
  1099. mutex_lock(&raw_event_mutex);
  1100. pev = mipspmu.map_raw_event(event->attr.config);
  1101. } else {
  1102. /* The event type is not (yet) supported. */
  1103. return -EOPNOTSUPP;
  1104. }
  1105. if (IS_ERR(pev)) {
  1106. if (PERF_TYPE_RAW == event->attr.type)
  1107. mutex_unlock(&raw_event_mutex);
  1108. return PTR_ERR(pev);
  1109. }
  1110. /*
  1111. * We allow max flexibility on how each individual counter shared
  1112. * by the single CPU operates (the mode exclusion and the range).
  1113. */
  1114. hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE;
  1115. /* Calculate range bits and validate it. */
  1116. if (num_possible_cpus() > 1)
  1117. check_and_calc_range(event, pev);
  1118. hwc->event_base = mipspmu_perf_event_encode(pev);
  1119. if (PERF_TYPE_RAW == event->attr.type)
  1120. mutex_unlock(&raw_event_mutex);
  1121. if (!attr->exclude_user)
  1122. hwc->config_base |= M_PERFCTL_USER;
  1123. if (!attr->exclude_kernel) {
  1124. hwc->config_base |= M_PERFCTL_KERNEL;
  1125. /* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */
  1126. hwc->config_base |= M_PERFCTL_EXL;
  1127. }
  1128. if (!attr->exclude_hv)
  1129. hwc->config_base |= M_PERFCTL_SUPERVISOR;
  1130. hwc->config_base &= M_PERFCTL_CONFIG_MASK;
  1131. /*
  1132. * The event can belong to another cpu. We do not assign a local
  1133. * counter for it for now.
  1134. */
  1135. hwc->idx = -1;
  1136. hwc->config = 0;
  1137. if (!hwc->sample_period) {
  1138. hwc->sample_period = mipspmu.max_period;
  1139. hwc->last_period = hwc->sample_period;
  1140. local64_set(&hwc->period_left, hwc->sample_period);
  1141. }
  1142. err = 0;
  1143. if (event->group_leader != event)
  1144. err = validate_group(event);
  1145. event->destroy = hw_perf_event_destroy;
  1146. if (err)
  1147. event->destroy(event);
  1148. return err;
  1149. }
  1150. static void pause_local_counters(void)
  1151. {
  1152. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1153. int ctr = mipspmu.num_counters;
  1154. unsigned long flags;
  1155. local_irq_save(flags);
  1156. do {
  1157. ctr--;
  1158. cpuc->saved_ctrl[ctr] = mipsxx_pmu_read_control(ctr);
  1159. mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] &
  1160. ~M_PERFCTL_COUNT_EVENT_WHENEVER);
  1161. } while (ctr > 0);
  1162. local_irq_restore(flags);
  1163. }
  1164. static void resume_local_counters(void)
  1165. {
  1166. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1167. int ctr = mipspmu.num_counters;
  1168. do {
  1169. ctr--;
  1170. mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]);
  1171. } while (ctr > 0);
  1172. }
  1173. static int mipsxx_pmu_handle_shared_irq(void)
  1174. {
  1175. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1176. struct perf_sample_data data;
  1177. unsigned int counters = mipspmu.num_counters;
  1178. u64 counter;
  1179. int handled = IRQ_NONE;
  1180. struct pt_regs *regs;
  1181. if (cpu_has_perf_cntr_intr_bit && !(read_c0_cause() & CAUSEF_PCI))
  1182. return handled;
  1183. /*
  1184. * First we pause the local counters, so that when we are locked
  1185. * here, the counters are all paused. When it gets locked due to
  1186. * perf_disable(), the timer interrupt handler will be delayed.
  1187. *
  1188. * See also mipsxx_pmu_start().
  1189. */
  1190. pause_local_counters();
  1191. #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
  1192. read_lock(&pmuint_rwlock);
  1193. #endif
  1194. regs = get_irq_regs();
  1195. perf_sample_data_init(&data, 0, 0);
  1196. switch (counters) {
  1197. #define HANDLE_COUNTER(n) \
  1198. case n + 1: \
  1199. if (test_bit(n, cpuc->used_mask)) { \
  1200. counter = mipspmu.read_counter(n); \
  1201. if (counter & mipspmu.overflow) { \
  1202. handle_associated_event(cpuc, n, &data, regs); \
  1203. handled = IRQ_HANDLED; \
  1204. } \
  1205. }
  1206. HANDLE_COUNTER(3)
  1207. HANDLE_COUNTER(2)
  1208. HANDLE_COUNTER(1)
  1209. HANDLE_COUNTER(0)
  1210. }
  1211. /*
  1212. * Do all the work for the pending perf events. We can do this
  1213. * in here because the performance counter interrupt is a regular
  1214. * interrupt, not NMI.
  1215. */
  1216. if (handled == IRQ_HANDLED)
  1217. irq_work_run();
  1218. #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
  1219. read_unlock(&pmuint_rwlock);
  1220. #endif
  1221. resume_local_counters();
  1222. return handled;
  1223. }
  1224. static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev)
  1225. {
  1226. return mipsxx_pmu_handle_shared_irq();
  1227. }
  1228. /* 24K */
  1229. #define IS_BOTH_COUNTERS_24K_EVENT(b) \
  1230. ((b) == 0 || (b) == 1 || (b) == 11)
  1231. /* 34K */
  1232. #define IS_BOTH_COUNTERS_34K_EVENT(b) \
  1233. ((b) == 0 || (b) == 1 || (b) == 11)
  1234. #ifdef CONFIG_MIPS_MT_SMP
  1235. #define IS_RANGE_P_34K_EVENT(r, b) \
  1236. ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \
  1237. (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 || \
  1238. (r) == 176 || ((b) >= 50 && (b) <= 55) || \
  1239. ((b) >= 64 && (b) <= 67))
  1240. #define IS_RANGE_V_34K_EVENT(r) ((r) == 47)
  1241. #endif
  1242. /* 74K */
  1243. #define IS_BOTH_COUNTERS_74K_EVENT(b) \
  1244. ((b) == 0 || (b) == 1)
  1245. /* 1004K */
  1246. #define IS_BOTH_COUNTERS_1004K_EVENT(b) \
  1247. ((b) == 0 || (b) == 1 || (b) == 11)
  1248. #ifdef CONFIG_MIPS_MT_SMP
  1249. #define IS_RANGE_P_1004K_EVENT(r, b) \
  1250. ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \
  1251. (b) == 25 || (b) == 36 || (b) == 39 || (r) == 44 || \
  1252. (r) == 174 || (r) == 176 || ((b) >= 50 && (b) <= 59) || \
  1253. (r) == 188 || (b) == 61 || (b) == 62 || \
  1254. ((b) >= 64 && (b) <= 67))
  1255. #define IS_RANGE_V_1004K_EVENT(r) ((r) == 47)
  1256. #endif
  1257. /* BMIPS5000 */
  1258. #define IS_BOTH_COUNTERS_BMIPS5000_EVENT(b) \
  1259. ((b) == 0 || (b) == 1)
  1260. /*
  1261. * User can use 0-255 raw events, where 0-127 for the events of even
  1262. * counters, and 128-255 for odd counters. Note that bit 7 is used to
  1263. * indicate the parity. So, for example, when user wants to take the
  1264. * Event Num of 15 for odd counters (by referring to the user manual),
  1265. * then 128 needs to be added to 15 as the input for the event config,
  1266. * i.e., 143 (0x8F) to be used.
  1267. */
  1268. static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config)
  1269. {
  1270. unsigned int raw_id = config & 0xff;
  1271. unsigned int base_id = raw_id & 0x7f;
  1272. raw_event.event_id = base_id;
  1273. switch (current_cpu_type()) {
  1274. case CPU_24K:
  1275. if (IS_BOTH_COUNTERS_24K_EVENT(base_id))
  1276. raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
  1277. else
  1278. raw_event.cntr_mask =
  1279. raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
  1280. #ifdef CONFIG_MIPS_MT_SMP
  1281. /*
  1282. * This is actually doing nothing. Non-multithreading
  1283. * CPUs will not check and calculate the range.
  1284. */
  1285. raw_event.range = P;
  1286. #endif
  1287. break;
  1288. case CPU_34K:
  1289. if (IS_BOTH_COUNTERS_34K_EVENT(base_id))
  1290. raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
  1291. else
  1292. raw_event.cntr_mask =
  1293. raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
  1294. #ifdef CONFIG_MIPS_MT_SMP
  1295. if (IS_RANGE_P_34K_EVENT(raw_id, base_id))
  1296. raw_event.range = P;
  1297. else if (unlikely(IS_RANGE_V_34K_EVENT(raw_id)))
  1298. raw_event.range = V;
  1299. else
  1300. raw_event.range = T;
  1301. #endif
  1302. break;
  1303. case CPU_74K:
  1304. if (IS_BOTH_COUNTERS_74K_EVENT(base_id))
  1305. raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
  1306. else
  1307. raw_event.cntr_mask =
  1308. raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
  1309. #ifdef CONFIG_MIPS_MT_SMP
  1310. raw_event.range = P;
  1311. #endif
  1312. break;
  1313. case CPU_1004K:
  1314. if (IS_BOTH_COUNTERS_1004K_EVENT(base_id))
  1315. raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
  1316. else
  1317. raw_event.cntr_mask =
  1318. raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
  1319. #ifdef CONFIG_MIPS_MT_SMP
  1320. if (IS_RANGE_P_1004K_EVENT(raw_id, base_id))
  1321. raw_event.range = P;
  1322. else if (unlikely(IS_RANGE_V_1004K_EVENT(raw_id)))
  1323. raw_event.range = V;
  1324. else
  1325. raw_event.range = T;
  1326. #endif
  1327. break;
  1328. case CPU_BMIPS5000:
  1329. if (IS_BOTH_COUNTERS_BMIPS5000_EVENT(base_id))
  1330. raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
  1331. else
  1332. raw_event.cntr_mask =
  1333. raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
  1334. }
  1335. return &raw_event;
  1336. }
  1337. static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config)
  1338. {
  1339. unsigned int raw_id = config & 0xff;
  1340. unsigned int base_id = raw_id & 0x7f;
  1341. raw_event.cntr_mask = CNTR_ALL;
  1342. raw_event.event_id = base_id;
  1343. if (current_cpu_type() == CPU_CAVIUM_OCTEON2) {
  1344. if (base_id > 0x42)
  1345. return ERR_PTR(-EOPNOTSUPP);
  1346. } else {
  1347. if (base_id > 0x3a)
  1348. return ERR_PTR(-EOPNOTSUPP);
  1349. }
  1350. switch (base_id) {
  1351. case 0x00:
  1352. case 0x0f:
  1353. case 0x1e:
  1354. case 0x1f:
  1355. case 0x2f:
  1356. case 0x34:
  1357. case 0x3b ... 0x3f:
  1358. return ERR_PTR(-EOPNOTSUPP);
  1359. default:
  1360. break;
  1361. }
  1362. return &raw_event;
  1363. }
  1364. static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config)
  1365. {
  1366. unsigned int raw_id = config & 0xff;
  1367. /* Only 1-63 are defined */
  1368. if ((raw_id < 0x01) || (raw_id > 0x3f))
  1369. return ERR_PTR(-EOPNOTSUPP);
  1370. raw_event.cntr_mask = CNTR_ALL;
  1371. raw_event.event_id = raw_id;
  1372. return &raw_event;
  1373. }
  1374. static int __init
  1375. init_hw_perf_events(void)
  1376. {
  1377. int counters, irq;
  1378. int counter_bits;
  1379. pr_info("Performance counters: ");
  1380. counters = n_counters();
  1381. if (counters == 0) {
  1382. pr_cont("No available PMU.\n");
  1383. return -ENODEV;
  1384. }
  1385. #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS
  1386. cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19);
  1387. if (!cpu_has_mipsmt_pertccounters)
  1388. counters = counters_total_to_per_cpu(counters);
  1389. #endif
  1390. #ifdef MSC01E_INT_BASE
  1391. if (cpu_has_veic) {
  1392. /*
  1393. * Using platform specific interrupt controller defines.
  1394. */
  1395. irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR;
  1396. } else {
  1397. #endif
  1398. if ((cp0_perfcount_irq >= 0) &&
  1399. (cp0_compare_irq != cp0_perfcount_irq))
  1400. irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
  1401. else
  1402. irq = -1;
  1403. #ifdef MSC01E_INT_BASE
  1404. }
  1405. #endif
  1406. mipspmu.map_raw_event = mipsxx_pmu_map_raw_event;
  1407. switch (current_cpu_type()) {
  1408. case CPU_24K:
  1409. mipspmu.name = "mips/24K";
  1410. mipspmu.general_event_map = &mipsxxcore_event_map;
  1411. mipspmu.cache_event_map = &mipsxxcore_cache_map;
  1412. break;
  1413. case CPU_34K:
  1414. mipspmu.name = "mips/34K";
  1415. mipspmu.general_event_map = &mipsxxcore_event_map;
  1416. mipspmu.cache_event_map = &mipsxxcore_cache_map;
  1417. break;
  1418. case CPU_74K:
  1419. mipspmu.name = "mips/74K";
  1420. mipspmu.general_event_map = &mipsxx74Kcore_event_map;
  1421. mipspmu.cache_event_map = &mipsxx74Kcore_cache_map;
  1422. break;
  1423. case CPU_1004K:
  1424. mipspmu.name = "mips/1004K";
  1425. mipspmu.general_event_map = &mipsxxcore_event_map;
  1426. mipspmu.cache_event_map = &mipsxxcore_cache_map;
  1427. break;
  1428. case CPU_LOONGSON1:
  1429. mipspmu.name = "mips/loongson1";
  1430. mipspmu.general_event_map = &mipsxxcore_event_map;
  1431. mipspmu.cache_event_map = &mipsxxcore_cache_map;
  1432. break;
  1433. case CPU_CAVIUM_OCTEON:
  1434. case CPU_CAVIUM_OCTEON_PLUS:
  1435. case CPU_CAVIUM_OCTEON2:
  1436. mipspmu.name = "octeon";
  1437. mipspmu.general_event_map = &octeon_event_map;
  1438. mipspmu.cache_event_map = &octeon_cache_map;
  1439. mipspmu.map_raw_event = octeon_pmu_map_raw_event;
  1440. break;
  1441. case CPU_BMIPS5000:
  1442. mipspmu.name = "BMIPS5000";
  1443. mipspmu.general_event_map = &bmips5000_event_map;
  1444. mipspmu.cache_event_map = &bmips5000_cache_map;
  1445. break;
  1446. case CPU_XLP:
  1447. mipspmu.name = "xlp";
  1448. mipspmu.general_event_map = &xlp_event_map;
  1449. mipspmu.cache_event_map = &xlp_cache_map;
  1450. mipspmu.map_raw_event = xlp_pmu_map_raw_event;
  1451. break;
  1452. default:
  1453. pr_cont("Either hardware does not support performance "
  1454. "counters, or not yet implemented.\n");
  1455. return -ENODEV;
  1456. }
  1457. mipspmu.num_counters = counters;
  1458. mipspmu.irq = irq;
  1459. if (read_c0_perfctrl0() & M_PERFCTL_WIDE) {
  1460. mipspmu.max_period = (1ULL << 63) - 1;
  1461. mipspmu.valid_count = (1ULL << 63) - 1;
  1462. mipspmu.overflow = 1ULL << 63;
  1463. mipspmu.read_counter = mipsxx_pmu_read_counter_64;
  1464. mipspmu.write_counter = mipsxx_pmu_write_counter_64;
  1465. counter_bits = 64;
  1466. } else {
  1467. mipspmu.max_period = (1ULL << 31) - 1;
  1468. mipspmu.valid_count = (1ULL << 31) - 1;
  1469. mipspmu.overflow = 1ULL << 31;
  1470. mipspmu.read_counter = mipsxx_pmu_read_counter;
  1471. mipspmu.write_counter = mipsxx_pmu_write_counter;
  1472. counter_bits = 32;
  1473. }
  1474. on_each_cpu(reset_counters, (void *)(long)counters, 1);
  1475. pr_cont("%s PMU enabled, %d %d-bit counters available to each "
  1476. "CPU, irq %d%s\n", mipspmu.name, counters, counter_bits, irq,
  1477. irq < 0 ? " (share with timer interrupt)" : "");
  1478. perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
  1479. return 0;
  1480. }
  1481. early_initcall(init_hw_perf_events);