core-book3s.c 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979
  1. /*
  2. * Performance event support - powerpc architecture code
  3. *
  4. * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the License, or (at your option) any later version.
  10. */
  11. #include <linux/kernel.h>
  12. #include <linux/sched.h>
  13. #include <linux/perf_event.h>
  14. #include <linux/percpu.h>
  15. #include <linux/hardirq.h>
  16. #include <linux/uaccess.h>
  17. #include <asm/reg.h>
  18. #include <asm/pmc.h>
  19. #include <asm/machdep.h>
  20. #include <asm/firmware.h>
  21. #include <asm/ptrace.h>
  22. #include <asm/code-patching.h>
  23. #define BHRB_MAX_ENTRIES 32
  24. #define BHRB_TARGET 0x0000000000000002
  25. #define BHRB_PREDICTION 0x0000000000000001
  26. #define BHRB_EA 0xFFFFFFFFFFFFFFFC
  27. struct cpu_hw_events {
  28. int n_events;
  29. int n_percpu;
  30. int disabled;
  31. int n_added;
  32. int n_limited;
  33. u8 pmcs_enabled;
  34. struct perf_event *event[MAX_HWEVENTS];
  35. u64 events[MAX_HWEVENTS];
  36. unsigned int flags[MAX_HWEVENTS];
  37. unsigned long mmcr[3];
  38. struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
  39. u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
  40. u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
  41. unsigned long amasks[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
  42. unsigned long avalues[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
  43. unsigned int group_flag;
  44. int n_txn_start;
  45. /* BHRB bits */
  46. u64 bhrb_filter; /* BHRB HW branch filter */
  47. int bhrb_users;
  48. void *bhrb_context;
  49. struct perf_branch_stack bhrb_stack;
  50. struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES];
  51. };
  52. DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
  53. struct power_pmu *ppmu;
  54. /*
  55. * Normally, to ignore kernel events we set the FCS (freeze counters
  56. * in supervisor mode) bit in MMCR0, but if the kernel runs with the
  57. * hypervisor bit set in the MSR, or if we are running on a processor
  58. * where the hypervisor bit is forced to 1 (as on Apple G5 processors),
  59. * then we need to use the FCHV bit to ignore kernel events.
  60. */
  61. static unsigned int freeze_events_kernel = MMCR0_FCS;
  62. /*
  63. * 32-bit doesn't have MMCRA but does have an MMCR2,
  64. * and a few other names are different.
  65. */
  66. #ifdef CONFIG_PPC32
  67. #define MMCR0_FCHV 0
  68. #define MMCR0_PMCjCE MMCR0_PMCnCE
  69. #define MMCR0_FC56 0
  70. #define MMCR0_PMAO 0
  71. #define MMCR0_EBE 0
  72. #define MMCR0_PMCC 0
  73. #define MMCR0_PMCC_U6 0
  74. #define SPRN_MMCRA SPRN_MMCR2
  75. #define MMCRA_SAMPLE_ENABLE 0
  76. static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
  77. {
  78. return 0;
  79. }
  80. static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { }
  81. static inline u32 perf_get_misc_flags(struct pt_regs *regs)
  82. {
  83. return 0;
  84. }
  85. static inline void perf_read_regs(struct pt_regs *regs)
  86. {
  87. regs->result = 0;
  88. }
  89. static inline int perf_intr_is_nmi(struct pt_regs *regs)
  90. {
  91. return 0;
  92. }
  93. static inline int siar_valid(struct pt_regs *regs)
  94. {
  95. return 1;
  96. }
  97. static bool is_ebb_event(struct perf_event *event) { return false; }
  98. static int ebb_event_check(struct perf_event *event) { return 0; }
  99. static void ebb_event_add(struct perf_event *event) { }
  100. static void ebb_switch_out(unsigned long mmcr0) { }
  101. static unsigned long ebb_switch_in(bool ebb, unsigned long mmcr0)
  102. {
  103. return mmcr0;
  104. }
  105. static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
  106. static inline void power_pmu_bhrb_disable(struct perf_event *event) {}
  107. void power_pmu_flush_branch_stack(void) {}
  108. static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {}
  109. #endif /* CONFIG_PPC32 */
  110. static bool regs_use_siar(struct pt_regs *regs)
  111. {
  112. return !!regs->result;
  113. }
  114. /*
  115. * Things that are specific to 64-bit implementations.
  116. */
  117. #ifdef CONFIG_PPC64
  118. static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
  119. {
  120. unsigned long mmcra = regs->dsisr;
  121. if ((ppmu->flags & PPMU_HAS_SSLOT) && (mmcra & MMCRA_SAMPLE_ENABLE)) {
  122. unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
  123. if (slot > 1)
  124. return 4 * (slot - 1);
  125. }
  126. return 0;
  127. }
  128. /*
  129. * The user wants a data address recorded.
  130. * If we're not doing instruction sampling, give them the SDAR
  131. * (sampled data address). If we are doing instruction sampling, then
  132. * only give them the SDAR if it corresponds to the instruction
  133. * pointed to by SIAR; this is indicated by the [POWER6_]MMCRA_SDSYNC, the
  134. * [POWER7P_]MMCRA_SDAR_VALID bit in MMCRA, or the SDAR_VALID bit in SIER.
  135. */
  136. static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp)
  137. {
  138. unsigned long mmcra = regs->dsisr;
  139. bool sdar_valid;
  140. if (ppmu->flags & PPMU_HAS_SIER)
  141. sdar_valid = regs->dar & SIER_SDAR_VALID;
  142. else {
  143. unsigned long sdsync;
  144. if (ppmu->flags & PPMU_SIAR_VALID)
  145. sdsync = POWER7P_MMCRA_SDAR_VALID;
  146. else if (ppmu->flags & PPMU_ALT_SIPR)
  147. sdsync = POWER6_MMCRA_SDSYNC;
  148. else
  149. sdsync = MMCRA_SDSYNC;
  150. sdar_valid = mmcra & sdsync;
  151. }
  152. if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid)
  153. *addrp = mfspr(SPRN_SDAR);
  154. }
  155. static bool regs_sihv(struct pt_regs *regs)
  156. {
  157. unsigned long sihv = MMCRA_SIHV;
  158. if (ppmu->flags & PPMU_HAS_SIER)
  159. return !!(regs->dar & SIER_SIHV);
  160. if (ppmu->flags & PPMU_ALT_SIPR)
  161. sihv = POWER6_MMCRA_SIHV;
  162. return !!(regs->dsisr & sihv);
  163. }
  164. static bool regs_sipr(struct pt_regs *regs)
  165. {
  166. unsigned long sipr = MMCRA_SIPR;
  167. if (ppmu->flags & PPMU_HAS_SIER)
  168. return !!(regs->dar & SIER_SIPR);
  169. if (ppmu->flags & PPMU_ALT_SIPR)
  170. sipr = POWER6_MMCRA_SIPR;
  171. return !!(regs->dsisr & sipr);
  172. }
  173. static inline u32 perf_flags_from_msr(struct pt_regs *regs)
  174. {
  175. if (regs->msr & MSR_PR)
  176. return PERF_RECORD_MISC_USER;
  177. if ((regs->msr & MSR_HV) && freeze_events_kernel != MMCR0_FCHV)
  178. return PERF_RECORD_MISC_HYPERVISOR;
  179. return PERF_RECORD_MISC_KERNEL;
  180. }
  181. static inline u32 perf_get_misc_flags(struct pt_regs *regs)
  182. {
  183. bool use_siar = regs_use_siar(regs);
  184. if (!use_siar)
  185. return perf_flags_from_msr(regs);
  186. /*
  187. * If we don't have flags in MMCRA, rather than using
  188. * the MSR, we intuit the flags from the address in
  189. * SIAR which should give slightly more reliable
  190. * results
  191. */
  192. if (ppmu->flags & PPMU_NO_SIPR) {
  193. unsigned long siar = mfspr(SPRN_SIAR);
  194. if (siar >= PAGE_OFFSET)
  195. return PERF_RECORD_MISC_KERNEL;
  196. return PERF_RECORD_MISC_USER;
  197. }
  198. /* PR has priority over HV, so order below is important */
  199. if (regs_sipr(regs))
  200. return PERF_RECORD_MISC_USER;
  201. if (regs_sihv(regs) && (freeze_events_kernel != MMCR0_FCHV))
  202. return PERF_RECORD_MISC_HYPERVISOR;
  203. return PERF_RECORD_MISC_KERNEL;
  204. }
  205. /*
  206. * Overload regs->dsisr to store MMCRA so we only need to read it once
  207. * on each interrupt.
  208. * Overload regs->dar to store SIER if we have it.
  209. * Overload regs->result to specify whether we should use the MSR (result
  210. * is zero) or the SIAR (result is non zero).
  211. */
  212. static inline void perf_read_regs(struct pt_regs *regs)
  213. {
  214. unsigned long mmcra = mfspr(SPRN_MMCRA);
  215. int marked = mmcra & MMCRA_SAMPLE_ENABLE;
  216. int use_siar;
  217. regs->dsisr = mmcra;
  218. if (ppmu->flags & PPMU_HAS_SIER)
  219. regs->dar = mfspr(SPRN_SIER);
  220. /*
  221. * If this isn't a PMU exception (eg a software event) the SIAR is
  222. * not valid. Use pt_regs.
  223. *
  224. * If it is a marked event use the SIAR.
  225. *
  226. * If the PMU doesn't update the SIAR for non marked events use
  227. * pt_regs.
  228. *
  229. * If the PMU has HV/PR flags then check to see if they
  230. * place the exception in userspace. If so, use pt_regs. In
  231. * continuous sampling mode the SIAR and the PMU exception are
  232. * not synchronised, so they may be many instructions apart.
  233. * This can result in confusing backtraces. We still want
  234. * hypervisor samples as well as samples in the kernel with
  235. * interrupts off hence the userspace check.
  236. */
  237. if (TRAP(regs) != 0xf00)
  238. use_siar = 0;
  239. else if (marked)
  240. use_siar = 1;
  241. else if ((ppmu->flags & PPMU_NO_CONT_SAMPLING))
  242. use_siar = 0;
  243. else if (!(ppmu->flags & PPMU_NO_SIPR) && regs_sipr(regs))
  244. use_siar = 0;
  245. else
  246. use_siar = 1;
  247. regs->result = use_siar;
  248. }
  249. /*
  250. * If interrupts were soft-disabled when a PMU interrupt occurs, treat
  251. * it as an NMI.
  252. */
  253. static inline int perf_intr_is_nmi(struct pt_regs *regs)
  254. {
  255. return !regs->softe;
  256. }
  257. /*
  258. * On processors like P7+ that have the SIAR-Valid bit, marked instructions
  259. * must be sampled only if the SIAR-valid bit is set.
  260. *
  261. * For unmarked instructions and for processors that don't have the SIAR-Valid
  262. * bit, assume that SIAR is valid.
  263. */
  264. static inline int siar_valid(struct pt_regs *regs)
  265. {
  266. unsigned long mmcra = regs->dsisr;
  267. int marked = mmcra & MMCRA_SAMPLE_ENABLE;
  268. if (marked) {
  269. if (ppmu->flags & PPMU_HAS_SIER)
  270. return regs->dar & SIER_SIAR_VALID;
  271. if (ppmu->flags & PPMU_SIAR_VALID)
  272. return mmcra & POWER7P_MMCRA_SIAR_VALID;
  273. }
  274. return 1;
  275. }
  276. /* Reset all possible BHRB entries */
  277. static void power_pmu_bhrb_reset(void)
  278. {
  279. asm volatile(PPC_CLRBHRB);
  280. }
  281. static void power_pmu_bhrb_enable(struct perf_event *event)
  282. {
  283. struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
  284. if (!ppmu->bhrb_nr)
  285. return;
  286. /* Clear BHRB if we changed task context to avoid data leaks */
  287. if (event->ctx->task && cpuhw->bhrb_context != event->ctx) {
  288. power_pmu_bhrb_reset();
  289. cpuhw->bhrb_context = event->ctx;
  290. }
  291. cpuhw->bhrb_users++;
  292. }
  293. static void power_pmu_bhrb_disable(struct perf_event *event)
  294. {
  295. struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
  296. if (!ppmu->bhrb_nr)
  297. return;
  298. cpuhw->bhrb_users--;
  299. WARN_ON_ONCE(cpuhw->bhrb_users < 0);
  300. if (!cpuhw->disabled && !cpuhw->bhrb_users) {
  301. /* BHRB cannot be turned off when other
  302. * events are active on the PMU.
  303. */
  304. /* avoid stale pointer */
  305. cpuhw->bhrb_context = NULL;
  306. }
  307. }
  308. /* Called from ctxsw to prevent one process's branch entries to
  309. * mingle with the other process's entries during context switch.
  310. */
  311. void power_pmu_flush_branch_stack(void)
  312. {
  313. if (ppmu->bhrb_nr)
  314. power_pmu_bhrb_reset();
  315. }
  316. /* Calculate the to address for a branch */
  317. static __u64 power_pmu_bhrb_to(u64 addr)
  318. {
  319. unsigned int instr;
  320. int ret;
  321. __u64 target;
  322. if (is_kernel_addr(addr))
  323. return branch_target((unsigned int *)addr);
  324. /* Userspace: need copy instruction here then translate it */
  325. pagefault_disable();
  326. ret = __get_user_inatomic(instr, (unsigned int __user *)addr);
  327. if (ret) {
  328. pagefault_enable();
  329. return 0;
  330. }
  331. pagefault_enable();
  332. target = branch_target(&instr);
  333. if ((!target) || (instr & BRANCH_ABSOLUTE))
  334. return target;
  335. /* Translate relative branch target from kernel to user address */
  336. return target - (unsigned long)&instr + addr;
  337. }
  338. /* Processing BHRB entries */
  339. void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
  340. {
  341. u64 val;
  342. u64 addr;
  343. int r_index, u_index, pred;
  344. r_index = 0;
  345. u_index = 0;
  346. while (r_index < ppmu->bhrb_nr) {
  347. /* Assembly read function */
  348. val = read_bhrb(r_index++);
  349. if (!val)
  350. /* Terminal marker: End of valid BHRB entries */
  351. break;
  352. else {
  353. addr = val & BHRB_EA;
  354. pred = val & BHRB_PREDICTION;
  355. if (!addr)
  356. /* invalid entry */
  357. continue;
  358. /* Branches are read most recent first (ie. mfbhrb 0 is
  359. * the most recent branch).
  360. * There are two types of valid entries:
  361. * 1) a target entry which is the to address of a
  362. * computed goto like a blr,bctr,btar. The next
  363. * entry read from the bhrb will be branch
  364. * corresponding to this target (ie. the actual
  365. * blr/bctr/btar instruction).
  366. * 2) a from address which is an actual branch. If a
  367. * target entry proceeds this, then this is the
  368. * matching branch for that target. If this is not
  369. * following a target entry, then this is a branch
  370. * where the target is given as an immediate field
  371. * in the instruction (ie. an i or b form branch).
  372. * In this case we need to read the instruction from
  373. * memory to determine the target/to address.
  374. */
  375. if (val & BHRB_TARGET) {
  376. /* Target branches use two entries
  377. * (ie. computed gotos/XL form)
  378. */
  379. cpuhw->bhrb_entries[u_index].to = addr;
  380. cpuhw->bhrb_entries[u_index].mispred = pred;
  381. cpuhw->bhrb_entries[u_index].predicted = ~pred;
  382. /* Get from address in next entry */
  383. val = read_bhrb(r_index++);
  384. addr = val & BHRB_EA;
  385. if (val & BHRB_TARGET) {
  386. /* Shouldn't have two targets in a
  387. row.. Reset index and try again */
  388. r_index--;
  389. addr = 0;
  390. }
  391. cpuhw->bhrb_entries[u_index].from = addr;
  392. } else {
  393. /* Branches to immediate field
  394. (ie I or B form) */
  395. cpuhw->bhrb_entries[u_index].from = addr;
  396. cpuhw->bhrb_entries[u_index].to =
  397. power_pmu_bhrb_to(addr);
  398. cpuhw->bhrb_entries[u_index].mispred = pred;
  399. cpuhw->bhrb_entries[u_index].predicted = ~pred;
  400. }
  401. u_index++;
  402. }
  403. }
  404. cpuhw->bhrb_stack.nr = u_index;
  405. return;
  406. }
  407. static bool is_ebb_event(struct perf_event *event)
  408. {
  409. /*
  410. * This could be a per-PMU callback, but we'd rather avoid the cost. We
  411. * check that the PMU supports EBB, meaning those that don't can still
  412. * use bit 63 of the event code for something else if they wish.
  413. */
  414. return (ppmu->flags & PPMU_EBB) &&
  415. ((event->attr.config >> PERF_EVENT_CONFIG_EBB_SHIFT) & 1);
  416. }
  417. static int ebb_event_check(struct perf_event *event)
  418. {
  419. struct perf_event *leader = event->group_leader;
  420. /* Event and group leader must agree on EBB */
  421. if (is_ebb_event(leader) != is_ebb_event(event))
  422. return -EINVAL;
  423. if (is_ebb_event(event)) {
  424. if (!(event->attach_state & PERF_ATTACH_TASK))
  425. return -EINVAL;
  426. if (!leader->attr.pinned || !leader->attr.exclusive)
  427. return -EINVAL;
  428. if (event->attr.inherit || event->attr.sample_period ||
  429. event->attr.enable_on_exec || event->attr.freq)
  430. return -EINVAL;
  431. }
  432. return 0;
  433. }
  434. static void ebb_event_add(struct perf_event *event)
  435. {
  436. if (!is_ebb_event(event) || current->thread.used_ebb)
  437. return;
  438. /*
  439. * IFF this is the first time we've added an EBB event, set
  440. * PMXE in the user MMCR0 so we can detect when it's cleared by
  441. * userspace. We need this so that we can context switch while
  442. * userspace is in the EBB handler (where PMXE is 0).
  443. */
  444. current->thread.used_ebb = 1;
  445. current->thread.mmcr0 |= MMCR0_PMXE;
  446. }
  447. static void ebb_switch_out(unsigned long mmcr0)
  448. {
  449. if (!(mmcr0 & MMCR0_EBE))
  450. return;
  451. current->thread.siar = mfspr(SPRN_SIAR);
  452. current->thread.sier = mfspr(SPRN_SIER);
  453. current->thread.sdar = mfspr(SPRN_SDAR);
  454. current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
  455. current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
  456. }
  457. static unsigned long ebb_switch_in(bool ebb, unsigned long mmcr0)
  458. {
  459. if (!ebb)
  460. goto out;
  461. /* Enable EBB and read/write to all 6 PMCs for userspace */
  462. mmcr0 |= MMCR0_EBE | MMCR0_PMCC_U6;
  463. /* Add any bits from the user reg, FC or PMAO */
  464. mmcr0 |= current->thread.mmcr0;
  465. /* Be careful not to set PMXE if userspace had it cleared */
  466. if (!(current->thread.mmcr0 & MMCR0_PMXE))
  467. mmcr0 &= ~MMCR0_PMXE;
  468. mtspr(SPRN_SIAR, current->thread.siar);
  469. mtspr(SPRN_SIER, current->thread.sier);
  470. mtspr(SPRN_SDAR, current->thread.sdar);
  471. mtspr(SPRN_MMCR2, current->thread.mmcr2);
  472. out:
  473. return mmcr0;
  474. }
  475. #endif /* CONFIG_PPC64 */
  476. static void perf_event_interrupt(struct pt_regs *regs);
  477. void perf_event_print_debug(void)
  478. {
  479. }
  480. /*
  481. * Read one performance monitor counter (PMC).
  482. */
  483. static unsigned long read_pmc(int idx)
  484. {
  485. unsigned long val;
  486. switch (idx) {
  487. case 1:
  488. val = mfspr(SPRN_PMC1);
  489. break;
  490. case 2:
  491. val = mfspr(SPRN_PMC2);
  492. break;
  493. case 3:
  494. val = mfspr(SPRN_PMC3);
  495. break;
  496. case 4:
  497. val = mfspr(SPRN_PMC4);
  498. break;
  499. case 5:
  500. val = mfspr(SPRN_PMC5);
  501. break;
  502. case 6:
  503. val = mfspr(SPRN_PMC6);
  504. break;
  505. #ifdef CONFIG_PPC64
  506. case 7:
  507. val = mfspr(SPRN_PMC7);
  508. break;
  509. case 8:
  510. val = mfspr(SPRN_PMC8);
  511. break;
  512. #endif /* CONFIG_PPC64 */
  513. default:
  514. printk(KERN_ERR "oops trying to read PMC%d\n", idx);
  515. val = 0;
  516. }
  517. return val;
  518. }
  519. /*
  520. * Write one PMC.
  521. */
  522. static void write_pmc(int idx, unsigned long val)
  523. {
  524. switch (idx) {
  525. case 1:
  526. mtspr(SPRN_PMC1, val);
  527. break;
  528. case 2:
  529. mtspr(SPRN_PMC2, val);
  530. break;
  531. case 3:
  532. mtspr(SPRN_PMC3, val);
  533. break;
  534. case 4:
  535. mtspr(SPRN_PMC4, val);
  536. break;
  537. case 5:
  538. mtspr(SPRN_PMC5, val);
  539. break;
  540. case 6:
  541. mtspr(SPRN_PMC6, val);
  542. break;
  543. #ifdef CONFIG_PPC64
  544. case 7:
  545. mtspr(SPRN_PMC7, val);
  546. break;
  547. case 8:
  548. mtspr(SPRN_PMC8, val);
  549. break;
  550. #endif /* CONFIG_PPC64 */
  551. default:
  552. printk(KERN_ERR "oops trying to write PMC%d\n", idx);
  553. }
  554. }
  555. /*
  556. * Check if a set of events can all go on the PMU at once.
  557. * If they can't, this will look at alternative codes for the events
  558. * and see if any combination of alternative codes is feasible.
  559. * The feasible set is returned in event_id[].
  560. */
  561. static int power_check_constraints(struct cpu_hw_events *cpuhw,
  562. u64 event_id[], unsigned int cflags[],
  563. int n_ev)
  564. {
  565. unsigned long mask, value, nv;
  566. unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS];
  567. int n_alt[MAX_HWEVENTS], choice[MAX_HWEVENTS];
  568. int i, j;
  569. unsigned long addf = ppmu->add_fields;
  570. unsigned long tadd = ppmu->test_adder;
  571. if (n_ev > ppmu->n_counter)
  572. return -1;
  573. /* First see if the events will go on as-is */
  574. for (i = 0; i < n_ev; ++i) {
  575. if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
  576. && !ppmu->limited_pmc_event(event_id[i])) {
  577. ppmu->get_alternatives(event_id[i], cflags[i],
  578. cpuhw->alternatives[i]);
  579. event_id[i] = cpuhw->alternatives[i][0];
  580. }
  581. if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0],
  582. &cpuhw->avalues[i][0]))
  583. return -1;
  584. }
  585. value = mask = 0;
  586. for (i = 0; i < n_ev; ++i) {
  587. nv = (value | cpuhw->avalues[i][0]) +
  588. (value & cpuhw->avalues[i][0] & addf);
  589. if ((((nv + tadd) ^ value) & mask) != 0 ||
  590. (((nv + tadd) ^ cpuhw->avalues[i][0]) &
  591. cpuhw->amasks[i][0]) != 0)
  592. break;
  593. value = nv;
  594. mask |= cpuhw->amasks[i][0];
  595. }
  596. if (i == n_ev)
  597. return 0; /* all OK */
  598. /* doesn't work, gather alternatives... */
  599. if (!ppmu->get_alternatives)
  600. return -1;
  601. for (i = 0; i < n_ev; ++i) {
  602. choice[i] = 0;
  603. n_alt[i] = ppmu->get_alternatives(event_id[i], cflags[i],
  604. cpuhw->alternatives[i]);
  605. for (j = 1; j < n_alt[i]; ++j)
  606. ppmu->get_constraint(cpuhw->alternatives[i][j],
  607. &cpuhw->amasks[i][j],
  608. &cpuhw->avalues[i][j]);
  609. }
  610. /* enumerate all possibilities and see if any will work */
  611. i = 0;
  612. j = -1;
  613. value = mask = nv = 0;
  614. while (i < n_ev) {
  615. if (j >= 0) {
  616. /* we're backtracking, restore context */
  617. value = svalues[i];
  618. mask = smasks[i];
  619. j = choice[i];
  620. }
  621. /*
  622. * See if any alternative k for event_id i,
  623. * where k > j, will satisfy the constraints.
  624. */
  625. while (++j < n_alt[i]) {
  626. nv = (value | cpuhw->avalues[i][j]) +
  627. (value & cpuhw->avalues[i][j] & addf);
  628. if ((((nv + tadd) ^ value) & mask) == 0 &&
  629. (((nv + tadd) ^ cpuhw->avalues[i][j])
  630. & cpuhw->amasks[i][j]) == 0)
  631. break;
  632. }
  633. if (j >= n_alt[i]) {
  634. /*
  635. * No feasible alternative, backtrack
  636. * to event_id i-1 and continue enumerating its
  637. * alternatives from where we got up to.
  638. */
  639. if (--i < 0)
  640. return -1;
  641. } else {
  642. /*
  643. * Found a feasible alternative for event_id i,
  644. * remember where we got up to with this event_id,
  645. * go on to the next event_id, and start with
  646. * the first alternative for it.
  647. */
  648. choice[i] = j;
  649. svalues[i] = value;
  650. smasks[i] = mask;
  651. value = nv;
  652. mask |= cpuhw->amasks[i][j];
  653. ++i;
  654. j = -1;
  655. }
  656. }
  657. /* OK, we have a feasible combination, tell the caller the solution */
  658. for (i = 0; i < n_ev; ++i)
  659. event_id[i] = cpuhw->alternatives[i][choice[i]];
  660. return 0;
  661. }
  662. /*
  663. * Check if newly-added events have consistent settings for
  664. * exclude_{user,kernel,hv} with each other and any previously
  665. * added events.
  666. */
  667. static int check_excludes(struct perf_event **ctrs, unsigned int cflags[],
  668. int n_prev, int n_new)
  669. {
  670. int eu = 0, ek = 0, eh = 0;
  671. int i, n, first;
  672. struct perf_event *event;
  673. n = n_prev + n_new;
  674. if (n <= 1)
  675. return 0;
  676. first = 1;
  677. for (i = 0; i < n; ++i) {
  678. if (cflags[i] & PPMU_LIMITED_PMC_OK) {
  679. cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
  680. continue;
  681. }
  682. event = ctrs[i];
  683. if (first) {
  684. eu = event->attr.exclude_user;
  685. ek = event->attr.exclude_kernel;
  686. eh = event->attr.exclude_hv;
  687. first = 0;
  688. } else if (event->attr.exclude_user != eu ||
  689. event->attr.exclude_kernel != ek ||
  690. event->attr.exclude_hv != eh) {
  691. return -EAGAIN;
  692. }
  693. }
  694. if (eu || ek || eh)
  695. for (i = 0; i < n; ++i)
  696. if (cflags[i] & PPMU_LIMITED_PMC_OK)
  697. cflags[i] |= PPMU_LIMITED_PMC_REQD;
  698. return 0;
  699. }
  700. static u64 check_and_compute_delta(u64 prev, u64 val)
  701. {
  702. u64 delta = (val - prev) & 0xfffffffful;
  703. /*
  704. * POWER7 can roll back counter values, if the new value is smaller
  705. * than the previous value it will cause the delta and the counter to
  706. * have bogus values unless we rolled a counter over. If a coutner is
  707. * rolled back, it will be smaller, but within 256, which is the maximum
  708. * number of events to rollback at once. If we dectect a rollback
  709. * return 0. This can lead to a small lack of precision in the
  710. * counters.
  711. */
  712. if (prev > val && (prev - val) < 256)
  713. delta = 0;
  714. return delta;
  715. }
  716. static void power_pmu_read(struct perf_event *event)
  717. {
  718. s64 val, delta, prev;
  719. if (event->hw.state & PERF_HES_STOPPED)
  720. return;
  721. if (!event->hw.idx)
  722. return;
  723. if (is_ebb_event(event)) {
  724. val = read_pmc(event->hw.idx);
  725. local64_set(&event->hw.prev_count, val);
  726. return;
  727. }
  728. /*
  729. * Performance monitor interrupts come even when interrupts
  730. * are soft-disabled, as long as interrupts are hard-enabled.
  731. * Therefore we treat them like NMIs.
  732. */
  733. do {
  734. prev = local64_read(&event->hw.prev_count);
  735. barrier();
  736. val = read_pmc(event->hw.idx);
  737. delta = check_and_compute_delta(prev, val);
  738. if (!delta)
  739. return;
  740. } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
  741. local64_add(delta, &event->count);
  742. local64_sub(delta, &event->hw.period_left);
  743. }
  744. /*
  745. * On some machines, PMC5 and PMC6 can't be written, don't respect
  746. * the freeze conditions, and don't generate interrupts. This tells
  747. * us if `event' is using such a PMC.
  748. */
  749. static int is_limited_pmc(int pmcnum)
  750. {
  751. return (ppmu->flags & PPMU_LIMITED_PMC5_6)
  752. && (pmcnum == 5 || pmcnum == 6);
  753. }
  754. static void freeze_limited_counters(struct cpu_hw_events *cpuhw,
  755. unsigned long pmc5, unsigned long pmc6)
  756. {
  757. struct perf_event *event;
  758. u64 val, prev, delta;
  759. int i;
  760. for (i = 0; i < cpuhw->n_limited; ++i) {
  761. event = cpuhw->limited_counter[i];
  762. if (!event->hw.idx)
  763. continue;
  764. val = (event->hw.idx == 5) ? pmc5 : pmc6;
  765. prev = local64_read(&event->hw.prev_count);
  766. event->hw.idx = 0;
  767. delta = check_and_compute_delta(prev, val);
  768. if (delta)
  769. local64_add(delta, &event->count);
  770. }
  771. }
  772. static void thaw_limited_counters(struct cpu_hw_events *cpuhw,
  773. unsigned long pmc5, unsigned long pmc6)
  774. {
  775. struct perf_event *event;
  776. u64 val, prev;
  777. int i;
  778. for (i = 0; i < cpuhw->n_limited; ++i) {
  779. event = cpuhw->limited_counter[i];
  780. event->hw.idx = cpuhw->limited_hwidx[i];
  781. val = (event->hw.idx == 5) ? pmc5 : pmc6;
  782. prev = local64_read(&event->hw.prev_count);
  783. if (check_and_compute_delta(prev, val))
  784. local64_set(&event->hw.prev_count, val);
  785. perf_event_update_userpage(event);
  786. }
  787. }
  788. /*
  789. * Since limited events don't respect the freeze conditions, we
  790. * have to read them immediately after freezing or unfreezing the
  791. * other events. We try to keep the values from the limited
  792. * events as consistent as possible by keeping the delay (in
  793. * cycles and instructions) between freezing/unfreezing and reading
  794. * the limited events as small and consistent as possible.
  795. * Therefore, if any limited events are in use, we read them
  796. * both, and always in the same order, to minimize variability,
  797. * and do it inside the same asm that writes MMCR0.
  798. */
  799. static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0)
  800. {
  801. unsigned long pmc5, pmc6;
  802. if (!cpuhw->n_limited) {
  803. mtspr(SPRN_MMCR0, mmcr0);
  804. return;
  805. }
  806. /*
  807. * Write MMCR0, then read PMC5 and PMC6 immediately.
  808. * To ensure we don't get a performance monitor interrupt
  809. * between writing MMCR0 and freezing/thawing the limited
  810. * events, we first write MMCR0 with the event overflow
  811. * interrupt enable bits turned off.
  812. */
  813. asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
  814. : "=&r" (pmc5), "=&r" (pmc6)
  815. : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
  816. "i" (SPRN_MMCR0),
  817. "i" (SPRN_PMC5), "i" (SPRN_PMC6));
  818. if (mmcr0 & MMCR0_FC)
  819. freeze_limited_counters(cpuhw, pmc5, pmc6);
  820. else
  821. thaw_limited_counters(cpuhw, pmc5, pmc6);
  822. /*
  823. * Write the full MMCR0 including the event overflow interrupt
  824. * enable bits, if necessary.
  825. */
  826. if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
  827. mtspr(SPRN_MMCR0, mmcr0);
  828. }
  829. /*
  830. * Disable all events to prevent PMU interrupts and to allow
  831. * events to be added or removed.
  832. */
  833. static void power_pmu_disable(struct pmu *pmu)
  834. {
  835. struct cpu_hw_events *cpuhw;
  836. unsigned long flags, mmcr0, val;
  837. if (!ppmu)
  838. return;
  839. local_irq_save(flags);
  840. cpuhw = &__get_cpu_var(cpu_hw_events);
  841. if (!cpuhw->disabled) {
  842. /*
  843. * Check if we ever enabled the PMU on this cpu.
  844. */
  845. if (!cpuhw->pmcs_enabled) {
  846. ppc_enable_pmcs();
  847. cpuhw->pmcs_enabled = 1;
  848. }
  849. /*
  850. * Set the 'freeze counters' bit, clear EBE/PMCC/PMAO/FC56.
  851. */
  852. val = mmcr0 = mfspr(SPRN_MMCR0);
  853. val |= MMCR0_FC;
  854. val &= ~(MMCR0_EBE | MMCR0_PMCC | MMCR0_PMAO | MMCR0_FC56);
  855. /*
  856. * The barrier is to make sure the mtspr has been
  857. * executed and the PMU has frozen the events etc.
  858. * before we return.
  859. */
  860. write_mmcr0(cpuhw, val);
  861. mb();
  862. /*
  863. * Disable instruction sampling if it was enabled
  864. */
  865. if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
  866. mtspr(SPRN_MMCRA,
  867. cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
  868. mb();
  869. }
  870. cpuhw->disabled = 1;
  871. cpuhw->n_added = 0;
  872. ebb_switch_out(mmcr0);
  873. }
  874. local_irq_restore(flags);
  875. }
  876. /*
  877. * Re-enable all events if disable == 0.
  878. * If we were previously disabled and events were added, then
  879. * put the new config on the PMU.
  880. */
  881. static void power_pmu_enable(struct pmu *pmu)
  882. {
  883. struct perf_event *event;
  884. struct cpu_hw_events *cpuhw;
  885. unsigned long flags;
  886. long i;
  887. unsigned long val, mmcr0;
  888. s64 left;
  889. unsigned int hwc_index[MAX_HWEVENTS];
  890. int n_lim;
  891. int idx;
  892. bool ebb;
  893. if (!ppmu)
  894. return;
  895. local_irq_save(flags);
  896. cpuhw = &__get_cpu_var(cpu_hw_events);
  897. if (!cpuhw->disabled)
  898. goto out;
  899. if (cpuhw->n_events == 0) {
  900. ppc_set_pmu_inuse(0);
  901. goto out;
  902. }
  903. cpuhw->disabled = 0;
  904. /*
  905. * EBB requires an exclusive group and all events must have the EBB
  906. * flag set, or not set, so we can just check a single event. Also we
  907. * know we have at least one event.
  908. */
  909. ebb = is_ebb_event(cpuhw->event[0]);
  910. /*
  911. * If we didn't change anything, or only removed events,
  912. * no need to recalculate MMCR* settings and reset the PMCs.
  913. * Just reenable the PMU with the current MMCR* settings
  914. * (possibly updated for removal of events).
  915. */
  916. if (!cpuhw->n_added) {
  917. mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
  918. mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
  919. goto out_enable;
  920. }
  921. /*
  922. * Compute MMCR* values for the new set of events
  923. */
  924. if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
  925. cpuhw->mmcr)) {
  926. /* shouldn't ever get here */
  927. printk(KERN_ERR "oops compute_mmcr failed\n");
  928. goto out;
  929. }
  930. /*
  931. * Add in MMCR0 freeze bits corresponding to the
  932. * attr.exclude_* bits for the first event.
  933. * We have already checked that all events have the
  934. * same values for these bits as the first event.
  935. */
  936. event = cpuhw->event[0];
  937. if (event->attr.exclude_user)
  938. cpuhw->mmcr[0] |= MMCR0_FCP;
  939. if (event->attr.exclude_kernel)
  940. cpuhw->mmcr[0] |= freeze_events_kernel;
  941. if (event->attr.exclude_hv)
  942. cpuhw->mmcr[0] |= MMCR0_FCHV;
  943. /*
  944. * Write the new configuration to MMCR* with the freeze
  945. * bit set and set the hardware events to their initial values.
  946. * Then unfreeze the events.
  947. */
  948. ppc_set_pmu_inuse(1);
  949. mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
  950. mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
  951. mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
  952. | MMCR0_FC);
  953. /*
  954. * Read off any pre-existing events that need to move
  955. * to another PMC.
  956. */
  957. for (i = 0; i < cpuhw->n_events; ++i) {
  958. event = cpuhw->event[i];
  959. if (event->hw.idx && event->hw.idx != hwc_index[i] + 1) {
  960. power_pmu_read(event);
  961. write_pmc(event->hw.idx, 0);
  962. event->hw.idx = 0;
  963. }
  964. }
  965. /*
  966. * Initialize the PMCs for all the new and moved events.
  967. */
  968. cpuhw->n_limited = n_lim = 0;
  969. for (i = 0; i < cpuhw->n_events; ++i) {
  970. event = cpuhw->event[i];
  971. if (event->hw.idx)
  972. continue;
  973. idx = hwc_index[i] + 1;
  974. if (is_limited_pmc(idx)) {
  975. cpuhw->limited_counter[n_lim] = event;
  976. cpuhw->limited_hwidx[n_lim] = idx;
  977. ++n_lim;
  978. continue;
  979. }
  980. if (ebb)
  981. val = local64_read(&event->hw.prev_count);
  982. else {
  983. val = 0;
  984. if (event->hw.sample_period) {
  985. left = local64_read(&event->hw.period_left);
  986. if (left < 0x80000000L)
  987. val = 0x80000000L - left;
  988. }
  989. local64_set(&event->hw.prev_count, val);
  990. }
  991. event->hw.idx = idx;
  992. if (event->hw.state & PERF_HES_STOPPED)
  993. val = 0;
  994. write_pmc(idx, val);
  995. perf_event_update_userpage(event);
  996. }
  997. cpuhw->n_limited = n_lim;
  998. cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
  999. out_enable:
  1000. mmcr0 = ebb_switch_in(ebb, cpuhw->mmcr[0]);
  1001. mb();
  1002. write_mmcr0(cpuhw, mmcr0);
  1003. /*
  1004. * Enable instruction sampling if necessary
  1005. */
  1006. if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
  1007. mb();
  1008. mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
  1009. }
  1010. out:
  1011. if (cpuhw->bhrb_users)
  1012. ppmu->config_bhrb(cpuhw->bhrb_filter);
  1013. local_irq_restore(flags);
  1014. }
  1015. static int collect_events(struct perf_event *group, int max_count,
  1016. struct perf_event *ctrs[], u64 *events,
  1017. unsigned int *flags)
  1018. {
  1019. int n = 0;
  1020. struct perf_event *event;
  1021. if (!is_software_event(group)) {
  1022. if (n >= max_count)
  1023. return -1;
  1024. ctrs[n] = group;
  1025. flags[n] = group->hw.event_base;
  1026. events[n++] = group->hw.config;
  1027. }
  1028. list_for_each_entry(event, &group->sibling_list, group_entry) {
  1029. if (!is_software_event(event) &&
  1030. event->state != PERF_EVENT_STATE_OFF) {
  1031. if (n >= max_count)
  1032. return -1;
  1033. ctrs[n] = event;
  1034. flags[n] = event->hw.event_base;
  1035. events[n++] = event->hw.config;
  1036. }
  1037. }
  1038. return n;
  1039. }
  1040. /*
  1041. * Add a event to the PMU.
  1042. * If all events are not already frozen, then we disable and
  1043. * re-enable the PMU in order to get hw_perf_enable to do the
  1044. * actual work of reconfiguring the PMU.
  1045. */
  1046. static int power_pmu_add(struct perf_event *event, int ef_flags)
  1047. {
  1048. struct cpu_hw_events *cpuhw;
  1049. unsigned long flags;
  1050. int n0;
  1051. int ret = -EAGAIN;
  1052. local_irq_save(flags);
  1053. perf_pmu_disable(event->pmu);
  1054. /*
  1055. * Add the event to the list (if there is room)
  1056. * and check whether the total set is still feasible.
  1057. */
  1058. cpuhw = &__get_cpu_var(cpu_hw_events);
  1059. n0 = cpuhw->n_events;
  1060. if (n0 >= ppmu->n_counter)
  1061. goto out;
  1062. cpuhw->event[n0] = event;
  1063. cpuhw->events[n0] = event->hw.config;
  1064. cpuhw->flags[n0] = event->hw.event_base;
  1065. /*
  1066. * This event may have been disabled/stopped in record_and_restart()
  1067. * because we exceeded the ->event_limit. If re-starting the event,
  1068. * clear the ->hw.state (STOPPED and UPTODATE flags), so the user
  1069. * notification is re-enabled.
  1070. */
  1071. if (!(ef_flags & PERF_EF_START))
  1072. event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  1073. else
  1074. event->hw.state = 0;
  1075. /*
  1076. * If group events scheduling transaction was started,
  1077. * skip the schedulability test here, it will be performed
  1078. * at commit time(->commit_txn) as a whole
  1079. */
  1080. if (cpuhw->group_flag & PERF_EVENT_TXN)
  1081. goto nocheck;
  1082. if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1))
  1083. goto out;
  1084. if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1))
  1085. goto out;
  1086. event->hw.config = cpuhw->events[n0];
  1087. nocheck:
  1088. ebb_event_add(event);
  1089. ++cpuhw->n_events;
  1090. ++cpuhw->n_added;
  1091. ret = 0;
  1092. out:
  1093. if (has_branch_stack(event)) {
  1094. power_pmu_bhrb_enable(event);
  1095. cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
  1096. event->attr.branch_sample_type);
  1097. }
  1098. perf_pmu_enable(event->pmu);
  1099. local_irq_restore(flags);
  1100. return ret;
  1101. }
  1102. /*
  1103. * Remove a event from the PMU.
  1104. */
  1105. static void power_pmu_del(struct perf_event *event, int ef_flags)
  1106. {
  1107. struct cpu_hw_events *cpuhw;
  1108. long i;
  1109. unsigned long flags;
  1110. local_irq_save(flags);
  1111. perf_pmu_disable(event->pmu);
  1112. power_pmu_read(event);
  1113. cpuhw = &__get_cpu_var(cpu_hw_events);
  1114. for (i = 0; i < cpuhw->n_events; ++i) {
  1115. if (event == cpuhw->event[i]) {
  1116. while (++i < cpuhw->n_events) {
  1117. cpuhw->event[i-1] = cpuhw->event[i];
  1118. cpuhw->events[i-1] = cpuhw->events[i];
  1119. cpuhw->flags[i-1] = cpuhw->flags[i];
  1120. }
  1121. --cpuhw->n_events;
  1122. ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr);
  1123. if (event->hw.idx) {
  1124. write_pmc(event->hw.idx, 0);
  1125. event->hw.idx = 0;
  1126. }
  1127. perf_event_update_userpage(event);
  1128. break;
  1129. }
  1130. }
  1131. for (i = 0; i < cpuhw->n_limited; ++i)
  1132. if (event == cpuhw->limited_counter[i])
  1133. break;
  1134. if (i < cpuhw->n_limited) {
  1135. while (++i < cpuhw->n_limited) {
  1136. cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
  1137. cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
  1138. }
  1139. --cpuhw->n_limited;
  1140. }
  1141. if (cpuhw->n_events == 0) {
  1142. /* disable exceptions if no events are running */
  1143. cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
  1144. }
  1145. if (has_branch_stack(event))
  1146. power_pmu_bhrb_disable(event);
  1147. perf_pmu_enable(event->pmu);
  1148. local_irq_restore(flags);
  1149. }
  1150. /*
  1151. * POWER-PMU does not support disabling individual counters, hence
  1152. * program their cycle counter to their max value and ignore the interrupts.
  1153. */
  1154. static void power_pmu_start(struct perf_event *event, int ef_flags)
  1155. {
  1156. unsigned long flags;
  1157. s64 left;
  1158. unsigned long val;
  1159. if (!event->hw.idx || !event->hw.sample_period)
  1160. return;
  1161. if (!(event->hw.state & PERF_HES_STOPPED))
  1162. return;
  1163. if (ef_flags & PERF_EF_RELOAD)
  1164. WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
  1165. local_irq_save(flags);
  1166. perf_pmu_disable(event->pmu);
  1167. event->hw.state = 0;
  1168. left = local64_read(&event->hw.period_left);
  1169. val = 0;
  1170. if (left < 0x80000000L)
  1171. val = 0x80000000L - left;
  1172. write_pmc(event->hw.idx, val);
  1173. perf_event_update_userpage(event);
  1174. perf_pmu_enable(event->pmu);
  1175. local_irq_restore(flags);
  1176. }
  1177. static void power_pmu_stop(struct perf_event *event, int ef_flags)
  1178. {
  1179. unsigned long flags;
  1180. if (!event->hw.idx || !event->hw.sample_period)
  1181. return;
  1182. if (event->hw.state & PERF_HES_STOPPED)
  1183. return;
  1184. local_irq_save(flags);
  1185. perf_pmu_disable(event->pmu);
  1186. power_pmu_read(event);
  1187. event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  1188. write_pmc(event->hw.idx, 0);
  1189. perf_event_update_userpage(event);
  1190. perf_pmu_enable(event->pmu);
  1191. local_irq_restore(flags);
  1192. }
  1193. /*
  1194. * Start group events scheduling transaction
  1195. * Set the flag to make pmu::enable() not perform the
  1196. * schedulability test, it will be performed at commit time
  1197. */
  1198. void power_pmu_start_txn(struct pmu *pmu)
  1199. {
  1200. struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
  1201. perf_pmu_disable(pmu);
  1202. cpuhw->group_flag |= PERF_EVENT_TXN;
  1203. cpuhw->n_txn_start = cpuhw->n_events;
  1204. }
  1205. /*
  1206. * Stop group events scheduling transaction
  1207. * Clear the flag and pmu::enable() will perform the
  1208. * schedulability test.
  1209. */
  1210. void power_pmu_cancel_txn(struct pmu *pmu)
  1211. {
  1212. struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
  1213. cpuhw->group_flag &= ~PERF_EVENT_TXN;
  1214. perf_pmu_enable(pmu);
  1215. }
  1216. /*
  1217. * Commit group events scheduling transaction
  1218. * Perform the group schedulability test as a whole
  1219. * Return 0 if success
  1220. */
  1221. int power_pmu_commit_txn(struct pmu *pmu)
  1222. {
  1223. struct cpu_hw_events *cpuhw;
  1224. long i, n;
  1225. if (!ppmu)
  1226. return -EAGAIN;
  1227. cpuhw = &__get_cpu_var(cpu_hw_events);
  1228. n = cpuhw->n_events;
  1229. if (check_excludes(cpuhw->event, cpuhw->flags, 0, n))
  1230. return -EAGAIN;
  1231. i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n);
  1232. if (i < 0)
  1233. return -EAGAIN;
  1234. for (i = cpuhw->n_txn_start; i < n; ++i)
  1235. cpuhw->event[i]->hw.config = cpuhw->events[i];
  1236. cpuhw->group_flag &= ~PERF_EVENT_TXN;
  1237. perf_pmu_enable(pmu);
  1238. return 0;
  1239. }
  1240. /*
  1241. * Return 1 if we might be able to put event on a limited PMC,
  1242. * or 0 if not.
  1243. * A event can only go on a limited PMC if it counts something
  1244. * that a limited PMC can count, doesn't require interrupts, and
  1245. * doesn't exclude any processor mode.
  1246. */
  1247. static int can_go_on_limited_pmc(struct perf_event *event, u64 ev,
  1248. unsigned int flags)
  1249. {
  1250. int n;
  1251. u64 alt[MAX_EVENT_ALTERNATIVES];
  1252. if (event->attr.exclude_user
  1253. || event->attr.exclude_kernel
  1254. || event->attr.exclude_hv
  1255. || event->attr.sample_period)
  1256. return 0;
  1257. if (ppmu->limited_pmc_event(ev))
  1258. return 1;
  1259. /*
  1260. * The requested event_id isn't on a limited PMC already;
  1261. * see if any alternative code goes on a limited PMC.
  1262. */
  1263. if (!ppmu->get_alternatives)
  1264. return 0;
  1265. flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
  1266. n = ppmu->get_alternatives(ev, flags, alt);
  1267. return n > 0;
  1268. }
  1269. /*
  1270. * Find an alternative event_id that goes on a normal PMC, if possible,
  1271. * and return the event_id code, or 0 if there is no such alternative.
  1272. * (Note: event_id code 0 is "don't count" on all machines.)
  1273. */
  1274. static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
  1275. {
  1276. u64 alt[MAX_EVENT_ALTERNATIVES];
  1277. int n;
  1278. flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
  1279. n = ppmu->get_alternatives(ev, flags, alt);
  1280. if (!n)
  1281. return 0;
  1282. return alt[0];
  1283. }
  1284. /* Number of perf_events counting hardware events */
  1285. static atomic_t num_events;
  1286. /* Used to avoid races in calling reserve/release_pmc_hardware */
  1287. static DEFINE_MUTEX(pmc_reserve_mutex);
  1288. /*
  1289. * Release the PMU if this is the last perf_event.
  1290. */
  1291. static void hw_perf_event_destroy(struct perf_event *event)
  1292. {
  1293. if (!atomic_add_unless(&num_events, -1, 1)) {
  1294. mutex_lock(&pmc_reserve_mutex);
  1295. if (atomic_dec_return(&num_events) == 0)
  1296. release_pmc_hardware();
  1297. mutex_unlock(&pmc_reserve_mutex);
  1298. }
  1299. }
  1300. /*
  1301. * Translate a generic cache event_id config to a raw event_id code.
  1302. */
  1303. static int hw_perf_cache_event(u64 config, u64 *eventp)
  1304. {
  1305. unsigned long type, op, result;
  1306. int ev;
  1307. if (!ppmu->cache_events)
  1308. return -EINVAL;
  1309. /* unpack config */
  1310. type = config & 0xff;
  1311. op = (config >> 8) & 0xff;
  1312. result = (config >> 16) & 0xff;
  1313. if (type >= PERF_COUNT_HW_CACHE_MAX ||
  1314. op >= PERF_COUNT_HW_CACHE_OP_MAX ||
  1315. result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  1316. return -EINVAL;
  1317. ev = (*ppmu->cache_events)[type][op][result];
  1318. if (ev == 0)
  1319. return -EOPNOTSUPP;
  1320. if (ev == -1)
  1321. return -EINVAL;
  1322. *eventp = ev;
  1323. return 0;
  1324. }
  1325. static int power_pmu_event_init(struct perf_event *event)
  1326. {
  1327. u64 ev;
  1328. unsigned long flags;
  1329. struct perf_event *ctrs[MAX_HWEVENTS];
  1330. u64 events[MAX_HWEVENTS];
  1331. unsigned int cflags[MAX_HWEVENTS];
  1332. int n;
  1333. int err;
  1334. struct cpu_hw_events *cpuhw;
  1335. if (!ppmu)
  1336. return -ENOENT;
  1337. if (has_branch_stack(event)) {
  1338. /* PMU has BHRB enabled */
  1339. if (!(ppmu->flags & PPMU_BHRB))
  1340. return -EOPNOTSUPP;
  1341. }
  1342. switch (event->attr.type) {
  1343. case PERF_TYPE_HARDWARE:
  1344. ev = event->attr.config;
  1345. if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
  1346. return -EOPNOTSUPP;
  1347. ev = ppmu->generic_events[ev];
  1348. break;
  1349. case PERF_TYPE_HW_CACHE:
  1350. err = hw_perf_cache_event(event->attr.config, &ev);
  1351. if (err)
  1352. return err;
  1353. break;
  1354. case PERF_TYPE_RAW:
  1355. ev = event->attr.config;
  1356. break;
  1357. default:
  1358. return -ENOENT;
  1359. }
  1360. event->hw.config_base = ev;
  1361. event->hw.idx = 0;
  1362. /*
  1363. * If we are not running on a hypervisor, force the
  1364. * exclude_hv bit to 0 so that we don't care what
  1365. * the user set it to.
  1366. */
  1367. if (!firmware_has_feature(FW_FEATURE_LPAR))
  1368. event->attr.exclude_hv = 0;
  1369. /*
  1370. * If this is a per-task event, then we can use
  1371. * PM_RUN_* events interchangeably with their non RUN_*
  1372. * equivalents, e.g. PM_RUN_CYC instead of PM_CYC.
  1373. * XXX we should check if the task is an idle task.
  1374. */
  1375. flags = 0;
  1376. if (event->attach_state & PERF_ATTACH_TASK)
  1377. flags |= PPMU_ONLY_COUNT_RUN;
  1378. /*
  1379. * If this machine has limited events, check whether this
  1380. * event_id could go on a limited event.
  1381. */
  1382. if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
  1383. if (can_go_on_limited_pmc(event, ev, flags)) {
  1384. flags |= PPMU_LIMITED_PMC_OK;
  1385. } else if (ppmu->limited_pmc_event(ev)) {
  1386. /*
  1387. * The requested event_id is on a limited PMC,
  1388. * but we can't use a limited PMC; see if any
  1389. * alternative goes on a normal PMC.
  1390. */
  1391. ev = normal_pmc_alternative(ev, flags);
  1392. if (!ev)
  1393. return -EINVAL;
  1394. }
  1395. }
  1396. /* Extra checks for EBB */
  1397. err = ebb_event_check(event);
  1398. if (err)
  1399. return err;
  1400. /*
  1401. * If this is in a group, check if it can go on with all the
  1402. * other hardware events in the group. We assume the event
  1403. * hasn't been linked into its leader's sibling list at this point.
  1404. */
  1405. n = 0;
  1406. if (event->group_leader != event) {
  1407. n = collect_events(event->group_leader, ppmu->n_counter - 1,
  1408. ctrs, events, cflags);
  1409. if (n < 0)
  1410. return -EINVAL;
  1411. }
  1412. events[n] = ev;
  1413. ctrs[n] = event;
  1414. cflags[n] = flags;
  1415. if (check_excludes(ctrs, cflags, n, 1))
  1416. return -EINVAL;
  1417. cpuhw = &get_cpu_var(cpu_hw_events);
  1418. err = power_check_constraints(cpuhw, events, cflags, n + 1);
  1419. if (has_branch_stack(event)) {
  1420. cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
  1421. event->attr.branch_sample_type);
  1422. if(cpuhw->bhrb_filter == -1)
  1423. return -EOPNOTSUPP;
  1424. }
  1425. put_cpu_var(cpu_hw_events);
  1426. if (err)
  1427. return -EINVAL;
  1428. event->hw.config = events[n];
  1429. event->hw.event_base = cflags[n];
  1430. event->hw.last_period = event->hw.sample_period;
  1431. local64_set(&event->hw.period_left, event->hw.last_period);
  1432. /*
  1433. * For EBB events we just context switch the PMC value, we don't do any
  1434. * of the sample_period logic. We use hw.prev_count for this.
  1435. */
  1436. if (is_ebb_event(event))
  1437. local64_set(&event->hw.prev_count, 0);
  1438. /*
  1439. * See if we need to reserve the PMU.
  1440. * If no events are currently in use, then we have to take a
  1441. * mutex to ensure that we don't race with another task doing
  1442. * reserve_pmc_hardware or release_pmc_hardware.
  1443. */
  1444. err = 0;
  1445. if (!atomic_inc_not_zero(&num_events)) {
  1446. mutex_lock(&pmc_reserve_mutex);
  1447. if (atomic_read(&num_events) == 0 &&
  1448. reserve_pmc_hardware(perf_event_interrupt))
  1449. err = -EBUSY;
  1450. else
  1451. atomic_inc(&num_events);
  1452. mutex_unlock(&pmc_reserve_mutex);
  1453. }
  1454. event->destroy = hw_perf_event_destroy;
  1455. return err;
  1456. }
  1457. static int power_pmu_event_idx(struct perf_event *event)
  1458. {
  1459. return event->hw.idx;
  1460. }
  1461. ssize_t power_events_sysfs_show(struct device *dev,
  1462. struct device_attribute *attr, char *page)
  1463. {
  1464. struct perf_pmu_events_attr *pmu_attr;
  1465. pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
  1466. return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
  1467. }
  1468. struct pmu power_pmu = {
  1469. .pmu_enable = power_pmu_enable,
  1470. .pmu_disable = power_pmu_disable,
  1471. .event_init = power_pmu_event_init,
  1472. .add = power_pmu_add,
  1473. .del = power_pmu_del,
  1474. .start = power_pmu_start,
  1475. .stop = power_pmu_stop,
  1476. .read = power_pmu_read,
  1477. .start_txn = power_pmu_start_txn,
  1478. .cancel_txn = power_pmu_cancel_txn,
  1479. .commit_txn = power_pmu_commit_txn,
  1480. .event_idx = power_pmu_event_idx,
  1481. .flush_branch_stack = power_pmu_flush_branch_stack,
  1482. };
  1483. /*
  1484. * A counter has overflowed; update its count and record
  1485. * things if requested. Note that interrupts are hard-disabled
  1486. * here so there is no possibility of being interrupted.
  1487. */
  1488. static void record_and_restart(struct perf_event *event, unsigned long val,
  1489. struct pt_regs *regs)
  1490. {
  1491. u64 period = event->hw.sample_period;
  1492. s64 prev, delta, left;
  1493. int record = 0;
  1494. if (event->hw.state & PERF_HES_STOPPED) {
  1495. write_pmc(event->hw.idx, 0);
  1496. return;
  1497. }
  1498. /* we don't have to worry about interrupts here */
  1499. prev = local64_read(&event->hw.prev_count);
  1500. delta = check_and_compute_delta(prev, val);
  1501. local64_add(delta, &event->count);
  1502. /*
  1503. * See if the total period for this event has expired,
  1504. * and update for the next period.
  1505. */
  1506. val = 0;
  1507. left = local64_read(&event->hw.period_left) - delta;
  1508. if (delta == 0)
  1509. left++;
  1510. if (period) {
  1511. if (left <= 0) {
  1512. left += period;
  1513. if (left <= 0)
  1514. left = period;
  1515. record = siar_valid(regs);
  1516. event->hw.last_period = event->hw.sample_period;
  1517. }
  1518. if (left < 0x80000000LL)
  1519. val = 0x80000000LL - left;
  1520. }
  1521. write_pmc(event->hw.idx, val);
  1522. local64_set(&event->hw.prev_count, val);
  1523. local64_set(&event->hw.period_left, left);
  1524. perf_event_update_userpage(event);
  1525. /*
  1526. * Finally record data if requested.
  1527. */
  1528. if (record) {
  1529. struct perf_sample_data data;
  1530. perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
  1531. if (event->attr.sample_type & PERF_SAMPLE_ADDR)
  1532. perf_get_data_addr(regs, &data.addr);
  1533. if (event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK) {
  1534. struct cpu_hw_events *cpuhw;
  1535. cpuhw = &__get_cpu_var(cpu_hw_events);
  1536. power_pmu_bhrb_read(cpuhw);
  1537. data.br_stack = &cpuhw->bhrb_stack;
  1538. }
  1539. if (perf_event_overflow(event, &data, regs))
  1540. power_pmu_stop(event, 0);
  1541. }
  1542. }
  1543. /*
  1544. * Called from generic code to get the misc flags (i.e. processor mode)
  1545. * for an event_id.
  1546. */
  1547. unsigned long perf_misc_flags(struct pt_regs *regs)
  1548. {
  1549. u32 flags = perf_get_misc_flags(regs);
  1550. if (flags)
  1551. return flags;
  1552. return user_mode(regs) ? PERF_RECORD_MISC_USER :
  1553. PERF_RECORD_MISC_KERNEL;
  1554. }
  1555. /*
  1556. * Called from generic code to get the instruction pointer
  1557. * for an event_id.
  1558. */
  1559. unsigned long perf_instruction_pointer(struct pt_regs *regs)
  1560. {
  1561. bool use_siar = regs_use_siar(regs);
  1562. if (use_siar && siar_valid(regs))
  1563. return mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
  1564. else if (use_siar)
  1565. return 0; // no valid instruction pointer
  1566. else
  1567. return regs->nip;
  1568. }
  1569. static bool pmc_overflow_power7(unsigned long val)
  1570. {
  1571. /*
  1572. * Events on POWER7 can roll back if a speculative event doesn't
  1573. * eventually complete. Unfortunately in some rare cases they will
  1574. * raise a performance monitor exception. We need to catch this to
  1575. * ensure we reset the PMC. In all cases the PMC will be 256 or less
  1576. * cycles from overflow.
  1577. *
  1578. * We only do this if the first pass fails to find any overflowing
  1579. * PMCs because a user might set a period of less than 256 and we
  1580. * don't want to mistakenly reset them.
  1581. */
  1582. if ((0x80000000 - val) <= 256)
  1583. return true;
  1584. return false;
  1585. }
  1586. static bool pmc_overflow(unsigned long val)
  1587. {
  1588. if ((int)val < 0)
  1589. return true;
  1590. return false;
  1591. }
  1592. /*
  1593. * Performance monitor interrupt stuff
  1594. */
  1595. static void perf_event_interrupt(struct pt_regs *regs)
  1596. {
  1597. int i, j;
  1598. struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
  1599. struct perf_event *event;
  1600. unsigned long val[8];
  1601. int found, active;
  1602. int nmi;
  1603. if (cpuhw->n_limited)
  1604. freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
  1605. mfspr(SPRN_PMC6));
  1606. perf_read_regs(regs);
  1607. nmi = perf_intr_is_nmi(regs);
  1608. if (nmi)
  1609. nmi_enter();
  1610. else
  1611. irq_enter();
  1612. /* Read all the PMCs since we'll need them a bunch of times */
  1613. for (i = 0; i < ppmu->n_counter; ++i)
  1614. val[i] = read_pmc(i + 1);
  1615. /* Try to find what caused the IRQ */
  1616. found = 0;
  1617. for (i = 0; i < ppmu->n_counter; ++i) {
  1618. if (!pmc_overflow(val[i]))
  1619. continue;
  1620. if (is_limited_pmc(i + 1))
  1621. continue; /* these won't generate IRQs */
  1622. /*
  1623. * We've found one that's overflowed. For active
  1624. * counters we need to log this. For inactive
  1625. * counters, we need to reset it anyway
  1626. */
  1627. found = 1;
  1628. active = 0;
  1629. for (j = 0; j < cpuhw->n_events; ++j) {
  1630. event = cpuhw->event[j];
  1631. if (event->hw.idx == (i + 1)) {
  1632. active = 1;
  1633. record_and_restart(event, val[i], regs);
  1634. break;
  1635. }
  1636. }
  1637. if (!active)
  1638. /* reset non active counters that have overflowed */
  1639. write_pmc(i + 1, 0);
  1640. }
  1641. if (!found && pvr_version_is(PVR_POWER7)) {
  1642. /* check active counters for special buggy p7 overflow */
  1643. for (i = 0; i < cpuhw->n_events; ++i) {
  1644. event = cpuhw->event[i];
  1645. if (!event->hw.idx || is_limited_pmc(event->hw.idx))
  1646. continue;
  1647. if (pmc_overflow_power7(val[event->hw.idx - 1])) {
  1648. /* event has overflowed in a buggy way*/
  1649. found = 1;
  1650. record_and_restart(event,
  1651. val[event->hw.idx - 1],
  1652. regs);
  1653. }
  1654. }
  1655. }
  1656. if (!found && !nmi && printk_ratelimit())
  1657. printk(KERN_WARNING "Can't find PMC that caused IRQ\n");
  1658. /*
  1659. * Reset MMCR0 to its normal value. This will set PMXE and
  1660. * clear FC (freeze counters) and PMAO (perf mon alert occurred)
  1661. * and thus allow interrupts to occur again.
  1662. * XXX might want to use MSR.PM to keep the events frozen until
  1663. * we get back out of this interrupt.
  1664. */
  1665. write_mmcr0(cpuhw, cpuhw->mmcr[0]);
  1666. if (nmi)
  1667. nmi_exit();
  1668. else
  1669. irq_exit();
  1670. }
  1671. static void power_pmu_setup(int cpu)
  1672. {
  1673. struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
  1674. if (!ppmu)
  1675. return;
  1676. memset(cpuhw, 0, sizeof(*cpuhw));
  1677. cpuhw->mmcr[0] = MMCR0_FC;
  1678. }
  1679. static int
  1680. power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
  1681. {
  1682. unsigned int cpu = (long)hcpu;
  1683. switch (action & ~CPU_TASKS_FROZEN) {
  1684. case CPU_UP_PREPARE:
  1685. power_pmu_setup(cpu);
  1686. break;
  1687. default:
  1688. break;
  1689. }
  1690. return NOTIFY_OK;
  1691. }
  1692. int register_power_pmu(struct power_pmu *pmu)
  1693. {
  1694. if (ppmu)
  1695. return -EBUSY; /* something's already registered */
  1696. ppmu = pmu;
  1697. pr_info("%s performance monitor hardware support registered\n",
  1698. pmu->name);
  1699. power_pmu.attr_groups = ppmu->attr_groups;
  1700. #ifdef MSR_HV
  1701. /*
  1702. * Use FCHV to ignore kernel events if MSR.HV is set.
  1703. */
  1704. if (mfmsr() & MSR_HV)
  1705. freeze_events_kernel = MMCR0_FCHV;
  1706. #endif /* CONFIG_PPC64 */
  1707. perf_pmu_register(&power_pmu, "cpu", PERF_TYPE_RAW);
  1708. perf_cpu_notifier(power_pmu_notifier);
  1709. return 0;
  1710. }