perf_event_intel_ds.c 26 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028
  1. #include <linux/bitops.h>
  2. #include <linux/types.h>
  3. #include <linux/slab.h>
  4. #include <asm/perf_event.h>
  5. #include <asm/insn.h>
  6. #include "perf_event.h"
  7. /* The size of a BTS record in bytes: */
  8. #define BTS_RECORD_SIZE 24
  9. #define BTS_BUFFER_SIZE (PAGE_SIZE << 4)
  10. #define PEBS_BUFFER_SIZE PAGE_SIZE
  11. /*
  12. * pebs_record_32 for p4 and core not supported
  13. struct pebs_record_32 {
  14. u32 flags, ip;
  15. u32 ax, bc, cx, dx;
  16. u32 si, di, bp, sp;
  17. };
  18. */
  19. union intel_x86_pebs_dse {
  20. u64 val;
  21. struct {
  22. unsigned int ld_dse:4;
  23. unsigned int ld_stlb_miss:1;
  24. unsigned int ld_locked:1;
  25. unsigned int ld_reserved:26;
  26. };
  27. struct {
  28. unsigned int st_l1d_hit:1;
  29. unsigned int st_reserved1:3;
  30. unsigned int st_stlb_miss:1;
  31. unsigned int st_locked:1;
  32. unsigned int st_reserved2:26;
  33. };
  34. };
  35. /*
  36. * Map PEBS Load Latency Data Source encodings to generic
  37. * memory data source information
  38. */
  39. #define P(a, b) PERF_MEM_S(a, b)
  40. #define OP_LH (P(OP, LOAD) | P(LVL, HIT))
  41. #define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS))
  42. static const u64 pebs_data_source[] = {
  43. P(OP, LOAD) | P(LVL, MISS) | P(LVL, L3) | P(SNOOP, NA),/* 0x00:ukn L3 */
  44. OP_LH | P(LVL, L1) | P(SNOOP, NONE), /* 0x01: L1 local */
  45. OP_LH | P(LVL, LFB) | P(SNOOP, NONE), /* 0x02: LFB hit */
  46. OP_LH | P(LVL, L2) | P(SNOOP, NONE), /* 0x03: L2 hit */
  47. OP_LH | P(LVL, L3) | P(SNOOP, NONE), /* 0x04: L3 hit */
  48. OP_LH | P(LVL, L3) | P(SNOOP, MISS), /* 0x05: L3 hit, snoop miss */
  49. OP_LH | P(LVL, L3) | P(SNOOP, HIT), /* 0x06: L3 hit, snoop hit */
  50. OP_LH | P(LVL, L3) | P(SNOOP, HITM), /* 0x07: L3 hit, snoop hitm */
  51. OP_LH | P(LVL, REM_CCE1) | P(SNOOP, HIT), /* 0x08: L3 miss snoop hit */
  52. OP_LH | P(LVL, REM_CCE1) | P(SNOOP, HITM), /* 0x09: L3 miss snoop hitm*/
  53. OP_LH | P(LVL, LOC_RAM) | P(SNOOP, HIT), /* 0x0a: L3 miss, shared */
  54. OP_LH | P(LVL, REM_RAM1) | P(SNOOP, HIT), /* 0x0b: L3 miss, shared */
  55. OP_LH | P(LVL, LOC_RAM) | SNOOP_NONE_MISS,/* 0x0c: L3 miss, excl */
  56. OP_LH | P(LVL, REM_RAM1) | SNOOP_NONE_MISS,/* 0x0d: L3 miss, excl */
  57. OP_LH | P(LVL, IO) | P(SNOOP, NONE), /* 0x0e: I/O */
  58. OP_LH | P(LVL, UNC) | P(SNOOP, NONE), /* 0x0f: uncached */
  59. };
  60. static u64 precise_store_data(u64 status)
  61. {
  62. union intel_x86_pebs_dse dse;
  63. u64 val = P(OP, STORE) | P(SNOOP, NA) | P(LVL, L1) | P(TLB, L2);
  64. dse.val = status;
  65. /*
  66. * bit 4: TLB access
  67. * 1 = stored missed 2nd level TLB
  68. *
  69. * so it either hit the walker or the OS
  70. * otherwise hit 2nd level TLB
  71. */
  72. if (dse.st_stlb_miss)
  73. val |= P(TLB, MISS);
  74. else
  75. val |= P(TLB, HIT);
  76. /*
  77. * bit 0: hit L1 data cache
  78. * if not set, then all we know is that
  79. * it missed L1D
  80. */
  81. if (dse.st_l1d_hit)
  82. val |= P(LVL, HIT);
  83. else
  84. val |= P(LVL, MISS);
  85. /*
  86. * bit 5: Locked prefix
  87. */
  88. if (dse.st_locked)
  89. val |= P(LOCK, LOCKED);
  90. return val;
  91. }
  92. static u64 precise_store_data_hsw(u64 status)
  93. {
  94. union perf_mem_data_src dse;
  95. dse.val = 0;
  96. dse.mem_op = PERF_MEM_OP_STORE;
  97. dse.mem_lvl = PERF_MEM_LVL_NA;
  98. if (status & 1)
  99. dse.mem_lvl = PERF_MEM_LVL_L1;
  100. /* Nothing else supported. Sorry. */
  101. return dse.val;
  102. }
  103. static u64 load_latency_data(u64 status)
  104. {
  105. union intel_x86_pebs_dse dse;
  106. u64 val;
  107. int model = boot_cpu_data.x86_model;
  108. int fam = boot_cpu_data.x86;
  109. dse.val = status;
  110. /*
  111. * use the mapping table for bit 0-3
  112. */
  113. val = pebs_data_source[dse.ld_dse];
  114. /*
  115. * Nehalem models do not support TLB, Lock infos
  116. */
  117. if (fam == 0x6 && (model == 26 || model == 30
  118. || model == 31 || model == 46)) {
  119. val |= P(TLB, NA) | P(LOCK, NA);
  120. return val;
  121. }
  122. /*
  123. * bit 4: TLB access
  124. * 0 = did not miss 2nd level TLB
  125. * 1 = missed 2nd level TLB
  126. */
  127. if (dse.ld_stlb_miss)
  128. val |= P(TLB, MISS) | P(TLB, L2);
  129. else
  130. val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
  131. /*
  132. * bit 5: locked prefix
  133. */
  134. if (dse.ld_locked)
  135. val |= P(LOCK, LOCKED);
  136. return val;
  137. }
  138. struct pebs_record_core {
  139. u64 flags, ip;
  140. u64 ax, bx, cx, dx;
  141. u64 si, di, bp, sp;
  142. u64 r8, r9, r10, r11;
  143. u64 r12, r13, r14, r15;
  144. };
  145. struct pebs_record_nhm {
  146. u64 flags, ip;
  147. u64 ax, bx, cx, dx;
  148. u64 si, di, bp, sp;
  149. u64 r8, r9, r10, r11;
  150. u64 r12, r13, r14, r15;
  151. u64 status, dla, dse, lat;
  152. };
  153. /*
  154. * Same as pebs_record_nhm, with two additional fields.
  155. */
  156. struct pebs_record_hsw {
  157. struct pebs_record_nhm nhm;
  158. /*
  159. * Real IP of the event. In the Intel documentation this
  160. * is called eventingrip.
  161. */
  162. u64 real_ip;
  163. /*
  164. * TSX tuning information field: abort cycles and abort flags.
  165. */
  166. u64 tsx_tuning;
  167. };
  168. void init_debug_store_on_cpu(int cpu)
  169. {
  170. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  171. if (!ds)
  172. return;
  173. wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
  174. (u32)((u64)(unsigned long)ds),
  175. (u32)((u64)(unsigned long)ds >> 32));
  176. }
  177. void fini_debug_store_on_cpu(int cpu)
  178. {
  179. if (!per_cpu(cpu_hw_events, cpu).ds)
  180. return;
  181. wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
  182. }
  183. static int alloc_pebs_buffer(int cpu)
  184. {
  185. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  186. int node = cpu_to_node(cpu);
  187. int max, thresh = 1; /* always use a single PEBS record */
  188. void *buffer;
  189. if (!x86_pmu.pebs)
  190. return 0;
  191. buffer = kmalloc_node(PEBS_BUFFER_SIZE, GFP_KERNEL | __GFP_ZERO, node);
  192. if (unlikely(!buffer))
  193. return -ENOMEM;
  194. max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
  195. ds->pebs_buffer_base = (u64)(unsigned long)buffer;
  196. ds->pebs_index = ds->pebs_buffer_base;
  197. ds->pebs_absolute_maximum = ds->pebs_buffer_base +
  198. max * x86_pmu.pebs_record_size;
  199. ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
  200. thresh * x86_pmu.pebs_record_size;
  201. return 0;
  202. }
  203. static void release_pebs_buffer(int cpu)
  204. {
  205. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  206. if (!ds || !x86_pmu.pebs)
  207. return;
  208. kfree((void *)(unsigned long)ds->pebs_buffer_base);
  209. ds->pebs_buffer_base = 0;
  210. }
  211. static int alloc_bts_buffer(int cpu)
  212. {
  213. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  214. int node = cpu_to_node(cpu);
  215. int max, thresh;
  216. void *buffer;
  217. if (!x86_pmu.bts)
  218. return 0;
  219. buffer = kmalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_ZERO, node);
  220. if (unlikely(!buffer))
  221. return -ENOMEM;
  222. max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
  223. thresh = max / 16;
  224. ds->bts_buffer_base = (u64)(unsigned long)buffer;
  225. ds->bts_index = ds->bts_buffer_base;
  226. ds->bts_absolute_maximum = ds->bts_buffer_base +
  227. max * BTS_RECORD_SIZE;
  228. ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
  229. thresh * BTS_RECORD_SIZE;
  230. return 0;
  231. }
  232. static void release_bts_buffer(int cpu)
  233. {
  234. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  235. if (!ds || !x86_pmu.bts)
  236. return;
  237. kfree((void *)(unsigned long)ds->bts_buffer_base);
  238. ds->bts_buffer_base = 0;
  239. }
  240. static int alloc_ds_buffer(int cpu)
  241. {
  242. int node = cpu_to_node(cpu);
  243. struct debug_store *ds;
  244. ds = kmalloc_node(sizeof(*ds), GFP_KERNEL | __GFP_ZERO, node);
  245. if (unlikely(!ds))
  246. return -ENOMEM;
  247. per_cpu(cpu_hw_events, cpu).ds = ds;
  248. return 0;
  249. }
  250. static void release_ds_buffer(int cpu)
  251. {
  252. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  253. if (!ds)
  254. return;
  255. per_cpu(cpu_hw_events, cpu).ds = NULL;
  256. kfree(ds);
  257. }
  258. void release_ds_buffers(void)
  259. {
  260. int cpu;
  261. if (!x86_pmu.bts && !x86_pmu.pebs)
  262. return;
  263. get_online_cpus();
  264. for_each_online_cpu(cpu)
  265. fini_debug_store_on_cpu(cpu);
  266. for_each_possible_cpu(cpu) {
  267. release_pebs_buffer(cpu);
  268. release_bts_buffer(cpu);
  269. release_ds_buffer(cpu);
  270. }
  271. put_online_cpus();
  272. }
  273. void reserve_ds_buffers(void)
  274. {
  275. int bts_err = 0, pebs_err = 0;
  276. int cpu;
  277. x86_pmu.bts_active = 0;
  278. x86_pmu.pebs_active = 0;
  279. if (!x86_pmu.bts && !x86_pmu.pebs)
  280. return;
  281. if (!x86_pmu.bts)
  282. bts_err = 1;
  283. if (!x86_pmu.pebs)
  284. pebs_err = 1;
  285. get_online_cpus();
  286. for_each_possible_cpu(cpu) {
  287. if (alloc_ds_buffer(cpu)) {
  288. bts_err = 1;
  289. pebs_err = 1;
  290. }
  291. if (!bts_err && alloc_bts_buffer(cpu))
  292. bts_err = 1;
  293. if (!pebs_err && alloc_pebs_buffer(cpu))
  294. pebs_err = 1;
  295. if (bts_err && pebs_err)
  296. break;
  297. }
  298. if (bts_err) {
  299. for_each_possible_cpu(cpu)
  300. release_bts_buffer(cpu);
  301. }
  302. if (pebs_err) {
  303. for_each_possible_cpu(cpu)
  304. release_pebs_buffer(cpu);
  305. }
  306. if (bts_err && pebs_err) {
  307. for_each_possible_cpu(cpu)
  308. release_ds_buffer(cpu);
  309. } else {
  310. if (x86_pmu.bts && !bts_err)
  311. x86_pmu.bts_active = 1;
  312. if (x86_pmu.pebs && !pebs_err)
  313. x86_pmu.pebs_active = 1;
  314. for_each_online_cpu(cpu)
  315. init_debug_store_on_cpu(cpu);
  316. }
  317. put_online_cpus();
  318. }
  319. /*
  320. * BTS
  321. */
  322. struct event_constraint bts_constraint =
  323. EVENT_CONSTRAINT(0, 1ULL << INTEL_PMC_IDX_FIXED_BTS, 0);
  324. void intel_pmu_enable_bts(u64 config)
  325. {
  326. unsigned long debugctlmsr;
  327. debugctlmsr = get_debugctlmsr();
  328. debugctlmsr |= DEBUGCTLMSR_TR;
  329. debugctlmsr |= DEBUGCTLMSR_BTS;
  330. debugctlmsr |= DEBUGCTLMSR_BTINT;
  331. if (!(config & ARCH_PERFMON_EVENTSEL_OS))
  332. debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
  333. if (!(config & ARCH_PERFMON_EVENTSEL_USR))
  334. debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
  335. update_debugctlmsr(debugctlmsr);
  336. }
  337. void intel_pmu_disable_bts(void)
  338. {
  339. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  340. unsigned long debugctlmsr;
  341. if (!cpuc->ds)
  342. return;
  343. debugctlmsr = get_debugctlmsr();
  344. debugctlmsr &=
  345. ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
  346. DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
  347. update_debugctlmsr(debugctlmsr);
  348. }
  349. int intel_pmu_drain_bts_buffer(void)
  350. {
  351. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  352. struct debug_store *ds = cpuc->ds;
  353. struct bts_record {
  354. u64 from;
  355. u64 to;
  356. u64 flags;
  357. };
  358. struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
  359. struct bts_record *at, *top;
  360. struct perf_output_handle handle;
  361. struct perf_event_header header;
  362. struct perf_sample_data data;
  363. struct pt_regs regs;
  364. if (!event)
  365. return 0;
  366. if (!x86_pmu.bts_active)
  367. return 0;
  368. at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
  369. top = (struct bts_record *)(unsigned long)ds->bts_index;
  370. if (top <= at)
  371. return 0;
  372. memset(&regs, 0, sizeof(regs));
  373. ds->bts_index = ds->bts_buffer_base;
  374. perf_sample_data_init(&data, 0, event->hw.last_period);
  375. /*
  376. * Prepare a generic sample, i.e. fill in the invariant fields.
  377. * We will overwrite the from and to address before we output
  378. * the sample.
  379. */
  380. perf_prepare_sample(&header, &data, event, &regs);
  381. if (perf_output_begin(&handle, event, header.size * (top - at)))
  382. return 1;
  383. for (; at < top; at++) {
  384. data.ip = at->from;
  385. data.addr = at->to;
  386. perf_output_sample(&handle, &header, &data, event);
  387. }
  388. perf_output_end(&handle);
  389. /* There's new data available. */
  390. event->hw.interrupts++;
  391. event->pending_kill = POLL_IN;
  392. return 1;
  393. }
  394. /*
  395. * PEBS
  396. */
  397. struct event_constraint intel_core2_pebs_event_constraints[] = {
  398. INTEL_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
  399. INTEL_UEVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
  400. INTEL_UEVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
  401. INTEL_UEVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
  402. INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED.* */
  403. EVENT_CONSTRAINT_END
  404. };
  405. struct event_constraint intel_atom_pebs_event_constraints[] = {
  406. INTEL_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
  407. INTEL_UEVENT_CONSTRAINT(0x00c5, 0x1), /* MISPREDICTED_BRANCH_RETIRED */
  408. INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED.* */
  409. EVENT_CONSTRAINT_END
  410. };
  411. struct event_constraint intel_nehalem_pebs_event_constraints[] = {
  412. INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */
  413. INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
  414. INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
  415. INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INST_RETIRED.ANY */
  416. INTEL_EVENT_CONSTRAINT(0xc2, 0xf), /* UOPS_RETIRED.* */
  417. INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
  418. INTEL_UEVENT_CONSTRAINT(0x02c5, 0xf), /* BR_MISP_RETIRED.NEAR_CALL */
  419. INTEL_EVENT_CONSTRAINT(0xc7, 0xf), /* SSEX_UOPS_RETIRED.* */
  420. INTEL_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
  421. INTEL_EVENT_CONSTRAINT(0xcb, 0xf), /* MEM_LOAD_RETIRED.* */
  422. INTEL_EVENT_CONSTRAINT(0xf7, 0xf), /* FP_ASSIST.* */
  423. EVENT_CONSTRAINT_END
  424. };
  425. struct event_constraint intel_westmere_pebs_event_constraints[] = {
  426. INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */
  427. INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
  428. INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
  429. INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INSTR_RETIRED.* */
  430. INTEL_EVENT_CONSTRAINT(0xc2, 0xf), /* UOPS_RETIRED.* */
  431. INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
  432. INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
  433. INTEL_EVENT_CONSTRAINT(0xc7, 0xf), /* SSEX_UOPS_RETIRED.* */
  434. INTEL_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
  435. INTEL_EVENT_CONSTRAINT(0xcb, 0xf), /* MEM_LOAD_RETIRED.* */
  436. INTEL_EVENT_CONSTRAINT(0xf7, 0xf), /* FP_ASSIST.* */
  437. EVENT_CONSTRAINT_END
  438. };
  439. struct event_constraint intel_snb_pebs_event_constraints[] = {
  440. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
  441. INTEL_UEVENT_CONSTRAINT(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
  442. INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
  443. INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
  444. INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
  445. INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
  446. INTEL_PST_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORES */
  447. INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
  448. INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
  449. INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
  450. INTEL_UEVENT_CONSTRAINT(0x02d4, 0xf), /* MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS */
  451. EVENT_CONSTRAINT_END
  452. };
  453. struct event_constraint intel_ivb_pebs_event_constraints[] = {
  454. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
  455. INTEL_UEVENT_CONSTRAINT(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
  456. INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
  457. INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
  458. INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
  459. INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
  460. INTEL_PST_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORES */
  461. INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
  462. INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
  463. INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
  464. INTEL_EVENT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
  465. EVENT_CONSTRAINT_END
  466. };
  467. struct event_constraint intel_hsw_pebs_event_constraints[] = {
  468. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
  469. INTEL_PST_HSW_CONSTRAINT(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
  470. INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
  471. INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
  472. INTEL_UEVENT_CONSTRAINT(0x01c5, 0xf), /* BR_MISP_RETIRED.CONDITIONAL */
  473. INTEL_UEVENT_CONSTRAINT(0x04c5, 0xf), /* BR_MISP_RETIRED.ALL_BRANCHES */
  474. INTEL_UEVENT_CONSTRAINT(0x20c5, 0xf), /* BR_MISP_RETIRED.NEAR_TAKEN */
  475. INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.* */
  476. /* MEM_UOPS_RETIRED.STLB_MISS_LOADS */
  477. INTEL_UEVENT_CONSTRAINT(0x11d0, 0xf),
  478. /* MEM_UOPS_RETIRED.STLB_MISS_STORES */
  479. INTEL_UEVENT_CONSTRAINT(0x12d0, 0xf),
  480. INTEL_UEVENT_CONSTRAINT(0x21d0, 0xf), /* MEM_UOPS_RETIRED.LOCK_LOADS */
  481. INTEL_UEVENT_CONSTRAINT(0x41d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_LOADS */
  482. /* MEM_UOPS_RETIRED.SPLIT_STORES */
  483. INTEL_UEVENT_CONSTRAINT(0x42d0, 0xf),
  484. INTEL_UEVENT_CONSTRAINT(0x81d0, 0xf), /* MEM_UOPS_RETIRED.ALL_LOADS */
  485. INTEL_PST_HSW_CONSTRAINT(0x82d0, 0xf), /* MEM_UOPS_RETIRED.ALL_STORES */
  486. INTEL_UEVENT_CONSTRAINT(0x01d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.L1_HIT */
  487. INTEL_UEVENT_CONSTRAINT(0x02d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.L2_HIT */
  488. INTEL_UEVENT_CONSTRAINT(0x04d1, 0xf), /* MEM_LOAD_UOPS_RETIRED.L3_HIT */
  489. /* MEM_LOAD_UOPS_RETIRED.HIT_LFB */
  490. INTEL_UEVENT_CONSTRAINT(0x40d1, 0xf),
  491. /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS */
  492. INTEL_UEVENT_CONSTRAINT(0x01d2, 0xf),
  493. /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT */
  494. INTEL_UEVENT_CONSTRAINT(0x02d2, 0xf),
  495. /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM */
  496. INTEL_UEVENT_CONSTRAINT(0x01d3, 0xf),
  497. INTEL_UEVENT_CONSTRAINT(0x04c8, 0xf), /* HLE_RETIRED.Abort */
  498. INTEL_UEVENT_CONSTRAINT(0x04c9, 0xf), /* RTM_RETIRED.Abort */
  499. EVENT_CONSTRAINT_END
  500. };
  501. struct event_constraint *intel_pebs_constraints(struct perf_event *event)
  502. {
  503. struct event_constraint *c;
  504. if (!event->attr.precise_ip)
  505. return NULL;
  506. if (x86_pmu.pebs_constraints) {
  507. for_each_event_constraint(c, x86_pmu.pebs_constraints) {
  508. if ((event->hw.config & c->cmask) == c->code) {
  509. event->hw.flags |= c->flags;
  510. return c;
  511. }
  512. }
  513. }
  514. return &emptyconstraint;
  515. }
  516. void intel_pmu_pebs_enable(struct perf_event *event)
  517. {
  518. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  519. struct hw_perf_event *hwc = &event->hw;
  520. hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
  521. cpuc->pebs_enabled |= 1ULL << hwc->idx;
  522. if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
  523. cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
  524. else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
  525. cpuc->pebs_enabled |= 1ULL << 63;
  526. }
  527. void intel_pmu_pebs_disable(struct perf_event *event)
  528. {
  529. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  530. struct hw_perf_event *hwc = &event->hw;
  531. cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
  532. if (cpuc->enabled)
  533. wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
  534. hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
  535. }
  536. void intel_pmu_pebs_enable_all(void)
  537. {
  538. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  539. if (cpuc->pebs_enabled)
  540. wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
  541. }
  542. void intel_pmu_pebs_disable_all(void)
  543. {
  544. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  545. if (cpuc->pebs_enabled)
  546. wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
  547. }
  548. static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
  549. {
  550. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  551. unsigned long from = cpuc->lbr_entries[0].from;
  552. unsigned long old_to, to = cpuc->lbr_entries[0].to;
  553. unsigned long ip = regs->ip;
  554. int is_64bit = 0;
  555. /*
  556. * We don't need to fixup if the PEBS assist is fault like
  557. */
  558. if (!x86_pmu.intel_cap.pebs_trap)
  559. return 1;
  560. /*
  561. * No LBR entry, no basic block, no rewinding
  562. */
  563. if (!cpuc->lbr_stack.nr || !from || !to)
  564. return 0;
  565. /*
  566. * Basic blocks should never cross user/kernel boundaries
  567. */
  568. if (kernel_ip(ip) != kernel_ip(to))
  569. return 0;
  570. /*
  571. * unsigned math, either ip is before the start (impossible) or
  572. * the basic block is larger than 1 page (sanity)
  573. */
  574. if ((ip - to) > PAGE_SIZE)
  575. return 0;
  576. /*
  577. * We sampled a branch insn, rewind using the LBR stack
  578. */
  579. if (ip == to) {
  580. set_linear_ip(regs, from);
  581. return 1;
  582. }
  583. do {
  584. struct insn insn;
  585. u8 buf[MAX_INSN_SIZE];
  586. void *kaddr;
  587. old_to = to;
  588. if (!kernel_ip(ip)) {
  589. int bytes, size = MAX_INSN_SIZE;
  590. bytes = copy_from_user_nmi(buf, (void __user *)to, size);
  591. if (bytes != size)
  592. return 0;
  593. kaddr = buf;
  594. } else
  595. kaddr = (void *)to;
  596. #ifdef CONFIG_X86_64
  597. is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
  598. #endif
  599. insn_init(&insn, kaddr, is_64bit);
  600. insn_get_length(&insn);
  601. to += insn.length;
  602. } while (to < ip);
  603. if (to == ip) {
  604. set_linear_ip(regs, old_to);
  605. return 1;
  606. }
  607. /*
  608. * Even though we decoded the basic block, the instruction stream
  609. * never matched the given IP, either the TO or the IP got corrupted.
  610. */
  611. return 0;
  612. }
  613. static void __intel_pmu_pebs_event(struct perf_event *event,
  614. struct pt_regs *iregs, void *__pebs)
  615. {
  616. /*
  617. * We cast to pebs_record_nhm to get the load latency data
  618. * if extra_reg MSR_PEBS_LD_LAT_THRESHOLD used
  619. */
  620. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  621. struct pebs_record_nhm *pebs = __pebs;
  622. struct pebs_record_hsw *pebs_hsw = __pebs;
  623. struct perf_sample_data data;
  624. struct pt_regs regs;
  625. u64 sample_type;
  626. int fll, fst;
  627. if (!intel_pmu_save_and_restart(event))
  628. return;
  629. fll = event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT;
  630. fst = event->hw.flags & (PERF_X86_EVENT_PEBS_ST |
  631. PERF_X86_EVENT_PEBS_ST_HSW);
  632. perf_sample_data_init(&data, 0, event->hw.last_period);
  633. data.period = event->hw.last_period;
  634. sample_type = event->attr.sample_type;
  635. /*
  636. * if PEBS-LL or PreciseStore
  637. */
  638. if (fll || fst) {
  639. /*
  640. * Use latency for weight (only avail with PEBS-LL)
  641. */
  642. if (fll && (sample_type & PERF_SAMPLE_WEIGHT))
  643. data.weight = pebs->lat;
  644. /*
  645. * data.data_src encodes the data source
  646. */
  647. if (sample_type & PERF_SAMPLE_DATA_SRC) {
  648. if (fll)
  649. data.data_src.val = load_latency_data(pebs->dse);
  650. else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
  651. data.data_src.val =
  652. precise_store_data_hsw(pebs->dse);
  653. else
  654. data.data_src.val = precise_store_data(pebs->dse);
  655. }
  656. }
  657. /*
  658. * We use the interrupt regs as a base because the PEBS record
  659. * does not contain a full regs set, specifically it seems to
  660. * lack segment descriptors, which get used by things like
  661. * user_mode().
  662. *
  663. * In the simple case fix up only the IP and BP,SP regs, for
  664. * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
  665. * A possible PERF_SAMPLE_REGS will have to transfer all regs.
  666. */
  667. regs = *iregs;
  668. regs.flags = pebs->flags;
  669. set_linear_ip(&regs, pebs->ip);
  670. regs.bp = pebs->bp;
  671. regs.sp = pebs->sp;
  672. if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format >= 2) {
  673. regs.ip = pebs_hsw->real_ip;
  674. regs.flags |= PERF_EFLAGS_EXACT;
  675. } else if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(&regs))
  676. regs.flags |= PERF_EFLAGS_EXACT;
  677. else
  678. regs.flags &= ~PERF_EFLAGS_EXACT;
  679. if ((event->attr.sample_type & PERF_SAMPLE_ADDR) &&
  680. x86_pmu.intel_cap.pebs_format >= 1)
  681. data.addr = pebs->dla;
  682. if (has_branch_stack(event))
  683. data.br_stack = &cpuc->lbr_stack;
  684. if (perf_event_overflow(event, &data, &regs))
  685. x86_pmu_stop(event, 0);
  686. }
  687. static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
  688. {
  689. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  690. struct debug_store *ds = cpuc->ds;
  691. struct perf_event *event = cpuc->events[0]; /* PMC0 only */
  692. struct pebs_record_core *at, *top;
  693. int n;
  694. if (!x86_pmu.pebs_active)
  695. return;
  696. at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
  697. top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
  698. /*
  699. * Whatever else happens, drain the thing
  700. */
  701. ds->pebs_index = ds->pebs_buffer_base;
  702. if (!test_bit(0, cpuc->active_mask))
  703. return;
  704. WARN_ON_ONCE(!event);
  705. if (!event->attr.precise_ip)
  706. return;
  707. n = top - at;
  708. if (n <= 0)
  709. return;
  710. /*
  711. * Should not happen, we program the threshold at 1 and do not
  712. * set a reset value.
  713. */
  714. WARN_ONCE(n > 1, "bad leftover pebs %d\n", n);
  715. at += n - 1;
  716. __intel_pmu_pebs_event(event, iregs, at);
  717. }
  718. static void __intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, void *at,
  719. void *top)
  720. {
  721. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  722. struct debug_store *ds = cpuc->ds;
  723. struct perf_event *event = NULL;
  724. u64 status = 0;
  725. int bit;
  726. ds->pebs_index = ds->pebs_buffer_base;
  727. for (; at < top; at += x86_pmu.pebs_record_size) {
  728. struct pebs_record_nhm *p = at;
  729. for_each_set_bit(bit, (unsigned long *)&p->status,
  730. x86_pmu.max_pebs_events) {
  731. event = cpuc->events[bit];
  732. if (!test_bit(bit, cpuc->active_mask))
  733. continue;
  734. WARN_ON_ONCE(!event);
  735. if (!event->attr.precise_ip)
  736. continue;
  737. if (__test_and_set_bit(bit, (unsigned long *)&status))
  738. continue;
  739. break;
  740. }
  741. if (!event || bit >= x86_pmu.max_pebs_events)
  742. continue;
  743. __intel_pmu_pebs_event(event, iregs, at);
  744. }
  745. }
  746. static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
  747. {
  748. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  749. struct debug_store *ds = cpuc->ds;
  750. struct pebs_record_nhm *at, *top;
  751. int n;
  752. if (!x86_pmu.pebs_active)
  753. return;
  754. at = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
  755. top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
  756. ds->pebs_index = ds->pebs_buffer_base;
  757. n = top - at;
  758. if (n <= 0)
  759. return;
  760. /*
  761. * Should not happen, we program the threshold at 1 and do not
  762. * set a reset value.
  763. */
  764. WARN_ONCE(n > x86_pmu.max_pebs_events,
  765. "Unexpected number of pebs records %d\n", n);
  766. return __intel_pmu_drain_pebs_nhm(iregs, at, top);
  767. }
  768. static void intel_pmu_drain_pebs_hsw(struct pt_regs *iregs)
  769. {
  770. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  771. struct debug_store *ds = cpuc->ds;
  772. struct pebs_record_hsw *at, *top;
  773. int n;
  774. if (!x86_pmu.pebs_active)
  775. return;
  776. at = (struct pebs_record_hsw *)(unsigned long)ds->pebs_buffer_base;
  777. top = (struct pebs_record_hsw *)(unsigned long)ds->pebs_index;
  778. n = top - at;
  779. if (n <= 0)
  780. return;
  781. /*
  782. * Should not happen, we program the threshold at 1 and do not
  783. * set a reset value.
  784. */
  785. WARN_ONCE(n > x86_pmu.max_pebs_events,
  786. "Unexpected number of pebs records %d\n", n);
  787. return __intel_pmu_drain_pebs_nhm(iregs, at, top);
  788. }
  789. /*
  790. * BTS, PEBS probe and setup
  791. */
  792. void intel_ds_init(void)
  793. {
  794. /*
  795. * No support for 32bit formats
  796. */
  797. if (!boot_cpu_has(X86_FEATURE_DTES64))
  798. return;
  799. x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
  800. x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
  801. if (x86_pmu.pebs) {
  802. char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-';
  803. int format = x86_pmu.intel_cap.pebs_format;
  804. switch (format) {
  805. case 0:
  806. printk(KERN_CONT "PEBS fmt0%c, ", pebs_type);
  807. x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
  808. x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
  809. break;
  810. case 1:
  811. printk(KERN_CONT "PEBS fmt1%c, ", pebs_type);
  812. x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
  813. x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
  814. break;
  815. case 2:
  816. pr_cont("PEBS fmt2%c, ", pebs_type);
  817. x86_pmu.pebs_record_size = sizeof(struct pebs_record_hsw);
  818. x86_pmu.drain_pebs = intel_pmu_drain_pebs_hsw;
  819. break;
  820. default:
  821. printk(KERN_CONT "no PEBS fmt%d%c, ", format, pebs_type);
  822. x86_pmu.pebs = 0;
  823. }
  824. }
  825. }
  826. void perf_restore_debug_store(void)
  827. {
  828. struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
  829. if (!x86_pmu.bts && !x86_pmu.pebs)
  830. return;
  831. wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);
  832. }