perf_event_intel_lbr.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761
  1. #include <linux/perf_event.h>
  2. #include <linux/types.h>
  3. #include <asm/perf_event.h>
  4. #include <asm/msr.h>
  5. #include <asm/insn.h>
  6. #include "perf_event.h"
  7. enum {
  8. LBR_FORMAT_32 = 0x00,
  9. LBR_FORMAT_LIP = 0x01,
  10. LBR_FORMAT_EIP = 0x02,
  11. LBR_FORMAT_EIP_FLAGS = 0x03,
  12. LBR_FORMAT_EIP_FLAGS2 = 0x04,
  13. LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_EIP_FLAGS2,
  14. };
  15. static enum {
  16. LBR_EIP_FLAGS = 1,
  17. LBR_TSX = 2,
  18. } lbr_desc[LBR_FORMAT_MAX_KNOWN + 1] = {
  19. [LBR_FORMAT_EIP_FLAGS] = LBR_EIP_FLAGS,
  20. [LBR_FORMAT_EIP_FLAGS2] = LBR_EIP_FLAGS | LBR_TSX,
  21. };
  22. /*
  23. * Intel LBR_SELECT bits
  24. * Intel Vol3a, April 2011, Section 16.7 Table 16-10
  25. *
  26. * Hardware branch filter (not available on all CPUs)
  27. */
  28. #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */
  29. #define LBR_USER_BIT 1 /* do not capture at ring > 0 */
  30. #define LBR_JCC_BIT 2 /* do not capture conditional branches */
  31. #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */
  32. #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */
  33. #define LBR_RETURN_BIT 5 /* do not capture near returns */
  34. #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */
  35. #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */
  36. #define LBR_FAR_BIT 8 /* do not capture far branches */
  37. #define LBR_KERNEL (1 << LBR_KERNEL_BIT)
  38. #define LBR_USER (1 << LBR_USER_BIT)
  39. #define LBR_JCC (1 << LBR_JCC_BIT)
  40. #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT)
  41. #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT)
  42. #define LBR_RETURN (1 << LBR_RETURN_BIT)
  43. #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT)
  44. #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT)
  45. #define LBR_FAR (1 << LBR_FAR_BIT)
  46. #define LBR_PLM (LBR_KERNEL | LBR_USER)
  47. #define LBR_SEL_MASK 0x1ff /* valid bits in LBR_SELECT */
  48. #define LBR_NOT_SUPP -1 /* LBR filter not supported */
  49. #define LBR_IGN 0 /* ignored */
  50. #define LBR_ANY \
  51. (LBR_JCC |\
  52. LBR_REL_CALL |\
  53. LBR_IND_CALL |\
  54. LBR_RETURN |\
  55. LBR_REL_JMP |\
  56. LBR_IND_JMP |\
  57. LBR_FAR)
  58. #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
  59. #define LBR_FROM_FLAG_IN_TX (1ULL << 62)
  60. #define LBR_FROM_FLAG_ABORT (1ULL << 61)
  61. #define for_each_branch_sample_type(x) \
  62. for ((x) = PERF_SAMPLE_BRANCH_USER; \
  63. (x) < PERF_SAMPLE_BRANCH_MAX; (x) <<= 1)
  64. /*
  65. * x86control flow change classification
  66. * x86control flow changes include branches, interrupts, traps, faults
  67. */
  68. enum {
  69. X86_BR_NONE = 0, /* unknown */
  70. X86_BR_USER = 1 << 0, /* branch target is user */
  71. X86_BR_KERNEL = 1 << 1, /* branch target is kernel */
  72. X86_BR_CALL = 1 << 2, /* call */
  73. X86_BR_RET = 1 << 3, /* return */
  74. X86_BR_SYSCALL = 1 << 4, /* syscall */
  75. X86_BR_SYSRET = 1 << 5, /* syscall return */
  76. X86_BR_INT = 1 << 6, /* sw interrupt */
  77. X86_BR_IRET = 1 << 7, /* return from interrupt */
  78. X86_BR_JCC = 1 << 8, /* conditional */
  79. X86_BR_JMP = 1 << 9, /* jump */
  80. X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */
  81. X86_BR_IND_CALL = 1 << 11,/* indirect calls */
  82. X86_BR_ABORT = 1 << 12,/* transaction abort */
  83. X86_BR_IN_TX = 1 << 13,/* in transaction */
  84. X86_BR_NO_TX = 1 << 14,/* not in transaction */
  85. };
  86. #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
  87. #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
  88. #define X86_BR_ANY \
  89. (X86_BR_CALL |\
  90. X86_BR_RET |\
  91. X86_BR_SYSCALL |\
  92. X86_BR_SYSRET |\
  93. X86_BR_INT |\
  94. X86_BR_IRET |\
  95. X86_BR_JCC |\
  96. X86_BR_JMP |\
  97. X86_BR_IRQ |\
  98. X86_BR_ABORT |\
  99. X86_BR_IND_CALL)
  100. #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
  101. #define X86_BR_ANY_CALL \
  102. (X86_BR_CALL |\
  103. X86_BR_IND_CALL |\
  104. X86_BR_SYSCALL |\
  105. X86_BR_IRQ |\
  106. X86_BR_INT)
  107. static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
  108. /*
  109. * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  110. * otherwise it becomes near impossible to get a reliable stack.
  111. */
  112. static void __intel_pmu_lbr_enable(void)
  113. {
  114. u64 debugctl;
  115. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  116. if (cpuc->lbr_sel)
  117. wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config);
  118. rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  119. debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  120. wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  121. }
  122. static void __intel_pmu_lbr_disable(void)
  123. {
  124. u64 debugctl;
  125. rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  126. debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  127. wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  128. }
  129. static void intel_pmu_lbr_reset_32(void)
  130. {
  131. int i;
  132. for (i = 0; i < x86_pmu.lbr_nr; i++)
  133. wrmsrl(x86_pmu.lbr_from + i, 0);
  134. }
  135. static void intel_pmu_lbr_reset_64(void)
  136. {
  137. int i;
  138. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  139. wrmsrl(x86_pmu.lbr_from + i, 0);
  140. wrmsrl(x86_pmu.lbr_to + i, 0);
  141. }
  142. }
  143. void intel_pmu_lbr_reset(void)
  144. {
  145. if (!x86_pmu.lbr_nr)
  146. return;
  147. if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  148. intel_pmu_lbr_reset_32();
  149. else
  150. intel_pmu_lbr_reset_64();
  151. }
  152. void intel_pmu_lbr_enable(struct perf_event *event)
  153. {
  154. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  155. if (!x86_pmu.lbr_nr)
  156. return;
  157. /*
  158. * Reset the LBR stack if we changed task context to
  159. * avoid data leaks.
  160. */
  161. if (event->ctx->task && cpuc->lbr_context != event->ctx) {
  162. intel_pmu_lbr_reset();
  163. cpuc->lbr_context = event->ctx;
  164. }
  165. cpuc->br_sel = event->hw.branch_reg.reg;
  166. cpuc->lbr_users++;
  167. }
  168. void intel_pmu_lbr_disable(struct perf_event *event)
  169. {
  170. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  171. if (!x86_pmu.lbr_nr)
  172. return;
  173. cpuc->lbr_users--;
  174. WARN_ON_ONCE(cpuc->lbr_users < 0);
  175. if (cpuc->enabled && !cpuc->lbr_users) {
  176. __intel_pmu_lbr_disable();
  177. /* avoid stale pointer */
  178. cpuc->lbr_context = NULL;
  179. }
  180. }
  181. void intel_pmu_lbr_enable_all(void)
  182. {
  183. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  184. if (cpuc->lbr_users)
  185. __intel_pmu_lbr_enable();
  186. }
  187. void intel_pmu_lbr_disable_all(void)
  188. {
  189. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  190. if (cpuc->lbr_users)
  191. __intel_pmu_lbr_disable();
  192. }
  193. /*
  194. * TOS = most recently recorded branch
  195. */
  196. static inline u64 intel_pmu_lbr_tos(void)
  197. {
  198. u64 tos;
  199. rdmsrl(x86_pmu.lbr_tos, tos);
  200. return tos;
  201. }
  202. static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
  203. {
  204. unsigned long mask = x86_pmu.lbr_nr - 1;
  205. u64 tos = intel_pmu_lbr_tos();
  206. int i;
  207. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  208. unsigned long lbr_idx = (tos - i) & mask;
  209. union {
  210. struct {
  211. u32 from;
  212. u32 to;
  213. };
  214. u64 lbr;
  215. } msr_lastbranch;
  216. rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
  217. cpuc->lbr_entries[i].from = msr_lastbranch.from;
  218. cpuc->lbr_entries[i].to = msr_lastbranch.to;
  219. cpuc->lbr_entries[i].mispred = 0;
  220. cpuc->lbr_entries[i].predicted = 0;
  221. cpuc->lbr_entries[i].reserved = 0;
  222. }
  223. cpuc->lbr_stack.nr = i;
  224. }
  225. /*
  226. * Due to lack of segmentation in Linux the effective address (offset)
  227. * is the same as the linear address, allowing us to merge the LIP and EIP
  228. * LBR formats.
  229. */
  230. static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
  231. {
  232. unsigned long mask = x86_pmu.lbr_nr - 1;
  233. int lbr_format = x86_pmu.intel_cap.lbr_format;
  234. u64 tos = intel_pmu_lbr_tos();
  235. int i;
  236. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  237. unsigned long lbr_idx = (tos - i) & mask;
  238. u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
  239. int skip = 0;
  240. int lbr_flags = lbr_desc[lbr_format];
  241. rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
  242. rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
  243. if (lbr_flags & LBR_EIP_FLAGS) {
  244. mis = !!(from & LBR_FROM_FLAG_MISPRED);
  245. pred = !mis;
  246. skip = 1;
  247. }
  248. if (lbr_flags & LBR_TSX) {
  249. in_tx = !!(from & LBR_FROM_FLAG_IN_TX);
  250. abort = !!(from & LBR_FROM_FLAG_ABORT);
  251. skip = 3;
  252. }
  253. from = (u64)((((s64)from) << skip) >> skip);
  254. cpuc->lbr_entries[i].from = from;
  255. cpuc->lbr_entries[i].to = to;
  256. cpuc->lbr_entries[i].mispred = mis;
  257. cpuc->lbr_entries[i].predicted = pred;
  258. cpuc->lbr_entries[i].in_tx = in_tx;
  259. cpuc->lbr_entries[i].abort = abort;
  260. cpuc->lbr_entries[i].reserved = 0;
  261. }
  262. cpuc->lbr_stack.nr = i;
  263. }
  264. void intel_pmu_lbr_read(void)
  265. {
  266. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  267. if (!cpuc->lbr_users)
  268. return;
  269. if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  270. intel_pmu_lbr_read_32(cpuc);
  271. else
  272. intel_pmu_lbr_read_64(cpuc);
  273. intel_pmu_lbr_filter(cpuc);
  274. }
  275. /*
  276. * SW filter is used:
  277. * - in case there is no HW filter
  278. * - in case the HW filter has errata or limitations
  279. */
  280. static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
  281. {
  282. u64 br_type = event->attr.branch_sample_type;
  283. int mask = 0;
  284. if (br_type & PERF_SAMPLE_BRANCH_USER)
  285. mask |= X86_BR_USER;
  286. if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
  287. mask |= X86_BR_KERNEL;
  288. /* we ignore BRANCH_HV here */
  289. if (br_type & PERF_SAMPLE_BRANCH_ANY)
  290. mask |= X86_BR_ANY;
  291. if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
  292. mask |= X86_BR_ANY_CALL;
  293. if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
  294. mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
  295. if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
  296. mask |= X86_BR_IND_CALL;
  297. if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX)
  298. mask |= X86_BR_ABORT;
  299. if (br_type & PERF_SAMPLE_BRANCH_IN_TX)
  300. mask |= X86_BR_IN_TX;
  301. if (br_type & PERF_SAMPLE_BRANCH_NO_TX)
  302. mask |= X86_BR_NO_TX;
  303. /*
  304. * stash actual user request into reg, it may
  305. * be used by fixup code for some CPU
  306. */
  307. event->hw.branch_reg.reg = mask;
  308. }
  309. /*
  310. * setup the HW LBR filter
  311. * Used only when available, may not be enough to disambiguate
  312. * all branches, may need the help of the SW filter
  313. */
  314. static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
  315. {
  316. struct hw_perf_event_extra *reg;
  317. u64 br_type = event->attr.branch_sample_type;
  318. u64 mask = 0, m;
  319. u64 v;
  320. for_each_branch_sample_type(m) {
  321. if (!(br_type & m))
  322. continue;
  323. v = x86_pmu.lbr_sel_map[m];
  324. if (v == LBR_NOT_SUPP)
  325. return -EOPNOTSUPP;
  326. if (v != LBR_IGN)
  327. mask |= v;
  328. }
  329. reg = &event->hw.branch_reg;
  330. reg->idx = EXTRA_REG_LBR;
  331. /* LBR_SELECT operates in suppress mode so invert mask */
  332. reg->config = ~mask & x86_pmu.lbr_sel_mask;
  333. return 0;
  334. }
  335. int intel_pmu_setup_lbr_filter(struct perf_event *event)
  336. {
  337. int ret = 0;
  338. /*
  339. * no LBR on this PMU
  340. */
  341. if (!x86_pmu.lbr_nr)
  342. return -EOPNOTSUPP;
  343. /*
  344. * setup SW LBR filter
  345. */
  346. intel_pmu_setup_sw_lbr_filter(event);
  347. /*
  348. * setup HW LBR filter, if any
  349. */
  350. if (x86_pmu.lbr_sel_map)
  351. ret = intel_pmu_setup_hw_lbr_filter(event);
  352. return ret;
  353. }
  354. /*
  355. * return the type of control flow change at address "from"
  356. * intruction is not necessarily a branch (in case of interrupt).
  357. *
  358. * The branch type returned also includes the priv level of the
  359. * target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
  360. *
  361. * If a branch type is unknown OR the instruction cannot be
  362. * decoded (e.g., text page not present), then X86_BR_NONE is
  363. * returned.
  364. */
  365. static int branch_type(unsigned long from, unsigned long to, int abort)
  366. {
  367. struct insn insn;
  368. void *addr;
  369. int bytes, size = MAX_INSN_SIZE;
  370. int ret = X86_BR_NONE;
  371. int ext, to_plm, from_plm;
  372. u8 buf[MAX_INSN_SIZE];
  373. int is64 = 0;
  374. to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
  375. from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
  376. /*
  377. * maybe zero if lbr did not fill up after a reset by the time
  378. * we get a PMU interrupt
  379. */
  380. if (from == 0 || to == 0)
  381. return X86_BR_NONE;
  382. if (abort)
  383. return X86_BR_ABORT | to_plm;
  384. if (from_plm == X86_BR_USER) {
  385. /*
  386. * can happen if measuring at the user level only
  387. * and we interrupt in a kernel thread, e.g., idle.
  388. */
  389. if (!current->mm)
  390. return X86_BR_NONE;
  391. /* may fail if text not present */
  392. bytes = copy_from_user_nmi(buf, (void __user *)from, size);
  393. if (bytes != size)
  394. return X86_BR_NONE;
  395. addr = buf;
  396. } else {
  397. /*
  398. * The LBR logs any address in the IP, even if the IP just
  399. * faulted. This means userspace can control the from address.
  400. * Ensure we don't blindy read any address by validating it is
  401. * a known text address.
  402. */
  403. if (kernel_text_address(from))
  404. addr = (void *)from;
  405. else
  406. return X86_BR_NONE;
  407. }
  408. /*
  409. * decoder needs to know the ABI especially
  410. * on 64-bit systems running 32-bit apps
  411. */
  412. #ifdef CONFIG_X86_64
  413. is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32);
  414. #endif
  415. insn_init(&insn, addr, is64);
  416. insn_get_opcode(&insn);
  417. switch (insn.opcode.bytes[0]) {
  418. case 0xf:
  419. switch (insn.opcode.bytes[1]) {
  420. case 0x05: /* syscall */
  421. case 0x34: /* sysenter */
  422. ret = X86_BR_SYSCALL;
  423. break;
  424. case 0x07: /* sysret */
  425. case 0x35: /* sysexit */
  426. ret = X86_BR_SYSRET;
  427. break;
  428. case 0x80 ... 0x8f: /* conditional */
  429. ret = X86_BR_JCC;
  430. break;
  431. default:
  432. ret = X86_BR_NONE;
  433. }
  434. break;
  435. case 0x70 ... 0x7f: /* conditional */
  436. ret = X86_BR_JCC;
  437. break;
  438. case 0xc2: /* near ret */
  439. case 0xc3: /* near ret */
  440. case 0xca: /* far ret */
  441. case 0xcb: /* far ret */
  442. ret = X86_BR_RET;
  443. break;
  444. case 0xcf: /* iret */
  445. ret = X86_BR_IRET;
  446. break;
  447. case 0xcc ... 0xce: /* int */
  448. ret = X86_BR_INT;
  449. break;
  450. case 0xe8: /* call near rel */
  451. case 0x9a: /* call far absolute */
  452. ret = X86_BR_CALL;
  453. break;
  454. case 0xe0 ... 0xe3: /* loop jmp */
  455. ret = X86_BR_JCC;
  456. break;
  457. case 0xe9 ... 0xeb: /* jmp */
  458. ret = X86_BR_JMP;
  459. break;
  460. case 0xff: /* call near absolute, call far absolute ind */
  461. insn_get_modrm(&insn);
  462. ext = (insn.modrm.bytes[0] >> 3) & 0x7;
  463. switch (ext) {
  464. case 2: /* near ind call */
  465. case 3: /* far ind call */
  466. ret = X86_BR_IND_CALL;
  467. break;
  468. case 4:
  469. case 5:
  470. ret = X86_BR_JMP;
  471. break;
  472. }
  473. break;
  474. default:
  475. ret = X86_BR_NONE;
  476. }
  477. /*
  478. * interrupts, traps, faults (and thus ring transition) may
  479. * occur on any instructions. Thus, to classify them correctly,
  480. * we need to first look at the from and to priv levels. If they
  481. * are different and to is in the kernel, then it indicates
  482. * a ring transition. If the from instruction is not a ring
  483. * transition instr (syscall, systenter, int), then it means
  484. * it was a irq, trap or fault.
  485. *
  486. * we have no way of detecting kernel to kernel faults.
  487. */
  488. if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
  489. && ret != X86_BR_SYSCALL && ret != X86_BR_INT)
  490. ret = X86_BR_IRQ;
  491. /*
  492. * branch priv level determined by target as
  493. * is done by HW when LBR_SELECT is implemented
  494. */
  495. if (ret != X86_BR_NONE)
  496. ret |= to_plm;
  497. return ret;
  498. }
  499. /*
  500. * implement actual branch filter based on user demand.
  501. * Hardware may not exactly satisfy that request, thus
  502. * we need to inspect opcodes. Mismatched branches are
  503. * discarded. Therefore, the number of branches returned
  504. * in PERF_SAMPLE_BRANCH_STACK sample may vary.
  505. */
  506. static void
  507. intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
  508. {
  509. u64 from, to;
  510. int br_sel = cpuc->br_sel;
  511. int i, j, type;
  512. bool compress = false;
  513. /* if sampling all branches, then nothing to filter */
  514. if ((br_sel & X86_BR_ALL) == X86_BR_ALL)
  515. return;
  516. for (i = 0; i < cpuc->lbr_stack.nr; i++) {
  517. from = cpuc->lbr_entries[i].from;
  518. to = cpuc->lbr_entries[i].to;
  519. type = branch_type(from, to, cpuc->lbr_entries[i].abort);
  520. if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) {
  521. if (cpuc->lbr_entries[i].in_tx)
  522. type |= X86_BR_IN_TX;
  523. else
  524. type |= X86_BR_NO_TX;
  525. }
  526. /* if type does not correspond, then discard */
  527. if (type == X86_BR_NONE || (br_sel & type) != type) {
  528. cpuc->lbr_entries[i].from = 0;
  529. compress = true;
  530. }
  531. }
  532. if (!compress)
  533. return;
  534. /* remove all entries with from=0 */
  535. for (i = 0; i < cpuc->lbr_stack.nr; ) {
  536. if (!cpuc->lbr_entries[i].from) {
  537. j = i;
  538. while (++j < cpuc->lbr_stack.nr)
  539. cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
  540. cpuc->lbr_stack.nr--;
  541. if (!cpuc->lbr_entries[i].from)
  542. continue;
  543. }
  544. i++;
  545. }
  546. }
  547. /*
  548. * Map interface branch filters onto LBR filters
  549. */
  550. static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
  551. [PERF_SAMPLE_BRANCH_ANY] = LBR_ANY,
  552. [PERF_SAMPLE_BRANCH_USER] = LBR_USER,
  553. [PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL,
  554. [PERF_SAMPLE_BRANCH_HV] = LBR_IGN,
  555. [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_REL_JMP
  556. | LBR_IND_JMP | LBR_FAR,
  557. /*
  558. * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
  559. */
  560. [PERF_SAMPLE_BRANCH_ANY_CALL] =
  561. LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
  562. /*
  563. * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
  564. */
  565. [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP,
  566. };
  567. static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
  568. [PERF_SAMPLE_BRANCH_ANY] = LBR_ANY,
  569. [PERF_SAMPLE_BRANCH_USER] = LBR_USER,
  570. [PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL,
  571. [PERF_SAMPLE_BRANCH_HV] = LBR_IGN,
  572. [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_FAR,
  573. [PERF_SAMPLE_BRANCH_ANY_CALL] = LBR_REL_CALL | LBR_IND_CALL
  574. | LBR_FAR,
  575. [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL,
  576. };
  577. /* core */
  578. void intel_pmu_lbr_init_core(void)
  579. {
  580. x86_pmu.lbr_nr = 4;
  581. x86_pmu.lbr_tos = MSR_LBR_TOS;
  582. x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
  583. x86_pmu.lbr_to = MSR_LBR_CORE_TO;
  584. /*
  585. * SW branch filter usage:
  586. * - compensate for lack of HW filter
  587. */
  588. pr_cont("4-deep LBR, ");
  589. }
  590. /* nehalem/westmere */
  591. void intel_pmu_lbr_init_nhm(void)
  592. {
  593. x86_pmu.lbr_nr = 16;
  594. x86_pmu.lbr_tos = MSR_LBR_TOS;
  595. x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
  596. x86_pmu.lbr_to = MSR_LBR_NHM_TO;
  597. x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
  598. x86_pmu.lbr_sel_map = nhm_lbr_sel_map;
  599. /*
  600. * SW branch filter usage:
  601. * - workaround LBR_SEL errata (see above)
  602. * - support syscall, sysret capture.
  603. * That requires LBR_FAR but that means far
  604. * jmp need to be filtered out
  605. */
  606. pr_cont("16-deep LBR, ");
  607. }
  608. /* sandy bridge */
  609. void intel_pmu_lbr_init_snb(void)
  610. {
  611. x86_pmu.lbr_nr = 16;
  612. x86_pmu.lbr_tos = MSR_LBR_TOS;
  613. x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
  614. x86_pmu.lbr_to = MSR_LBR_NHM_TO;
  615. x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
  616. x86_pmu.lbr_sel_map = snb_lbr_sel_map;
  617. /*
  618. * SW branch filter usage:
  619. * - support syscall, sysret capture.
  620. * That requires LBR_FAR but that means far
  621. * jmp need to be filtered out
  622. */
  623. pr_cont("16-deep LBR, ");
  624. }
  625. /* atom */
  626. void intel_pmu_lbr_init_atom(void)
  627. {
  628. /*
  629. * only models starting at stepping 10 seems
  630. * to have an operational LBR which can freeze
  631. * on PMU interrupt
  632. */
  633. if (boot_cpu_data.x86_model == 28
  634. && boot_cpu_data.x86_mask < 10) {
  635. pr_cont("LBR disabled due to erratum");
  636. return;
  637. }
  638. x86_pmu.lbr_nr = 8;
  639. x86_pmu.lbr_tos = MSR_LBR_TOS;
  640. x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
  641. x86_pmu.lbr_to = MSR_LBR_CORE_TO;
  642. /*
  643. * SW branch filter usage:
  644. * - compensate for lack of HW filter
  645. */
  646. pr_cont("8-deep LBR, ");
  647. }