perf_event_intel_lbr.c 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218
  1. #ifdef CONFIG_CPU_SUP_INTEL
  2. enum {
  3. LBR_FORMAT_32 = 0x00,
  4. LBR_FORMAT_LIP = 0x01,
  5. LBR_FORMAT_EIP = 0x02,
  6. LBR_FORMAT_EIP_FLAGS = 0x03,
  7. };
  8. /*
  9. * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  10. * otherwise it becomes near impossible to get a reliable stack.
  11. */
  12. static void __intel_pmu_lbr_enable(void)
  13. {
  14. u64 debugctl;
  15. rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  16. debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  17. wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  18. }
  19. static void __intel_pmu_lbr_disable(void)
  20. {
  21. u64 debugctl;
  22. rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  23. debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  24. wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  25. }
  26. static void intel_pmu_lbr_reset_32(void)
  27. {
  28. int i;
  29. for (i = 0; i < x86_pmu.lbr_nr; i++)
  30. wrmsrl(x86_pmu.lbr_from + i, 0);
  31. }
  32. static void intel_pmu_lbr_reset_64(void)
  33. {
  34. int i;
  35. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  36. wrmsrl(x86_pmu.lbr_from + i, 0);
  37. wrmsrl(x86_pmu.lbr_to + i, 0);
  38. }
  39. }
  40. static void intel_pmu_lbr_reset(void)
  41. {
  42. if (!x86_pmu.lbr_nr)
  43. return;
  44. if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  45. intel_pmu_lbr_reset_32();
  46. else
  47. intel_pmu_lbr_reset_64();
  48. }
  49. static void intel_pmu_lbr_enable(struct perf_event *event)
  50. {
  51. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  52. if (!x86_pmu.lbr_nr)
  53. return;
  54. WARN_ON_ONCE(cpuc->enabled);
  55. /*
  56. * Reset the LBR stack if we changed task context to
  57. * avoid data leaks.
  58. */
  59. if (event->ctx->task && cpuc->lbr_context != event->ctx) {
  60. intel_pmu_lbr_reset();
  61. cpuc->lbr_context = event->ctx;
  62. }
  63. cpuc->lbr_users++;
  64. }
  65. static void intel_pmu_lbr_disable(struct perf_event *event)
  66. {
  67. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  68. if (!x86_pmu.lbr_nr)
  69. return;
  70. cpuc->lbr_users--;
  71. WARN_ON_ONCE(cpuc->lbr_users < 0);
  72. if (cpuc->enabled && !cpuc->lbr_users)
  73. __intel_pmu_lbr_disable();
  74. }
  75. static void intel_pmu_lbr_enable_all(void)
  76. {
  77. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  78. if (cpuc->lbr_users)
  79. __intel_pmu_lbr_enable();
  80. }
  81. static void intel_pmu_lbr_disable_all(void)
  82. {
  83. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  84. if (cpuc->lbr_users)
  85. __intel_pmu_lbr_disable();
  86. }
  87. static inline u64 intel_pmu_lbr_tos(void)
  88. {
  89. u64 tos;
  90. rdmsrl(x86_pmu.lbr_tos, tos);
  91. return tos;
  92. }
  93. static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
  94. {
  95. unsigned long mask = x86_pmu.lbr_nr - 1;
  96. u64 tos = intel_pmu_lbr_tos();
  97. int i;
  98. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  99. unsigned long lbr_idx = (tos - i) & mask;
  100. union {
  101. struct {
  102. u32 from;
  103. u32 to;
  104. };
  105. u64 lbr;
  106. } msr_lastbranch;
  107. rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
  108. cpuc->lbr_entries[i].from = msr_lastbranch.from;
  109. cpuc->lbr_entries[i].to = msr_lastbranch.to;
  110. cpuc->lbr_entries[i].flags = 0;
  111. }
  112. cpuc->lbr_stack.nr = i;
  113. }
  114. #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
  115. /*
  116. * Due to lack of segmentation in Linux the effective address (offset)
  117. * is the same as the linear address, allowing us to merge the LIP and EIP
  118. * LBR formats.
  119. */
  120. static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
  121. {
  122. unsigned long mask = x86_pmu.lbr_nr - 1;
  123. int lbr_format = x86_pmu.intel_cap.lbr_format;
  124. u64 tos = intel_pmu_lbr_tos();
  125. int i;
  126. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  127. unsigned long lbr_idx = (tos - i) & mask;
  128. u64 from, to, flags = 0;
  129. rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
  130. rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
  131. if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
  132. flags = !!(from & LBR_FROM_FLAG_MISPRED);
  133. from = (u64)((((s64)from) << 1) >> 1);
  134. }
  135. cpuc->lbr_entries[i].from = from;
  136. cpuc->lbr_entries[i].to = to;
  137. cpuc->lbr_entries[i].flags = flags;
  138. }
  139. cpuc->lbr_stack.nr = i;
  140. }
  141. static void intel_pmu_lbr_read(void)
  142. {
  143. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  144. if (!cpuc->lbr_users)
  145. return;
  146. if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  147. intel_pmu_lbr_read_32(cpuc);
  148. else
  149. intel_pmu_lbr_read_64(cpuc);
  150. }
  151. static void intel_pmu_lbr_init_core(void)
  152. {
  153. x86_pmu.lbr_nr = 4;
  154. x86_pmu.lbr_tos = 0x01c9;
  155. x86_pmu.lbr_from = 0x40;
  156. x86_pmu.lbr_to = 0x60;
  157. }
  158. static void intel_pmu_lbr_init_nhm(void)
  159. {
  160. x86_pmu.lbr_nr = 16;
  161. x86_pmu.lbr_tos = 0x01c9;
  162. x86_pmu.lbr_from = 0x680;
  163. x86_pmu.lbr_to = 0x6c0;
  164. }
  165. static void intel_pmu_lbr_init_atom(void)
  166. {
  167. x86_pmu.lbr_nr = 8;
  168. x86_pmu.lbr_tos = 0x01c9;
  169. x86_pmu.lbr_from = 0x40;
  170. x86_pmu.lbr_to = 0x60;
  171. }
  172. #endif /* CONFIG_CPU_SUP_INTEL */