perf_event_intel.c 61 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219
  1. /*
  2. * Per core/cpu state
  3. *
  4. * Used to coordinate shared registers between HT threads or
  5. * among events on a single PMU.
  6. */
  7. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  8. #include <linux/stddef.h>
  9. #include <linux/types.h>
  10. #include <linux/init.h>
  11. #include <linux/slab.h>
  12. #include <linux/export.h>
  13. #include <asm/hardirq.h>
  14. #include <asm/apic.h>
  15. #include "perf_event.h"
  16. /*
  17. * Intel PerfMon, used on Core and later.
  18. */
  19. static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
  20. {
  21. [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
  22. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  23. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
  24. [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
  25. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  26. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  27. [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
  28. [PERF_COUNT_HW_REF_CPU_CYCLES] = 0x0300, /* pseudo-encoding */
  29. };
  30. static struct event_constraint intel_core_event_constraints[] __read_mostly =
  31. {
  32. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  33. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  34. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  35. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  36. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  37. INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
  38. EVENT_CONSTRAINT_END
  39. };
  40. static struct event_constraint intel_core2_event_constraints[] __read_mostly =
  41. {
  42. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  43. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  44. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  45. INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  46. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  47. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  48. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  49. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  50. INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
  51. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  52. INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
  53. INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */
  54. INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
  55. EVENT_CONSTRAINT_END
  56. };
  57. static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
  58. {
  59. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  60. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  61. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  62. INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
  63. INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
  64. INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
  65. INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
  66. INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
  67. INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
  68. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  69. INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  70. EVENT_CONSTRAINT_END
  71. };
  72. static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
  73. {
  74. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  75. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  76. EVENT_EXTRA_END
  77. };
  78. static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
  79. {
  80. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  81. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  82. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  83. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  84. INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
  85. INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  86. INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */
  87. EVENT_CONSTRAINT_END
  88. };
  89. static struct event_constraint intel_snb_event_constraints[] __read_mostly =
  90. {
  91. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  92. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  93. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  94. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
  95. INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
  96. INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  97. INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  98. INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
  99. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  100. INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
  101. EVENT_CONSTRAINT_END
  102. };
  103. static struct event_constraint intel_ivb_event_constraints[] __read_mostly =
  104. {
  105. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  106. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  107. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  108. INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */
  109. INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */
  110. INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */
  111. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
  112. INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
  113. INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */
  114. INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  115. INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  116. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  117. INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
  118. INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
  119. INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
  120. INTEL_EVENT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
  121. EVENT_CONSTRAINT_END
  122. };
  123. static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
  124. {
  125. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  126. INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1),
  127. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  128. EVENT_EXTRA_END
  129. };
  130. static struct event_constraint intel_v1_event_constraints[] __read_mostly =
  131. {
  132. EVENT_CONSTRAINT_END
  133. };
  134. static struct event_constraint intel_gen_event_constraints[] __read_mostly =
  135. {
  136. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  137. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  138. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  139. EVENT_CONSTRAINT_END
  140. };
  141. static struct extra_reg intel_snb_extra_regs[] __read_mostly = {
  142. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
  143. INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1),
  144. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
  145. EVENT_EXTRA_END
  146. };
  147. EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3");
  148. EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3");
  149. struct attribute *nhm_events_attrs[] = {
  150. EVENT_PTR(mem_ld_nhm),
  151. NULL,
  152. };
  153. struct attribute *snb_events_attrs[] = {
  154. EVENT_PTR(mem_ld_snb),
  155. NULL,
  156. };
  157. static u64 intel_pmu_event_map(int hw_event)
  158. {
  159. return intel_perfmon_event_map[hw_event];
  160. }
  161. #define SNB_DMND_DATA_RD (1ULL << 0)
  162. #define SNB_DMND_RFO (1ULL << 1)
  163. #define SNB_DMND_IFETCH (1ULL << 2)
  164. #define SNB_DMND_WB (1ULL << 3)
  165. #define SNB_PF_DATA_RD (1ULL << 4)
  166. #define SNB_PF_RFO (1ULL << 5)
  167. #define SNB_PF_IFETCH (1ULL << 6)
  168. #define SNB_LLC_DATA_RD (1ULL << 7)
  169. #define SNB_LLC_RFO (1ULL << 8)
  170. #define SNB_LLC_IFETCH (1ULL << 9)
  171. #define SNB_BUS_LOCKS (1ULL << 10)
  172. #define SNB_STRM_ST (1ULL << 11)
  173. #define SNB_OTHER (1ULL << 15)
  174. #define SNB_RESP_ANY (1ULL << 16)
  175. #define SNB_NO_SUPP (1ULL << 17)
  176. #define SNB_LLC_HITM (1ULL << 18)
  177. #define SNB_LLC_HITE (1ULL << 19)
  178. #define SNB_LLC_HITS (1ULL << 20)
  179. #define SNB_LLC_HITF (1ULL << 21)
  180. #define SNB_LOCAL (1ULL << 22)
  181. #define SNB_REMOTE (0xffULL << 23)
  182. #define SNB_SNP_NONE (1ULL << 31)
  183. #define SNB_SNP_NOT_NEEDED (1ULL << 32)
  184. #define SNB_SNP_MISS (1ULL << 33)
  185. #define SNB_NO_FWD (1ULL << 34)
  186. #define SNB_SNP_FWD (1ULL << 35)
  187. #define SNB_HITM (1ULL << 36)
  188. #define SNB_NON_DRAM (1ULL << 37)
  189. #define SNB_DMND_READ (SNB_DMND_DATA_RD|SNB_LLC_DATA_RD)
  190. #define SNB_DMND_WRITE (SNB_DMND_RFO|SNB_LLC_RFO)
  191. #define SNB_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)
  192. #define SNB_SNP_ANY (SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \
  193. SNB_SNP_MISS|SNB_NO_FWD|SNB_SNP_FWD| \
  194. SNB_HITM)
  195. #define SNB_DRAM_ANY (SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY)
  196. #define SNB_DRAM_REMOTE (SNB_REMOTE|SNB_SNP_ANY)
  197. #define SNB_L3_ACCESS SNB_RESP_ANY
  198. #define SNB_L3_MISS (SNB_DRAM_ANY|SNB_NON_DRAM)
  199. static __initconst const u64 snb_hw_cache_extra_regs
  200. [PERF_COUNT_HW_CACHE_MAX]
  201. [PERF_COUNT_HW_CACHE_OP_MAX]
  202. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  203. {
  204. [ C(LL ) ] = {
  205. [ C(OP_READ) ] = {
  206. [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_L3_ACCESS,
  207. [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_L3_MISS,
  208. },
  209. [ C(OP_WRITE) ] = {
  210. [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_L3_ACCESS,
  211. [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_L3_MISS,
  212. },
  213. [ C(OP_PREFETCH) ] = {
  214. [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_L3_ACCESS,
  215. [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_L3_MISS,
  216. },
  217. },
  218. [ C(NODE) ] = {
  219. [ C(OP_READ) ] = {
  220. [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_DRAM_ANY,
  221. [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_DRAM_REMOTE,
  222. },
  223. [ C(OP_WRITE) ] = {
  224. [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_DRAM_ANY,
  225. [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_DRAM_REMOTE,
  226. },
  227. [ C(OP_PREFETCH) ] = {
  228. [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_DRAM_ANY,
  229. [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_DRAM_REMOTE,
  230. },
  231. },
  232. };
  233. static __initconst const u64 snb_hw_cache_event_ids
  234. [PERF_COUNT_HW_CACHE_MAX]
  235. [PERF_COUNT_HW_CACHE_OP_MAX]
  236. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  237. {
  238. [ C(L1D) ] = {
  239. [ C(OP_READ) ] = {
  240. [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS */
  241. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPLACEMENT */
  242. },
  243. [ C(OP_WRITE) ] = {
  244. [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES */
  245. [ C(RESULT_MISS) ] = 0x0851, /* L1D.ALL_M_REPLACEMENT */
  246. },
  247. [ C(OP_PREFETCH) ] = {
  248. [ C(RESULT_ACCESS) ] = 0x0,
  249. [ C(RESULT_MISS) ] = 0x024e, /* HW_PRE_REQ.DL1_MISS */
  250. },
  251. },
  252. [ C(L1I ) ] = {
  253. [ C(OP_READ) ] = {
  254. [ C(RESULT_ACCESS) ] = 0x0,
  255. [ C(RESULT_MISS) ] = 0x0280, /* ICACHE.MISSES */
  256. },
  257. [ C(OP_WRITE) ] = {
  258. [ C(RESULT_ACCESS) ] = -1,
  259. [ C(RESULT_MISS) ] = -1,
  260. },
  261. [ C(OP_PREFETCH) ] = {
  262. [ C(RESULT_ACCESS) ] = 0x0,
  263. [ C(RESULT_MISS) ] = 0x0,
  264. },
  265. },
  266. [ C(LL ) ] = {
  267. [ C(OP_READ) ] = {
  268. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  269. [ C(RESULT_ACCESS) ] = 0x01b7,
  270. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  271. [ C(RESULT_MISS) ] = 0x01b7,
  272. },
  273. [ C(OP_WRITE) ] = {
  274. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  275. [ C(RESULT_ACCESS) ] = 0x01b7,
  276. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  277. [ C(RESULT_MISS) ] = 0x01b7,
  278. },
  279. [ C(OP_PREFETCH) ] = {
  280. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  281. [ C(RESULT_ACCESS) ] = 0x01b7,
  282. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  283. [ C(RESULT_MISS) ] = 0x01b7,
  284. },
  285. },
  286. [ C(DTLB) ] = {
  287. [ C(OP_READ) ] = {
  288. [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
  289. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
  290. },
  291. [ C(OP_WRITE) ] = {
  292. [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
  293. [ C(RESULT_MISS) ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
  294. },
  295. [ C(OP_PREFETCH) ] = {
  296. [ C(RESULT_ACCESS) ] = 0x0,
  297. [ C(RESULT_MISS) ] = 0x0,
  298. },
  299. },
  300. [ C(ITLB) ] = {
  301. [ C(OP_READ) ] = {
  302. [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT */
  303. [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK */
  304. },
  305. [ C(OP_WRITE) ] = {
  306. [ C(RESULT_ACCESS) ] = -1,
  307. [ C(RESULT_MISS) ] = -1,
  308. },
  309. [ C(OP_PREFETCH) ] = {
  310. [ C(RESULT_ACCESS) ] = -1,
  311. [ C(RESULT_MISS) ] = -1,
  312. },
  313. },
  314. [ C(BPU ) ] = {
  315. [ C(OP_READ) ] = {
  316. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  317. [ C(RESULT_MISS) ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
  318. },
  319. [ C(OP_WRITE) ] = {
  320. [ C(RESULT_ACCESS) ] = -1,
  321. [ C(RESULT_MISS) ] = -1,
  322. },
  323. [ C(OP_PREFETCH) ] = {
  324. [ C(RESULT_ACCESS) ] = -1,
  325. [ C(RESULT_MISS) ] = -1,
  326. },
  327. },
  328. [ C(NODE) ] = {
  329. [ C(OP_READ) ] = {
  330. [ C(RESULT_ACCESS) ] = 0x01b7,
  331. [ C(RESULT_MISS) ] = 0x01b7,
  332. },
  333. [ C(OP_WRITE) ] = {
  334. [ C(RESULT_ACCESS) ] = 0x01b7,
  335. [ C(RESULT_MISS) ] = 0x01b7,
  336. },
  337. [ C(OP_PREFETCH) ] = {
  338. [ C(RESULT_ACCESS) ] = 0x01b7,
  339. [ C(RESULT_MISS) ] = 0x01b7,
  340. },
  341. },
  342. };
  343. static __initconst const u64 westmere_hw_cache_event_ids
  344. [PERF_COUNT_HW_CACHE_MAX]
  345. [PERF_COUNT_HW_CACHE_OP_MAX]
  346. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  347. {
  348. [ C(L1D) ] = {
  349. [ C(OP_READ) ] = {
  350. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  351. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */
  352. },
  353. [ C(OP_WRITE) ] = {
  354. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  355. [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */
  356. },
  357. [ C(OP_PREFETCH) ] = {
  358. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  359. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  360. },
  361. },
  362. [ C(L1I ) ] = {
  363. [ C(OP_READ) ] = {
  364. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  365. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  366. },
  367. [ C(OP_WRITE) ] = {
  368. [ C(RESULT_ACCESS) ] = -1,
  369. [ C(RESULT_MISS) ] = -1,
  370. },
  371. [ C(OP_PREFETCH) ] = {
  372. [ C(RESULT_ACCESS) ] = 0x0,
  373. [ C(RESULT_MISS) ] = 0x0,
  374. },
  375. },
  376. [ C(LL ) ] = {
  377. [ C(OP_READ) ] = {
  378. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  379. [ C(RESULT_ACCESS) ] = 0x01b7,
  380. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  381. [ C(RESULT_MISS) ] = 0x01b7,
  382. },
  383. /*
  384. * Use RFO, not WRITEBACK, because a write miss would typically occur
  385. * on RFO.
  386. */
  387. [ C(OP_WRITE) ] = {
  388. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  389. [ C(RESULT_ACCESS) ] = 0x01b7,
  390. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  391. [ C(RESULT_MISS) ] = 0x01b7,
  392. },
  393. [ C(OP_PREFETCH) ] = {
  394. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  395. [ C(RESULT_ACCESS) ] = 0x01b7,
  396. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  397. [ C(RESULT_MISS) ] = 0x01b7,
  398. },
  399. },
  400. [ C(DTLB) ] = {
  401. [ C(OP_READ) ] = {
  402. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  403. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  404. },
  405. [ C(OP_WRITE) ] = {
  406. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  407. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  408. },
  409. [ C(OP_PREFETCH) ] = {
  410. [ C(RESULT_ACCESS) ] = 0x0,
  411. [ C(RESULT_MISS) ] = 0x0,
  412. },
  413. },
  414. [ C(ITLB) ] = {
  415. [ C(OP_READ) ] = {
  416. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  417. [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.ANY */
  418. },
  419. [ C(OP_WRITE) ] = {
  420. [ C(RESULT_ACCESS) ] = -1,
  421. [ C(RESULT_MISS) ] = -1,
  422. },
  423. [ C(OP_PREFETCH) ] = {
  424. [ C(RESULT_ACCESS) ] = -1,
  425. [ C(RESULT_MISS) ] = -1,
  426. },
  427. },
  428. [ C(BPU ) ] = {
  429. [ C(OP_READ) ] = {
  430. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  431. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  432. },
  433. [ C(OP_WRITE) ] = {
  434. [ C(RESULT_ACCESS) ] = -1,
  435. [ C(RESULT_MISS) ] = -1,
  436. },
  437. [ C(OP_PREFETCH) ] = {
  438. [ C(RESULT_ACCESS) ] = -1,
  439. [ C(RESULT_MISS) ] = -1,
  440. },
  441. },
  442. [ C(NODE) ] = {
  443. [ C(OP_READ) ] = {
  444. [ C(RESULT_ACCESS) ] = 0x01b7,
  445. [ C(RESULT_MISS) ] = 0x01b7,
  446. },
  447. [ C(OP_WRITE) ] = {
  448. [ C(RESULT_ACCESS) ] = 0x01b7,
  449. [ C(RESULT_MISS) ] = 0x01b7,
  450. },
  451. [ C(OP_PREFETCH) ] = {
  452. [ C(RESULT_ACCESS) ] = 0x01b7,
  453. [ C(RESULT_MISS) ] = 0x01b7,
  454. },
  455. },
  456. };
  457. /*
  458. * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits;
  459. * See IA32 SDM Vol 3B 30.6.1.3
  460. */
  461. #define NHM_DMND_DATA_RD (1 << 0)
  462. #define NHM_DMND_RFO (1 << 1)
  463. #define NHM_DMND_IFETCH (1 << 2)
  464. #define NHM_DMND_WB (1 << 3)
  465. #define NHM_PF_DATA_RD (1 << 4)
  466. #define NHM_PF_DATA_RFO (1 << 5)
  467. #define NHM_PF_IFETCH (1 << 6)
  468. #define NHM_OFFCORE_OTHER (1 << 7)
  469. #define NHM_UNCORE_HIT (1 << 8)
  470. #define NHM_OTHER_CORE_HIT_SNP (1 << 9)
  471. #define NHM_OTHER_CORE_HITM (1 << 10)
  472. /* reserved */
  473. #define NHM_REMOTE_CACHE_FWD (1 << 12)
  474. #define NHM_REMOTE_DRAM (1 << 13)
  475. #define NHM_LOCAL_DRAM (1 << 14)
  476. #define NHM_NON_DRAM (1 << 15)
  477. #define NHM_LOCAL (NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD)
  478. #define NHM_REMOTE (NHM_REMOTE_DRAM)
  479. #define NHM_DMND_READ (NHM_DMND_DATA_RD)
  480. #define NHM_DMND_WRITE (NHM_DMND_RFO|NHM_DMND_WB)
  481. #define NHM_DMND_PREFETCH (NHM_PF_DATA_RD|NHM_PF_DATA_RFO)
  482. #define NHM_L3_HIT (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)
  483. #define NHM_L3_MISS (NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD)
  484. #define NHM_L3_ACCESS (NHM_L3_HIT|NHM_L3_MISS)
  485. static __initconst const u64 nehalem_hw_cache_extra_regs
  486. [PERF_COUNT_HW_CACHE_MAX]
  487. [PERF_COUNT_HW_CACHE_OP_MAX]
  488. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  489. {
  490. [ C(LL ) ] = {
  491. [ C(OP_READ) ] = {
  492. [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS,
  493. [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_L3_MISS,
  494. },
  495. [ C(OP_WRITE) ] = {
  496. [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS,
  497. [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_L3_MISS,
  498. },
  499. [ C(OP_PREFETCH) ] = {
  500. [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS,
  501. [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_L3_MISS,
  502. },
  503. },
  504. [ C(NODE) ] = {
  505. [ C(OP_READ) ] = {
  506. [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_LOCAL|NHM_REMOTE,
  507. [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_REMOTE,
  508. },
  509. [ C(OP_WRITE) ] = {
  510. [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_LOCAL|NHM_REMOTE,
  511. [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_REMOTE,
  512. },
  513. [ C(OP_PREFETCH) ] = {
  514. [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_LOCAL|NHM_REMOTE,
  515. [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_REMOTE,
  516. },
  517. },
  518. };
  519. static __initconst const u64 nehalem_hw_cache_event_ids
  520. [PERF_COUNT_HW_CACHE_MAX]
  521. [PERF_COUNT_HW_CACHE_OP_MAX]
  522. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  523. {
  524. [ C(L1D) ] = {
  525. [ C(OP_READ) ] = {
  526. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  527. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */
  528. },
  529. [ C(OP_WRITE) ] = {
  530. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  531. [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */
  532. },
  533. [ C(OP_PREFETCH) ] = {
  534. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  535. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  536. },
  537. },
  538. [ C(L1I ) ] = {
  539. [ C(OP_READ) ] = {
  540. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  541. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  542. },
  543. [ C(OP_WRITE) ] = {
  544. [ C(RESULT_ACCESS) ] = -1,
  545. [ C(RESULT_MISS) ] = -1,
  546. },
  547. [ C(OP_PREFETCH) ] = {
  548. [ C(RESULT_ACCESS) ] = 0x0,
  549. [ C(RESULT_MISS) ] = 0x0,
  550. },
  551. },
  552. [ C(LL ) ] = {
  553. [ C(OP_READ) ] = {
  554. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  555. [ C(RESULT_ACCESS) ] = 0x01b7,
  556. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  557. [ C(RESULT_MISS) ] = 0x01b7,
  558. },
  559. /*
  560. * Use RFO, not WRITEBACK, because a write miss would typically occur
  561. * on RFO.
  562. */
  563. [ C(OP_WRITE) ] = {
  564. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  565. [ C(RESULT_ACCESS) ] = 0x01b7,
  566. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  567. [ C(RESULT_MISS) ] = 0x01b7,
  568. },
  569. [ C(OP_PREFETCH) ] = {
  570. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  571. [ C(RESULT_ACCESS) ] = 0x01b7,
  572. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  573. [ C(RESULT_MISS) ] = 0x01b7,
  574. },
  575. },
  576. [ C(DTLB) ] = {
  577. [ C(OP_READ) ] = {
  578. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  579. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  580. },
  581. [ C(OP_WRITE) ] = {
  582. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  583. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  584. },
  585. [ C(OP_PREFETCH) ] = {
  586. [ C(RESULT_ACCESS) ] = 0x0,
  587. [ C(RESULT_MISS) ] = 0x0,
  588. },
  589. },
  590. [ C(ITLB) ] = {
  591. [ C(OP_READ) ] = {
  592. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  593. [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
  594. },
  595. [ C(OP_WRITE) ] = {
  596. [ C(RESULT_ACCESS) ] = -1,
  597. [ C(RESULT_MISS) ] = -1,
  598. },
  599. [ C(OP_PREFETCH) ] = {
  600. [ C(RESULT_ACCESS) ] = -1,
  601. [ C(RESULT_MISS) ] = -1,
  602. },
  603. },
  604. [ C(BPU ) ] = {
  605. [ C(OP_READ) ] = {
  606. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  607. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  608. },
  609. [ C(OP_WRITE) ] = {
  610. [ C(RESULT_ACCESS) ] = -1,
  611. [ C(RESULT_MISS) ] = -1,
  612. },
  613. [ C(OP_PREFETCH) ] = {
  614. [ C(RESULT_ACCESS) ] = -1,
  615. [ C(RESULT_MISS) ] = -1,
  616. },
  617. },
  618. [ C(NODE) ] = {
  619. [ C(OP_READ) ] = {
  620. [ C(RESULT_ACCESS) ] = 0x01b7,
  621. [ C(RESULT_MISS) ] = 0x01b7,
  622. },
  623. [ C(OP_WRITE) ] = {
  624. [ C(RESULT_ACCESS) ] = 0x01b7,
  625. [ C(RESULT_MISS) ] = 0x01b7,
  626. },
  627. [ C(OP_PREFETCH) ] = {
  628. [ C(RESULT_ACCESS) ] = 0x01b7,
  629. [ C(RESULT_MISS) ] = 0x01b7,
  630. },
  631. },
  632. };
  633. static __initconst const u64 core2_hw_cache_event_ids
  634. [PERF_COUNT_HW_CACHE_MAX]
  635. [PERF_COUNT_HW_CACHE_OP_MAX]
  636. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  637. {
  638. [ C(L1D) ] = {
  639. [ C(OP_READ) ] = {
  640. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  641. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  642. },
  643. [ C(OP_WRITE) ] = {
  644. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  645. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  646. },
  647. [ C(OP_PREFETCH) ] = {
  648. [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
  649. [ C(RESULT_MISS) ] = 0,
  650. },
  651. },
  652. [ C(L1I ) ] = {
  653. [ C(OP_READ) ] = {
  654. [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
  655. [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
  656. },
  657. [ C(OP_WRITE) ] = {
  658. [ C(RESULT_ACCESS) ] = -1,
  659. [ C(RESULT_MISS) ] = -1,
  660. },
  661. [ C(OP_PREFETCH) ] = {
  662. [ C(RESULT_ACCESS) ] = 0,
  663. [ C(RESULT_MISS) ] = 0,
  664. },
  665. },
  666. [ C(LL ) ] = {
  667. [ C(OP_READ) ] = {
  668. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  669. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  670. },
  671. [ C(OP_WRITE) ] = {
  672. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  673. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  674. },
  675. [ C(OP_PREFETCH) ] = {
  676. [ C(RESULT_ACCESS) ] = 0,
  677. [ C(RESULT_MISS) ] = 0,
  678. },
  679. },
  680. [ C(DTLB) ] = {
  681. [ C(OP_READ) ] = {
  682. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  683. [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
  684. },
  685. [ C(OP_WRITE) ] = {
  686. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  687. [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
  688. },
  689. [ C(OP_PREFETCH) ] = {
  690. [ C(RESULT_ACCESS) ] = 0,
  691. [ C(RESULT_MISS) ] = 0,
  692. },
  693. },
  694. [ C(ITLB) ] = {
  695. [ C(OP_READ) ] = {
  696. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  697. [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
  698. },
  699. [ C(OP_WRITE) ] = {
  700. [ C(RESULT_ACCESS) ] = -1,
  701. [ C(RESULT_MISS) ] = -1,
  702. },
  703. [ C(OP_PREFETCH) ] = {
  704. [ C(RESULT_ACCESS) ] = -1,
  705. [ C(RESULT_MISS) ] = -1,
  706. },
  707. },
  708. [ C(BPU ) ] = {
  709. [ C(OP_READ) ] = {
  710. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  711. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  712. },
  713. [ C(OP_WRITE) ] = {
  714. [ C(RESULT_ACCESS) ] = -1,
  715. [ C(RESULT_MISS) ] = -1,
  716. },
  717. [ C(OP_PREFETCH) ] = {
  718. [ C(RESULT_ACCESS) ] = -1,
  719. [ C(RESULT_MISS) ] = -1,
  720. },
  721. },
  722. };
  723. static __initconst const u64 atom_hw_cache_event_ids
  724. [PERF_COUNT_HW_CACHE_MAX]
  725. [PERF_COUNT_HW_CACHE_OP_MAX]
  726. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  727. {
  728. [ C(L1D) ] = {
  729. [ C(OP_READ) ] = {
  730. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
  731. [ C(RESULT_MISS) ] = 0,
  732. },
  733. [ C(OP_WRITE) ] = {
  734. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
  735. [ C(RESULT_MISS) ] = 0,
  736. },
  737. [ C(OP_PREFETCH) ] = {
  738. [ C(RESULT_ACCESS) ] = 0x0,
  739. [ C(RESULT_MISS) ] = 0,
  740. },
  741. },
  742. [ C(L1I ) ] = {
  743. [ C(OP_READ) ] = {
  744. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  745. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  746. },
  747. [ C(OP_WRITE) ] = {
  748. [ C(RESULT_ACCESS) ] = -1,
  749. [ C(RESULT_MISS) ] = -1,
  750. },
  751. [ C(OP_PREFETCH) ] = {
  752. [ C(RESULT_ACCESS) ] = 0,
  753. [ C(RESULT_MISS) ] = 0,
  754. },
  755. },
  756. [ C(LL ) ] = {
  757. [ C(OP_READ) ] = {
  758. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  759. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  760. },
  761. [ C(OP_WRITE) ] = {
  762. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  763. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  764. },
  765. [ C(OP_PREFETCH) ] = {
  766. [ C(RESULT_ACCESS) ] = 0,
  767. [ C(RESULT_MISS) ] = 0,
  768. },
  769. },
  770. [ C(DTLB) ] = {
  771. [ C(OP_READ) ] = {
  772. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
  773. [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
  774. },
  775. [ C(OP_WRITE) ] = {
  776. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
  777. [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
  778. },
  779. [ C(OP_PREFETCH) ] = {
  780. [ C(RESULT_ACCESS) ] = 0,
  781. [ C(RESULT_MISS) ] = 0,
  782. },
  783. },
  784. [ C(ITLB) ] = {
  785. [ C(OP_READ) ] = {
  786. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  787. [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
  788. },
  789. [ C(OP_WRITE) ] = {
  790. [ C(RESULT_ACCESS) ] = -1,
  791. [ C(RESULT_MISS) ] = -1,
  792. },
  793. [ C(OP_PREFETCH) ] = {
  794. [ C(RESULT_ACCESS) ] = -1,
  795. [ C(RESULT_MISS) ] = -1,
  796. },
  797. },
  798. [ C(BPU ) ] = {
  799. [ C(OP_READ) ] = {
  800. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  801. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  802. },
  803. [ C(OP_WRITE) ] = {
  804. [ C(RESULT_ACCESS) ] = -1,
  805. [ C(RESULT_MISS) ] = -1,
  806. },
  807. [ C(OP_PREFETCH) ] = {
  808. [ C(RESULT_ACCESS) ] = -1,
  809. [ C(RESULT_MISS) ] = -1,
  810. },
  811. },
  812. };
  813. static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
  814. {
  815. /* user explicitly requested branch sampling */
  816. if (has_branch_stack(event))
  817. return true;
  818. /* implicit branch sampling to correct PEBS skid */
  819. if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
  820. return true;
  821. return false;
  822. }
  823. static void intel_pmu_disable_all(void)
  824. {
  825. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  826. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
  827. if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
  828. intel_pmu_disable_bts();
  829. intel_pmu_pebs_disable_all();
  830. intel_pmu_lbr_disable_all();
  831. }
  832. static void intel_pmu_enable_all(int added)
  833. {
  834. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  835. intel_pmu_pebs_enable_all();
  836. intel_pmu_lbr_enable_all();
  837. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
  838. x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
  839. if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
  840. struct perf_event *event =
  841. cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
  842. if (WARN_ON_ONCE(!event))
  843. return;
  844. intel_pmu_enable_bts(event->hw.config);
  845. }
  846. }
  847. /*
  848. * Workaround for:
  849. * Intel Errata AAK100 (model 26)
  850. * Intel Errata AAP53 (model 30)
  851. * Intel Errata BD53 (model 44)
  852. *
  853. * The official story:
  854. * These chips need to be 'reset' when adding counters by programming the
  855. * magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either
  856. * in sequence on the same PMC or on different PMCs.
  857. *
  858. * In practise it appears some of these events do in fact count, and
  859. * we need to programm all 4 events.
  860. */
  861. static void intel_pmu_nhm_workaround(void)
  862. {
  863. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  864. static const unsigned long nhm_magic[4] = {
  865. 0x4300B5,
  866. 0x4300D2,
  867. 0x4300B1,
  868. 0x4300B1
  869. };
  870. struct perf_event *event;
  871. int i;
  872. /*
  873. * The Errata requires below steps:
  874. * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL;
  875. * 2) Configure 4 PERFEVTSELx with the magic events and clear
  876. * the corresponding PMCx;
  877. * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL;
  878. * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL;
  879. * 5) Clear 4 pairs of ERFEVTSELx and PMCx;
  880. */
  881. /*
  882. * The real steps we choose are a little different from above.
  883. * A) To reduce MSR operations, we don't run step 1) as they
  884. * are already cleared before this function is called;
  885. * B) Call x86_perf_event_update to save PMCx before configuring
  886. * PERFEVTSELx with magic number;
  887. * C) With step 5), we do clear only when the PERFEVTSELx is
  888. * not used currently.
  889. * D) Call x86_perf_event_set_period to restore PMCx;
  890. */
  891. /* We always operate 4 pairs of PERF Counters */
  892. for (i = 0; i < 4; i++) {
  893. event = cpuc->events[i];
  894. if (event)
  895. x86_perf_event_update(event);
  896. }
  897. for (i = 0; i < 4; i++) {
  898. wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
  899. wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
  900. }
  901. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
  902. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
  903. for (i = 0; i < 4; i++) {
  904. event = cpuc->events[i];
  905. if (event) {
  906. x86_perf_event_set_period(event);
  907. __x86_pmu_enable_event(&event->hw,
  908. ARCH_PERFMON_EVENTSEL_ENABLE);
  909. } else
  910. wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
  911. }
  912. }
  913. static void intel_pmu_nhm_enable_all(int added)
  914. {
  915. if (added)
  916. intel_pmu_nhm_workaround();
  917. intel_pmu_enable_all(added);
  918. }
  919. static inline u64 intel_pmu_get_status(void)
  920. {
  921. u64 status;
  922. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  923. return status;
  924. }
  925. static inline void intel_pmu_ack_status(u64 ack)
  926. {
  927. wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
  928. }
  929. static void intel_pmu_disable_fixed(struct hw_perf_event *hwc)
  930. {
  931. int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
  932. u64 ctrl_val, mask;
  933. mask = 0xfULL << (idx * 4);
  934. rdmsrl(hwc->config_base, ctrl_val);
  935. ctrl_val &= ~mask;
  936. wrmsrl(hwc->config_base, ctrl_val);
  937. }
  938. static void intel_pmu_disable_event(struct perf_event *event)
  939. {
  940. struct hw_perf_event *hwc = &event->hw;
  941. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  942. if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
  943. intel_pmu_disable_bts();
  944. intel_pmu_drain_bts_buffer();
  945. return;
  946. }
  947. cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
  948. cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
  949. /*
  950. * must disable before any actual event
  951. * because any event may be combined with LBR
  952. */
  953. if (intel_pmu_needs_lbr_smpl(event))
  954. intel_pmu_lbr_disable(event);
  955. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  956. intel_pmu_disable_fixed(hwc);
  957. return;
  958. }
  959. x86_pmu_disable_event(event);
  960. if (unlikely(event->attr.precise_ip))
  961. intel_pmu_pebs_disable(event);
  962. }
  963. static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
  964. {
  965. int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
  966. u64 ctrl_val, bits, mask;
  967. /*
  968. * Enable IRQ generation (0x8),
  969. * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
  970. * if requested:
  971. */
  972. bits = 0x8ULL;
  973. if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
  974. bits |= 0x2;
  975. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  976. bits |= 0x1;
  977. /*
  978. * ANY bit is supported in v3 and up
  979. */
  980. if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
  981. bits |= 0x4;
  982. bits <<= (idx * 4);
  983. mask = 0xfULL << (idx * 4);
  984. rdmsrl(hwc->config_base, ctrl_val);
  985. ctrl_val &= ~mask;
  986. ctrl_val |= bits;
  987. wrmsrl(hwc->config_base, ctrl_val);
  988. }
  989. static void intel_pmu_enable_event(struct perf_event *event)
  990. {
  991. struct hw_perf_event *hwc = &event->hw;
  992. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  993. if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
  994. if (!__this_cpu_read(cpu_hw_events.enabled))
  995. return;
  996. intel_pmu_enable_bts(hwc->config);
  997. return;
  998. }
  999. /*
  1000. * must enabled before any actual event
  1001. * because any event may be combined with LBR
  1002. */
  1003. if (intel_pmu_needs_lbr_smpl(event))
  1004. intel_pmu_lbr_enable(event);
  1005. if (event->attr.exclude_host)
  1006. cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
  1007. if (event->attr.exclude_guest)
  1008. cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);
  1009. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1010. intel_pmu_enable_fixed(hwc);
  1011. return;
  1012. }
  1013. if (unlikely(event->attr.precise_ip))
  1014. intel_pmu_pebs_enable(event);
  1015. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  1016. }
  1017. /*
  1018. * Save and restart an expired event. Called by NMI contexts,
  1019. * so it has to be careful about preempting normal event ops:
  1020. */
  1021. int intel_pmu_save_and_restart(struct perf_event *event)
  1022. {
  1023. x86_perf_event_update(event);
  1024. return x86_perf_event_set_period(event);
  1025. }
  1026. static void intel_pmu_reset(void)
  1027. {
  1028. struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
  1029. unsigned long flags;
  1030. int idx;
  1031. if (!x86_pmu.num_counters)
  1032. return;
  1033. local_irq_save(flags);
  1034. pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());
  1035. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1036. wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
  1037. wrmsrl_safe(x86_pmu_event_addr(idx), 0ull);
  1038. }
  1039. for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
  1040. wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
  1041. if (ds)
  1042. ds->bts_index = ds->bts_buffer_base;
  1043. local_irq_restore(flags);
  1044. }
  1045. /*
  1046. * This handler is triggered by the local APIC, so the APIC IRQ handling
  1047. * rules apply:
  1048. */
  1049. static int intel_pmu_handle_irq(struct pt_regs *regs)
  1050. {
  1051. struct perf_sample_data data;
  1052. struct cpu_hw_events *cpuc;
  1053. int bit, loops;
  1054. u64 status;
  1055. int handled;
  1056. cpuc = &__get_cpu_var(cpu_hw_events);
  1057. /*
  1058. * Some chipsets need to unmask the LVTPC in a particular spot
  1059. * inside the nmi handler. As a result, the unmasking was pushed
  1060. * into all the nmi handlers.
  1061. *
  1062. * This handler doesn't seem to have any issues with the unmasking
  1063. * so it was left at the top.
  1064. */
  1065. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1066. intel_pmu_disable_all();
  1067. handled = intel_pmu_drain_bts_buffer();
  1068. status = intel_pmu_get_status();
  1069. if (!status) {
  1070. intel_pmu_enable_all(0);
  1071. return handled;
  1072. }
  1073. loops = 0;
  1074. again:
  1075. intel_pmu_ack_status(status);
  1076. if (++loops > 100) {
  1077. WARN_ONCE(1, "perfevents: irq loop stuck!\n");
  1078. perf_event_print_debug();
  1079. intel_pmu_reset();
  1080. goto done;
  1081. }
  1082. inc_irq_stat(apic_perf_irqs);
  1083. intel_pmu_lbr_read();
  1084. /*
  1085. * PEBS overflow sets bit 62 in the global status register
  1086. */
  1087. if (__test_and_clear_bit(62, (unsigned long *)&status)) {
  1088. handled++;
  1089. x86_pmu.drain_pebs(regs);
  1090. }
  1091. for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
  1092. struct perf_event *event = cpuc->events[bit];
  1093. handled++;
  1094. if (!test_bit(bit, cpuc->active_mask))
  1095. continue;
  1096. if (!intel_pmu_save_and_restart(event))
  1097. continue;
  1098. perf_sample_data_init(&data, 0, event->hw.last_period);
  1099. if (has_branch_stack(event))
  1100. data.br_stack = &cpuc->lbr_stack;
  1101. if (perf_event_overflow(event, &data, regs))
  1102. x86_pmu_stop(event, 0);
  1103. }
  1104. /*
  1105. * Repeat if there is more work to be done:
  1106. */
  1107. status = intel_pmu_get_status();
  1108. if (status)
  1109. goto again;
  1110. done:
  1111. intel_pmu_enable_all(0);
  1112. return handled;
  1113. }
  1114. static struct event_constraint *
  1115. intel_bts_constraints(struct perf_event *event)
  1116. {
  1117. struct hw_perf_event *hwc = &event->hw;
  1118. unsigned int hw_event, bts_event;
  1119. if (event->attr.freq)
  1120. return NULL;
  1121. hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
  1122. bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
  1123. if (unlikely(hw_event == bts_event && hwc->sample_period == 1))
  1124. return &bts_constraint;
  1125. return NULL;
  1126. }
  1127. static int intel_alt_er(int idx)
  1128. {
  1129. if (!(x86_pmu.er_flags & ERF_HAS_RSP_1))
  1130. return idx;
  1131. if (idx == EXTRA_REG_RSP_0)
  1132. return EXTRA_REG_RSP_1;
  1133. if (idx == EXTRA_REG_RSP_1)
  1134. return EXTRA_REG_RSP_0;
  1135. return idx;
  1136. }
  1137. static void intel_fixup_er(struct perf_event *event, int idx)
  1138. {
  1139. event->hw.extra_reg.idx = idx;
  1140. if (idx == EXTRA_REG_RSP_0) {
  1141. event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
  1142. event->hw.config |= 0x01b7;
  1143. event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0;
  1144. } else if (idx == EXTRA_REG_RSP_1) {
  1145. event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
  1146. event->hw.config |= 0x01bb;
  1147. event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1;
  1148. }
  1149. }
  1150. /*
  1151. * manage allocation of shared extra msr for certain events
  1152. *
  1153. * sharing can be:
  1154. * per-cpu: to be shared between the various events on a single PMU
  1155. * per-core: per-cpu + shared by HT threads
  1156. */
  1157. static struct event_constraint *
  1158. __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
  1159. struct perf_event *event,
  1160. struct hw_perf_event_extra *reg)
  1161. {
  1162. struct event_constraint *c = &emptyconstraint;
  1163. struct er_account *era;
  1164. unsigned long flags;
  1165. int idx = reg->idx;
  1166. /*
  1167. * reg->alloc can be set due to existing state, so for fake cpuc we
  1168. * need to ignore this, otherwise we might fail to allocate proper fake
  1169. * state for this extra reg constraint. Also see the comment below.
  1170. */
  1171. if (reg->alloc && !cpuc->is_fake)
  1172. return NULL; /* call x86_get_event_constraint() */
  1173. again:
  1174. era = &cpuc->shared_regs->regs[idx];
  1175. /*
  1176. * we use spin_lock_irqsave() to avoid lockdep issues when
  1177. * passing a fake cpuc
  1178. */
  1179. raw_spin_lock_irqsave(&era->lock, flags);
  1180. if (!atomic_read(&era->ref) || era->config == reg->config) {
  1181. /*
  1182. * If its a fake cpuc -- as per validate_{group,event}() we
  1183. * shouldn't touch event state and we can avoid doing so
  1184. * since both will only call get_event_constraints() once
  1185. * on each event, this avoids the need for reg->alloc.
  1186. *
  1187. * Not doing the ER fixup will only result in era->reg being
  1188. * wrong, but since we won't actually try and program hardware
  1189. * this isn't a problem either.
  1190. */
  1191. if (!cpuc->is_fake) {
  1192. if (idx != reg->idx)
  1193. intel_fixup_er(event, idx);
  1194. /*
  1195. * x86_schedule_events() can call get_event_constraints()
  1196. * multiple times on events in the case of incremental
  1197. * scheduling(). reg->alloc ensures we only do the ER
  1198. * allocation once.
  1199. */
  1200. reg->alloc = 1;
  1201. }
  1202. /* lock in msr value */
  1203. era->config = reg->config;
  1204. era->reg = reg->reg;
  1205. /* one more user */
  1206. atomic_inc(&era->ref);
  1207. /*
  1208. * need to call x86_get_event_constraint()
  1209. * to check if associated event has constraints
  1210. */
  1211. c = NULL;
  1212. } else {
  1213. idx = intel_alt_er(idx);
  1214. if (idx != reg->idx) {
  1215. raw_spin_unlock_irqrestore(&era->lock, flags);
  1216. goto again;
  1217. }
  1218. }
  1219. raw_spin_unlock_irqrestore(&era->lock, flags);
  1220. return c;
  1221. }
  1222. static void
  1223. __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,
  1224. struct hw_perf_event_extra *reg)
  1225. {
  1226. struct er_account *era;
  1227. /*
  1228. * Only put constraint if extra reg was actually allocated. Also takes
  1229. * care of event which do not use an extra shared reg.
  1230. *
  1231. * Also, if this is a fake cpuc we shouldn't touch any event state
  1232. * (reg->alloc) and we don't care about leaving inconsistent cpuc state
  1233. * either since it'll be thrown out.
  1234. */
  1235. if (!reg->alloc || cpuc->is_fake)
  1236. return;
  1237. era = &cpuc->shared_regs->regs[reg->idx];
  1238. /* one fewer user */
  1239. atomic_dec(&era->ref);
  1240. /* allocate again next time */
  1241. reg->alloc = 0;
  1242. }
  1243. static struct event_constraint *
  1244. intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
  1245. struct perf_event *event)
  1246. {
  1247. struct event_constraint *c = NULL, *d;
  1248. struct hw_perf_event_extra *xreg, *breg;
  1249. xreg = &event->hw.extra_reg;
  1250. if (xreg->idx != EXTRA_REG_NONE) {
  1251. c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
  1252. if (c == &emptyconstraint)
  1253. return c;
  1254. }
  1255. breg = &event->hw.branch_reg;
  1256. if (breg->idx != EXTRA_REG_NONE) {
  1257. d = __intel_shared_reg_get_constraints(cpuc, event, breg);
  1258. if (d == &emptyconstraint) {
  1259. __intel_shared_reg_put_constraints(cpuc, xreg);
  1260. c = d;
  1261. }
  1262. }
  1263. return c;
  1264. }
  1265. struct event_constraint *
  1266. x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1267. {
  1268. struct event_constraint *c;
  1269. if (x86_pmu.event_constraints) {
  1270. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1271. if ((event->hw.config & c->cmask) == c->code) {
  1272. /* hw.flags zeroed at initialization */
  1273. event->hw.flags |= c->flags;
  1274. return c;
  1275. }
  1276. }
  1277. }
  1278. return &unconstrained;
  1279. }
  1280. static struct event_constraint *
  1281. intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1282. {
  1283. struct event_constraint *c;
  1284. c = intel_bts_constraints(event);
  1285. if (c)
  1286. return c;
  1287. c = intel_pebs_constraints(event);
  1288. if (c)
  1289. return c;
  1290. c = intel_shared_regs_constraints(cpuc, event);
  1291. if (c)
  1292. return c;
  1293. return x86_get_event_constraints(cpuc, event);
  1294. }
  1295. static void
  1296. intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
  1297. struct perf_event *event)
  1298. {
  1299. struct hw_perf_event_extra *reg;
  1300. reg = &event->hw.extra_reg;
  1301. if (reg->idx != EXTRA_REG_NONE)
  1302. __intel_shared_reg_put_constraints(cpuc, reg);
  1303. reg = &event->hw.branch_reg;
  1304. if (reg->idx != EXTRA_REG_NONE)
  1305. __intel_shared_reg_put_constraints(cpuc, reg);
  1306. }
  1307. static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
  1308. struct perf_event *event)
  1309. {
  1310. event->hw.flags = 0;
  1311. intel_put_shared_regs_event_constraints(cpuc, event);
  1312. }
  1313. static void intel_pebs_aliases_core2(struct perf_event *event)
  1314. {
  1315. if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
  1316. /*
  1317. * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
  1318. * (0x003c) so that we can use it with PEBS.
  1319. *
  1320. * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
  1321. * PEBS capable. However we can use INST_RETIRED.ANY_P
  1322. * (0x00c0), which is a PEBS capable event, to get the same
  1323. * count.
  1324. *
  1325. * INST_RETIRED.ANY_P counts the number of cycles that retires
  1326. * CNTMASK instructions. By setting CNTMASK to a value (16)
  1327. * larger than the maximum number of instructions that can be
  1328. * retired per cycle (4) and then inverting the condition, we
  1329. * count all cycles that retire 16 or less instructions, which
  1330. * is every cycle.
  1331. *
  1332. * Thereby we gain a PEBS capable cycle counter.
  1333. */
  1334. u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16);
  1335. alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
  1336. event->hw.config = alt_config;
  1337. }
  1338. }
  1339. static void intel_pebs_aliases_snb(struct perf_event *event)
  1340. {
  1341. if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
  1342. /*
  1343. * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
  1344. * (0x003c) so that we can use it with PEBS.
  1345. *
  1346. * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
  1347. * PEBS capable. However we can use UOPS_RETIRED.ALL
  1348. * (0x01c2), which is a PEBS capable event, to get the same
  1349. * count.
  1350. *
  1351. * UOPS_RETIRED.ALL counts the number of cycles that retires
  1352. * CNTMASK micro-ops. By setting CNTMASK to a value (16)
  1353. * larger than the maximum number of micro-ops that can be
  1354. * retired per cycle (4) and then inverting the condition, we
  1355. * count all cycles that retire 16 or less micro-ops, which
  1356. * is every cycle.
  1357. *
  1358. * Thereby we gain a PEBS capable cycle counter.
  1359. */
  1360. u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16);
  1361. alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
  1362. event->hw.config = alt_config;
  1363. }
  1364. }
  1365. static int intel_pmu_hw_config(struct perf_event *event)
  1366. {
  1367. int ret = x86_pmu_hw_config(event);
  1368. if (ret)
  1369. return ret;
  1370. if (event->attr.precise_ip && x86_pmu.pebs_aliases)
  1371. x86_pmu.pebs_aliases(event);
  1372. if (intel_pmu_needs_lbr_smpl(event)) {
  1373. ret = intel_pmu_setup_lbr_filter(event);
  1374. if (ret)
  1375. return ret;
  1376. }
  1377. if (event->attr.type != PERF_TYPE_RAW)
  1378. return 0;
  1379. if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
  1380. return 0;
  1381. if (x86_pmu.version < 3)
  1382. return -EINVAL;
  1383. if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
  1384. return -EACCES;
  1385. event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;
  1386. return 0;
  1387. }
  1388. struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
  1389. {
  1390. if (x86_pmu.guest_get_msrs)
  1391. return x86_pmu.guest_get_msrs(nr);
  1392. *nr = 0;
  1393. return NULL;
  1394. }
  1395. EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
  1396. static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
  1397. {
  1398. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1399. struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
  1400. arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
  1401. arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;
  1402. arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;
  1403. /*
  1404. * If PMU counter has PEBS enabled it is not enough to disable counter
  1405. * on a guest entry since PEBS memory write can overshoot guest entry
  1406. * and corrupt guest memory. Disabling PEBS solves the problem.
  1407. */
  1408. arr[1].msr = MSR_IA32_PEBS_ENABLE;
  1409. arr[1].host = cpuc->pebs_enabled;
  1410. arr[1].guest = 0;
  1411. *nr = 2;
  1412. return arr;
  1413. }
  1414. static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr)
  1415. {
  1416. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1417. struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
  1418. int idx;
  1419. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1420. struct perf_event *event = cpuc->events[idx];
  1421. arr[idx].msr = x86_pmu_config_addr(idx);
  1422. arr[idx].host = arr[idx].guest = 0;
  1423. if (!test_bit(idx, cpuc->active_mask))
  1424. continue;
  1425. arr[idx].host = arr[idx].guest =
  1426. event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE;
  1427. if (event->attr.exclude_host)
  1428. arr[idx].host &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  1429. else if (event->attr.exclude_guest)
  1430. arr[idx].guest &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  1431. }
  1432. *nr = x86_pmu.num_counters;
  1433. return arr;
  1434. }
  1435. static void core_pmu_enable_event(struct perf_event *event)
  1436. {
  1437. if (!event->attr.exclude_host)
  1438. x86_pmu_enable_event(event);
  1439. }
  1440. static void core_pmu_enable_all(int added)
  1441. {
  1442. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1443. int idx;
  1444. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1445. struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
  1446. if (!test_bit(idx, cpuc->active_mask) ||
  1447. cpuc->events[idx]->attr.exclude_host)
  1448. continue;
  1449. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  1450. }
  1451. }
  1452. PMU_FORMAT_ATTR(event, "config:0-7" );
  1453. PMU_FORMAT_ATTR(umask, "config:8-15" );
  1454. PMU_FORMAT_ATTR(edge, "config:18" );
  1455. PMU_FORMAT_ATTR(pc, "config:19" );
  1456. PMU_FORMAT_ATTR(any, "config:21" ); /* v3 + */
  1457. PMU_FORMAT_ATTR(inv, "config:23" );
  1458. PMU_FORMAT_ATTR(cmask, "config:24-31" );
  1459. static struct attribute *intel_arch_formats_attr[] = {
  1460. &format_attr_event.attr,
  1461. &format_attr_umask.attr,
  1462. &format_attr_edge.attr,
  1463. &format_attr_pc.attr,
  1464. &format_attr_inv.attr,
  1465. &format_attr_cmask.attr,
  1466. NULL,
  1467. };
  1468. ssize_t intel_event_sysfs_show(char *page, u64 config)
  1469. {
  1470. u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
  1471. return x86_event_sysfs_show(page, config, event);
  1472. }
  1473. static __initconst const struct x86_pmu core_pmu = {
  1474. .name = "core",
  1475. .handle_irq = x86_pmu_handle_irq,
  1476. .disable_all = x86_pmu_disable_all,
  1477. .enable_all = core_pmu_enable_all,
  1478. .enable = core_pmu_enable_event,
  1479. .disable = x86_pmu_disable_event,
  1480. .hw_config = x86_pmu_hw_config,
  1481. .schedule_events = x86_schedule_events,
  1482. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1483. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1484. .event_map = intel_pmu_event_map,
  1485. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1486. .apic = 1,
  1487. /*
  1488. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1489. * so we install an artificial 1<<31 period regardless of
  1490. * the generic event period:
  1491. */
  1492. .max_period = (1ULL << 31) - 1,
  1493. .get_event_constraints = intel_get_event_constraints,
  1494. .put_event_constraints = intel_put_event_constraints,
  1495. .event_constraints = intel_core_event_constraints,
  1496. .guest_get_msrs = core_guest_get_msrs,
  1497. .format_attrs = intel_arch_formats_attr,
  1498. .events_sysfs_show = intel_event_sysfs_show,
  1499. };
  1500. struct intel_shared_regs *allocate_shared_regs(int cpu)
  1501. {
  1502. struct intel_shared_regs *regs;
  1503. int i;
  1504. regs = kzalloc_node(sizeof(struct intel_shared_regs),
  1505. GFP_KERNEL, cpu_to_node(cpu));
  1506. if (regs) {
  1507. /*
  1508. * initialize the locks to keep lockdep happy
  1509. */
  1510. for (i = 0; i < EXTRA_REG_MAX; i++)
  1511. raw_spin_lock_init(&regs->regs[i].lock);
  1512. regs->core_id = -1;
  1513. }
  1514. return regs;
  1515. }
  1516. static int intel_pmu_cpu_prepare(int cpu)
  1517. {
  1518. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1519. if (!(x86_pmu.extra_regs || x86_pmu.lbr_sel_map))
  1520. return NOTIFY_OK;
  1521. cpuc->shared_regs = allocate_shared_regs(cpu);
  1522. if (!cpuc->shared_regs)
  1523. return NOTIFY_BAD;
  1524. return NOTIFY_OK;
  1525. }
  1526. static void intel_pmu_cpu_starting(int cpu)
  1527. {
  1528. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1529. int core_id = topology_core_id(cpu);
  1530. int i;
  1531. init_debug_store_on_cpu(cpu);
  1532. /*
  1533. * Deal with CPUs that don't clear their LBRs on power-up.
  1534. */
  1535. intel_pmu_lbr_reset();
  1536. cpuc->lbr_sel = NULL;
  1537. if (!cpuc->shared_regs)
  1538. return;
  1539. if (!(x86_pmu.er_flags & ERF_NO_HT_SHARING)) {
  1540. for_each_cpu(i, topology_thread_cpumask(cpu)) {
  1541. struct intel_shared_regs *pc;
  1542. pc = per_cpu(cpu_hw_events, i).shared_regs;
  1543. if (pc && pc->core_id == core_id) {
  1544. cpuc->kfree_on_online = cpuc->shared_regs;
  1545. cpuc->shared_regs = pc;
  1546. break;
  1547. }
  1548. }
  1549. cpuc->shared_regs->core_id = core_id;
  1550. cpuc->shared_regs->refcnt++;
  1551. }
  1552. if (x86_pmu.lbr_sel_map)
  1553. cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
  1554. }
  1555. static void intel_pmu_cpu_dying(int cpu)
  1556. {
  1557. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1558. struct intel_shared_regs *pc;
  1559. pc = cpuc->shared_regs;
  1560. if (pc) {
  1561. if (pc->core_id == -1 || --pc->refcnt == 0)
  1562. kfree(pc);
  1563. cpuc->shared_regs = NULL;
  1564. }
  1565. fini_debug_store_on_cpu(cpu);
  1566. }
  1567. static void intel_pmu_flush_branch_stack(void)
  1568. {
  1569. /*
  1570. * Intel LBR does not tag entries with the
  1571. * PID of the current task, then we need to
  1572. * flush it on ctxsw
  1573. * For now, we simply reset it
  1574. */
  1575. if (x86_pmu.lbr_nr)
  1576. intel_pmu_lbr_reset();
  1577. }
  1578. PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
  1579. PMU_FORMAT_ATTR(ldlat, "config1:0-15");
  1580. static struct attribute *intel_arch3_formats_attr[] = {
  1581. &format_attr_event.attr,
  1582. &format_attr_umask.attr,
  1583. &format_attr_edge.attr,
  1584. &format_attr_pc.attr,
  1585. &format_attr_any.attr,
  1586. &format_attr_inv.attr,
  1587. &format_attr_cmask.attr,
  1588. &format_attr_offcore_rsp.attr, /* XXX do NHM/WSM + SNB breakout */
  1589. &format_attr_ldlat.attr, /* PEBS load latency */
  1590. NULL,
  1591. };
  1592. static __initconst const struct x86_pmu intel_pmu = {
  1593. .name = "Intel",
  1594. .handle_irq = intel_pmu_handle_irq,
  1595. .disable_all = intel_pmu_disable_all,
  1596. .enable_all = intel_pmu_enable_all,
  1597. .enable = intel_pmu_enable_event,
  1598. .disable = intel_pmu_disable_event,
  1599. .hw_config = intel_pmu_hw_config,
  1600. .schedule_events = x86_schedule_events,
  1601. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1602. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1603. .event_map = intel_pmu_event_map,
  1604. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1605. .apic = 1,
  1606. /*
  1607. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1608. * so we install an artificial 1<<31 period regardless of
  1609. * the generic event period:
  1610. */
  1611. .max_period = (1ULL << 31) - 1,
  1612. .get_event_constraints = intel_get_event_constraints,
  1613. .put_event_constraints = intel_put_event_constraints,
  1614. .pebs_aliases = intel_pebs_aliases_core2,
  1615. .format_attrs = intel_arch3_formats_attr,
  1616. .events_sysfs_show = intel_event_sysfs_show,
  1617. .cpu_prepare = intel_pmu_cpu_prepare,
  1618. .cpu_starting = intel_pmu_cpu_starting,
  1619. .cpu_dying = intel_pmu_cpu_dying,
  1620. .guest_get_msrs = intel_guest_get_msrs,
  1621. .flush_branch_stack = intel_pmu_flush_branch_stack,
  1622. };
  1623. static __init void intel_clovertown_quirk(void)
  1624. {
  1625. /*
  1626. * PEBS is unreliable due to:
  1627. *
  1628. * AJ67 - PEBS may experience CPL leaks
  1629. * AJ68 - PEBS PMI may be delayed by one event
  1630. * AJ69 - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12]
  1631. * AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS
  1632. *
  1633. * AJ67 could be worked around by restricting the OS/USR flags.
  1634. * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI.
  1635. *
  1636. * AJ106 could possibly be worked around by not allowing LBR
  1637. * usage from PEBS, including the fixup.
  1638. * AJ68 could possibly be worked around by always programming
  1639. * a pebs_event_reset[0] value and coping with the lost events.
  1640. *
  1641. * But taken together it might just make sense to not enable PEBS on
  1642. * these chips.
  1643. */
  1644. pr_warn("PEBS disabled due to CPU errata\n");
  1645. x86_pmu.pebs = 0;
  1646. x86_pmu.pebs_constraints = NULL;
  1647. }
  1648. static int intel_snb_pebs_broken(int cpu)
  1649. {
  1650. u32 rev = UINT_MAX; /* default to broken for unknown models */
  1651. switch (cpu_data(cpu).x86_model) {
  1652. case 42: /* SNB */
  1653. rev = 0x28;
  1654. break;
  1655. case 45: /* SNB-EP */
  1656. switch (cpu_data(cpu).x86_mask) {
  1657. case 6: rev = 0x618; break;
  1658. case 7: rev = 0x70c; break;
  1659. }
  1660. }
  1661. return (cpu_data(cpu).microcode < rev);
  1662. }
  1663. static void intel_snb_check_microcode(void)
  1664. {
  1665. int pebs_broken = 0;
  1666. int cpu;
  1667. get_online_cpus();
  1668. for_each_online_cpu(cpu) {
  1669. if ((pebs_broken = intel_snb_pebs_broken(cpu)))
  1670. break;
  1671. }
  1672. put_online_cpus();
  1673. if (pebs_broken == x86_pmu.pebs_broken)
  1674. return;
  1675. /*
  1676. * Serialized by the microcode lock..
  1677. */
  1678. if (x86_pmu.pebs_broken) {
  1679. pr_info("PEBS enabled due to microcode update\n");
  1680. x86_pmu.pebs_broken = 0;
  1681. } else {
  1682. pr_info("PEBS disabled due to CPU errata, please upgrade microcode\n");
  1683. x86_pmu.pebs_broken = 1;
  1684. }
  1685. }
  1686. static __init void intel_sandybridge_quirk(void)
  1687. {
  1688. x86_pmu.check_microcode = intel_snb_check_microcode;
  1689. intel_snb_check_microcode();
  1690. }
  1691. static const struct { int id; char *name; } intel_arch_events_map[] __initconst = {
  1692. { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
  1693. { PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
  1694. { PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
  1695. { PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
  1696. { PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
  1697. { PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
  1698. { PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
  1699. };
  1700. static __init void intel_arch_events_quirk(void)
  1701. {
  1702. int bit;
  1703. /* disable event that reported as not presend by cpuid */
  1704. for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(intel_arch_events_map)) {
  1705. intel_perfmon_event_map[intel_arch_events_map[bit].id] = 0;
  1706. pr_warn("CPUID marked event: \'%s\' unavailable\n",
  1707. intel_arch_events_map[bit].name);
  1708. }
  1709. }
  1710. static __init void intel_nehalem_quirk(void)
  1711. {
  1712. union cpuid10_ebx ebx;
  1713. ebx.full = x86_pmu.events_maskl;
  1714. if (ebx.split.no_branch_misses_retired) {
  1715. /*
  1716. * Erratum AAJ80 detected, we work it around by using
  1717. * the BR_MISP_EXEC.ANY event. This will over-count
  1718. * branch-misses, but it's still much better than the
  1719. * architectural event which is often completely bogus:
  1720. */
  1721. intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89;
  1722. ebx.split.no_branch_misses_retired = 0;
  1723. x86_pmu.events_maskl = ebx.full;
  1724. pr_info("CPU erratum AAJ80 worked around\n");
  1725. }
  1726. }
  1727. __init int intel_pmu_init(void)
  1728. {
  1729. union cpuid10_edx edx;
  1730. union cpuid10_eax eax;
  1731. union cpuid10_ebx ebx;
  1732. struct event_constraint *c;
  1733. unsigned int unused;
  1734. int version;
  1735. if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
  1736. switch (boot_cpu_data.x86) {
  1737. case 0x6:
  1738. return p6_pmu_init();
  1739. case 0xb:
  1740. return knc_pmu_init();
  1741. case 0xf:
  1742. return p4_pmu_init();
  1743. }
  1744. return -ENODEV;
  1745. }
  1746. /*
  1747. * Check whether the Architectural PerfMon supports
  1748. * Branch Misses Retired hw_event or not.
  1749. */
  1750. cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);
  1751. if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT)
  1752. return -ENODEV;
  1753. version = eax.split.version_id;
  1754. if (version < 2)
  1755. x86_pmu = core_pmu;
  1756. else
  1757. x86_pmu = intel_pmu;
  1758. x86_pmu.version = version;
  1759. x86_pmu.num_counters = eax.split.num_counters;
  1760. x86_pmu.cntval_bits = eax.split.bit_width;
  1761. x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1;
  1762. x86_pmu.events_maskl = ebx.full;
  1763. x86_pmu.events_mask_len = eax.split.mask_length;
  1764. x86_pmu.max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, x86_pmu.num_counters);
  1765. /*
  1766. * Quirk: v2 perfmon does not report fixed-purpose events, so
  1767. * assume at least 3 events:
  1768. */
  1769. if (version > 1)
  1770. x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
  1771. /*
  1772. * v2 and above have a perf capabilities MSR
  1773. */
  1774. if (version > 1) {
  1775. u64 capabilities;
  1776. rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
  1777. x86_pmu.intel_cap.capabilities = capabilities;
  1778. }
  1779. intel_ds_init();
  1780. x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
  1781. /*
  1782. * Install the hw-cache-events table:
  1783. */
  1784. switch (boot_cpu_data.x86_model) {
  1785. case 14: /* 65 nm core solo/duo, "Yonah" */
  1786. pr_cont("Core events, ");
  1787. break;
  1788. case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
  1789. x86_add_quirk(intel_clovertown_quirk);
  1790. case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
  1791. case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
  1792. case 29: /* six-core 45 nm xeon "Dunnington" */
  1793. memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
  1794. sizeof(hw_cache_event_ids));
  1795. intel_pmu_lbr_init_core();
  1796. x86_pmu.event_constraints = intel_core2_event_constraints;
  1797. x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
  1798. pr_cont("Core2 events, ");
  1799. break;
  1800. case 26: /* 45 nm nehalem, "Bloomfield" */
  1801. case 30: /* 45 nm nehalem, "Lynnfield" */
  1802. case 46: /* 45 nm nehalem-ex, "Beckton" */
  1803. memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
  1804. sizeof(hw_cache_event_ids));
  1805. memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
  1806. sizeof(hw_cache_extra_regs));
  1807. intel_pmu_lbr_init_nhm();
  1808. x86_pmu.event_constraints = intel_nehalem_event_constraints;
  1809. x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
  1810. x86_pmu.enable_all = intel_pmu_nhm_enable_all;
  1811. x86_pmu.extra_regs = intel_nehalem_extra_regs;
  1812. x86_pmu.cpu_events = nhm_events_attrs;
  1813. /* UOPS_ISSUED.STALLED_CYCLES */
  1814. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1815. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1816. /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
  1817. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  1818. X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
  1819. x86_add_quirk(intel_nehalem_quirk);
  1820. pr_cont("Nehalem events, ");
  1821. break;
  1822. case 28: /* Atom */
  1823. case 38: /* Lincroft */
  1824. case 39: /* Penwell */
  1825. case 53: /* Cloverview */
  1826. case 54: /* Cedarview */
  1827. memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
  1828. sizeof(hw_cache_event_ids));
  1829. intel_pmu_lbr_init_atom();
  1830. x86_pmu.event_constraints = intel_gen_event_constraints;
  1831. x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
  1832. pr_cont("Atom events, ");
  1833. break;
  1834. case 37: /* 32 nm nehalem, "Clarkdale" */
  1835. case 44: /* 32 nm nehalem, "Gulftown" */
  1836. case 47: /* 32 nm Xeon E7 */
  1837. memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
  1838. sizeof(hw_cache_event_ids));
  1839. memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
  1840. sizeof(hw_cache_extra_regs));
  1841. intel_pmu_lbr_init_nhm();
  1842. x86_pmu.event_constraints = intel_westmere_event_constraints;
  1843. x86_pmu.enable_all = intel_pmu_nhm_enable_all;
  1844. x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
  1845. x86_pmu.extra_regs = intel_westmere_extra_regs;
  1846. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  1847. x86_pmu.cpu_events = nhm_events_attrs;
  1848. /* UOPS_ISSUED.STALLED_CYCLES */
  1849. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1850. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1851. /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
  1852. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  1853. X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
  1854. pr_cont("Westmere events, ");
  1855. break;
  1856. case 42: /* SandyBridge */
  1857. case 45: /* SandyBridge, "Romely-EP" */
  1858. x86_add_quirk(intel_sandybridge_quirk);
  1859. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
  1860. sizeof(hw_cache_event_ids));
  1861. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
  1862. sizeof(hw_cache_extra_regs));
  1863. intel_pmu_lbr_init_snb();
  1864. x86_pmu.event_constraints = intel_snb_event_constraints;
  1865. x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
  1866. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  1867. x86_pmu.extra_regs = intel_snb_extra_regs;
  1868. /* all extra regs are per-cpu when HT is on */
  1869. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  1870. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  1871. x86_pmu.cpu_events = snb_events_attrs;
  1872. /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
  1873. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1874. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1875. /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
  1876. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  1877. X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1);
  1878. pr_cont("SandyBridge events, ");
  1879. break;
  1880. case 58: /* IvyBridge */
  1881. case 62: /* IvyBridge EP */
  1882. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
  1883. sizeof(hw_cache_event_ids));
  1884. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
  1885. sizeof(hw_cache_extra_regs));
  1886. intel_pmu_lbr_init_snb();
  1887. x86_pmu.event_constraints = intel_ivb_event_constraints;
  1888. x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
  1889. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  1890. x86_pmu.extra_regs = intel_snb_extra_regs;
  1891. /* all extra regs are per-cpu when HT is on */
  1892. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  1893. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  1894. x86_pmu.cpu_events = snb_events_attrs;
  1895. /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
  1896. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1897. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1898. pr_cont("IvyBridge events, ");
  1899. break;
  1900. default:
  1901. switch (x86_pmu.version) {
  1902. case 1:
  1903. x86_pmu.event_constraints = intel_v1_event_constraints;
  1904. pr_cont("generic architected perfmon v1, ");
  1905. break;
  1906. default:
  1907. /*
  1908. * default constraints for v2 and up
  1909. */
  1910. x86_pmu.event_constraints = intel_gen_event_constraints;
  1911. pr_cont("generic architected perfmon, ");
  1912. break;
  1913. }
  1914. }
  1915. if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) {
  1916. WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
  1917. x86_pmu.num_counters, INTEL_PMC_MAX_GENERIC);
  1918. x86_pmu.num_counters = INTEL_PMC_MAX_GENERIC;
  1919. }
  1920. x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
  1921. if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) {
  1922. WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
  1923. x86_pmu.num_counters_fixed, INTEL_PMC_MAX_FIXED);
  1924. x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED;
  1925. }
  1926. x86_pmu.intel_ctrl |=
  1927. ((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
  1928. if (x86_pmu.event_constraints) {
  1929. /*
  1930. * event on fixed counter2 (REF_CYCLES) only works on this
  1931. * counter, so do not extend mask to generic counters
  1932. */
  1933. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1934. if (c->cmask != X86_RAW_EVENT_MASK
  1935. || c->idxmsk64 == INTEL_PMC_MSK_FIXED_REF_CYCLES) {
  1936. continue;
  1937. }
  1938. c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
  1939. c->weight += x86_pmu.num_counters;
  1940. }
  1941. }
  1942. return 0;
  1943. }