perf_event_intel.c 61 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241
  1. /*
  2. * Per core/cpu state
  3. *
  4. * Used to coordinate shared registers between HT threads or
  5. * among events on a single PMU.
  6. */
  7. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  8. #include <linux/stddef.h>
  9. #include <linux/types.h>
  10. #include <linux/init.h>
  11. #include <linux/slab.h>
  12. #include <linux/export.h>
  13. #include <asm/hardirq.h>
  14. #include <asm/apic.h>
  15. #include "perf_event.h"
  16. /*
  17. * Intel PerfMon, used on Core and later.
  18. */
  19. static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
  20. {
  21. [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
  22. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  23. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
  24. [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
  25. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  26. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  27. [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
  28. [PERF_COUNT_HW_REF_CPU_CYCLES] = 0x0300, /* pseudo-encoding */
  29. };
  30. static struct event_constraint intel_core_event_constraints[] __read_mostly =
  31. {
  32. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  33. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  34. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  35. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  36. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  37. INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
  38. EVENT_CONSTRAINT_END
  39. };
  40. static struct event_constraint intel_core2_event_constraints[] __read_mostly =
  41. {
  42. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  43. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  44. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  45. INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  46. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  47. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  48. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  49. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  50. INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
  51. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  52. INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
  53. INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */
  54. INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
  55. EVENT_CONSTRAINT_END
  56. };
  57. static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
  58. {
  59. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  60. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  61. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  62. INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
  63. INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
  64. INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
  65. INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
  66. INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
  67. INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
  68. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  69. INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  70. EVENT_CONSTRAINT_END
  71. };
  72. static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
  73. {
  74. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  75. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  76. EVENT_EXTRA_END
  77. };
  78. static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
  79. {
  80. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  81. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  82. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  83. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  84. INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
  85. INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  86. INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */
  87. EVENT_CONSTRAINT_END
  88. };
  89. static struct event_constraint intel_snb_event_constraints[] __read_mostly =
  90. {
  91. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  92. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  93. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  94. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
  95. INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
  96. INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  97. INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  98. INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
  99. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  100. INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
  101. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
  102. INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  103. EVENT_CONSTRAINT_END
  104. };
  105. static struct event_constraint intel_ivb_event_constraints[] __read_mostly =
  106. {
  107. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  108. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  109. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  110. INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */
  111. INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */
  112. INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */
  113. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
  114. INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
  115. INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */
  116. INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  117. INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  118. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  119. /*
  120. * Errata BV98 -- MEM_*_RETIRED events can leak between counters of SMT
  121. * siblings; disable these events because they can corrupt unrelated
  122. * counters.
  123. */
  124. INTEL_EVENT_CONSTRAINT(0xd0, 0x0), /* MEM_UOPS_RETIRED.* */
  125. INTEL_EVENT_CONSTRAINT(0xd1, 0x0), /* MEM_LOAD_UOPS_RETIRED.* */
  126. INTEL_EVENT_CONSTRAINT(0xd2, 0x0), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
  127. INTEL_EVENT_CONSTRAINT(0xd3, 0x0), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
  128. EVENT_CONSTRAINT_END
  129. };
  130. static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
  131. {
  132. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  133. INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1),
  134. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  135. EVENT_EXTRA_END
  136. };
  137. static struct event_constraint intel_v1_event_constraints[] __read_mostly =
  138. {
  139. EVENT_CONSTRAINT_END
  140. };
  141. static struct event_constraint intel_gen_event_constraints[] __read_mostly =
  142. {
  143. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  144. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  145. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  146. EVENT_CONSTRAINT_END
  147. };
  148. static struct extra_reg intel_snb_extra_regs[] __read_mostly = {
  149. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0),
  150. INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1),
  151. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
  152. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
  153. EVENT_EXTRA_END
  154. };
  155. static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {
  156. INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
  157. INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
  158. EVENT_EXTRA_END
  159. };
  160. EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3");
  161. EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3");
  162. EVENT_ATTR_STR(mem-stores, mem_st_snb, "event=0xcd,umask=0x2");
  163. struct attribute *nhm_events_attrs[] = {
  164. EVENT_PTR(mem_ld_nhm),
  165. NULL,
  166. };
  167. struct attribute *snb_events_attrs[] = {
  168. EVENT_PTR(mem_ld_snb),
  169. EVENT_PTR(mem_st_snb),
  170. NULL,
  171. };
  172. static u64 intel_pmu_event_map(int hw_event)
  173. {
  174. return intel_perfmon_event_map[hw_event];
  175. }
  176. #define SNB_DMND_DATA_RD (1ULL << 0)
  177. #define SNB_DMND_RFO (1ULL << 1)
  178. #define SNB_DMND_IFETCH (1ULL << 2)
  179. #define SNB_DMND_WB (1ULL << 3)
  180. #define SNB_PF_DATA_RD (1ULL << 4)
  181. #define SNB_PF_RFO (1ULL << 5)
  182. #define SNB_PF_IFETCH (1ULL << 6)
  183. #define SNB_LLC_DATA_RD (1ULL << 7)
  184. #define SNB_LLC_RFO (1ULL << 8)
  185. #define SNB_LLC_IFETCH (1ULL << 9)
  186. #define SNB_BUS_LOCKS (1ULL << 10)
  187. #define SNB_STRM_ST (1ULL << 11)
  188. #define SNB_OTHER (1ULL << 15)
  189. #define SNB_RESP_ANY (1ULL << 16)
  190. #define SNB_NO_SUPP (1ULL << 17)
  191. #define SNB_LLC_HITM (1ULL << 18)
  192. #define SNB_LLC_HITE (1ULL << 19)
  193. #define SNB_LLC_HITS (1ULL << 20)
  194. #define SNB_LLC_HITF (1ULL << 21)
  195. #define SNB_LOCAL (1ULL << 22)
  196. #define SNB_REMOTE (0xffULL << 23)
  197. #define SNB_SNP_NONE (1ULL << 31)
  198. #define SNB_SNP_NOT_NEEDED (1ULL << 32)
  199. #define SNB_SNP_MISS (1ULL << 33)
  200. #define SNB_NO_FWD (1ULL << 34)
  201. #define SNB_SNP_FWD (1ULL << 35)
  202. #define SNB_HITM (1ULL << 36)
  203. #define SNB_NON_DRAM (1ULL << 37)
  204. #define SNB_DMND_READ (SNB_DMND_DATA_RD|SNB_LLC_DATA_RD)
  205. #define SNB_DMND_WRITE (SNB_DMND_RFO|SNB_LLC_RFO)
  206. #define SNB_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)
  207. #define SNB_SNP_ANY (SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \
  208. SNB_SNP_MISS|SNB_NO_FWD|SNB_SNP_FWD| \
  209. SNB_HITM)
  210. #define SNB_DRAM_ANY (SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY)
  211. #define SNB_DRAM_REMOTE (SNB_REMOTE|SNB_SNP_ANY)
  212. #define SNB_L3_ACCESS SNB_RESP_ANY
  213. #define SNB_L3_MISS (SNB_DRAM_ANY|SNB_NON_DRAM)
  214. static __initconst const u64 snb_hw_cache_extra_regs
  215. [PERF_COUNT_HW_CACHE_MAX]
  216. [PERF_COUNT_HW_CACHE_OP_MAX]
  217. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  218. {
  219. [ C(LL ) ] = {
  220. [ C(OP_READ) ] = {
  221. [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_L3_ACCESS,
  222. [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_L3_MISS,
  223. },
  224. [ C(OP_WRITE) ] = {
  225. [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_L3_ACCESS,
  226. [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_L3_MISS,
  227. },
  228. [ C(OP_PREFETCH) ] = {
  229. [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_L3_ACCESS,
  230. [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_L3_MISS,
  231. },
  232. },
  233. [ C(NODE) ] = {
  234. [ C(OP_READ) ] = {
  235. [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_DRAM_ANY,
  236. [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_DRAM_REMOTE,
  237. },
  238. [ C(OP_WRITE) ] = {
  239. [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_DRAM_ANY,
  240. [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_DRAM_REMOTE,
  241. },
  242. [ C(OP_PREFETCH) ] = {
  243. [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_DRAM_ANY,
  244. [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_DRAM_REMOTE,
  245. },
  246. },
  247. };
  248. static __initconst const u64 snb_hw_cache_event_ids
  249. [PERF_COUNT_HW_CACHE_MAX]
  250. [PERF_COUNT_HW_CACHE_OP_MAX]
  251. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  252. {
  253. [ C(L1D) ] = {
  254. [ C(OP_READ) ] = {
  255. [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS */
  256. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPLACEMENT */
  257. },
  258. [ C(OP_WRITE) ] = {
  259. [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES */
  260. [ C(RESULT_MISS) ] = 0x0851, /* L1D.ALL_M_REPLACEMENT */
  261. },
  262. [ C(OP_PREFETCH) ] = {
  263. [ C(RESULT_ACCESS) ] = 0x0,
  264. [ C(RESULT_MISS) ] = 0x024e, /* HW_PRE_REQ.DL1_MISS */
  265. },
  266. },
  267. [ C(L1I ) ] = {
  268. [ C(OP_READ) ] = {
  269. [ C(RESULT_ACCESS) ] = 0x0,
  270. [ C(RESULT_MISS) ] = 0x0280, /* ICACHE.MISSES */
  271. },
  272. [ C(OP_WRITE) ] = {
  273. [ C(RESULT_ACCESS) ] = -1,
  274. [ C(RESULT_MISS) ] = -1,
  275. },
  276. [ C(OP_PREFETCH) ] = {
  277. [ C(RESULT_ACCESS) ] = 0x0,
  278. [ C(RESULT_MISS) ] = 0x0,
  279. },
  280. },
  281. [ C(LL ) ] = {
  282. [ C(OP_READ) ] = {
  283. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  284. [ C(RESULT_ACCESS) ] = 0x01b7,
  285. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  286. [ C(RESULT_MISS) ] = 0x01b7,
  287. },
  288. [ C(OP_WRITE) ] = {
  289. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  290. [ C(RESULT_ACCESS) ] = 0x01b7,
  291. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  292. [ C(RESULT_MISS) ] = 0x01b7,
  293. },
  294. [ C(OP_PREFETCH) ] = {
  295. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  296. [ C(RESULT_ACCESS) ] = 0x01b7,
  297. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  298. [ C(RESULT_MISS) ] = 0x01b7,
  299. },
  300. },
  301. [ C(DTLB) ] = {
  302. [ C(OP_READ) ] = {
  303. [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
  304. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
  305. },
  306. [ C(OP_WRITE) ] = {
  307. [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
  308. [ C(RESULT_MISS) ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
  309. },
  310. [ C(OP_PREFETCH) ] = {
  311. [ C(RESULT_ACCESS) ] = 0x0,
  312. [ C(RESULT_MISS) ] = 0x0,
  313. },
  314. },
  315. [ C(ITLB) ] = {
  316. [ C(OP_READ) ] = {
  317. [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT */
  318. [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK */
  319. },
  320. [ C(OP_WRITE) ] = {
  321. [ C(RESULT_ACCESS) ] = -1,
  322. [ C(RESULT_MISS) ] = -1,
  323. },
  324. [ C(OP_PREFETCH) ] = {
  325. [ C(RESULT_ACCESS) ] = -1,
  326. [ C(RESULT_MISS) ] = -1,
  327. },
  328. },
  329. [ C(BPU ) ] = {
  330. [ C(OP_READ) ] = {
  331. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  332. [ C(RESULT_MISS) ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
  333. },
  334. [ C(OP_WRITE) ] = {
  335. [ C(RESULT_ACCESS) ] = -1,
  336. [ C(RESULT_MISS) ] = -1,
  337. },
  338. [ C(OP_PREFETCH) ] = {
  339. [ C(RESULT_ACCESS) ] = -1,
  340. [ C(RESULT_MISS) ] = -1,
  341. },
  342. },
  343. [ C(NODE) ] = {
  344. [ C(OP_READ) ] = {
  345. [ C(RESULT_ACCESS) ] = 0x01b7,
  346. [ C(RESULT_MISS) ] = 0x01b7,
  347. },
  348. [ C(OP_WRITE) ] = {
  349. [ C(RESULT_ACCESS) ] = 0x01b7,
  350. [ C(RESULT_MISS) ] = 0x01b7,
  351. },
  352. [ C(OP_PREFETCH) ] = {
  353. [ C(RESULT_ACCESS) ] = 0x01b7,
  354. [ C(RESULT_MISS) ] = 0x01b7,
  355. },
  356. },
  357. };
  358. static __initconst const u64 westmere_hw_cache_event_ids
  359. [PERF_COUNT_HW_CACHE_MAX]
  360. [PERF_COUNT_HW_CACHE_OP_MAX]
  361. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  362. {
  363. [ C(L1D) ] = {
  364. [ C(OP_READ) ] = {
  365. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  366. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */
  367. },
  368. [ C(OP_WRITE) ] = {
  369. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  370. [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */
  371. },
  372. [ C(OP_PREFETCH) ] = {
  373. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  374. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  375. },
  376. },
  377. [ C(L1I ) ] = {
  378. [ C(OP_READ) ] = {
  379. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  380. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  381. },
  382. [ C(OP_WRITE) ] = {
  383. [ C(RESULT_ACCESS) ] = -1,
  384. [ C(RESULT_MISS) ] = -1,
  385. },
  386. [ C(OP_PREFETCH) ] = {
  387. [ C(RESULT_ACCESS) ] = 0x0,
  388. [ C(RESULT_MISS) ] = 0x0,
  389. },
  390. },
  391. [ C(LL ) ] = {
  392. [ C(OP_READ) ] = {
  393. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  394. [ C(RESULT_ACCESS) ] = 0x01b7,
  395. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  396. [ C(RESULT_MISS) ] = 0x01b7,
  397. },
  398. /*
  399. * Use RFO, not WRITEBACK, because a write miss would typically occur
  400. * on RFO.
  401. */
  402. [ C(OP_WRITE) ] = {
  403. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  404. [ C(RESULT_ACCESS) ] = 0x01b7,
  405. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  406. [ C(RESULT_MISS) ] = 0x01b7,
  407. },
  408. [ C(OP_PREFETCH) ] = {
  409. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  410. [ C(RESULT_ACCESS) ] = 0x01b7,
  411. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  412. [ C(RESULT_MISS) ] = 0x01b7,
  413. },
  414. },
  415. [ C(DTLB) ] = {
  416. [ C(OP_READ) ] = {
  417. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  418. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  419. },
  420. [ C(OP_WRITE) ] = {
  421. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  422. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  423. },
  424. [ C(OP_PREFETCH) ] = {
  425. [ C(RESULT_ACCESS) ] = 0x0,
  426. [ C(RESULT_MISS) ] = 0x0,
  427. },
  428. },
  429. [ C(ITLB) ] = {
  430. [ C(OP_READ) ] = {
  431. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  432. [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.ANY */
  433. },
  434. [ C(OP_WRITE) ] = {
  435. [ C(RESULT_ACCESS) ] = -1,
  436. [ C(RESULT_MISS) ] = -1,
  437. },
  438. [ C(OP_PREFETCH) ] = {
  439. [ C(RESULT_ACCESS) ] = -1,
  440. [ C(RESULT_MISS) ] = -1,
  441. },
  442. },
  443. [ C(BPU ) ] = {
  444. [ C(OP_READ) ] = {
  445. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  446. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  447. },
  448. [ C(OP_WRITE) ] = {
  449. [ C(RESULT_ACCESS) ] = -1,
  450. [ C(RESULT_MISS) ] = -1,
  451. },
  452. [ C(OP_PREFETCH) ] = {
  453. [ C(RESULT_ACCESS) ] = -1,
  454. [ C(RESULT_MISS) ] = -1,
  455. },
  456. },
  457. [ C(NODE) ] = {
  458. [ C(OP_READ) ] = {
  459. [ C(RESULT_ACCESS) ] = 0x01b7,
  460. [ C(RESULT_MISS) ] = 0x01b7,
  461. },
  462. [ C(OP_WRITE) ] = {
  463. [ C(RESULT_ACCESS) ] = 0x01b7,
  464. [ C(RESULT_MISS) ] = 0x01b7,
  465. },
  466. [ C(OP_PREFETCH) ] = {
  467. [ C(RESULT_ACCESS) ] = 0x01b7,
  468. [ C(RESULT_MISS) ] = 0x01b7,
  469. },
  470. },
  471. };
  472. /*
  473. * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits;
  474. * See IA32 SDM Vol 3B 30.6.1.3
  475. */
  476. #define NHM_DMND_DATA_RD (1 << 0)
  477. #define NHM_DMND_RFO (1 << 1)
  478. #define NHM_DMND_IFETCH (1 << 2)
  479. #define NHM_DMND_WB (1 << 3)
  480. #define NHM_PF_DATA_RD (1 << 4)
  481. #define NHM_PF_DATA_RFO (1 << 5)
  482. #define NHM_PF_IFETCH (1 << 6)
  483. #define NHM_OFFCORE_OTHER (1 << 7)
  484. #define NHM_UNCORE_HIT (1 << 8)
  485. #define NHM_OTHER_CORE_HIT_SNP (1 << 9)
  486. #define NHM_OTHER_CORE_HITM (1 << 10)
  487. /* reserved */
  488. #define NHM_REMOTE_CACHE_FWD (1 << 12)
  489. #define NHM_REMOTE_DRAM (1 << 13)
  490. #define NHM_LOCAL_DRAM (1 << 14)
  491. #define NHM_NON_DRAM (1 << 15)
  492. #define NHM_LOCAL (NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD)
  493. #define NHM_REMOTE (NHM_REMOTE_DRAM)
  494. #define NHM_DMND_READ (NHM_DMND_DATA_RD)
  495. #define NHM_DMND_WRITE (NHM_DMND_RFO|NHM_DMND_WB)
  496. #define NHM_DMND_PREFETCH (NHM_PF_DATA_RD|NHM_PF_DATA_RFO)
  497. #define NHM_L3_HIT (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)
  498. #define NHM_L3_MISS (NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD)
  499. #define NHM_L3_ACCESS (NHM_L3_HIT|NHM_L3_MISS)
  500. static __initconst const u64 nehalem_hw_cache_extra_regs
  501. [PERF_COUNT_HW_CACHE_MAX]
  502. [PERF_COUNT_HW_CACHE_OP_MAX]
  503. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  504. {
  505. [ C(LL ) ] = {
  506. [ C(OP_READ) ] = {
  507. [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS,
  508. [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_L3_MISS,
  509. },
  510. [ C(OP_WRITE) ] = {
  511. [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS,
  512. [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_L3_MISS,
  513. },
  514. [ C(OP_PREFETCH) ] = {
  515. [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS,
  516. [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_L3_MISS,
  517. },
  518. },
  519. [ C(NODE) ] = {
  520. [ C(OP_READ) ] = {
  521. [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_LOCAL|NHM_REMOTE,
  522. [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_REMOTE,
  523. },
  524. [ C(OP_WRITE) ] = {
  525. [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_LOCAL|NHM_REMOTE,
  526. [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_REMOTE,
  527. },
  528. [ C(OP_PREFETCH) ] = {
  529. [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_LOCAL|NHM_REMOTE,
  530. [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_REMOTE,
  531. },
  532. },
  533. };
  534. static __initconst const u64 nehalem_hw_cache_event_ids
  535. [PERF_COUNT_HW_CACHE_MAX]
  536. [PERF_COUNT_HW_CACHE_OP_MAX]
  537. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  538. {
  539. [ C(L1D) ] = {
  540. [ C(OP_READ) ] = {
  541. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  542. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */
  543. },
  544. [ C(OP_WRITE) ] = {
  545. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  546. [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */
  547. },
  548. [ C(OP_PREFETCH) ] = {
  549. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  550. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  551. },
  552. },
  553. [ C(L1I ) ] = {
  554. [ C(OP_READ) ] = {
  555. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  556. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  557. },
  558. [ C(OP_WRITE) ] = {
  559. [ C(RESULT_ACCESS) ] = -1,
  560. [ C(RESULT_MISS) ] = -1,
  561. },
  562. [ C(OP_PREFETCH) ] = {
  563. [ C(RESULT_ACCESS) ] = 0x0,
  564. [ C(RESULT_MISS) ] = 0x0,
  565. },
  566. },
  567. [ C(LL ) ] = {
  568. [ C(OP_READ) ] = {
  569. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  570. [ C(RESULT_ACCESS) ] = 0x01b7,
  571. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  572. [ C(RESULT_MISS) ] = 0x01b7,
  573. },
  574. /*
  575. * Use RFO, not WRITEBACK, because a write miss would typically occur
  576. * on RFO.
  577. */
  578. [ C(OP_WRITE) ] = {
  579. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  580. [ C(RESULT_ACCESS) ] = 0x01b7,
  581. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  582. [ C(RESULT_MISS) ] = 0x01b7,
  583. },
  584. [ C(OP_PREFETCH) ] = {
  585. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  586. [ C(RESULT_ACCESS) ] = 0x01b7,
  587. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  588. [ C(RESULT_MISS) ] = 0x01b7,
  589. },
  590. },
  591. [ C(DTLB) ] = {
  592. [ C(OP_READ) ] = {
  593. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  594. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  595. },
  596. [ C(OP_WRITE) ] = {
  597. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  598. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  599. },
  600. [ C(OP_PREFETCH) ] = {
  601. [ C(RESULT_ACCESS) ] = 0x0,
  602. [ C(RESULT_MISS) ] = 0x0,
  603. },
  604. },
  605. [ C(ITLB) ] = {
  606. [ C(OP_READ) ] = {
  607. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  608. [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
  609. },
  610. [ C(OP_WRITE) ] = {
  611. [ C(RESULT_ACCESS) ] = -1,
  612. [ C(RESULT_MISS) ] = -1,
  613. },
  614. [ C(OP_PREFETCH) ] = {
  615. [ C(RESULT_ACCESS) ] = -1,
  616. [ C(RESULT_MISS) ] = -1,
  617. },
  618. },
  619. [ C(BPU ) ] = {
  620. [ C(OP_READ) ] = {
  621. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  622. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  623. },
  624. [ C(OP_WRITE) ] = {
  625. [ C(RESULT_ACCESS) ] = -1,
  626. [ C(RESULT_MISS) ] = -1,
  627. },
  628. [ C(OP_PREFETCH) ] = {
  629. [ C(RESULT_ACCESS) ] = -1,
  630. [ C(RESULT_MISS) ] = -1,
  631. },
  632. },
  633. [ C(NODE) ] = {
  634. [ C(OP_READ) ] = {
  635. [ C(RESULT_ACCESS) ] = 0x01b7,
  636. [ C(RESULT_MISS) ] = 0x01b7,
  637. },
  638. [ C(OP_WRITE) ] = {
  639. [ C(RESULT_ACCESS) ] = 0x01b7,
  640. [ C(RESULT_MISS) ] = 0x01b7,
  641. },
  642. [ C(OP_PREFETCH) ] = {
  643. [ C(RESULT_ACCESS) ] = 0x01b7,
  644. [ C(RESULT_MISS) ] = 0x01b7,
  645. },
  646. },
  647. };
  648. static __initconst const u64 core2_hw_cache_event_ids
  649. [PERF_COUNT_HW_CACHE_MAX]
  650. [PERF_COUNT_HW_CACHE_OP_MAX]
  651. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  652. {
  653. [ C(L1D) ] = {
  654. [ C(OP_READ) ] = {
  655. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  656. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  657. },
  658. [ C(OP_WRITE) ] = {
  659. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  660. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  661. },
  662. [ C(OP_PREFETCH) ] = {
  663. [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
  664. [ C(RESULT_MISS) ] = 0,
  665. },
  666. },
  667. [ C(L1I ) ] = {
  668. [ C(OP_READ) ] = {
  669. [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
  670. [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
  671. },
  672. [ C(OP_WRITE) ] = {
  673. [ C(RESULT_ACCESS) ] = -1,
  674. [ C(RESULT_MISS) ] = -1,
  675. },
  676. [ C(OP_PREFETCH) ] = {
  677. [ C(RESULT_ACCESS) ] = 0,
  678. [ C(RESULT_MISS) ] = 0,
  679. },
  680. },
  681. [ C(LL ) ] = {
  682. [ C(OP_READ) ] = {
  683. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  684. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  685. },
  686. [ C(OP_WRITE) ] = {
  687. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  688. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  689. },
  690. [ C(OP_PREFETCH) ] = {
  691. [ C(RESULT_ACCESS) ] = 0,
  692. [ C(RESULT_MISS) ] = 0,
  693. },
  694. },
  695. [ C(DTLB) ] = {
  696. [ C(OP_READ) ] = {
  697. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  698. [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
  699. },
  700. [ C(OP_WRITE) ] = {
  701. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  702. [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
  703. },
  704. [ C(OP_PREFETCH) ] = {
  705. [ C(RESULT_ACCESS) ] = 0,
  706. [ C(RESULT_MISS) ] = 0,
  707. },
  708. },
  709. [ C(ITLB) ] = {
  710. [ C(OP_READ) ] = {
  711. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  712. [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
  713. },
  714. [ C(OP_WRITE) ] = {
  715. [ C(RESULT_ACCESS) ] = -1,
  716. [ C(RESULT_MISS) ] = -1,
  717. },
  718. [ C(OP_PREFETCH) ] = {
  719. [ C(RESULT_ACCESS) ] = -1,
  720. [ C(RESULT_MISS) ] = -1,
  721. },
  722. },
  723. [ C(BPU ) ] = {
  724. [ C(OP_READ) ] = {
  725. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  726. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  727. },
  728. [ C(OP_WRITE) ] = {
  729. [ C(RESULT_ACCESS) ] = -1,
  730. [ C(RESULT_MISS) ] = -1,
  731. },
  732. [ C(OP_PREFETCH) ] = {
  733. [ C(RESULT_ACCESS) ] = -1,
  734. [ C(RESULT_MISS) ] = -1,
  735. },
  736. },
  737. };
  738. static __initconst const u64 atom_hw_cache_event_ids
  739. [PERF_COUNT_HW_CACHE_MAX]
  740. [PERF_COUNT_HW_CACHE_OP_MAX]
  741. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  742. {
  743. [ C(L1D) ] = {
  744. [ C(OP_READ) ] = {
  745. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
  746. [ C(RESULT_MISS) ] = 0,
  747. },
  748. [ C(OP_WRITE) ] = {
  749. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
  750. [ C(RESULT_MISS) ] = 0,
  751. },
  752. [ C(OP_PREFETCH) ] = {
  753. [ C(RESULT_ACCESS) ] = 0x0,
  754. [ C(RESULT_MISS) ] = 0,
  755. },
  756. },
  757. [ C(L1I ) ] = {
  758. [ C(OP_READ) ] = {
  759. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  760. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  761. },
  762. [ C(OP_WRITE) ] = {
  763. [ C(RESULT_ACCESS) ] = -1,
  764. [ C(RESULT_MISS) ] = -1,
  765. },
  766. [ C(OP_PREFETCH) ] = {
  767. [ C(RESULT_ACCESS) ] = 0,
  768. [ C(RESULT_MISS) ] = 0,
  769. },
  770. },
  771. [ C(LL ) ] = {
  772. [ C(OP_READ) ] = {
  773. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  774. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  775. },
  776. [ C(OP_WRITE) ] = {
  777. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  778. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  779. },
  780. [ C(OP_PREFETCH) ] = {
  781. [ C(RESULT_ACCESS) ] = 0,
  782. [ C(RESULT_MISS) ] = 0,
  783. },
  784. },
  785. [ C(DTLB) ] = {
  786. [ C(OP_READ) ] = {
  787. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
  788. [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
  789. },
  790. [ C(OP_WRITE) ] = {
  791. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
  792. [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
  793. },
  794. [ C(OP_PREFETCH) ] = {
  795. [ C(RESULT_ACCESS) ] = 0,
  796. [ C(RESULT_MISS) ] = 0,
  797. },
  798. },
  799. [ C(ITLB) ] = {
  800. [ C(OP_READ) ] = {
  801. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  802. [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
  803. },
  804. [ C(OP_WRITE) ] = {
  805. [ C(RESULT_ACCESS) ] = -1,
  806. [ C(RESULT_MISS) ] = -1,
  807. },
  808. [ C(OP_PREFETCH) ] = {
  809. [ C(RESULT_ACCESS) ] = -1,
  810. [ C(RESULT_MISS) ] = -1,
  811. },
  812. },
  813. [ C(BPU ) ] = {
  814. [ C(OP_READ) ] = {
  815. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  816. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  817. },
  818. [ C(OP_WRITE) ] = {
  819. [ C(RESULT_ACCESS) ] = -1,
  820. [ C(RESULT_MISS) ] = -1,
  821. },
  822. [ C(OP_PREFETCH) ] = {
  823. [ C(RESULT_ACCESS) ] = -1,
  824. [ C(RESULT_MISS) ] = -1,
  825. },
  826. },
  827. };
  828. static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
  829. {
  830. /* user explicitly requested branch sampling */
  831. if (has_branch_stack(event))
  832. return true;
  833. /* implicit branch sampling to correct PEBS skid */
  834. if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
  835. return true;
  836. return false;
  837. }
  838. static void intel_pmu_disable_all(void)
  839. {
  840. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  841. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
  842. if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
  843. intel_pmu_disable_bts();
  844. intel_pmu_pebs_disable_all();
  845. intel_pmu_lbr_disable_all();
  846. }
  847. static void intel_pmu_enable_all(int added)
  848. {
  849. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  850. intel_pmu_pebs_enable_all();
  851. intel_pmu_lbr_enable_all();
  852. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
  853. x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
  854. if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
  855. struct perf_event *event =
  856. cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
  857. if (WARN_ON_ONCE(!event))
  858. return;
  859. intel_pmu_enable_bts(event->hw.config);
  860. }
  861. }
  862. /*
  863. * Workaround for:
  864. * Intel Errata AAK100 (model 26)
  865. * Intel Errata AAP53 (model 30)
  866. * Intel Errata BD53 (model 44)
  867. *
  868. * The official story:
  869. * These chips need to be 'reset' when adding counters by programming the
  870. * magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either
  871. * in sequence on the same PMC or on different PMCs.
  872. *
  873. * In practise it appears some of these events do in fact count, and
  874. * we need to programm all 4 events.
  875. */
  876. static void intel_pmu_nhm_workaround(void)
  877. {
  878. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  879. static const unsigned long nhm_magic[4] = {
  880. 0x4300B5,
  881. 0x4300D2,
  882. 0x4300B1,
  883. 0x4300B1
  884. };
  885. struct perf_event *event;
  886. int i;
  887. /*
  888. * The Errata requires below steps:
  889. * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL;
  890. * 2) Configure 4 PERFEVTSELx with the magic events and clear
  891. * the corresponding PMCx;
  892. * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL;
  893. * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL;
  894. * 5) Clear 4 pairs of ERFEVTSELx and PMCx;
  895. */
  896. /*
  897. * The real steps we choose are a little different from above.
  898. * A) To reduce MSR operations, we don't run step 1) as they
  899. * are already cleared before this function is called;
  900. * B) Call x86_perf_event_update to save PMCx before configuring
  901. * PERFEVTSELx with magic number;
  902. * C) With step 5), we do clear only when the PERFEVTSELx is
  903. * not used currently.
  904. * D) Call x86_perf_event_set_period to restore PMCx;
  905. */
  906. /* We always operate 4 pairs of PERF Counters */
  907. for (i = 0; i < 4; i++) {
  908. event = cpuc->events[i];
  909. if (event)
  910. x86_perf_event_update(event);
  911. }
  912. for (i = 0; i < 4; i++) {
  913. wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
  914. wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
  915. }
  916. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
  917. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
  918. for (i = 0; i < 4; i++) {
  919. event = cpuc->events[i];
  920. if (event) {
  921. x86_perf_event_set_period(event);
  922. __x86_pmu_enable_event(&event->hw,
  923. ARCH_PERFMON_EVENTSEL_ENABLE);
  924. } else
  925. wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
  926. }
  927. }
  928. static void intel_pmu_nhm_enable_all(int added)
  929. {
  930. if (added)
  931. intel_pmu_nhm_workaround();
  932. intel_pmu_enable_all(added);
  933. }
  934. static inline u64 intel_pmu_get_status(void)
  935. {
  936. u64 status;
  937. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  938. return status;
  939. }
  940. static inline void intel_pmu_ack_status(u64 ack)
  941. {
  942. wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
  943. }
  944. static void intel_pmu_disable_fixed(struct hw_perf_event *hwc)
  945. {
  946. int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
  947. u64 ctrl_val, mask;
  948. mask = 0xfULL << (idx * 4);
  949. rdmsrl(hwc->config_base, ctrl_val);
  950. ctrl_val &= ~mask;
  951. wrmsrl(hwc->config_base, ctrl_val);
  952. }
  953. static void intel_pmu_disable_event(struct perf_event *event)
  954. {
  955. struct hw_perf_event *hwc = &event->hw;
  956. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  957. if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
  958. intel_pmu_disable_bts();
  959. intel_pmu_drain_bts_buffer();
  960. return;
  961. }
  962. cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
  963. cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
  964. /*
  965. * must disable before any actual event
  966. * because any event may be combined with LBR
  967. */
  968. if (intel_pmu_needs_lbr_smpl(event))
  969. intel_pmu_lbr_disable(event);
  970. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  971. intel_pmu_disable_fixed(hwc);
  972. return;
  973. }
  974. x86_pmu_disable_event(event);
  975. if (unlikely(event->attr.precise_ip))
  976. intel_pmu_pebs_disable(event);
  977. }
  978. static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
  979. {
  980. int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
  981. u64 ctrl_val, bits, mask;
  982. /*
  983. * Enable IRQ generation (0x8),
  984. * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
  985. * if requested:
  986. */
  987. bits = 0x8ULL;
  988. if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
  989. bits |= 0x2;
  990. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  991. bits |= 0x1;
  992. /*
  993. * ANY bit is supported in v3 and up
  994. */
  995. if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
  996. bits |= 0x4;
  997. bits <<= (idx * 4);
  998. mask = 0xfULL << (idx * 4);
  999. rdmsrl(hwc->config_base, ctrl_val);
  1000. ctrl_val &= ~mask;
  1001. ctrl_val |= bits;
  1002. wrmsrl(hwc->config_base, ctrl_val);
  1003. }
  1004. static void intel_pmu_enable_event(struct perf_event *event)
  1005. {
  1006. struct hw_perf_event *hwc = &event->hw;
  1007. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1008. if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
  1009. if (!__this_cpu_read(cpu_hw_events.enabled))
  1010. return;
  1011. intel_pmu_enable_bts(hwc->config);
  1012. return;
  1013. }
  1014. /*
  1015. * must enabled before any actual event
  1016. * because any event may be combined with LBR
  1017. */
  1018. if (intel_pmu_needs_lbr_smpl(event))
  1019. intel_pmu_lbr_enable(event);
  1020. if (event->attr.exclude_host)
  1021. cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
  1022. if (event->attr.exclude_guest)
  1023. cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);
  1024. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1025. intel_pmu_enable_fixed(hwc);
  1026. return;
  1027. }
  1028. if (unlikely(event->attr.precise_ip))
  1029. intel_pmu_pebs_enable(event);
  1030. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  1031. }
  1032. /*
  1033. * Save and restart an expired event. Called by NMI contexts,
  1034. * so it has to be careful about preempting normal event ops:
  1035. */
  1036. int intel_pmu_save_and_restart(struct perf_event *event)
  1037. {
  1038. x86_perf_event_update(event);
  1039. return x86_perf_event_set_period(event);
  1040. }
  1041. static void intel_pmu_reset(void)
  1042. {
  1043. struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
  1044. unsigned long flags;
  1045. int idx;
  1046. if (!x86_pmu.num_counters)
  1047. return;
  1048. local_irq_save(flags);
  1049. pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());
  1050. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1051. wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
  1052. wrmsrl_safe(x86_pmu_event_addr(idx), 0ull);
  1053. }
  1054. for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
  1055. wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
  1056. if (ds)
  1057. ds->bts_index = ds->bts_buffer_base;
  1058. local_irq_restore(flags);
  1059. }
  1060. /*
  1061. * This handler is triggered by the local APIC, so the APIC IRQ handling
  1062. * rules apply:
  1063. */
  1064. static int intel_pmu_handle_irq(struct pt_regs *regs)
  1065. {
  1066. struct perf_sample_data data;
  1067. struct cpu_hw_events *cpuc;
  1068. int bit, loops;
  1069. u64 status;
  1070. int handled;
  1071. cpuc = &__get_cpu_var(cpu_hw_events);
  1072. /*
  1073. * Some chipsets need to unmask the LVTPC in a particular spot
  1074. * inside the nmi handler. As a result, the unmasking was pushed
  1075. * into all the nmi handlers.
  1076. *
  1077. * This handler doesn't seem to have any issues with the unmasking
  1078. * so it was left at the top.
  1079. */
  1080. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1081. intel_pmu_disable_all();
  1082. handled = intel_pmu_drain_bts_buffer();
  1083. status = intel_pmu_get_status();
  1084. if (!status) {
  1085. intel_pmu_enable_all(0);
  1086. return handled;
  1087. }
  1088. loops = 0;
  1089. again:
  1090. intel_pmu_ack_status(status);
  1091. if (++loops > 100) {
  1092. WARN_ONCE(1, "perfevents: irq loop stuck!\n");
  1093. perf_event_print_debug();
  1094. intel_pmu_reset();
  1095. goto done;
  1096. }
  1097. inc_irq_stat(apic_perf_irqs);
  1098. intel_pmu_lbr_read();
  1099. /*
  1100. * PEBS overflow sets bit 62 in the global status register
  1101. */
  1102. if (__test_and_clear_bit(62, (unsigned long *)&status)) {
  1103. handled++;
  1104. x86_pmu.drain_pebs(regs);
  1105. }
  1106. for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
  1107. struct perf_event *event = cpuc->events[bit];
  1108. handled++;
  1109. if (!test_bit(bit, cpuc->active_mask))
  1110. continue;
  1111. if (!intel_pmu_save_and_restart(event))
  1112. continue;
  1113. perf_sample_data_init(&data, 0, event->hw.last_period);
  1114. if (has_branch_stack(event))
  1115. data.br_stack = &cpuc->lbr_stack;
  1116. if (perf_event_overflow(event, &data, regs))
  1117. x86_pmu_stop(event, 0);
  1118. }
  1119. /*
  1120. * Repeat if there is more work to be done:
  1121. */
  1122. status = intel_pmu_get_status();
  1123. if (status)
  1124. goto again;
  1125. done:
  1126. intel_pmu_enable_all(0);
  1127. return handled;
  1128. }
  1129. static struct event_constraint *
  1130. intel_bts_constraints(struct perf_event *event)
  1131. {
  1132. struct hw_perf_event *hwc = &event->hw;
  1133. unsigned int hw_event, bts_event;
  1134. if (event->attr.freq)
  1135. return NULL;
  1136. hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
  1137. bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
  1138. if (unlikely(hw_event == bts_event && hwc->sample_period == 1))
  1139. return &bts_constraint;
  1140. return NULL;
  1141. }
  1142. static int intel_alt_er(int idx)
  1143. {
  1144. if (!(x86_pmu.er_flags & ERF_HAS_RSP_1))
  1145. return idx;
  1146. if (idx == EXTRA_REG_RSP_0)
  1147. return EXTRA_REG_RSP_1;
  1148. if (idx == EXTRA_REG_RSP_1)
  1149. return EXTRA_REG_RSP_0;
  1150. return idx;
  1151. }
  1152. static void intel_fixup_er(struct perf_event *event, int idx)
  1153. {
  1154. event->hw.extra_reg.idx = idx;
  1155. if (idx == EXTRA_REG_RSP_0) {
  1156. event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
  1157. event->hw.config |= 0x01b7;
  1158. event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0;
  1159. } else if (idx == EXTRA_REG_RSP_1) {
  1160. event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
  1161. event->hw.config |= 0x01bb;
  1162. event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1;
  1163. }
  1164. }
  1165. /*
  1166. * manage allocation of shared extra msr for certain events
  1167. *
  1168. * sharing can be:
  1169. * per-cpu: to be shared between the various events on a single PMU
  1170. * per-core: per-cpu + shared by HT threads
  1171. */
  1172. static struct event_constraint *
  1173. __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
  1174. struct perf_event *event,
  1175. struct hw_perf_event_extra *reg)
  1176. {
  1177. struct event_constraint *c = &emptyconstraint;
  1178. struct er_account *era;
  1179. unsigned long flags;
  1180. int idx = reg->idx;
  1181. /*
  1182. * reg->alloc can be set due to existing state, so for fake cpuc we
  1183. * need to ignore this, otherwise we might fail to allocate proper fake
  1184. * state for this extra reg constraint. Also see the comment below.
  1185. */
  1186. if (reg->alloc && !cpuc->is_fake)
  1187. return NULL; /* call x86_get_event_constraint() */
  1188. again:
  1189. era = &cpuc->shared_regs->regs[idx];
  1190. /*
  1191. * we use spin_lock_irqsave() to avoid lockdep issues when
  1192. * passing a fake cpuc
  1193. */
  1194. raw_spin_lock_irqsave(&era->lock, flags);
  1195. if (!atomic_read(&era->ref) || era->config == reg->config) {
  1196. /*
  1197. * If its a fake cpuc -- as per validate_{group,event}() we
  1198. * shouldn't touch event state and we can avoid doing so
  1199. * since both will only call get_event_constraints() once
  1200. * on each event, this avoids the need for reg->alloc.
  1201. *
  1202. * Not doing the ER fixup will only result in era->reg being
  1203. * wrong, but since we won't actually try and program hardware
  1204. * this isn't a problem either.
  1205. */
  1206. if (!cpuc->is_fake) {
  1207. if (idx != reg->idx)
  1208. intel_fixup_er(event, idx);
  1209. /*
  1210. * x86_schedule_events() can call get_event_constraints()
  1211. * multiple times on events in the case of incremental
  1212. * scheduling(). reg->alloc ensures we only do the ER
  1213. * allocation once.
  1214. */
  1215. reg->alloc = 1;
  1216. }
  1217. /* lock in msr value */
  1218. era->config = reg->config;
  1219. era->reg = reg->reg;
  1220. /* one more user */
  1221. atomic_inc(&era->ref);
  1222. /*
  1223. * need to call x86_get_event_constraint()
  1224. * to check if associated event has constraints
  1225. */
  1226. c = NULL;
  1227. } else {
  1228. idx = intel_alt_er(idx);
  1229. if (idx != reg->idx) {
  1230. raw_spin_unlock_irqrestore(&era->lock, flags);
  1231. goto again;
  1232. }
  1233. }
  1234. raw_spin_unlock_irqrestore(&era->lock, flags);
  1235. return c;
  1236. }
  1237. static void
  1238. __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,
  1239. struct hw_perf_event_extra *reg)
  1240. {
  1241. struct er_account *era;
  1242. /*
  1243. * Only put constraint if extra reg was actually allocated. Also takes
  1244. * care of event which do not use an extra shared reg.
  1245. *
  1246. * Also, if this is a fake cpuc we shouldn't touch any event state
  1247. * (reg->alloc) and we don't care about leaving inconsistent cpuc state
  1248. * either since it'll be thrown out.
  1249. */
  1250. if (!reg->alloc || cpuc->is_fake)
  1251. return;
  1252. era = &cpuc->shared_regs->regs[reg->idx];
  1253. /* one fewer user */
  1254. atomic_dec(&era->ref);
  1255. /* allocate again next time */
  1256. reg->alloc = 0;
  1257. }
  1258. static struct event_constraint *
  1259. intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
  1260. struct perf_event *event)
  1261. {
  1262. struct event_constraint *c = NULL, *d;
  1263. struct hw_perf_event_extra *xreg, *breg;
  1264. xreg = &event->hw.extra_reg;
  1265. if (xreg->idx != EXTRA_REG_NONE) {
  1266. c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
  1267. if (c == &emptyconstraint)
  1268. return c;
  1269. }
  1270. breg = &event->hw.branch_reg;
  1271. if (breg->idx != EXTRA_REG_NONE) {
  1272. d = __intel_shared_reg_get_constraints(cpuc, event, breg);
  1273. if (d == &emptyconstraint) {
  1274. __intel_shared_reg_put_constraints(cpuc, xreg);
  1275. c = d;
  1276. }
  1277. }
  1278. return c;
  1279. }
  1280. struct event_constraint *
  1281. x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1282. {
  1283. struct event_constraint *c;
  1284. if (x86_pmu.event_constraints) {
  1285. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1286. if ((event->hw.config & c->cmask) == c->code) {
  1287. /* hw.flags zeroed at initialization */
  1288. event->hw.flags |= c->flags;
  1289. return c;
  1290. }
  1291. }
  1292. }
  1293. return &unconstrained;
  1294. }
  1295. static struct event_constraint *
  1296. intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1297. {
  1298. struct event_constraint *c;
  1299. c = intel_bts_constraints(event);
  1300. if (c)
  1301. return c;
  1302. c = intel_pebs_constraints(event);
  1303. if (c)
  1304. return c;
  1305. c = intel_shared_regs_constraints(cpuc, event);
  1306. if (c)
  1307. return c;
  1308. return x86_get_event_constraints(cpuc, event);
  1309. }
  1310. static void
  1311. intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
  1312. struct perf_event *event)
  1313. {
  1314. struct hw_perf_event_extra *reg;
  1315. reg = &event->hw.extra_reg;
  1316. if (reg->idx != EXTRA_REG_NONE)
  1317. __intel_shared_reg_put_constraints(cpuc, reg);
  1318. reg = &event->hw.branch_reg;
  1319. if (reg->idx != EXTRA_REG_NONE)
  1320. __intel_shared_reg_put_constraints(cpuc, reg);
  1321. }
  1322. static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
  1323. struct perf_event *event)
  1324. {
  1325. event->hw.flags = 0;
  1326. intel_put_shared_regs_event_constraints(cpuc, event);
  1327. }
  1328. static void intel_pebs_aliases_core2(struct perf_event *event)
  1329. {
  1330. if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
  1331. /*
  1332. * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
  1333. * (0x003c) so that we can use it with PEBS.
  1334. *
  1335. * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
  1336. * PEBS capable. However we can use INST_RETIRED.ANY_P
  1337. * (0x00c0), which is a PEBS capable event, to get the same
  1338. * count.
  1339. *
  1340. * INST_RETIRED.ANY_P counts the number of cycles that retires
  1341. * CNTMASK instructions. By setting CNTMASK to a value (16)
  1342. * larger than the maximum number of instructions that can be
  1343. * retired per cycle (4) and then inverting the condition, we
  1344. * count all cycles that retire 16 or less instructions, which
  1345. * is every cycle.
  1346. *
  1347. * Thereby we gain a PEBS capable cycle counter.
  1348. */
  1349. u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16);
  1350. alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
  1351. event->hw.config = alt_config;
  1352. }
  1353. }
  1354. static void intel_pebs_aliases_snb(struct perf_event *event)
  1355. {
  1356. if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
  1357. /*
  1358. * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
  1359. * (0x003c) so that we can use it with PEBS.
  1360. *
  1361. * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
  1362. * PEBS capable. However we can use UOPS_RETIRED.ALL
  1363. * (0x01c2), which is a PEBS capable event, to get the same
  1364. * count.
  1365. *
  1366. * UOPS_RETIRED.ALL counts the number of cycles that retires
  1367. * CNTMASK micro-ops. By setting CNTMASK to a value (16)
  1368. * larger than the maximum number of micro-ops that can be
  1369. * retired per cycle (4) and then inverting the condition, we
  1370. * count all cycles that retire 16 or less micro-ops, which
  1371. * is every cycle.
  1372. *
  1373. * Thereby we gain a PEBS capable cycle counter.
  1374. */
  1375. u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16);
  1376. alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
  1377. event->hw.config = alt_config;
  1378. }
  1379. }
  1380. static int intel_pmu_hw_config(struct perf_event *event)
  1381. {
  1382. int ret = x86_pmu_hw_config(event);
  1383. if (ret)
  1384. return ret;
  1385. if (event->attr.precise_ip && x86_pmu.pebs_aliases)
  1386. x86_pmu.pebs_aliases(event);
  1387. if (intel_pmu_needs_lbr_smpl(event)) {
  1388. ret = intel_pmu_setup_lbr_filter(event);
  1389. if (ret)
  1390. return ret;
  1391. }
  1392. if (event->attr.type != PERF_TYPE_RAW)
  1393. return 0;
  1394. if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
  1395. return 0;
  1396. if (x86_pmu.version < 3)
  1397. return -EINVAL;
  1398. if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
  1399. return -EACCES;
  1400. event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;
  1401. return 0;
  1402. }
  1403. struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
  1404. {
  1405. if (x86_pmu.guest_get_msrs)
  1406. return x86_pmu.guest_get_msrs(nr);
  1407. *nr = 0;
  1408. return NULL;
  1409. }
  1410. EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
  1411. static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
  1412. {
  1413. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1414. struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
  1415. arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
  1416. arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;
  1417. arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;
  1418. /*
  1419. * If PMU counter has PEBS enabled it is not enough to disable counter
  1420. * on a guest entry since PEBS memory write can overshoot guest entry
  1421. * and corrupt guest memory. Disabling PEBS solves the problem.
  1422. */
  1423. arr[1].msr = MSR_IA32_PEBS_ENABLE;
  1424. arr[1].host = cpuc->pebs_enabled;
  1425. arr[1].guest = 0;
  1426. *nr = 2;
  1427. return arr;
  1428. }
  1429. static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr)
  1430. {
  1431. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1432. struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
  1433. int idx;
  1434. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1435. struct perf_event *event = cpuc->events[idx];
  1436. arr[idx].msr = x86_pmu_config_addr(idx);
  1437. arr[idx].host = arr[idx].guest = 0;
  1438. if (!test_bit(idx, cpuc->active_mask))
  1439. continue;
  1440. arr[idx].host = arr[idx].guest =
  1441. event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE;
  1442. if (event->attr.exclude_host)
  1443. arr[idx].host &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  1444. else if (event->attr.exclude_guest)
  1445. arr[idx].guest &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  1446. }
  1447. *nr = x86_pmu.num_counters;
  1448. return arr;
  1449. }
  1450. static void core_pmu_enable_event(struct perf_event *event)
  1451. {
  1452. if (!event->attr.exclude_host)
  1453. x86_pmu_enable_event(event);
  1454. }
  1455. static void core_pmu_enable_all(int added)
  1456. {
  1457. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1458. int idx;
  1459. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1460. struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
  1461. if (!test_bit(idx, cpuc->active_mask) ||
  1462. cpuc->events[idx]->attr.exclude_host)
  1463. continue;
  1464. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  1465. }
  1466. }
  1467. PMU_FORMAT_ATTR(event, "config:0-7" );
  1468. PMU_FORMAT_ATTR(umask, "config:8-15" );
  1469. PMU_FORMAT_ATTR(edge, "config:18" );
  1470. PMU_FORMAT_ATTR(pc, "config:19" );
  1471. PMU_FORMAT_ATTR(any, "config:21" ); /* v3 + */
  1472. PMU_FORMAT_ATTR(inv, "config:23" );
  1473. PMU_FORMAT_ATTR(cmask, "config:24-31" );
  1474. static struct attribute *intel_arch_formats_attr[] = {
  1475. &format_attr_event.attr,
  1476. &format_attr_umask.attr,
  1477. &format_attr_edge.attr,
  1478. &format_attr_pc.attr,
  1479. &format_attr_inv.attr,
  1480. &format_attr_cmask.attr,
  1481. NULL,
  1482. };
  1483. ssize_t intel_event_sysfs_show(char *page, u64 config)
  1484. {
  1485. u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
  1486. return x86_event_sysfs_show(page, config, event);
  1487. }
  1488. static __initconst const struct x86_pmu core_pmu = {
  1489. .name = "core",
  1490. .handle_irq = x86_pmu_handle_irq,
  1491. .disable_all = x86_pmu_disable_all,
  1492. .enable_all = core_pmu_enable_all,
  1493. .enable = core_pmu_enable_event,
  1494. .disable = x86_pmu_disable_event,
  1495. .hw_config = x86_pmu_hw_config,
  1496. .schedule_events = x86_schedule_events,
  1497. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1498. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1499. .event_map = intel_pmu_event_map,
  1500. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1501. .apic = 1,
  1502. /*
  1503. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1504. * so we install an artificial 1<<31 period regardless of
  1505. * the generic event period:
  1506. */
  1507. .max_period = (1ULL << 31) - 1,
  1508. .get_event_constraints = intel_get_event_constraints,
  1509. .put_event_constraints = intel_put_event_constraints,
  1510. .event_constraints = intel_core_event_constraints,
  1511. .guest_get_msrs = core_guest_get_msrs,
  1512. .format_attrs = intel_arch_formats_attr,
  1513. .events_sysfs_show = intel_event_sysfs_show,
  1514. };
  1515. struct intel_shared_regs *allocate_shared_regs(int cpu)
  1516. {
  1517. struct intel_shared_regs *regs;
  1518. int i;
  1519. regs = kzalloc_node(sizeof(struct intel_shared_regs),
  1520. GFP_KERNEL, cpu_to_node(cpu));
  1521. if (regs) {
  1522. /*
  1523. * initialize the locks to keep lockdep happy
  1524. */
  1525. for (i = 0; i < EXTRA_REG_MAX; i++)
  1526. raw_spin_lock_init(&regs->regs[i].lock);
  1527. regs->core_id = -1;
  1528. }
  1529. return regs;
  1530. }
  1531. static int intel_pmu_cpu_prepare(int cpu)
  1532. {
  1533. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1534. if (!(x86_pmu.extra_regs || x86_pmu.lbr_sel_map))
  1535. return NOTIFY_OK;
  1536. cpuc->shared_regs = allocate_shared_regs(cpu);
  1537. if (!cpuc->shared_regs)
  1538. return NOTIFY_BAD;
  1539. return NOTIFY_OK;
  1540. }
  1541. static void intel_pmu_cpu_starting(int cpu)
  1542. {
  1543. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1544. int core_id = topology_core_id(cpu);
  1545. int i;
  1546. init_debug_store_on_cpu(cpu);
  1547. /*
  1548. * Deal with CPUs that don't clear their LBRs on power-up.
  1549. */
  1550. intel_pmu_lbr_reset();
  1551. cpuc->lbr_sel = NULL;
  1552. if (!cpuc->shared_regs)
  1553. return;
  1554. if (!(x86_pmu.er_flags & ERF_NO_HT_SHARING)) {
  1555. for_each_cpu(i, topology_thread_cpumask(cpu)) {
  1556. struct intel_shared_regs *pc;
  1557. pc = per_cpu(cpu_hw_events, i).shared_regs;
  1558. if (pc && pc->core_id == core_id) {
  1559. cpuc->kfree_on_online = cpuc->shared_regs;
  1560. cpuc->shared_regs = pc;
  1561. break;
  1562. }
  1563. }
  1564. cpuc->shared_regs->core_id = core_id;
  1565. cpuc->shared_regs->refcnt++;
  1566. }
  1567. if (x86_pmu.lbr_sel_map)
  1568. cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
  1569. }
  1570. static void intel_pmu_cpu_dying(int cpu)
  1571. {
  1572. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1573. struct intel_shared_regs *pc;
  1574. pc = cpuc->shared_regs;
  1575. if (pc) {
  1576. if (pc->core_id == -1 || --pc->refcnt == 0)
  1577. kfree(pc);
  1578. cpuc->shared_regs = NULL;
  1579. }
  1580. fini_debug_store_on_cpu(cpu);
  1581. }
  1582. static void intel_pmu_flush_branch_stack(void)
  1583. {
  1584. /*
  1585. * Intel LBR does not tag entries with the
  1586. * PID of the current task, then we need to
  1587. * flush it on ctxsw
  1588. * For now, we simply reset it
  1589. */
  1590. if (x86_pmu.lbr_nr)
  1591. intel_pmu_lbr_reset();
  1592. }
  1593. PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
  1594. PMU_FORMAT_ATTR(ldlat, "config1:0-15");
  1595. static struct attribute *intel_arch3_formats_attr[] = {
  1596. &format_attr_event.attr,
  1597. &format_attr_umask.attr,
  1598. &format_attr_edge.attr,
  1599. &format_attr_pc.attr,
  1600. &format_attr_any.attr,
  1601. &format_attr_inv.attr,
  1602. &format_attr_cmask.attr,
  1603. &format_attr_offcore_rsp.attr, /* XXX do NHM/WSM + SNB breakout */
  1604. &format_attr_ldlat.attr, /* PEBS load latency */
  1605. NULL,
  1606. };
  1607. static __initconst const struct x86_pmu intel_pmu = {
  1608. .name = "Intel",
  1609. .handle_irq = intel_pmu_handle_irq,
  1610. .disable_all = intel_pmu_disable_all,
  1611. .enable_all = intel_pmu_enable_all,
  1612. .enable = intel_pmu_enable_event,
  1613. .disable = intel_pmu_disable_event,
  1614. .hw_config = intel_pmu_hw_config,
  1615. .schedule_events = x86_schedule_events,
  1616. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1617. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1618. .event_map = intel_pmu_event_map,
  1619. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1620. .apic = 1,
  1621. /*
  1622. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1623. * so we install an artificial 1<<31 period regardless of
  1624. * the generic event period:
  1625. */
  1626. .max_period = (1ULL << 31) - 1,
  1627. .get_event_constraints = intel_get_event_constraints,
  1628. .put_event_constraints = intel_put_event_constraints,
  1629. .pebs_aliases = intel_pebs_aliases_core2,
  1630. .format_attrs = intel_arch3_formats_attr,
  1631. .events_sysfs_show = intel_event_sysfs_show,
  1632. .cpu_prepare = intel_pmu_cpu_prepare,
  1633. .cpu_starting = intel_pmu_cpu_starting,
  1634. .cpu_dying = intel_pmu_cpu_dying,
  1635. .guest_get_msrs = intel_guest_get_msrs,
  1636. .flush_branch_stack = intel_pmu_flush_branch_stack,
  1637. };
  1638. static __init void intel_clovertown_quirk(void)
  1639. {
  1640. /*
  1641. * PEBS is unreliable due to:
  1642. *
  1643. * AJ67 - PEBS may experience CPL leaks
  1644. * AJ68 - PEBS PMI may be delayed by one event
  1645. * AJ69 - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12]
  1646. * AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS
  1647. *
  1648. * AJ67 could be worked around by restricting the OS/USR flags.
  1649. * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI.
  1650. *
  1651. * AJ106 could possibly be worked around by not allowing LBR
  1652. * usage from PEBS, including the fixup.
  1653. * AJ68 could possibly be worked around by always programming
  1654. * a pebs_event_reset[0] value and coping with the lost events.
  1655. *
  1656. * But taken together it might just make sense to not enable PEBS on
  1657. * these chips.
  1658. */
  1659. pr_warn("PEBS disabled due to CPU errata\n");
  1660. x86_pmu.pebs = 0;
  1661. x86_pmu.pebs_constraints = NULL;
  1662. }
  1663. static int intel_snb_pebs_broken(int cpu)
  1664. {
  1665. u32 rev = UINT_MAX; /* default to broken for unknown models */
  1666. switch (cpu_data(cpu).x86_model) {
  1667. case 42: /* SNB */
  1668. rev = 0x28;
  1669. break;
  1670. case 45: /* SNB-EP */
  1671. switch (cpu_data(cpu).x86_mask) {
  1672. case 6: rev = 0x618; break;
  1673. case 7: rev = 0x70c; break;
  1674. }
  1675. }
  1676. return (cpu_data(cpu).microcode < rev);
  1677. }
  1678. static void intel_snb_check_microcode(void)
  1679. {
  1680. int pebs_broken = 0;
  1681. int cpu;
  1682. get_online_cpus();
  1683. for_each_online_cpu(cpu) {
  1684. if ((pebs_broken = intel_snb_pebs_broken(cpu)))
  1685. break;
  1686. }
  1687. put_online_cpus();
  1688. if (pebs_broken == x86_pmu.pebs_broken)
  1689. return;
  1690. /*
  1691. * Serialized by the microcode lock..
  1692. */
  1693. if (x86_pmu.pebs_broken) {
  1694. pr_info("PEBS enabled due to microcode update\n");
  1695. x86_pmu.pebs_broken = 0;
  1696. } else {
  1697. pr_info("PEBS disabled due to CPU errata, please upgrade microcode\n");
  1698. x86_pmu.pebs_broken = 1;
  1699. }
  1700. }
  1701. static __init void intel_sandybridge_quirk(void)
  1702. {
  1703. x86_pmu.check_microcode = intel_snb_check_microcode;
  1704. intel_snb_check_microcode();
  1705. }
  1706. static const struct { int id; char *name; } intel_arch_events_map[] __initconst = {
  1707. { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
  1708. { PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
  1709. { PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
  1710. { PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
  1711. { PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
  1712. { PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
  1713. { PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
  1714. };
  1715. static __init void intel_arch_events_quirk(void)
  1716. {
  1717. int bit;
  1718. /* disable event that reported as not presend by cpuid */
  1719. for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(intel_arch_events_map)) {
  1720. intel_perfmon_event_map[intel_arch_events_map[bit].id] = 0;
  1721. pr_warn("CPUID marked event: \'%s\' unavailable\n",
  1722. intel_arch_events_map[bit].name);
  1723. }
  1724. }
  1725. static __init void intel_nehalem_quirk(void)
  1726. {
  1727. union cpuid10_ebx ebx;
  1728. ebx.full = x86_pmu.events_maskl;
  1729. if (ebx.split.no_branch_misses_retired) {
  1730. /*
  1731. * Erratum AAJ80 detected, we work it around by using
  1732. * the BR_MISP_EXEC.ANY event. This will over-count
  1733. * branch-misses, but it's still much better than the
  1734. * architectural event which is often completely bogus:
  1735. */
  1736. intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89;
  1737. ebx.split.no_branch_misses_retired = 0;
  1738. x86_pmu.events_maskl = ebx.full;
  1739. pr_info("CPU erratum AAJ80 worked around\n");
  1740. }
  1741. }
  1742. __init int intel_pmu_init(void)
  1743. {
  1744. union cpuid10_edx edx;
  1745. union cpuid10_eax eax;
  1746. union cpuid10_ebx ebx;
  1747. struct event_constraint *c;
  1748. unsigned int unused;
  1749. int version;
  1750. if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
  1751. switch (boot_cpu_data.x86) {
  1752. case 0x6:
  1753. return p6_pmu_init();
  1754. case 0xb:
  1755. return knc_pmu_init();
  1756. case 0xf:
  1757. return p4_pmu_init();
  1758. }
  1759. return -ENODEV;
  1760. }
  1761. /*
  1762. * Check whether the Architectural PerfMon supports
  1763. * Branch Misses Retired hw_event or not.
  1764. */
  1765. cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);
  1766. if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT)
  1767. return -ENODEV;
  1768. version = eax.split.version_id;
  1769. if (version < 2)
  1770. x86_pmu = core_pmu;
  1771. else
  1772. x86_pmu = intel_pmu;
  1773. x86_pmu.version = version;
  1774. x86_pmu.num_counters = eax.split.num_counters;
  1775. x86_pmu.cntval_bits = eax.split.bit_width;
  1776. x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1;
  1777. x86_pmu.events_maskl = ebx.full;
  1778. x86_pmu.events_mask_len = eax.split.mask_length;
  1779. x86_pmu.max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, x86_pmu.num_counters);
  1780. /*
  1781. * Quirk: v2 perfmon does not report fixed-purpose events, so
  1782. * assume at least 3 events:
  1783. */
  1784. if (version > 1)
  1785. x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
  1786. /*
  1787. * v2 and above have a perf capabilities MSR
  1788. */
  1789. if (version > 1) {
  1790. u64 capabilities;
  1791. rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
  1792. x86_pmu.intel_cap.capabilities = capabilities;
  1793. }
  1794. intel_ds_init();
  1795. x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
  1796. /*
  1797. * Install the hw-cache-events table:
  1798. */
  1799. switch (boot_cpu_data.x86_model) {
  1800. case 14: /* 65 nm core solo/duo, "Yonah" */
  1801. pr_cont("Core events, ");
  1802. break;
  1803. case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
  1804. x86_add_quirk(intel_clovertown_quirk);
  1805. case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
  1806. case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
  1807. case 29: /* six-core 45 nm xeon "Dunnington" */
  1808. memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
  1809. sizeof(hw_cache_event_ids));
  1810. intel_pmu_lbr_init_core();
  1811. x86_pmu.event_constraints = intel_core2_event_constraints;
  1812. x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
  1813. pr_cont("Core2 events, ");
  1814. break;
  1815. case 26: /* 45 nm nehalem, "Bloomfield" */
  1816. case 30: /* 45 nm nehalem, "Lynnfield" */
  1817. case 46: /* 45 nm nehalem-ex, "Beckton" */
  1818. memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
  1819. sizeof(hw_cache_event_ids));
  1820. memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
  1821. sizeof(hw_cache_extra_regs));
  1822. intel_pmu_lbr_init_nhm();
  1823. x86_pmu.event_constraints = intel_nehalem_event_constraints;
  1824. x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
  1825. x86_pmu.enable_all = intel_pmu_nhm_enable_all;
  1826. x86_pmu.extra_regs = intel_nehalem_extra_regs;
  1827. x86_pmu.cpu_events = nhm_events_attrs;
  1828. /* UOPS_ISSUED.STALLED_CYCLES */
  1829. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1830. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1831. /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
  1832. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  1833. X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
  1834. x86_add_quirk(intel_nehalem_quirk);
  1835. pr_cont("Nehalem events, ");
  1836. break;
  1837. case 28: /* Atom */
  1838. case 38: /* Lincroft */
  1839. case 39: /* Penwell */
  1840. case 53: /* Cloverview */
  1841. case 54: /* Cedarview */
  1842. memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
  1843. sizeof(hw_cache_event_ids));
  1844. intel_pmu_lbr_init_atom();
  1845. x86_pmu.event_constraints = intel_gen_event_constraints;
  1846. x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
  1847. pr_cont("Atom events, ");
  1848. break;
  1849. case 37: /* 32 nm nehalem, "Clarkdale" */
  1850. case 44: /* 32 nm nehalem, "Gulftown" */
  1851. case 47: /* 32 nm Xeon E7 */
  1852. memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
  1853. sizeof(hw_cache_event_ids));
  1854. memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
  1855. sizeof(hw_cache_extra_regs));
  1856. intel_pmu_lbr_init_nhm();
  1857. x86_pmu.event_constraints = intel_westmere_event_constraints;
  1858. x86_pmu.enable_all = intel_pmu_nhm_enable_all;
  1859. x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
  1860. x86_pmu.extra_regs = intel_westmere_extra_regs;
  1861. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  1862. x86_pmu.cpu_events = nhm_events_attrs;
  1863. /* UOPS_ISSUED.STALLED_CYCLES */
  1864. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1865. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1866. /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
  1867. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  1868. X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
  1869. pr_cont("Westmere events, ");
  1870. break;
  1871. case 42: /* SandyBridge */
  1872. case 45: /* SandyBridge, "Romely-EP" */
  1873. x86_add_quirk(intel_sandybridge_quirk);
  1874. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
  1875. sizeof(hw_cache_event_ids));
  1876. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
  1877. sizeof(hw_cache_extra_regs));
  1878. intel_pmu_lbr_init_snb();
  1879. x86_pmu.event_constraints = intel_snb_event_constraints;
  1880. x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
  1881. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  1882. if (boot_cpu_data.x86_model == 45)
  1883. x86_pmu.extra_regs = intel_snbep_extra_regs;
  1884. else
  1885. x86_pmu.extra_regs = intel_snb_extra_regs;
  1886. /* all extra regs are per-cpu when HT is on */
  1887. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  1888. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  1889. x86_pmu.cpu_events = snb_events_attrs;
  1890. /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
  1891. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1892. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1893. /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
  1894. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  1895. X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1);
  1896. pr_cont("SandyBridge events, ");
  1897. break;
  1898. case 58: /* IvyBridge */
  1899. case 62: /* IvyBridge EP */
  1900. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
  1901. sizeof(hw_cache_event_ids));
  1902. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
  1903. sizeof(hw_cache_extra_regs));
  1904. intel_pmu_lbr_init_snb();
  1905. x86_pmu.event_constraints = intel_ivb_event_constraints;
  1906. x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
  1907. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  1908. if (boot_cpu_data.x86_model == 62)
  1909. x86_pmu.extra_regs = intel_snbep_extra_regs;
  1910. else
  1911. x86_pmu.extra_regs = intel_snb_extra_regs;
  1912. /* all extra regs are per-cpu when HT is on */
  1913. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  1914. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  1915. x86_pmu.cpu_events = snb_events_attrs;
  1916. /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
  1917. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  1918. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  1919. pr_cont("IvyBridge events, ");
  1920. break;
  1921. default:
  1922. switch (x86_pmu.version) {
  1923. case 1:
  1924. x86_pmu.event_constraints = intel_v1_event_constraints;
  1925. pr_cont("generic architected perfmon v1, ");
  1926. break;
  1927. default:
  1928. /*
  1929. * default constraints for v2 and up
  1930. */
  1931. x86_pmu.event_constraints = intel_gen_event_constraints;
  1932. pr_cont("generic architected perfmon, ");
  1933. break;
  1934. }
  1935. }
  1936. if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) {
  1937. WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
  1938. x86_pmu.num_counters, INTEL_PMC_MAX_GENERIC);
  1939. x86_pmu.num_counters = INTEL_PMC_MAX_GENERIC;
  1940. }
  1941. x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
  1942. if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) {
  1943. WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
  1944. x86_pmu.num_counters_fixed, INTEL_PMC_MAX_FIXED);
  1945. x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED;
  1946. }
  1947. x86_pmu.intel_ctrl |=
  1948. ((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
  1949. if (x86_pmu.event_constraints) {
  1950. /*
  1951. * event on fixed counter2 (REF_CYCLES) only works on this
  1952. * counter, so do not extend mask to generic counters
  1953. */
  1954. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1955. if (c->cmask != X86_RAW_EVENT_MASK
  1956. || c->idxmsk64 == INTEL_PMC_MSK_FIXED_REF_CYCLES) {
  1957. continue;
  1958. }
  1959. c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
  1960. c->weight += x86_pmu.num_counters;
  1961. }
  1962. }
  1963. return 0;
  1964. }