perf_event_intel.c 69 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513
  1. /*
  2. * Per core/cpu state
  3. *
  4. * Used to coordinate shared registers between HT threads or
  5. * among events on a single PMU.
  6. */
  7. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  8. #include <linux/stddef.h>
  9. #include <linux/types.h>
  10. #include <linux/init.h>
  11. #include <linux/slab.h>
  12. #include <linux/export.h>
  13. #include <asm/cpufeature.h>
  14. #include <asm/hardirq.h>
  15. #include <asm/apic.h>
  16. #include "perf_event.h"
  17. /*
  18. * Intel PerfMon, used on Core and later.
  19. */
  20. static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
  21. {
  22. [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
  23. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  24. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
  25. [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
  26. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  27. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  28. [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
  29. [PERF_COUNT_HW_REF_CPU_CYCLES] = 0x0300, /* pseudo-encoding */
  30. };
  31. static struct event_constraint intel_core_event_constraints[] __read_mostly =
  32. {
  33. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  34. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  35. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  36. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  37. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  38. INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
  39. EVENT_CONSTRAINT_END
  40. };
  41. static struct event_constraint intel_core2_event_constraints[] __read_mostly =
  42. {
  43. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  44. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  45. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  46. INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  47. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  48. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  49. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  50. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  51. INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
  52. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  53. INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
  54. INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */
  55. INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
  56. EVENT_CONSTRAINT_END
  57. };
  58. static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
  59. {
  60. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  61. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  62. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  63. INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
  64. INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
  65. INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
  66. INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
  67. INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
  68. INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
  69. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  70. INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  71. EVENT_CONSTRAINT_END
  72. };
  73. static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
  74. {
  75. /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
  76. INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  77. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  78. EVENT_EXTRA_END
  79. };
  80. static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
  81. {
  82. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  83. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  84. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  85. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  86. INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
  87. INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  88. INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */
  89. EVENT_CONSTRAINT_END
  90. };
  91. static struct event_constraint intel_snb_event_constraints[] __read_mostly =
  92. {
  93. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  94. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  95. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  96. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
  97. INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
  98. INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  99. INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  100. INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
  101. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  102. INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
  103. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
  104. INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  105. EVENT_CONSTRAINT_END
  106. };
  107. static struct event_constraint intel_ivb_event_constraints[] __read_mostly =
  108. {
  109. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  110. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  111. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  112. INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */
  113. INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */
  114. INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */
  115. INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_LDM_PENDING */
  116. INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
  117. INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
  118. INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */
  119. INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  120. INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  121. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  122. /*
  123. * Errata BV98 -- MEM_*_RETIRED events can leak between counters of SMT
  124. * siblings; disable these events because they can corrupt unrelated
  125. * counters.
  126. */
  127. INTEL_EVENT_CONSTRAINT(0xd0, 0x0), /* MEM_UOPS_RETIRED.* */
  128. INTEL_EVENT_CONSTRAINT(0xd1, 0x0), /* MEM_LOAD_UOPS_RETIRED.* */
  129. INTEL_EVENT_CONSTRAINT(0xd2, 0x0), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
  130. INTEL_EVENT_CONSTRAINT(0xd3, 0x0), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
  131. EVENT_CONSTRAINT_END
  132. };
  133. static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
  134. {
  135. /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
  136. INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  137. INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1),
  138. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  139. EVENT_EXTRA_END
  140. };
  141. static struct event_constraint intel_v1_event_constraints[] __read_mostly =
  142. {
  143. EVENT_CONSTRAINT_END
  144. };
  145. static struct event_constraint intel_gen_event_constraints[] __read_mostly =
  146. {
  147. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  148. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  149. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  150. EVENT_CONSTRAINT_END
  151. };
  152. static struct event_constraint intel_slm_event_constraints[] __read_mostly =
  153. {
  154. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  155. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  156. FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF */
  157. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
  158. EVENT_CONSTRAINT_END
  159. };
  160. static struct extra_reg intel_snb_extra_regs[] __read_mostly = {
  161. /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
  162. INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0),
  163. INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1),
  164. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
  165. EVENT_EXTRA_END
  166. };
  167. static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {
  168. /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
  169. INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
  170. INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
  171. INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
  172. EVENT_EXTRA_END
  173. };
  174. EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3");
  175. EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3");
  176. EVENT_ATTR_STR(mem-stores, mem_st_snb, "event=0xcd,umask=0x2");
  177. struct attribute *nhm_events_attrs[] = {
  178. EVENT_PTR(mem_ld_nhm),
  179. NULL,
  180. };
  181. struct attribute *snb_events_attrs[] = {
  182. EVENT_PTR(mem_ld_snb),
  183. EVENT_PTR(mem_st_snb),
  184. NULL,
  185. };
  186. static struct event_constraint intel_hsw_event_constraints[] = {
  187. FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  188. FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  189. FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  190. INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.* */
  191. INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
  192. INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
  193. /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
  194. INTEL_EVENT_CONSTRAINT(0x08a3, 0x4),
  195. /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
  196. INTEL_EVENT_CONSTRAINT(0x0ca3, 0x4),
  197. /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
  198. INTEL_EVENT_CONSTRAINT(0x04a3, 0xf),
  199. EVENT_CONSTRAINT_END
  200. };
  201. static u64 intel_pmu_event_map(int hw_event)
  202. {
  203. return intel_perfmon_event_map[hw_event];
  204. }
  205. #define SNB_DMND_DATA_RD (1ULL << 0)
  206. #define SNB_DMND_RFO (1ULL << 1)
  207. #define SNB_DMND_IFETCH (1ULL << 2)
  208. #define SNB_DMND_WB (1ULL << 3)
  209. #define SNB_PF_DATA_RD (1ULL << 4)
  210. #define SNB_PF_RFO (1ULL << 5)
  211. #define SNB_PF_IFETCH (1ULL << 6)
  212. #define SNB_LLC_DATA_RD (1ULL << 7)
  213. #define SNB_LLC_RFO (1ULL << 8)
  214. #define SNB_LLC_IFETCH (1ULL << 9)
  215. #define SNB_BUS_LOCKS (1ULL << 10)
  216. #define SNB_STRM_ST (1ULL << 11)
  217. #define SNB_OTHER (1ULL << 15)
  218. #define SNB_RESP_ANY (1ULL << 16)
  219. #define SNB_NO_SUPP (1ULL << 17)
  220. #define SNB_LLC_HITM (1ULL << 18)
  221. #define SNB_LLC_HITE (1ULL << 19)
  222. #define SNB_LLC_HITS (1ULL << 20)
  223. #define SNB_LLC_HITF (1ULL << 21)
  224. #define SNB_LOCAL (1ULL << 22)
  225. #define SNB_REMOTE (0xffULL << 23)
  226. #define SNB_SNP_NONE (1ULL << 31)
  227. #define SNB_SNP_NOT_NEEDED (1ULL << 32)
  228. #define SNB_SNP_MISS (1ULL << 33)
  229. #define SNB_NO_FWD (1ULL << 34)
  230. #define SNB_SNP_FWD (1ULL << 35)
  231. #define SNB_HITM (1ULL << 36)
  232. #define SNB_NON_DRAM (1ULL << 37)
  233. #define SNB_DMND_READ (SNB_DMND_DATA_RD|SNB_LLC_DATA_RD)
  234. #define SNB_DMND_WRITE (SNB_DMND_RFO|SNB_LLC_RFO)
  235. #define SNB_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)
  236. #define SNB_SNP_ANY (SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \
  237. SNB_SNP_MISS|SNB_NO_FWD|SNB_SNP_FWD| \
  238. SNB_HITM)
  239. #define SNB_DRAM_ANY (SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY)
  240. #define SNB_DRAM_REMOTE (SNB_REMOTE|SNB_SNP_ANY)
  241. #define SNB_L3_ACCESS SNB_RESP_ANY
  242. #define SNB_L3_MISS (SNB_DRAM_ANY|SNB_NON_DRAM)
  243. static __initconst const u64 snb_hw_cache_extra_regs
  244. [PERF_COUNT_HW_CACHE_MAX]
  245. [PERF_COUNT_HW_CACHE_OP_MAX]
  246. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  247. {
  248. [ C(LL ) ] = {
  249. [ C(OP_READ) ] = {
  250. [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_L3_ACCESS,
  251. [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_L3_MISS,
  252. },
  253. [ C(OP_WRITE) ] = {
  254. [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_L3_ACCESS,
  255. [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_L3_MISS,
  256. },
  257. [ C(OP_PREFETCH) ] = {
  258. [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_L3_ACCESS,
  259. [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_L3_MISS,
  260. },
  261. },
  262. [ C(NODE) ] = {
  263. [ C(OP_READ) ] = {
  264. [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_DRAM_ANY,
  265. [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_DRAM_REMOTE,
  266. },
  267. [ C(OP_WRITE) ] = {
  268. [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_DRAM_ANY,
  269. [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_DRAM_REMOTE,
  270. },
  271. [ C(OP_PREFETCH) ] = {
  272. [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_DRAM_ANY,
  273. [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_DRAM_REMOTE,
  274. },
  275. },
  276. };
  277. static __initconst const u64 snb_hw_cache_event_ids
  278. [PERF_COUNT_HW_CACHE_MAX]
  279. [PERF_COUNT_HW_CACHE_OP_MAX]
  280. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  281. {
  282. [ C(L1D) ] = {
  283. [ C(OP_READ) ] = {
  284. [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS */
  285. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPLACEMENT */
  286. },
  287. [ C(OP_WRITE) ] = {
  288. [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES */
  289. [ C(RESULT_MISS) ] = 0x0851, /* L1D.ALL_M_REPLACEMENT */
  290. },
  291. [ C(OP_PREFETCH) ] = {
  292. [ C(RESULT_ACCESS) ] = 0x0,
  293. [ C(RESULT_MISS) ] = 0x024e, /* HW_PRE_REQ.DL1_MISS */
  294. },
  295. },
  296. [ C(L1I ) ] = {
  297. [ C(OP_READ) ] = {
  298. [ C(RESULT_ACCESS) ] = 0x0,
  299. [ C(RESULT_MISS) ] = 0x0280, /* ICACHE.MISSES */
  300. },
  301. [ C(OP_WRITE) ] = {
  302. [ C(RESULT_ACCESS) ] = -1,
  303. [ C(RESULT_MISS) ] = -1,
  304. },
  305. [ C(OP_PREFETCH) ] = {
  306. [ C(RESULT_ACCESS) ] = 0x0,
  307. [ C(RESULT_MISS) ] = 0x0,
  308. },
  309. },
  310. [ C(LL ) ] = {
  311. [ C(OP_READ) ] = {
  312. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  313. [ C(RESULT_ACCESS) ] = 0x01b7,
  314. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  315. [ C(RESULT_MISS) ] = 0x01b7,
  316. },
  317. [ C(OP_WRITE) ] = {
  318. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  319. [ C(RESULT_ACCESS) ] = 0x01b7,
  320. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  321. [ C(RESULT_MISS) ] = 0x01b7,
  322. },
  323. [ C(OP_PREFETCH) ] = {
  324. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  325. [ C(RESULT_ACCESS) ] = 0x01b7,
  326. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  327. [ C(RESULT_MISS) ] = 0x01b7,
  328. },
  329. },
  330. [ C(DTLB) ] = {
  331. [ C(OP_READ) ] = {
  332. [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
  333. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
  334. },
  335. [ C(OP_WRITE) ] = {
  336. [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
  337. [ C(RESULT_MISS) ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
  338. },
  339. [ C(OP_PREFETCH) ] = {
  340. [ C(RESULT_ACCESS) ] = 0x0,
  341. [ C(RESULT_MISS) ] = 0x0,
  342. },
  343. },
  344. [ C(ITLB) ] = {
  345. [ C(OP_READ) ] = {
  346. [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT */
  347. [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK */
  348. },
  349. [ C(OP_WRITE) ] = {
  350. [ C(RESULT_ACCESS) ] = -1,
  351. [ C(RESULT_MISS) ] = -1,
  352. },
  353. [ C(OP_PREFETCH) ] = {
  354. [ C(RESULT_ACCESS) ] = -1,
  355. [ C(RESULT_MISS) ] = -1,
  356. },
  357. },
  358. [ C(BPU ) ] = {
  359. [ C(OP_READ) ] = {
  360. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  361. [ C(RESULT_MISS) ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
  362. },
  363. [ C(OP_WRITE) ] = {
  364. [ C(RESULT_ACCESS) ] = -1,
  365. [ C(RESULT_MISS) ] = -1,
  366. },
  367. [ C(OP_PREFETCH) ] = {
  368. [ C(RESULT_ACCESS) ] = -1,
  369. [ C(RESULT_MISS) ] = -1,
  370. },
  371. },
  372. [ C(NODE) ] = {
  373. [ C(OP_READ) ] = {
  374. [ C(RESULT_ACCESS) ] = 0x01b7,
  375. [ C(RESULT_MISS) ] = 0x01b7,
  376. },
  377. [ C(OP_WRITE) ] = {
  378. [ C(RESULT_ACCESS) ] = 0x01b7,
  379. [ C(RESULT_MISS) ] = 0x01b7,
  380. },
  381. [ C(OP_PREFETCH) ] = {
  382. [ C(RESULT_ACCESS) ] = 0x01b7,
  383. [ C(RESULT_MISS) ] = 0x01b7,
  384. },
  385. },
  386. };
  387. static __initconst const u64 westmere_hw_cache_event_ids
  388. [PERF_COUNT_HW_CACHE_MAX]
  389. [PERF_COUNT_HW_CACHE_OP_MAX]
  390. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  391. {
  392. [ C(L1D) ] = {
  393. [ C(OP_READ) ] = {
  394. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  395. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */
  396. },
  397. [ C(OP_WRITE) ] = {
  398. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  399. [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */
  400. },
  401. [ C(OP_PREFETCH) ] = {
  402. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  403. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  404. },
  405. },
  406. [ C(L1I ) ] = {
  407. [ C(OP_READ) ] = {
  408. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  409. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  410. },
  411. [ C(OP_WRITE) ] = {
  412. [ C(RESULT_ACCESS) ] = -1,
  413. [ C(RESULT_MISS) ] = -1,
  414. },
  415. [ C(OP_PREFETCH) ] = {
  416. [ C(RESULT_ACCESS) ] = 0x0,
  417. [ C(RESULT_MISS) ] = 0x0,
  418. },
  419. },
  420. [ C(LL ) ] = {
  421. [ C(OP_READ) ] = {
  422. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  423. [ C(RESULT_ACCESS) ] = 0x01b7,
  424. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  425. [ C(RESULT_MISS) ] = 0x01b7,
  426. },
  427. /*
  428. * Use RFO, not WRITEBACK, because a write miss would typically occur
  429. * on RFO.
  430. */
  431. [ C(OP_WRITE) ] = {
  432. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  433. [ C(RESULT_ACCESS) ] = 0x01b7,
  434. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  435. [ C(RESULT_MISS) ] = 0x01b7,
  436. },
  437. [ C(OP_PREFETCH) ] = {
  438. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  439. [ C(RESULT_ACCESS) ] = 0x01b7,
  440. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  441. [ C(RESULT_MISS) ] = 0x01b7,
  442. },
  443. },
  444. [ C(DTLB) ] = {
  445. [ C(OP_READ) ] = {
  446. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  447. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  448. },
  449. [ C(OP_WRITE) ] = {
  450. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  451. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  452. },
  453. [ C(OP_PREFETCH) ] = {
  454. [ C(RESULT_ACCESS) ] = 0x0,
  455. [ C(RESULT_MISS) ] = 0x0,
  456. },
  457. },
  458. [ C(ITLB) ] = {
  459. [ C(OP_READ) ] = {
  460. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  461. [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.ANY */
  462. },
  463. [ C(OP_WRITE) ] = {
  464. [ C(RESULT_ACCESS) ] = -1,
  465. [ C(RESULT_MISS) ] = -1,
  466. },
  467. [ C(OP_PREFETCH) ] = {
  468. [ C(RESULT_ACCESS) ] = -1,
  469. [ C(RESULT_MISS) ] = -1,
  470. },
  471. },
  472. [ C(BPU ) ] = {
  473. [ C(OP_READ) ] = {
  474. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  475. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  476. },
  477. [ C(OP_WRITE) ] = {
  478. [ C(RESULT_ACCESS) ] = -1,
  479. [ C(RESULT_MISS) ] = -1,
  480. },
  481. [ C(OP_PREFETCH) ] = {
  482. [ C(RESULT_ACCESS) ] = -1,
  483. [ C(RESULT_MISS) ] = -1,
  484. },
  485. },
  486. [ C(NODE) ] = {
  487. [ C(OP_READ) ] = {
  488. [ C(RESULT_ACCESS) ] = 0x01b7,
  489. [ C(RESULT_MISS) ] = 0x01b7,
  490. },
  491. [ C(OP_WRITE) ] = {
  492. [ C(RESULT_ACCESS) ] = 0x01b7,
  493. [ C(RESULT_MISS) ] = 0x01b7,
  494. },
  495. [ C(OP_PREFETCH) ] = {
  496. [ C(RESULT_ACCESS) ] = 0x01b7,
  497. [ C(RESULT_MISS) ] = 0x01b7,
  498. },
  499. },
  500. };
  501. /*
  502. * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits;
  503. * See IA32 SDM Vol 3B 30.6.1.3
  504. */
  505. #define NHM_DMND_DATA_RD (1 << 0)
  506. #define NHM_DMND_RFO (1 << 1)
  507. #define NHM_DMND_IFETCH (1 << 2)
  508. #define NHM_DMND_WB (1 << 3)
  509. #define NHM_PF_DATA_RD (1 << 4)
  510. #define NHM_PF_DATA_RFO (1 << 5)
  511. #define NHM_PF_IFETCH (1 << 6)
  512. #define NHM_OFFCORE_OTHER (1 << 7)
  513. #define NHM_UNCORE_HIT (1 << 8)
  514. #define NHM_OTHER_CORE_HIT_SNP (1 << 9)
  515. #define NHM_OTHER_CORE_HITM (1 << 10)
  516. /* reserved */
  517. #define NHM_REMOTE_CACHE_FWD (1 << 12)
  518. #define NHM_REMOTE_DRAM (1 << 13)
  519. #define NHM_LOCAL_DRAM (1 << 14)
  520. #define NHM_NON_DRAM (1 << 15)
  521. #define NHM_LOCAL (NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD)
  522. #define NHM_REMOTE (NHM_REMOTE_DRAM)
  523. #define NHM_DMND_READ (NHM_DMND_DATA_RD)
  524. #define NHM_DMND_WRITE (NHM_DMND_RFO|NHM_DMND_WB)
  525. #define NHM_DMND_PREFETCH (NHM_PF_DATA_RD|NHM_PF_DATA_RFO)
  526. #define NHM_L3_HIT (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)
  527. #define NHM_L3_MISS (NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD)
  528. #define NHM_L3_ACCESS (NHM_L3_HIT|NHM_L3_MISS)
  529. static __initconst const u64 nehalem_hw_cache_extra_regs
  530. [PERF_COUNT_HW_CACHE_MAX]
  531. [PERF_COUNT_HW_CACHE_OP_MAX]
  532. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  533. {
  534. [ C(LL ) ] = {
  535. [ C(OP_READ) ] = {
  536. [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS,
  537. [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_L3_MISS,
  538. },
  539. [ C(OP_WRITE) ] = {
  540. [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS,
  541. [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_L3_MISS,
  542. },
  543. [ C(OP_PREFETCH) ] = {
  544. [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS,
  545. [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_L3_MISS,
  546. },
  547. },
  548. [ C(NODE) ] = {
  549. [ C(OP_READ) ] = {
  550. [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_LOCAL|NHM_REMOTE,
  551. [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_REMOTE,
  552. },
  553. [ C(OP_WRITE) ] = {
  554. [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_LOCAL|NHM_REMOTE,
  555. [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_REMOTE,
  556. },
  557. [ C(OP_PREFETCH) ] = {
  558. [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_LOCAL|NHM_REMOTE,
  559. [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_REMOTE,
  560. },
  561. },
  562. };
  563. static __initconst const u64 nehalem_hw_cache_event_ids
  564. [PERF_COUNT_HW_CACHE_MAX]
  565. [PERF_COUNT_HW_CACHE_OP_MAX]
  566. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  567. {
  568. [ C(L1D) ] = {
  569. [ C(OP_READ) ] = {
  570. [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */
  571. [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */
  572. },
  573. [ C(OP_WRITE) ] = {
  574. [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */
  575. [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */
  576. },
  577. [ C(OP_PREFETCH) ] = {
  578. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  579. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  580. },
  581. },
  582. [ C(L1I ) ] = {
  583. [ C(OP_READ) ] = {
  584. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  585. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  586. },
  587. [ C(OP_WRITE) ] = {
  588. [ C(RESULT_ACCESS) ] = -1,
  589. [ C(RESULT_MISS) ] = -1,
  590. },
  591. [ C(OP_PREFETCH) ] = {
  592. [ C(RESULT_ACCESS) ] = 0x0,
  593. [ C(RESULT_MISS) ] = 0x0,
  594. },
  595. },
  596. [ C(LL ) ] = {
  597. [ C(OP_READ) ] = {
  598. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  599. [ C(RESULT_ACCESS) ] = 0x01b7,
  600. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  601. [ C(RESULT_MISS) ] = 0x01b7,
  602. },
  603. /*
  604. * Use RFO, not WRITEBACK, because a write miss would typically occur
  605. * on RFO.
  606. */
  607. [ C(OP_WRITE) ] = {
  608. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  609. [ C(RESULT_ACCESS) ] = 0x01b7,
  610. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  611. [ C(RESULT_MISS) ] = 0x01b7,
  612. },
  613. [ C(OP_PREFETCH) ] = {
  614. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  615. [ C(RESULT_ACCESS) ] = 0x01b7,
  616. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  617. [ C(RESULT_MISS) ] = 0x01b7,
  618. },
  619. },
  620. [ C(DTLB) ] = {
  621. [ C(OP_READ) ] = {
  622. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  623. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  624. },
  625. [ C(OP_WRITE) ] = {
  626. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  627. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  628. },
  629. [ C(OP_PREFETCH) ] = {
  630. [ C(RESULT_ACCESS) ] = 0x0,
  631. [ C(RESULT_MISS) ] = 0x0,
  632. },
  633. },
  634. [ C(ITLB) ] = {
  635. [ C(OP_READ) ] = {
  636. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  637. [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
  638. },
  639. [ C(OP_WRITE) ] = {
  640. [ C(RESULT_ACCESS) ] = -1,
  641. [ C(RESULT_MISS) ] = -1,
  642. },
  643. [ C(OP_PREFETCH) ] = {
  644. [ C(RESULT_ACCESS) ] = -1,
  645. [ C(RESULT_MISS) ] = -1,
  646. },
  647. },
  648. [ C(BPU ) ] = {
  649. [ C(OP_READ) ] = {
  650. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  651. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  652. },
  653. [ C(OP_WRITE) ] = {
  654. [ C(RESULT_ACCESS) ] = -1,
  655. [ C(RESULT_MISS) ] = -1,
  656. },
  657. [ C(OP_PREFETCH) ] = {
  658. [ C(RESULT_ACCESS) ] = -1,
  659. [ C(RESULT_MISS) ] = -1,
  660. },
  661. },
  662. [ C(NODE) ] = {
  663. [ C(OP_READ) ] = {
  664. [ C(RESULT_ACCESS) ] = 0x01b7,
  665. [ C(RESULT_MISS) ] = 0x01b7,
  666. },
  667. [ C(OP_WRITE) ] = {
  668. [ C(RESULT_ACCESS) ] = 0x01b7,
  669. [ C(RESULT_MISS) ] = 0x01b7,
  670. },
  671. [ C(OP_PREFETCH) ] = {
  672. [ C(RESULT_ACCESS) ] = 0x01b7,
  673. [ C(RESULT_MISS) ] = 0x01b7,
  674. },
  675. },
  676. };
  677. static __initconst const u64 core2_hw_cache_event_ids
  678. [PERF_COUNT_HW_CACHE_MAX]
  679. [PERF_COUNT_HW_CACHE_OP_MAX]
  680. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  681. {
  682. [ C(L1D) ] = {
  683. [ C(OP_READ) ] = {
  684. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  685. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  686. },
  687. [ C(OP_WRITE) ] = {
  688. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  689. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  690. },
  691. [ C(OP_PREFETCH) ] = {
  692. [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
  693. [ C(RESULT_MISS) ] = 0,
  694. },
  695. },
  696. [ C(L1I ) ] = {
  697. [ C(OP_READ) ] = {
  698. [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
  699. [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
  700. },
  701. [ C(OP_WRITE) ] = {
  702. [ C(RESULT_ACCESS) ] = -1,
  703. [ C(RESULT_MISS) ] = -1,
  704. },
  705. [ C(OP_PREFETCH) ] = {
  706. [ C(RESULT_ACCESS) ] = 0,
  707. [ C(RESULT_MISS) ] = 0,
  708. },
  709. },
  710. [ C(LL ) ] = {
  711. [ C(OP_READ) ] = {
  712. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  713. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  714. },
  715. [ C(OP_WRITE) ] = {
  716. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  717. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  718. },
  719. [ C(OP_PREFETCH) ] = {
  720. [ C(RESULT_ACCESS) ] = 0,
  721. [ C(RESULT_MISS) ] = 0,
  722. },
  723. },
  724. [ C(DTLB) ] = {
  725. [ C(OP_READ) ] = {
  726. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  727. [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
  728. },
  729. [ C(OP_WRITE) ] = {
  730. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  731. [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
  732. },
  733. [ C(OP_PREFETCH) ] = {
  734. [ C(RESULT_ACCESS) ] = 0,
  735. [ C(RESULT_MISS) ] = 0,
  736. },
  737. },
  738. [ C(ITLB) ] = {
  739. [ C(OP_READ) ] = {
  740. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  741. [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
  742. },
  743. [ C(OP_WRITE) ] = {
  744. [ C(RESULT_ACCESS) ] = -1,
  745. [ C(RESULT_MISS) ] = -1,
  746. },
  747. [ C(OP_PREFETCH) ] = {
  748. [ C(RESULT_ACCESS) ] = -1,
  749. [ C(RESULT_MISS) ] = -1,
  750. },
  751. },
  752. [ C(BPU ) ] = {
  753. [ C(OP_READ) ] = {
  754. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  755. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  756. },
  757. [ C(OP_WRITE) ] = {
  758. [ C(RESULT_ACCESS) ] = -1,
  759. [ C(RESULT_MISS) ] = -1,
  760. },
  761. [ C(OP_PREFETCH) ] = {
  762. [ C(RESULT_ACCESS) ] = -1,
  763. [ C(RESULT_MISS) ] = -1,
  764. },
  765. },
  766. };
  767. static __initconst const u64 atom_hw_cache_event_ids
  768. [PERF_COUNT_HW_CACHE_MAX]
  769. [PERF_COUNT_HW_CACHE_OP_MAX]
  770. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  771. {
  772. [ C(L1D) ] = {
  773. [ C(OP_READ) ] = {
  774. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
  775. [ C(RESULT_MISS) ] = 0,
  776. },
  777. [ C(OP_WRITE) ] = {
  778. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
  779. [ C(RESULT_MISS) ] = 0,
  780. },
  781. [ C(OP_PREFETCH) ] = {
  782. [ C(RESULT_ACCESS) ] = 0x0,
  783. [ C(RESULT_MISS) ] = 0,
  784. },
  785. },
  786. [ C(L1I ) ] = {
  787. [ C(OP_READ) ] = {
  788. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  789. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  790. },
  791. [ C(OP_WRITE) ] = {
  792. [ C(RESULT_ACCESS) ] = -1,
  793. [ C(RESULT_MISS) ] = -1,
  794. },
  795. [ C(OP_PREFETCH) ] = {
  796. [ C(RESULT_ACCESS) ] = 0,
  797. [ C(RESULT_MISS) ] = 0,
  798. },
  799. },
  800. [ C(LL ) ] = {
  801. [ C(OP_READ) ] = {
  802. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  803. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  804. },
  805. [ C(OP_WRITE) ] = {
  806. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  807. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  808. },
  809. [ C(OP_PREFETCH) ] = {
  810. [ C(RESULT_ACCESS) ] = 0,
  811. [ C(RESULT_MISS) ] = 0,
  812. },
  813. },
  814. [ C(DTLB) ] = {
  815. [ C(OP_READ) ] = {
  816. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
  817. [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
  818. },
  819. [ C(OP_WRITE) ] = {
  820. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
  821. [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
  822. },
  823. [ C(OP_PREFETCH) ] = {
  824. [ C(RESULT_ACCESS) ] = 0,
  825. [ C(RESULT_MISS) ] = 0,
  826. },
  827. },
  828. [ C(ITLB) ] = {
  829. [ C(OP_READ) ] = {
  830. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  831. [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
  832. },
  833. [ C(OP_WRITE) ] = {
  834. [ C(RESULT_ACCESS) ] = -1,
  835. [ C(RESULT_MISS) ] = -1,
  836. },
  837. [ C(OP_PREFETCH) ] = {
  838. [ C(RESULT_ACCESS) ] = -1,
  839. [ C(RESULT_MISS) ] = -1,
  840. },
  841. },
  842. [ C(BPU ) ] = {
  843. [ C(OP_READ) ] = {
  844. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  845. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  846. },
  847. [ C(OP_WRITE) ] = {
  848. [ C(RESULT_ACCESS) ] = -1,
  849. [ C(RESULT_MISS) ] = -1,
  850. },
  851. [ C(OP_PREFETCH) ] = {
  852. [ C(RESULT_ACCESS) ] = -1,
  853. [ C(RESULT_MISS) ] = -1,
  854. },
  855. },
  856. };
  857. static struct extra_reg intel_slm_extra_regs[] __read_mostly =
  858. {
  859. /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
  860. INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffffull, RSP_0),
  861. INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x768005ffffull, RSP_1),
  862. EVENT_EXTRA_END
  863. };
  864. #define SLM_DMND_READ SNB_DMND_DATA_RD
  865. #define SLM_DMND_WRITE SNB_DMND_RFO
  866. #define SLM_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)
  867. #define SLM_SNP_ANY (SNB_SNP_NONE|SNB_SNP_MISS|SNB_NO_FWD|SNB_HITM)
  868. #define SLM_LLC_ACCESS SNB_RESP_ANY
  869. #define SLM_LLC_MISS (SLM_SNP_ANY|SNB_NON_DRAM)
  870. static __initconst const u64 slm_hw_cache_extra_regs
  871. [PERF_COUNT_HW_CACHE_MAX]
  872. [PERF_COUNT_HW_CACHE_OP_MAX]
  873. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  874. {
  875. [ C(LL ) ] = {
  876. [ C(OP_READ) ] = {
  877. [ C(RESULT_ACCESS) ] = SLM_DMND_READ|SLM_LLC_ACCESS,
  878. [ C(RESULT_MISS) ] = SLM_DMND_READ|SLM_LLC_MISS,
  879. },
  880. [ C(OP_WRITE) ] = {
  881. [ C(RESULT_ACCESS) ] = SLM_DMND_WRITE|SLM_LLC_ACCESS,
  882. [ C(RESULT_MISS) ] = SLM_DMND_WRITE|SLM_LLC_MISS,
  883. },
  884. [ C(OP_PREFETCH) ] = {
  885. [ C(RESULT_ACCESS) ] = SLM_DMND_PREFETCH|SLM_LLC_ACCESS,
  886. [ C(RESULT_MISS) ] = SLM_DMND_PREFETCH|SLM_LLC_MISS,
  887. },
  888. },
  889. };
  890. static __initconst const u64 slm_hw_cache_event_ids
  891. [PERF_COUNT_HW_CACHE_MAX]
  892. [PERF_COUNT_HW_CACHE_OP_MAX]
  893. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  894. {
  895. [ C(L1D) ] = {
  896. [ C(OP_READ) ] = {
  897. [ C(RESULT_ACCESS) ] = 0,
  898. [ C(RESULT_MISS) ] = 0x0104, /* LD_DCU_MISS */
  899. },
  900. [ C(OP_WRITE) ] = {
  901. [ C(RESULT_ACCESS) ] = 0,
  902. [ C(RESULT_MISS) ] = 0,
  903. },
  904. [ C(OP_PREFETCH) ] = {
  905. [ C(RESULT_ACCESS) ] = 0,
  906. [ C(RESULT_MISS) ] = 0,
  907. },
  908. },
  909. [ C(L1I ) ] = {
  910. [ C(OP_READ) ] = {
  911. [ C(RESULT_ACCESS) ] = 0x0380, /* ICACHE.ACCESSES */
  912. [ C(RESULT_MISS) ] = 0x0280, /* ICACGE.MISSES */
  913. },
  914. [ C(OP_WRITE) ] = {
  915. [ C(RESULT_ACCESS) ] = -1,
  916. [ C(RESULT_MISS) ] = -1,
  917. },
  918. [ C(OP_PREFETCH) ] = {
  919. [ C(RESULT_ACCESS) ] = 0,
  920. [ C(RESULT_MISS) ] = 0,
  921. },
  922. },
  923. [ C(LL ) ] = {
  924. [ C(OP_READ) ] = {
  925. /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
  926. [ C(RESULT_ACCESS) ] = 0x01b7,
  927. /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
  928. [ C(RESULT_MISS) ] = 0x01b7,
  929. },
  930. [ C(OP_WRITE) ] = {
  931. /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
  932. [ C(RESULT_ACCESS) ] = 0x01b7,
  933. /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
  934. [ C(RESULT_MISS) ] = 0x01b7,
  935. },
  936. [ C(OP_PREFETCH) ] = {
  937. /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
  938. [ C(RESULT_ACCESS) ] = 0x01b7,
  939. /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
  940. [ C(RESULT_MISS) ] = 0x01b7,
  941. },
  942. },
  943. [ C(DTLB) ] = {
  944. [ C(OP_READ) ] = {
  945. [ C(RESULT_ACCESS) ] = 0,
  946. [ C(RESULT_MISS) ] = 0x0804, /* LD_DTLB_MISS */
  947. },
  948. [ C(OP_WRITE) ] = {
  949. [ C(RESULT_ACCESS) ] = 0,
  950. [ C(RESULT_MISS) ] = 0,
  951. },
  952. [ C(OP_PREFETCH) ] = {
  953. [ C(RESULT_ACCESS) ] = 0,
  954. [ C(RESULT_MISS) ] = 0,
  955. },
  956. },
  957. [ C(ITLB) ] = {
  958. [ C(OP_READ) ] = {
  959. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  960. [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
  961. },
  962. [ C(OP_WRITE) ] = {
  963. [ C(RESULT_ACCESS) ] = -1,
  964. [ C(RESULT_MISS) ] = -1,
  965. },
  966. [ C(OP_PREFETCH) ] = {
  967. [ C(RESULT_ACCESS) ] = -1,
  968. [ C(RESULT_MISS) ] = -1,
  969. },
  970. },
  971. [ C(BPU ) ] = {
  972. [ C(OP_READ) ] = {
  973. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  974. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  975. },
  976. [ C(OP_WRITE) ] = {
  977. [ C(RESULT_ACCESS) ] = -1,
  978. [ C(RESULT_MISS) ] = -1,
  979. },
  980. [ C(OP_PREFETCH) ] = {
  981. [ C(RESULT_ACCESS) ] = -1,
  982. [ C(RESULT_MISS) ] = -1,
  983. },
  984. },
  985. };
  986. static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
  987. {
  988. /* user explicitly requested branch sampling */
  989. if (has_branch_stack(event))
  990. return true;
  991. /* implicit branch sampling to correct PEBS skid */
  992. if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1 &&
  993. x86_pmu.intel_cap.pebs_format < 2)
  994. return true;
  995. return false;
  996. }
  997. static void intel_pmu_disable_all(void)
  998. {
  999. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1000. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
  1001. if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
  1002. intel_pmu_disable_bts();
  1003. intel_pmu_pebs_disable_all();
  1004. intel_pmu_lbr_disable_all();
  1005. }
  1006. static void intel_pmu_enable_all(int added)
  1007. {
  1008. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1009. intel_pmu_pebs_enable_all();
  1010. intel_pmu_lbr_enable_all();
  1011. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
  1012. x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
  1013. if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
  1014. struct perf_event *event =
  1015. cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
  1016. if (WARN_ON_ONCE(!event))
  1017. return;
  1018. intel_pmu_enable_bts(event->hw.config);
  1019. }
  1020. }
  1021. /*
  1022. * Workaround for:
  1023. * Intel Errata AAK100 (model 26)
  1024. * Intel Errata AAP53 (model 30)
  1025. * Intel Errata BD53 (model 44)
  1026. *
  1027. * The official story:
  1028. * These chips need to be 'reset' when adding counters by programming the
  1029. * magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either
  1030. * in sequence on the same PMC or on different PMCs.
  1031. *
  1032. * In practise it appears some of these events do in fact count, and
  1033. * we need to programm all 4 events.
  1034. */
  1035. static void intel_pmu_nhm_workaround(void)
  1036. {
  1037. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1038. static const unsigned long nhm_magic[4] = {
  1039. 0x4300B5,
  1040. 0x4300D2,
  1041. 0x4300B1,
  1042. 0x4300B1
  1043. };
  1044. struct perf_event *event;
  1045. int i;
  1046. /*
  1047. * The Errata requires below steps:
  1048. * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL;
  1049. * 2) Configure 4 PERFEVTSELx with the magic events and clear
  1050. * the corresponding PMCx;
  1051. * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL;
  1052. * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL;
  1053. * 5) Clear 4 pairs of ERFEVTSELx and PMCx;
  1054. */
  1055. /*
  1056. * The real steps we choose are a little different from above.
  1057. * A) To reduce MSR operations, we don't run step 1) as they
  1058. * are already cleared before this function is called;
  1059. * B) Call x86_perf_event_update to save PMCx before configuring
  1060. * PERFEVTSELx with magic number;
  1061. * C) With step 5), we do clear only when the PERFEVTSELx is
  1062. * not used currently.
  1063. * D) Call x86_perf_event_set_period to restore PMCx;
  1064. */
  1065. /* We always operate 4 pairs of PERF Counters */
  1066. for (i = 0; i < 4; i++) {
  1067. event = cpuc->events[i];
  1068. if (event)
  1069. x86_perf_event_update(event);
  1070. }
  1071. for (i = 0; i < 4; i++) {
  1072. wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
  1073. wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
  1074. }
  1075. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
  1076. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
  1077. for (i = 0; i < 4; i++) {
  1078. event = cpuc->events[i];
  1079. if (event) {
  1080. x86_perf_event_set_period(event);
  1081. __x86_pmu_enable_event(&event->hw,
  1082. ARCH_PERFMON_EVENTSEL_ENABLE);
  1083. } else
  1084. wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
  1085. }
  1086. }
  1087. static void intel_pmu_nhm_enable_all(int added)
  1088. {
  1089. if (added)
  1090. intel_pmu_nhm_workaround();
  1091. intel_pmu_enable_all(added);
  1092. }
  1093. static inline u64 intel_pmu_get_status(void)
  1094. {
  1095. u64 status;
  1096. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  1097. return status;
  1098. }
  1099. static inline void intel_pmu_ack_status(u64 ack)
  1100. {
  1101. wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
  1102. }
  1103. static void intel_pmu_disable_fixed(struct hw_perf_event *hwc)
  1104. {
  1105. int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
  1106. u64 ctrl_val, mask;
  1107. mask = 0xfULL << (idx * 4);
  1108. rdmsrl(hwc->config_base, ctrl_val);
  1109. ctrl_val &= ~mask;
  1110. wrmsrl(hwc->config_base, ctrl_val);
  1111. }
  1112. static void intel_pmu_disable_event(struct perf_event *event)
  1113. {
  1114. struct hw_perf_event *hwc = &event->hw;
  1115. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1116. if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
  1117. intel_pmu_disable_bts();
  1118. intel_pmu_drain_bts_buffer();
  1119. return;
  1120. }
  1121. cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
  1122. cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
  1123. /*
  1124. * must disable before any actual event
  1125. * because any event may be combined with LBR
  1126. */
  1127. if (intel_pmu_needs_lbr_smpl(event))
  1128. intel_pmu_lbr_disable(event);
  1129. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1130. intel_pmu_disable_fixed(hwc);
  1131. return;
  1132. }
  1133. x86_pmu_disable_event(event);
  1134. if (unlikely(event->attr.precise_ip))
  1135. intel_pmu_pebs_disable(event);
  1136. }
  1137. static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
  1138. {
  1139. int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
  1140. u64 ctrl_val, bits, mask;
  1141. /*
  1142. * Enable IRQ generation (0x8),
  1143. * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
  1144. * if requested:
  1145. */
  1146. bits = 0x8ULL;
  1147. if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
  1148. bits |= 0x2;
  1149. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  1150. bits |= 0x1;
  1151. /*
  1152. * ANY bit is supported in v3 and up
  1153. */
  1154. if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
  1155. bits |= 0x4;
  1156. bits <<= (idx * 4);
  1157. mask = 0xfULL << (idx * 4);
  1158. rdmsrl(hwc->config_base, ctrl_val);
  1159. ctrl_val &= ~mask;
  1160. ctrl_val |= bits;
  1161. wrmsrl(hwc->config_base, ctrl_val);
  1162. }
  1163. static void intel_pmu_enable_event(struct perf_event *event)
  1164. {
  1165. struct hw_perf_event *hwc = &event->hw;
  1166. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1167. if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
  1168. if (!__this_cpu_read(cpu_hw_events.enabled))
  1169. return;
  1170. intel_pmu_enable_bts(hwc->config);
  1171. return;
  1172. }
  1173. /*
  1174. * must enabled before any actual event
  1175. * because any event may be combined with LBR
  1176. */
  1177. if (intel_pmu_needs_lbr_smpl(event))
  1178. intel_pmu_lbr_enable(event);
  1179. if (event->attr.exclude_host)
  1180. cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
  1181. if (event->attr.exclude_guest)
  1182. cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);
  1183. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1184. intel_pmu_enable_fixed(hwc);
  1185. return;
  1186. }
  1187. if (unlikely(event->attr.precise_ip))
  1188. intel_pmu_pebs_enable(event);
  1189. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  1190. }
  1191. /*
  1192. * Save and restart an expired event. Called by NMI contexts,
  1193. * so it has to be careful about preempting normal event ops:
  1194. */
  1195. int intel_pmu_save_and_restart(struct perf_event *event)
  1196. {
  1197. x86_perf_event_update(event);
  1198. return x86_perf_event_set_period(event);
  1199. }
  1200. static void intel_pmu_reset(void)
  1201. {
  1202. struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
  1203. unsigned long flags;
  1204. int idx;
  1205. if (!x86_pmu.num_counters)
  1206. return;
  1207. local_irq_save(flags);
  1208. pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());
  1209. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1210. wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
  1211. wrmsrl_safe(x86_pmu_event_addr(idx), 0ull);
  1212. }
  1213. for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
  1214. wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
  1215. if (ds)
  1216. ds->bts_index = ds->bts_buffer_base;
  1217. local_irq_restore(flags);
  1218. }
  1219. /*
  1220. * This handler is triggered by the local APIC, so the APIC IRQ handling
  1221. * rules apply:
  1222. */
  1223. static int intel_pmu_handle_irq(struct pt_regs *regs)
  1224. {
  1225. struct perf_sample_data data;
  1226. struct cpu_hw_events *cpuc;
  1227. int bit, loops;
  1228. u64 status;
  1229. int handled;
  1230. cpuc = &__get_cpu_var(cpu_hw_events);
  1231. /*
  1232. * No known reason to not always do late ACK,
  1233. * but just in case do it opt-in.
  1234. */
  1235. if (!x86_pmu.late_ack)
  1236. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1237. intel_pmu_disable_all();
  1238. handled = intel_pmu_drain_bts_buffer();
  1239. status = intel_pmu_get_status();
  1240. if (!status) {
  1241. intel_pmu_enable_all(0);
  1242. return handled;
  1243. }
  1244. loops = 0;
  1245. again:
  1246. intel_pmu_ack_status(status);
  1247. if (++loops > 100) {
  1248. static bool warned = false;
  1249. if (!warned) {
  1250. WARN(1, "perfevents: irq loop stuck!\n");
  1251. perf_event_print_debug();
  1252. warned = true;
  1253. }
  1254. intel_pmu_reset();
  1255. goto done;
  1256. }
  1257. inc_irq_stat(apic_perf_irqs);
  1258. intel_pmu_lbr_read();
  1259. /*
  1260. * PEBS overflow sets bit 62 in the global status register
  1261. */
  1262. if (__test_and_clear_bit(62, (unsigned long *)&status)) {
  1263. handled++;
  1264. x86_pmu.drain_pebs(regs);
  1265. }
  1266. for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
  1267. struct perf_event *event = cpuc->events[bit];
  1268. handled++;
  1269. if (!test_bit(bit, cpuc->active_mask))
  1270. continue;
  1271. if (!intel_pmu_save_and_restart(event))
  1272. continue;
  1273. perf_sample_data_init(&data, 0, event->hw.last_period);
  1274. if (has_branch_stack(event))
  1275. data.br_stack = &cpuc->lbr_stack;
  1276. if (perf_event_overflow(event, &data, regs))
  1277. x86_pmu_stop(event, 0);
  1278. }
  1279. /*
  1280. * Repeat if there is more work to be done:
  1281. */
  1282. status = intel_pmu_get_status();
  1283. if (status)
  1284. goto again;
  1285. done:
  1286. intel_pmu_enable_all(0);
  1287. /*
  1288. * Only unmask the NMI after the overflow counters
  1289. * have been reset. This avoids spurious NMIs on
  1290. * Haswell CPUs.
  1291. */
  1292. if (x86_pmu.late_ack)
  1293. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1294. return handled;
  1295. }
  1296. static struct event_constraint *
  1297. intel_bts_constraints(struct perf_event *event)
  1298. {
  1299. struct hw_perf_event *hwc = &event->hw;
  1300. unsigned int hw_event, bts_event;
  1301. if (event->attr.freq)
  1302. return NULL;
  1303. hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
  1304. bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
  1305. if (unlikely(hw_event == bts_event && hwc->sample_period == 1))
  1306. return &bts_constraint;
  1307. return NULL;
  1308. }
  1309. static int intel_alt_er(int idx)
  1310. {
  1311. if (!(x86_pmu.er_flags & ERF_HAS_RSP_1))
  1312. return idx;
  1313. if (idx == EXTRA_REG_RSP_0)
  1314. return EXTRA_REG_RSP_1;
  1315. if (idx == EXTRA_REG_RSP_1)
  1316. return EXTRA_REG_RSP_0;
  1317. return idx;
  1318. }
  1319. static void intel_fixup_er(struct perf_event *event, int idx)
  1320. {
  1321. event->hw.extra_reg.idx = idx;
  1322. if (idx == EXTRA_REG_RSP_0) {
  1323. event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
  1324. event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_0].event;
  1325. event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0;
  1326. } else if (idx == EXTRA_REG_RSP_1) {
  1327. event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
  1328. event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_1].event;
  1329. event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1;
  1330. }
  1331. }
  1332. /*
  1333. * manage allocation of shared extra msr for certain events
  1334. *
  1335. * sharing can be:
  1336. * per-cpu: to be shared between the various events on a single PMU
  1337. * per-core: per-cpu + shared by HT threads
  1338. */
  1339. static struct event_constraint *
  1340. __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
  1341. struct perf_event *event,
  1342. struct hw_perf_event_extra *reg)
  1343. {
  1344. struct event_constraint *c = &emptyconstraint;
  1345. struct er_account *era;
  1346. unsigned long flags;
  1347. int idx = reg->idx;
  1348. /*
  1349. * reg->alloc can be set due to existing state, so for fake cpuc we
  1350. * need to ignore this, otherwise we might fail to allocate proper fake
  1351. * state for this extra reg constraint. Also see the comment below.
  1352. */
  1353. if (reg->alloc && !cpuc->is_fake)
  1354. return NULL; /* call x86_get_event_constraint() */
  1355. again:
  1356. era = &cpuc->shared_regs->regs[idx];
  1357. /*
  1358. * we use spin_lock_irqsave() to avoid lockdep issues when
  1359. * passing a fake cpuc
  1360. */
  1361. raw_spin_lock_irqsave(&era->lock, flags);
  1362. if (!atomic_read(&era->ref) || era->config == reg->config) {
  1363. /*
  1364. * If its a fake cpuc -- as per validate_{group,event}() we
  1365. * shouldn't touch event state and we can avoid doing so
  1366. * since both will only call get_event_constraints() once
  1367. * on each event, this avoids the need for reg->alloc.
  1368. *
  1369. * Not doing the ER fixup will only result in era->reg being
  1370. * wrong, but since we won't actually try and program hardware
  1371. * this isn't a problem either.
  1372. */
  1373. if (!cpuc->is_fake) {
  1374. if (idx != reg->idx)
  1375. intel_fixup_er(event, idx);
  1376. /*
  1377. * x86_schedule_events() can call get_event_constraints()
  1378. * multiple times on events in the case of incremental
  1379. * scheduling(). reg->alloc ensures we only do the ER
  1380. * allocation once.
  1381. */
  1382. reg->alloc = 1;
  1383. }
  1384. /* lock in msr value */
  1385. era->config = reg->config;
  1386. era->reg = reg->reg;
  1387. /* one more user */
  1388. atomic_inc(&era->ref);
  1389. /*
  1390. * need to call x86_get_event_constraint()
  1391. * to check if associated event has constraints
  1392. */
  1393. c = NULL;
  1394. } else {
  1395. idx = intel_alt_er(idx);
  1396. if (idx != reg->idx) {
  1397. raw_spin_unlock_irqrestore(&era->lock, flags);
  1398. goto again;
  1399. }
  1400. }
  1401. raw_spin_unlock_irqrestore(&era->lock, flags);
  1402. return c;
  1403. }
  1404. static void
  1405. __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,
  1406. struct hw_perf_event_extra *reg)
  1407. {
  1408. struct er_account *era;
  1409. /*
  1410. * Only put constraint if extra reg was actually allocated. Also takes
  1411. * care of event which do not use an extra shared reg.
  1412. *
  1413. * Also, if this is a fake cpuc we shouldn't touch any event state
  1414. * (reg->alloc) and we don't care about leaving inconsistent cpuc state
  1415. * either since it'll be thrown out.
  1416. */
  1417. if (!reg->alloc || cpuc->is_fake)
  1418. return;
  1419. era = &cpuc->shared_regs->regs[reg->idx];
  1420. /* one fewer user */
  1421. atomic_dec(&era->ref);
  1422. /* allocate again next time */
  1423. reg->alloc = 0;
  1424. }
  1425. static struct event_constraint *
  1426. intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
  1427. struct perf_event *event)
  1428. {
  1429. struct event_constraint *c = NULL, *d;
  1430. struct hw_perf_event_extra *xreg, *breg;
  1431. xreg = &event->hw.extra_reg;
  1432. if (xreg->idx != EXTRA_REG_NONE) {
  1433. c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
  1434. if (c == &emptyconstraint)
  1435. return c;
  1436. }
  1437. breg = &event->hw.branch_reg;
  1438. if (breg->idx != EXTRA_REG_NONE) {
  1439. d = __intel_shared_reg_get_constraints(cpuc, event, breg);
  1440. if (d == &emptyconstraint) {
  1441. __intel_shared_reg_put_constraints(cpuc, xreg);
  1442. c = d;
  1443. }
  1444. }
  1445. return c;
  1446. }
  1447. struct event_constraint *
  1448. x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1449. {
  1450. struct event_constraint *c;
  1451. if (x86_pmu.event_constraints) {
  1452. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1453. if ((event->hw.config & c->cmask) == c->code) {
  1454. event->hw.flags |= c->flags;
  1455. return c;
  1456. }
  1457. }
  1458. }
  1459. return &unconstrained;
  1460. }
  1461. static struct event_constraint *
  1462. intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1463. {
  1464. struct event_constraint *c;
  1465. c = intel_bts_constraints(event);
  1466. if (c)
  1467. return c;
  1468. c = intel_pebs_constraints(event);
  1469. if (c)
  1470. return c;
  1471. c = intel_shared_regs_constraints(cpuc, event);
  1472. if (c)
  1473. return c;
  1474. return x86_get_event_constraints(cpuc, event);
  1475. }
  1476. static void
  1477. intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
  1478. struct perf_event *event)
  1479. {
  1480. struct hw_perf_event_extra *reg;
  1481. reg = &event->hw.extra_reg;
  1482. if (reg->idx != EXTRA_REG_NONE)
  1483. __intel_shared_reg_put_constraints(cpuc, reg);
  1484. reg = &event->hw.branch_reg;
  1485. if (reg->idx != EXTRA_REG_NONE)
  1486. __intel_shared_reg_put_constraints(cpuc, reg);
  1487. }
  1488. static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
  1489. struct perf_event *event)
  1490. {
  1491. intel_put_shared_regs_event_constraints(cpuc, event);
  1492. }
  1493. static void intel_pebs_aliases_core2(struct perf_event *event)
  1494. {
  1495. if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
  1496. /*
  1497. * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
  1498. * (0x003c) so that we can use it with PEBS.
  1499. *
  1500. * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
  1501. * PEBS capable. However we can use INST_RETIRED.ANY_P
  1502. * (0x00c0), which is a PEBS capable event, to get the same
  1503. * count.
  1504. *
  1505. * INST_RETIRED.ANY_P counts the number of cycles that retires
  1506. * CNTMASK instructions. By setting CNTMASK to a value (16)
  1507. * larger than the maximum number of instructions that can be
  1508. * retired per cycle (4) and then inverting the condition, we
  1509. * count all cycles that retire 16 or less instructions, which
  1510. * is every cycle.
  1511. *
  1512. * Thereby we gain a PEBS capable cycle counter.
  1513. */
  1514. u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16);
  1515. alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
  1516. event->hw.config = alt_config;
  1517. }
  1518. }
  1519. static void intel_pebs_aliases_snb(struct perf_event *event)
  1520. {
  1521. if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
  1522. /*
  1523. * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
  1524. * (0x003c) so that we can use it with PEBS.
  1525. *
  1526. * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
  1527. * PEBS capable. However we can use UOPS_RETIRED.ALL
  1528. * (0x01c2), which is a PEBS capable event, to get the same
  1529. * count.
  1530. *
  1531. * UOPS_RETIRED.ALL counts the number of cycles that retires
  1532. * CNTMASK micro-ops. By setting CNTMASK to a value (16)
  1533. * larger than the maximum number of micro-ops that can be
  1534. * retired per cycle (4) and then inverting the condition, we
  1535. * count all cycles that retire 16 or less micro-ops, which
  1536. * is every cycle.
  1537. *
  1538. * Thereby we gain a PEBS capable cycle counter.
  1539. */
  1540. u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16);
  1541. alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
  1542. event->hw.config = alt_config;
  1543. }
  1544. }
  1545. static int intel_pmu_hw_config(struct perf_event *event)
  1546. {
  1547. int ret = x86_pmu_hw_config(event);
  1548. if (ret)
  1549. return ret;
  1550. if (event->attr.precise_ip && x86_pmu.pebs_aliases)
  1551. x86_pmu.pebs_aliases(event);
  1552. if (intel_pmu_needs_lbr_smpl(event)) {
  1553. ret = intel_pmu_setup_lbr_filter(event);
  1554. if (ret)
  1555. return ret;
  1556. }
  1557. if (event->attr.type != PERF_TYPE_RAW)
  1558. return 0;
  1559. if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
  1560. return 0;
  1561. if (x86_pmu.version < 3)
  1562. return -EINVAL;
  1563. if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
  1564. return -EACCES;
  1565. event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;
  1566. return 0;
  1567. }
  1568. struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
  1569. {
  1570. if (x86_pmu.guest_get_msrs)
  1571. return x86_pmu.guest_get_msrs(nr);
  1572. *nr = 0;
  1573. return NULL;
  1574. }
  1575. EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
  1576. static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
  1577. {
  1578. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1579. struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
  1580. arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
  1581. arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;
  1582. arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;
  1583. /*
  1584. * If PMU counter has PEBS enabled it is not enough to disable counter
  1585. * on a guest entry since PEBS memory write can overshoot guest entry
  1586. * and corrupt guest memory. Disabling PEBS solves the problem.
  1587. */
  1588. arr[1].msr = MSR_IA32_PEBS_ENABLE;
  1589. arr[1].host = cpuc->pebs_enabled;
  1590. arr[1].guest = 0;
  1591. *nr = 2;
  1592. return arr;
  1593. }
  1594. static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr)
  1595. {
  1596. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1597. struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
  1598. int idx;
  1599. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1600. struct perf_event *event = cpuc->events[idx];
  1601. arr[idx].msr = x86_pmu_config_addr(idx);
  1602. arr[idx].host = arr[idx].guest = 0;
  1603. if (!test_bit(idx, cpuc->active_mask))
  1604. continue;
  1605. arr[idx].host = arr[idx].guest =
  1606. event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE;
  1607. if (event->attr.exclude_host)
  1608. arr[idx].host &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  1609. else if (event->attr.exclude_guest)
  1610. arr[idx].guest &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  1611. }
  1612. *nr = x86_pmu.num_counters;
  1613. return arr;
  1614. }
  1615. static void core_pmu_enable_event(struct perf_event *event)
  1616. {
  1617. if (!event->attr.exclude_host)
  1618. x86_pmu_enable_event(event);
  1619. }
  1620. static void core_pmu_enable_all(int added)
  1621. {
  1622. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1623. int idx;
  1624. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1625. struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
  1626. if (!test_bit(idx, cpuc->active_mask) ||
  1627. cpuc->events[idx]->attr.exclude_host)
  1628. continue;
  1629. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  1630. }
  1631. }
  1632. static int hsw_hw_config(struct perf_event *event)
  1633. {
  1634. int ret = intel_pmu_hw_config(event);
  1635. if (ret)
  1636. return ret;
  1637. if (!boot_cpu_has(X86_FEATURE_RTM) && !boot_cpu_has(X86_FEATURE_HLE))
  1638. return 0;
  1639. event->hw.config |= event->attr.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
  1640. /*
  1641. * IN_TX/IN_TX-CP filters are not supported by the Haswell PMU with
  1642. * PEBS or in ANY thread mode. Since the results are non-sensical forbid
  1643. * this combination.
  1644. */
  1645. if ((event->hw.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)) &&
  1646. ((event->hw.config & ARCH_PERFMON_EVENTSEL_ANY) ||
  1647. event->attr.precise_ip > 0))
  1648. return -EOPNOTSUPP;
  1649. return 0;
  1650. }
  1651. static struct event_constraint counter2_constraint =
  1652. EVENT_CONSTRAINT(0, 0x4, 0);
  1653. static struct event_constraint *
  1654. hsw_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1655. {
  1656. struct event_constraint *c = intel_get_event_constraints(cpuc, event);
  1657. /* Handle special quirk on in_tx_checkpointed only in counter 2 */
  1658. if (event->hw.config & HSW_IN_TX_CHECKPOINTED) {
  1659. if (c->idxmsk64 & (1U << 2))
  1660. return &counter2_constraint;
  1661. return &emptyconstraint;
  1662. }
  1663. return c;
  1664. }
  1665. PMU_FORMAT_ATTR(event, "config:0-7" );
  1666. PMU_FORMAT_ATTR(umask, "config:8-15" );
  1667. PMU_FORMAT_ATTR(edge, "config:18" );
  1668. PMU_FORMAT_ATTR(pc, "config:19" );
  1669. PMU_FORMAT_ATTR(any, "config:21" ); /* v3 + */
  1670. PMU_FORMAT_ATTR(inv, "config:23" );
  1671. PMU_FORMAT_ATTR(cmask, "config:24-31" );
  1672. PMU_FORMAT_ATTR(in_tx, "config:32");
  1673. PMU_FORMAT_ATTR(in_tx_cp, "config:33");
  1674. static struct attribute *intel_arch_formats_attr[] = {
  1675. &format_attr_event.attr,
  1676. &format_attr_umask.attr,
  1677. &format_attr_edge.attr,
  1678. &format_attr_pc.attr,
  1679. &format_attr_inv.attr,
  1680. &format_attr_cmask.attr,
  1681. NULL,
  1682. };
  1683. ssize_t intel_event_sysfs_show(char *page, u64 config)
  1684. {
  1685. u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
  1686. return x86_event_sysfs_show(page, config, event);
  1687. }
  1688. static __initconst const struct x86_pmu core_pmu = {
  1689. .name = "core",
  1690. .handle_irq = x86_pmu_handle_irq,
  1691. .disable_all = x86_pmu_disable_all,
  1692. .enable_all = core_pmu_enable_all,
  1693. .enable = core_pmu_enable_event,
  1694. .disable = x86_pmu_disable_event,
  1695. .hw_config = x86_pmu_hw_config,
  1696. .schedule_events = x86_schedule_events,
  1697. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1698. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1699. .event_map = intel_pmu_event_map,
  1700. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1701. .apic = 1,
  1702. /*
  1703. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1704. * so we install an artificial 1<<31 period regardless of
  1705. * the generic event period:
  1706. */
  1707. .max_period = (1ULL << 31) - 1,
  1708. .get_event_constraints = intel_get_event_constraints,
  1709. .put_event_constraints = intel_put_event_constraints,
  1710. .event_constraints = intel_core_event_constraints,
  1711. .guest_get_msrs = core_guest_get_msrs,
  1712. .format_attrs = intel_arch_formats_attr,
  1713. .events_sysfs_show = intel_event_sysfs_show,
  1714. };
  1715. struct intel_shared_regs *allocate_shared_regs(int cpu)
  1716. {
  1717. struct intel_shared_regs *regs;
  1718. int i;
  1719. regs = kzalloc_node(sizeof(struct intel_shared_regs),
  1720. GFP_KERNEL, cpu_to_node(cpu));
  1721. if (regs) {
  1722. /*
  1723. * initialize the locks to keep lockdep happy
  1724. */
  1725. for (i = 0; i < EXTRA_REG_MAX; i++)
  1726. raw_spin_lock_init(&regs->regs[i].lock);
  1727. regs->core_id = -1;
  1728. }
  1729. return regs;
  1730. }
  1731. static int intel_pmu_cpu_prepare(int cpu)
  1732. {
  1733. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1734. if (!(x86_pmu.extra_regs || x86_pmu.lbr_sel_map))
  1735. return NOTIFY_OK;
  1736. cpuc->shared_regs = allocate_shared_regs(cpu);
  1737. if (!cpuc->shared_regs)
  1738. return NOTIFY_BAD;
  1739. return NOTIFY_OK;
  1740. }
  1741. static void intel_pmu_cpu_starting(int cpu)
  1742. {
  1743. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1744. int core_id = topology_core_id(cpu);
  1745. int i;
  1746. init_debug_store_on_cpu(cpu);
  1747. /*
  1748. * Deal with CPUs that don't clear their LBRs on power-up.
  1749. */
  1750. intel_pmu_lbr_reset();
  1751. cpuc->lbr_sel = NULL;
  1752. if (!cpuc->shared_regs)
  1753. return;
  1754. if (!(x86_pmu.er_flags & ERF_NO_HT_SHARING)) {
  1755. for_each_cpu(i, topology_thread_cpumask(cpu)) {
  1756. struct intel_shared_regs *pc;
  1757. pc = per_cpu(cpu_hw_events, i).shared_regs;
  1758. if (pc && pc->core_id == core_id) {
  1759. cpuc->kfree_on_online = cpuc->shared_regs;
  1760. cpuc->shared_regs = pc;
  1761. break;
  1762. }
  1763. }
  1764. cpuc->shared_regs->core_id = core_id;
  1765. cpuc->shared_regs->refcnt++;
  1766. }
  1767. if (x86_pmu.lbr_sel_map)
  1768. cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
  1769. }
  1770. static void intel_pmu_cpu_dying(int cpu)
  1771. {
  1772. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1773. struct intel_shared_regs *pc;
  1774. pc = cpuc->shared_regs;
  1775. if (pc) {
  1776. if (pc->core_id == -1 || --pc->refcnt == 0)
  1777. kfree(pc);
  1778. cpuc->shared_regs = NULL;
  1779. }
  1780. fini_debug_store_on_cpu(cpu);
  1781. }
  1782. static void intel_pmu_flush_branch_stack(void)
  1783. {
  1784. /*
  1785. * Intel LBR does not tag entries with the
  1786. * PID of the current task, then we need to
  1787. * flush it on ctxsw
  1788. * For now, we simply reset it
  1789. */
  1790. if (x86_pmu.lbr_nr)
  1791. intel_pmu_lbr_reset();
  1792. }
  1793. PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
  1794. PMU_FORMAT_ATTR(ldlat, "config1:0-15");
  1795. static struct attribute *intel_arch3_formats_attr[] = {
  1796. &format_attr_event.attr,
  1797. &format_attr_umask.attr,
  1798. &format_attr_edge.attr,
  1799. &format_attr_pc.attr,
  1800. &format_attr_any.attr,
  1801. &format_attr_inv.attr,
  1802. &format_attr_cmask.attr,
  1803. &format_attr_in_tx.attr,
  1804. &format_attr_in_tx_cp.attr,
  1805. &format_attr_offcore_rsp.attr, /* XXX do NHM/WSM + SNB breakout */
  1806. &format_attr_ldlat.attr, /* PEBS load latency */
  1807. NULL,
  1808. };
  1809. static __initconst const struct x86_pmu intel_pmu = {
  1810. .name = "Intel",
  1811. .handle_irq = intel_pmu_handle_irq,
  1812. .disable_all = intel_pmu_disable_all,
  1813. .enable_all = intel_pmu_enable_all,
  1814. .enable = intel_pmu_enable_event,
  1815. .disable = intel_pmu_disable_event,
  1816. .hw_config = intel_pmu_hw_config,
  1817. .schedule_events = x86_schedule_events,
  1818. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1819. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1820. .event_map = intel_pmu_event_map,
  1821. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1822. .apic = 1,
  1823. /*
  1824. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1825. * so we install an artificial 1<<31 period regardless of
  1826. * the generic event period:
  1827. */
  1828. .max_period = (1ULL << 31) - 1,
  1829. .get_event_constraints = intel_get_event_constraints,
  1830. .put_event_constraints = intel_put_event_constraints,
  1831. .pebs_aliases = intel_pebs_aliases_core2,
  1832. .format_attrs = intel_arch3_formats_attr,
  1833. .events_sysfs_show = intel_event_sysfs_show,
  1834. .cpu_prepare = intel_pmu_cpu_prepare,
  1835. .cpu_starting = intel_pmu_cpu_starting,
  1836. .cpu_dying = intel_pmu_cpu_dying,
  1837. .guest_get_msrs = intel_guest_get_msrs,
  1838. .flush_branch_stack = intel_pmu_flush_branch_stack,
  1839. };
  1840. static __init void intel_clovertown_quirk(void)
  1841. {
  1842. /*
  1843. * PEBS is unreliable due to:
  1844. *
  1845. * AJ67 - PEBS may experience CPL leaks
  1846. * AJ68 - PEBS PMI may be delayed by one event
  1847. * AJ69 - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12]
  1848. * AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS
  1849. *
  1850. * AJ67 could be worked around by restricting the OS/USR flags.
  1851. * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI.
  1852. *
  1853. * AJ106 could possibly be worked around by not allowing LBR
  1854. * usage from PEBS, including the fixup.
  1855. * AJ68 could possibly be worked around by always programming
  1856. * a pebs_event_reset[0] value and coping with the lost events.
  1857. *
  1858. * But taken together it might just make sense to not enable PEBS on
  1859. * these chips.
  1860. */
  1861. pr_warn("PEBS disabled due to CPU errata\n");
  1862. x86_pmu.pebs = 0;
  1863. x86_pmu.pebs_constraints = NULL;
  1864. }
  1865. static int intel_snb_pebs_broken(int cpu)
  1866. {
  1867. u32 rev = UINT_MAX; /* default to broken for unknown models */
  1868. switch (cpu_data(cpu).x86_model) {
  1869. case 42: /* SNB */
  1870. rev = 0x28;
  1871. break;
  1872. case 45: /* SNB-EP */
  1873. switch (cpu_data(cpu).x86_mask) {
  1874. case 6: rev = 0x618; break;
  1875. case 7: rev = 0x70c; break;
  1876. }
  1877. }
  1878. return (cpu_data(cpu).microcode < rev);
  1879. }
  1880. static void intel_snb_check_microcode(void)
  1881. {
  1882. int pebs_broken = 0;
  1883. int cpu;
  1884. get_online_cpus();
  1885. for_each_online_cpu(cpu) {
  1886. if ((pebs_broken = intel_snb_pebs_broken(cpu)))
  1887. break;
  1888. }
  1889. put_online_cpus();
  1890. if (pebs_broken == x86_pmu.pebs_broken)
  1891. return;
  1892. /*
  1893. * Serialized by the microcode lock..
  1894. */
  1895. if (x86_pmu.pebs_broken) {
  1896. pr_info("PEBS enabled due to microcode update\n");
  1897. x86_pmu.pebs_broken = 0;
  1898. } else {
  1899. pr_info("PEBS disabled due to CPU errata, please upgrade microcode\n");
  1900. x86_pmu.pebs_broken = 1;
  1901. }
  1902. }
  1903. static __init void intel_sandybridge_quirk(void)
  1904. {
  1905. x86_pmu.check_microcode = intel_snb_check_microcode;
  1906. intel_snb_check_microcode();
  1907. }
  1908. static const struct { int id; char *name; } intel_arch_events_map[] __initconst = {
  1909. { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
  1910. { PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
  1911. { PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
  1912. { PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
  1913. { PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
  1914. { PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
  1915. { PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
  1916. };
  1917. static __init void intel_arch_events_quirk(void)
  1918. {
  1919. int bit;
  1920. /* disable event that reported as not presend by cpuid */
  1921. for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(intel_arch_events_map)) {
  1922. intel_perfmon_event_map[intel_arch_events_map[bit].id] = 0;
  1923. pr_warn("CPUID marked event: \'%s\' unavailable\n",
  1924. intel_arch_events_map[bit].name);
  1925. }
  1926. }
  1927. static __init void intel_nehalem_quirk(void)
  1928. {
  1929. union cpuid10_ebx ebx;
  1930. ebx.full = x86_pmu.events_maskl;
  1931. if (ebx.split.no_branch_misses_retired) {
  1932. /*
  1933. * Erratum AAJ80 detected, we work it around by using
  1934. * the BR_MISP_EXEC.ANY event. This will over-count
  1935. * branch-misses, but it's still much better than the
  1936. * architectural event which is often completely bogus:
  1937. */
  1938. intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89;
  1939. ebx.split.no_branch_misses_retired = 0;
  1940. x86_pmu.events_maskl = ebx.full;
  1941. pr_info("CPU erratum AAJ80 worked around\n");
  1942. }
  1943. }
  1944. EVENT_ATTR_STR(mem-loads, mem_ld_hsw, "event=0xcd,umask=0x1,ldlat=3");
  1945. EVENT_ATTR_STR(mem-stores, mem_st_hsw, "event=0xd0,umask=0x82")
  1946. static struct attribute *hsw_events_attrs[] = {
  1947. EVENT_PTR(mem_ld_hsw),
  1948. EVENT_PTR(mem_st_hsw),
  1949. NULL
  1950. };
  1951. __init int intel_pmu_init(void)
  1952. {
  1953. union cpuid10_edx edx;
  1954. union cpuid10_eax eax;
  1955. union cpuid10_ebx ebx;
  1956. struct event_constraint *c;
  1957. unsigned int unused;
  1958. int version;
  1959. if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
  1960. switch (boot_cpu_data.x86) {
  1961. case 0x6:
  1962. return p6_pmu_init();
  1963. case 0xb:
  1964. return knc_pmu_init();
  1965. case 0xf:
  1966. return p4_pmu_init();
  1967. }
  1968. return -ENODEV;
  1969. }
  1970. /*
  1971. * Check whether the Architectural PerfMon supports
  1972. * Branch Misses Retired hw_event or not.
  1973. */
  1974. cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);
  1975. if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT)
  1976. return -ENODEV;
  1977. version = eax.split.version_id;
  1978. if (version < 2)
  1979. x86_pmu = core_pmu;
  1980. else
  1981. x86_pmu = intel_pmu;
  1982. x86_pmu.version = version;
  1983. x86_pmu.num_counters = eax.split.num_counters;
  1984. x86_pmu.cntval_bits = eax.split.bit_width;
  1985. x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1;
  1986. x86_pmu.events_maskl = ebx.full;
  1987. x86_pmu.events_mask_len = eax.split.mask_length;
  1988. x86_pmu.max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, x86_pmu.num_counters);
  1989. /*
  1990. * Quirk: v2 perfmon does not report fixed-purpose events, so
  1991. * assume at least 3 events:
  1992. */
  1993. if (version > 1)
  1994. x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
  1995. /*
  1996. * v2 and above have a perf capabilities MSR
  1997. */
  1998. if (version > 1) {
  1999. u64 capabilities;
  2000. rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
  2001. x86_pmu.intel_cap.capabilities = capabilities;
  2002. }
  2003. intel_ds_init();
  2004. x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
  2005. /*
  2006. * Install the hw-cache-events table:
  2007. */
  2008. switch (boot_cpu_data.x86_model) {
  2009. case 14: /* 65 nm core solo/duo, "Yonah" */
  2010. pr_cont("Core events, ");
  2011. break;
  2012. case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
  2013. x86_add_quirk(intel_clovertown_quirk);
  2014. case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
  2015. case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
  2016. case 29: /* six-core 45 nm xeon "Dunnington" */
  2017. memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
  2018. sizeof(hw_cache_event_ids));
  2019. intel_pmu_lbr_init_core();
  2020. x86_pmu.event_constraints = intel_core2_event_constraints;
  2021. x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
  2022. pr_cont("Core2 events, ");
  2023. break;
  2024. case 26: /* 45 nm nehalem, "Bloomfield" */
  2025. case 30: /* 45 nm nehalem, "Lynnfield" */
  2026. case 46: /* 45 nm nehalem-ex, "Beckton" */
  2027. memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
  2028. sizeof(hw_cache_event_ids));
  2029. memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
  2030. sizeof(hw_cache_extra_regs));
  2031. intel_pmu_lbr_init_nhm();
  2032. x86_pmu.event_constraints = intel_nehalem_event_constraints;
  2033. x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
  2034. x86_pmu.enable_all = intel_pmu_nhm_enable_all;
  2035. x86_pmu.extra_regs = intel_nehalem_extra_regs;
  2036. x86_pmu.cpu_events = nhm_events_attrs;
  2037. /* UOPS_ISSUED.STALLED_CYCLES */
  2038. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  2039. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  2040. /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
  2041. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  2042. X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
  2043. x86_add_quirk(intel_nehalem_quirk);
  2044. pr_cont("Nehalem events, ");
  2045. break;
  2046. case 28: /* Atom */
  2047. case 38: /* Lincroft */
  2048. case 39: /* Penwell */
  2049. case 53: /* Cloverview */
  2050. case 54: /* Cedarview */
  2051. memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
  2052. sizeof(hw_cache_event_ids));
  2053. intel_pmu_lbr_init_atom();
  2054. x86_pmu.event_constraints = intel_gen_event_constraints;
  2055. x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
  2056. pr_cont("Atom events, ");
  2057. break;
  2058. case 55: /* Atom 22nm "Silvermont" */
  2059. memcpy(hw_cache_event_ids, slm_hw_cache_event_ids,
  2060. sizeof(hw_cache_event_ids));
  2061. memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs,
  2062. sizeof(hw_cache_extra_regs));
  2063. intel_pmu_lbr_init_atom();
  2064. x86_pmu.event_constraints = intel_slm_event_constraints;
  2065. x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints;
  2066. x86_pmu.extra_regs = intel_slm_extra_regs;
  2067. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  2068. pr_cont("Silvermont events, ");
  2069. break;
  2070. case 37: /* 32 nm nehalem, "Clarkdale" */
  2071. case 44: /* 32 nm nehalem, "Gulftown" */
  2072. case 47: /* 32 nm Xeon E7 */
  2073. memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
  2074. sizeof(hw_cache_event_ids));
  2075. memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
  2076. sizeof(hw_cache_extra_regs));
  2077. intel_pmu_lbr_init_nhm();
  2078. x86_pmu.event_constraints = intel_westmere_event_constraints;
  2079. x86_pmu.enable_all = intel_pmu_nhm_enable_all;
  2080. x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
  2081. x86_pmu.extra_regs = intel_westmere_extra_regs;
  2082. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  2083. x86_pmu.cpu_events = nhm_events_attrs;
  2084. /* UOPS_ISSUED.STALLED_CYCLES */
  2085. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  2086. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  2087. /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
  2088. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  2089. X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
  2090. pr_cont("Westmere events, ");
  2091. break;
  2092. case 42: /* SandyBridge */
  2093. case 45: /* SandyBridge, "Romely-EP" */
  2094. x86_add_quirk(intel_sandybridge_quirk);
  2095. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
  2096. sizeof(hw_cache_event_ids));
  2097. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
  2098. sizeof(hw_cache_extra_regs));
  2099. intel_pmu_lbr_init_snb();
  2100. x86_pmu.event_constraints = intel_snb_event_constraints;
  2101. x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
  2102. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  2103. if (boot_cpu_data.x86_model == 45)
  2104. x86_pmu.extra_regs = intel_snbep_extra_regs;
  2105. else
  2106. x86_pmu.extra_regs = intel_snb_extra_regs;
  2107. /* all extra regs are per-cpu when HT is on */
  2108. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  2109. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  2110. x86_pmu.cpu_events = snb_events_attrs;
  2111. /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
  2112. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  2113. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  2114. /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
  2115. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
  2116. X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1);
  2117. pr_cont("SandyBridge events, ");
  2118. break;
  2119. case 58: /* IvyBridge */
  2120. case 62: /* IvyBridge EP */
  2121. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
  2122. sizeof(hw_cache_event_ids));
  2123. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
  2124. sizeof(hw_cache_extra_regs));
  2125. intel_pmu_lbr_init_snb();
  2126. x86_pmu.event_constraints = intel_ivb_event_constraints;
  2127. x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
  2128. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  2129. if (boot_cpu_data.x86_model == 62)
  2130. x86_pmu.extra_regs = intel_snbep_extra_regs;
  2131. else
  2132. x86_pmu.extra_regs = intel_snb_extra_regs;
  2133. /* all extra regs are per-cpu when HT is on */
  2134. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  2135. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  2136. x86_pmu.cpu_events = snb_events_attrs;
  2137. /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
  2138. intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
  2139. X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
  2140. pr_cont("IvyBridge events, ");
  2141. break;
  2142. case 60: /* Haswell Client */
  2143. case 70:
  2144. case 71:
  2145. case 63:
  2146. case 69:
  2147. x86_pmu.late_ack = true;
  2148. memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, sizeof(hw_cache_event_ids));
  2149. memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
  2150. intel_pmu_lbr_init_snb();
  2151. x86_pmu.event_constraints = intel_hsw_event_constraints;
  2152. x86_pmu.pebs_constraints = intel_hsw_pebs_event_constraints;
  2153. x86_pmu.extra_regs = intel_snb_extra_regs;
  2154. x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
  2155. /* all extra regs are per-cpu when HT is on */
  2156. x86_pmu.er_flags |= ERF_HAS_RSP_1;
  2157. x86_pmu.er_flags |= ERF_NO_HT_SHARING;
  2158. x86_pmu.hw_config = hsw_hw_config;
  2159. x86_pmu.get_event_constraints = hsw_get_event_constraints;
  2160. x86_pmu.cpu_events = hsw_events_attrs;
  2161. pr_cont("Haswell events, ");
  2162. break;
  2163. default:
  2164. switch (x86_pmu.version) {
  2165. case 1:
  2166. x86_pmu.event_constraints = intel_v1_event_constraints;
  2167. pr_cont("generic architected perfmon v1, ");
  2168. break;
  2169. default:
  2170. /*
  2171. * default constraints for v2 and up
  2172. */
  2173. x86_pmu.event_constraints = intel_gen_event_constraints;
  2174. pr_cont("generic architected perfmon, ");
  2175. break;
  2176. }
  2177. }
  2178. if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) {
  2179. WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
  2180. x86_pmu.num_counters, INTEL_PMC_MAX_GENERIC);
  2181. x86_pmu.num_counters = INTEL_PMC_MAX_GENERIC;
  2182. }
  2183. x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
  2184. if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) {
  2185. WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
  2186. x86_pmu.num_counters_fixed, INTEL_PMC_MAX_FIXED);
  2187. x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED;
  2188. }
  2189. x86_pmu.intel_ctrl |=
  2190. ((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
  2191. if (x86_pmu.event_constraints) {
  2192. /*
  2193. * event on fixed counter2 (REF_CYCLES) only works on this
  2194. * counter, so do not extend mask to generic counters
  2195. */
  2196. for_each_event_constraint(c, x86_pmu.event_constraints) {
  2197. if (c->cmask != FIXED_EVENT_FLAGS
  2198. || c->idxmsk64 == INTEL_PMC_MSK_FIXED_REF_CYCLES) {
  2199. continue;
  2200. }
  2201. c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
  2202. c->weight += x86_pmu.num_counters;
  2203. }
  2204. }
  2205. /* Support full width counters using alternative MSR range */
  2206. if (x86_pmu.intel_cap.full_width_write) {
  2207. x86_pmu.max_period = x86_pmu.cntval_mask;
  2208. x86_pmu.perfctr = MSR_IA32_PMC0;
  2209. pr_cont("full-width counters, ");
  2210. }
  2211. return 0;
  2212. }