perf_event.c 66 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841
  1. /*
  2. * Performance events x86 architecture code
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2009 Jaswinder Singh Rajput
  7. * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  8. * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  9. * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10. * Copyright (C) 2009 Google, Inc., Stephane Eranian
  11. *
  12. * For licencing details see kernel-base/COPYING
  13. */
  14. #include <linux/perf_event.h>
  15. #include <linux/capability.h>
  16. #include <linux/notifier.h>
  17. #include <linux/hardirq.h>
  18. #include <linux/kprobes.h>
  19. #include <linux/module.h>
  20. #include <linux/kdebug.h>
  21. #include <linux/sched.h>
  22. #include <linux/uaccess.h>
  23. #include <linux/highmem.h>
  24. #include <linux/cpu.h>
  25. #include <asm/apic.h>
  26. #include <asm/stacktrace.h>
  27. #include <asm/nmi.h>
  28. static u64 perf_event_mask __read_mostly;
  29. /* The maximal number of PEBS events: */
  30. #define MAX_PEBS_EVENTS 4
  31. /* The size of a BTS record in bytes: */
  32. #define BTS_RECORD_SIZE 24
  33. /* The size of a per-cpu BTS buffer in bytes: */
  34. #define BTS_BUFFER_SIZE (BTS_RECORD_SIZE * 2048)
  35. /* The BTS overflow threshold in bytes from the end of the buffer: */
  36. #define BTS_OVFL_TH (BTS_RECORD_SIZE * 128)
  37. /*
  38. * Bits in the debugctlmsr controlling branch tracing.
  39. */
  40. #define X86_DEBUGCTL_TR (1 << 6)
  41. #define X86_DEBUGCTL_BTS (1 << 7)
  42. #define X86_DEBUGCTL_BTINT (1 << 8)
  43. #define X86_DEBUGCTL_BTS_OFF_OS (1 << 9)
  44. #define X86_DEBUGCTL_BTS_OFF_USR (1 << 10)
  45. /*
  46. * A debug store configuration.
  47. *
  48. * We only support architectures that use 64bit fields.
  49. */
  50. struct debug_store {
  51. u64 bts_buffer_base;
  52. u64 bts_index;
  53. u64 bts_absolute_maximum;
  54. u64 bts_interrupt_threshold;
  55. u64 pebs_buffer_base;
  56. u64 pebs_index;
  57. u64 pebs_absolute_maximum;
  58. u64 pebs_interrupt_threshold;
  59. u64 pebs_event_reset[MAX_PEBS_EVENTS];
  60. };
  61. struct event_constraint {
  62. union {
  63. unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  64. u64 idxmsk64[1];
  65. };
  66. int code;
  67. int cmask;
  68. };
  69. struct cpu_hw_events {
  70. struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
  71. unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  72. unsigned long interrupts;
  73. int enabled;
  74. struct debug_store *ds;
  75. int n_events;
  76. int n_added;
  77. int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
  78. struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
  79. };
  80. #define EVENT_CONSTRAINT(c, n, m) { \
  81. { .idxmsk64[0] = (n) }, \
  82. .code = (c), \
  83. .cmask = (m), \
  84. }
  85. #define EVENT_CONSTRAINT_END \
  86. EVENT_CONSTRAINT(0, 0, 0)
  87. #define for_each_event_constraint(e, c) \
  88. for ((e) = (c); (e)->cmask; (e)++)
  89. /*
  90. * struct x86_pmu - generic x86 pmu
  91. */
  92. struct x86_pmu {
  93. const char *name;
  94. int version;
  95. int (*handle_irq)(struct pt_regs *);
  96. void (*disable_all)(void);
  97. void (*enable_all)(void);
  98. void (*enable)(struct hw_perf_event *, int);
  99. void (*disable)(struct hw_perf_event *, int);
  100. unsigned eventsel;
  101. unsigned perfctr;
  102. u64 (*event_map)(int);
  103. u64 (*raw_event)(u64);
  104. int max_events;
  105. int num_events;
  106. int num_events_fixed;
  107. int event_bits;
  108. u64 event_mask;
  109. int apic;
  110. u64 max_period;
  111. u64 intel_ctrl;
  112. void (*enable_bts)(u64 config);
  113. void (*disable_bts)(void);
  114. void (*get_event_constraints)(struct cpu_hw_events *cpuc,
  115. struct perf_event *event,
  116. unsigned long *idxmsk);
  117. void (*put_event_constraints)(struct cpu_hw_events *cpuc,
  118. struct perf_event *event);
  119. const struct event_constraint *event_constraints;
  120. };
  121. static struct x86_pmu x86_pmu __read_mostly;
  122. static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  123. .enabled = 1,
  124. };
  125. static int x86_perf_event_set_period(struct perf_event *event,
  126. struct hw_perf_event *hwc, int idx);
  127. /*
  128. * Not sure about some of these
  129. */
  130. static const u64 p6_perfmon_event_map[] =
  131. {
  132. [PERF_COUNT_HW_CPU_CYCLES] = 0x0079,
  133. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  134. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0f2e,
  135. [PERF_COUNT_HW_CACHE_MISSES] = 0x012e,
  136. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  137. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  138. [PERF_COUNT_HW_BUS_CYCLES] = 0x0062,
  139. };
  140. static u64 p6_pmu_event_map(int hw_event)
  141. {
  142. return p6_perfmon_event_map[hw_event];
  143. }
  144. /*
  145. * Event setting that is specified not to count anything.
  146. * We use this to effectively disable a counter.
  147. *
  148. * L2_RQSTS with 0 MESI unit mask.
  149. */
  150. #define P6_NOP_EVENT 0x0000002EULL
  151. static u64 p6_pmu_raw_event(u64 hw_event)
  152. {
  153. #define P6_EVNTSEL_EVENT_MASK 0x000000FFULL
  154. #define P6_EVNTSEL_UNIT_MASK 0x0000FF00ULL
  155. #define P6_EVNTSEL_EDGE_MASK 0x00040000ULL
  156. #define P6_EVNTSEL_INV_MASK 0x00800000ULL
  157. #define P6_EVNTSEL_REG_MASK 0xFF000000ULL
  158. #define P6_EVNTSEL_MASK \
  159. (P6_EVNTSEL_EVENT_MASK | \
  160. P6_EVNTSEL_UNIT_MASK | \
  161. P6_EVNTSEL_EDGE_MASK | \
  162. P6_EVNTSEL_INV_MASK | \
  163. P6_EVNTSEL_REG_MASK)
  164. return hw_event & P6_EVNTSEL_MASK;
  165. }
  166. static struct event_constraint intel_p6_event_constraints[] =
  167. {
  168. EVENT_CONSTRAINT(0xc1, 0x1, INTEL_ARCH_EVENT_MASK), /* FLOPS */
  169. EVENT_CONSTRAINT(0x10, 0x1, INTEL_ARCH_EVENT_MASK), /* FP_COMP_OPS_EXE */
  170. EVENT_CONSTRAINT(0x11, 0x1, INTEL_ARCH_EVENT_MASK), /* FP_ASSIST */
  171. EVENT_CONSTRAINT(0x12, 0x2, INTEL_ARCH_EVENT_MASK), /* MUL */
  172. EVENT_CONSTRAINT(0x13, 0x2, INTEL_ARCH_EVENT_MASK), /* DIV */
  173. EVENT_CONSTRAINT(0x14, 0x1, INTEL_ARCH_EVENT_MASK), /* CYCLES_DIV_BUSY */
  174. EVENT_CONSTRAINT_END
  175. };
  176. /*
  177. * Intel PerfMon v3. Used on Core2 and later.
  178. */
  179. static const u64 intel_perfmon_event_map[] =
  180. {
  181. [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
  182. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  183. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
  184. [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
  185. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  186. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  187. [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
  188. };
  189. static struct event_constraint intel_core_event_constraints[] =
  190. {
  191. EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32)), INTEL_ARCH_FIXED_MASK), /* INSTRUCTIONS_RETIRED */
  192. EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33)), INTEL_ARCH_FIXED_MASK), /* UNHALTED_CORE_CYCLES */
  193. EVENT_CONSTRAINT(0x10, 0x1, INTEL_ARCH_EVENT_MASK), /* FP_COMP_OPS_EXE */
  194. EVENT_CONSTRAINT(0x11, 0x2, INTEL_ARCH_EVENT_MASK), /* FP_ASSIST */
  195. EVENT_CONSTRAINT(0x12, 0x2, INTEL_ARCH_EVENT_MASK), /* MUL */
  196. EVENT_CONSTRAINT(0x13, 0x2, INTEL_ARCH_EVENT_MASK), /* DIV */
  197. EVENT_CONSTRAINT(0x14, 0x1, INTEL_ARCH_EVENT_MASK), /* CYCLES_DIV_BUSY */
  198. EVENT_CONSTRAINT(0x18, 0x1, INTEL_ARCH_EVENT_MASK), /* IDLE_DURING_DIV */
  199. EVENT_CONSTRAINT(0x19, 0x2, INTEL_ARCH_EVENT_MASK), /* DELAYED_BYPASS */
  200. EVENT_CONSTRAINT(0xa1, 0x1, INTEL_ARCH_EVENT_MASK), /* RS_UOPS_DISPATCH_CYCLES */
  201. EVENT_CONSTRAINT(0xcb, 0x1, INTEL_ARCH_EVENT_MASK), /* MEM_LOAD_RETIRED */
  202. EVENT_CONSTRAINT_END
  203. };
  204. static struct event_constraint intel_nehalem_event_constraints[] =
  205. {
  206. EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32)), INTEL_ARCH_FIXED_MASK), /* INSTRUCTIONS_RETIRED */
  207. EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33)), INTEL_ARCH_FIXED_MASK), /* UNHALTED_CORE_CYCLES */
  208. EVENT_CONSTRAINT(0x40, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_LD */
  209. EVENT_CONSTRAINT(0x41, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_ST */
  210. EVENT_CONSTRAINT(0x42, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_LOCK */
  211. EVENT_CONSTRAINT(0x43, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_ALL_REF */
  212. EVENT_CONSTRAINT(0x4e, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_PREFETCH */
  213. EVENT_CONSTRAINT(0x4c, 0x3, INTEL_ARCH_EVENT_MASK), /* LOAD_HIT_PRE */
  214. EVENT_CONSTRAINT(0x51, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D */
  215. EVENT_CONSTRAINT(0x52, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
  216. EVENT_CONSTRAINT(0x53, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_LOCK_FB_HIT */
  217. EVENT_CONSTRAINT(0xc5, 0x3, INTEL_ARCH_EVENT_MASK), /* CACHE_LOCK_CYCLES */
  218. EVENT_CONSTRAINT_END
  219. };
  220. static struct event_constraint intel_gen_event_constraints[] =
  221. {
  222. EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32)), INTEL_ARCH_FIXED_MASK), /* INSTRUCTIONS_RETIRED */
  223. EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33)), INTEL_ARCH_FIXED_MASK), /* UNHALTED_CORE_CYCLES */
  224. EVENT_CONSTRAINT_END
  225. };
  226. static u64 intel_pmu_event_map(int hw_event)
  227. {
  228. return intel_perfmon_event_map[hw_event];
  229. }
  230. /*
  231. * Generalized hw caching related hw_event table, filled
  232. * in on a per model basis. A value of 0 means
  233. * 'not supported', -1 means 'hw_event makes no sense on
  234. * this CPU', any other value means the raw hw_event
  235. * ID.
  236. */
  237. #define C(x) PERF_COUNT_HW_CACHE_##x
  238. static u64 __read_mostly hw_cache_event_ids
  239. [PERF_COUNT_HW_CACHE_MAX]
  240. [PERF_COUNT_HW_CACHE_OP_MAX]
  241. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  242. static __initconst u64 nehalem_hw_cache_event_ids
  243. [PERF_COUNT_HW_CACHE_MAX]
  244. [PERF_COUNT_HW_CACHE_OP_MAX]
  245. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  246. {
  247. [ C(L1D) ] = {
  248. [ C(OP_READ) ] = {
  249. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  250. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  251. },
  252. [ C(OP_WRITE) ] = {
  253. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  254. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  255. },
  256. [ C(OP_PREFETCH) ] = {
  257. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  258. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  259. },
  260. },
  261. [ C(L1I ) ] = {
  262. [ C(OP_READ) ] = {
  263. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  264. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  265. },
  266. [ C(OP_WRITE) ] = {
  267. [ C(RESULT_ACCESS) ] = -1,
  268. [ C(RESULT_MISS) ] = -1,
  269. },
  270. [ C(OP_PREFETCH) ] = {
  271. [ C(RESULT_ACCESS) ] = 0x0,
  272. [ C(RESULT_MISS) ] = 0x0,
  273. },
  274. },
  275. [ C(LL ) ] = {
  276. [ C(OP_READ) ] = {
  277. [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS */
  278. [ C(RESULT_MISS) ] = 0x0224, /* L2_RQSTS.LD_MISS */
  279. },
  280. [ C(OP_WRITE) ] = {
  281. [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS */
  282. [ C(RESULT_MISS) ] = 0x0824, /* L2_RQSTS.RFO_MISS */
  283. },
  284. [ C(OP_PREFETCH) ] = {
  285. [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference */
  286. [ C(RESULT_MISS) ] = 0x412e, /* LLC Misses */
  287. },
  288. },
  289. [ C(DTLB) ] = {
  290. [ C(OP_READ) ] = {
  291. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  292. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  293. },
  294. [ C(OP_WRITE) ] = {
  295. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  296. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  297. },
  298. [ C(OP_PREFETCH) ] = {
  299. [ C(RESULT_ACCESS) ] = 0x0,
  300. [ C(RESULT_MISS) ] = 0x0,
  301. },
  302. },
  303. [ C(ITLB) ] = {
  304. [ C(OP_READ) ] = {
  305. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  306. [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
  307. },
  308. [ C(OP_WRITE) ] = {
  309. [ C(RESULT_ACCESS) ] = -1,
  310. [ C(RESULT_MISS) ] = -1,
  311. },
  312. [ C(OP_PREFETCH) ] = {
  313. [ C(RESULT_ACCESS) ] = -1,
  314. [ C(RESULT_MISS) ] = -1,
  315. },
  316. },
  317. [ C(BPU ) ] = {
  318. [ C(OP_READ) ] = {
  319. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  320. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  321. },
  322. [ C(OP_WRITE) ] = {
  323. [ C(RESULT_ACCESS) ] = -1,
  324. [ C(RESULT_MISS) ] = -1,
  325. },
  326. [ C(OP_PREFETCH) ] = {
  327. [ C(RESULT_ACCESS) ] = -1,
  328. [ C(RESULT_MISS) ] = -1,
  329. },
  330. },
  331. };
  332. static __initconst u64 core2_hw_cache_event_ids
  333. [PERF_COUNT_HW_CACHE_MAX]
  334. [PERF_COUNT_HW_CACHE_OP_MAX]
  335. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  336. {
  337. [ C(L1D) ] = {
  338. [ C(OP_READ) ] = {
  339. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  340. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  341. },
  342. [ C(OP_WRITE) ] = {
  343. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  344. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  345. },
  346. [ C(OP_PREFETCH) ] = {
  347. [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
  348. [ C(RESULT_MISS) ] = 0,
  349. },
  350. },
  351. [ C(L1I ) ] = {
  352. [ C(OP_READ) ] = {
  353. [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
  354. [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
  355. },
  356. [ C(OP_WRITE) ] = {
  357. [ C(RESULT_ACCESS) ] = -1,
  358. [ C(RESULT_MISS) ] = -1,
  359. },
  360. [ C(OP_PREFETCH) ] = {
  361. [ C(RESULT_ACCESS) ] = 0,
  362. [ C(RESULT_MISS) ] = 0,
  363. },
  364. },
  365. [ C(LL ) ] = {
  366. [ C(OP_READ) ] = {
  367. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  368. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  369. },
  370. [ C(OP_WRITE) ] = {
  371. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  372. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  373. },
  374. [ C(OP_PREFETCH) ] = {
  375. [ C(RESULT_ACCESS) ] = 0,
  376. [ C(RESULT_MISS) ] = 0,
  377. },
  378. },
  379. [ C(DTLB) ] = {
  380. [ C(OP_READ) ] = {
  381. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  382. [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
  383. },
  384. [ C(OP_WRITE) ] = {
  385. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  386. [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
  387. },
  388. [ C(OP_PREFETCH) ] = {
  389. [ C(RESULT_ACCESS) ] = 0,
  390. [ C(RESULT_MISS) ] = 0,
  391. },
  392. },
  393. [ C(ITLB) ] = {
  394. [ C(OP_READ) ] = {
  395. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  396. [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
  397. },
  398. [ C(OP_WRITE) ] = {
  399. [ C(RESULT_ACCESS) ] = -1,
  400. [ C(RESULT_MISS) ] = -1,
  401. },
  402. [ C(OP_PREFETCH) ] = {
  403. [ C(RESULT_ACCESS) ] = -1,
  404. [ C(RESULT_MISS) ] = -1,
  405. },
  406. },
  407. [ C(BPU ) ] = {
  408. [ C(OP_READ) ] = {
  409. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  410. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  411. },
  412. [ C(OP_WRITE) ] = {
  413. [ C(RESULT_ACCESS) ] = -1,
  414. [ C(RESULT_MISS) ] = -1,
  415. },
  416. [ C(OP_PREFETCH) ] = {
  417. [ C(RESULT_ACCESS) ] = -1,
  418. [ C(RESULT_MISS) ] = -1,
  419. },
  420. },
  421. };
  422. static __initconst u64 atom_hw_cache_event_ids
  423. [PERF_COUNT_HW_CACHE_MAX]
  424. [PERF_COUNT_HW_CACHE_OP_MAX]
  425. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  426. {
  427. [ C(L1D) ] = {
  428. [ C(OP_READ) ] = {
  429. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
  430. [ C(RESULT_MISS) ] = 0,
  431. },
  432. [ C(OP_WRITE) ] = {
  433. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
  434. [ C(RESULT_MISS) ] = 0,
  435. },
  436. [ C(OP_PREFETCH) ] = {
  437. [ C(RESULT_ACCESS) ] = 0x0,
  438. [ C(RESULT_MISS) ] = 0,
  439. },
  440. },
  441. [ C(L1I ) ] = {
  442. [ C(OP_READ) ] = {
  443. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  444. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  445. },
  446. [ C(OP_WRITE) ] = {
  447. [ C(RESULT_ACCESS) ] = -1,
  448. [ C(RESULT_MISS) ] = -1,
  449. },
  450. [ C(OP_PREFETCH) ] = {
  451. [ C(RESULT_ACCESS) ] = 0,
  452. [ C(RESULT_MISS) ] = 0,
  453. },
  454. },
  455. [ C(LL ) ] = {
  456. [ C(OP_READ) ] = {
  457. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  458. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  459. },
  460. [ C(OP_WRITE) ] = {
  461. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  462. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  463. },
  464. [ C(OP_PREFETCH) ] = {
  465. [ C(RESULT_ACCESS) ] = 0,
  466. [ C(RESULT_MISS) ] = 0,
  467. },
  468. },
  469. [ C(DTLB) ] = {
  470. [ C(OP_READ) ] = {
  471. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
  472. [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
  473. },
  474. [ C(OP_WRITE) ] = {
  475. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
  476. [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
  477. },
  478. [ C(OP_PREFETCH) ] = {
  479. [ C(RESULT_ACCESS) ] = 0,
  480. [ C(RESULT_MISS) ] = 0,
  481. },
  482. },
  483. [ C(ITLB) ] = {
  484. [ C(OP_READ) ] = {
  485. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  486. [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
  487. },
  488. [ C(OP_WRITE) ] = {
  489. [ C(RESULT_ACCESS) ] = -1,
  490. [ C(RESULT_MISS) ] = -1,
  491. },
  492. [ C(OP_PREFETCH) ] = {
  493. [ C(RESULT_ACCESS) ] = -1,
  494. [ C(RESULT_MISS) ] = -1,
  495. },
  496. },
  497. [ C(BPU ) ] = {
  498. [ C(OP_READ) ] = {
  499. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  500. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  501. },
  502. [ C(OP_WRITE) ] = {
  503. [ C(RESULT_ACCESS) ] = -1,
  504. [ C(RESULT_MISS) ] = -1,
  505. },
  506. [ C(OP_PREFETCH) ] = {
  507. [ C(RESULT_ACCESS) ] = -1,
  508. [ C(RESULT_MISS) ] = -1,
  509. },
  510. },
  511. };
  512. static u64 intel_pmu_raw_event(u64 hw_event)
  513. {
  514. #define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
  515. #define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
  516. #define CORE_EVNTSEL_EDGE_MASK 0x00040000ULL
  517. #define CORE_EVNTSEL_INV_MASK 0x00800000ULL
  518. #define CORE_EVNTSEL_REG_MASK 0xFF000000ULL
  519. #define CORE_EVNTSEL_MASK \
  520. (INTEL_ARCH_EVTSEL_MASK | \
  521. INTEL_ARCH_UNIT_MASK | \
  522. INTEL_ARCH_EDGE_MASK | \
  523. INTEL_ARCH_INV_MASK | \
  524. INTEL_ARCH_CNT_MASK)
  525. return hw_event & CORE_EVNTSEL_MASK;
  526. }
  527. static __initconst u64 amd_hw_cache_event_ids
  528. [PERF_COUNT_HW_CACHE_MAX]
  529. [PERF_COUNT_HW_CACHE_OP_MAX]
  530. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  531. {
  532. [ C(L1D) ] = {
  533. [ C(OP_READ) ] = {
  534. [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
  535. [ C(RESULT_MISS) ] = 0x0041, /* Data Cache Misses */
  536. },
  537. [ C(OP_WRITE) ] = {
  538. [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
  539. [ C(RESULT_MISS) ] = 0,
  540. },
  541. [ C(OP_PREFETCH) ] = {
  542. [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */
  543. [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */
  544. },
  545. },
  546. [ C(L1I ) ] = {
  547. [ C(OP_READ) ] = {
  548. [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
  549. [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
  550. },
  551. [ C(OP_WRITE) ] = {
  552. [ C(RESULT_ACCESS) ] = -1,
  553. [ C(RESULT_MISS) ] = -1,
  554. },
  555. [ C(OP_PREFETCH) ] = {
  556. [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
  557. [ C(RESULT_MISS) ] = 0,
  558. },
  559. },
  560. [ C(LL ) ] = {
  561. [ C(OP_READ) ] = {
  562. [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
  563. [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */
  564. },
  565. [ C(OP_WRITE) ] = {
  566. [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */
  567. [ C(RESULT_MISS) ] = 0,
  568. },
  569. [ C(OP_PREFETCH) ] = {
  570. [ C(RESULT_ACCESS) ] = 0,
  571. [ C(RESULT_MISS) ] = 0,
  572. },
  573. },
  574. [ C(DTLB) ] = {
  575. [ C(OP_READ) ] = {
  576. [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
  577. [ C(RESULT_MISS) ] = 0x0046, /* L1 DTLB and L2 DLTB Miss */
  578. },
  579. [ C(OP_WRITE) ] = {
  580. [ C(RESULT_ACCESS) ] = 0,
  581. [ C(RESULT_MISS) ] = 0,
  582. },
  583. [ C(OP_PREFETCH) ] = {
  584. [ C(RESULT_ACCESS) ] = 0,
  585. [ C(RESULT_MISS) ] = 0,
  586. },
  587. },
  588. [ C(ITLB) ] = {
  589. [ C(OP_READ) ] = {
  590. [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
  591. [ C(RESULT_MISS) ] = 0x0085, /* Instr. fetch ITLB misses */
  592. },
  593. [ C(OP_WRITE) ] = {
  594. [ C(RESULT_ACCESS) ] = -1,
  595. [ C(RESULT_MISS) ] = -1,
  596. },
  597. [ C(OP_PREFETCH) ] = {
  598. [ C(RESULT_ACCESS) ] = -1,
  599. [ C(RESULT_MISS) ] = -1,
  600. },
  601. },
  602. [ C(BPU ) ] = {
  603. [ C(OP_READ) ] = {
  604. [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
  605. [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
  606. },
  607. [ C(OP_WRITE) ] = {
  608. [ C(RESULT_ACCESS) ] = -1,
  609. [ C(RESULT_MISS) ] = -1,
  610. },
  611. [ C(OP_PREFETCH) ] = {
  612. [ C(RESULT_ACCESS) ] = -1,
  613. [ C(RESULT_MISS) ] = -1,
  614. },
  615. },
  616. };
  617. /*
  618. * AMD Performance Monitor K7 and later.
  619. */
  620. static const u64 amd_perfmon_event_map[] =
  621. {
  622. [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
  623. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  624. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
  625. [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
  626. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  627. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  628. };
  629. static u64 amd_pmu_event_map(int hw_event)
  630. {
  631. return amd_perfmon_event_map[hw_event];
  632. }
  633. static u64 amd_pmu_raw_event(u64 hw_event)
  634. {
  635. #define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
  636. #define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
  637. #define K7_EVNTSEL_EDGE_MASK 0x000040000ULL
  638. #define K7_EVNTSEL_INV_MASK 0x000800000ULL
  639. #define K7_EVNTSEL_REG_MASK 0x0FF000000ULL
  640. #define K7_EVNTSEL_MASK \
  641. (K7_EVNTSEL_EVENT_MASK | \
  642. K7_EVNTSEL_UNIT_MASK | \
  643. K7_EVNTSEL_EDGE_MASK | \
  644. K7_EVNTSEL_INV_MASK | \
  645. K7_EVNTSEL_REG_MASK)
  646. return hw_event & K7_EVNTSEL_MASK;
  647. }
  648. /*
  649. * Propagate event elapsed time into the generic event.
  650. * Can only be executed on the CPU where the event is active.
  651. * Returns the delta events processed.
  652. */
  653. static u64
  654. x86_perf_event_update(struct perf_event *event,
  655. struct hw_perf_event *hwc, int idx)
  656. {
  657. int shift = 64 - x86_pmu.event_bits;
  658. u64 prev_raw_count, new_raw_count;
  659. s64 delta;
  660. if (idx == X86_PMC_IDX_FIXED_BTS)
  661. return 0;
  662. /*
  663. * Careful: an NMI might modify the previous event value.
  664. *
  665. * Our tactic to handle this is to first atomically read and
  666. * exchange a new raw count - then add that new-prev delta
  667. * count to the generic event atomically:
  668. */
  669. again:
  670. prev_raw_count = atomic64_read(&hwc->prev_count);
  671. rdmsrl(hwc->event_base + idx, new_raw_count);
  672. if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
  673. new_raw_count) != prev_raw_count)
  674. goto again;
  675. /*
  676. * Now we have the new raw value and have updated the prev
  677. * timestamp already. We can now calculate the elapsed delta
  678. * (event-)time and add that to the generic event.
  679. *
  680. * Careful, not all hw sign-extends above the physical width
  681. * of the count.
  682. */
  683. delta = (new_raw_count << shift) - (prev_raw_count << shift);
  684. delta >>= shift;
  685. atomic64_add(delta, &event->count);
  686. atomic64_sub(delta, &hwc->period_left);
  687. return new_raw_count;
  688. }
  689. static atomic_t active_events;
  690. static DEFINE_MUTEX(pmc_reserve_mutex);
  691. static bool reserve_pmc_hardware(void)
  692. {
  693. #ifdef CONFIG_X86_LOCAL_APIC
  694. int i;
  695. if (nmi_watchdog == NMI_LOCAL_APIC)
  696. disable_lapic_nmi_watchdog();
  697. for (i = 0; i < x86_pmu.num_events; i++) {
  698. if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
  699. goto perfctr_fail;
  700. }
  701. for (i = 0; i < x86_pmu.num_events; i++) {
  702. if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
  703. goto eventsel_fail;
  704. }
  705. #endif
  706. return true;
  707. #ifdef CONFIG_X86_LOCAL_APIC
  708. eventsel_fail:
  709. for (i--; i >= 0; i--)
  710. release_evntsel_nmi(x86_pmu.eventsel + i);
  711. i = x86_pmu.num_events;
  712. perfctr_fail:
  713. for (i--; i >= 0; i--)
  714. release_perfctr_nmi(x86_pmu.perfctr + i);
  715. if (nmi_watchdog == NMI_LOCAL_APIC)
  716. enable_lapic_nmi_watchdog();
  717. return false;
  718. #endif
  719. }
  720. static void release_pmc_hardware(void)
  721. {
  722. #ifdef CONFIG_X86_LOCAL_APIC
  723. int i;
  724. for (i = 0; i < x86_pmu.num_events; i++) {
  725. release_perfctr_nmi(x86_pmu.perfctr + i);
  726. release_evntsel_nmi(x86_pmu.eventsel + i);
  727. }
  728. if (nmi_watchdog == NMI_LOCAL_APIC)
  729. enable_lapic_nmi_watchdog();
  730. #endif
  731. }
  732. static inline bool bts_available(void)
  733. {
  734. return x86_pmu.enable_bts != NULL;
  735. }
  736. static inline void init_debug_store_on_cpu(int cpu)
  737. {
  738. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  739. if (!ds)
  740. return;
  741. wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
  742. (u32)((u64)(unsigned long)ds),
  743. (u32)((u64)(unsigned long)ds >> 32));
  744. }
  745. static inline void fini_debug_store_on_cpu(int cpu)
  746. {
  747. if (!per_cpu(cpu_hw_events, cpu).ds)
  748. return;
  749. wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
  750. }
  751. static void release_bts_hardware(void)
  752. {
  753. int cpu;
  754. if (!bts_available())
  755. return;
  756. get_online_cpus();
  757. for_each_online_cpu(cpu)
  758. fini_debug_store_on_cpu(cpu);
  759. for_each_possible_cpu(cpu) {
  760. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  761. if (!ds)
  762. continue;
  763. per_cpu(cpu_hw_events, cpu).ds = NULL;
  764. kfree((void *)(unsigned long)ds->bts_buffer_base);
  765. kfree(ds);
  766. }
  767. put_online_cpus();
  768. }
  769. static int reserve_bts_hardware(void)
  770. {
  771. int cpu, err = 0;
  772. if (!bts_available())
  773. return 0;
  774. get_online_cpus();
  775. for_each_possible_cpu(cpu) {
  776. struct debug_store *ds;
  777. void *buffer;
  778. err = -ENOMEM;
  779. buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
  780. if (unlikely(!buffer))
  781. break;
  782. ds = kzalloc(sizeof(*ds), GFP_KERNEL);
  783. if (unlikely(!ds)) {
  784. kfree(buffer);
  785. break;
  786. }
  787. ds->bts_buffer_base = (u64)(unsigned long)buffer;
  788. ds->bts_index = ds->bts_buffer_base;
  789. ds->bts_absolute_maximum =
  790. ds->bts_buffer_base + BTS_BUFFER_SIZE;
  791. ds->bts_interrupt_threshold =
  792. ds->bts_absolute_maximum - BTS_OVFL_TH;
  793. per_cpu(cpu_hw_events, cpu).ds = ds;
  794. err = 0;
  795. }
  796. if (err)
  797. release_bts_hardware();
  798. else {
  799. for_each_online_cpu(cpu)
  800. init_debug_store_on_cpu(cpu);
  801. }
  802. put_online_cpus();
  803. return err;
  804. }
  805. static void hw_perf_event_destroy(struct perf_event *event)
  806. {
  807. if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
  808. release_pmc_hardware();
  809. release_bts_hardware();
  810. mutex_unlock(&pmc_reserve_mutex);
  811. }
  812. }
  813. static inline int x86_pmu_initialized(void)
  814. {
  815. return x86_pmu.handle_irq != NULL;
  816. }
  817. static inline int
  818. set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
  819. {
  820. unsigned int cache_type, cache_op, cache_result;
  821. u64 config, val;
  822. config = attr->config;
  823. cache_type = (config >> 0) & 0xff;
  824. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  825. return -EINVAL;
  826. cache_op = (config >> 8) & 0xff;
  827. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  828. return -EINVAL;
  829. cache_result = (config >> 16) & 0xff;
  830. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  831. return -EINVAL;
  832. val = hw_cache_event_ids[cache_type][cache_op][cache_result];
  833. if (val == 0)
  834. return -ENOENT;
  835. if (val == -1)
  836. return -EINVAL;
  837. hwc->config |= val;
  838. return 0;
  839. }
  840. static void intel_pmu_enable_bts(u64 config)
  841. {
  842. unsigned long debugctlmsr;
  843. debugctlmsr = get_debugctlmsr();
  844. debugctlmsr |= X86_DEBUGCTL_TR;
  845. debugctlmsr |= X86_DEBUGCTL_BTS;
  846. debugctlmsr |= X86_DEBUGCTL_BTINT;
  847. if (!(config & ARCH_PERFMON_EVENTSEL_OS))
  848. debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
  849. if (!(config & ARCH_PERFMON_EVENTSEL_USR))
  850. debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
  851. update_debugctlmsr(debugctlmsr);
  852. }
  853. static void intel_pmu_disable_bts(void)
  854. {
  855. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  856. unsigned long debugctlmsr;
  857. if (!cpuc->ds)
  858. return;
  859. debugctlmsr = get_debugctlmsr();
  860. debugctlmsr &=
  861. ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
  862. X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
  863. update_debugctlmsr(debugctlmsr);
  864. }
  865. /*
  866. * Setup the hardware configuration for a given attr_type
  867. */
  868. static int __hw_perf_event_init(struct perf_event *event)
  869. {
  870. struct perf_event_attr *attr = &event->attr;
  871. struct hw_perf_event *hwc = &event->hw;
  872. u64 config;
  873. int err;
  874. if (!x86_pmu_initialized())
  875. return -ENODEV;
  876. err = 0;
  877. if (!atomic_inc_not_zero(&active_events)) {
  878. mutex_lock(&pmc_reserve_mutex);
  879. if (atomic_read(&active_events) == 0) {
  880. if (!reserve_pmc_hardware())
  881. err = -EBUSY;
  882. else
  883. err = reserve_bts_hardware();
  884. }
  885. if (!err)
  886. atomic_inc(&active_events);
  887. mutex_unlock(&pmc_reserve_mutex);
  888. }
  889. if (err)
  890. return err;
  891. event->destroy = hw_perf_event_destroy;
  892. /*
  893. * Generate PMC IRQs:
  894. * (keep 'enabled' bit clear for now)
  895. */
  896. hwc->config = ARCH_PERFMON_EVENTSEL_INT;
  897. hwc->idx = -1;
  898. /*
  899. * Count user and OS events unless requested not to.
  900. */
  901. if (!attr->exclude_user)
  902. hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
  903. if (!attr->exclude_kernel)
  904. hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
  905. if (!hwc->sample_period) {
  906. hwc->sample_period = x86_pmu.max_period;
  907. hwc->last_period = hwc->sample_period;
  908. atomic64_set(&hwc->period_left, hwc->sample_period);
  909. } else {
  910. /*
  911. * If we have a PMU initialized but no APIC
  912. * interrupts, we cannot sample hardware
  913. * events (user-space has to fall back and
  914. * sample via a hrtimer based software event):
  915. */
  916. if (!x86_pmu.apic)
  917. return -EOPNOTSUPP;
  918. }
  919. /*
  920. * Raw hw_event type provide the config in the hw_event structure
  921. */
  922. if (attr->type == PERF_TYPE_RAW) {
  923. hwc->config |= x86_pmu.raw_event(attr->config);
  924. return 0;
  925. }
  926. if (attr->type == PERF_TYPE_HW_CACHE)
  927. return set_ext_hw_attr(hwc, attr);
  928. if (attr->config >= x86_pmu.max_events)
  929. return -EINVAL;
  930. /*
  931. * The generic map:
  932. */
  933. config = x86_pmu.event_map(attr->config);
  934. if (config == 0)
  935. return -ENOENT;
  936. if (config == -1LL)
  937. return -EINVAL;
  938. /*
  939. * Branch tracing:
  940. */
  941. if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
  942. (hwc->sample_period == 1)) {
  943. /* BTS is not supported by this architecture. */
  944. if (!bts_available())
  945. return -EOPNOTSUPP;
  946. /* BTS is currently only allowed for user-mode. */
  947. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  948. return -EOPNOTSUPP;
  949. }
  950. hwc->config |= config;
  951. return 0;
  952. }
  953. static void p6_pmu_disable_all(void)
  954. {
  955. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  956. u64 val;
  957. if (!cpuc->enabled)
  958. return;
  959. cpuc->enabled = 0;
  960. barrier();
  961. /* p6 only has one enable register */
  962. rdmsrl(MSR_P6_EVNTSEL0, val);
  963. val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
  964. wrmsrl(MSR_P6_EVNTSEL0, val);
  965. }
  966. static void intel_pmu_disable_all(void)
  967. {
  968. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  969. if (!cpuc->enabled)
  970. return;
  971. cpuc->enabled = 0;
  972. barrier();
  973. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
  974. if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
  975. intel_pmu_disable_bts();
  976. }
  977. static void amd_pmu_disable_all(void)
  978. {
  979. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  980. int idx;
  981. if (!cpuc->enabled)
  982. return;
  983. cpuc->enabled = 0;
  984. /*
  985. * ensure we write the disable before we start disabling the
  986. * events proper, so that amd_pmu_enable_event() does the
  987. * right thing.
  988. */
  989. barrier();
  990. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  991. u64 val;
  992. if (!test_bit(idx, cpuc->active_mask))
  993. continue;
  994. rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
  995. if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
  996. continue;
  997. val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
  998. wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
  999. }
  1000. }
  1001. void hw_perf_disable(void)
  1002. {
  1003. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1004. if (!x86_pmu_initialized())
  1005. return;
  1006. if (cpuc->enabled)
  1007. cpuc->n_added = 0;
  1008. x86_pmu.disable_all();
  1009. }
  1010. static void p6_pmu_enable_all(void)
  1011. {
  1012. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1013. unsigned long val;
  1014. if (cpuc->enabled)
  1015. return;
  1016. cpuc->enabled = 1;
  1017. barrier();
  1018. /* p6 only has one enable register */
  1019. rdmsrl(MSR_P6_EVNTSEL0, val);
  1020. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1021. wrmsrl(MSR_P6_EVNTSEL0, val);
  1022. }
  1023. static void intel_pmu_enable_all(void)
  1024. {
  1025. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1026. if (cpuc->enabled)
  1027. return;
  1028. cpuc->enabled = 1;
  1029. barrier();
  1030. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
  1031. if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
  1032. struct perf_event *event =
  1033. cpuc->events[X86_PMC_IDX_FIXED_BTS];
  1034. if (WARN_ON_ONCE(!event))
  1035. return;
  1036. intel_pmu_enable_bts(event->hw.config);
  1037. }
  1038. }
  1039. static void amd_pmu_enable_all(void)
  1040. {
  1041. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1042. int idx;
  1043. if (cpuc->enabled)
  1044. return;
  1045. cpuc->enabled = 1;
  1046. barrier();
  1047. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1048. struct perf_event *event = cpuc->events[idx];
  1049. u64 val;
  1050. if (!test_bit(idx, cpuc->active_mask))
  1051. continue;
  1052. val = event->hw.config;
  1053. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1054. wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
  1055. }
  1056. }
  1057. static const struct pmu pmu;
  1058. static inline int is_x86_event(struct perf_event *event)
  1059. {
  1060. return event->pmu == &pmu;
  1061. }
  1062. static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
  1063. {
  1064. int i, j , w, num;
  1065. int weight, wmax;
  1066. unsigned long *c;
  1067. unsigned long constraints[X86_PMC_IDX_MAX][BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  1068. unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  1069. struct hw_perf_event *hwc;
  1070. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  1071. for (i = 0; i < n; i++) {
  1072. x86_pmu.get_event_constraints(cpuc,
  1073. cpuc->event_list[i],
  1074. constraints[i]);
  1075. }
  1076. /*
  1077. * fastpath, try to reuse previous register
  1078. */
  1079. for (i = 0, num = n; i < n; i++, num--) {
  1080. hwc = &cpuc->event_list[i]->hw;
  1081. c = constraints[i];
  1082. /* never assigned */
  1083. if (hwc->idx == -1)
  1084. break;
  1085. /* constraint still honored */
  1086. if (!test_bit(hwc->idx, c))
  1087. break;
  1088. /* not already used */
  1089. if (test_bit(hwc->idx, used_mask))
  1090. break;
  1091. #if 0
  1092. pr_debug("CPU%d fast config=0x%llx idx=%d assign=%c\n",
  1093. smp_processor_id(),
  1094. hwc->config,
  1095. hwc->idx,
  1096. assign ? 'y' : 'n');
  1097. #endif
  1098. set_bit(hwc->idx, used_mask);
  1099. if (assign)
  1100. assign[i] = hwc->idx;
  1101. }
  1102. if (!num)
  1103. goto done;
  1104. /*
  1105. * begin slow path
  1106. */
  1107. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  1108. /*
  1109. * weight = number of possible counters
  1110. *
  1111. * 1 = most constrained, only works on one counter
  1112. * wmax = least constrained, works on any counter
  1113. *
  1114. * assign events to counters starting with most
  1115. * constrained events.
  1116. */
  1117. wmax = x86_pmu.num_events;
  1118. /*
  1119. * when fixed event counters are present,
  1120. * wmax is incremented by 1 to account
  1121. * for one more choice
  1122. */
  1123. if (x86_pmu.num_events_fixed)
  1124. wmax++;
  1125. for (w = 1, num = n; num && w <= wmax; w++) {
  1126. /* for each event */
  1127. for (i = 0; num && i < n; i++) {
  1128. c = constraints[i];
  1129. hwc = &cpuc->event_list[i]->hw;
  1130. weight = bitmap_weight(c, X86_PMC_IDX_MAX);
  1131. if (weight != w)
  1132. continue;
  1133. for_each_bit(j, c, X86_PMC_IDX_MAX) {
  1134. if (!test_bit(j, used_mask))
  1135. break;
  1136. }
  1137. if (j == X86_PMC_IDX_MAX)
  1138. break;
  1139. #if 0
  1140. pr_debug("CPU%d slow config=0x%llx idx=%d assign=%c\n",
  1141. smp_processor_id(),
  1142. hwc->config,
  1143. j,
  1144. assign ? 'y' : 'n');
  1145. #endif
  1146. set_bit(j, used_mask);
  1147. if (assign)
  1148. assign[i] = j;
  1149. num--;
  1150. }
  1151. }
  1152. done:
  1153. /*
  1154. * scheduling failed or is just a simulation,
  1155. * free resources if necessary
  1156. */
  1157. if (!assign || num) {
  1158. for (i = 0; i < n; i++) {
  1159. if (x86_pmu.put_event_constraints)
  1160. x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
  1161. }
  1162. }
  1163. return num ? -ENOSPC : 0;
  1164. }
  1165. /*
  1166. * dogrp: true if must collect siblings events (group)
  1167. * returns total number of events and error code
  1168. */
  1169. static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
  1170. {
  1171. struct perf_event *event;
  1172. int n, max_count;
  1173. max_count = x86_pmu.num_events + x86_pmu.num_events_fixed;
  1174. /* current number of events already accepted */
  1175. n = cpuc->n_events;
  1176. if (is_x86_event(leader)) {
  1177. if (n >= max_count)
  1178. return -ENOSPC;
  1179. cpuc->event_list[n] = leader;
  1180. n++;
  1181. }
  1182. if (!dogrp)
  1183. return n;
  1184. list_for_each_entry(event, &leader->sibling_list, group_entry) {
  1185. if (!is_x86_event(event) ||
  1186. event->state <= PERF_EVENT_STATE_OFF)
  1187. continue;
  1188. if (n >= max_count)
  1189. return -ENOSPC;
  1190. cpuc->event_list[n] = event;
  1191. n++;
  1192. }
  1193. return n;
  1194. }
  1195. static inline void x86_assign_hw_event(struct perf_event *event,
  1196. struct hw_perf_event *hwc, int idx)
  1197. {
  1198. hwc->idx = idx;
  1199. if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
  1200. hwc->config_base = 0;
  1201. hwc->event_base = 0;
  1202. } else if (hwc->idx >= X86_PMC_IDX_FIXED) {
  1203. hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
  1204. /*
  1205. * We set it so that event_base + idx in wrmsr/rdmsr maps to
  1206. * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
  1207. */
  1208. hwc->event_base =
  1209. MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
  1210. } else {
  1211. hwc->config_base = x86_pmu.eventsel;
  1212. hwc->event_base = x86_pmu.perfctr;
  1213. }
  1214. }
  1215. void hw_perf_enable(void)
  1216. {
  1217. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1218. struct perf_event *event;
  1219. struct hw_perf_event *hwc;
  1220. int i;
  1221. if (!x86_pmu_initialized())
  1222. return;
  1223. if (cpuc->n_added) {
  1224. /*
  1225. * apply assignment obtained either from
  1226. * hw_perf_group_sched_in() or x86_pmu_enable()
  1227. *
  1228. * step1: save events moving to new counters
  1229. * step2: reprogram moved events into new counters
  1230. */
  1231. for (i = 0; i < cpuc->n_events; i++) {
  1232. event = cpuc->event_list[i];
  1233. hwc = &event->hw;
  1234. if (hwc->idx == -1 || hwc->idx == cpuc->assign[i])
  1235. continue;
  1236. x86_pmu.disable(hwc, hwc->idx);
  1237. clear_bit(hwc->idx, cpuc->active_mask);
  1238. barrier();
  1239. cpuc->events[hwc->idx] = NULL;
  1240. x86_perf_event_update(event, hwc, hwc->idx);
  1241. hwc->idx = -1;
  1242. }
  1243. for (i = 0; i < cpuc->n_events; i++) {
  1244. event = cpuc->event_list[i];
  1245. hwc = &event->hw;
  1246. if (hwc->idx == -1) {
  1247. x86_assign_hw_event(event, hwc, cpuc->assign[i]);
  1248. x86_perf_event_set_period(event, hwc, hwc->idx);
  1249. }
  1250. /*
  1251. * need to mark as active because x86_pmu_disable()
  1252. * clear active_mask and eventsp[] yet it preserves
  1253. * idx
  1254. */
  1255. set_bit(hwc->idx, cpuc->active_mask);
  1256. cpuc->events[hwc->idx] = event;
  1257. x86_pmu.enable(hwc, hwc->idx);
  1258. perf_event_update_userpage(event);
  1259. }
  1260. cpuc->n_added = 0;
  1261. perf_events_lapic_init();
  1262. }
  1263. x86_pmu.enable_all();
  1264. }
  1265. static inline u64 intel_pmu_get_status(void)
  1266. {
  1267. u64 status;
  1268. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  1269. return status;
  1270. }
  1271. static inline void intel_pmu_ack_status(u64 ack)
  1272. {
  1273. wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
  1274. }
  1275. static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1276. {
  1277. (void)checking_wrmsrl(hwc->config_base + idx,
  1278. hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
  1279. }
  1280. static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1281. {
  1282. (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
  1283. }
  1284. static inline void
  1285. intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
  1286. {
  1287. int idx = __idx - X86_PMC_IDX_FIXED;
  1288. u64 ctrl_val, mask;
  1289. mask = 0xfULL << (idx * 4);
  1290. rdmsrl(hwc->config_base, ctrl_val);
  1291. ctrl_val &= ~mask;
  1292. (void)checking_wrmsrl(hwc->config_base, ctrl_val);
  1293. }
  1294. static inline void
  1295. p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1296. {
  1297. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1298. u64 val = P6_NOP_EVENT;
  1299. if (cpuc->enabled)
  1300. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1301. (void)checking_wrmsrl(hwc->config_base + idx, val);
  1302. }
  1303. static inline void
  1304. intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1305. {
  1306. if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
  1307. intel_pmu_disable_bts();
  1308. return;
  1309. }
  1310. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1311. intel_pmu_disable_fixed(hwc, idx);
  1312. return;
  1313. }
  1314. x86_pmu_disable_event(hwc, idx);
  1315. }
  1316. static inline void
  1317. amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1318. {
  1319. x86_pmu_disable_event(hwc, idx);
  1320. }
  1321. static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
  1322. /*
  1323. * Set the next IRQ period, based on the hwc->period_left value.
  1324. * To be called with the event disabled in hw:
  1325. */
  1326. static int
  1327. x86_perf_event_set_period(struct perf_event *event,
  1328. struct hw_perf_event *hwc, int idx)
  1329. {
  1330. s64 left = atomic64_read(&hwc->period_left);
  1331. s64 period = hwc->sample_period;
  1332. int err, ret = 0;
  1333. if (idx == X86_PMC_IDX_FIXED_BTS)
  1334. return 0;
  1335. /*
  1336. * If we are way outside a reasonable range then just skip forward:
  1337. */
  1338. if (unlikely(left <= -period)) {
  1339. left = period;
  1340. atomic64_set(&hwc->period_left, left);
  1341. hwc->last_period = period;
  1342. ret = 1;
  1343. }
  1344. if (unlikely(left <= 0)) {
  1345. left += period;
  1346. atomic64_set(&hwc->period_left, left);
  1347. hwc->last_period = period;
  1348. ret = 1;
  1349. }
  1350. /*
  1351. * Quirk: certain CPUs dont like it if just 1 hw_event is left:
  1352. */
  1353. if (unlikely(left < 2))
  1354. left = 2;
  1355. if (left > x86_pmu.max_period)
  1356. left = x86_pmu.max_period;
  1357. per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
  1358. /*
  1359. * The hw event starts counting from this event offset,
  1360. * mark it to be able to extra future deltas:
  1361. */
  1362. atomic64_set(&hwc->prev_count, (u64)-left);
  1363. err = checking_wrmsrl(hwc->event_base + idx,
  1364. (u64)(-left) & x86_pmu.event_mask);
  1365. perf_event_update_userpage(event);
  1366. return ret;
  1367. }
  1368. static inline void
  1369. intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
  1370. {
  1371. int idx = __idx - X86_PMC_IDX_FIXED;
  1372. u64 ctrl_val, bits, mask;
  1373. int err;
  1374. /*
  1375. * Enable IRQ generation (0x8),
  1376. * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
  1377. * if requested:
  1378. */
  1379. bits = 0x8ULL;
  1380. if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
  1381. bits |= 0x2;
  1382. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  1383. bits |= 0x1;
  1384. bits <<= (idx * 4);
  1385. mask = 0xfULL << (idx * 4);
  1386. rdmsrl(hwc->config_base, ctrl_val);
  1387. ctrl_val &= ~mask;
  1388. ctrl_val |= bits;
  1389. err = checking_wrmsrl(hwc->config_base, ctrl_val);
  1390. }
  1391. static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1392. {
  1393. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1394. u64 val;
  1395. val = hwc->config;
  1396. if (cpuc->enabled)
  1397. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1398. (void)checking_wrmsrl(hwc->config_base + idx, val);
  1399. }
  1400. static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1401. {
  1402. if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
  1403. if (!__get_cpu_var(cpu_hw_events).enabled)
  1404. return;
  1405. intel_pmu_enable_bts(hwc->config);
  1406. return;
  1407. }
  1408. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1409. intel_pmu_enable_fixed(hwc, idx);
  1410. return;
  1411. }
  1412. x86_pmu_enable_event(hwc, idx);
  1413. }
  1414. static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1415. {
  1416. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1417. if (cpuc->enabled)
  1418. x86_pmu_enable_event(hwc, idx);
  1419. }
  1420. /*
  1421. * activate a single event
  1422. *
  1423. * The event is added to the group of enabled events
  1424. * but only if it can be scehduled with existing events.
  1425. *
  1426. * Called with PMU disabled. If successful and return value 1,
  1427. * then guaranteed to call perf_enable() and hw_perf_enable()
  1428. */
  1429. static int x86_pmu_enable(struct perf_event *event)
  1430. {
  1431. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1432. struct hw_perf_event *hwc;
  1433. int assign[X86_PMC_IDX_MAX];
  1434. int n, n0, ret;
  1435. hwc = &event->hw;
  1436. n0 = cpuc->n_events;
  1437. n = collect_events(cpuc, event, false);
  1438. if (n < 0)
  1439. return n;
  1440. ret = x86_schedule_events(cpuc, n, assign);
  1441. if (ret)
  1442. return ret;
  1443. /*
  1444. * copy new assignment, now we know it is possible
  1445. * will be used by hw_perf_enable()
  1446. */
  1447. memcpy(cpuc->assign, assign, n*sizeof(int));
  1448. cpuc->n_events = n;
  1449. cpuc->n_added = n - n0;
  1450. if (hwc->idx != -1)
  1451. x86_perf_event_set_period(event, hwc, hwc->idx);
  1452. return 0;
  1453. }
  1454. static void x86_pmu_unthrottle(struct perf_event *event)
  1455. {
  1456. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1457. struct hw_perf_event *hwc = &event->hw;
  1458. if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
  1459. cpuc->events[hwc->idx] != event))
  1460. return;
  1461. x86_pmu.enable(hwc, hwc->idx);
  1462. }
  1463. void perf_event_print_debug(void)
  1464. {
  1465. u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
  1466. struct cpu_hw_events *cpuc;
  1467. unsigned long flags;
  1468. int cpu, idx;
  1469. if (!x86_pmu.num_events)
  1470. return;
  1471. local_irq_save(flags);
  1472. cpu = smp_processor_id();
  1473. cpuc = &per_cpu(cpu_hw_events, cpu);
  1474. if (x86_pmu.version >= 2) {
  1475. rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
  1476. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  1477. rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
  1478. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
  1479. pr_info("\n");
  1480. pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
  1481. pr_info("CPU#%d: status: %016llx\n", cpu, status);
  1482. pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
  1483. pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
  1484. }
  1485. pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
  1486. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1487. rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
  1488. rdmsrl(x86_pmu.perfctr + idx, pmc_count);
  1489. prev_left = per_cpu(pmc_prev_left[idx], cpu);
  1490. pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
  1491. cpu, idx, pmc_ctrl);
  1492. pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
  1493. cpu, idx, pmc_count);
  1494. pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
  1495. cpu, idx, prev_left);
  1496. }
  1497. for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
  1498. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
  1499. pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
  1500. cpu, idx, pmc_count);
  1501. }
  1502. local_irq_restore(flags);
  1503. }
  1504. static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
  1505. {
  1506. struct debug_store *ds = cpuc->ds;
  1507. struct bts_record {
  1508. u64 from;
  1509. u64 to;
  1510. u64 flags;
  1511. };
  1512. struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
  1513. struct bts_record *at, *top;
  1514. struct perf_output_handle handle;
  1515. struct perf_event_header header;
  1516. struct perf_sample_data data;
  1517. struct pt_regs regs;
  1518. if (!event)
  1519. return;
  1520. if (!ds)
  1521. return;
  1522. at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
  1523. top = (struct bts_record *)(unsigned long)ds->bts_index;
  1524. if (top <= at)
  1525. return;
  1526. ds->bts_index = ds->bts_buffer_base;
  1527. data.period = event->hw.last_period;
  1528. data.addr = 0;
  1529. data.raw = NULL;
  1530. regs.ip = 0;
  1531. /*
  1532. * Prepare a generic sample, i.e. fill in the invariant fields.
  1533. * We will overwrite the from and to address before we output
  1534. * the sample.
  1535. */
  1536. perf_prepare_sample(&header, &data, event, &regs);
  1537. if (perf_output_begin(&handle, event,
  1538. header.size * (top - at), 1, 1))
  1539. return;
  1540. for (; at < top; at++) {
  1541. data.ip = at->from;
  1542. data.addr = at->to;
  1543. perf_output_sample(&handle, &header, &data, event);
  1544. }
  1545. perf_output_end(&handle);
  1546. /* There's new data available. */
  1547. event->hw.interrupts++;
  1548. event->pending_kill = POLL_IN;
  1549. }
  1550. static void x86_pmu_disable(struct perf_event *event)
  1551. {
  1552. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1553. struct hw_perf_event *hwc = &event->hw;
  1554. int i, idx = hwc->idx;
  1555. /*
  1556. * Must be done before we disable, otherwise the nmi handler
  1557. * could reenable again:
  1558. */
  1559. clear_bit(idx, cpuc->active_mask);
  1560. x86_pmu.disable(hwc, idx);
  1561. /*
  1562. * Make sure the cleared pointer becomes visible before we
  1563. * (potentially) free the event:
  1564. */
  1565. barrier();
  1566. /*
  1567. * Drain the remaining delta count out of a event
  1568. * that we are disabling:
  1569. */
  1570. x86_perf_event_update(event, hwc, idx);
  1571. /* Drain the remaining BTS records. */
  1572. if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
  1573. intel_pmu_drain_bts_buffer(cpuc);
  1574. cpuc->events[idx] = NULL;
  1575. for (i = 0; i < cpuc->n_events; i++) {
  1576. if (event == cpuc->event_list[i]) {
  1577. if (x86_pmu.put_event_constraints)
  1578. x86_pmu.put_event_constraints(cpuc, event);
  1579. while (++i < cpuc->n_events)
  1580. cpuc->event_list[i-1] = cpuc->event_list[i];
  1581. --cpuc->n_events;
  1582. }
  1583. }
  1584. perf_event_update_userpage(event);
  1585. }
  1586. /*
  1587. * Save and restart an expired event. Called by NMI contexts,
  1588. * so it has to be careful about preempting normal event ops:
  1589. */
  1590. static int intel_pmu_save_and_restart(struct perf_event *event)
  1591. {
  1592. struct hw_perf_event *hwc = &event->hw;
  1593. int idx = hwc->idx;
  1594. int ret;
  1595. x86_perf_event_update(event, hwc, idx);
  1596. ret = x86_perf_event_set_period(event, hwc, idx);
  1597. if (event->state == PERF_EVENT_STATE_ACTIVE)
  1598. intel_pmu_enable_event(hwc, idx);
  1599. return ret;
  1600. }
  1601. static void intel_pmu_reset(void)
  1602. {
  1603. struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
  1604. unsigned long flags;
  1605. int idx;
  1606. if (!x86_pmu.num_events)
  1607. return;
  1608. local_irq_save(flags);
  1609. printk("clearing PMU state on CPU#%d\n", smp_processor_id());
  1610. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1611. checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
  1612. checking_wrmsrl(x86_pmu.perfctr + idx, 0ull);
  1613. }
  1614. for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
  1615. checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
  1616. }
  1617. if (ds)
  1618. ds->bts_index = ds->bts_buffer_base;
  1619. local_irq_restore(flags);
  1620. }
  1621. static int p6_pmu_handle_irq(struct pt_regs *regs)
  1622. {
  1623. struct perf_sample_data data;
  1624. struct cpu_hw_events *cpuc;
  1625. struct perf_event *event;
  1626. struct hw_perf_event *hwc;
  1627. int idx, handled = 0;
  1628. u64 val;
  1629. data.addr = 0;
  1630. data.raw = NULL;
  1631. cpuc = &__get_cpu_var(cpu_hw_events);
  1632. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1633. if (!test_bit(idx, cpuc->active_mask))
  1634. continue;
  1635. event = cpuc->events[idx];
  1636. hwc = &event->hw;
  1637. val = x86_perf_event_update(event, hwc, idx);
  1638. if (val & (1ULL << (x86_pmu.event_bits - 1)))
  1639. continue;
  1640. /*
  1641. * event overflow
  1642. */
  1643. handled = 1;
  1644. data.period = event->hw.last_period;
  1645. if (!x86_perf_event_set_period(event, hwc, idx))
  1646. continue;
  1647. if (perf_event_overflow(event, 1, &data, regs))
  1648. p6_pmu_disable_event(hwc, idx);
  1649. }
  1650. if (handled)
  1651. inc_irq_stat(apic_perf_irqs);
  1652. return handled;
  1653. }
  1654. /*
  1655. * This handler is triggered by the local APIC, so the APIC IRQ handling
  1656. * rules apply:
  1657. */
  1658. static int intel_pmu_handle_irq(struct pt_regs *regs)
  1659. {
  1660. struct perf_sample_data data;
  1661. struct cpu_hw_events *cpuc;
  1662. int bit, loops;
  1663. u64 ack, status;
  1664. data.addr = 0;
  1665. data.raw = NULL;
  1666. cpuc = &__get_cpu_var(cpu_hw_events);
  1667. perf_disable();
  1668. intel_pmu_drain_bts_buffer(cpuc);
  1669. status = intel_pmu_get_status();
  1670. if (!status) {
  1671. perf_enable();
  1672. return 0;
  1673. }
  1674. loops = 0;
  1675. again:
  1676. if (++loops > 100) {
  1677. WARN_ONCE(1, "perfevents: irq loop stuck!\n");
  1678. perf_event_print_debug();
  1679. intel_pmu_reset();
  1680. perf_enable();
  1681. return 1;
  1682. }
  1683. inc_irq_stat(apic_perf_irqs);
  1684. ack = status;
  1685. for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
  1686. struct perf_event *event = cpuc->events[bit];
  1687. clear_bit(bit, (unsigned long *) &status);
  1688. if (!test_bit(bit, cpuc->active_mask))
  1689. continue;
  1690. if (!intel_pmu_save_and_restart(event))
  1691. continue;
  1692. data.period = event->hw.last_period;
  1693. if (perf_event_overflow(event, 1, &data, regs))
  1694. intel_pmu_disable_event(&event->hw, bit);
  1695. }
  1696. intel_pmu_ack_status(ack);
  1697. /*
  1698. * Repeat if there is more work to be done:
  1699. */
  1700. status = intel_pmu_get_status();
  1701. if (status)
  1702. goto again;
  1703. perf_enable();
  1704. return 1;
  1705. }
  1706. static int amd_pmu_handle_irq(struct pt_regs *regs)
  1707. {
  1708. struct perf_sample_data data;
  1709. struct cpu_hw_events *cpuc;
  1710. struct perf_event *event;
  1711. struct hw_perf_event *hwc;
  1712. int idx, handled = 0;
  1713. u64 val;
  1714. data.addr = 0;
  1715. data.raw = NULL;
  1716. cpuc = &__get_cpu_var(cpu_hw_events);
  1717. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1718. if (!test_bit(idx, cpuc->active_mask))
  1719. continue;
  1720. event = cpuc->events[idx];
  1721. hwc = &event->hw;
  1722. val = x86_perf_event_update(event, hwc, idx);
  1723. if (val & (1ULL << (x86_pmu.event_bits - 1)))
  1724. continue;
  1725. /*
  1726. * event overflow
  1727. */
  1728. handled = 1;
  1729. data.period = event->hw.last_period;
  1730. if (!x86_perf_event_set_period(event, hwc, idx))
  1731. continue;
  1732. if (perf_event_overflow(event, 1, &data, regs))
  1733. amd_pmu_disable_event(hwc, idx);
  1734. }
  1735. if (handled)
  1736. inc_irq_stat(apic_perf_irqs);
  1737. return handled;
  1738. }
  1739. void smp_perf_pending_interrupt(struct pt_regs *regs)
  1740. {
  1741. irq_enter();
  1742. ack_APIC_irq();
  1743. inc_irq_stat(apic_pending_irqs);
  1744. perf_event_do_pending();
  1745. irq_exit();
  1746. }
  1747. void set_perf_event_pending(void)
  1748. {
  1749. #ifdef CONFIG_X86_LOCAL_APIC
  1750. if (!x86_pmu.apic || !x86_pmu_initialized())
  1751. return;
  1752. apic->send_IPI_self(LOCAL_PENDING_VECTOR);
  1753. #endif
  1754. }
  1755. void perf_events_lapic_init(void)
  1756. {
  1757. #ifdef CONFIG_X86_LOCAL_APIC
  1758. if (!x86_pmu.apic || !x86_pmu_initialized())
  1759. return;
  1760. /*
  1761. * Always use NMI for PMU
  1762. */
  1763. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1764. #endif
  1765. }
  1766. static int __kprobes
  1767. perf_event_nmi_handler(struct notifier_block *self,
  1768. unsigned long cmd, void *__args)
  1769. {
  1770. struct die_args *args = __args;
  1771. struct pt_regs *regs;
  1772. if (!atomic_read(&active_events))
  1773. return NOTIFY_DONE;
  1774. switch (cmd) {
  1775. case DIE_NMI:
  1776. case DIE_NMI_IPI:
  1777. break;
  1778. default:
  1779. return NOTIFY_DONE;
  1780. }
  1781. regs = args->regs;
  1782. #ifdef CONFIG_X86_LOCAL_APIC
  1783. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1784. #endif
  1785. /*
  1786. * Can't rely on the handled return value to say it was our NMI, two
  1787. * events could trigger 'simultaneously' raising two back-to-back NMIs.
  1788. *
  1789. * If the first NMI handles both, the latter will be empty and daze
  1790. * the CPU.
  1791. */
  1792. x86_pmu.handle_irq(regs);
  1793. return NOTIFY_STOP;
  1794. }
  1795. static struct event_constraint bts_constraint =
  1796. EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
  1797. static int intel_special_constraints(struct perf_event *event,
  1798. unsigned long *idxmsk)
  1799. {
  1800. unsigned int hw_event;
  1801. hw_event = event->hw.config & INTEL_ARCH_EVENT_MASK;
  1802. if (unlikely((hw_event ==
  1803. x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
  1804. (event->hw.sample_period == 1))) {
  1805. bitmap_copy((unsigned long *)idxmsk,
  1806. (unsigned long *)bts_constraint.idxmsk,
  1807. X86_PMC_IDX_MAX);
  1808. return 1;
  1809. }
  1810. return 0;
  1811. }
  1812. static void intel_get_event_constraints(struct cpu_hw_events *cpuc,
  1813. struct perf_event *event,
  1814. unsigned long *idxmsk)
  1815. {
  1816. const struct event_constraint *c;
  1817. /*
  1818. * cleanup bitmask
  1819. */
  1820. bitmap_zero(idxmsk, X86_PMC_IDX_MAX);
  1821. if (intel_special_constraints(event, idxmsk))
  1822. return;
  1823. if (x86_pmu.event_constraints) {
  1824. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1825. if ((event->hw.config & c->cmask) == c->code) {
  1826. bitmap_copy(idxmsk, c->idxmsk, X86_PMC_IDX_MAX);
  1827. return;
  1828. }
  1829. }
  1830. }
  1831. /* no constraints, means supports all generic counters */
  1832. bitmap_fill((unsigned long *)idxmsk, x86_pmu.num_events);
  1833. }
  1834. static void amd_get_event_constraints(struct cpu_hw_events *cpuc,
  1835. struct perf_event *event,
  1836. unsigned long *idxmsk)
  1837. {
  1838. /* no constraints, means supports all generic counters */
  1839. bitmap_fill(idxmsk, x86_pmu.num_events);
  1840. }
  1841. static int x86_event_sched_in(struct perf_event *event,
  1842. struct perf_cpu_context *cpuctx, int cpu)
  1843. {
  1844. int ret = 0;
  1845. event->state = PERF_EVENT_STATE_ACTIVE;
  1846. event->oncpu = cpu;
  1847. event->tstamp_running += event->ctx->time - event->tstamp_stopped;
  1848. if (!is_x86_event(event))
  1849. ret = event->pmu->enable(event);
  1850. if (!ret && !is_software_event(event))
  1851. cpuctx->active_oncpu++;
  1852. if (!ret && event->attr.exclusive)
  1853. cpuctx->exclusive = 1;
  1854. return ret;
  1855. }
  1856. static void x86_event_sched_out(struct perf_event *event,
  1857. struct perf_cpu_context *cpuctx, int cpu)
  1858. {
  1859. event->state = PERF_EVENT_STATE_INACTIVE;
  1860. event->oncpu = -1;
  1861. if (!is_x86_event(event))
  1862. event->pmu->disable(event);
  1863. event->tstamp_running -= event->ctx->time - event->tstamp_stopped;
  1864. if (!is_software_event(event))
  1865. cpuctx->active_oncpu--;
  1866. if (event->attr.exclusive || !cpuctx->active_oncpu)
  1867. cpuctx->exclusive = 0;
  1868. }
  1869. /*
  1870. * Called to enable a whole group of events.
  1871. * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
  1872. * Assumes the caller has disabled interrupts and has
  1873. * frozen the PMU with hw_perf_save_disable.
  1874. *
  1875. * called with PMU disabled. If successful and return value 1,
  1876. * then guaranteed to call perf_enable() and hw_perf_enable()
  1877. */
  1878. int hw_perf_group_sched_in(struct perf_event *leader,
  1879. struct perf_cpu_context *cpuctx,
  1880. struct perf_event_context *ctx, int cpu)
  1881. {
  1882. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1883. struct perf_event *sub;
  1884. int assign[X86_PMC_IDX_MAX];
  1885. int n0, n1, ret;
  1886. /* n0 = total number of events */
  1887. n0 = collect_events(cpuc, leader, true);
  1888. if (n0 < 0)
  1889. return n0;
  1890. ret = x86_schedule_events(cpuc, n0, assign);
  1891. if (ret)
  1892. return ret;
  1893. ret = x86_event_sched_in(leader, cpuctx, cpu);
  1894. if (ret)
  1895. return ret;
  1896. n1 = 1;
  1897. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  1898. if (sub->state > PERF_EVENT_STATE_OFF) {
  1899. ret = x86_event_sched_in(sub, cpuctx, cpu);
  1900. if (ret)
  1901. goto undo;
  1902. ++n1;
  1903. }
  1904. }
  1905. /*
  1906. * copy new assignment, now we know it is possible
  1907. * will be used by hw_perf_enable()
  1908. */
  1909. memcpy(cpuc->assign, assign, n0*sizeof(int));
  1910. cpuc->n_events = n0;
  1911. cpuc->n_added = n1;
  1912. ctx->nr_active += n1;
  1913. /*
  1914. * 1 means successful and events are active
  1915. * This is not quite true because we defer
  1916. * actual activation until hw_perf_enable() but
  1917. * this way we* ensure caller won't try to enable
  1918. * individual events
  1919. */
  1920. return 1;
  1921. undo:
  1922. x86_event_sched_out(leader, cpuctx, cpu);
  1923. n0 = 1;
  1924. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  1925. if (sub->state == PERF_EVENT_STATE_ACTIVE) {
  1926. x86_event_sched_out(sub, cpuctx, cpu);
  1927. if (++n0 == n1)
  1928. break;
  1929. }
  1930. }
  1931. return ret;
  1932. }
  1933. static __read_mostly struct notifier_block perf_event_nmi_notifier = {
  1934. .notifier_call = perf_event_nmi_handler,
  1935. .next = NULL,
  1936. .priority = 1
  1937. };
  1938. static __initconst struct x86_pmu p6_pmu = {
  1939. .name = "p6",
  1940. .handle_irq = p6_pmu_handle_irq,
  1941. .disable_all = p6_pmu_disable_all,
  1942. .enable_all = p6_pmu_enable_all,
  1943. .enable = p6_pmu_enable_event,
  1944. .disable = p6_pmu_disable_event,
  1945. .eventsel = MSR_P6_EVNTSEL0,
  1946. .perfctr = MSR_P6_PERFCTR0,
  1947. .event_map = p6_pmu_event_map,
  1948. .raw_event = p6_pmu_raw_event,
  1949. .max_events = ARRAY_SIZE(p6_perfmon_event_map),
  1950. .apic = 1,
  1951. .max_period = (1ULL << 31) - 1,
  1952. .version = 0,
  1953. .num_events = 2,
  1954. /*
  1955. * Events have 40 bits implemented. However they are designed such
  1956. * that bits [32-39] are sign extensions of bit 31. As such the
  1957. * effective width of a event for P6-like PMU is 32 bits only.
  1958. *
  1959. * See IA-32 Intel Architecture Software developer manual Vol 3B
  1960. */
  1961. .event_bits = 32,
  1962. .event_mask = (1ULL << 32) - 1,
  1963. .get_event_constraints = intel_get_event_constraints,
  1964. .event_constraints = intel_p6_event_constraints
  1965. };
  1966. static __initconst struct x86_pmu intel_pmu = {
  1967. .name = "Intel",
  1968. .handle_irq = intel_pmu_handle_irq,
  1969. .disable_all = intel_pmu_disable_all,
  1970. .enable_all = intel_pmu_enable_all,
  1971. .enable = intel_pmu_enable_event,
  1972. .disable = intel_pmu_disable_event,
  1973. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1974. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1975. .event_map = intel_pmu_event_map,
  1976. .raw_event = intel_pmu_raw_event,
  1977. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1978. .apic = 1,
  1979. /*
  1980. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1981. * so we install an artificial 1<<31 period regardless of
  1982. * the generic event period:
  1983. */
  1984. .max_period = (1ULL << 31) - 1,
  1985. .enable_bts = intel_pmu_enable_bts,
  1986. .disable_bts = intel_pmu_disable_bts,
  1987. .get_event_constraints = intel_get_event_constraints
  1988. };
  1989. static __initconst struct x86_pmu amd_pmu = {
  1990. .name = "AMD",
  1991. .handle_irq = amd_pmu_handle_irq,
  1992. .disable_all = amd_pmu_disable_all,
  1993. .enable_all = amd_pmu_enable_all,
  1994. .enable = amd_pmu_enable_event,
  1995. .disable = amd_pmu_disable_event,
  1996. .eventsel = MSR_K7_EVNTSEL0,
  1997. .perfctr = MSR_K7_PERFCTR0,
  1998. .event_map = amd_pmu_event_map,
  1999. .raw_event = amd_pmu_raw_event,
  2000. .max_events = ARRAY_SIZE(amd_perfmon_event_map),
  2001. .num_events = 4,
  2002. .event_bits = 48,
  2003. .event_mask = (1ULL << 48) - 1,
  2004. .apic = 1,
  2005. /* use highest bit to detect overflow */
  2006. .max_period = (1ULL << 47) - 1,
  2007. .get_event_constraints = amd_get_event_constraints
  2008. };
  2009. static __init int p6_pmu_init(void)
  2010. {
  2011. switch (boot_cpu_data.x86_model) {
  2012. case 1:
  2013. case 3: /* Pentium Pro */
  2014. case 5:
  2015. case 6: /* Pentium II */
  2016. case 7:
  2017. case 8:
  2018. case 11: /* Pentium III */
  2019. case 9:
  2020. case 13:
  2021. /* Pentium M */
  2022. break;
  2023. default:
  2024. pr_cont("unsupported p6 CPU model %d ",
  2025. boot_cpu_data.x86_model);
  2026. return -ENODEV;
  2027. }
  2028. x86_pmu = p6_pmu;
  2029. return 0;
  2030. }
  2031. static __init int intel_pmu_init(void)
  2032. {
  2033. union cpuid10_edx edx;
  2034. union cpuid10_eax eax;
  2035. unsigned int unused;
  2036. unsigned int ebx;
  2037. int version;
  2038. if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
  2039. /* check for P6 processor family */
  2040. if (boot_cpu_data.x86 == 6) {
  2041. return p6_pmu_init();
  2042. } else {
  2043. return -ENODEV;
  2044. }
  2045. }
  2046. /*
  2047. * Check whether the Architectural PerfMon supports
  2048. * Branch Misses Retired hw_event or not.
  2049. */
  2050. cpuid(10, &eax.full, &ebx, &unused, &edx.full);
  2051. if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
  2052. return -ENODEV;
  2053. version = eax.split.version_id;
  2054. if (version < 2)
  2055. return -ENODEV;
  2056. x86_pmu = intel_pmu;
  2057. x86_pmu.version = version;
  2058. x86_pmu.num_events = eax.split.num_events;
  2059. x86_pmu.event_bits = eax.split.bit_width;
  2060. x86_pmu.event_mask = (1ULL << eax.split.bit_width) - 1;
  2061. /*
  2062. * Quirk: v2 perfmon does not report fixed-purpose events, so
  2063. * assume at least 3 events:
  2064. */
  2065. x86_pmu.num_events_fixed = max((int)edx.split.num_events_fixed, 3);
  2066. /*
  2067. * Install the hw-cache-events table:
  2068. */
  2069. switch (boot_cpu_data.x86_model) {
  2070. case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
  2071. case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
  2072. case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
  2073. case 29: /* six-core 45 nm xeon "Dunnington" */
  2074. memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
  2075. sizeof(hw_cache_event_ids));
  2076. x86_pmu.event_constraints = intel_core_event_constraints;
  2077. pr_cont("Core2 events, ");
  2078. break;
  2079. case 26:
  2080. memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
  2081. sizeof(hw_cache_event_ids));
  2082. x86_pmu.event_constraints = intel_nehalem_event_constraints;
  2083. pr_cont("Nehalem/Corei7 events, ");
  2084. break;
  2085. case 28:
  2086. memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
  2087. sizeof(hw_cache_event_ids));
  2088. x86_pmu.event_constraints = intel_gen_event_constraints;
  2089. pr_cont("Atom events, ");
  2090. break;
  2091. default:
  2092. /*
  2093. * default constraints for v2 and up
  2094. */
  2095. x86_pmu.event_constraints = intel_gen_event_constraints;
  2096. pr_cont("generic architected perfmon, ");
  2097. }
  2098. return 0;
  2099. }
  2100. static __init int amd_pmu_init(void)
  2101. {
  2102. /* Performance-monitoring supported from K7 and later: */
  2103. if (boot_cpu_data.x86 < 6)
  2104. return -ENODEV;
  2105. x86_pmu = amd_pmu;
  2106. /* Events are common for all AMDs */
  2107. memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
  2108. sizeof(hw_cache_event_ids));
  2109. return 0;
  2110. }
  2111. static void __init pmu_check_apic(void)
  2112. {
  2113. if (cpu_has_apic)
  2114. return;
  2115. x86_pmu.apic = 0;
  2116. pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
  2117. pr_info("no hardware sampling interrupt available.\n");
  2118. }
  2119. void __init init_hw_perf_events(void)
  2120. {
  2121. int err;
  2122. pr_info("Performance Events: ");
  2123. switch (boot_cpu_data.x86_vendor) {
  2124. case X86_VENDOR_INTEL:
  2125. err = intel_pmu_init();
  2126. break;
  2127. case X86_VENDOR_AMD:
  2128. err = amd_pmu_init();
  2129. break;
  2130. default:
  2131. return;
  2132. }
  2133. if (err != 0) {
  2134. pr_cont("no PMU driver, software events only.\n");
  2135. return;
  2136. }
  2137. pmu_check_apic();
  2138. pr_cont("%s PMU driver.\n", x86_pmu.name);
  2139. if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
  2140. WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
  2141. x86_pmu.num_events, X86_PMC_MAX_GENERIC);
  2142. x86_pmu.num_events = X86_PMC_MAX_GENERIC;
  2143. }
  2144. perf_event_mask = (1 << x86_pmu.num_events) - 1;
  2145. perf_max_events = x86_pmu.num_events;
  2146. if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
  2147. WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
  2148. x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
  2149. x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
  2150. }
  2151. perf_event_mask |=
  2152. ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
  2153. x86_pmu.intel_ctrl = perf_event_mask;
  2154. perf_events_lapic_init();
  2155. register_die_notifier(&perf_event_nmi_notifier);
  2156. pr_info("... version: %d\n", x86_pmu.version);
  2157. pr_info("... bit width: %d\n", x86_pmu.event_bits);
  2158. pr_info("... generic registers: %d\n", x86_pmu.num_events);
  2159. pr_info("... value mask: %016Lx\n", x86_pmu.event_mask);
  2160. pr_info("... max period: %016Lx\n", x86_pmu.max_period);
  2161. pr_info("... fixed-purpose events: %d\n", x86_pmu.num_events_fixed);
  2162. pr_info("... event mask: %016Lx\n", perf_event_mask);
  2163. }
  2164. static inline void x86_pmu_read(struct perf_event *event)
  2165. {
  2166. x86_perf_event_update(event, &event->hw, event->hw.idx);
  2167. }
  2168. static const struct pmu pmu = {
  2169. .enable = x86_pmu_enable,
  2170. .disable = x86_pmu_disable,
  2171. .read = x86_pmu_read,
  2172. .unthrottle = x86_pmu_unthrottle,
  2173. };
  2174. /*
  2175. * validate a single event group
  2176. *
  2177. * validation include:
  2178. * - check events are compatible which each other
  2179. * - events do not compete for the same counter
  2180. * - number of events <= number of counters
  2181. *
  2182. * validation ensures the group can be loaded onto the
  2183. * PMU if it was the only group available.
  2184. */
  2185. static int validate_group(struct perf_event *event)
  2186. {
  2187. struct perf_event *leader = event->group_leader;
  2188. struct cpu_hw_events *fake_cpuc;
  2189. int ret, n;
  2190. ret = -ENOMEM;
  2191. fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
  2192. if (!fake_cpuc)
  2193. goto out;
  2194. /*
  2195. * the event is not yet connected with its
  2196. * siblings therefore we must first collect
  2197. * existing siblings, then add the new event
  2198. * before we can simulate the scheduling
  2199. */
  2200. ret = -ENOSPC;
  2201. n = collect_events(fake_cpuc, leader, true);
  2202. if (n < 0)
  2203. goto out_free;
  2204. fake_cpuc->n_events = n;
  2205. n = collect_events(fake_cpuc, event, false);
  2206. if (n < 0)
  2207. goto out_free;
  2208. fake_cpuc->n_events = n;
  2209. ret = x86_schedule_events(fake_cpuc, n, NULL);
  2210. out_free:
  2211. kfree(fake_cpuc);
  2212. out:
  2213. return ret;
  2214. }
  2215. const struct pmu *hw_perf_event_init(struct perf_event *event)
  2216. {
  2217. const struct pmu *tmp;
  2218. int err;
  2219. err = __hw_perf_event_init(event);
  2220. if (!err) {
  2221. /*
  2222. * we temporarily connect event to its pmu
  2223. * such that validate_group() can classify
  2224. * it as an x86 event using is_x86_event()
  2225. */
  2226. tmp = event->pmu;
  2227. event->pmu = &pmu;
  2228. if (event->group_leader != event)
  2229. err = validate_group(event);
  2230. event->pmu = tmp;
  2231. }
  2232. if (err) {
  2233. if (event->destroy)
  2234. event->destroy(event);
  2235. return ERR_PTR(err);
  2236. }
  2237. return &pmu;
  2238. }
  2239. /*
  2240. * callchain support
  2241. */
  2242. static inline
  2243. void callchain_store(struct perf_callchain_entry *entry, u64 ip)
  2244. {
  2245. if (entry->nr < PERF_MAX_STACK_DEPTH)
  2246. entry->ip[entry->nr++] = ip;
  2247. }
  2248. static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
  2249. static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
  2250. static void
  2251. backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
  2252. {
  2253. /* Ignore warnings */
  2254. }
  2255. static void backtrace_warning(void *data, char *msg)
  2256. {
  2257. /* Ignore warnings */
  2258. }
  2259. static int backtrace_stack(void *data, char *name)
  2260. {
  2261. return 0;
  2262. }
  2263. static void backtrace_address(void *data, unsigned long addr, int reliable)
  2264. {
  2265. struct perf_callchain_entry *entry = data;
  2266. if (reliable)
  2267. callchain_store(entry, addr);
  2268. }
  2269. static const struct stacktrace_ops backtrace_ops = {
  2270. .warning = backtrace_warning,
  2271. .warning_symbol = backtrace_warning_symbol,
  2272. .stack = backtrace_stack,
  2273. .address = backtrace_address,
  2274. .walk_stack = print_context_stack_bp,
  2275. };
  2276. #include "../dumpstack.h"
  2277. static void
  2278. perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
  2279. {
  2280. callchain_store(entry, PERF_CONTEXT_KERNEL);
  2281. callchain_store(entry, regs->ip);
  2282. dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
  2283. }
  2284. /*
  2285. * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
  2286. */
  2287. static unsigned long
  2288. copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
  2289. {
  2290. unsigned long offset, addr = (unsigned long)from;
  2291. int type = in_nmi() ? KM_NMI : KM_IRQ0;
  2292. unsigned long size, len = 0;
  2293. struct page *page;
  2294. void *map;
  2295. int ret;
  2296. do {
  2297. ret = __get_user_pages_fast(addr, 1, 0, &page);
  2298. if (!ret)
  2299. break;
  2300. offset = addr & (PAGE_SIZE - 1);
  2301. size = min(PAGE_SIZE - offset, n - len);
  2302. map = kmap_atomic(page, type);
  2303. memcpy(to, map+offset, size);
  2304. kunmap_atomic(map, type);
  2305. put_page(page);
  2306. len += size;
  2307. to += size;
  2308. addr += size;
  2309. } while (len < n);
  2310. return len;
  2311. }
  2312. static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
  2313. {
  2314. unsigned long bytes;
  2315. bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
  2316. return bytes == sizeof(*frame);
  2317. }
  2318. static void
  2319. perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
  2320. {
  2321. struct stack_frame frame;
  2322. const void __user *fp;
  2323. if (!user_mode(regs))
  2324. regs = task_pt_regs(current);
  2325. fp = (void __user *)regs->bp;
  2326. callchain_store(entry, PERF_CONTEXT_USER);
  2327. callchain_store(entry, regs->ip);
  2328. while (entry->nr < PERF_MAX_STACK_DEPTH) {
  2329. frame.next_frame = NULL;
  2330. frame.return_address = 0;
  2331. if (!copy_stack_frame(fp, &frame))
  2332. break;
  2333. if ((unsigned long)fp < regs->sp)
  2334. break;
  2335. callchain_store(entry, frame.return_address);
  2336. fp = frame.next_frame;
  2337. }
  2338. }
  2339. static void
  2340. perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
  2341. {
  2342. int is_user;
  2343. if (!regs)
  2344. return;
  2345. is_user = user_mode(regs);
  2346. if (is_user && current->state != TASK_RUNNING)
  2347. return;
  2348. if (!is_user)
  2349. perf_callchain_kernel(regs, entry);
  2350. if (current->mm)
  2351. perf_callchain_user(regs, entry);
  2352. }
  2353. struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
  2354. {
  2355. struct perf_callchain_entry *entry;
  2356. if (in_nmi())
  2357. entry = &__get_cpu_var(pmc_nmi_entry);
  2358. else
  2359. entry = &__get_cpu_var(pmc_irq_entry);
  2360. entry->nr = 0;
  2361. perf_do_callchain(regs, entry);
  2362. return entry;
  2363. }
  2364. void hw_perf_event_setup_online(int cpu)
  2365. {
  2366. init_debug_store_on_cpu(cpu);
  2367. }