perf_event.c 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821
  1. /*
  2. * Performance events x86 architecture code
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2009 Jaswinder Singh Rajput
  7. * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  8. * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  9. * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10. * Copyright (C) 2009 Google, Inc., Stephane Eranian
  11. *
  12. * For licencing details see kernel-base/COPYING
  13. */
  14. #include <linux/perf_event.h>
  15. #include <linux/capability.h>
  16. #include <linux/notifier.h>
  17. #include <linux/hardirq.h>
  18. #include <linux/kprobes.h>
  19. #include <linux/module.h>
  20. #include <linux/kdebug.h>
  21. #include <linux/sched.h>
  22. #include <linux/uaccess.h>
  23. #include <linux/highmem.h>
  24. #include <linux/cpu.h>
  25. #include <linux/bitops.h>
  26. #include <asm/apic.h>
  27. #include <asm/stacktrace.h>
  28. #include <asm/nmi.h>
  29. static u64 perf_event_mask __read_mostly;
  30. /* The maximal number of PEBS events: */
  31. #define MAX_PEBS_EVENTS 4
  32. /* The size of a BTS record in bytes: */
  33. #define BTS_RECORD_SIZE 24
  34. /* The size of a per-cpu BTS buffer in bytes: */
  35. #define BTS_BUFFER_SIZE (BTS_RECORD_SIZE * 2048)
  36. /* The BTS overflow threshold in bytes from the end of the buffer: */
  37. #define BTS_OVFL_TH (BTS_RECORD_SIZE * 128)
  38. /*
  39. * Bits in the debugctlmsr controlling branch tracing.
  40. */
  41. #define X86_DEBUGCTL_TR (1 << 6)
  42. #define X86_DEBUGCTL_BTS (1 << 7)
  43. #define X86_DEBUGCTL_BTINT (1 << 8)
  44. #define X86_DEBUGCTL_BTS_OFF_OS (1 << 9)
  45. #define X86_DEBUGCTL_BTS_OFF_USR (1 << 10)
  46. /*
  47. * A debug store configuration.
  48. *
  49. * We only support architectures that use 64bit fields.
  50. */
  51. struct debug_store {
  52. u64 bts_buffer_base;
  53. u64 bts_index;
  54. u64 bts_absolute_maximum;
  55. u64 bts_interrupt_threshold;
  56. u64 pebs_buffer_base;
  57. u64 pebs_index;
  58. u64 pebs_absolute_maximum;
  59. u64 pebs_interrupt_threshold;
  60. u64 pebs_event_reset[MAX_PEBS_EVENTS];
  61. };
  62. struct event_constraint {
  63. union {
  64. unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  65. u64 idxmsk64[1];
  66. };
  67. int code;
  68. int cmask;
  69. int weight;
  70. };
  71. struct cpu_hw_events {
  72. struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
  73. unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  74. unsigned long interrupts;
  75. int enabled;
  76. struct debug_store *ds;
  77. int n_events;
  78. int n_added;
  79. int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
  80. struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
  81. };
  82. #define EVENT_CONSTRAINT(c, n, m) { \
  83. { .idxmsk64[0] = (n) }, \
  84. .code = (c), \
  85. .cmask = (m), \
  86. .weight = HWEIGHT64((u64)(n)), \
  87. }
  88. #define INTEL_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
  89. #define FIXED_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, INTEL_ARCH_FIXED_MASK)
  90. #define EVENT_CONSTRAINT_END EVENT_CONSTRAINT(0, 0, 0)
  91. #define for_each_event_constraint(e, c) for ((e) = (c); (e)->cmask; (e)++)
  92. /*
  93. * struct x86_pmu - generic x86 pmu
  94. */
  95. struct x86_pmu {
  96. const char *name;
  97. int version;
  98. int (*handle_irq)(struct pt_regs *);
  99. void (*disable_all)(void);
  100. void (*enable_all)(void);
  101. void (*enable)(struct hw_perf_event *, int);
  102. void (*disable)(struct hw_perf_event *, int);
  103. unsigned eventsel;
  104. unsigned perfctr;
  105. u64 (*event_map)(int);
  106. u64 (*raw_event)(u64);
  107. int max_events;
  108. int num_events;
  109. int num_events_fixed;
  110. int event_bits;
  111. u64 event_mask;
  112. int apic;
  113. u64 max_period;
  114. u64 intel_ctrl;
  115. void (*enable_bts)(u64 config);
  116. void (*disable_bts)(void);
  117. struct event_constraint *
  118. (*get_event_constraints)(struct cpu_hw_events *cpuc,
  119. struct perf_event *event);
  120. void (*put_event_constraints)(struct cpu_hw_events *cpuc,
  121. struct perf_event *event);
  122. struct event_constraint *event_constraints;
  123. };
  124. static struct x86_pmu x86_pmu __read_mostly;
  125. static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  126. .enabled = 1,
  127. };
  128. static int x86_perf_event_set_period(struct perf_event *event,
  129. struct hw_perf_event *hwc, int idx);
  130. /*
  131. * Not sure about some of these
  132. */
  133. static const u64 p6_perfmon_event_map[] =
  134. {
  135. [PERF_COUNT_HW_CPU_CYCLES] = 0x0079,
  136. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  137. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0f2e,
  138. [PERF_COUNT_HW_CACHE_MISSES] = 0x012e,
  139. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  140. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  141. [PERF_COUNT_HW_BUS_CYCLES] = 0x0062,
  142. };
  143. static u64 p6_pmu_event_map(int hw_event)
  144. {
  145. return p6_perfmon_event_map[hw_event];
  146. }
  147. /*
  148. * Event setting that is specified not to count anything.
  149. * We use this to effectively disable a counter.
  150. *
  151. * L2_RQSTS with 0 MESI unit mask.
  152. */
  153. #define P6_NOP_EVENT 0x0000002EULL
  154. static u64 p6_pmu_raw_event(u64 hw_event)
  155. {
  156. #define P6_EVNTSEL_EVENT_MASK 0x000000FFULL
  157. #define P6_EVNTSEL_UNIT_MASK 0x0000FF00ULL
  158. #define P6_EVNTSEL_EDGE_MASK 0x00040000ULL
  159. #define P6_EVNTSEL_INV_MASK 0x00800000ULL
  160. #define P6_EVNTSEL_REG_MASK 0xFF000000ULL
  161. #define P6_EVNTSEL_MASK \
  162. (P6_EVNTSEL_EVENT_MASK | \
  163. P6_EVNTSEL_UNIT_MASK | \
  164. P6_EVNTSEL_EDGE_MASK | \
  165. P6_EVNTSEL_INV_MASK | \
  166. P6_EVNTSEL_REG_MASK)
  167. return hw_event & P6_EVNTSEL_MASK;
  168. }
  169. static struct event_constraint intel_p6_event_constraints[] =
  170. {
  171. INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FLOPS */
  172. INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  173. INTEL_EVENT_CONSTRAINT(0x11, 0x1), /* FP_ASSIST */
  174. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  175. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  176. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  177. EVENT_CONSTRAINT_END
  178. };
  179. /*
  180. * Intel PerfMon v3. Used on Core2 and later.
  181. */
  182. static const u64 intel_perfmon_event_map[] =
  183. {
  184. [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
  185. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  186. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
  187. [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
  188. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  189. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  190. [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
  191. };
  192. static struct event_constraint intel_core_event_constraints[] =
  193. {
  194. FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
  195. FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
  196. INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  197. INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  198. INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  199. INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  200. INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  201. INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
  202. INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  203. INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
  204. INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
  205. EVENT_CONSTRAINT_END
  206. };
  207. static struct event_constraint intel_nehalem_event_constraints[] =
  208. {
  209. FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
  210. FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
  211. INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
  212. INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
  213. INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
  214. INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
  215. INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
  216. INTEL_EVENT_CONSTRAINT(0x4c, 0x3), /* LOAD_HIT_PRE */
  217. INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  218. INTEL_EVENT_CONSTRAINT(0x52, 0x3), /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
  219. INTEL_EVENT_CONSTRAINT(0x53, 0x3), /* L1D_CACHE_LOCK_FB_HIT */
  220. INTEL_EVENT_CONSTRAINT(0xc5, 0x3), /* CACHE_LOCK_CYCLES */
  221. EVENT_CONSTRAINT_END
  222. };
  223. static struct event_constraint intel_gen_event_constraints[] =
  224. {
  225. FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
  226. FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
  227. EVENT_CONSTRAINT_END
  228. };
  229. static u64 intel_pmu_event_map(int hw_event)
  230. {
  231. return intel_perfmon_event_map[hw_event];
  232. }
  233. /*
  234. * Generalized hw caching related hw_event table, filled
  235. * in on a per model basis. A value of 0 means
  236. * 'not supported', -1 means 'hw_event makes no sense on
  237. * this CPU', any other value means the raw hw_event
  238. * ID.
  239. */
  240. #define C(x) PERF_COUNT_HW_CACHE_##x
  241. static u64 __read_mostly hw_cache_event_ids
  242. [PERF_COUNT_HW_CACHE_MAX]
  243. [PERF_COUNT_HW_CACHE_OP_MAX]
  244. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  245. static __initconst u64 nehalem_hw_cache_event_ids
  246. [PERF_COUNT_HW_CACHE_MAX]
  247. [PERF_COUNT_HW_CACHE_OP_MAX]
  248. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  249. {
  250. [ C(L1D) ] = {
  251. [ C(OP_READ) ] = {
  252. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  253. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  254. },
  255. [ C(OP_WRITE) ] = {
  256. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  257. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  258. },
  259. [ C(OP_PREFETCH) ] = {
  260. [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
  261. [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
  262. },
  263. },
  264. [ C(L1I ) ] = {
  265. [ C(OP_READ) ] = {
  266. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  267. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  268. },
  269. [ C(OP_WRITE) ] = {
  270. [ C(RESULT_ACCESS) ] = -1,
  271. [ C(RESULT_MISS) ] = -1,
  272. },
  273. [ C(OP_PREFETCH) ] = {
  274. [ C(RESULT_ACCESS) ] = 0x0,
  275. [ C(RESULT_MISS) ] = 0x0,
  276. },
  277. },
  278. [ C(LL ) ] = {
  279. [ C(OP_READ) ] = {
  280. [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS */
  281. [ C(RESULT_MISS) ] = 0x0224, /* L2_RQSTS.LD_MISS */
  282. },
  283. [ C(OP_WRITE) ] = {
  284. [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS */
  285. [ C(RESULT_MISS) ] = 0x0824, /* L2_RQSTS.RFO_MISS */
  286. },
  287. [ C(OP_PREFETCH) ] = {
  288. [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference */
  289. [ C(RESULT_MISS) ] = 0x412e, /* LLC Misses */
  290. },
  291. },
  292. [ C(DTLB) ] = {
  293. [ C(OP_READ) ] = {
  294. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  295. [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
  296. },
  297. [ C(OP_WRITE) ] = {
  298. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  299. [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
  300. },
  301. [ C(OP_PREFETCH) ] = {
  302. [ C(RESULT_ACCESS) ] = 0x0,
  303. [ C(RESULT_MISS) ] = 0x0,
  304. },
  305. },
  306. [ C(ITLB) ] = {
  307. [ C(OP_READ) ] = {
  308. [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
  309. [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
  310. },
  311. [ C(OP_WRITE) ] = {
  312. [ C(RESULT_ACCESS) ] = -1,
  313. [ C(RESULT_MISS) ] = -1,
  314. },
  315. [ C(OP_PREFETCH) ] = {
  316. [ C(RESULT_ACCESS) ] = -1,
  317. [ C(RESULT_MISS) ] = -1,
  318. },
  319. },
  320. [ C(BPU ) ] = {
  321. [ C(OP_READ) ] = {
  322. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
  323. [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
  324. },
  325. [ C(OP_WRITE) ] = {
  326. [ C(RESULT_ACCESS) ] = -1,
  327. [ C(RESULT_MISS) ] = -1,
  328. },
  329. [ C(OP_PREFETCH) ] = {
  330. [ C(RESULT_ACCESS) ] = -1,
  331. [ C(RESULT_MISS) ] = -1,
  332. },
  333. },
  334. };
  335. static __initconst u64 core2_hw_cache_event_ids
  336. [PERF_COUNT_HW_CACHE_MAX]
  337. [PERF_COUNT_HW_CACHE_OP_MAX]
  338. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  339. {
  340. [ C(L1D) ] = {
  341. [ C(OP_READ) ] = {
  342. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
  343. [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
  344. },
  345. [ C(OP_WRITE) ] = {
  346. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
  347. [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
  348. },
  349. [ C(OP_PREFETCH) ] = {
  350. [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
  351. [ C(RESULT_MISS) ] = 0,
  352. },
  353. },
  354. [ C(L1I ) ] = {
  355. [ C(OP_READ) ] = {
  356. [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
  357. [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
  358. },
  359. [ C(OP_WRITE) ] = {
  360. [ C(RESULT_ACCESS) ] = -1,
  361. [ C(RESULT_MISS) ] = -1,
  362. },
  363. [ C(OP_PREFETCH) ] = {
  364. [ C(RESULT_ACCESS) ] = 0,
  365. [ C(RESULT_MISS) ] = 0,
  366. },
  367. },
  368. [ C(LL ) ] = {
  369. [ C(OP_READ) ] = {
  370. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  371. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  372. },
  373. [ C(OP_WRITE) ] = {
  374. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  375. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  376. },
  377. [ C(OP_PREFETCH) ] = {
  378. [ C(RESULT_ACCESS) ] = 0,
  379. [ C(RESULT_MISS) ] = 0,
  380. },
  381. },
  382. [ C(DTLB) ] = {
  383. [ C(OP_READ) ] = {
  384. [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
  385. [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
  386. },
  387. [ C(OP_WRITE) ] = {
  388. [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
  389. [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
  390. },
  391. [ C(OP_PREFETCH) ] = {
  392. [ C(RESULT_ACCESS) ] = 0,
  393. [ C(RESULT_MISS) ] = 0,
  394. },
  395. },
  396. [ C(ITLB) ] = {
  397. [ C(OP_READ) ] = {
  398. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  399. [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
  400. },
  401. [ C(OP_WRITE) ] = {
  402. [ C(RESULT_ACCESS) ] = -1,
  403. [ C(RESULT_MISS) ] = -1,
  404. },
  405. [ C(OP_PREFETCH) ] = {
  406. [ C(RESULT_ACCESS) ] = -1,
  407. [ C(RESULT_MISS) ] = -1,
  408. },
  409. },
  410. [ C(BPU ) ] = {
  411. [ C(OP_READ) ] = {
  412. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  413. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  414. },
  415. [ C(OP_WRITE) ] = {
  416. [ C(RESULT_ACCESS) ] = -1,
  417. [ C(RESULT_MISS) ] = -1,
  418. },
  419. [ C(OP_PREFETCH) ] = {
  420. [ C(RESULT_ACCESS) ] = -1,
  421. [ C(RESULT_MISS) ] = -1,
  422. },
  423. },
  424. };
  425. static __initconst u64 atom_hw_cache_event_ids
  426. [PERF_COUNT_HW_CACHE_MAX]
  427. [PERF_COUNT_HW_CACHE_OP_MAX]
  428. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  429. {
  430. [ C(L1D) ] = {
  431. [ C(OP_READ) ] = {
  432. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
  433. [ C(RESULT_MISS) ] = 0,
  434. },
  435. [ C(OP_WRITE) ] = {
  436. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
  437. [ C(RESULT_MISS) ] = 0,
  438. },
  439. [ C(OP_PREFETCH) ] = {
  440. [ C(RESULT_ACCESS) ] = 0x0,
  441. [ C(RESULT_MISS) ] = 0,
  442. },
  443. },
  444. [ C(L1I ) ] = {
  445. [ C(OP_READ) ] = {
  446. [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
  447. [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
  448. },
  449. [ C(OP_WRITE) ] = {
  450. [ C(RESULT_ACCESS) ] = -1,
  451. [ C(RESULT_MISS) ] = -1,
  452. },
  453. [ C(OP_PREFETCH) ] = {
  454. [ C(RESULT_ACCESS) ] = 0,
  455. [ C(RESULT_MISS) ] = 0,
  456. },
  457. },
  458. [ C(LL ) ] = {
  459. [ C(OP_READ) ] = {
  460. [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
  461. [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
  462. },
  463. [ C(OP_WRITE) ] = {
  464. [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
  465. [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
  466. },
  467. [ C(OP_PREFETCH) ] = {
  468. [ C(RESULT_ACCESS) ] = 0,
  469. [ C(RESULT_MISS) ] = 0,
  470. },
  471. },
  472. [ C(DTLB) ] = {
  473. [ C(OP_READ) ] = {
  474. [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
  475. [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
  476. },
  477. [ C(OP_WRITE) ] = {
  478. [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
  479. [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
  480. },
  481. [ C(OP_PREFETCH) ] = {
  482. [ C(RESULT_ACCESS) ] = 0,
  483. [ C(RESULT_MISS) ] = 0,
  484. },
  485. },
  486. [ C(ITLB) ] = {
  487. [ C(OP_READ) ] = {
  488. [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
  489. [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
  490. },
  491. [ C(OP_WRITE) ] = {
  492. [ C(RESULT_ACCESS) ] = -1,
  493. [ C(RESULT_MISS) ] = -1,
  494. },
  495. [ C(OP_PREFETCH) ] = {
  496. [ C(RESULT_ACCESS) ] = -1,
  497. [ C(RESULT_MISS) ] = -1,
  498. },
  499. },
  500. [ C(BPU ) ] = {
  501. [ C(OP_READ) ] = {
  502. [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
  503. [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
  504. },
  505. [ C(OP_WRITE) ] = {
  506. [ C(RESULT_ACCESS) ] = -1,
  507. [ C(RESULT_MISS) ] = -1,
  508. },
  509. [ C(OP_PREFETCH) ] = {
  510. [ C(RESULT_ACCESS) ] = -1,
  511. [ C(RESULT_MISS) ] = -1,
  512. },
  513. },
  514. };
  515. static u64 intel_pmu_raw_event(u64 hw_event)
  516. {
  517. #define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
  518. #define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
  519. #define CORE_EVNTSEL_EDGE_MASK 0x00040000ULL
  520. #define CORE_EVNTSEL_INV_MASK 0x00800000ULL
  521. #define CORE_EVNTSEL_REG_MASK 0xFF000000ULL
  522. #define CORE_EVNTSEL_MASK \
  523. (INTEL_ARCH_EVTSEL_MASK | \
  524. INTEL_ARCH_UNIT_MASK | \
  525. INTEL_ARCH_EDGE_MASK | \
  526. INTEL_ARCH_INV_MASK | \
  527. INTEL_ARCH_CNT_MASK)
  528. return hw_event & CORE_EVNTSEL_MASK;
  529. }
  530. static __initconst u64 amd_hw_cache_event_ids
  531. [PERF_COUNT_HW_CACHE_MAX]
  532. [PERF_COUNT_HW_CACHE_OP_MAX]
  533. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  534. {
  535. [ C(L1D) ] = {
  536. [ C(OP_READ) ] = {
  537. [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
  538. [ C(RESULT_MISS) ] = 0x0041, /* Data Cache Misses */
  539. },
  540. [ C(OP_WRITE) ] = {
  541. [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
  542. [ C(RESULT_MISS) ] = 0,
  543. },
  544. [ C(OP_PREFETCH) ] = {
  545. [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */
  546. [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */
  547. },
  548. },
  549. [ C(L1I ) ] = {
  550. [ C(OP_READ) ] = {
  551. [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
  552. [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
  553. },
  554. [ C(OP_WRITE) ] = {
  555. [ C(RESULT_ACCESS) ] = -1,
  556. [ C(RESULT_MISS) ] = -1,
  557. },
  558. [ C(OP_PREFETCH) ] = {
  559. [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
  560. [ C(RESULT_MISS) ] = 0,
  561. },
  562. },
  563. [ C(LL ) ] = {
  564. [ C(OP_READ) ] = {
  565. [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
  566. [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */
  567. },
  568. [ C(OP_WRITE) ] = {
  569. [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */
  570. [ C(RESULT_MISS) ] = 0,
  571. },
  572. [ C(OP_PREFETCH) ] = {
  573. [ C(RESULT_ACCESS) ] = 0,
  574. [ C(RESULT_MISS) ] = 0,
  575. },
  576. },
  577. [ C(DTLB) ] = {
  578. [ C(OP_READ) ] = {
  579. [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
  580. [ C(RESULT_MISS) ] = 0x0046, /* L1 DTLB and L2 DLTB Miss */
  581. },
  582. [ C(OP_WRITE) ] = {
  583. [ C(RESULT_ACCESS) ] = 0,
  584. [ C(RESULT_MISS) ] = 0,
  585. },
  586. [ C(OP_PREFETCH) ] = {
  587. [ C(RESULT_ACCESS) ] = 0,
  588. [ C(RESULT_MISS) ] = 0,
  589. },
  590. },
  591. [ C(ITLB) ] = {
  592. [ C(OP_READ) ] = {
  593. [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
  594. [ C(RESULT_MISS) ] = 0x0085, /* Instr. fetch ITLB misses */
  595. },
  596. [ C(OP_WRITE) ] = {
  597. [ C(RESULT_ACCESS) ] = -1,
  598. [ C(RESULT_MISS) ] = -1,
  599. },
  600. [ C(OP_PREFETCH) ] = {
  601. [ C(RESULT_ACCESS) ] = -1,
  602. [ C(RESULT_MISS) ] = -1,
  603. },
  604. },
  605. [ C(BPU ) ] = {
  606. [ C(OP_READ) ] = {
  607. [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
  608. [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
  609. },
  610. [ C(OP_WRITE) ] = {
  611. [ C(RESULT_ACCESS) ] = -1,
  612. [ C(RESULT_MISS) ] = -1,
  613. },
  614. [ C(OP_PREFETCH) ] = {
  615. [ C(RESULT_ACCESS) ] = -1,
  616. [ C(RESULT_MISS) ] = -1,
  617. },
  618. },
  619. };
  620. /*
  621. * AMD Performance Monitor K7 and later.
  622. */
  623. static const u64 amd_perfmon_event_map[] =
  624. {
  625. [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
  626. [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
  627. [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
  628. [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
  629. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
  630. [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
  631. };
  632. static u64 amd_pmu_event_map(int hw_event)
  633. {
  634. return amd_perfmon_event_map[hw_event];
  635. }
  636. static u64 amd_pmu_raw_event(u64 hw_event)
  637. {
  638. #define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
  639. #define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
  640. #define K7_EVNTSEL_EDGE_MASK 0x000040000ULL
  641. #define K7_EVNTSEL_INV_MASK 0x000800000ULL
  642. #define K7_EVNTSEL_REG_MASK 0x0FF000000ULL
  643. #define K7_EVNTSEL_MASK \
  644. (K7_EVNTSEL_EVENT_MASK | \
  645. K7_EVNTSEL_UNIT_MASK | \
  646. K7_EVNTSEL_EDGE_MASK | \
  647. K7_EVNTSEL_INV_MASK | \
  648. K7_EVNTSEL_REG_MASK)
  649. return hw_event & K7_EVNTSEL_MASK;
  650. }
  651. /*
  652. * Propagate event elapsed time into the generic event.
  653. * Can only be executed on the CPU where the event is active.
  654. * Returns the delta events processed.
  655. */
  656. static u64
  657. x86_perf_event_update(struct perf_event *event,
  658. struct hw_perf_event *hwc, int idx)
  659. {
  660. int shift = 64 - x86_pmu.event_bits;
  661. u64 prev_raw_count, new_raw_count;
  662. s64 delta;
  663. if (idx == X86_PMC_IDX_FIXED_BTS)
  664. return 0;
  665. /*
  666. * Careful: an NMI might modify the previous event value.
  667. *
  668. * Our tactic to handle this is to first atomically read and
  669. * exchange a new raw count - then add that new-prev delta
  670. * count to the generic event atomically:
  671. */
  672. again:
  673. prev_raw_count = atomic64_read(&hwc->prev_count);
  674. rdmsrl(hwc->event_base + idx, new_raw_count);
  675. if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
  676. new_raw_count) != prev_raw_count)
  677. goto again;
  678. /*
  679. * Now we have the new raw value and have updated the prev
  680. * timestamp already. We can now calculate the elapsed delta
  681. * (event-)time and add that to the generic event.
  682. *
  683. * Careful, not all hw sign-extends above the physical width
  684. * of the count.
  685. */
  686. delta = (new_raw_count << shift) - (prev_raw_count << shift);
  687. delta >>= shift;
  688. atomic64_add(delta, &event->count);
  689. atomic64_sub(delta, &hwc->period_left);
  690. return new_raw_count;
  691. }
  692. static atomic_t active_events;
  693. static DEFINE_MUTEX(pmc_reserve_mutex);
  694. static bool reserve_pmc_hardware(void)
  695. {
  696. #ifdef CONFIG_X86_LOCAL_APIC
  697. int i;
  698. if (nmi_watchdog == NMI_LOCAL_APIC)
  699. disable_lapic_nmi_watchdog();
  700. for (i = 0; i < x86_pmu.num_events; i++) {
  701. if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
  702. goto perfctr_fail;
  703. }
  704. for (i = 0; i < x86_pmu.num_events; i++) {
  705. if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
  706. goto eventsel_fail;
  707. }
  708. #endif
  709. return true;
  710. #ifdef CONFIG_X86_LOCAL_APIC
  711. eventsel_fail:
  712. for (i--; i >= 0; i--)
  713. release_evntsel_nmi(x86_pmu.eventsel + i);
  714. i = x86_pmu.num_events;
  715. perfctr_fail:
  716. for (i--; i >= 0; i--)
  717. release_perfctr_nmi(x86_pmu.perfctr + i);
  718. if (nmi_watchdog == NMI_LOCAL_APIC)
  719. enable_lapic_nmi_watchdog();
  720. return false;
  721. #endif
  722. }
  723. static void release_pmc_hardware(void)
  724. {
  725. #ifdef CONFIG_X86_LOCAL_APIC
  726. int i;
  727. for (i = 0; i < x86_pmu.num_events; i++) {
  728. release_perfctr_nmi(x86_pmu.perfctr + i);
  729. release_evntsel_nmi(x86_pmu.eventsel + i);
  730. }
  731. if (nmi_watchdog == NMI_LOCAL_APIC)
  732. enable_lapic_nmi_watchdog();
  733. #endif
  734. }
  735. static inline bool bts_available(void)
  736. {
  737. return x86_pmu.enable_bts != NULL;
  738. }
  739. static inline void init_debug_store_on_cpu(int cpu)
  740. {
  741. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  742. if (!ds)
  743. return;
  744. wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
  745. (u32)((u64)(unsigned long)ds),
  746. (u32)((u64)(unsigned long)ds >> 32));
  747. }
  748. static inline void fini_debug_store_on_cpu(int cpu)
  749. {
  750. if (!per_cpu(cpu_hw_events, cpu).ds)
  751. return;
  752. wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
  753. }
  754. static void release_bts_hardware(void)
  755. {
  756. int cpu;
  757. if (!bts_available())
  758. return;
  759. get_online_cpus();
  760. for_each_online_cpu(cpu)
  761. fini_debug_store_on_cpu(cpu);
  762. for_each_possible_cpu(cpu) {
  763. struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  764. if (!ds)
  765. continue;
  766. per_cpu(cpu_hw_events, cpu).ds = NULL;
  767. kfree((void *)(unsigned long)ds->bts_buffer_base);
  768. kfree(ds);
  769. }
  770. put_online_cpus();
  771. }
  772. static int reserve_bts_hardware(void)
  773. {
  774. int cpu, err = 0;
  775. if (!bts_available())
  776. return 0;
  777. get_online_cpus();
  778. for_each_possible_cpu(cpu) {
  779. struct debug_store *ds;
  780. void *buffer;
  781. err = -ENOMEM;
  782. buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
  783. if (unlikely(!buffer))
  784. break;
  785. ds = kzalloc(sizeof(*ds), GFP_KERNEL);
  786. if (unlikely(!ds)) {
  787. kfree(buffer);
  788. break;
  789. }
  790. ds->bts_buffer_base = (u64)(unsigned long)buffer;
  791. ds->bts_index = ds->bts_buffer_base;
  792. ds->bts_absolute_maximum =
  793. ds->bts_buffer_base + BTS_BUFFER_SIZE;
  794. ds->bts_interrupt_threshold =
  795. ds->bts_absolute_maximum - BTS_OVFL_TH;
  796. per_cpu(cpu_hw_events, cpu).ds = ds;
  797. err = 0;
  798. }
  799. if (err)
  800. release_bts_hardware();
  801. else {
  802. for_each_online_cpu(cpu)
  803. init_debug_store_on_cpu(cpu);
  804. }
  805. put_online_cpus();
  806. return err;
  807. }
  808. static void hw_perf_event_destroy(struct perf_event *event)
  809. {
  810. if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
  811. release_pmc_hardware();
  812. release_bts_hardware();
  813. mutex_unlock(&pmc_reserve_mutex);
  814. }
  815. }
  816. static inline int x86_pmu_initialized(void)
  817. {
  818. return x86_pmu.handle_irq != NULL;
  819. }
  820. static inline int
  821. set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
  822. {
  823. unsigned int cache_type, cache_op, cache_result;
  824. u64 config, val;
  825. config = attr->config;
  826. cache_type = (config >> 0) & 0xff;
  827. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  828. return -EINVAL;
  829. cache_op = (config >> 8) & 0xff;
  830. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  831. return -EINVAL;
  832. cache_result = (config >> 16) & 0xff;
  833. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  834. return -EINVAL;
  835. val = hw_cache_event_ids[cache_type][cache_op][cache_result];
  836. if (val == 0)
  837. return -ENOENT;
  838. if (val == -1)
  839. return -EINVAL;
  840. hwc->config |= val;
  841. return 0;
  842. }
  843. static void intel_pmu_enable_bts(u64 config)
  844. {
  845. unsigned long debugctlmsr;
  846. debugctlmsr = get_debugctlmsr();
  847. debugctlmsr |= X86_DEBUGCTL_TR;
  848. debugctlmsr |= X86_DEBUGCTL_BTS;
  849. debugctlmsr |= X86_DEBUGCTL_BTINT;
  850. if (!(config & ARCH_PERFMON_EVENTSEL_OS))
  851. debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
  852. if (!(config & ARCH_PERFMON_EVENTSEL_USR))
  853. debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
  854. update_debugctlmsr(debugctlmsr);
  855. }
  856. static void intel_pmu_disable_bts(void)
  857. {
  858. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  859. unsigned long debugctlmsr;
  860. if (!cpuc->ds)
  861. return;
  862. debugctlmsr = get_debugctlmsr();
  863. debugctlmsr &=
  864. ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
  865. X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
  866. update_debugctlmsr(debugctlmsr);
  867. }
  868. /*
  869. * Setup the hardware configuration for a given attr_type
  870. */
  871. static int __hw_perf_event_init(struct perf_event *event)
  872. {
  873. struct perf_event_attr *attr = &event->attr;
  874. struct hw_perf_event *hwc = &event->hw;
  875. u64 config;
  876. int err;
  877. if (!x86_pmu_initialized())
  878. return -ENODEV;
  879. err = 0;
  880. if (!atomic_inc_not_zero(&active_events)) {
  881. mutex_lock(&pmc_reserve_mutex);
  882. if (atomic_read(&active_events) == 0) {
  883. if (!reserve_pmc_hardware())
  884. err = -EBUSY;
  885. else
  886. err = reserve_bts_hardware();
  887. }
  888. if (!err)
  889. atomic_inc(&active_events);
  890. mutex_unlock(&pmc_reserve_mutex);
  891. }
  892. if (err)
  893. return err;
  894. event->destroy = hw_perf_event_destroy;
  895. /*
  896. * Generate PMC IRQs:
  897. * (keep 'enabled' bit clear for now)
  898. */
  899. hwc->config = ARCH_PERFMON_EVENTSEL_INT;
  900. hwc->idx = -1;
  901. /*
  902. * Count user and OS events unless requested not to.
  903. */
  904. if (!attr->exclude_user)
  905. hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
  906. if (!attr->exclude_kernel)
  907. hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
  908. if (!hwc->sample_period) {
  909. hwc->sample_period = x86_pmu.max_period;
  910. hwc->last_period = hwc->sample_period;
  911. atomic64_set(&hwc->period_left, hwc->sample_period);
  912. } else {
  913. /*
  914. * If we have a PMU initialized but no APIC
  915. * interrupts, we cannot sample hardware
  916. * events (user-space has to fall back and
  917. * sample via a hrtimer based software event):
  918. */
  919. if (!x86_pmu.apic)
  920. return -EOPNOTSUPP;
  921. }
  922. /*
  923. * Raw hw_event type provide the config in the hw_event structure
  924. */
  925. if (attr->type == PERF_TYPE_RAW) {
  926. hwc->config |= x86_pmu.raw_event(attr->config);
  927. return 0;
  928. }
  929. if (attr->type == PERF_TYPE_HW_CACHE)
  930. return set_ext_hw_attr(hwc, attr);
  931. if (attr->config >= x86_pmu.max_events)
  932. return -EINVAL;
  933. /*
  934. * The generic map:
  935. */
  936. config = x86_pmu.event_map(attr->config);
  937. if (config == 0)
  938. return -ENOENT;
  939. if (config == -1LL)
  940. return -EINVAL;
  941. /*
  942. * Branch tracing:
  943. */
  944. if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
  945. (hwc->sample_period == 1)) {
  946. /* BTS is not supported by this architecture. */
  947. if (!bts_available())
  948. return -EOPNOTSUPP;
  949. /* BTS is currently only allowed for user-mode. */
  950. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  951. return -EOPNOTSUPP;
  952. }
  953. hwc->config |= config;
  954. return 0;
  955. }
  956. static void p6_pmu_disable_all(void)
  957. {
  958. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  959. u64 val;
  960. if (!cpuc->enabled)
  961. return;
  962. cpuc->enabled = 0;
  963. barrier();
  964. /* p6 only has one enable register */
  965. rdmsrl(MSR_P6_EVNTSEL0, val);
  966. val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
  967. wrmsrl(MSR_P6_EVNTSEL0, val);
  968. }
  969. static void intel_pmu_disable_all(void)
  970. {
  971. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  972. if (!cpuc->enabled)
  973. return;
  974. cpuc->enabled = 0;
  975. barrier();
  976. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
  977. if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
  978. intel_pmu_disable_bts();
  979. }
  980. static void amd_pmu_disable_all(void)
  981. {
  982. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  983. int idx;
  984. if (!cpuc->enabled)
  985. return;
  986. cpuc->enabled = 0;
  987. /*
  988. * ensure we write the disable before we start disabling the
  989. * events proper, so that amd_pmu_enable_event() does the
  990. * right thing.
  991. */
  992. barrier();
  993. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  994. u64 val;
  995. if (!test_bit(idx, cpuc->active_mask))
  996. continue;
  997. rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
  998. if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
  999. continue;
  1000. val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
  1001. wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
  1002. }
  1003. }
  1004. void hw_perf_disable(void)
  1005. {
  1006. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1007. if (!x86_pmu_initialized())
  1008. return;
  1009. if (cpuc->enabled)
  1010. cpuc->n_added = 0;
  1011. x86_pmu.disable_all();
  1012. }
  1013. static void p6_pmu_enable_all(void)
  1014. {
  1015. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1016. unsigned long val;
  1017. if (cpuc->enabled)
  1018. return;
  1019. cpuc->enabled = 1;
  1020. barrier();
  1021. /* p6 only has one enable register */
  1022. rdmsrl(MSR_P6_EVNTSEL0, val);
  1023. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1024. wrmsrl(MSR_P6_EVNTSEL0, val);
  1025. }
  1026. static void intel_pmu_enable_all(void)
  1027. {
  1028. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1029. if (cpuc->enabled)
  1030. return;
  1031. cpuc->enabled = 1;
  1032. barrier();
  1033. wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
  1034. if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
  1035. struct perf_event *event =
  1036. cpuc->events[X86_PMC_IDX_FIXED_BTS];
  1037. if (WARN_ON_ONCE(!event))
  1038. return;
  1039. intel_pmu_enable_bts(event->hw.config);
  1040. }
  1041. }
  1042. static void amd_pmu_enable_all(void)
  1043. {
  1044. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1045. int idx;
  1046. if (cpuc->enabled)
  1047. return;
  1048. cpuc->enabled = 1;
  1049. barrier();
  1050. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1051. struct perf_event *event = cpuc->events[idx];
  1052. u64 val;
  1053. if (!test_bit(idx, cpuc->active_mask))
  1054. continue;
  1055. val = event->hw.config;
  1056. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1057. wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
  1058. }
  1059. }
  1060. static const struct pmu pmu;
  1061. static inline int is_x86_event(struct perf_event *event)
  1062. {
  1063. return event->pmu == &pmu;
  1064. }
  1065. static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
  1066. {
  1067. struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
  1068. unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  1069. int i, j, w, wmax, num = 0;
  1070. struct hw_perf_event *hwc;
  1071. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  1072. for (i = 0; i < n; i++) {
  1073. constraints[i] =
  1074. x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
  1075. }
  1076. /*
  1077. * fastpath, try to reuse previous register
  1078. */
  1079. for (i = 0; i < n; i++) {
  1080. hwc = &cpuc->event_list[i]->hw;
  1081. c = constraints[i];
  1082. /* never assigned */
  1083. if (hwc->idx == -1)
  1084. break;
  1085. /* constraint still honored */
  1086. if (!test_bit(hwc->idx, c->idxmsk))
  1087. break;
  1088. /* not already used */
  1089. if (test_bit(hwc->idx, used_mask))
  1090. break;
  1091. set_bit(hwc->idx, used_mask);
  1092. if (assign)
  1093. assign[i] = hwc->idx;
  1094. }
  1095. if (i == n)
  1096. goto done;
  1097. /*
  1098. * begin slow path
  1099. */
  1100. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  1101. /*
  1102. * weight = number of possible counters
  1103. *
  1104. * 1 = most constrained, only works on one counter
  1105. * wmax = least constrained, works on any counter
  1106. *
  1107. * assign events to counters starting with most
  1108. * constrained events.
  1109. */
  1110. wmax = x86_pmu.num_events;
  1111. /*
  1112. * when fixed event counters are present,
  1113. * wmax is incremented by 1 to account
  1114. * for one more choice
  1115. */
  1116. if (x86_pmu.num_events_fixed)
  1117. wmax++;
  1118. for (w = 1, num = n; num && w <= wmax; w++) {
  1119. /* for each event */
  1120. for (i = 0; num && i < n; i++) {
  1121. c = constraints[i];
  1122. hwc = &cpuc->event_list[i]->hw;
  1123. if (c->weight != w)
  1124. continue;
  1125. for_each_bit(j, c->idxmsk, X86_PMC_IDX_MAX) {
  1126. if (!test_bit(j, used_mask))
  1127. break;
  1128. }
  1129. if (j == X86_PMC_IDX_MAX)
  1130. break;
  1131. set_bit(j, used_mask);
  1132. if (assign)
  1133. assign[i] = j;
  1134. num--;
  1135. }
  1136. }
  1137. done:
  1138. /*
  1139. * scheduling failed or is just a simulation,
  1140. * free resources if necessary
  1141. */
  1142. if (!assign || num) {
  1143. for (i = 0; i < n; i++) {
  1144. if (x86_pmu.put_event_constraints)
  1145. x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
  1146. }
  1147. }
  1148. return num ? -ENOSPC : 0;
  1149. }
  1150. /*
  1151. * dogrp: true if must collect siblings events (group)
  1152. * returns total number of events and error code
  1153. */
  1154. static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
  1155. {
  1156. struct perf_event *event;
  1157. int n, max_count;
  1158. max_count = x86_pmu.num_events + x86_pmu.num_events_fixed;
  1159. /* current number of events already accepted */
  1160. n = cpuc->n_events;
  1161. if (is_x86_event(leader)) {
  1162. if (n >= max_count)
  1163. return -ENOSPC;
  1164. cpuc->event_list[n] = leader;
  1165. n++;
  1166. }
  1167. if (!dogrp)
  1168. return n;
  1169. list_for_each_entry(event, &leader->sibling_list, group_entry) {
  1170. if (!is_x86_event(event) ||
  1171. event->state <= PERF_EVENT_STATE_OFF)
  1172. continue;
  1173. if (n >= max_count)
  1174. return -ENOSPC;
  1175. cpuc->event_list[n] = event;
  1176. n++;
  1177. }
  1178. return n;
  1179. }
  1180. static inline void x86_assign_hw_event(struct perf_event *event,
  1181. struct hw_perf_event *hwc, int idx)
  1182. {
  1183. hwc->idx = idx;
  1184. if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
  1185. hwc->config_base = 0;
  1186. hwc->event_base = 0;
  1187. } else if (hwc->idx >= X86_PMC_IDX_FIXED) {
  1188. hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
  1189. /*
  1190. * We set it so that event_base + idx in wrmsr/rdmsr maps to
  1191. * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
  1192. */
  1193. hwc->event_base =
  1194. MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
  1195. } else {
  1196. hwc->config_base = x86_pmu.eventsel;
  1197. hwc->event_base = x86_pmu.perfctr;
  1198. }
  1199. }
  1200. void hw_perf_enable(void)
  1201. {
  1202. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1203. struct perf_event *event;
  1204. struct hw_perf_event *hwc;
  1205. int i;
  1206. if (!x86_pmu_initialized())
  1207. return;
  1208. if (cpuc->n_added) {
  1209. /*
  1210. * apply assignment obtained either from
  1211. * hw_perf_group_sched_in() or x86_pmu_enable()
  1212. *
  1213. * step1: save events moving to new counters
  1214. * step2: reprogram moved events into new counters
  1215. */
  1216. for (i = 0; i < cpuc->n_events; i++) {
  1217. event = cpuc->event_list[i];
  1218. hwc = &event->hw;
  1219. if (hwc->idx == -1 || hwc->idx == cpuc->assign[i])
  1220. continue;
  1221. x86_pmu.disable(hwc, hwc->idx);
  1222. clear_bit(hwc->idx, cpuc->active_mask);
  1223. barrier();
  1224. cpuc->events[hwc->idx] = NULL;
  1225. x86_perf_event_update(event, hwc, hwc->idx);
  1226. hwc->idx = -1;
  1227. }
  1228. for (i = 0; i < cpuc->n_events; i++) {
  1229. event = cpuc->event_list[i];
  1230. hwc = &event->hw;
  1231. if (hwc->idx == -1) {
  1232. x86_assign_hw_event(event, hwc, cpuc->assign[i]);
  1233. x86_perf_event_set_period(event, hwc, hwc->idx);
  1234. }
  1235. /*
  1236. * need to mark as active because x86_pmu_disable()
  1237. * clear active_mask and eventsp[] yet it preserves
  1238. * idx
  1239. */
  1240. set_bit(hwc->idx, cpuc->active_mask);
  1241. cpuc->events[hwc->idx] = event;
  1242. x86_pmu.enable(hwc, hwc->idx);
  1243. perf_event_update_userpage(event);
  1244. }
  1245. cpuc->n_added = 0;
  1246. perf_events_lapic_init();
  1247. }
  1248. x86_pmu.enable_all();
  1249. }
  1250. static inline u64 intel_pmu_get_status(void)
  1251. {
  1252. u64 status;
  1253. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  1254. return status;
  1255. }
  1256. static inline void intel_pmu_ack_status(u64 ack)
  1257. {
  1258. wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
  1259. }
  1260. static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1261. {
  1262. (void)checking_wrmsrl(hwc->config_base + idx,
  1263. hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
  1264. }
  1265. static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1266. {
  1267. (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
  1268. }
  1269. static inline void
  1270. intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
  1271. {
  1272. int idx = __idx - X86_PMC_IDX_FIXED;
  1273. u64 ctrl_val, mask;
  1274. mask = 0xfULL << (idx * 4);
  1275. rdmsrl(hwc->config_base, ctrl_val);
  1276. ctrl_val &= ~mask;
  1277. (void)checking_wrmsrl(hwc->config_base, ctrl_val);
  1278. }
  1279. static inline void
  1280. p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1281. {
  1282. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1283. u64 val = P6_NOP_EVENT;
  1284. if (cpuc->enabled)
  1285. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1286. (void)checking_wrmsrl(hwc->config_base + idx, val);
  1287. }
  1288. static inline void
  1289. intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1290. {
  1291. if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
  1292. intel_pmu_disable_bts();
  1293. return;
  1294. }
  1295. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1296. intel_pmu_disable_fixed(hwc, idx);
  1297. return;
  1298. }
  1299. x86_pmu_disable_event(hwc, idx);
  1300. }
  1301. static inline void
  1302. amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
  1303. {
  1304. x86_pmu_disable_event(hwc, idx);
  1305. }
  1306. static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
  1307. /*
  1308. * Set the next IRQ period, based on the hwc->period_left value.
  1309. * To be called with the event disabled in hw:
  1310. */
  1311. static int
  1312. x86_perf_event_set_period(struct perf_event *event,
  1313. struct hw_perf_event *hwc, int idx)
  1314. {
  1315. s64 left = atomic64_read(&hwc->period_left);
  1316. s64 period = hwc->sample_period;
  1317. int err, ret = 0;
  1318. if (idx == X86_PMC_IDX_FIXED_BTS)
  1319. return 0;
  1320. /*
  1321. * If we are way outside a reasonable range then just skip forward:
  1322. */
  1323. if (unlikely(left <= -period)) {
  1324. left = period;
  1325. atomic64_set(&hwc->period_left, left);
  1326. hwc->last_period = period;
  1327. ret = 1;
  1328. }
  1329. if (unlikely(left <= 0)) {
  1330. left += period;
  1331. atomic64_set(&hwc->period_left, left);
  1332. hwc->last_period = period;
  1333. ret = 1;
  1334. }
  1335. /*
  1336. * Quirk: certain CPUs dont like it if just 1 hw_event is left:
  1337. */
  1338. if (unlikely(left < 2))
  1339. left = 2;
  1340. if (left > x86_pmu.max_period)
  1341. left = x86_pmu.max_period;
  1342. per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
  1343. /*
  1344. * The hw event starts counting from this event offset,
  1345. * mark it to be able to extra future deltas:
  1346. */
  1347. atomic64_set(&hwc->prev_count, (u64)-left);
  1348. err = checking_wrmsrl(hwc->event_base + idx,
  1349. (u64)(-left) & x86_pmu.event_mask);
  1350. perf_event_update_userpage(event);
  1351. return ret;
  1352. }
  1353. static inline void
  1354. intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
  1355. {
  1356. int idx = __idx - X86_PMC_IDX_FIXED;
  1357. u64 ctrl_val, bits, mask;
  1358. int err;
  1359. /*
  1360. * Enable IRQ generation (0x8),
  1361. * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
  1362. * if requested:
  1363. */
  1364. bits = 0x8ULL;
  1365. if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
  1366. bits |= 0x2;
  1367. if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
  1368. bits |= 0x1;
  1369. bits <<= (idx * 4);
  1370. mask = 0xfULL << (idx * 4);
  1371. rdmsrl(hwc->config_base, ctrl_val);
  1372. ctrl_val &= ~mask;
  1373. ctrl_val |= bits;
  1374. err = checking_wrmsrl(hwc->config_base, ctrl_val);
  1375. }
  1376. static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1377. {
  1378. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1379. u64 val;
  1380. val = hwc->config;
  1381. if (cpuc->enabled)
  1382. val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
  1383. (void)checking_wrmsrl(hwc->config_base + idx, val);
  1384. }
  1385. static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1386. {
  1387. if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
  1388. if (!__get_cpu_var(cpu_hw_events).enabled)
  1389. return;
  1390. intel_pmu_enable_bts(hwc->config);
  1391. return;
  1392. }
  1393. if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
  1394. intel_pmu_enable_fixed(hwc, idx);
  1395. return;
  1396. }
  1397. x86_pmu_enable_event(hwc, idx);
  1398. }
  1399. static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
  1400. {
  1401. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1402. if (cpuc->enabled)
  1403. x86_pmu_enable_event(hwc, idx);
  1404. }
  1405. /*
  1406. * activate a single event
  1407. *
  1408. * The event is added to the group of enabled events
  1409. * but only if it can be scehduled with existing events.
  1410. *
  1411. * Called with PMU disabled. If successful and return value 1,
  1412. * then guaranteed to call perf_enable() and hw_perf_enable()
  1413. */
  1414. static int x86_pmu_enable(struct perf_event *event)
  1415. {
  1416. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1417. struct hw_perf_event *hwc;
  1418. int assign[X86_PMC_IDX_MAX];
  1419. int n, n0, ret;
  1420. hwc = &event->hw;
  1421. n0 = cpuc->n_events;
  1422. n = collect_events(cpuc, event, false);
  1423. if (n < 0)
  1424. return n;
  1425. ret = x86_schedule_events(cpuc, n, assign);
  1426. if (ret)
  1427. return ret;
  1428. /*
  1429. * copy new assignment, now we know it is possible
  1430. * will be used by hw_perf_enable()
  1431. */
  1432. memcpy(cpuc->assign, assign, n*sizeof(int));
  1433. cpuc->n_events = n;
  1434. cpuc->n_added = n - n0;
  1435. if (hwc->idx != -1)
  1436. x86_perf_event_set_period(event, hwc, hwc->idx);
  1437. return 0;
  1438. }
  1439. static void x86_pmu_unthrottle(struct perf_event *event)
  1440. {
  1441. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1442. struct hw_perf_event *hwc = &event->hw;
  1443. if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
  1444. cpuc->events[hwc->idx] != event))
  1445. return;
  1446. x86_pmu.enable(hwc, hwc->idx);
  1447. }
  1448. void perf_event_print_debug(void)
  1449. {
  1450. u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
  1451. struct cpu_hw_events *cpuc;
  1452. unsigned long flags;
  1453. int cpu, idx;
  1454. if (!x86_pmu.num_events)
  1455. return;
  1456. local_irq_save(flags);
  1457. cpu = smp_processor_id();
  1458. cpuc = &per_cpu(cpu_hw_events, cpu);
  1459. if (x86_pmu.version >= 2) {
  1460. rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
  1461. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  1462. rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
  1463. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
  1464. pr_info("\n");
  1465. pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
  1466. pr_info("CPU#%d: status: %016llx\n", cpu, status);
  1467. pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
  1468. pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
  1469. }
  1470. pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
  1471. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1472. rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
  1473. rdmsrl(x86_pmu.perfctr + idx, pmc_count);
  1474. prev_left = per_cpu(pmc_prev_left[idx], cpu);
  1475. pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
  1476. cpu, idx, pmc_ctrl);
  1477. pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
  1478. cpu, idx, pmc_count);
  1479. pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
  1480. cpu, idx, prev_left);
  1481. }
  1482. for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
  1483. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
  1484. pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
  1485. cpu, idx, pmc_count);
  1486. }
  1487. local_irq_restore(flags);
  1488. }
  1489. static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
  1490. {
  1491. struct debug_store *ds = cpuc->ds;
  1492. struct bts_record {
  1493. u64 from;
  1494. u64 to;
  1495. u64 flags;
  1496. };
  1497. struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
  1498. struct bts_record *at, *top;
  1499. struct perf_output_handle handle;
  1500. struct perf_event_header header;
  1501. struct perf_sample_data data;
  1502. struct pt_regs regs;
  1503. if (!event)
  1504. return;
  1505. if (!ds)
  1506. return;
  1507. at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
  1508. top = (struct bts_record *)(unsigned long)ds->bts_index;
  1509. if (top <= at)
  1510. return;
  1511. ds->bts_index = ds->bts_buffer_base;
  1512. data.period = event->hw.last_period;
  1513. data.addr = 0;
  1514. data.raw = NULL;
  1515. regs.ip = 0;
  1516. /*
  1517. * Prepare a generic sample, i.e. fill in the invariant fields.
  1518. * We will overwrite the from and to address before we output
  1519. * the sample.
  1520. */
  1521. perf_prepare_sample(&header, &data, event, &regs);
  1522. if (perf_output_begin(&handle, event,
  1523. header.size * (top - at), 1, 1))
  1524. return;
  1525. for (; at < top; at++) {
  1526. data.ip = at->from;
  1527. data.addr = at->to;
  1528. perf_output_sample(&handle, &header, &data, event);
  1529. }
  1530. perf_output_end(&handle);
  1531. /* There's new data available. */
  1532. event->hw.interrupts++;
  1533. event->pending_kill = POLL_IN;
  1534. }
  1535. static void x86_pmu_disable(struct perf_event *event)
  1536. {
  1537. struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  1538. struct hw_perf_event *hwc = &event->hw;
  1539. int i, idx = hwc->idx;
  1540. /*
  1541. * Must be done before we disable, otherwise the nmi handler
  1542. * could reenable again:
  1543. */
  1544. clear_bit(idx, cpuc->active_mask);
  1545. x86_pmu.disable(hwc, idx);
  1546. /*
  1547. * Make sure the cleared pointer becomes visible before we
  1548. * (potentially) free the event:
  1549. */
  1550. barrier();
  1551. /*
  1552. * Drain the remaining delta count out of a event
  1553. * that we are disabling:
  1554. */
  1555. x86_perf_event_update(event, hwc, idx);
  1556. /* Drain the remaining BTS records. */
  1557. if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
  1558. intel_pmu_drain_bts_buffer(cpuc);
  1559. cpuc->events[idx] = NULL;
  1560. for (i = 0; i < cpuc->n_events; i++) {
  1561. if (event == cpuc->event_list[i]) {
  1562. if (x86_pmu.put_event_constraints)
  1563. x86_pmu.put_event_constraints(cpuc, event);
  1564. while (++i < cpuc->n_events)
  1565. cpuc->event_list[i-1] = cpuc->event_list[i];
  1566. --cpuc->n_events;
  1567. break;
  1568. }
  1569. }
  1570. perf_event_update_userpage(event);
  1571. }
  1572. /*
  1573. * Save and restart an expired event. Called by NMI contexts,
  1574. * so it has to be careful about preempting normal event ops:
  1575. */
  1576. static int intel_pmu_save_and_restart(struct perf_event *event)
  1577. {
  1578. struct hw_perf_event *hwc = &event->hw;
  1579. int idx = hwc->idx;
  1580. int ret;
  1581. x86_perf_event_update(event, hwc, idx);
  1582. ret = x86_perf_event_set_period(event, hwc, idx);
  1583. if (event->state == PERF_EVENT_STATE_ACTIVE)
  1584. intel_pmu_enable_event(hwc, idx);
  1585. return ret;
  1586. }
  1587. static void intel_pmu_reset(void)
  1588. {
  1589. struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
  1590. unsigned long flags;
  1591. int idx;
  1592. if (!x86_pmu.num_events)
  1593. return;
  1594. local_irq_save(flags);
  1595. printk("clearing PMU state on CPU#%d\n", smp_processor_id());
  1596. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1597. checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
  1598. checking_wrmsrl(x86_pmu.perfctr + idx, 0ull);
  1599. }
  1600. for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
  1601. checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
  1602. }
  1603. if (ds)
  1604. ds->bts_index = ds->bts_buffer_base;
  1605. local_irq_restore(flags);
  1606. }
  1607. static int p6_pmu_handle_irq(struct pt_regs *regs)
  1608. {
  1609. struct perf_sample_data data;
  1610. struct cpu_hw_events *cpuc;
  1611. struct perf_event *event;
  1612. struct hw_perf_event *hwc;
  1613. int idx, handled = 0;
  1614. u64 val;
  1615. data.addr = 0;
  1616. data.raw = NULL;
  1617. cpuc = &__get_cpu_var(cpu_hw_events);
  1618. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1619. if (!test_bit(idx, cpuc->active_mask))
  1620. continue;
  1621. event = cpuc->events[idx];
  1622. hwc = &event->hw;
  1623. val = x86_perf_event_update(event, hwc, idx);
  1624. if (val & (1ULL << (x86_pmu.event_bits - 1)))
  1625. continue;
  1626. /*
  1627. * event overflow
  1628. */
  1629. handled = 1;
  1630. data.period = event->hw.last_period;
  1631. if (!x86_perf_event_set_period(event, hwc, idx))
  1632. continue;
  1633. if (perf_event_overflow(event, 1, &data, regs))
  1634. p6_pmu_disable_event(hwc, idx);
  1635. }
  1636. if (handled)
  1637. inc_irq_stat(apic_perf_irqs);
  1638. return handled;
  1639. }
  1640. /*
  1641. * This handler is triggered by the local APIC, so the APIC IRQ handling
  1642. * rules apply:
  1643. */
  1644. static int intel_pmu_handle_irq(struct pt_regs *regs)
  1645. {
  1646. struct perf_sample_data data;
  1647. struct cpu_hw_events *cpuc;
  1648. int bit, loops;
  1649. u64 ack, status;
  1650. data.addr = 0;
  1651. data.raw = NULL;
  1652. cpuc = &__get_cpu_var(cpu_hw_events);
  1653. perf_disable();
  1654. intel_pmu_drain_bts_buffer(cpuc);
  1655. status = intel_pmu_get_status();
  1656. if (!status) {
  1657. perf_enable();
  1658. return 0;
  1659. }
  1660. loops = 0;
  1661. again:
  1662. if (++loops > 100) {
  1663. WARN_ONCE(1, "perfevents: irq loop stuck!\n");
  1664. perf_event_print_debug();
  1665. intel_pmu_reset();
  1666. perf_enable();
  1667. return 1;
  1668. }
  1669. inc_irq_stat(apic_perf_irqs);
  1670. ack = status;
  1671. for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
  1672. struct perf_event *event = cpuc->events[bit];
  1673. clear_bit(bit, (unsigned long *) &status);
  1674. if (!test_bit(bit, cpuc->active_mask))
  1675. continue;
  1676. if (!intel_pmu_save_and_restart(event))
  1677. continue;
  1678. data.period = event->hw.last_period;
  1679. if (perf_event_overflow(event, 1, &data, regs))
  1680. intel_pmu_disable_event(&event->hw, bit);
  1681. }
  1682. intel_pmu_ack_status(ack);
  1683. /*
  1684. * Repeat if there is more work to be done:
  1685. */
  1686. status = intel_pmu_get_status();
  1687. if (status)
  1688. goto again;
  1689. perf_enable();
  1690. return 1;
  1691. }
  1692. static int amd_pmu_handle_irq(struct pt_regs *regs)
  1693. {
  1694. struct perf_sample_data data;
  1695. struct cpu_hw_events *cpuc;
  1696. struct perf_event *event;
  1697. struct hw_perf_event *hwc;
  1698. int idx, handled = 0;
  1699. u64 val;
  1700. data.addr = 0;
  1701. data.raw = NULL;
  1702. cpuc = &__get_cpu_var(cpu_hw_events);
  1703. for (idx = 0; idx < x86_pmu.num_events; idx++) {
  1704. if (!test_bit(idx, cpuc->active_mask))
  1705. continue;
  1706. event = cpuc->events[idx];
  1707. hwc = &event->hw;
  1708. val = x86_perf_event_update(event, hwc, idx);
  1709. if (val & (1ULL << (x86_pmu.event_bits - 1)))
  1710. continue;
  1711. /*
  1712. * event overflow
  1713. */
  1714. handled = 1;
  1715. data.period = event->hw.last_period;
  1716. if (!x86_perf_event_set_period(event, hwc, idx))
  1717. continue;
  1718. if (perf_event_overflow(event, 1, &data, regs))
  1719. amd_pmu_disable_event(hwc, idx);
  1720. }
  1721. if (handled)
  1722. inc_irq_stat(apic_perf_irqs);
  1723. return handled;
  1724. }
  1725. void smp_perf_pending_interrupt(struct pt_regs *regs)
  1726. {
  1727. irq_enter();
  1728. ack_APIC_irq();
  1729. inc_irq_stat(apic_pending_irqs);
  1730. perf_event_do_pending();
  1731. irq_exit();
  1732. }
  1733. void set_perf_event_pending(void)
  1734. {
  1735. #ifdef CONFIG_X86_LOCAL_APIC
  1736. if (!x86_pmu.apic || !x86_pmu_initialized())
  1737. return;
  1738. apic->send_IPI_self(LOCAL_PENDING_VECTOR);
  1739. #endif
  1740. }
  1741. void perf_events_lapic_init(void)
  1742. {
  1743. #ifdef CONFIG_X86_LOCAL_APIC
  1744. if (!x86_pmu.apic || !x86_pmu_initialized())
  1745. return;
  1746. /*
  1747. * Always use NMI for PMU
  1748. */
  1749. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1750. #endif
  1751. }
  1752. static int __kprobes
  1753. perf_event_nmi_handler(struct notifier_block *self,
  1754. unsigned long cmd, void *__args)
  1755. {
  1756. struct die_args *args = __args;
  1757. struct pt_regs *regs;
  1758. if (!atomic_read(&active_events))
  1759. return NOTIFY_DONE;
  1760. switch (cmd) {
  1761. case DIE_NMI:
  1762. case DIE_NMI_IPI:
  1763. break;
  1764. default:
  1765. return NOTIFY_DONE;
  1766. }
  1767. regs = args->regs;
  1768. #ifdef CONFIG_X86_LOCAL_APIC
  1769. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1770. #endif
  1771. /*
  1772. * Can't rely on the handled return value to say it was our NMI, two
  1773. * events could trigger 'simultaneously' raising two back-to-back NMIs.
  1774. *
  1775. * If the first NMI handles both, the latter will be empty and daze
  1776. * the CPU.
  1777. */
  1778. x86_pmu.handle_irq(regs);
  1779. return NOTIFY_STOP;
  1780. }
  1781. static struct event_constraint unconstrained;
  1782. static struct event_constraint bts_constraint =
  1783. EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
  1784. static struct event_constraint *
  1785. intel_special_constraints(struct perf_event *event)
  1786. {
  1787. unsigned int hw_event;
  1788. hw_event = event->hw.config & INTEL_ARCH_EVENT_MASK;
  1789. if (unlikely((hw_event ==
  1790. x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
  1791. (event->hw.sample_period == 1))) {
  1792. return &bts_constraint;
  1793. }
  1794. return NULL;
  1795. }
  1796. static struct event_constraint *
  1797. intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1798. {
  1799. struct event_constraint *c;
  1800. c = intel_special_constraints(event);
  1801. if (c)
  1802. return c;
  1803. if (x86_pmu.event_constraints) {
  1804. for_each_event_constraint(c, x86_pmu.event_constraints) {
  1805. if ((event->hw.config & c->cmask) == c->code)
  1806. return c;
  1807. }
  1808. }
  1809. return &unconstrained;
  1810. }
  1811. static struct event_constraint *
  1812. amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
  1813. {
  1814. return &unconstrained;
  1815. }
  1816. static int x86_event_sched_in(struct perf_event *event,
  1817. struct perf_cpu_context *cpuctx, int cpu)
  1818. {
  1819. int ret = 0;
  1820. event->state = PERF_EVENT_STATE_ACTIVE;
  1821. event->oncpu = cpu;
  1822. event->tstamp_running += event->ctx->time - event->tstamp_stopped;
  1823. if (!is_x86_event(event))
  1824. ret = event->pmu->enable(event);
  1825. if (!ret && !is_software_event(event))
  1826. cpuctx->active_oncpu++;
  1827. if (!ret && event->attr.exclusive)
  1828. cpuctx->exclusive = 1;
  1829. return ret;
  1830. }
  1831. static void x86_event_sched_out(struct perf_event *event,
  1832. struct perf_cpu_context *cpuctx, int cpu)
  1833. {
  1834. event->state = PERF_EVENT_STATE_INACTIVE;
  1835. event->oncpu = -1;
  1836. if (!is_x86_event(event))
  1837. event->pmu->disable(event);
  1838. event->tstamp_running -= event->ctx->time - event->tstamp_stopped;
  1839. if (!is_software_event(event))
  1840. cpuctx->active_oncpu--;
  1841. if (event->attr.exclusive || !cpuctx->active_oncpu)
  1842. cpuctx->exclusive = 0;
  1843. }
  1844. /*
  1845. * Called to enable a whole group of events.
  1846. * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
  1847. * Assumes the caller has disabled interrupts and has
  1848. * frozen the PMU with hw_perf_save_disable.
  1849. *
  1850. * called with PMU disabled. If successful and return value 1,
  1851. * then guaranteed to call perf_enable() and hw_perf_enable()
  1852. */
  1853. int hw_perf_group_sched_in(struct perf_event *leader,
  1854. struct perf_cpu_context *cpuctx,
  1855. struct perf_event_context *ctx, int cpu)
  1856. {
  1857. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1858. struct perf_event *sub;
  1859. int assign[X86_PMC_IDX_MAX];
  1860. int n0, n1, ret;
  1861. /* n0 = total number of events */
  1862. n0 = collect_events(cpuc, leader, true);
  1863. if (n0 < 0)
  1864. return n0;
  1865. ret = x86_schedule_events(cpuc, n0, assign);
  1866. if (ret)
  1867. return ret;
  1868. ret = x86_event_sched_in(leader, cpuctx, cpu);
  1869. if (ret)
  1870. return ret;
  1871. n1 = 1;
  1872. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  1873. if (sub->state > PERF_EVENT_STATE_OFF) {
  1874. ret = x86_event_sched_in(sub, cpuctx, cpu);
  1875. if (ret)
  1876. goto undo;
  1877. ++n1;
  1878. }
  1879. }
  1880. /*
  1881. * copy new assignment, now we know it is possible
  1882. * will be used by hw_perf_enable()
  1883. */
  1884. memcpy(cpuc->assign, assign, n0*sizeof(int));
  1885. cpuc->n_events = n0;
  1886. cpuc->n_added = n1;
  1887. ctx->nr_active += n1;
  1888. /*
  1889. * 1 means successful and events are active
  1890. * This is not quite true because we defer
  1891. * actual activation until hw_perf_enable() but
  1892. * this way we* ensure caller won't try to enable
  1893. * individual events
  1894. */
  1895. return 1;
  1896. undo:
  1897. x86_event_sched_out(leader, cpuctx, cpu);
  1898. n0 = 1;
  1899. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  1900. if (sub->state == PERF_EVENT_STATE_ACTIVE) {
  1901. x86_event_sched_out(sub, cpuctx, cpu);
  1902. if (++n0 == n1)
  1903. break;
  1904. }
  1905. }
  1906. return ret;
  1907. }
  1908. static __read_mostly struct notifier_block perf_event_nmi_notifier = {
  1909. .notifier_call = perf_event_nmi_handler,
  1910. .next = NULL,
  1911. .priority = 1
  1912. };
  1913. static __initconst struct x86_pmu p6_pmu = {
  1914. .name = "p6",
  1915. .handle_irq = p6_pmu_handle_irq,
  1916. .disable_all = p6_pmu_disable_all,
  1917. .enable_all = p6_pmu_enable_all,
  1918. .enable = p6_pmu_enable_event,
  1919. .disable = p6_pmu_disable_event,
  1920. .eventsel = MSR_P6_EVNTSEL0,
  1921. .perfctr = MSR_P6_PERFCTR0,
  1922. .event_map = p6_pmu_event_map,
  1923. .raw_event = p6_pmu_raw_event,
  1924. .max_events = ARRAY_SIZE(p6_perfmon_event_map),
  1925. .apic = 1,
  1926. .max_period = (1ULL << 31) - 1,
  1927. .version = 0,
  1928. .num_events = 2,
  1929. /*
  1930. * Events have 40 bits implemented. However they are designed such
  1931. * that bits [32-39] are sign extensions of bit 31. As such the
  1932. * effective width of a event for P6-like PMU is 32 bits only.
  1933. *
  1934. * See IA-32 Intel Architecture Software developer manual Vol 3B
  1935. */
  1936. .event_bits = 32,
  1937. .event_mask = (1ULL << 32) - 1,
  1938. .get_event_constraints = intel_get_event_constraints,
  1939. .event_constraints = intel_p6_event_constraints
  1940. };
  1941. static __initconst struct x86_pmu intel_pmu = {
  1942. .name = "Intel",
  1943. .handle_irq = intel_pmu_handle_irq,
  1944. .disable_all = intel_pmu_disable_all,
  1945. .enable_all = intel_pmu_enable_all,
  1946. .enable = intel_pmu_enable_event,
  1947. .disable = intel_pmu_disable_event,
  1948. .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
  1949. .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
  1950. .event_map = intel_pmu_event_map,
  1951. .raw_event = intel_pmu_raw_event,
  1952. .max_events = ARRAY_SIZE(intel_perfmon_event_map),
  1953. .apic = 1,
  1954. /*
  1955. * Intel PMCs cannot be accessed sanely above 32 bit width,
  1956. * so we install an artificial 1<<31 period regardless of
  1957. * the generic event period:
  1958. */
  1959. .max_period = (1ULL << 31) - 1,
  1960. .enable_bts = intel_pmu_enable_bts,
  1961. .disable_bts = intel_pmu_disable_bts,
  1962. .get_event_constraints = intel_get_event_constraints
  1963. };
  1964. static __initconst struct x86_pmu amd_pmu = {
  1965. .name = "AMD",
  1966. .handle_irq = amd_pmu_handle_irq,
  1967. .disable_all = amd_pmu_disable_all,
  1968. .enable_all = amd_pmu_enable_all,
  1969. .enable = amd_pmu_enable_event,
  1970. .disable = amd_pmu_disable_event,
  1971. .eventsel = MSR_K7_EVNTSEL0,
  1972. .perfctr = MSR_K7_PERFCTR0,
  1973. .event_map = amd_pmu_event_map,
  1974. .raw_event = amd_pmu_raw_event,
  1975. .max_events = ARRAY_SIZE(amd_perfmon_event_map),
  1976. .num_events = 4,
  1977. .event_bits = 48,
  1978. .event_mask = (1ULL << 48) - 1,
  1979. .apic = 1,
  1980. /* use highest bit to detect overflow */
  1981. .max_period = (1ULL << 47) - 1,
  1982. .get_event_constraints = amd_get_event_constraints
  1983. };
  1984. static __init int p6_pmu_init(void)
  1985. {
  1986. switch (boot_cpu_data.x86_model) {
  1987. case 1:
  1988. case 3: /* Pentium Pro */
  1989. case 5:
  1990. case 6: /* Pentium II */
  1991. case 7:
  1992. case 8:
  1993. case 11: /* Pentium III */
  1994. case 9:
  1995. case 13:
  1996. /* Pentium M */
  1997. break;
  1998. default:
  1999. pr_cont("unsupported p6 CPU model %d ",
  2000. boot_cpu_data.x86_model);
  2001. return -ENODEV;
  2002. }
  2003. x86_pmu = p6_pmu;
  2004. return 0;
  2005. }
  2006. static __init int intel_pmu_init(void)
  2007. {
  2008. union cpuid10_edx edx;
  2009. union cpuid10_eax eax;
  2010. unsigned int unused;
  2011. unsigned int ebx;
  2012. int version;
  2013. if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
  2014. /* check for P6 processor family */
  2015. if (boot_cpu_data.x86 == 6) {
  2016. return p6_pmu_init();
  2017. } else {
  2018. return -ENODEV;
  2019. }
  2020. }
  2021. /*
  2022. * Check whether the Architectural PerfMon supports
  2023. * Branch Misses Retired hw_event or not.
  2024. */
  2025. cpuid(10, &eax.full, &ebx, &unused, &edx.full);
  2026. if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
  2027. return -ENODEV;
  2028. version = eax.split.version_id;
  2029. if (version < 2)
  2030. return -ENODEV;
  2031. x86_pmu = intel_pmu;
  2032. x86_pmu.version = version;
  2033. x86_pmu.num_events = eax.split.num_events;
  2034. x86_pmu.event_bits = eax.split.bit_width;
  2035. x86_pmu.event_mask = (1ULL << eax.split.bit_width) - 1;
  2036. /*
  2037. * Quirk: v2 perfmon does not report fixed-purpose events, so
  2038. * assume at least 3 events:
  2039. */
  2040. x86_pmu.num_events_fixed = max((int)edx.split.num_events_fixed, 3);
  2041. /*
  2042. * Install the hw-cache-events table:
  2043. */
  2044. switch (boot_cpu_data.x86_model) {
  2045. case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
  2046. case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
  2047. case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
  2048. case 29: /* six-core 45 nm xeon "Dunnington" */
  2049. memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
  2050. sizeof(hw_cache_event_ids));
  2051. x86_pmu.event_constraints = intel_core_event_constraints;
  2052. pr_cont("Core2 events, ");
  2053. break;
  2054. case 26:
  2055. memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
  2056. sizeof(hw_cache_event_ids));
  2057. x86_pmu.event_constraints = intel_nehalem_event_constraints;
  2058. pr_cont("Nehalem/Corei7 events, ");
  2059. break;
  2060. case 28:
  2061. memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
  2062. sizeof(hw_cache_event_ids));
  2063. x86_pmu.event_constraints = intel_gen_event_constraints;
  2064. pr_cont("Atom events, ");
  2065. break;
  2066. default:
  2067. /*
  2068. * default constraints for v2 and up
  2069. */
  2070. x86_pmu.event_constraints = intel_gen_event_constraints;
  2071. pr_cont("generic architected perfmon, ");
  2072. }
  2073. return 0;
  2074. }
  2075. static __init int amd_pmu_init(void)
  2076. {
  2077. /* Performance-monitoring supported from K7 and later: */
  2078. if (boot_cpu_data.x86 < 6)
  2079. return -ENODEV;
  2080. x86_pmu = amd_pmu;
  2081. /* Events are common for all AMDs */
  2082. memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
  2083. sizeof(hw_cache_event_ids));
  2084. return 0;
  2085. }
  2086. static void __init pmu_check_apic(void)
  2087. {
  2088. if (cpu_has_apic)
  2089. return;
  2090. x86_pmu.apic = 0;
  2091. pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
  2092. pr_info("no hardware sampling interrupt available.\n");
  2093. }
  2094. void __init init_hw_perf_events(void)
  2095. {
  2096. int err;
  2097. pr_info("Performance Events: ");
  2098. switch (boot_cpu_data.x86_vendor) {
  2099. case X86_VENDOR_INTEL:
  2100. err = intel_pmu_init();
  2101. break;
  2102. case X86_VENDOR_AMD:
  2103. err = amd_pmu_init();
  2104. break;
  2105. default:
  2106. return;
  2107. }
  2108. if (err != 0) {
  2109. pr_cont("no PMU driver, software events only.\n");
  2110. return;
  2111. }
  2112. pmu_check_apic();
  2113. pr_cont("%s PMU driver.\n", x86_pmu.name);
  2114. if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
  2115. WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
  2116. x86_pmu.num_events, X86_PMC_MAX_GENERIC);
  2117. x86_pmu.num_events = X86_PMC_MAX_GENERIC;
  2118. }
  2119. perf_event_mask = (1 << x86_pmu.num_events) - 1;
  2120. perf_max_events = x86_pmu.num_events;
  2121. if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
  2122. WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
  2123. x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
  2124. x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
  2125. }
  2126. perf_event_mask |=
  2127. ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
  2128. x86_pmu.intel_ctrl = perf_event_mask;
  2129. perf_events_lapic_init();
  2130. register_die_notifier(&perf_event_nmi_notifier);
  2131. unconstrained = (struct event_constraint)
  2132. EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_events) - 1, 0);
  2133. pr_info("... version: %d\n", x86_pmu.version);
  2134. pr_info("... bit width: %d\n", x86_pmu.event_bits);
  2135. pr_info("... generic registers: %d\n", x86_pmu.num_events);
  2136. pr_info("... value mask: %016Lx\n", x86_pmu.event_mask);
  2137. pr_info("... max period: %016Lx\n", x86_pmu.max_period);
  2138. pr_info("... fixed-purpose events: %d\n", x86_pmu.num_events_fixed);
  2139. pr_info("... event mask: %016Lx\n", perf_event_mask);
  2140. }
  2141. static inline void x86_pmu_read(struct perf_event *event)
  2142. {
  2143. x86_perf_event_update(event, &event->hw, event->hw.idx);
  2144. }
  2145. static const struct pmu pmu = {
  2146. .enable = x86_pmu_enable,
  2147. .disable = x86_pmu_disable,
  2148. .read = x86_pmu_read,
  2149. .unthrottle = x86_pmu_unthrottle,
  2150. };
  2151. /*
  2152. * validate a single event group
  2153. *
  2154. * validation include:
  2155. * - check events are compatible which each other
  2156. * - events do not compete for the same counter
  2157. * - number of events <= number of counters
  2158. *
  2159. * validation ensures the group can be loaded onto the
  2160. * PMU if it was the only group available.
  2161. */
  2162. static int validate_group(struct perf_event *event)
  2163. {
  2164. struct perf_event *leader = event->group_leader;
  2165. struct cpu_hw_events *fake_cpuc;
  2166. int ret, n;
  2167. ret = -ENOMEM;
  2168. fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
  2169. if (!fake_cpuc)
  2170. goto out;
  2171. /*
  2172. * the event is not yet connected with its
  2173. * siblings therefore we must first collect
  2174. * existing siblings, then add the new event
  2175. * before we can simulate the scheduling
  2176. */
  2177. ret = -ENOSPC;
  2178. n = collect_events(fake_cpuc, leader, true);
  2179. if (n < 0)
  2180. goto out_free;
  2181. fake_cpuc->n_events = n;
  2182. n = collect_events(fake_cpuc, event, false);
  2183. if (n < 0)
  2184. goto out_free;
  2185. fake_cpuc->n_events = n;
  2186. ret = x86_schedule_events(fake_cpuc, n, NULL);
  2187. out_free:
  2188. kfree(fake_cpuc);
  2189. out:
  2190. return ret;
  2191. }
  2192. const struct pmu *hw_perf_event_init(struct perf_event *event)
  2193. {
  2194. const struct pmu *tmp;
  2195. int err;
  2196. err = __hw_perf_event_init(event);
  2197. if (!err) {
  2198. /*
  2199. * we temporarily connect event to its pmu
  2200. * such that validate_group() can classify
  2201. * it as an x86 event using is_x86_event()
  2202. */
  2203. tmp = event->pmu;
  2204. event->pmu = &pmu;
  2205. if (event->group_leader != event)
  2206. err = validate_group(event);
  2207. event->pmu = tmp;
  2208. }
  2209. if (err) {
  2210. if (event->destroy)
  2211. event->destroy(event);
  2212. return ERR_PTR(err);
  2213. }
  2214. return &pmu;
  2215. }
  2216. /*
  2217. * callchain support
  2218. */
  2219. static inline
  2220. void callchain_store(struct perf_callchain_entry *entry, u64 ip)
  2221. {
  2222. if (entry->nr < PERF_MAX_STACK_DEPTH)
  2223. entry->ip[entry->nr++] = ip;
  2224. }
  2225. static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
  2226. static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
  2227. static void
  2228. backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
  2229. {
  2230. /* Ignore warnings */
  2231. }
  2232. static void backtrace_warning(void *data, char *msg)
  2233. {
  2234. /* Ignore warnings */
  2235. }
  2236. static int backtrace_stack(void *data, char *name)
  2237. {
  2238. return 0;
  2239. }
  2240. static void backtrace_address(void *data, unsigned long addr, int reliable)
  2241. {
  2242. struct perf_callchain_entry *entry = data;
  2243. if (reliable)
  2244. callchain_store(entry, addr);
  2245. }
  2246. static const struct stacktrace_ops backtrace_ops = {
  2247. .warning = backtrace_warning,
  2248. .warning_symbol = backtrace_warning_symbol,
  2249. .stack = backtrace_stack,
  2250. .address = backtrace_address,
  2251. .walk_stack = print_context_stack_bp,
  2252. };
  2253. #include "../dumpstack.h"
  2254. static void
  2255. perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
  2256. {
  2257. callchain_store(entry, PERF_CONTEXT_KERNEL);
  2258. callchain_store(entry, regs->ip);
  2259. dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
  2260. }
  2261. /*
  2262. * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
  2263. */
  2264. static unsigned long
  2265. copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
  2266. {
  2267. unsigned long offset, addr = (unsigned long)from;
  2268. int type = in_nmi() ? KM_NMI : KM_IRQ0;
  2269. unsigned long size, len = 0;
  2270. struct page *page;
  2271. void *map;
  2272. int ret;
  2273. do {
  2274. ret = __get_user_pages_fast(addr, 1, 0, &page);
  2275. if (!ret)
  2276. break;
  2277. offset = addr & (PAGE_SIZE - 1);
  2278. size = min(PAGE_SIZE - offset, n - len);
  2279. map = kmap_atomic(page, type);
  2280. memcpy(to, map+offset, size);
  2281. kunmap_atomic(map, type);
  2282. put_page(page);
  2283. len += size;
  2284. to += size;
  2285. addr += size;
  2286. } while (len < n);
  2287. return len;
  2288. }
  2289. static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
  2290. {
  2291. unsigned long bytes;
  2292. bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
  2293. return bytes == sizeof(*frame);
  2294. }
  2295. static void
  2296. perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
  2297. {
  2298. struct stack_frame frame;
  2299. const void __user *fp;
  2300. if (!user_mode(regs))
  2301. regs = task_pt_regs(current);
  2302. fp = (void __user *)regs->bp;
  2303. callchain_store(entry, PERF_CONTEXT_USER);
  2304. callchain_store(entry, regs->ip);
  2305. while (entry->nr < PERF_MAX_STACK_DEPTH) {
  2306. frame.next_frame = NULL;
  2307. frame.return_address = 0;
  2308. if (!copy_stack_frame(fp, &frame))
  2309. break;
  2310. if ((unsigned long)fp < regs->sp)
  2311. break;
  2312. callchain_store(entry, frame.return_address);
  2313. fp = frame.next_frame;
  2314. }
  2315. }
  2316. static void
  2317. perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
  2318. {
  2319. int is_user;
  2320. if (!regs)
  2321. return;
  2322. is_user = user_mode(regs);
  2323. if (is_user && current->state != TASK_RUNNING)
  2324. return;
  2325. if (!is_user)
  2326. perf_callchain_kernel(regs, entry);
  2327. if (current->mm)
  2328. perf_callchain_user(regs, entry);
  2329. }
  2330. struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
  2331. {
  2332. struct perf_callchain_entry *entry;
  2333. if (in_nmi())
  2334. entry = &__get_cpu_var(pmc_nmi_entry);
  2335. else
  2336. entry = &__get_cpu_var(pmc_irq_entry);
  2337. entry->nr = 0;
  2338. perf_do_callchain(regs, entry);
  2339. return entry;
  2340. }
  2341. void hw_perf_event_setup_online(int cpu)
  2342. {
  2343. init_debug_store_on_cpu(cpu);
  2344. }